def test_fetch_from_empty_cache(self):
"""Test if there are not any event returned when the cache is empty"""
cache = Cache(self.tmp_path)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2, cache=cache)
cached_events = [event for event in meetup.fetch_from_cache()]
self.assertEqual(len(cached_events), 0)
def test_fetch_from_non_set_cache(self):
"""Test if a error is raised when the cache was not set"""
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
with self.assertRaises(CacheError):
_ = [event for event in meetup.fetch_from_cache()]
def test_initialization(self):
"""Test whether attributes are initializated"""
meetup = Meetup('mygroup',
'aaaa',
max_items=5,
tag='test',
sleep_for_rate=True,
min_rate_to_sleep=10,
sleep_time=60)
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'test')
self.assertEqual(meetup.group, 'mygroup')
self.assertEqual(meetup.max_items, 5)
self.assertIsNone(meetup.client)
# When tag is empty or None it will be set to
# the value in URL
meetup = Meetup('mygroup', 'aaaa')
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'https://meetup.com/')
meetup = Meetup('mygroup', 'aaaa', tag='')
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'https://meetup.com/')
def test_fetch_empty(self):
"""Test if nothing is returned when there are no events"""
http_requests = setup_http_server()
from_date = datetime.datetime(2017, 1, 1)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [event for event in meetup.fetch(from_date=from_date)]
self.assertEqual(len(events), 0)
# Check requests
expected = {
'fields': [
'event_hosts,featured,group_topics,plain_text_description,rsvpable,series'
],
'key': ['aaaa'],
'order': ['updated'],
'page': ['2'],
'scroll': ['since:2017-01-01T00:00:00.000Z'],
'sign': ['true'],
'status': ['cancelled,upcoming,past,proposed,suggested']
}
self.assertEqual(len(http_requests), 1)
self.assertDictEqual(http_requests[0].querystring, expected)
def test_get_repository_filter_raw(self):
"""Test whether the repository filter raw works properly"""
expected_filter = {
'name': 'origin',
'value': 'http://example.com'
}
eitems = ElasticItems(self.perceval_backend)
fltr = eitems.get_repository_filter_raw()
self.assertDictEqual(fltr, expected_filter)
expected_filter = {
'name': 'tag',
'value': 'test'
}
perceval_backend_meetup = Meetup('mygroup', 'aaaa', tag='test')
eitems = MeetupOcean(perceval_backend_meetup)
fltr = eitems.get_repository_filter_raw()
self.assertDictEqual(fltr, expected_filter)
expected_filter = {}
perceval_backend_meetup = Meetup('mygroup', 'aaaa', tag='https://meetup.com/')
eitems = MeetupOcean(perceval_backend_meetup)
fltr = eitems.get_repository_filter_raw()
self.assertDictEqual(fltr, expected_filter)
def test_fetch_from_date(self):
"""Test whether if fetches a set of events from the given date"""
http_requests = setup_http_server()
from_date = datetime.datetime(2016, 9, 25)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [event for event in meetup.fetch(from_date=from_date)]
expected = [('3', 'a42b7cf556c17b17f05b951e2eb5e07a7cb0a731', 1474842748.0, 2, 3)]
self.assertEqual(len(events), len(expected))
for x in range(len(events)):
event = events[x]
expc = expected[x]
self.assertEqual(event['data']['id'], expc[0])
self.assertEqual(event['uuid'], expc[1])
self.assertEqual(event['origin'], 'https://meetup.com/')
self.assertEqual(event['updated_on'], expc[2])
self.assertEqual(event['category'], 'event')
self.assertEqual(event['tag'], 'https://meetup.com/')
self.assertEqual(event['classified_fields_filtered'], None)
self.assertEqual(len(event['data']['comments']), expc[3])
self.assertEqual(len(event['data']['rsvps']), expc[4])
# Check requests
expected = [
{
'fields': ['event_hosts,featured,group_topics,plain_text_description,rsvpable,series'],
'key': ['aaaa'],
'order': ['updated'],
'page': ['2'],
'scroll': ['since:2016-09-25T00:00:00.000Z'],
'sign': ['true'],
'status': ['cancelled,upcoming,past,proposed,suggested']
},
{
'key': ['aaaa'],
'page': ['2'],
'sign': ['true']
},
{
'fields': ['attendance_status'],
'key': ['aaaa'],
'page': ['2'],
'response': ['yes,no'],
'sign': ['true']
}
]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertDictEqual(http_requests[i].querystring, expected[i])
def setUp(self):
super().setUp()
self.backend_write_archive = Meetup('sqlpass-es',
'aaaa',
max_items=2,
archive=self.archive)
self.backend_read_archive = Meetup('sqlpass-es',
'aaaa',
max_items=2,
archive=self.archive)
def test_fetch_date_range(self):
"""Test whether if fetches a set of events updated withing the given range"""
http_requests = setup_http_server()
from_date = datetime.datetime(2016, 4, 8)
to_date = datetime.datetime(2016, 9, 25)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [
event
for event in meetup.fetch(from_date=from_date, to_date=to_date)
]
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event['data']['id'], '2')
self.assertEqual(event['uuid'],
'24b47b622eb33965676dd951b18eea7689b1d81c')
self.assertEqual(event['origin'], 'https://meetup.com/')
self.assertEqual(event['updated_on'], 1465503498.0)
self.assertEqual(event['category'], 'event')
self.assertEqual(event['tag'], 'https://meetup.com/')
self.assertEqual(event['classified_fields_filtered'], None)
self.assertEqual(len(event['data']['comments']), 2)
self.assertEqual(len(event['data']['rsvps']), 3)
# Check requests
expected = [{
'fields': [
'event_hosts,featured,group_topics,plain_text_description,rsvpable,series'
],
'key': ['aaaa'],
'order': ['updated'],
'page': ['2'],
'scroll': ['since:2016-04-08T00:00:00.000Z'],
'sign': ['true'],
'status': ['cancelled,upcoming,past,proposed,suggested']
}, {
'key': ['aaaa'],
'page': ['2'],
'sign': ['true']
}, {
'fields': ['attendance_status'],
'key': ['aaaa'],
'page': ['2'],
'response': ['yes,no'],
'sign': ['true']
}]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertDictEqual(http_requests[i].querystring, expected[i])
def test_fetch_from_cache(self):
"""Test whether the cache works"""
http_requests = setup_http_server()
# First, we fetch the events from the server,
# storing them in a cache
cache = Cache(self.tmp_path)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2, cache=cache)
events = [event for event in meetup.fetch()]
self.assertEqual(len(http_requests), 8)
# Now, we get the events from the cache.
# The events should be the same and there won't be
# any new request to the server
cached_events = [event for event in meetup.fetch_from_cache()]
self.assertEqual(len(cached_events), len(events))
expected = [('1', '0d07fe36f994a6c78dfcf60fb73674bcf158cb5a',
1460065164.0, 2, 3),
('2', '24b47b622eb33965676dd951b18eea7689b1d81c',
1465503498.0, 2, 3),
('3', 'a42b7cf556c17b17f05b951e2eb5e07a7cb0a731',
1474842748.0, 2, 3)]
self.assertEqual(len(cached_events), len(expected))
for x in range(len(cached_events)):
event = cached_events[x]
expc = expected[x]
self.assertEqual(event['data']['id'], expc[0])
self.assertEqual(event['uuid'], expc[1])
self.assertEqual(event['origin'], 'https://meetup.com/')
self.assertEqual(event['updated_on'], expc[2])
self.assertEqual(event['category'], 'event')
self.assertEqual(event['tag'], 'https://meetup.com/')
self.assertEqual(len(event['data']['comments']), expc[3])
self.assertEqual(len(event['data']['rsvps']), expc[4])
# Compare chached and fetched task
self.assertDictEqual(event['data'], events[x]['data'])
# No more requests were sent
self.assertEqual(len(http_requests), 8)
def test_parse_json(self):
"""Test if it parses a JSON stream"""
raw_json = read_file('data/meetup/meetup_events.json')
items = Meetup.parse_json(raw_json)
results = [item for item in items]
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['id'], '1')
self.assertEqual(results[1]['id'], '2')
# Parse a file without results
raw_json = read_file('data/meetup/meetup_events_empty.json')
items = Meetup.parse_json(raw_json)
results = [item for item in items]
self.assertEqual(len(results), 0)
def test_initialization(self):
"""Test whether attributes are initializated"""
meetup = Meetup('mygroup', 'aaaa', max_items=5, tag='test')
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'test')
self.assertEqual(meetup.group, 'mygroup')
self.assertEqual(meetup.max_items, 5)
self.assertIsInstance(meetup.client, MeetupClient)
self.assertEqual(meetup.client.api_key, 'aaaa')
# When tag is empty or None it will be set to
# the value in URL
meetup = Meetup('mygroup', 'aaaa')
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'https://meetup.com/')
meetup = Meetup('mygroup', 'aaaa', tag='')
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'https://meetup.com/')
def test_search_fields(self):
"""Test whether the search_fields is properly set"""
http_requests = setup_http_server()
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [event for event in meetup.fetch(from_date=None)]
event = events[0]
self.assertEqual(meetup.metadata_id(event['data']), event['search_fields']['item_id'])
self.assertEqual(event['data']['group']['name'], 'sqlpass.es')
self.assertEqual(event['data']['group']['name'], event['search_fields']['group_name'])
self.assertEqual(event['data']['group']['id'], 19734270)
self.assertEqual(event['data']['group']['id'], event['search_fields']['group_id'])
event = events[1]
self.assertEqual(meetup.metadata_id(event['data']), event['search_fields']['item_id'])
self.assertEqual(event['data']['group']['name'], 'sqlpass.es')
self.assertEqual(event['data']['group']['name'], event['search_fields']['group_name'])
self.assertEqual(event['data']['group']['id'], 19734270)
self.assertEqual(event['data']['group']['id'], event['search_fields']['group_id'])
event = events[2]
self.assertEqual(meetup.metadata_id(event['data']), event['search_fields']['item_id'])
self.assertEqual(event['data']['group']['name'], 'sqlpass.es')
self.assertEqual(event['data']['group']['name'], event['search_fields']['group_name'])
self.assertEqual(event['data']['group']['id'], 19734270)
self.assertEqual(event['data']['group']['id'], event['search_fields']['group_id'])
def test_initialization_oauth(self):
"""Test whether attributes are initialized with an oauth token"""
meetup = Meetup('mygroup',
'aaaa',
is_oauth_token=True,
max_items=5,
tag='test',
sleep_for_rate=True,
min_rate_to_sleep=10,
sleep_time=60)
self.assertEqual(meetup.origin, 'https://meetup.com/')
self.assertEqual(meetup.tag, 'test')
self.assertEqual(meetup.group, 'mygroup')
self.assertEqual(meetup.max_items, 5)
self.assertIsNone(meetup.client)
self.assertTrue(meetup.is_oauth_token)
def test_has_archiving(self):
"""Test if it returns True when has_archiving is called"""
self.assertTrue(Meetup.has_archiving())
def test_fetch_to_date(self):
"""Test whether if fetches a set of events updated before the given date"""
http_requests = setup_http_server()
to_date = datetime.datetime(2016, 9, 25)
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [event for event in meetup.fetch(to_date=to_date)]
expected = [('1', '0d07fe36f994a6c78dfcf60fb73674bcf158cb5a',
1460065164.0, 2, 3),
('2', '24b47b622eb33965676dd951b18eea7689b1d81c',
1465503498.0, 2, 3)]
self.assertEqual(len(events), len(expected))
for x in range(len(events)):
event = events[x]
expc = expected[x]
self.assertEqual(event['data']['id'], expc[0])
self.assertEqual(event['uuid'], expc[1])
self.assertEqual(event['origin'], 'https://meetup.com/')
self.assertEqual(event['updated_on'], expc[2])
self.assertEqual(event['category'], 'event')
self.assertEqual(event['tag'], 'https://meetup.com/')
self.assertEqual(len(event['data']['comments']), expc[3])
self.assertEqual(len(event['data']['rsvps']), expc[4])
# Check requests
expected = [{
'fields': [
'event_hosts,featured,group_topics,plain_text_description,rsvpable,series'
],
'key': ['aaaa'],
'order': ['updated'],
'page': ['2'],
'scroll': ['since:1970-01-01T00:00:00.000Z'],
'sign': ['true'],
'status': ['cancelled,upcoming,past,proposed,suggested']
}, {
'key': ['aaaa'],
'page': ['2'],
'sign': ['true']
}, {
'fields': ['attendance_status'],
'key': ['aaaa'],
'page': ['2'],
'response': ['yes,no'],
'sign': ['true']
}, {
'key': ['aaaa'],
'page': ['2'],
'sign': ['true']
}, {
'fields': ['attendance_status'],
'key': ['aaaa'],
'page': ['2'],
'response': ['yes,no'],
'sign': ['true']
}, {
'key': ['aaaa'],
'sign': ['true']
}, {
'key': ['aaaa'],
'page': ['2'],
'sign': ['true']
}, {
'fields': ['attendance_status'],
'key': ['aaaa'],
'page': ['2'],
'response': ['yes,no'],
'sign': ['true']
}]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertDictEqual(http_requests[i].querystring, expected[i])
def test_has_resuming(self):
"""Test if it returns True when has_resuming is called"""
self.assertEqual(Meetup.has_resuming(), True)
def test_initialization_warning(self):
with self.assertWarns(DeprecationWarning):
_ = Meetup('mygroup', 'aaaa')
def test_has_caching(self):
"""Test if it returns False when has_caching is called"""
self.assertEqual(Meetup.has_caching(), False)
def test_fetch_filtering_classified_fields(self):
"""Test it it removes classified fields from a set of fetched items"""
http_requests = setup_http_server()
meetup = Meetup('sqlpass-es', 'aaaa', max_items=2)
events = [
event
for event in meetup.fetch(from_date=None, filter_classified=True)
]
expected = [('1', '0d07fe36f994a6c78dfcf60fb73674bcf158cb5a',
1460065164.0, 2, 3),
('2', '24b47b622eb33965676dd951b18eea7689b1d81c',
1465503498.0, 2, 3),
('3', 'a42b7cf556c17b17f05b951e2eb5e07a7cb0a731',
1474842748.0, 2, 3)]
self.assertEqual(len(events), len(expected))
for x in range(len(events)):
event = events[x]
expc = expected[x]
self.assertEqual(event['data']['id'], expc[0])
self.assertEqual(event['uuid'], expc[1])
self.assertEqual(event['origin'], 'https://meetup.com/')
self.assertEqual(event['updated_on'], expc[2])
self.assertEqual(event['category'], 'event')
self.assertEqual(event['tag'], 'https://meetup.com/')
self.assertEqual(len(event['data']['comments']), expc[3])
# Check classified items filtering
self.assertEqual(event['classified_fields_filtered'],
['group.topics', 'event_hosts', 'rsvps', 'venue'])
self.assertNotIn('topics', event['data']['group'])
self.assertNotIn('event_hosts', event['data'])
self.assertNotIn('rsvps', event['data'])
self.assertNotIn('venue', event['data'])
# Check requests
expected = [{
'fields': [
'event_hosts,featured,group_topics,plain_text_description,rsvpable,series'
],
'order': ['updated'],
'page': ['2'],
'scroll': ['since:1970-01-01T00:00:00.000Z'],
'status': ['cancelled,upcoming,past,proposed,suggested']
}, {
'page': ['2']
}, {
'fields': ['attendance_status'],
'page': ['2'],
'response': ['yes,no']
}, {
'page': ['2']
}, {
'fields': ['attendance_status'],
'page': ['2'],
'response': ['yes,no']
}, {
'order': ['updated'],
'page': ['2'],
'scroll': ['since:1970-01-01T00:00:00.000Z']
}, {
'page': ['2']
}, {
'fields': ['attendance_status'],
'page': ['2'],
'response': ['yes,no']
}]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertIn((MeetupClient.PKEY_OAUTH2, 'Bearer aaaa'),
http_requests[i].headers._headers)
self.assertDictEqual(http_requests[i].querystring, expected[i])
