def harvest():
query = DBSession.query(Source)
for source in query:
error_message = u"Failed to harvest source {id} with URL {url}".format(
id=source.id,
url=source.url,
)
try:
response = requests.get(source.url)
if response.status_code != 200:
continue
content = response.text
try:
json.loads(content)
is_json = True
except ValueError:
is_json = False
request = HarvestRequest(content)
if is_json:
cstruct = json_extractor(request)
else:
cstruct = icalendar_extractor(request)
harvest_cstruct(cstruct, source)
if request.errors:
log.warning(error_message, exc_info=True)
for error in request.errors:
log.warning(error['description'])
except Exception:
log.warning(error_message, exc_info=True)
DBSession.flush()
def test_bogus_icalendar_data_does_not_crash_harvesting(self):
log_mock = self.patch('ode.harvesting.log')
source1 = self.make_source(url=u"http://example.com/a",
provider_id='123')
self.make_source(url=u"http://example.com/b", provider_id='456')
self.setup_requests_mock()
self.mock_requests.get.side_effect = [
Mock(
status_code=200,
content_type='text/calendar',
text='*** BOGUS DATA ***',
),
Mock(
status_code=200,
content_type='text/calendar',
text=valid_icalendar,
),
]
harvest()
self.assertEqual(DBSession.query(Event).count(), 1)
expected_messasge = "Invalid iCalendar request body: "
actual_message = log_mock.warning.call_args[0][0]
self.assertTrue(actual_message.startswith(expected_messasge))
log_mock.warning.assert_any_call(
u"Failed to harvest source {} with URL {}".format(
source1.id, source1.url),
exc_info=True)
def test_post_single_event(self):
calendar = self.make_icalendar(titles=[u'Événement'])
response = self.post(calendar)
items = response.json['collection']['items']
event = DBSession.query(Event).filter_by(title=u'Événement').one()
self.assertEqual(items[0]['href'],
'http://localhost/v1/events/%s' % quote(event.id))
def harvest():
query = DBSession.query(Source)
for source in query:
error_message = u"Failed to harvest source {id} with URL {url}".format(
id=source.id,
url=source.url,
)
try:
response = requests.get(source.url)
if response.status_code != 200:
continue
content = response.text
try:
json.loads(content)
is_json = True
except ValueError:
is_json = False
request = HarvestRequest(content)
if is_json:
cstruct = json_extractor(request)
else:
cstruct = icalendar_extractor(request)
harvest_cstruct(cstruct, source)
if request.errors:
log.warning(error_message, exc_info=True)
for error in request.errors:
log.warning(error['description'])
except Exception:
log.warning(error_message, exc_info=True)
DBSession.flush()
def test_post_single_event(self):
calendar = self.make_icalendar(titles=[u'Événement'])
response = self.post(calendar)
items = response.json['collection']['items']
event = DBSession.query(Event).filter_by(title=u'Événement').one()
self.assertEqual(items[0]['href'],
'http://localhost/v1/events/%s' % quote(event.id))
def test_post(self):
body_data = """title,description,start_time,end_time
Événement,A great event,2014-01-25T15:00,2014-01-25T19:00
Autre Événement,Another great event,2014-01-26T15:00,2014-01-26T19:00
"""
self.app.post('/v1/events',
body_data,
status=201,
headers={
'Content-Type': 'text/csv',
'X-ODE-Provider-Id': '123',
})
self.assertEqual(DBSession.query(Event).count(), 2)
event = DBSession.query(Event).filter_by(title=u'Événement').one()
self.assertEqual(event.description, u'A great event')
def test_bogus_icalendar_data_does_not_crash_harvesting(self):
log_mock = self.patch('ode.harvesting.log')
source1 = self.make_source(url=u"http://example.com/a",
provider_id='123')
self.make_source(url=u"http://example.com/b", provider_id='456')
self.setup_requests_mock()
self.mock_requests.get.side_effect = [
Mock(
status_code=200,
content_type='text/calendar',
text='*** BOGUS DATA ***',
),
Mock(
status_code=200,
content_type='text/calendar',
text=valid_icalendar,
),
]
harvest()
self.assertEqual(DBSession.query(Event).count(), 1)
expected_messasge = "Invalid iCalendar request body: "
actual_message = log_mock.warning.call_args[0][0]
self.assertTrue(actual_message.startswith(expected_messasge))
log_mock.warning.assert_any_call(
u"Failed to harvest source {} with URL {}".format(
source1.id, source1.url),
exc_info=True)
def put(self):
"""Update existing event by id"""
event_id = self.request.matchdict['id']
query = DBSession.query(Event).filter_by(id=event_id)
if not query.update(self.request.validated):
raise HTTPNotFound()
return {'status': 'updated'}
def test_bogus_json_data(self):
log_mock = self.patch('ode.harvesting.log')
self.make_source(url=u"http://example.com/a", provider_id='123')
self.setup_requests_mock(content_type='text/json',
body_text=u'{"foo": 42}')
harvest()
self.assertEqual(DBSession.query(Event).count(), 0)
log_mock.warning.assert_any_call(u"Invalid Collection+JSON input")
def test_bogus_json_data(self):
log_mock = self.patch('ode.harvesting.log')
self.make_source(url=u"http://example.com/a", provider_id='123')
self.setup_requests_mock(content_type='text/json',
body_text=u'{"foo": 42}')
harvest()
self.assertEqual(DBSession.query(Event).count(), 0)
log_mock.warning.assert_any_call(u"Invalid Collection+JSON input")
def test_update_from_uid_with_domain_part(self):
self.create_event(title=u'Existing event', id=u'*****@*****.**')
DBSession.flush()
self.setup_requests_mock(body_text=valid_icalendar)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Capitole du Libre")
def delete(self):
"""Delete a specific event by id"""
id = self.request.matchdict['id']
try:
event = DBSession.query(Event).filter_by(id=id).one()
except NoResultFound:
raise HTTPNotFound()
DBSession.delete(event)
return {'status': 'deleted'}
def test_update_from_uid_with_domain_part(self):
self.create_event(title=u'Existing event', id=u'*****@*****.**')
DBSession.flush()
self.setup_requests_mock(body_text=valid_icalendar)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Capitole du Libre")
def test_fetch_json_data_from_source(self):
self.setup_requests_mock(content_type='text/json',
body_text=valid_json)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Test medias")
self.assertEqual(event.description, u"Description")
def get(self):
"""Get a specific resource by id"""
id = self.request.matchdict['id']
try:
resource = DBSession.query(self.model).filter_by(id=id).one()
except NoResultFound:
raise HTTPNotFound()
items = [resource.to_item(self.request)]
return self.collection_json(items)
def get(self):
"""Get a specific resource by id"""
id = self.request.matchdict['id']
try:
resource = DBSession.query(self.model).filter_by(id=id).one()
except NoResultFound:
raise HTTPNotFound()
items = [resource.to_item(self.request)]
return self.collection_json(items)
def test_fetch_json_data_from_source(self):
self.setup_requests_mock(content_type='text/json',
body_text=valid_json)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Test medias")
self.assertEqual(event.description,
u"Description")
def delete(self):
"""Delete a resource by id"""
id = self.request.matchdict['id']
try:
event = DBSession.query(self.model).filter_by(
id=id,
provider_id=self.request.validated['provider_id'],
).one()
except NoResultFound:
raise HTTPNotFound()
DBSession.delete(event)
def test_fetch_data_from_source(self):
self.setup_requests_mock()
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Capitole du Libre")
self.assertEqual(
event.url, u"http://www.agendadulibre.org/showevent.php?id=7064")
self.assertEqual(event.description,
u"Un évènement de l'Agenda du Libre")
def test_fetch_data_from_source(self):
self.setup_requests_mock()
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
event = DBSession.query(Event).one()
self.assertEqual(event.title, u"Capitole du Libre")
self.assertEqual(event.url,
u"http://www.agendadulibre.org/showevent.php?id=7064")
self.assertEqual(event.description,
u"Un évènement de l'Agenda du Libre")
def delete(self):
"""Delete a resource by id"""
id = self.request.matchdict['id']
try:
event = DBSession.query(self.model).filter_by(
id=id,
provider_id=self.request.validated['provider_id'],
).one()
except NoResultFound:
raise HTTPNotFound()
DBSession.delete(event)
def get(self):
"""Get a specific event by id"""
id = self.request.matchdict['id']
try:
event = DBSession.query(Event).filter_by(id=id).one()
except NoResultFound:
raise HTTPNotFound()
return {
'status': 'success',
'event': event.json_data(),
}
def test_post_multiple_events(self):
calendar = self.make_icalendar(titles=[u'Événement 1', u'Événement 2'])
response = self.post(calendar)
items = response.json['collection']['items']
event = DBSession.query(Event).filter_by(title=u'Événement 1').one()
for item in items:
for name, value in item['data']:
if name == u'title' and value == u'Événement 1':
expected_url = u'http://localhost/v1/events/%s' % quote(
event.id)
self.assertEqual(items[0]['href'], expected_url)
def test_post_multiple_events(self):
calendar = self.make_icalendar(titles=[u'Événement 1', u'Événement 2'])
response = self.post(calendar)
items = response.json['collection']['items']
event = DBSession.query(Event).filter_by(title=u'Événement 1').one()
for item in items:
for name, value in item['data']:
if name == u'title' and value == u'Événement 1':
expected_url = u'http://localhost/v1/events/%s' % quote(
event.id)
self.assertEqual(items[0]['href'], expected_url)
def put(self):
"""Update an existing resource by id"""
resouce_id = self.request.matchdict['id']
query = DBSession.query(self.model).filter_by(
id=resouce_id,
provider_id=self.request.validated['provider_id'],
)
self.request.validated['provider_id'] = {
'value': self.request.validated['provider_id']
}
event = query.first()
if not event:
raise HTTPNotFound()
event.update_from_appstruct(self.request.validated['items'][0]['data'])
return {'status': 'updated'}
def put(self):
"""Update an existing resource by id"""
resouce_id = self.request.matchdict['id']
query = DBSession.query(self.model).filter_by(
id=resouce_id,
provider_id=self.request.validated['provider_id'],
)
self.request.validated['provider_id'] = {
'value': self.request.validated['provider_id']
}
event = query.first()
if not event:
raise HTTPNotFound()
event.update_from_appstruct(
self.request.validated['items'][0]['data'])
return {'status': 'updated'}
def test_create_source(self):
sources_info = {
'template': {
'data': [
{'name': 'url', 'value': u'http://example.com/mysource'},
{'name': 'active', 'value': True},
]
}
}
response = self.app.post_json('/v1/sources', sources_info,
headers=self.WRITE_HEADERS, status=201)
source = DBSession.query(Source).one()
self.assertEqual(response.headers['location'],
'http://localhost/v1/sources/%s' % source.id)
self.assertEqual(source.url, u'http://example.com/mysource')
self.assertEqual(source.active, True)
def test_update_source(self):
source = self.make_source()
response = self.app.put_json(
'/v1/sources/%s' % source.id,
{
'template': {
'data': [
{'name': 'url',
'value': 'http://example.com/myothersource'},
{'name': 'active', 'value': False}
]
}
},
headers=self.WRITE_HEADERS)
self.assertEqual(response.json['status'], 'updated')
source = DBSession.query(Source).one()
self.assertEqual(source.url, u'http://example.com/myothersource')
self.assertEqual(source.active, False)
def test_update_source(self):
source = self.make_source()
response = self.app.put_json('/v1/sources/%s' % source.id, {
'template': {
'data': [{
'name': 'url',
'value': 'http://example.com/myothersource'
}, {
'name': 'active',
'value': False
}]
}
},
headers=self.WRITE_HEADERS)
self.assertEqual(response.json['status'], 'updated')
source = DBSession.query(Source).one()
self.assertEqual(source.url, u'http://example.com/myothersource')
self.assertEqual(source.active, False)
def collection_get_query(self):
query = DBSession.query(self.model)
query = self.collection_get_filter_query(query)
sort_by = self.request.validated.get('sort_by')
if sort_by:
order_criterion = getattr(self.model, sort_by, None)
if order_criterion:
if self.request.validated['sort_direction'] == 'desc':
order_criterion = order_criterion.desc()
query = query.order_by(order_criterion)
else:
message = self._(
u"${sort_by} is not a valid sorting criterion",
mapping={'sort_by': sort_by})
raise HTTPBadRequest(message)
total_count = query.count()
limit = self.request.validated.get('limit', COLLECTION_MAX_LENGTH)
query = query.limit(limit)
offset = self.request.validated.get('offset')
if offset:
query = query.offset(offset)
return (query, total_count)
def collection_get_query(self):
query = DBSession.query(self.model)
query = self.collection_get_filter_query(query)
sort_by = self.request.validated.get('sort_by')
if sort_by:
order_criterion = getattr(self.model, sort_by, None)
if order_criterion:
if self.request.validated['sort_direction'] == 'desc':
order_criterion = order_criterion.desc()
query = query.order_by(order_criterion)
else:
message = self._(
u"${sort_by} is not a valid sorting criterion",
mapping={'sort_by': sort_by})
raise HTTPBadRequest(message)
total_count = query.count()
limit = self.request.validated.get('limit', COLLECTION_MAX_LENGTH)
query = query.limit(limit)
offset = self.request.validated.get('offset')
if offset:
query = query.offset(offset)
return (query, total_count)
def test_create_source(self):
sources_info = {
'template': {
'data': [
{
'name': 'url',
'value': u'http://example.com/mysource'
},
{
'name': 'active',
'value': True
},
]
}
}
response = self.app.post_json('/v1/sources',
sources_info,
headers=self.WRITE_HEADERS,
status=201)
source = DBSession.query(Source).one()
self.assertEqual(response.headers['location'],
'http://localhost/v1/sources/%s' % source.id)
self.assertEqual(source.url, u'http://example.com/mysource')
self.assertEqual(source.active, True)
def test_delete_event(self):
id = self.post_event()
self.app.delete('/events/%s' % id)
self.assertEqual(DBSession.query(Event).count(), 0)
def exists_in_database(self):
uid = self.cstruct['data']['id']
return DBSession.query(Event).filter_by(id=uid).count() > 0
def assertSourceCount(self, count):
self.assertEqual(DBSession.query(Source).count(), count)
def test_invalid_calendar(self):
self.setup_requests_mock(body_text=start_time_missing)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
self.assertEqual(DBSession.query(Event).count(), 0)
def test_delete_event(self):
id = self.post_event()
self.app.delete('/v1/events/%s' % id,
headers=self.PROVIDER_ID_HEADER,
status=204)
self.assertEqual(DBSession.query(Event).count(), 0)
def test_invalid_calendar(self):
self.setup_requests_mock(body_text=start_time_missing)
source = self.make_source()
harvest()
self.mock_requests.get.assert_called_with(source.url)
self.assertEqual(DBSession.query(Event).count(), 0)
def exists_in_database(self):
uid = self.cstruct['data']['id']
return DBSession.query(Event).filter_by(id=uid).count() > 0
def collection_get(self):
"""Get list of all events"""
query = DBSession.query(Event).all()
events = [event.json_data() for event in query]
return {'events': events}
def test_delete_event(self):
id = self.post_event()
self.app.delete('/v1/events/%s' % id,
headers=self.PROVIDER_ID_HEADER,
status=204)
self.assertEqual(DBSession.query(Event).count(), 0)
def test_post_all_fields(self):
event_id = self.post_event(self.example_data)
event = DBSession.query(Event).filter_by(id=event_id).first()
self.assertDictEqual(event.json_data(), self.example_data)
def assertTitleEqual(self, event_id, title):
event = DBSession.query(Event).filter_by(id=event_id).first()
self.assertEqual(event.title, title)
def assertSourceCount(self, count):
self.assertEqual(DBSession.query(Source).count(), count)