def harvest(): query = DBSession.query(Source) for source in query: error_message = u"Failed to harvest source {id} with URL {url}".format( id=source.id, url=source.url, ) try: response = requests.get(source.url) if response.status_code != 200: continue content = response.text try: json.loads(content) is_json = True except ValueError: is_json = False request = HarvestRequest(content) if is_json: cstruct = json_extractor(request) else: cstruct = icalendar_extractor(request) harvest_cstruct(cstruct, source) if request.errors: log.warning(error_message, exc_info=True) for error in request.errors: log.warning(error['description']) except Exception: log.warning(error_message, exc_info=True) DBSession.flush()
def test_bogus_icalendar_data_does_not_crash_harvesting(self): log_mock = self.patch('ode.harvesting.log') source1 = self.make_source(url=u"http://example.com/a", provider_id='123') self.make_source(url=u"http://example.com/b", provider_id='456') self.setup_requests_mock() self.mock_requests.get.side_effect = [ Mock( status_code=200, content_type='text/calendar', text='*** BOGUS DATA ***', ), Mock( status_code=200, content_type='text/calendar', text=valid_icalendar, ), ] harvest() self.assertEqual(DBSession.query(Event).count(), 1) expected_messasge = "Invalid iCalendar request body: " actual_message = log_mock.warning.call_args[0][0] self.assertTrue(actual_message.startswith(expected_messasge)) log_mock.warning.assert_any_call( u"Failed to harvest source {} with URL {}".format( source1.id, source1.url), exc_info=True)
def test_post_single_event(self): calendar = self.make_icalendar(titles=[u'Événement']) response = self.post(calendar) items = response.json['collection']['items'] event = DBSession.query(Event).filter_by(title=u'Événement').one() self.assertEqual(items[0]['href'], 'http://localhost/v1/events/%s' % quote(event.id))
def test_post(self): body_data = """title,description,start_time,end_time Événement,A great event,2014-01-25T15:00,2014-01-25T19:00 Autre Événement,Another great event,2014-01-26T15:00,2014-01-26T19:00 """ self.app.post('/v1/events', body_data, status=201, headers={ 'Content-Type': 'text/csv', 'X-ODE-Provider-Id': '123', }) self.assertEqual(DBSession.query(Event).count(), 2) event = DBSession.query(Event).filter_by(title=u'Événement').one() self.assertEqual(event.description, u'A great event')
def put(self): """Update existing event by id""" event_id = self.request.matchdict['id'] query = DBSession.query(Event).filter_by(id=event_id) if not query.update(self.request.validated): raise HTTPNotFound() return {'status': 'updated'}
def test_bogus_json_data(self): log_mock = self.patch('ode.harvesting.log') self.make_source(url=u"http://example.com/a", provider_id='123') self.setup_requests_mock(content_type='text/json', body_text=u'{"foo": 42}') harvest() self.assertEqual(DBSession.query(Event).count(), 0) log_mock.warning.assert_any_call(u"Invalid Collection+JSON input")
def test_update_from_uid_with_domain_part(self): self.create_event(title=u'Existing event', id=u'*****@*****.**') DBSession.flush() self.setup_requests_mock(body_text=valid_icalendar) source = self.make_source() harvest() self.mock_requests.get.assert_called_with(source.url) event = DBSession.query(Event).one() self.assertEqual(event.title, u"Capitole du Libre")
def delete(self): """Delete a specific event by id""" id = self.request.matchdict['id'] try: event = DBSession.query(Event).filter_by(id=id).one() except NoResultFound: raise HTTPNotFound() DBSession.delete(event) return {'status': 'deleted'}
def test_fetch_json_data_from_source(self): self.setup_requests_mock(content_type='text/json', body_text=valid_json) source = self.make_source() harvest() self.mock_requests.get.assert_called_with(source.url) event = DBSession.query(Event).one() self.assertEqual(event.title, u"Test medias") self.assertEqual(event.description, u"Description")
def get(self): """Get a specific resource by id""" id = self.request.matchdict['id'] try: resource = DBSession.query(self.model).filter_by(id=id).one() except NoResultFound: raise HTTPNotFound() items = [resource.to_item(self.request)] return self.collection_json(items)
def delete(self): """Delete a resource by id""" id = self.request.matchdict['id'] try: event = DBSession.query(self.model).filter_by( id=id, provider_id=self.request.validated['provider_id'], ).one() except NoResultFound: raise HTTPNotFound() DBSession.delete(event)
def test_fetch_data_from_source(self): self.setup_requests_mock() source = self.make_source() harvest() self.mock_requests.get.assert_called_with(source.url) event = DBSession.query(Event).one() self.assertEqual(event.title, u"Capitole du Libre") self.assertEqual( event.url, u"http://www.agendadulibre.org/showevent.php?id=7064") self.assertEqual(event.description, u"Un évènement de l'Agenda du Libre")
def test_fetch_data_from_source(self): self.setup_requests_mock() source = self.make_source() harvest() self.mock_requests.get.assert_called_with(source.url) event = DBSession.query(Event).one() self.assertEqual(event.title, u"Capitole du Libre") self.assertEqual(event.url, u"http://www.agendadulibre.org/showevent.php?id=7064") self.assertEqual(event.description, u"Un évènement de l'Agenda du Libre")
def get(self): """Get a specific event by id""" id = self.request.matchdict['id'] try: event = DBSession.query(Event).filter_by(id=id).one() except NoResultFound: raise HTTPNotFound() return { 'status': 'success', 'event': event.json_data(), }
def test_post_multiple_events(self): calendar = self.make_icalendar(titles=[u'Événement 1', u'Événement 2']) response = self.post(calendar) items = response.json['collection']['items'] event = DBSession.query(Event).filter_by(title=u'Événement 1').one() for item in items: for name, value in item['data']: if name == u'title' and value == u'Événement 1': expected_url = u'http://localhost/v1/events/%s' % quote( event.id) self.assertEqual(items[0]['href'], expected_url)
def put(self): """Update an existing resource by id""" resouce_id = self.request.matchdict['id'] query = DBSession.query(self.model).filter_by( id=resouce_id, provider_id=self.request.validated['provider_id'], ) self.request.validated['provider_id'] = { 'value': self.request.validated['provider_id'] } event = query.first() if not event: raise HTTPNotFound() event.update_from_appstruct(self.request.validated['items'][0]['data']) return {'status': 'updated'}
def put(self): """Update an existing resource by id""" resouce_id = self.request.matchdict['id'] query = DBSession.query(self.model).filter_by( id=resouce_id, provider_id=self.request.validated['provider_id'], ) self.request.validated['provider_id'] = { 'value': self.request.validated['provider_id'] } event = query.first() if not event: raise HTTPNotFound() event.update_from_appstruct( self.request.validated['items'][0]['data']) return {'status': 'updated'}
def test_create_source(self): sources_info = { 'template': { 'data': [ {'name': 'url', 'value': u'http://example.com/mysource'}, {'name': 'active', 'value': True}, ] } } response = self.app.post_json('/v1/sources', sources_info, headers=self.WRITE_HEADERS, status=201) source = DBSession.query(Source).one() self.assertEqual(response.headers['location'], 'http://localhost/v1/sources/%s' % source.id) self.assertEqual(source.url, u'http://example.com/mysource') self.assertEqual(source.active, True)
def test_update_source(self): source = self.make_source() response = self.app.put_json( '/v1/sources/%s' % source.id, { 'template': { 'data': [ {'name': 'url', 'value': 'http://example.com/myothersource'}, {'name': 'active', 'value': False} ] } }, headers=self.WRITE_HEADERS) self.assertEqual(response.json['status'], 'updated') source = DBSession.query(Source).one() self.assertEqual(source.url, u'http://example.com/myothersource') self.assertEqual(source.active, False)
def test_update_source(self): source = self.make_source() response = self.app.put_json('/v1/sources/%s' % source.id, { 'template': { 'data': [{ 'name': 'url', 'value': 'http://example.com/myothersource' }, { 'name': 'active', 'value': False }] } }, headers=self.WRITE_HEADERS) self.assertEqual(response.json['status'], 'updated') source = DBSession.query(Source).one() self.assertEqual(source.url, u'http://example.com/myothersource') self.assertEqual(source.active, False)
def collection_get_query(self): query = DBSession.query(self.model) query = self.collection_get_filter_query(query) sort_by = self.request.validated.get('sort_by') if sort_by: order_criterion = getattr(self.model, sort_by, None) if order_criterion: if self.request.validated['sort_direction'] == 'desc': order_criterion = order_criterion.desc() query = query.order_by(order_criterion) else: message = self._( u"${sort_by} is not a valid sorting criterion", mapping={'sort_by': sort_by}) raise HTTPBadRequest(message) total_count = query.count() limit = self.request.validated.get('limit', COLLECTION_MAX_LENGTH) query = query.limit(limit) offset = self.request.validated.get('offset') if offset: query = query.offset(offset) return (query, total_count)
def test_create_source(self): sources_info = { 'template': { 'data': [ { 'name': 'url', 'value': u'http://example.com/mysource' }, { 'name': 'active', 'value': True }, ] } } response = self.app.post_json('/v1/sources', sources_info, headers=self.WRITE_HEADERS, status=201) source = DBSession.query(Source).one() self.assertEqual(response.headers['location'], 'http://localhost/v1/sources/%s' % source.id) self.assertEqual(source.url, u'http://example.com/mysource') self.assertEqual(source.active, True)
def test_delete_event(self): id = self.post_event() self.app.delete('/events/%s' % id) self.assertEqual(DBSession.query(Event).count(), 0)
def exists_in_database(self): uid = self.cstruct['data']['id'] return DBSession.query(Event).filter_by(id=uid).count() > 0
def assertSourceCount(self, count): self.assertEqual(DBSession.query(Source).count(), count)
def test_invalid_calendar(self): self.setup_requests_mock(body_text=start_time_missing) source = self.make_source() harvest() self.mock_requests.get.assert_called_with(source.url) self.assertEqual(DBSession.query(Event).count(), 0)
def test_delete_event(self): id = self.post_event() self.app.delete('/v1/events/%s' % id, headers=self.PROVIDER_ID_HEADER, status=204) self.assertEqual(DBSession.query(Event).count(), 0)
def collection_get(self): """Get list of all events""" query = DBSession.query(Event).all() events = [event.json_data() for event in query] return {'events': events}
def test_post_all_fields(self): event_id = self.post_event(self.example_data) event = DBSession.query(Event).filter_by(id=event_id).first() self.assertDictEqual(event.json_data(), self.example_data)
def assertTitleEqual(self, event_id, title): event = DBSession.query(Event).filter_by(id=event_id).first() self.assertEqual(event.title, title)