def test__raises_ScraperFailureException_when_GET_on_coracle_endpoint_fails(self):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, self.coracle_timestamp_endpoint,
status=500)
scrape = sap.scraper(coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
self.assertRaises(ex.ScrapeFailureException, scrape)
def test__raises_ScraperFailureException_when_GET_on_openahjo_endpoint_fails(self):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, self.coracle_timestamp_endpoint,
body=self.coracle_timestamp_response_body, status=200,
content_type="application/json")
rsps.add(responses.GET, self.openahjo_endpoint,
status=500)
scrape = sap.scraper(coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
self.assertRaises(ex.ScrapeFailureException, scrape)
def test__raises_ScraperFailureException_when_GET_on_coracle_endpoint_fails(
self):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
self.coracle_timestamp_endpoint,
status=500)
scrape = sap.scraper(
coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
self.assertRaises(ex.ScrapeFailureException, scrape)
def test__raises_ScraperFailureException_when_GET_on_openahjo_endpoint_fails(
self):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
self.coracle_timestamp_endpoint,
body=self.coracle_timestamp_response_body,
status=200,
content_type="application/json")
rsps.add(responses.GET, self.openahjo_endpoint, status=500)
scrape = sap.scraper(
coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
self.assertRaises(ex.ScrapeFailureException, scrape)
def test__scrapes_source_endpoint_without_last_modified_time_if_no_time_is_returned_by_coracle(self):
query_string = '?' + self.order_by
openahjo_endpoint_with_query_string = self.openahjo_endpoint + query_string
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, self.coracle_timestamp_endpoint,
body="{}", status=200,
content_type="application/json")
rsps.add(responses.GET, openahjo_endpoint_with_query_string,
body=self.openahjo_response_body, status=200,
content_type="application/json",
match_querystring=True)
scrape = sap.scraper(coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
agenda_items = scrape()
self.assertEquals(agenda_items, [{'Object': '123'}])
def create_scheduler():
logging.basicConfig(level=logging.INFO)
application = flask.Flask(__name__)
coracle_timestamp_endpoint = os.environ['CORACLE_TIMESTAMP_ENDPOINT']
coracle_post_activity_endpoint = os.environ['CORACLE_POST_ACTIVITY_ENDPOINT']
openahjo_endpoint = os.environ['OPENAHJO_ENDPOINT']
bearer_token = os.environ['BEARER_TOKEN']
scrape = scraper(coracle_timestamp_endpoint, openahjo_endpoint)
push = pusher(coracle_post_activity_endpoint, bearer_token)
logging.basicConfig(level=logging.INFO)
scrape_and_push_event_loop = scrape_and_push(scrape=scrape,
convert=convert.agenda_item_to_activity,
push=push)
clock = Clock()
s = Scheduler(interval=3600, clock=clock, stop_when=lambda: False, event=scrape_and_push_event_loop)
s.start()
return application
def create_scheduler():
logging.basicConfig(level=logging.INFO)
application = flask.Flask(__name__)
coracle_timestamp_endpoint = os.environ['CORACLE_TIMESTAMP_ENDPOINT']
coracle_post_activity_endpoint = os.environ[
'CORACLE_POST_ACTIVITY_ENDPOINT']
openahjo_endpoint = os.environ['OPENAHJO_ENDPOINT']
bearer_token = os.environ['BEARER_TOKEN']
scrape = scraper(coracle_timestamp_endpoint, openahjo_endpoint)
push = pusher(coracle_post_activity_endpoint, bearer_token)
logging.basicConfig(level=logging.INFO)
scrape_and_push_event_loop = scrape_and_push(
scrape=scrape, convert=convert.agenda_item_to_activity, push=push)
clock = Clock()
s = Scheduler(interval=3600,
clock=clock,
stop_when=lambda: False,
event=scrape_and_push_event_loop)
s.start()
return application
def test__scrapes_source_endpoint_without_last_modified_time_if_no_time_is_returned_by_coracle(
self):
query_string = '?' + self.order_by
openahjo_endpoint_with_query_string = self.openahjo_endpoint + query_string
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
self.coracle_timestamp_endpoint,
body="{}",
status=200,
content_type="application/json")
rsps.add(responses.GET,
openahjo_endpoint_with_query_string,
body=self.openahjo_response_body,
status=200,
content_type="application/json",
match_querystring=True)
scrape = sap.scraper(
coracle_endpoint=self.coracle_timestamp_endpoint,
openahjo_endpoint=self.openahjo_endpoint)
agenda_items = scrape()
self.assertEquals(agenda_items, [{'Object': '123'}])