def test_iterate_docket_item(self): scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) self.create_session(scrape_key) docket.add_to_query_docket(scrape_key, get_payload()[0]).result() docket.add_to_query_docket(scrape_key, get_payload()[1]).result() assert tracker.iterate_docket_item(scrape_key) == get_payload()[0] assert tracker.iterate_docket_item(scrape_key) == get_payload()[1]
def test_remove_item_from_session_and_docket(self): scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) docket.add_to_query_docket(scrape_key, get_payload()).result() self.create_session(scrape_key) tracker.iterate_docket_item(scrape_key) tracker.remove_item_from_session_and_docket(scrape_key) assert not sessions.get_current_session(scrape_key).docket_ack_id
def test_iterate_docket_item_no_matching_items(self, mock_docket): mock_docket.return_value = None payload = tracker.iterate_docket_item( ScrapeKey("us_fl", constants.ScrapeType.BACKGROUND) ) assert not payload
def test_iterate_docket_item_no_matching_items(self): docket_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) docket.add_to_query_docket(docket_key, get_payload()).result() session_key = ScrapeKey(REGIONS[1], constants.ScrapeType.BACKGROUND) self.create_session(session_key) assert not tracker.iterate_docket_item(session_key)
def test_iterate_docket_item(self, mock_session, mock_docket): mock_session.return_value = True mock_docket.return_value = create_pubsub_message(get_payload()) payload = tracker.iterate_docket_item( ScrapeKey("us_ny", constants.ScrapeType.BACKGROUND)) assert payload == get_payload()
def test_purge_docket_and_session(self): scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) docket.add_to_query_docket(scrape_key, get_payload()).result() self.create_session(scrape_key) tracker.purge_docket_and_session(scrape_key) assert not tracker.iterate_docket_item(scrape_key)
def test_iterate_docket_item_no_open_session_to_update( self, mock_session, mock_docket): mock_session.return_value = False mock_docket.return_value = create_pubsub_message(get_payload()) payload = tracker.iterate_docket_item( ScrapeKey("us_ny", constants.ScrapeType.BACKGROUND)) assert not payload
def iterate_docket_item(self, scrape_type): """Leases new docket item, updates current session, returns item contents Returns an entity to scrape as provided by the docket item. Args: scrape_type: (string) Type of docket item to retrieve Returns: False if there was any failure to retrieve a new docket item. If successful: Background scrape: ("surname", "given names") """ item_content = tracker.iterate_docket_item( ScrapeKey(self.get_region().region_code, scrape_type)) if item_content is None: return False return item_content
def test_iterate_docket_item_no_open_session_to_update(self): scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) docket.add_to_query_docket(scrape_key, get_payload()).result() assert not tracker.iterate_docket_item(scrape_key)