def test_write_to_datastore(self, mock_session_return): mock_session = mock_session_return.return_value = create_mock_session() scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) ii = ingest_info.IngestInfo() ii.create_person(full_name=TEST_NAME).create_booking( booking_id=TEST_ID) t = Task( task_type=constants.TaskType.SCRAPE_DATA, endpoint=TEST_ENDPOINT, response_type=constants.ResponseType.TEXT, ) task_hash = hash(json.dumps(t.to_serializable(), sort_keys=True)) expected_batch = BatchIngestInfoData(ingest_info=ii, task_hash=task_hash) batch_persistence.write(ii, scrape_key, t) batch_ingest_info_list = batch_persistence._get_batch_ingest_info_list( scrape_key.region_code, mock_session.start) self.assertEqual(len(batch_ingest_info_list), 1) self.assertEqual(expected_batch, batch_ingest_info_list[0])
def write(ingest_info: IngestInfo, scrape_key: ScrapeKey, task: Task): session = sessions.get_current_session(scrape_key) if not session: raise DatastoreError(scrape_key.region_code, "write") datastore_ingest_info.write_ingest_info( region=scrape_key.region_code, session_start_time=session.start, ingest_info=ingest_info, task_hash=hash(json.dumps(task.to_serializable(), sort_keys=True)))
def write_error(error: str, trace_id: Optional[str], task: Task, scrape_key: ScrapeKey): session = sessions.get_current_session(scrape_key) if not session: raise DatastoreError(scrape_key.region_code, "write_error") datastore_ingest_info.write_error(region=scrape_key.region_code, error=error, trace_id=trace_id, task_hash=hash(json.dumps( task.to_serializable(), sort_keys=True)), session_start_time=session.start)
def test_write_error_to_datastore(self, mock_session_return): mock_session = mock_session_return.return_value = create_mock_session() scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND) error = TEST_ERROR t = Task( task_type=constants.TaskType.SCRAPE_DATA, endpoint=TEST_ENDPOINT, response_type=constants.ResponseType.TEXT, ) task_hash = hash(json.dumps(t.to_serializable(), sort_keys=True)) expected_batch = BatchIngestInfoData(error=error, trace_id=TEST_TRACE, task_hash=task_hash) batch_persistence.write_error(error, TEST_TRACE, t, scrape_key) batch_ingest_info_list = batch_persistence._get_batch_ingest_info_list( scrape_key.region_code, mock_session.start) self.assertEqual(len(batch_ingest_info_list), 1) self.assertEqual(expected_batch, batch_ingest_info_list[0])