def test_write_to_datastore(self, mock_session_return):
mock_session = mock_session_return.return_value = create_mock_session()
scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)
ii = ingest_info.IngestInfo()
ii.create_person(full_name=TEST_NAME).create_booking(
booking_id=TEST_ID)
t = Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT,
)
task_hash = hash(json.dumps(t.to_serializable(), sort_keys=True))
expected_batch = BatchIngestInfoData(ingest_info=ii,
task_hash=task_hash)
batch_persistence.write(ii, scrape_key, t)
batch_ingest_info_list = batch_persistence._get_batch_ingest_info_list(
scrape_key.region_code, mock_session.start)
self.assertEqual(len(batch_ingest_info_list), 1)
self.assertEqual(expected_batch, batch_ingest_info_list[0])
def write(ingest_info: IngestInfo, scrape_key: ScrapeKey, task: Task):
session = sessions.get_current_session(scrape_key)
if not session:
raise DatastoreError(scrape_key.region_code, "write")
datastore_ingest_info.write_ingest_info(
region=scrape_key.region_code,
session_start_time=session.start,
ingest_info=ingest_info,
task_hash=hash(json.dumps(task.to_serializable(), sort_keys=True)))
def write_error(error: str, trace_id: Optional[str], task: Task,
scrape_key: ScrapeKey):
session = sessions.get_current_session(scrape_key)
if not session:
raise DatastoreError(scrape_key.region_code, "write_error")
datastore_ingest_info.write_error(region=scrape_key.region_code,
error=error, trace_id=trace_id,
task_hash=hash(json.dumps(
task.to_serializable(),
sort_keys=True)),
session_start_time=session.start)
def test_write_error_to_datastore(self, mock_session_return):
mock_session = mock_session_return.return_value = create_mock_session()
scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)
error = TEST_ERROR
t = Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT,
)
task_hash = hash(json.dumps(t.to_serializable(), sort_keys=True))
expected_batch = BatchIngestInfoData(error=error, trace_id=TEST_TRACE,
task_hash=task_hash)
batch_persistence.write_error(error, TEST_TRACE, t, scrape_key)
batch_ingest_info_list = batch_persistence._get_batch_ingest_info_list(
scrape_key.region_code, mock_session.start)
self.assertEqual(len(batch_ingest_info_list), 1)
self.assertEqual(expected_batch, batch_ingest_info_list[0])
