def test_batch_delete_over_500_ingest_infos_for_region(self):
task_hash = hash(
json.dumps(
Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT,
).to_serializable(),
sort_keys=True,
))
start_time = datetime.now()
# The Datastore limit for entity writes in one call is 500. Confirm
# that batch delete is properly handled when more than 500 entities
# exist for the same region.
for i in range(600):
datastore_ingest_info.write_ingest_info(
region="us_state_county",
session_start_time=start_time,
ingest_info=sample_ingest_info(str(i)),
task_hash=task_hash,
)
datastore_ingest_info.batch_delete_ingest_infos_for_region(
"us_state_county")
assert (datastore_ingest_info.batch_get_ingest_infos_for_region(
"us_state_county", start_time) == [])
def write(ingest_info: IngestInfo, scrape_key: ScrapeKey, task: Task):
session = sessions.get_current_session(scrape_key)
if not session:
raise DatastoreError(scrape_key.region_code, "write")
datastore_ingest_info.write_ingest_info(
region=scrape_key.region_code,
session_start_time=session.start,
ingest_info=ingest_info,
task_hash=hash(json.dumps(task.to_serializable(), sort_keys=True)))
def test_batch_get_ingest_infos_for_region(self):
task_hash = hash(
json.dumps(
Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT,
).to_serializable(),
sort_keys=True,
))
start_time = datetime.now()
first = datastore_ingest_info.write_ingest_info(
region="us_state_county",
session_start_time=start_time,
ingest_info=sample_ingest_info("1"),
task_hash=task_hash,
)
second = datastore_ingest_info.write_ingest_info(
region="us_state_county",
session_start_time=start_time,
ingest_info=sample_ingest_info("2"),
task_hash=task_hash,
)
third = datastore_ingest_info.write_ingest_info(
region="us_state_county",
session_start_time=start_time,
ingest_info=sample_ingest_info("3"),
task_hash=task_hash,
)
datastore_ingest_info.write_ingest_info(
region="unrelated",
session_start_time=start_time,
ingest_info=sample_ingest_info("n/a"),
task_hash=task_hash,
)
results = datastore_ingest_info.batch_get_ingest_infos_for_region(
"us_state_county", start_time)
assert results == [first, second, third]
datastore_ingest_info.batch_delete_ingest_infos_for_region(
"us_state_county")
def test_write_single_ingest_info(self):
task_hash = hash(
json.dumps(Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT).to_serializable(),
sort_keys=True))
start_time = datetime.now()
ingest_info = datastore_ingest_info.write_ingest_info(
region='us_state_county',
session_start_time=start_time,
ingest_info=sample_ingest_info('1'),
task_hash=task_hash)
results = datastore_ingest_info.batch_get_ingest_infos_for_region(
'us_state_county', start_time)
assert results == [ingest_info]
datastore_ingest_info.batch_delete_ingest_infos_for_region(
'us_state_county')
def test_batch_delete_ingest_infos_for_region(self):
task_hash = hash(
json.dumps(Task(
task_type=constants.TaskType.SCRAPE_DATA,
endpoint=TEST_ENDPOINT,
response_type=constants.ResponseType.TEXT).to_serializable(),
sort_keys=True))
start_time = datetime.now()
datastore_ingest_info.write_ingest_info(
region='us_state_county',
session_start_time=start_time,
ingest_info=sample_ingest_info('1'),
task_hash=task_hash)
datastore_ingest_info.write_ingest_info(
region='us_state_county',
session_start_time=start_time,
ingest_info=sample_ingest_info('2'),
task_hash=task_hash)
datastore_ingest_info.write_ingest_info(
region='us_state_county',
session_start_time=start_time,
ingest_info=sample_ingest_info('3'),
task_hash=task_hash)
unrelated = datastore_ingest_info \
.write_ingest_info(region='unrelated_us_state_county',
session_start_time=start_time,
ingest_info=sample_ingest_info('n/a'),
task_hash=task_hash)
datastore_ingest_info.batch_delete_ingest_infos_for_region(
'us_state_county')
assert datastore_ingest_info.batch_get_ingest_infos_for_region(
'us_state_county', start_time) == []
actual = datastore_ingest_info.batch_get_ingest_infos_for_region(
'unrelated_us_state_county', start_time)
assert actual == [unrelated]
datastore_ingest_info.batch_delete_ingest_infos_for_region(
'unrelated_us_state_county')