def test_get_most_recently_closed_session(self):
self.create_session(
region_code="us_ny",
scrape_type=constants.ScrapeType.BACKGROUND,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 17)),
end=fix_dt(datetime(2009, 6, 18)),
)
second = self.create_session(
region_code="us_ny",
scrape_type=constants.ScrapeType.SNAPSHOT,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 18)),
end=fix_dt(datetime(2009, 6, 21)),
)
self.create_session(
region_code="us_ny",
scrape_type=constants.ScrapeType.BACKGROUND,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 19)),
)
self.create_session(
region_code="us_fl",
scrape_type=constants.ScrapeType.SNAPSHOT,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 19)),
)
result = sessions.get_most_recent_completed_session("us_ny")
assert result.to_entity() == second.to_entity()
def infer_release():
"""Runs infer release for the given regions."""
region_codes = validate_regions(
get_str_param_values("region", request.args))
regions = [get_region(region_code) for region_code in region_codes]
for region in regions:
with monitoring.push_tags(
{monitoring.TagKey.REGION: region.region_code}):
if region.agency_type != "jail":
continue
session = sessions.get_most_recent_completed_session(
region.region_code)
if session:
logging.info(
"Got most recent completed session for [%s] with "
"start time [%s]",
region.region_code,
session.start,
)
persistence.infer_release_on_open_bookings(
region.region_code, session.start,
_get_custody_status(region))
sessions.update_phase(session, scrape_phase.ScrapePhase.DONE)
return "", HTTPStatus.OK
def read_and_persist() -> Tuple[str, HTTPStatus]:
"""Reads all of the messages from Datastore for a region and persists
them to the database.
"""
region = request.args.get("region")
if not isinstance(region, str):
raise ValueError(f"Expected string region, found [{region}]")
batch_tags = {
monitoring.TagKey.STATUS: "COMPLETED",
monitoring.TagKey.PERSISTED: False,
}
# Note: measurements must be second so it receives the region tag.
with monitoring.push_tags(
{monitoring.TagKey.REGION: region}
), monitoring.measurements(batch_tags) as measurements:
measurements.measure_int_put(m_batch_count, 1)
session = sessions.get_most_recent_completed_session(
region, ScrapeType.BACKGROUND
)
if not session:
raise ValueError(
f"Most recent session for region [{region}] is unexpectedly None"
)
scrape_type = session.scrape_type
try:
did_persist = persist_to_database(region, session.start)
batch_tags[monitoring.TagKey.PERSISTED] = did_persist
except Exception as e:
logging.exception(
"An exception occurred in read and persist: %s", type(e).__name__
)
batch_tags[monitoring.TagKey.STATUS] = "ERROR: {}".format(type(e).__name__)
sessions.update_phase(session, scrape_phase.ScrapePhase.DONE)
raise BatchPersistError(region, scrape_type) from e
if did_persist:
next_phase = scrape_phase.next_phase(request.endpoint)
sessions.update_phase(session, scrape_phase.ScrapePhase.RELEASE)
if next_phase:
logging.info("Enqueueing %s for region %s.", next_phase, region)
ScraperCloudTaskManager().create_scraper_phase_task(
region_code=region, url=url_for(next_phase)
)
return "", HTTPStatus.OK
sessions.update_phase(session, scrape_phase.ScrapePhase.DONE)
return "", HTTPStatus.ACCEPTED
def test_get_most_recently_closed_session_when_empty(self):
self.create_session(
region_code="us_fl", scrape_type=constants.ScrapeType.BACKGROUND,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 17)),
end=fix_dt(datetime(2009, 6, 18)))
self.create_session(
region_code="us_ny", scrape_type=constants.ScrapeType.BACKGROUND,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 19)))
self.create_session(
region_code="us_fl", scrape_type=constants.ScrapeType.SNAPSHOT,
phase=scrape_phase.ScrapePhase.START,
start=fix_dt(datetime(2009, 6, 19)))
assert not sessions.get_most_recent_completed_session("us_ny")
def read_and_persist():
"""Reads all of the messages from Datastore for a region and persists
them to the database.
"""
region = request.args.get('region')
batch_tags = {
monitoring.TagKey.STATUS: 'COMPLETED',
monitoring.TagKey.PERSISTED: False
}
# Note: measurements must be second so it receives the region tag.
with monitoring.push_tags({monitoring.TagKey.REGION: region}), \
monitoring.measurements(batch_tags) as measurements:
measurements.measure_int_put(m_batch_count, 1)
session = sessions.get_most_recent_completed_session(
region, ScrapeType.BACKGROUND)
scrape_type = session.scrape_type
try:
did_persist = persist_to_database(region, session.start)
batch_tags[monitoring.TagKey.PERSISTED] = did_persist
except Exception as e:
logging.exception("An exception occurred in read and persist: %s",
type(e).__name__)
batch_tags[monitoring.TagKey.STATUS] = 'ERROR: {}' \
.format(type(e).__name__)
sessions.update_phase(session, scrape_phase.ScrapePhase.DONE)
raise BatchPersistError(region, scrape_type)
if did_persist:
next_phase = scrape_phase.next_phase(request.endpoint)
sessions.update_phase(session, scrape_phase.ScrapePhase.RELEASE)
if next_phase:
logging.info("Enqueueing %s for region %s.", region,
next_phase)
queues.enqueue_scraper_phase(region_code=region,
url=url_for(next_phase))
return '', HTTPStatus.OK
sessions.update_phase(session, scrape_phase.ScrapePhase.DONE)
return '', HTTPStatus.ACCEPTED