def test_get_clusters_per_point_query(self):
dycast_parameters = test_helper_functions.get_dycast_parameters(
large_dataset=False)
risk_service = risk_service_module.RiskService(dycast_parameters)
comparative_test_service = comparative_test_service_module.ComparativeTestService(
dycast_parameters)
session = database_service.get_sqlalchemy_session()
riskdate = datetime.date(int(2016), int(3), int(25))
gridpoints = geography_service.generate_grid(dycast_parameters)
clusters_per_point_query = risk_service.get_clusters_per_point_query(
session, gridpoints, riskdate)
clusters_per_point = risk_service.get_clusters_per_point_from_query(
clusters_per_point_query)
daily_cases_query = comparative_test_service.get_daily_cases_query(
session, riskdate)
for cluster in clusters_per_point:
point_wkt_element = geography_service.get_wktelement_from_wkt(
cluster.point.wkt)
cases_in_cluster_query = comparative_test_service.get_cases_in_cluster_query(
daily_cases_query, point_wkt_element)
vector_count_new = cluster.get_case_count()
vector_count_old = database_service.get_count_for_query(
cases_in_cluster_query)
self.assertEqual(vector_count_new, vector_count_old)
def get_close_space_only_old(self, cases_in_cluster_query):
subquery = cases_in_cluster_query.subquery()
query = cases_in_cluster_query.join(subquery, literal(True)) \
.filter(func.ST_DWithin(Case.location, subquery.c.location, self.dycast_parameters.close_in_space),
Case.id < subquery.c.id)
return database_service.get_count_for_query(query)
def get_close_space_and_time(self, cases_in_cluster_query):
subquery = cases_in_cluster_query.subquery()
query = cases_in_cluster_query.join(subquery, literal(True)) \
.filter(func.ST_DWithin(Case.location, subquery.c.location, self.dycast_parameters.close_in_space),
func.abs(Case.report_date - subquery.c.report_date) <= self.dycast_parameters.close_in_time,
Case.id < subquery.c.id)
return database_service.get_count_for_query(query)
def export_risk(self, dycast_parameters):
session = database_service.get_sqlalchemy_session()
startdate = dycast_parameters.startdate
enddate = dycast_parameters.enddate
export_directory = dycast_parameters.export_directory
export_prefix = dycast_parameters.export_prefix
export_format = dycast_parameters.export_format
# Quick and dirty solution
if export_format != "tsv" and export_format != "csv":
logging.error("Incorrect export format: %s", export_format)
return 1
else:
separator = self.get_separator(export_format)
if export_directory is None:
export_directory = CONFIG.get("export_directory")
# dates are objects, not strings
startdate_string = conversion_service.get_string_from_date_object(
startdate)
enddate_string = conversion_service.get_string_from_date_object(
enddate)
export_time = strftime("%Y-%m-%d__%H-%M-%S")
filename = "exported_{0}__risk_{1}--{2}.{3}".format(
export_time, startdate_string, enddate_string, export_format)
if export_prefix:
filename = export_prefix + filename
filepath = os.path.join(export_directory, filename)
logging.info("Exporting risk for: %s - %s", startdate_string,
enddate_string)
risk_query = self.get_risk_query(session, startdate, enddate)
risk_count = database_service.get_count_for_query(risk_query)
if risk_count == 0:
logging.info("No risk found for the provided dates: %s - %s",
startdate_string, enddate_string)
return
risk_collection = risk_query.all()
table_content = file_service.TableContent()
header = self.get_header_as_string(separator)
table_content.set_header(header)
body = self.get_rows_as_string(risk_collection, separator)
table_content.set_body(body)
file_service.save_file(table_content.get_content(), filepath)
return filepath
def get_close_space_and_time_new(self, session, cluster):
cases_in_cluster = session.query(Case).filter(Case.id.in_(cluster.case_id_array))
subquery = cases_in_cluster.subquery()
close_in_space_and_time = cases_in_cluster.join(subquery, literal(True)) \
.filter(func.ST_DWithin(Case.location, subquery.c.location, self.dycast_parameters.close_in_space),
func.abs(Case.report_date - subquery.c.report_date) <= self.dycast_parameters.close_in_time,
Case.id < subquery.c.id)
return database_service.get_count_for_query(close_in_space_and_time)
def insert_test_cases():
import_service = import_service_module.ImportService()
dycast_model = dycast_parameters.DycastParameters()
dycast_model.srid_of_cases = '3857'
dycast_model.files_to_import = get_test_cases_import_files_latlong()
session = database_service.get_sqlalchemy_session()
case_query = session.query(Case)
case_count = database_service.get_count_for_query(case_query)
if case_count == 0:
import_service.load_case_files(dycast_model)
def test_get_daily_cases_query_old(self):
dycast_parameters = test_helper_functions.get_dycast_parameters()
comparative_test_service = comparative_test_service_module.ComparativeTestService(
dycast_parameters)
session = database_service.get_sqlalchemy_session()
riskdate = datetime.date(int(2016), int(3), int(25))
daily_cases_query = comparative_test_service.get_daily_cases_query(
session, riskdate)
count = database_service.get_count_for_query(daily_cases_query)
self.assertGreater(count, 0)
def test_load_case_correct(self):
session = database_service.get_sqlalchemy_session()
import_service = import_service_module.ImportService()
dycast_model = dycast_parameters.DycastParameters()
dycast_model.srid_of_cases = 3857
line_correct = "99999\t03/09/16\t1832445.278\t2118527.399"
location_type = enums.Location_type.LAT_LONG
import_service.load_case(session, dycast_model, line_correct,
location_type)
session.commit()
query = session.query(Case).filter(Case.id == '99999')
count = database_service.get_count_for_query(query)
self.assertEqual(count, 1)
session.delete(query.first())
session.commit()
def test_get_cases_in_cluster_query_old(self):
dycast_parameters = test_helper_functions.get_dycast_parameters()
comparative_test_service = comparative_test_service_module.ComparativeTestService(
dycast_parameters)
session = database_service.get_sqlalchemy_session()
riskdate = datetime.date(int(2016), int(3), int(25))
gridpoints = geography_service.generate_grid(dycast_parameters)
point = gridpoints[0]
daily_cases_query = comparative_test_service.get_daily_cases_query(
session, riskdate)
cases_in_cluster_query = comparative_test_service.get_cases_in_cluster_query(
daily_cases_query, point)
vector_count = database_service.get_count_for_query(
cases_in_cluster_query)
self.assertGreater(vector_count, 0)
def get_close_time_only(self, cases_in_cluster_query):
subquery = cases_in_cluster_query.subquery()
query = cases_in_cluster_query.join(subquery, literal(True)) \
.filter(func.abs(Case.report_date - subquery.c.report_date) <= self.dycast_parameters.close_in_time,
Case.id < subquery.c.id)
return database_service.get_count_for_query(query)
def generate_risk(self):
session = database_service.get_sqlalchemy_session()
logging_service.display_current_parameter_set(self.dycast_parameters)
case_threshold = self.dycast_parameters.case_threshold
gridpoints = geography_service.generate_grid(self.dycast_parameters)
day = self.dycast_parameters.startdate
delta = datetime.timedelta(days=1)
while day <= self.dycast_parameters.enddate:
daily_cases_query = self.get_daily_cases_query(session, day)
daily_case_count = database_service.get_count_for_query(
daily_cases_query)
if daily_case_count >= case_threshold:
start_time = time.time()
logging.info("Starting daily_risk for %s", day)
points_above_threshold = 0
clusters_per_point = self.get_clusters_per_point_query(
session, gridpoints, day)
for cluster in clusters_per_point:
vector_count = len(cluster.case_array)
if vector_count >= case_threshold:
points_above_threshold += 1
point = geography_service.get_shape_from_sqlalch_element(
cluster.point)
risk = Risk(risk_date=day,
number_of_cases=vector_count,
lat=point.x,
long=point.y,
location=point)
for point in gridpoints:
cases_in_cluster_query = self.get_cases_in_cluster_query(
daily_cases_query, point)
vector_count = database_service.get_count_for_query(
cases_in_cluster_query)
if vector_count >= case_threshold:
points_above_threshold += 1
risk = Risk(risk_date=day,
number_of_cases=vector_count,
lat=point.x,
long=point.y)
risk.close_pairs = self.get_close_space_and_time(
cases_in_cluster_query)
risk.close_space = self.get_close_space_only_old(
cases_in_cluster_query) - risk.close_pairs
risk.close_time = self.get_close_time_only(
cases_in_cluster_query) - risk.close_pairs
risk.cumulative_probability = self.get_cumulative_probability(
session, risk.number_of_cases, risk.close_pairs,
risk.close_space, risk.close_time)
self.insert_risk(session, risk)
logging.info("Finished daily_risk for %s: done %s points", day,
len(gridpoints))
logging.info("Total points above threshold of %s: %s",
case_threshold, points_above_threshold)
logging.info("Time elapsed: %.0f seconds",
time.time() - start_time)
else:
logging.info(
"Amount of cases for %s lower than threshold %s: %s, skipping.",
day, case_threshold, daily_case_count)
day += delta
try:
session.commit()
except SQLAlchemyError as e:
session.rollback()
logging.exception(
"There was a problem committing the risk data session")
logging.exception(e)
raise
finally:
session.close()
