def get_deserialized_filter(dp_id):
"""Cached, deserialized version.
"""
logger.debug("Getting filters")
key = 'clk-pkl-{}'.format(dp_id)
r = connect_to_redis(read_only=True)
# Check if this dp_id is already saved in redis?
if r.exists(key):
logger.debug("returning filters from cache")
return pickle.loads(r.get(key))
else:
logger.debug("Looking up popcounts and filename from database")
db = database.connect_db()
mc = connect_to_object_store()
serialized_filters_file = database.get_filter_metadata(db, dp_id)
logger.debug("Getting filters from object store")
# Note this uses already calculated popcounts unlike
# serialization.deserialize_filters()
raw_data_response = mc.get_object(config.MINIO_BUCKET,
serialized_filters_file)
python_filters = serialization.binary_unpack_filters(raw_data_response)
set_deserialized_filter(dp_id, python_filters)
return python_filters
def calculate_comparison_rate():
dbinstance = connect_db()
logger.info("Calculating global comparison rate")
total_comparisons = 0
total_time = timedelta(0)
for run in get_elapsed_run_times(dbinstance):
comparisons = get_total_comparisons_for_project(
dbinstance, run['project_id'])
if comparisons != 'NA':
total_comparisons += comparisons
else:
logger.debug("Skipping run as it hasn't completed")
total_time += run['elapsed']
if total_time.total_seconds() > 0:
rate = total_comparisons / total_time.total_seconds()
logger.info("Total comparisons: {}".format(total_comparisons))
logger.info("Total time: {}".format(total_time.total_seconds()))
logger.info("Comparison rate: {}".format(rate))
with dbinstance.cursor() as cur:
insert_comparison_rate(cur, rate)
dbinstance.commit()
else:
logger.warning("Can't compute comparison rate yet")
def get_db():
"""Opens a new database connection if there is none yet for the
current flask application context.
"""
conn = getattr(g, 'db', None)
if conn is None:
logger.debug("Caching a new database connection in application context")
conn = g.db = database.connect_db()
return conn
def clks_uploaded_to_project(project_id, check_data_ready=False):
""" See if the given project has had all parties contribute data.
"""
logger.info("Counting contributing parties")
conn = connect_db()
if check_data_ready:
parties_contributed = get_number_parties_ready(conn, project_id)
logger.info("Parties where data is ready: {}".format(parties_contributed))
else:
parties_contributed = get_number_parties_uploaded(conn, project_id)
logger.info("Parties where data is uploaded: {}".format(parties_contributed))
number_parties = get_project_column(conn, project_id, 'parties')
logger.info("{}/{} parties have contributed clks".format(parties_contributed, number_parties))
return parties_contributed == number_parties
def remove_project(project_id):
"""
"""
log = logger.bind(pid=project_id)
log.debug("Remove all project resources")
conn = db.connect_db()
log.debug("Deleting project resourced from database")
db.delete_project_data(conn, project_id)
log.debug(
"Getting object store files associated with project from database")
object_store_files = db.get_all_objects_for_project(conn, project_id)
log.debug(
f"Removing {len(object_store_files)} object store files associated with project."
)
delete_minio_objects.delay(object_store_files, project_id)
log.info("Project resources removed")