def calculate_comparison_rate():
with DBConn() as dbinstance:
logger.info("Calculating global comparison rate")
total_comparisons = 0
total_time = timedelta(0)
for run in get_elapsed_run_times(dbinstance):
comparisons = get_total_comparisons_for_project(dbinstance, run['project_id'])
if comparisons != 'NA':
total_comparisons += comparisons
else:
logger.debug("Skipping run as it hasn't completed")
total_time += run['elapsed']
if total_time.total_seconds() > 0:
rate = total_comparisons/total_time.total_seconds()
logger.info("Total comparisons: {}".format(total_comparisons))
logger.info("Total time: {}".format(total_time.total_seconds()))
logger.info("Comparison rate: {:.0f}".format(rate))
with dbinstance.cursor() as cur:
insert_comparison_rate(cur, rate)
else:
logger.warning("Can't compute comparison rate yet")
def stat_and_stream_object(bucket_name, object_name, credentials=None):
mc = connect_to_object_store(credentials)
logger.debug("Checking object exists in object store")
stat = mc.stat_object(bucket_name=bucket_name, object_name=object_name)
logger.debug("Retrieving file from object store")
response = mc.get_object(bucket_name=bucket_name, object_name=object_name)
return stat, response
def connect_to_object_store(credentials=None):
mc = minio.Minio(config.MINIO_SERVER,
config.MINIO_ACCESS_KEY,
config.MINIO_SECRET_KEY,
credentials=credentials,
secure=False)
logger.debug("Connected to minio")
mc.set_app_info("anonlink-client", "minio general client")
create_bucket(mc, config.MINIO_BUCKET)
return mc
def connect_to_upload_object_store():
"""
Instantiate a minio client with an upload only policy applied.
:return:
"""
mc = minio.Minio(config.UPLOAD_OBJECT_STORE_SERVER,
config.UPLOAD_OBJECT_STORE_ACCESS_KEY,
config.UPLOAD_OBJECT_STORE_SECRET_KEY,
region="us-east-1",
secure=config.UPLOAD_OBJECT_STORE_SECURE)
mc.set_app_info("anonlink-upload", "minio client for uploads")
logger.debug("Connected to minio upload account")
return mc
def __call__(self, *args, **kwargs):
args_dict = dict(zip(self.run.__code__.co_varnames, args))
args_dict.update(kwargs)
parent = args_dict.get(getattr(self, 'parent_span_arg', 'parent_span'),
None)
try:
parent = self.tracer.extract(Format.TEXT_MAP, parent)
except:
pass
if parent is None:
logger.debug('bugger, parent is None')
with self.tracer.start_span(getattr(self, 'span_name', self.name),
child_of=parent) as wrapper_span:
self._span = wrapper_span
for span_tag in getattr(self, 'args_as_tags', []):
wrapper_span.set_tag(span_tag, args_dict.get(span_tag, None))
return super(TracedTask, self).__call__(*args, **kwargs)