def upload_clk_data_binary(project_id, dp_id, raw_stream, count, size=128):
"""
Save the user provided raw CLK data.
"""
receipt_token = generate_code()
filename = Config.BIN_FILENAME_FMT.format(receipt_token)
# Set the state to 'pending' in the bloomingdata table
with DBConn() as conn:
db.insert_encoding_metadata(conn, filename, dp_id, receipt_token,
count)
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
logger.info(
f"Storing supplied binary clks of individual size {size} in file: {filename}"
)
num_bytes = count * (size + 6)
logger.debug(
"Directly storing binary file with index, base64 encoded CLK, popcount"
)
# Upload to object store
logger.info(
f"Uploading {count} binary encodings to object store. Total size: {fmt_bytes(num_bytes)}"
)
parent_span = g.flask_tracer.get_span()
with opentracing.tracer.start_span('save-to-minio',
child_of=parent_span) as span:
mc = connect_to_object_store()
try:
mc.put_object(Config.MINIO_BUCKET,
filename,
data=raw_stream,
length=num_bytes)
except (minio.error.InvalidSizeError, minio.error.InvalidArgumentError,
minio.error.ResponseError):
logger.info(
"Mismatch between expected stream length and header info")
raise ValueError(
"Mismatch between expected stream length and header info")
with opentracing.tracer.start_span('update-database',
child_of=parent_span) as span:
with DBConn() as conn:
db.update_encoding_metadata(conn, filename, dp_id, 'ready')
db.set_dataprovider_upload_state(conn, dp_id, True)
# Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id,
serialize_span(parent_span))
return receipt_token
def upload_clk_data_binary(project_id,
dp_id,
encoding_iter,
receipt_token,
count,
size=128):
"""
Save the user provided binary-packed CLK data.
"""
filename = None
# Set the state to 'pending' in the uploads table
with DBConn() as conn:
db.insert_encoding_metadata(conn,
filename,
dp_id,
receipt_token,
encoding_count=count,
block_count=1)
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
num_bytes = binary_format(size).size * count
logger.debug(
"Directly storing binary file with index, base64 encoded CLK, popcount"
)
# Upload to database
logger.info(
f"Uploading {count} binary encodings to database. Total size: {fmt_bytes(num_bytes)}"
)
parent_span = g.flask_tracer.get_span()
with DBConn() as conn:
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
with opentracing.tracer.start_span('create-default-block-in-db',
child_of=parent_span):
db.insert_blocking_metadata(conn, dp_id, {DEFAULT_BLOCK_ID: count})
with opentracing.tracer.start_span('upload-encodings-to-db',
child_of=parent_span):
store_encodings_in_db(conn, dp_id, encoding_iter, size)
with opentracing.tracer.start_span('update-encoding-metadata',
child_of=parent_span):
db.update_encoding_metadata(conn, filename, dp_id, 'ready')
def handle_invalid_encoding_data(project_id, dp_id):
with DBConn() as conn:
filename, _ = get_filter_metadata(conn, dp_id)
update_encoding_metadata(conn, 'DELETED', dp_id, state='error')
if filename is not None:
delete_minio_objects.delay([filename], project_id)