def upload_clk_data_binary(project_id, dp_id, raw_stream, count, size=128):
"""
Save the user provided raw CLK data.
"""
receipt_token = generate_code()
filename = Config.BIN_FILENAME_FMT.format(receipt_token)
# Set the state to 'pending' in the bloomingdata table
with DBConn() as conn:
db.insert_encoding_metadata(conn, filename, dp_id, receipt_token,
count)
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
logger.info(
f"Storing supplied binary clks of individual size {size} in file: {filename}"
)
num_bytes = count * (size + 6)
logger.debug(
"Directly storing binary file with index, base64 encoded CLK, popcount"
)
# Upload to object store
logger.info(
f"Uploading {count} binary encodings to object store. Total size: {fmt_bytes(num_bytes)}"
)
parent_span = g.flask_tracer.get_span()
with opentracing.tracer.start_span('save-to-minio',
child_of=parent_span) as span:
mc = connect_to_object_store()
try:
mc.put_object(Config.MINIO_BUCKET,
filename,
data=raw_stream,
length=num_bytes)
except (minio.error.InvalidSizeError, minio.error.InvalidArgumentError,
minio.error.ResponseError):
logger.info(
"Mismatch between expected stream length and header info")
raise ValueError(
"Mismatch between expected stream length and header info")
with opentracing.tracer.start_span('update-database',
child_of=parent_span) as span:
with DBConn() as conn:
db.update_encoding_metadata(conn, filename, dp_id, 'ready')
db.set_dataprovider_upload_state(conn, dp_id, True)
# Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id,
serialize_span(parent_span))
return receipt_token
def upload_clk_data_binary(project_id,
dp_id,
encoding_iter,
receipt_token,
count,
size=128):
"""
Save the user provided binary-packed CLK data.
"""
filename = None
# Set the state to 'pending' in the uploads table
with DBConn() as conn:
db.insert_encoding_metadata(conn,
filename,
dp_id,
receipt_token,
encoding_count=count,
block_count=1)
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
num_bytes = binary_format(size).size * count
logger.debug(
"Directly storing binary file with index, base64 encoded CLK, popcount"
)
# Upload to database
logger.info(
f"Uploading {count} binary encodings to database. Total size: {fmt_bytes(num_bytes)}"
)
parent_span = g.flask_tracer.get_span()
with DBConn() as conn:
db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
with opentracing.tracer.start_span('create-default-block-in-db',
child_of=parent_span):
db.insert_blocking_metadata(conn, dp_id, {DEFAULT_BLOCK_ID: count})
with opentracing.tracer.start_span('upload-encodings-to-db',
child_of=parent_span):
store_encodings_in_db(conn, dp_id, encoding_iter, size)
with opentracing.tracer.start_span('update-encoding-metadata',
child_of=parent_span):
db.update_encoding_metadata(conn, filename, dp_id, 'ready')
def upload_json_clk_data(dp_id, clk_json, parent_span):
"""
Convert user provided encodings from json array of base64 data into
a newline separated file of base64 data.
Note this implementation is non-streaming.
"""
if 'clks' not in clk_json or len(clk_json['clks']) < 1:
safe_fail_request(400, message="Missing CLKs information")
receipt_token = generate_code()
filename = Config.RAW_FILENAME_FMT.format(receipt_token)
logger.info("Storing user {} supplied clks from json".format(dp_id))
with opentracing.tracer.start_span('clk-splitting',
child_of=parent_span) as span:
count = len(clk_json['clks'])
span.set_tag("clks", count)
data = b''.join(''.join(clk.split('\n')).encode() + b'\n'
for clk in clk_json['clks'])
num_bytes = len(data)
span.set_tag("num_bytes", num_bytes)
buffer = BytesIO(data)
logger.info(
f"Received {count} encodings. Uploading {fmt_bytes(num_bytes)} to object store"
)
with opentracing.tracer.start_span('save-to-quarantine',
child_of=parent_span) as span:
span.set_tag('filename', filename)
mc = connect_to_object_store()
mc.put_object(Config.MINIO_BUCKET,
filename,
data=buffer,
length=num_bytes)
with opentracing.tracer.start_span('update-db',
child_of=parent_span) as span:
with DBConn() as conn:
db.insert_encoding_metadata(conn, filename, dp_id, receipt_token,
count)
return receipt_token, filename
def handle_encoding_upload_json(project_id, dp_id, clk_json, receipt_token,
uses_blocking, parent_span):
"""
Take user provided upload information - accepting multiple formats - and eventually
injest into the database.
Encodings uploaded directly in the JSON are first quarantined in the object store,
and a background task deserializes them.
Encodings that are in an object store are streamed directly into the database by
a background task.
"""
log = logger.bind(pid=project_id)
log.info("Checking json is consistent")
try:
abort_if_inconsistent_upload(uses_blocking, clk_json)
except ValueError as e:
safe_fail_request(403, e.args[0])
if "encodings" in clk_json and 'file' in clk_json['encodings']:
# external encodings
log.info("External encodings uploaded")
encoding_object_info = clk_json['encodings']['file']
object_name = encoding_object_info['path']
_check_object_path_allowed(project_id, dp_id, object_name, log)
encoding_credentials = clk_json['encodings'].get('credentials')
# Schedule a background task to pull the encodings from the object store
# This background task updates the database with encoding metadata assuming
# that there are no blocks.
if 'blocks' not in clk_json:
log.info("scheduling task to pull encodings from object store")
pull_external_data_encodings_only.delay(
project_id,
dp_id,
encoding_object_info,
encoding_credentials,
receipt_token,
parent_span=serialize_span(parent_span))
else:
# Need to deal with both encodings and blocks
if 'file' in clk_json['blocks']:
object_name = clk_json['blocks']['file']['path']
_check_object_path_allowed(project_id, dp_id, object_name, log)
# Blocks are in an external file
blocks_object_info = clk_json['blocks']['file']
blocks_credentials = clk_json['blocks'].get('credentials')
log.info(
"scheduling task to pull both encodings and blocking data from object store"
)
pull_external_data.delay(
project_id,
dp_id,
encoding_object_info,
encoding_credentials,
blocks_object_info,
blocks_credentials,
receipt_token,
parent_span=serialize_span(parent_span))
else:
raise NotImplementedError(
"Don't currently handle combination of external encodings and blocks"
)
return
# Convert uploaded JSON to common schema.
#
# The original JSON API simply accepted "clks", then came a combined encoding and
# blocking API expecting the top level element "clknblocks". Finally an API that
# specifies both "encodings" and "blocks" independently at the top level.
#
# We rewrite all into the "clknblocks" format.
if "encodings" in clk_json:
logger.debug(
"converting from 'encodings' & 'blocks' format to 'clknblocks'")
clk_json = convert_encoding_upload_to_clknblock(clk_json)
is_valid_clks = not uses_blocking and 'clks' in clk_json
element = 'clks' if is_valid_clks else 'clknblocks'
if len(clk_json[element]) < 1:
safe_fail_request(400, message="Missing CLKs information")
filename = Config.RAW_FILENAME_FMT.format(receipt_token)
logger.info("Storing user {} supplied {} from json".format(dp_id, element))
with opentracing.tracer.start_span('splitting-json-clks',
child_of=parent_span) as span:
encoding_count = len(clk_json[element])
span.set_tag(element, encoding_count)
logger.debug(f"Received {encoding_count} {element}")
if element == 'clks':
logger.info("Rewriting provided json into clknsblocks format")
clk_json = convert_clks_to_clknblocks(clk_json)
element = 'clknblocks'
logger.info("Counting block sizes and number of blocks")
# {'clknblocks': [['UG9vcA==', '001', '211'], [...]]}
block_sizes = {}
for _, *elements_blocks in clk_json[element]:
for el_block in elements_blocks:
block_sizes[el_block] = block_sizes.setdefault(el_block, 0) + 1
block_count = len(block_sizes)
logger.info(f"Received {encoding_count} encodings in {block_count} blocks")
for block in block_sizes:
logger.info(f"Block {block} has {block_sizes[block]} elements")
# write clk_json into a temp file
tmp = tempfile.NamedTemporaryFile(mode='w')
json.dump(clk_json, tmp)
tmp.flush()
with opentracing.tracer.start_span('save-clk-file-to-quarantine',
child_of=parent_span) as span:
span.set_tag('filename', filename)
mc = connect_to_object_store()
mc.fput_object(Config.MINIO_BUCKET,
filename,
tmp.name,
content_type='application/json')
logger.info('Saved uploaded {} JSON to file {} in object store.'.format(
element.upper(), filename))
with opentracing.tracer.start_span('update-encoding-metadata',
child_of=parent_span):
with DBConn() as conn:
db.insert_encoding_metadata(conn, filename, dp_id, receipt_token,
encoding_count, block_count)
db.insert_blocking_metadata(conn, dp_id, block_sizes)
# Schedule a task to deserialize the encodings
handle_raw_upload.delay(project_id,
dp_id,
receipt_token,
parent_span=serialize_span(parent_span))