def pull_external_data_encodings_only(project_id, dp_id, object_info, credentials, receipt_token, parent_span=None):
"""
"""
log = logger.bind(pid=project_id, dp_id=dp_id)
with DBConn() as conn:
if not check_project_exists(conn, project_id):
log.info("Project deleted, stopping immediately")
return
bucket_name = object_info['bucket']
object_name = object_info['path']
log.info("Pulling encoding data from an object store")
mc_credentials = parse_minio_credentials(credentials)
stat, stream = stat_and_stream_object(bucket_name, object_name, mc_credentials)
count = int(stat.metadata['X-Amz-Meta-Hash-Count'])
size = int(stat.metadata['X-Amz-Meta-Hash-Size'])
converted_stream = include_encoding_id_in_binary_stream(stream, size, count)
upload_clk_data_binary(project_id, dp_id, converted_stream, receipt_token, count, size)
# # Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id, serialize_span(parent_span))
def pull_external_data_encodings_only(project_id, dp_id, object_info, credentials, receipt_token, parent_span=None):
"""
"""
log = logger.bind(pid=project_id, dp_id=dp_id)
with DBConn() as conn:
if not check_project_exists(conn, project_id):
log.info("Project deleted, stopping immediately")
return
bucket_name = object_info['bucket']
object_name = object_info['path']
log.info("Pulling encoding data from an object store")
env_credentials = parse_minio_credentials({
'AccessKeyId': config.MINIO_ACCESS_KEY,
'SecretAccessKey': config.MINIO_SECRET_KEY
})
stat, stream = stat_and_stream_object(bucket_name, object_name, env_credentials)
count = int(stat.metadata['X-Amz-Meta-Hash-Count'])
size = int(stat.metadata['X-Amz-Meta-Hash-Size'])
if object_name.endswith('.json'):
encodings_stream = ijson.items(io.BytesIO(stream.data), 'clks.item')
converted_stream = include_encoding_id_in_json_stream(encodings_stream, size, count)
else:
converted_stream = include_encoding_id_in_binary_stream(stream, size, count)
upload_clk_data_binary(project_id, dp_id, converted_stream, receipt_token, count, size, parent_span=parent_span)
# # Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id, serialize_span(parent_span))
def project_clks_post(project_id):
"""
Update a project to provide encoded PII data.
"""
headers = request.headers
log, parent_span = bind_log_and_span(project_id)
log.debug("Starting data upload request")
token = precheck_upload_token(project_id, headers, parent_span)
receipt_token = generate_code()
with DBConn() as conn:
dp_id = db.get_dataprovider_id(conn, token)
project_encoding_size = db.get_project_schema_encoding_size(
conn, project_id)
upload_state_updated = db.is_dataprovider_allowed_to_upload_and_lock(
conn, dp_id)
# get flag use_blocking from table projects
uses_blocking = get_project_column(conn, project_id, 'uses_blocking')
if not upload_state_updated:
return safe_fail_request(
403, "This token has already been used to upload clks.")
log = log.bind(dp_id=dp_id)
log.info("Receiving CLK data.")
with opentracing.tracer.start_span('upload-clk-data',
child_of=parent_span) as span:
span.set_tag("project_id", project_id)
try:
if headers['Content-Type'] == "application/json":
span.set_tag("content-type", 'json')
# TODO: Previously, we were accessing the CLKs in a streaming fashion to avoid parsing the json in one hit. This
# enables running the web frontend with less memory.
# However, as connexion is very, very strict about input validation when it comes to json, it will always
# consume the stream first to validate it against the spec. Thus the backflip to fully reading the CLks as
# json into memory. -> issue #184
handle_encoding_upload_json(project_id,
dp_id,
get_json(),
receipt_token,
uses_blocking,
parent_span=span)
log.info("Job scheduled to handle users upload")
elif headers['Content-Type'] == "application/octet-stream":
span.set_tag("content-type", 'binary')
log.info("Handling binary CLK upload")
try:
count, size = check_binary_upload_headers(headers)
log.info(
f"Headers tell us to expect {count} encodings of {size} bytes"
)
span.log_kv({'count': count, 'size': size})
except Exception:
log.warning(
"Upload failed due to problem with headers in binary upload"
)
raise
# Check against project level encoding size (if it has been set)
if project_encoding_size is not None and size != project_encoding_size:
# fail fast - we haven't stored the encoded data yet
return safe_fail_request(
400,
"Upload 'Hash-Size' doesn't match project settings")
# TODO actually stream the upload data straight to Minio. Currently we can't because
# connexion has already read the data before our handler is called!
# https://github.com/zalando/connexion/issues/592
# stream = get_stream()
stream = BytesIO(request.data)
expected_bytes = binary_format(size).size * count
log.debug(
f"Stream size is {len(request.data)} B, and we expect {expected_bytes} B"
)
if len(request.data) != expected_bytes:
safe_fail_request(
400,
"Uploaded data did not match the expected size. Check request headers are correct"
)
try:
upload_clk_data_binary(project_id, dp_id, stream,
receipt_token, count, size)
except ValueError:
safe_fail_request(
400,
"Uploaded data did not match the expected size. Check request headers are correct."
)
else:
safe_fail_request(400, "Content Type not supported")
except ProblemException as e:
# Have an exception that is safe for the user. We reset the upload state to
# allow the user to try upload again.
log.info(
f"Problem occurred, returning status={e.status} - {e.detail}")
with DBConn() as conn:
db.set_dataprovider_upload_state(conn,
dp_id,
state='not_started')
raise
except Exception as e:
log.warning("Unhandled error occurred during data upload")
log.exception(e)
with DBConn() as conn:
db.set_dataprovider_upload_state(conn, dp_id, state='error')
safe_fail_request(
500, "Sorry, the server couldn't handle that request")
with DBConn() as conn:
db.set_dataprovider_upload_state(conn, dp_id, state='done')
# Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id,
serialize_span(parent_span))
return {'message': 'Updated', 'receipt_token': receipt_token}, 201
def project_binaryclks_post(project_id):
"""
Update a project to provide encoded PII data.
"""
log, parent_span = bind_log_and_span(project_id)
headers = request.headers
token = precheck_upload_token(project_id, headers, parent_span)
with DBConn() as conn:
dp_id = db.get_dataprovider_id(conn, token)
project_encoding_size = db.get_project_schema_encoding_size(
conn, project_id)
upload_state_updated = db.is_dataprovider_allowed_to_upload_and_lock(
conn, dp_id)
if not upload_state_updated:
return safe_fail_request(
403, "This token has already been used to upload clks.")
log = log.bind(dp_id=dp_id)
log.info("Receiving CLK data.")
receipt_token = generate_code()
with opentracing.tracer.start_span('upload-clk-data',
child_of=parent_span) as span:
span.set_tag("project_id", project_id)
try:
if headers['Content-Type'] == "application/octet-stream":
span.set_tag("content-type", 'binary')
log.info("Handling binary CLK upload")
try:
count, size = check_binary_upload_headers(headers)
log.info(
f"Headers tell us to expect {count} encodings of {size} bytes"
)
span.log_kv({'count': count, 'size': size})
except Exception:
log.warning(
"Upload failed due to problem with headers in binary upload"
)
raise
# Check against project level encoding size (if it has been set)
if project_encoding_size is not None and size != project_encoding_size:
# fail fast - we haven't stored the encoded data yet
return safe_fail_request(
400,
"Upload 'Hash-Size' doesn't match project settings")
# TODO actually stream the upload data straight to Minio. Currently we can't because
# connexion has already read the data before our handler is called!
# https://github.com/zalando/connexion/issues/592
# stream = get_stream()
stream = BytesIO(request.data)
converted_stream = include_encoding_id_in_binary_stream(
stream, size, count)
expected_bytes = size * count
log.debug(
f"Stream size is {len(request.data)} B, and we expect {expected_bytes} B"
)
if len(request.data) != expected_bytes:
safe_fail_request(
400,
"Uploaded data did not match the expected size. Check request headers are correct"
)
try:
upload_clk_data_binary(project_id, dp_id, converted_stream,
receipt_token, count, size)
except ValueError:
safe_fail_request(
400,
"Uploaded data did not match the expected size. Check request headers are correct."
)
else:
safe_fail_request(400, "Content Type not supported")
except Exception:
log.warning(
"The dataprovider was not able to upload their clks,"
" re-enable the corresponding upload token to be used.")
with DBConn() as conn:
db.set_dataprovider_upload_state(conn, dp_id, state='error')
raise
with DBConn() as conn:
db.set_dataprovider_upload_state(conn, dp_id, state='done')
# Now work out if all parties have added their data
if clks_uploaded_to_project(project_id):
logger.info("All parties data present. Scheduling any queued runs")
check_for_executable_runs.delay(project_id,
serialize_span(parent_span))
return {'message': 'Updated', 'receipt_token': receipt_token}, 201