def authorize_run_listing(project_id): logger.info("Looking up project") # Check the project resource exists abort_if_project_doesnt_exist(project_id) if request.headers is None or 'Authorization' not in request.headers: safe_fail_request(401, message="Authentication token required") auth_header = request.headers.get('Authorization') logger.info("Checking credentials to list project runs") # Check the caller has a valid results token (analyst token) abort_if_invalid_results_token(project_id, auth_header) logger.info("Caller is allowed to list project runs")
def precheck_upload_token(project_id, headers, parent_span): """ Raise a `ProblemException` if the project doesn't exist or the authentication token passed in the headers isn't valid. """ with opentracing.tracer.start_span('check-auth', child_of=parent_span) as span: abort_if_project_doesnt_exist(project_id) if headers is None or 'Authorization' not in headers: safe_fail_request(401, message="Authentication token required") token = headers['Authorization'] # Check the caller has valid token -> otherwise 403 abort_if_invalid_dataprovider_token(token) return token
def project_delete(project_id): log = logger.bind(pid=project_id) log.info('Request to delete project') # Check the resource exists and hasn't already been marked for deletion abort_if_project_doesnt_exist(project_id) # Check the caller has a valid results token. Yes it should be renamed. abort_if_invalid_results_token(project_id, request.headers.get('Authorization')) log.info("Marking project for deletion") with DBConn() as db_conn: db.mark_project_deleted(db_conn, project_id) log.info("Queuing authorized request to delete project resources") remove_project.delay(project_id) return '', 204
def authorise_get_request(project_id): if request.headers is None or 'Authorization' not in request.headers: safe_fail_request(401, message="Authentication token required") auth_header = request.headers.get('Authorization') dp_id = None # Check the resource exists abort_if_project_doesnt_exist(project_id) with DBConn() as dbinstance: project_object = db.get_project(dbinstance, project_id) logger.info("Checking credentials") if project_object['result_type'] == 'mapping' or project_object[ 'result_type'] == 'similarity_scores': # Check the caller has a valid results token if we are including results abort_if_invalid_results_token(project_id, auth_header) elif project_object['result_type'] == 'permutations': dp_id = get_authorization_token_type_or_abort(project_id, auth_header) else: safe_fail_request(500, "Unknown error") return dp_id, project_object
def post(project_id, run): log, span = bind_log_and_span(project_id) log.debug("Processing request to add a new run", run=run) # Check the resource exists abort_if_project_doesnt_exist(project_id) # Check the caller has a valid results token. Yes it should be renamed. abort_if_invalid_results_token(project_id, request.headers.get('Authorization')) abort_if_project_in_error_state(project_id) run_model = Run.from_json(run, project_id) log.debug("Saving run") with db.DBConn() as db_conn: run_model.save(db_conn) check_for_executable_runs.delay(project_id, serialize_span(span)) return RunDescription().dump(run_model), 201
def post(project_id, run): log = logger.bind(pid=project_id) log.debug("Processing request to add a new run", run=run) # Check the resource exists abort_if_project_doesnt_exist(project_id) # Check the caller has a valid results token. Yes it should be renamed. abort_if_invalid_results_token(project_id, request.headers.get('Authorization')) abort_if_project_in_error_state(project_id) run_model = Run.from_json(run, project_id) log.debug("Saving run") with db.DBConn() as db_conn: run_model.save(db_conn) project_object = db.get_project(db_conn, project_id) parties_contributed = db.get_number_parties_uploaded( db_conn, project_id) ready_to_run = parties_contributed == project_object['parties'] log.debug( "Expecting {} parties to upload data. Have received {}".format( project_object['parties'], parties_contributed)) if ready_to_run: log.info( "Scheduling task to carry out all runs for project {} now". format(project_id)) update_run_mark_queued(db_conn, run_model.run_id) else: log.info("Task queued but won't start until CLKs are all uploaded") if ready_to_run: span = g.flask_tracer.get_span() span.set_tag("run_id", run_model.run_id) span.set_tag("project_id", run_model.project_id) check_for_executable_runs.delay(project_id, serialize_span(span)) return RunDescription().dump(run_model), 201
def project_get(project_id): """ This endpoint describes a Project. """ log = logger.bind(pid=project_id) log.info("Getting detail for a project") abort_if_project_doesnt_exist(project_id) authorise_get_request(project_id) with DBConn() as db_conn: project_object = db.get_project(db_conn, project_id) # Expose the number of data providers who have uploaded clks parties_contributed = db.get_number_parties_uploaded( db_conn, project_id) num_parties_with_error = db.get_encoding_error_count( db_conn, project_id) log.info(f"{parties_contributed} parties have contributed hashes") project_object['parties_contributed'] = parties_contributed if num_parties_with_error > 0: log.warning( f"There are {num_parties_with_error} parties in error state") project_object['error'] = num_parties_with_error > 0 return ProjectDescription().dump(project_object)
def project_clks_post(project_id): """ Update a project to provide encoded PII data. """ log = logger.bind(pid=project_id) headers = request.headers parent_span = g.flask_tracer.get_span() with opentracing.tracer.start_span('check-auth', child_of=parent_span) as span: abort_if_project_doesnt_exist(project_id) if headers is None or 'Authorization' not in headers: safe_fail_request(401, message="Authentication token required") token = headers['Authorization'] # Check the caller has valid token -> otherwise 403 abort_if_invalid_dataprovider_token(token) with DBConn() as conn: dp_id = db.get_dataprovider_id(conn, token) project_encoding_size = db.get_project_schema_encoding_size( get_db(), project_id) log = log.bind(dp_id=dp_id) log.info("Receiving CLK data.") receipt_token = None with opentracing.tracer.start_span('upload-data', child_of=parent_span) as span: span.set_tag("project_id", project_id) if headers['Content-Type'] == "application/json": span.set_tag("content-type", 'json') # TODO: Previously, we were accessing the CLKs in a streaming fashion to avoid parsing the json in one hit. This # enables running the web frontend with less memory. # However, as connexion is very, very strict about input validation when it comes to json, it will always # consume the stream first to validate it against the spec. Thus the backflip to fully reading the CLks as # json into memory. -> issue #184 receipt_token, raw_file = upload_json_clk_data( dp_id, get_json(), span) # Schedule a task to deserialize the hashes, and carry # out a pop count. handle_raw_upload.delay(project_id, dp_id, receipt_token, parent_span=serialize_span(span)) log.info("Job scheduled to handle user uploaded hashes") elif headers['Content-Type'] == "application/octet-stream": span.set_tag("content-type", 'binary') log.info("Handling binary CLK upload") try: count, size = check_binary_upload_headers(headers) log.info( f"Headers tell us to expect {count} encodings of {size} bytes" ) span.log_kv({'count': count, 'size': size}) except Exception: log.warning( "Upload failed due to problem with headers in binary upload" ) raise # Check against project level encoding size (if it has been set) if project_encoding_size is not None and size != project_encoding_size: # fail fast - we haven't stored the encoded data yet return safe_fail_request( 400, "Upload 'Hash-Size' doesn't match project settings") # TODO actually stream the upload data straight to Minio. Currently we can't because # connexion has already read the data before our handler is called! # https://github.com/zalando/connexion/issues/592 # stream = get_stream() stream = BytesIO(request.data) log.debug( f"Stream size is {len(request.data)} B, and we expect {(6 + size)* count} B" ) if len(request.data) != (6 + size) * count: safe_fail_request( 400, "Uploaded data did not match the expected size. Check request headers are correct" ) try: receipt_token = upload_clk_data_binary(project_id, dp_id, stream, count, size) except ValueError: safe_fail_request( 400, "Uploaded data did not match the expected size. Check request headers are correct." ) else: safe_fail_request(400, "Content Type not supported") return {'message': 'Updated', 'receipt_token': receipt_token}, 201