def handle_single_transaction(role, program, project, **tx_kwargs): """ Main entry point for single file transactions. This function multiplexes on the content-type to call the appropriate transaction handler. """ doc = flask.request.get_data().decode("utf-8") content_type = flask.request.headers.get("Content-Type", "").lower() if content_type == "text/csv": doc_format = "csv" data, errors = utils.transforms.CSVToJSONConverter().convert(doc) elif content_type in ["text/tab-separated-values", "text/tsv"]: doc_format = "tsv" data, errors = utils.transforms.TSVToJSONConverter().convert(doc) else: doc_format = "json" data = utils.parse.parse_request_json() errors = None # TODO: use errors value? name = flask.request.headers.get("X-Document-Name", None) doc_args = [name, doc_format, doc, data] is_async = tx_kwargs.pop("is_async", utils.is_flag_set(FLAG_IS_ASYNC)) db_driver = tx_kwargs.pop("db_driver", flask.current_app.db) transaction = UploadTransaction( program=program, project=project, role=role, logger=flask.current_app.logger, flask_config=flask.current_app.config, index_client=flask.current_app.index_client, external_proxies=utils.get_external_proxies(), db_driver=db_driver, **tx_kwargs) if is_async: session = transaction.db_driver.session_scope(can_inherit=False) with session, transaction: response = { "code": 200, "message": "Transaction submitted.", "transaction_id": transaction.transaction_id, } flask.current_app.async_pool.schedule(single_transaction_worker, transaction, *doc_args) # create the resource in arborist auth.create_resource(program, project, doc_args[3]) return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) # create the resource in arborist auth.create_resource(program, project, doc_args[3]) return flask.jsonify(response), code
def handle_single_transaction(role, program, project, **tx_kwargs): """ Main entry point for single file transactions. This function multiplexes on the content-type to call the appropriate transaction handler. """ doc = flask.request.get_data() content_type = flask.request.headers.get('Content-Type', '').lower() if content_type == 'text/csv': doc_format = 'csv' data, errors = utils.transforms.CSVToJSONConverter().convert(doc) elif content_type in ['text/tab-separated-values', 'text/tsv']: doc_format = 'tsv' data, errors = utils.transforms.TSVToJSONConverter().convert(doc) else: doc_format = 'json' data = utils.parse.parse_request_json() errors = None # TODO: use errors value? name = flask.request.headers.get('X-Document-Name', None) doc_args = [name, doc_format, doc, data] is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC)) db_driver = tx_kwargs.pop('db_driver', flask.current_app.db) transaction = UploadTransaction(program=program, project=project, role=role, user=flask.g.user, logger=flask.current_app.logger, flask_config=flask.current_app.config, signpost=flask.current_app.signpost, external_proxies=utils.get_external_proxies(), db_driver=db_driver, **tx_kwargs) if is_async: session = transaction.db_driver.session_scope(can_inherit=False) with session, transaction: response = { "code": 200, "message": "Transaction submitted.", "transaction_id": transaction.transaction_id, } flask.current_app.async_pool.schedule( single_transaction_worker, transaction, *doc_args ) return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) return flask.jsonify(response), code
def handle_bulk_transaction(role, program, project, **tx_kwargs): """ TODO """ wrappers = utils.parse.parse_request_json() # Assert wrapper is list of JSON objects invalid_format_msg = ( 'Bulk transfers must be an array of JSON objects of format: {\n' ' "name": string,\n' ' "doc_format": string,\n' ' "doc": string,\n' '}' ) if not isinstance(wrappers, list): raise UserError(invalid_format_msg) for wrapper in wrappers: if not isinstance(wrapper, dict): raise UserError(invalid_format_msg) is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC)) transaction = BulkUploadTransaction( program=program, project=project, role=role, user=flask.g.user, logger=flask.current_app.logger, signpost=flask.current_app.signpost, db_driver=flask.current_app.db, external_proxies=utils.get_external_proxies(), **tx_kwargs ) if is_async: session = transaction.db_driver.session_scope(can_inherit=False) with session, transaction: response = { "code": 200, "message": "Transaction submitted.", "transaction_id": transaction.transaction_id, } flask.current_app.async_pool.schedule( bulk_transaction_worker, transaction, wrappers ) return flask.jsonify(response) else: response, code = bulk_transaction_worker(transaction, wrappers) return flask.jsonify(response), code
def _single_transaction(role, program, project, *doc_args, **tx_kwargs): """ Create and execute a single (not bulk) transaction. Most non-bulk variations of upload actions (based on content-type, etc.) should wrap this function. Return: Tuple[flask.Response, int]: (API response json, status code) """ is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC)) db_driver = tx_kwargs.pop('db_driver', flask.current_app.db) transaction = UploadTransaction( program=program, project=project, role=role, user=flask.g.user, logger=flask.current_app.logger, signpost=flask.current_app.signpost, flask_config=flask.current_app.config, db_driver=db_driver, external_proxies=utils.get_external_proxies(), **tx_kwargs ) if is_async: session = transaction.db_driver.session_scope(can_inherit=False) with session, transaction: response = { "code": 200, "message": "Transaction submitted.", "transaction_id": transaction.transaction_id, } flask.current_app.async_pool.schedule( single_transaction_worker, transaction, *doc_args ) return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) return flask.jsonify(response), code
def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter): """ Make sure that concurrent transactions don't cause duplicate submission. """ data = { "type": "experiment", "submitter_id": "BLGSP-71-06-00019", "projects.id": "daa208a7-f57a-562c-a04a-7a7c77542c98", } # convert to TSV (save to file) file_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data/experiment_tmp.tsv" ) with open(file_path, "w") as f: dw = csv.DictWriter(f, sorted(data.keys()), delimiter="\t") dw.writeheader() dw.writerow(data) # read the TSV data data = None with open(file_path, "r") as f: data = f.read() os.remove(file_path) # clean up (delete file) assert data program, project = BLGSP_PATH.split("/")[3:5] tsv_data = TSVToJSONConverter().convert(data)[0] doc_args = [None, "tsv", data, tsv_data] utx1, utx2 = [ UploadTransaction( program=program, project=project, role=ROLES["UPDATE"], logger=app.logger, flask_config=app.config, index_client=app.index_client, external_proxies=get_external_proxies(), db_driver=pg_driver, ) for _ in range(2) ] response = "" with pg_driver.session_scope(can_inherit=False) as s1: with utx1: utx1.parse_doc(*doc_args) with pg_driver.session_scope(can_inherit=False) as s2: with utx2: utx2.parse_doc(*doc_args) with pg_driver.session_scope(session=s2): utx2.flush() with pg_driver.session_scope(session=s2): utx2.post_validate() with pg_driver.session_scope(session=s2): utx2.commit() try: with pg_driver.session_scope(session=s1): utx1.flush() except IntegrityError: s1.rollback() utx1.integrity_check() response = utx1.json assert response["entity_error_count"] == 1 assert response["code"] == 400 assert ( response["entities"][0]["errors"][0]["message"] == "experiment with {'project_id': 'CGCI-BLGSP', 'submitter_id': 'BLGSP-71-06-00019'} already exists in the DB" ) with pg_driver.session_scope(): assert pg_driver.nodes(md.Experiment).count() == 1
def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter): """ Make sure that concurrent transactions don't cause duplicate submission. """ data = { "type": "experiment", "submitter_id": "BLGSP-71-06-00019", "projects.id": "daa208a7-f57a-562c-a04a-7a7c77542c98" } # convert to TSV (save to file) file_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data/experiment_tmp.tsv" ) with open(file_path, "w") as f: dw = csv.DictWriter(f, sorted(data.keys()), delimiter="\t") dw.writeheader() dw.writerow(data) # read the TSV data data = None with open(file_path, "r") as f: data = f.read() os.remove(file_path) # clean up (delete file) assert data program, project = BLGSP_PATH.split('/')[3:5] tsv_data = TSVToJSONConverter().convert(data)[0] doc_args = [None, 'tsv', data, tsv_data] utx1, utx2 = [UploadTransaction( program=program, project=project, role=ROLES['UPDATE'], logger=app.logger, flask_config=app.config, signpost=app.signpost, external_proxies=get_external_proxies(), db_driver=pg_driver, ) for _ in range(2)] with pg_driver.session_scope(can_inherit=False) as s1: with utx1: utx1.parse_doc(*doc_args) with pg_driver.session_scope(can_inherit=False) as s2: with utx2: utx2.parse_doc(*doc_args) with pg_driver.session_scope(session=s2): utx2.flush() with pg_driver.session_scope(session=s2): utx2.post_validate() with pg_driver.session_scope(session=s2): utx2.commit() try: with pg_driver.session_scope(session=s1): utx1.flush() with pg_driver.session_scope(session=s1): utx1.post_validate() with pg_driver.session_scope(session=s1): utx1.commit() except HandledIntegrityError: pass with pg_driver.session_scope(): assert pg_driver.nodes(md.Experiment).count() == 1