def handle_single_transaction(role, program, project, **tx_kwargs):
"""
Main entry point for single file transactions.
This function multiplexes on the content-type to call the appropriate
transaction handler.
"""
doc = flask.request.get_data().decode("utf-8")
content_type = flask.request.headers.get("Content-Type", "").lower()
if content_type == "text/csv":
doc_format = "csv"
data, errors = utils.transforms.CSVToJSONConverter().convert(doc)
elif content_type in ["text/tab-separated-values", "text/tsv"]:
doc_format = "tsv"
data, errors = utils.transforms.TSVToJSONConverter().convert(doc)
else:
doc_format = "json"
data = utils.parse.parse_request_json()
errors = None
# TODO: use errors value?
name = flask.request.headers.get("X-Document-Name", None)
doc_args = [name, doc_format, doc, data]
is_async = tx_kwargs.pop("is_async", utils.is_flag_set(FLAG_IS_ASYNC))
db_driver = tx_kwargs.pop("db_driver", flask.current_app.db)
transaction = UploadTransaction(
program=program,
project=project,
role=role,
logger=flask.current_app.logger,
flask_config=flask.current_app.config,
index_client=flask.current_app.index_client,
external_proxies=utils.get_external_proxies(),
db_driver=db_driver,
**tx_kwargs)
if is_async:
session = transaction.db_driver.session_scope(can_inherit=False)
with session, transaction:
response = {
"code": 200,
"message": "Transaction submitted.",
"transaction_id": transaction.transaction_id,
}
flask.current_app.async_pool.schedule(single_transaction_worker,
transaction, *doc_args)
# create the resource in arborist
auth.create_resource(program, project, doc_args[3])
return flask.jsonify(response)
else:
response, code = single_transaction_worker(transaction, *doc_args)
# create the resource in arborist
auth.create_resource(program, project, doc_args[3])
return flask.jsonify(response), code
def handle_single_transaction(role, program, project, **tx_kwargs):
"""
Main entry point for single file transactions.
This function multiplexes on the content-type to call the appropriate
transaction handler.
"""
doc = flask.request.get_data()
content_type = flask.request.headers.get('Content-Type', '').lower()
if content_type == 'text/csv':
doc_format = 'csv'
data, errors = utils.transforms.CSVToJSONConverter().convert(doc)
elif content_type in ['text/tab-separated-values', 'text/tsv']:
doc_format = 'tsv'
data, errors = utils.transforms.TSVToJSONConverter().convert(doc)
else:
doc_format = 'json'
data = utils.parse.parse_request_json()
errors = None
# TODO: use errors value?
name = flask.request.headers.get('X-Document-Name', None)
doc_args = [name, doc_format, doc, data]
is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC))
db_driver = tx_kwargs.pop('db_driver', flask.current_app.db)
transaction = UploadTransaction(program=program,
project=project,
role=role,
user=flask.g.user,
logger=flask.current_app.logger,
flask_config=flask.current_app.config,
signpost=flask.current_app.signpost,
external_proxies=utils.get_external_proxies(),
db_driver=db_driver,
**tx_kwargs)
if is_async:
session = transaction.db_driver.session_scope(can_inherit=False)
with session, transaction:
response = {
"code": 200,
"message": "Transaction submitted.",
"transaction_id": transaction.transaction_id,
}
flask.current_app.async_pool.schedule(
single_transaction_worker, transaction, *doc_args
)
return flask.jsonify(response)
else:
response, code = single_transaction_worker(transaction, *doc_args)
return flask.jsonify(response), code
def handle_bulk_transaction(role, program, project, **tx_kwargs):
"""
TODO
"""
wrappers = utils.parse.parse_request_json()
# Assert wrapper is list of JSON objects
invalid_format_msg = (
'Bulk transfers must be an array of JSON objects of format: {\n'
' "name": string,\n'
' "doc_format": string,\n'
' "doc": string,\n'
'}'
)
if not isinstance(wrappers, list):
raise UserError(invalid_format_msg)
for wrapper in wrappers:
if not isinstance(wrapper, dict):
raise UserError(invalid_format_msg)
is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC))
transaction = BulkUploadTransaction(
program=program,
project=project,
role=role,
user=flask.g.user,
logger=flask.current_app.logger,
signpost=flask.current_app.signpost,
db_driver=flask.current_app.db,
external_proxies=utils.get_external_proxies(),
**tx_kwargs
)
if is_async:
session = transaction.db_driver.session_scope(can_inherit=False)
with session, transaction:
response = {
"code": 200,
"message": "Transaction submitted.",
"transaction_id": transaction.transaction_id,
}
flask.current_app.async_pool.schedule(
bulk_transaction_worker, transaction, wrappers
)
return flask.jsonify(response)
else:
response, code = bulk_transaction_worker(transaction, wrappers)
return flask.jsonify(response), code
def _single_transaction(role, program, project, *doc_args, **tx_kwargs):
"""
Create and execute a single (not bulk) transaction.
Most non-bulk variations of upload actions (based on content-type,
etc.) should wrap this function.
Return:
Tuple[flask.Response, int]: (API response json, status code)
"""
is_async = tx_kwargs.pop('is_async', utils.is_flag_set(FLAG_IS_ASYNC))
db_driver = tx_kwargs.pop('db_driver', flask.current_app.db)
transaction = UploadTransaction(
program=program,
project=project,
role=role,
user=flask.g.user,
logger=flask.current_app.logger,
signpost=flask.current_app.signpost,
flask_config=flask.current_app.config,
db_driver=db_driver,
external_proxies=utils.get_external_proxies(),
**tx_kwargs
)
if is_async:
session = transaction.db_driver.session_scope(can_inherit=False)
with session, transaction:
response = {
"code": 200,
"message": "Transaction submitted.",
"transaction_id": transaction.transaction_id,
}
flask.current_app.async_pool.schedule(
single_transaction_worker, transaction, *doc_args
)
return flask.jsonify(response)
else:
response, code = single_transaction_worker(transaction, *doc_args)
return flask.jsonify(response), code
def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter):
"""
Make sure that concurrent transactions don't cause duplicate submission.
"""
data = {
"type": "experiment",
"submitter_id": "BLGSP-71-06-00019",
"projects.id": "daa208a7-f57a-562c-a04a-7a7c77542c98",
}
# convert to TSV (save to file)
file_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "data/experiment_tmp.tsv"
)
with open(file_path, "w") as f:
dw = csv.DictWriter(f, sorted(data.keys()), delimiter="\t")
dw.writeheader()
dw.writerow(data)
# read the TSV data
data = None
with open(file_path, "r") as f:
data = f.read()
os.remove(file_path) # clean up (delete file)
assert data
program, project = BLGSP_PATH.split("/")[3:5]
tsv_data = TSVToJSONConverter().convert(data)[0]
doc_args = [None, "tsv", data, tsv_data]
utx1, utx2 = [
UploadTransaction(
program=program,
project=project,
role=ROLES["UPDATE"],
logger=app.logger,
flask_config=app.config,
index_client=app.index_client,
external_proxies=get_external_proxies(),
db_driver=pg_driver,
)
for _ in range(2)
]
response = ""
with pg_driver.session_scope(can_inherit=False) as s1:
with utx1:
utx1.parse_doc(*doc_args)
with pg_driver.session_scope(can_inherit=False) as s2:
with utx2:
utx2.parse_doc(*doc_args)
with pg_driver.session_scope(session=s2):
utx2.flush()
with pg_driver.session_scope(session=s2):
utx2.post_validate()
with pg_driver.session_scope(session=s2):
utx2.commit()
try:
with pg_driver.session_scope(session=s1):
utx1.flush()
except IntegrityError:
s1.rollback()
utx1.integrity_check()
response = utx1.json
assert response["entity_error_count"] == 1
assert response["code"] == 400
assert (
response["entities"][0]["errors"][0]["message"]
== "experiment with {'project_id': 'CGCI-BLGSP', 'submitter_id': 'BLGSP-71-06-00019'} already exists in the DB"
)
with pg_driver.session_scope():
assert pg_driver.nodes(md.Experiment).count() == 1
def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter):
"""
Make sure that concurrent transactions don't cause duplicate submission.
"""
data = {
"type": "experiment",
"submitter_id": "BLGSP-71-06-00019",
"projects.id": "daa208a7-f57a-562c-a04a-7a7c77542c98"
}
# convert to TSV (save to file)
file_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"data/experiment_tmp.tsv"
)
with open(file_path, "w") as f:
dw = csv.DictWriter(f, sorted(data.keys()), delimiter="\t")
dw.writeheader()
dw.writerow(data)
# read the TSV data
data = None
with open(file_path, "r") as f:
data = f.read()
os.remove(file_path) # clean up (delete file)
assert data
program, project = BLGSP_PATH.split('/')[3:5]
tsv_data = TSVToJSONConverter().convert(data)[0]
doc_args = [None, 'tsv', data, tsv_data]
utx1, utx2 = [UploadTransaction(
program=program,
project=project,
role=ROLES['UPDATE'],
logger=app.logger,
flask_config=app.config,
signpost=app.signpost,
external_proxies=get_external_proxies(),
db_driver=pg_driver,
) for _ in range(2)]
with pg_driver.session_scope(can_inherit=False) as s1:
with utx1:
utx1.parse_doc(*doc_args)
with pg_driver.session_scope(can_inherit=False) as s2:
with utx2:
utx2.parse_doc(*doc_args)
with pg_driver.session_scope(session=s2):
utx2.flush()
with pg_driver.session_scope(session=s2):
utx2.post_validate()
with pg_driver.session_scope(session=s2):
utx2.commit()
try:
with pg_driver.session_scope(session=s1):
utx1.flush()
with pg_driver.session_scope(session=s1):
utx1.post_validate()
with pg_driver.session_scope(session=s1):
utx1.commit()
except HandledIntegrityError:
pass
with pg_driver.session_scope():
assert pg_driver.nodes(md.Experiment).count() == 1
