def test_upload_max_size(app):
# Test uploading a file with size greater than UPLOAD_MAX_SIZE
app.config.update({'UPLOAD_MAX_SIZE': 1000000})
with app.app_context():
user = User.query.first()
login_user(user)
recid = '12345'
get_or_create_hepsubmission(recid, 1)
base_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
response, code = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (code == 413)
assert (response.json == {
'message':
'TestHEPSubmission.zip too large (1818265 bytes > 1000000 bytes)'
})
def update_sandbox_payload(recid):
"""
Updates the Sandbox submission with a new file upload.
:param recid:
"""
if request.method == 'GET':
return redirect('/record/sandbox/' + str(recid))
file = request.files['hep_archive']
redirect_url = request.url_root + "record/sandbox/{}"
return process_payload(recid, file, redirect_url)
def consume_sandbox_payload():
"""
Creates a new sandbox submission with a new file upload.
:param recid:
"""
if request.method == 'GET':
return redirect('/record/sandbox')
id = (int(current_user.get_id())) + int(round(time.time()))
get_or_create_hepsubmission(id, current_user.get_id(), status="sandbox")
file = request.files['hep_archive']
redirect_url = request.url_root + "record/sandbox/{}"
return process_payload(id, file, redirect_url)
def consume_data_payload(recid):
"""
This method persists, then presents the loaded data back to the user.
:param recid: record id to attach the data to
:return: For POST requests, returns JSONResponse either containing 'url'
(for success cases) or 'message' (for error cases, which will
give a 400 error). For GET requests, redirects to the record.
"""
if request.method == 'POST':
file = request.files['hep_archive']
redirect_url = request.url_root + "record/{}"
return process_payload(recid, file, redirect_url)
else:
return redirect('/record/' + str(recid))
def update_sandbox_payload(recid):
"""
Updates the Sandbox submission with a new file upload.
:param recid:
"""
if request.method == 'GET':
return redirect('/record/sandbox/' + str(recid))
if not has_upload_permissions(recid, current_user, is_sandbox=True):
return jsonify({
"message": "Current user does not correspond to a confirmed uploader for this record."
}), 403
file = request.files['hep_archive']
redirect_url = request.url_root + "record/sandbox/{}"
return process_payload(recid, file, redirect_url)
def test_upload_invalid_file(app):
# Test uploading an invalid file
with app.app_context():
user = User.query.first()
login_user(user)
recid = '12345'
get_or_create_hepsubmission(recid, 1)
with StringIO("test") as stream:
test_file = FileStorage(stream=stream, filename="test.txt")
response, code = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (code == 400)
assert (response.json == {
'message':
'You must upload a .zip, .tar, .tar.gz or .tgz file'
' (or a .oldhepdata or single .yaml or .yaml.gz file).'
})
def consume_data_payload(recid):
"""
This method persists, then presents the loaded data back to the user.
:param recid: record id to attach the data to
:return: For POST requests, returns JSONResponse either containing 'url'
(for success cases) or 'message' (for error cases, which will
give a 400 error). For GET requests, redirects to the record.
"""
if request.method == 'POST':
if not has_upload_permissions(recid, current_user):
return jsonify({
"message": "Current user does not correspond to a confirmed uploader for this record."
}), 403
file = request.files['hep_archive']
redirect_url = request.url_root + "record/{}"
return process_payload(recid, file, redirect_url)
else:
return redirect('/record/' + str(recid))
def test_upload_valid_file_yaml_gz(app):
# Test uploading and processing a file for a record
with app.app_context():
base_dir = os.path.dirname(os.path.realpath(__file__))
user = User.query.first()
login_user(user)
recid = '1512299'
get_or_create_hepsubmission(recid, 1)
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission is not None)
assert (hepdata_submission.data_abstract is None)
assert (hepdata_submission.created < hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
with open(os.path.join(base_dir, 'test_data/1512299.yaml.gz'),
"rb") as stream:
test_file = FileStorage(stream=stream, filename="1512299.yaml.gz")
response = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (response.json == {'url': '/test_redirect_url'})
# Check the submission has been updated
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission.data_abstract.startswith(
'Unfolded differential decay rates of four kinematic variables'))
assert (hepdata_submission.created < hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
# Set the status to finished and try again, to check versioning
hepdata_submission.overall_status = 'finished'
db.session.add(hepdata_submission)
# Refresh user
user = User.query.first()
login_user(user)
with open(os.path.join(base_dir, 'test_data/1512299.yaml.gz'),
"rb") as stream:
test_file = FileStorage(stream=stream, filename="1512299.yaml.gz")
process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
# Check the submission has been updated
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 2)
assert (hepdata_submissions[0].version == 1)
assert (hepdata_submissions[0].overall_status == 'finished')
assert (hepdata_submissions[1].data_abstract.startswith(
'Unfolded differential decay rates of four kinematic variables'))
assert (hepdata_submissions[1].version == 2)
assert (hepdata_submissions[1].overall_status == 'todo')
def test_upload_valid_file(app):
# Test uploading and processing a file for a record
with app.app_context():
base_dir = os.path.dirname(os.path.realpath(__file__))
for i, status in enumerate(["todo", "sandbox"]):
user = User.query.first()
login_user(user)
recid = f'12345{i}'
get_or_create_hepsubmission(recid, 1, status=status)
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission is not None)
assert (hepdata_submission.data_abstract is None)
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
response = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (response.json == {'url': '/test_redirect_url'})
# Check the submission has been updated
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission.data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
# Set the status to finished and try again, to check versioning
if status == "todo":
hepdata_submission.overall_status = 'finished'
db.session.add(hepdata_submission)
# Sleep before uploading new version to avoid dir name conflict
sleep(1)
# Refresh user
user = User.query.first()
login_user(user)
# Upload a new version
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
# Check the submission has been updated (overridden for a sandbox;
# new version for normal submission)
expected_versions = 2 if status == "todo" else 1
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == expected_versions)
assert (hepdata_submissions[0].version == 1)
if status == "todo":
assert (hepdata_submissions[0].overall_status == 'finished')
assert (hepdata_submissions[-1].data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submissions[-1].version == expected_versions)
assert (hepdata_submissions[-1].overall_status == status)
# Check that there are the expected number of subdirectories and
# zip files under the record's main path
# For status = 'todo' (standard submission) there will be 1 file
# and 1 dir for each of 2 versions; for the sandbox submission
# there will just be 1 file and 1 dir.
directory = get_data_path_for_record(
hepdata_submission.publication_recid)
assert (os.path.exists(directory))
filepaths = os.listdir(directory)
assert (len(filepaths) == 2 * expected_versions)
dir_count = 0
file_count = 0
for path in filepaths:
if os.path.isdir(os.path.join(directory, path)):
dir_count += 1
assert (re.match(r"\d{10}", path) is not None)
else:
file_count += 1
assert (re.match(r"HEPData-%s-v[12]-yaml.zip" % recid,
path) is not None)
assert (dir_count == expected_versions)
assert (file_count == expected_versions)
if status == "todo":
# Delete the v2 submission and check db and v2 files have been removed
unload_submission(hepdata_submission.publication_recid,
version=2)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 1)
assert (hepdata_submissions[0].version == 1)
assert (hepdata_submissions[0].overall_status == 'finished')
filepaths = os.listdir(directory)
assert (len(filepaths) == 2)
assert (f"HEPData-12345{i}-v1-yaml.zip" in filepaths)
# Delete the submission and check everything has been removed
unload_submission(hepdata_submission.publication_recid, version=1)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 0)
assert (not os.path.exists(directory))