def get_data_path(record_id=None, inspire_id=None):
"""Gets the file path where data files for the given record are stored."""
if record_id:
# Check record exists
hepsubmission = get_latest_hepsubmission(publication_recid=record_id)
if hepsubmission is None:
click.echo("No record with id %s" % record_id)
return
elif inspire_id:
hepsubmission = get_latest_hepsubmission(inspire_id=inspire_id)
if hepsubmission is None:
click.echo("No record with inspire id %s" % inspire_id)
return
else:
record_id = hepsubmission.publication_recid
click.echo("Inspire ID %s maps to record id %s" %
(inspire_id, record_id))
else:
click.echo("Please provide either record-id or inspire-id.")
return
click.echo("Files for record %s are at:\t\t %s" %
(record_id, data_files.get_data_path_for_record(record_id)))
click.echo("Converted files for record %s are at:\t %s" %
(record_id, data_files.get_converted_directory_path(record_id)))
def package_submission(basepath, recid, hep_submission_obj):
"""
Zips up a submission directory. This is in advance of its download
for example by users.
:param basepath: path of directory containing all submission files
:param recid: the publication record ID
:param hep_submission_obj: the HEPSubmission object representing
the overall position
"""
path = get_data_path_for_record(str(recid))
if not os.path.exists(path):
os.makedirs(path)
version = hep_submission_obj.version
if version == 0:
version = 1
zip_location = os.path.join(
path,
current_app.config['SUBMISSION_FILE_NAME_PATTERN']
.format(recid, version))
if os.path.exists(zip_location):
os.remove(zip_location)
try:
shutil.make_archive(os.path.splitext(zip_location)[0], 'zip', basepath)
return {}
except Exception as e:
return {zip_location: [{"level": "error", "message": str(e)}]}
def check_and_convert_from_oldhepdata(input_directory, id, timestamp):
"""
Check if the input directory contains a .oldhepdata file
and convert it to YAML if it happens.
"""
converted_path = get_data_path_for_record(str(id), timestamp, 'yaml')
oldhepdata_found = find_file_in_directory(
input_directory,
lambda x: x.endswith('.oldhepdata'),
)
if not oldhepdata_found:
return {
"Converter": [{
"level":
"error",
"message":
"No file with .oldhepdata extension has been found."
}]
}
converted_temp_dir = tempfile.mkdtemp(dir=current_app.config["CFG_TMPDIR"])
converted_temp_path = os.path.join(converted_temp_dir, 'yaml')
try:
successful = convert_oldhepdata_to_yaml(oldhepdata_found[1],
converted_temp_path)
if not successful:
# Parse error message from title of HTML file, removing part of string after final "//".
soup = BeautifulSoup(open(converted_temp_path), "lxml")
errormsg = soup.title.string.rsplit("//", 1)[0]
except Error as error: # hepdata_converter_ws_client.Error
successful = False
errormsg = str(error)
if not successful:
shutil.rmtree(converted_temp_dir, ignore_errors=True
) # can uncomment when this is definitely working
return {
"Converter": [{
"level":
"error",
"message":
"The conversion from oldhepdata "
"to the YAML format has not succeeded. "
"Error message from converter follows:<br/><br/>" + errormsg
}]
}
else:
copy_errors = move_files(converted_temp_path, converted_path)
if copy_errors:
return copy_errors
return find_file_in_directory(converted_path,
lambda x: x == "submission.yaml")
def save_zip_file(file, id):
filename = secure_filename(file.filename)
time_stamp = str(int(round(time.time())))
file_save_directory = get_data_path_for_record(str(id), time_stamp)
if filename.endswith('.oldhepdata'):
file_save_directory = os.path.join(file_save_directory, 'oldhepdata')
if not os.path.exists(file_save_directory):
os.makedirs(file_save_directory)
file_path = os.path.join(file_save_directory, filename)
print('Saving file to {}'.format(file_path))
file.save(file_path)
return file_path
def create_original_with_resources(submission, data_filepath, output_path):
"""Copy or create 'original' zip file, i.e. yaml files with resources. If
resources were imported from hepdata.cedar.ac.uk we create a new zip
in a format that could be re-uploaded as a submission.
:param type submission: HEPSubmission object
:param type data_filepath: Path to original file
:param type output_path: Path to output file (in converted dir)
:return: None
"""
resource_location = os.path.join(
get_data_path_for_record(str(submission.publication_recid)),
'resources'
)
if os.path.isdir(resource_location):
# There is a resources directory from when this record was imported
# from the old hepdata site. We need to create a new zip with the
# contents of data_filepath and resources
with tempfile.TemporaryDirectory(dir=current_app.config['CFG_TMPDIR']) as tmpdir:
# Copy resources directory into 'contents' dir in temp directory
contents_path = os.path.join(tmpdir, 'contents')
shutil.copytree(resource_location, contents_path)
# Unzip data_filepath into contents path
shutil.unpack_archive(data_filepath, contents_path)
# Need to go through the submission file and update the paths so
# that all resources are at the top level. This should allow the
# zip to be re-uploaded or imported
submission_found = find_file_in_directory(
contents_path,
lambda x: x == "submission.yaml"
)
if submission_found:
with fileinput.FileInput(submission_found[1], inplace=True) as file:
p = re.compile(r'(\s+location: )\/resource\/.*\/([^\/]+)')
for line in file:
print(p.sub(r'\g<1>\g<2>', line), end='')
# Zip up contents dir into a new file
base, ext = os.path.splitext(output_path)
zip_type = 'zip' if ext == '.zip' else 'gztar'
print("Creating archive at %s" % output_path)
shutil.make_archive(base, zip_type, contents_path)
else:
shutil.copy2(data_filepath, output_path)
def test_create_submission(app, admin_idx):
"""
Test the whole submission pipeline in loading a file, ensuring the HEPSubmission object is created,
all the files have been added, and the record has been indexed.
:return:
"""
with app.app_context():
admin_idx.recreate_index()
# test submission part works
record = {
'inspire_id': '19999999',
'title': 'HEPData Testing 1',
'reviewer': {
'name': 'Testy McTester',
'email': '*****@*****.**'
},
'uploader': {
'name': 'Testy McTester',
'email': '*****@*****.**'
},
'message': 'This is ready',
'user_id': 1
}
hepdata_submission = process_submission_payload(**record)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
# test upload works
base_dir = os.path.dirname(os.path.realpath(__file__))
test_directory = os.path.join(base_dir, 'test_data/test_submission')
time_stamp = str(int(round(time.time())))
directory = get_data_path_for_record(
hepdata_submission.publication_recid, time_stamp)
shutil.copytree(test_directory, directory)
assert (os.path.exists(directory))
process_submission_directory(
directory, os.path.join(directory, 'submission.yaml'),
hepdata_submission.publication_recid)
admin_idx_results = admin_idx.search(
term=hepdata_submission.publication_recid, fields=['recid'])
assert (admin_idx_results is not None)
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepdata_submission.publication_recid).count()
assert (data_submissions == 8)
assert (len(hepdata_submission.resources) == 4)
assert (len(hepdata_submission.participants) == 4)
do_finalise(hepdata_submission.publication_recid,
force_finalise=True,
convert=False)
assert (record_exists(inspire_id=record['inspire_id']))
# Test record is in index...
index_records = get_records_matching_field('inspire_id',
record['inspire_id'],
doc_type='publication')
assert (len(index_records['hits']['hits']) == 1)
publication_record = get_record_contents(
hepdata_submission.publication_recid)
assert (publication_record is not None)
ctx = format_submission(hepdata_submission.publication_recid,
publication_record, hepdata_submission.version,
1, hepdata_submission)
assert (ctx is not None)
assert (ctx['version'] == 1)
assert (ctx['recid'] == hepdata_submission.publication_recid)
# remove the submission and test that all is remove
unload_submission(hepdata_submission.publication_recid)
assert (not record_exists(inspire_id=record['inspire_id']))
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepdata_submission.publication_recid).count()
assert (data_submissions == 0)
sleep(2)
admin_idx_results = admin_idx.search(
term=hepdata_submission.publication_recid, fields=['recid'])
assert (len(admin_idx_results) == 0)
# Check file dir has been deleted
assert (not os.path.exists(directory))
def test_upload_valid_file(app):
# Test uploading and processing a file for a record
with app.app_context():
base_dir = os.path.dirname(os.path.realpath(__file__))
for i, status in enumerate(["todo", "sandbox"]):
user = User.query.first()
login_user(user)
recid = f'12345{i}'
get_or_create_hepsubmission(recid, 1, status=status)
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission is not None)
assert (hepdata_submission.data_abstract is None)
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
response = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (response.json == {'url': '/test_redirect_url'})
# Check the submission has been updated
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission.data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
# Set the status to finished and try again, to check versioning
if status == "todo":
hepdata_submission.overall_status = 'finished'
db.session.add(hepdata_submission)
# Sleep before uploading new version to avoid dir name conflict
sleep(1)
# Refresh user
user = User.query.first()
login_user(user)
# Upload a new version
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
# Check the submission has been updated (overridden for a sandbox;
# new version for normal submission)
expected_versions = 2 if status == "todo" else 1
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == expected_versions)
assert (hepdata_submissions[0].version == 1)
if status == "todo":
assert (hepdata_submissions[0].overall_status == 'finished')
assert (hepdata_submissions[-1].data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submissions[-1].version == expected_versions)
assert (hepdata_submissions[-1].overall_status == status)
# Check that there are the expected number of subdirectories and
# zip files under the record's main path
# For status = 'todo' (standard submission) there will be 1 file
# and 1 dir for each of 2 versions; for the sandbox submission
# there will just be 1 file and 1 dir.
directory = get_data_path_for_record(
hepdata_submission.publication_recid)
assert (os.path.exists(directory))
filepaths = os.listdir(directory)
assert (len(filepaths) == 2 * expected_versions)
dir_count = 0
file_count = 0
for path in filepaths:
if os.path.isdir(os.path.join(directory, path)):
dir_count += 1
assert (re.match(r"\d{10}", path) is not None)
else:
file_count += 1
assert (re.match(r"HEPData-%s-v[12]-yaml.zip" % recid,
path) is not None)
assert (dir_count == expected_versions)
assert (file_count == expected_versions)
if status == "todo":
# Delete the v2 submission and check db and v2 files have been removed
unload_submission(hepdata_submission.publication_recid,
version=2)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 1)
assert (hepdata_submissions[0].version == 1)
assert (hepdata_submissions[0].overall_status == 'finished')
filepaths = os.listdir(directory)
assert (len(filepaths) == 2)
assert (f"HEPData-12345{i}-v1-yaml.zip" in filepaths)
# Delete the submission and check everything has been removed
unload_submission(hepdata_submission.publication_recid, version=1)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 0)
assert (not os.path.exists(directory))
def test_get_data_path_for_record(app):
data_dir = app.config['CFG_DATADIR']
assert (get_data_path_for_record('ins12345') == data_dir + '/96/ins12345')
assert (get_data_path_for_record('ins12345', 'mysubdir',
'file.xyz') == data_dir +
'/96/ins12345/mysubdir/file.xyz')
def mock_import_old_record(inspire_id=mock_inspire_ids[1], send_email=False):
"""Creates a submission but mimics the old migrated paths. (See hepdata
master branch at ccd691b for old migrator module.)
"""
if inspire_id not in mock_inspire_ids:
raise ValueError('Invalid inspire id %s. Accepted values are: %s' %
(inspire_id, ', '.join(mock_inspire_ids)))
# Use zipped test data for specific record(s)
publication_information, status = get_inspire_record_information(
inspire_id)
publication_information["inspire_id"] = inspire_id
# Create record
if status == "success":
record_information = create_record(publication_information)
else:
log.error("Failed to retrieve publication information for " +
inspire_id)
return False
# Unzip into correct data dir
data_path = get_data_path_for_record(record_information['recid'])
base_dir = os.path.dirname(os.path.realpath(__file__))
zip_path = os.path.join(base_dir, 'old_hepdata_zips',
'ins%s.zip' % inspire_id)
if os.path.isfile(zip_path):
log.info('Unzipping %s to %s' % (zip_path, data_path))
shutil.unpack_archive(zip_path, data_path)
time_stamp = str(int(round(time.time())))
yaml_path = os.path.join(data_path, time_stamp)
sub_zip_path = os.path.join(data_path, 'ins%s.zip' % inspire_id)
shutil.unpack_archive(sub_zip_path, yaml_path)
else:
log.error('Invalid path %s' % zip_path)
return False
# Create submission
admin_user_id = 1
# Consume data payload and store in db.
get_or_create_hepsubmission(record_information["recid"], admin_user_id)
errors = process_submission_directory(yaml_path,
os.path.join(yaml_path,
"submission.yaml"),
record_information["recid"],
old_submission_schema=True,
old_data_schema=True)
if errors:
log.error(
"Submission failed for {0}.".format(record_information["recid"]),
errors, record_information["recid"])
return False
do_finalise(record_information['recid'],
publication_record=record_information,
force_finalise=True,
convert=False,
send_email=send_email)
def _import_record(inspire_id,
update_existing=False,
base_url='https://hepdata.net',
send_email=False):
publication_information, status = get_inspire_record_information(
inspire_id)
if status != "success":
log.error("Failed to retrieve publication information for " +
inspire_id)
return False
current_submission = get_latest_hepsubmission(inspire_id=inspire_id)
if not current_submission:
log.info(
"The record with id {0} does not exist in the database, so we're loading it."
.format(inspire_id))
publication_information["inspire_id"] = inspire_id
record_information = create_record(publication_information)
recid = record_information['recid']
else:
log.info("The record with inspire id {0} already exists.".format(
inspire_id))
if update_existing:
log.info("Updating instead")
recid = current_submission.publication_recid
else:
log.info("Not updating as update_existing is False")
return False
try:
download_path = _download_file(base_url, inspire_id)
filename = os.path.basename(download_path)
time_stamp = str(int(round(time.time())))
file_save_directory = get_data_path_for_record(str(recid), time_stamp)
if not os.path.exists(file_save_directory):
os.makedirs(file_save_directory)
file_path = os.path.join(file_save_directory, filename)
log.info("Moving file to %s" % file_path)
shutil.copy(download_path, file_path)
# Create submission
admin_user_id = 1
hepsubmission = get_or_create_hepsubmission(recid, admin_user_id)
db.session.add(hepsubmission)
db.session.commit()
# Then process the payload as for any other record
errors = process_zip_archive(file_path, recid)
if errors:
log.info("Errors processing archive. Re-trying with old schema.")
# Try again with old schema
# Need to clean up first to avoid errors
# First delete tables
cleanup_submission(recid, 1, [])
# Next remove remaining files
file_save_directory = os.path.dirname(file_path)
submission_path = os.path.join(file_save_directory,
remove_file_extension(filename))
shutil.rmtree(submission_path)
errors = process_zip_archive(file_path,
recid,
old_submission_schema=True,
old_data_schema=True)
if errors:
log.error("Could not process zip archive: ")
for file, file_errors in errors.items():
log.error(" %s:" % file)
for error in file_errors:
log.error(" %s" % error['message'])
raise ValueError("Could not validate record.")
# Delete any previous upload folders
cleanup_old_files(hepsubmission)
log.info("Finalising record %s" % recid)
result_json = do_finalise(recid,
force_finalise=True,
update=(current_submission is not None),
convert=False,
send_email=send_email)
result = json.loads(result_json)
if result and result['success']:
log.info("Imported record %s with %s submissions" %
(recid, result['data_count']))
return True
else:
raise ValueError("Failed to finalise record.")
except Exception as e:
# Unload record
unload_submission(recid)
log.error(e)
return False