def convert_endpoint():
"""
Endpoint for general conversion, the file is passed as a GET parameter
and options ('from=' & 'to=') are query string arguments.
TO DO: is this function used anywhere? If not, it can probably be removed.
:return: display_error or send_file depending on success of conversion
"""
input_format, output_format = request.args.get(
'from', 'oldhepdata'), request.args.get('to', 'yaml')
if input_format not in ['yaml', 'oldhepdata'] or \
output_format not in ['root', 'yoda', 'csv', 'yaml']:
return display_error(
title="Chosen formats are not supported",
description="Supported input formats: oldhepdata, yaml\n" +
"Supported output formats: root, yoda, csv")
fileobject = request.files.get('file')
if not fileobject or not fileobject.filename.endswith('.zip'):
print("Fileobject: " + str(fileobject))
return display_error(
title="Please send a zip file for conversion",
description=
"No file has been sent or it does not have a zip extension")
filename = secure_filename(fileobject.filename)
timestamp = str(int(round(time.time())))
input_archive = os.path.join(current_app.config['CFG_TMPDIR'],
timestamp + '.zip')
fileobject.save(input_archive)
options = {
'input_format': input_format,
'output_format': output_format,
'filename': filename[:-4],
}
output_file = os.path.join(current_app.config['CFG_TMPDIR'],
timestamp + '.tar.gz')
try:
conversion_result = convert_zip_archive(input_archive, output_file,
options)
except Error as error: # hepdata_converter_ws_client.Error
return display_error(title='Report concerns to [email protected]',
description=str(error))
finally:
os.remove(input_archive)
if not conversion_result:
return display_error(
title="Your archive does not contain necessary files",
description=
"For YAML conversions a submission.yaml file is necessary,"
" and for conversions from the oldhepdata format"
" a file with .oldhepdata extension is required.")
return send_file(conversion_result, as_attachment=True)
def convert_endpoint():
""" Endpoint for general conversion, the file is passed as a GET parameter
and options ('from=' & 'to=') are query string arguments. """
input_format, output_format = request.args.get('from', 'oldhepdata'), request.args.get('to', 'yaml')
if input_format not in ['yaml', 'oldhepdata'] or \
output_format not in ['root', 'yoda', 'csv', 'yaml']:
return display_error(
title="Chosen formats are not supported",
description="Supported input formats: oldhepdata, yaml\n" +
"Supported output formats: root, yoda, csv"
)
fileobject = request.files.get('file')
if not fileobject or not fileobject.filename.endswith('.zip'):
print("Fileobject: " + str(fileobject))
return display_error(
title="Please send a zip file for conversion",
description="No file has been sent or it does not have a zip extension"
)
filename = secure_filename(fileobject.filename)
timestamp = str(int(round(time.time())))
input_archive = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.zip')
fileobject.save(input_archive)
options = {
'input_format': input_format,
'output_format': output_format,
'filename': filename[:-4],
}
output_file = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.tar.gz')
conversion_result = convert_zip_archive(input_archive, output_file, options)
os.remove(input_archive)
if not conversion_result:
return display_error(
title="Your archive does not contain necessary files",
description="For YAML conversions a submission.yaml file is necessary,"
" and for conversions from the oldhepdata format"
" a file with .oldhepdata extension is required."
)
return send_file(conversion_result, as_attachment=True)
def download_submission(submission,
file_format,
offline=False,
force=False,
rivet_analysis_name=''):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param submission: HEPSubmission
:param file_format: yaml, csv, root, or yoda
:param offline: offline creation of the conversion when a record is finalised
:param force: force recreation of the conversion
:param rivet_analysis_name: Rivet analysis name to override default written in YODA export
:return: display_error or send_file depending on success of conversion
"""
version = submission.version
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
if file_format == 'json':
return redirect('/record/{0}?version={1}&format=json'.format(
file_identifier, version))
elif file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
path = os.path.join(current_app.config['CFG_DATADIR'],
str(submission.publication_recid))
data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
submission.publication_recid, version)
output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier,
submission.version,
file_format)
converted_dir = os.path.join(current_app.config['CFG_DATADIR'],
'converted')
if not os.path.exists(converted_dir):
os.mkdir(converted_dir)
if file_format == 'yoda' and rivet_analysis_name:
# Don't store in converted_dir since rivet_analysis_name might possibly change between calls.
output_path = os.path.join(current_app.config['CFG_TMPDIR'],
output_file)
else:
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file or the submission is not finished.
if os.path.exists(
output_path
) and not force and submission.overall_status == 'finished':
if not offline:
return send_file(output_path, as_attachment=True)
else:
print('File already downloaded at {0}'.format(output_path))
return
converter_options = {
'input_format':
'yaml',
'output_format':
file_format,
'filename':
'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version,
file_format),
'validator_schema_version':
'0.1.0',
}
if submission.doi and submission.overall_status != 'sandbox':
converter_options['hepdata_doi'] = '{0}.v{1}'.format(
submission.doi, version)
if file_format == 'yoda':
if rivet_analysis_name:
converter_options['rivet_analysis_name'] = rivet_analysis_name
elif submission.inspire_id:
record = get_record_contents(submission.publication_recid)
if record:
# Check if this record has a Rivet analysis, then extract the Rivet analysis name from the URL.
if 'analyses' in record:
for analysis in record['analyses']:
if analysis['type'] == 'rivet':
converter_options[
'rivet_analysis_name'] = analysis[
'analysis'].split('/')[-1]
# Otherwise guess the Rivet analysis name using the collaboration name,
# the creation year of the INSPIRE record, and the INSPIRE ID.
if 'rivet_analysis_name' not in converter_options:
try:
year = parse(record['creation_date']).year
except:
year = record['year'] # publication year
converter_options[
'rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), year,
submission.inspire_id)
data_filepath = os.path.join(path, data_filename)
converted_file = convert_zip_archive(data_filepath, output_path,
converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} created successfully at {1}'.format(
file_identifier, output_path))
def download_submission(submission, file_format, offline=False, force=False, rivet_analysis_name=''):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param submission: HEPSubmission
:param file_format: json, yaml, csv, root, yoda or original
:param offline: offline creation of the conversion when a record is finalised
:param force: force recreation of the conversion
:param rivet_analysis_name: Rivet analysis name to override default written in YODA export
:return: display_error or send_file depending on success of conversion
"""
version = submission.version
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
if file_format == 'json':
return redirect('/record/{0}?version={1}&format=json'.format(file_identifier, version))
elif file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
data_filepath = find_submission_data_file_path(submission)
if file_format == 'original':
file_format_and_extension = os.path.splitext(data_filepath)[1]
else:
file_format_and_extension = '-{0}.tar.gz'.format(file_format)
output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension)
converted_dir = get_converted_directory_path(submission.publication_recid)
if not os.path.exists(converted_dir):
os.makedirs(converted_dir, exist_ok=True)
if file_format == 'yoda' and rivet_analysis_name:
# Don't store in converted_dir since rivet_analysis_name might possibly change between calls.
output_path = os.path.join(current_app.config['CFG_TMPDIR'], output_file)
else:
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file or the submission is not finished.
if os.path.exists(output_path) and not force and submission.overall_status == 'finished':
if not offline:
return send_file(output_path, as_attachment=True)
else:
print('File already downloaded at {0}'.format(output_path))
return
if file_format == 'original':
create_original_with_resources(submission, data_filepath, output_path)
if not offline:
return send_file(output_path, as_attachment=True)
else:
print('File created at {0}'.format(output_path))
return
converter_options = {
'input_format': 'yaml',
'output_format': file_format,
'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format),
'validator_schema_version': '0.1.0',
}
if submission.doi and not submission.overall_status.startswith('sandbox'):
converter_options['hepdata_doi'] = '{0}.v{1}'.format(submission.doi, version)
if file_format == 'yoda':
if not rivet_analysis_name:
rivet_analysis_name = guess_rivet_analysis_name(submission)
if rivet_analysis_name:
converter_options['rivet_analysis_name'] = rivet_analysis_name
try:
converted_file = convert_zip_archive(data_filepath, output_path, converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} created successfully at {1}'.format(file_identifier, output_path))
except Error as error: # hepdata_converter_ws_client.Error
if not offline:
return display_error(title='Report concerns to [email protected]', description=str(error))
else:
print('File conversion for {0} at {1} failed: {2}'.format(
file_identifier, output_path, str(error)
))
def download_submission(submission, file_format, offline=False, force=False):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param recid: submissions recid
:param version: version of submission to export. If -1, returns the latest.
:param file_format: yaml, csv, ROOT, or YODA
:return:
"""
if file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + CFG_SUPPORTED_FORMATS,
)
version = submission.version
path = os.path.join(current_app.config['CFG_DATADIR'], str(submission.publication_recid))
data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(submission.publication_recid, version)
# If a YAML file is requested, we just need to send this back.
if file_format == 'yaml' and os.path.exists(os.path.join(path, data_filename)):
if not offline:
return send_file(os.path.join(path, data_filename), as_attachment=True)
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
output_file = 'HEPData-{0}-{1}-{2}.tar.gz'.format(file_identifier, submission.version, file_format)
converted_dir = os.path.join(current_app.config['CFG_DATADIR'], 'converted')
if not os.path.exists(converted_dir):
os.mkdir(converted_dir)
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file.
if os.path.exists(output_path) and not force:
if not offline:
return send_file(
output_path,
as_attachment=True,
)
else:
print('File already downloaded at {0}'.format(output_path))
return
converter_options = {
'input_format': 'yaml',
'output_format': file_format,
'filename': 'HEPData-{0}-{1}-{2}'.format(file_identifier, submission.version, file_format),
}
data_filepath = os.path.join(path, data_filename)
converted_file = convert_zip_archive(data_filepath, output_path, converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} create successfully at {1}'.format(file_identifier, output_path))
def download_submission(submission, file_format, offline=False, force=False):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param submission: HEPSubmission
:param file_format: yaml, csv, root, or yoda
:return:
"""
version = submission.version
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
if file_format == 'json':
return redirect('/record/{0}?version={1}&format=json'.format(
file_identifier, version))
elif file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
path = os.path.join(current_app.config['CFG_DATADIR'],
str(submission.publication_recid))
data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
submission.publication_recid, version)
output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier,
submission.version,
file_format)
converted_dir = os.path.join(current_app.config['CFG_DATADIR'],
'converted')
if not os.path.exists(converted_dir):
os.mkdir(converted_dir)
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file.
if os.path.exists(output_path) and not force:
if not offline:
return send_file(
output_path,
as_attachment=True,
)
else:
print('File already downloaded at {0}'.format(output_path))
return
converter_options = {
'input_format':
'yaml',
'output_format':
file_format,
'filename':
'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version,
file_format),
}
if submission.doi:
converter_options['hepdata_doi'] = '{0}.v{1}'.format(
submission.doi, version)
if submission.inspire_id and file_format == 'yoda':
record = get_record_contents(submission.publication_recid)
if record:
converter_options['rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), record['year'],
submission.inspire_id)
data_filepath = os.path.join(path, data_filename)
converted_file = convert_zip_archive(data_filepath, output_path,
converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} created successfully at {1}'.format(
file_identifier, output_path))
