def get_records_participated_in_by_user():
_current_user_id = int(current_user.get_id())
as_uploader = SubmissionParticipant.query.filter_by(user_account=_current_user_id, role='uploader').order_by(
SubmissionParticipant.id.desc()).all()
as_reviewer = SubmissionParticipant.query.filter_by(user_account=_current_user_id, role='reviewer').order_by(
SubmissionParticipant.id.desc()).all()
as_coordinator_query = HEPSubmission.query.filter_by(coordinator=_current_user_id).order_by(
HEPSubmission.created.desc())
# special case, since this user ID is the one used for loading all submissions, which is in the 1000s.
if _current_user_id == 1:
as_coordinator_query = as_coordinator_query.limit(5)
as_coordinator = as_coordinator_query.all()
result = {'uploader': [], 'reviewer': [], 'coordinator': []}
if as_uploader:
_uploader = [get_record_contents(x.publication_recid) for x in as_uploader]
result['uploader'] = list(filter(partial(is_not, None), _uploader))
if as_reviewer:
_uploader = [get_record_contents(x.publication_recid) for x in as_reviewer]
result['reviewer'] = list(filter(partial(is_not, None), _uploader))
if as_coordinator:
_coordinator = [get_record_contents(x.publication_recid) for x in as_coordinator]
result['coordinator'] = list(filter(partial(is_not, None), _coordinator))
return result
def get_records_participated_in_by_user():
_current_user_id = int(current_user.get_id())
as_uploader = SubmissionParticipant.query.filter_by(user_account=_current_user_id, role='uploader').all()
as_reviewer = SubmissionParticipant.query.filter_by(user_account=_current_user_id, role='reviewer').all()
as_coordinator_query = HEPSubmission.query.filter_by(coordinator=_current_user_id)
# special case, since this user ID is the one used for loading all submissions, which is in the 1000s.
if _current_user_id == 1:
as_coordinator_query = as_coordinator_query.limit(5)
as_coordinator = as_coordinator_query.all()
result = {'uploader': [], 'reviewer': [], 'coordinator': []}
if as_uploader:
_uploader = [get_record_contents(x.publication_recid) for x in as_uploader]
result['uploader'] = filter(partial(is_not, None), _uploader)
if as_reviewer:
_uploader = [get_record_contents(x.publication_recid) for x in as_reviewer]
result['reviewer'] = filter(partial(is_not, None), _uploader)
if as_coordinator:
_coordinator = [get_record_contents(x.publication_recid) for x in as_coordinator]
result['coordinator'] = filter(partial(is_not, None), _coordinator)
return result
def index_submission(self, submission):
participants = []
for sub_participant in get_submission_participants_for_record(submission.publication_recid):
participants.append({'full_name': sub_participant.full_name, 'role': sub_participant.role})
record_information = get_record_contents(submission.publication_recid,
submission.overall_status)
data_count = DataSubmission.query.filter(DataSubmission.publication_recid == submission.publication_recid,
DataSubmission.version == submission.version).count()
if record_information:
collaboration = ','.join(record_information.get('collaborations', []))
self.add_to_index(_id=submission.publication_recid,
title=record_information['title'],
collaboration=collaboration,
recid=submission.publication_recid,
inspire_id=submission.inspire_id,
status=submission.overall_status,
data_count=data_count,
creation_date=submission.created,
last_updated=submission.last_updated,
version=submission.version,
participants=participants,
coordinator=submission.coordinator)
def guess_rivet_analysis_name(submission):
"""
Try to guess the Rivet analysis name.
:param submission: HEPSubmission object
:return: guessed Rivet analysis name
"""
rivet_analysis_name = ''
# Check if this submission has a Rivet analysis as additional resources,
# then extract the Rivet analysis name from the URL.
for resource in submission.resources:
if resource.file_type == 'rivet':
rivet_analysis_name = resource.file_location.split('/')[-1]
if not rivet_analysis_name:
# Otherwise guess the Rivet analysis name using the collaboration name,
# the creation year of the INSPIRE record, and the INSPIRE ID.
record = get_record_contents(submission.publication_recid,
submission.overall_status)
if record and 'inspire_id' in record:
try:
year = parse(record['creation_date']).year
except:
year = record['year'] # publication year
rivet_analysis_name = '{0}_{1}_I{2}'.format(''.join(
record['collaborations']).upper(), year, record['inspire_id'])
return rivet_analysis_name
def index_submission(self, submission):
participants = []
for sub_participant in submission.participants:
participants.append({"full_name": sub_participant.full_name, "role": sub_participant.role})
record_information = get_record_contents(submission.publication_recid)
data_count = DataSubmission.query.filter(
DataSubmission.publication_recid == submission.publication_recid,
DataSubmission.version == submission.version,
).count()
collaboration = ",".join(record_information.get("collaborations", []))
self.add_to_index(
_id=submission.publication_recid,
title=record_information["title"],
collaboration=collaboration,
recid=submission.publication_recid,
inspire_id=submission.inspire_id,
status=submission.overall_status,
data_count=data_count,
creation_date=submission.created,
last_updated=submission.last_updated,
version=submission.version,
participants=participants,
)
def get_records_subscribed_by_current_user():
subscriptions = Subscribers.query.filter(Subscribers.subscribers.contains(current_user)).all()
if subscriptions:
records = [get_record_contents(x.publication_recid) for x in subscriptions]
return filter(partial(is_not, None), records)
else:
return []
def get_records_subscribed_by_current_user():
subscriptions = Subscribers.query.filter(Subscribers.subscribers.contains(current_user)).all()
if subscriptions:
records = [get_record_contents(x.publication_recid) for x in subscriptions]
return list(filter(partial(is_not, None), records))
else:
return []
def test_create_submission(app):
"""
Test the whole submission pipeline in loading a file, ensuring the HEPSubmission object is created,
all the files have been added, and the record has been indexed.
:return:
"""
with app.app_context():
# test submission part works
record = {'inspire_id': '19999999',
'title': 'HEPData Testing 1',
'reviewer': {'name': 'Testy McTester', 'email': '*****@*****.**'},
'uploader': {'name': 'Testy McTester', 'email': '*****@*****.**'},
'message': 'This is ready',
'user_id': 1}
hepdata_submission = process_submission_payload(**record)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
# test upload works
base_dir = os.path.dirname(os.path.realpath(__file__))
directory = os.path.join(base_dir, 'test_data/test_submission')
process_submission_directory(directory, os.path.join(directory, 'submission.yaml'),
hepdata_submission.publication_recid)
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepdata_submission.publication_recid).count()
assert (data_submissions == 8)
assert (len(hepdata_submission.resources) == 4)
assert (len(hepdata_submission.participants) == 4)
do_finalise(hepdata_submission.publication_recid, force_finalise=True)
assert (record_exists(inspire_id=record['inspire_id']))
# Test record is in index...
index_records = get_records_matching_field('inspire_id', record['inspire_id'], doc_type='publication')
print(index_records)
assert (len(index_records['hits']['hits']) == 1)
publication_record = get_record_contents(hepdata_submission.publication_recid)
print(publication_record)
assert (publication_record is not None)
ctx = format_submission(hepdata_submission.publication_recid, publication_record, hepdata_submission.version, 1,
hepdata_submission)
assert(ctx is not None)
assert(ctx['version'] == 1)
assert (ctx['recid'] == hepdata_submission.publication_recid)
def test_get_record_contents(app, load_default_data, identifiers):
# Status finished - should use ES to get results
record1 = get_record_contents(1, status='finished')
for key in ["inspire_id", "title"]:
assert (record1[key] == identifiers[0][key])
# Status todo - should use DB for result
record2 = get_record_contents(1, status='todo')
# DB returns data from an Invenio RecordMetadata obj so has fewer fields
# than the ES dict
for key in record2.keys():
if key == 'last_updated':
# Date format is slightly different for DB vs ES
assert (record2[key].replace(' ', 'T') == record1[key])
else:
assert (record2[key] == record1[key])
record3 = get_record_contents(1)
assert (record3 == record1)
assert (get_record_contents(9999999) is None)
def render_record(recid, record, version, output_format, light_mode=False):
if user_allowed_to_perform_action(recid):
version_count = HEPSubmission.query.filter_by(
publication_recid=recid).count()
else:
version_count = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status='finished').count()
if version == -1:
version = version_count
hepdata_submission = get_latest_hepsubmission(publication_recid=recid, version=version)
if hepdata_submission is not None:
ctx = format_submission(recid, record, version, version_count, hepdata_submission)
increment(recid)
if output_format == "json":
ctx = process_ctx(ctx, light_mode)
return jsonify(ctx)
else:
return render_template('hepdata_records/publication_record.html',
ctx=ctx)
else: # this happens when we access an id of a data record
# in which case, we find the related publication, and
# make the front end focus on the relevant data table.
try:
publication_recid = int(record['related_publication'])
publication_record = get_record_contents(publication_recid)
hepdata_submission = get_latest_hepsubmission(recid=publication_recid)
ctx = format_submission(publication_recid, publication_record,
hepdata_submission.version, 1, hepdata_submission,
data_table=record['title'])
ctx['related_publication_id'] = publication_recid
ctx['table_name'] = record['title']
if output_format == "json":
ctx = process_ctx(ctx, light_mode)
return jsonify(ctx)
else:
return render_template('hepdata_records/data_record.html', ctx=ctx)
except Exception:
return render_template('hepdata_theme/404.html')
def test_load_file(app, migrator):
with app.app_context():
success = migrator.load_file('ins1487726')
assert (success)
hepsubmission = get_latest_hepsubmission(inspire_id='1487726')
assert (hepsubmission is not None)
assert (hepsubmission.inspire_id == '1487726')
index_records = get_records_matching_field('inspire_id',
hepsubmission.inspire_id,
doc_type='publication')
assert (len(index_records['hits']['hits']) == 1)
publication_record = get_record_contents(
hepsubmission.publication_recid)
assert (publication_record is not None)
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepsubmission.publication_recid).count()
assert (data_submissions == 5)
def download_datatable(datasubmission, file_format, *args, **kwargs):
"""
Download a particular data table given a ``datasubmission``.
:param datasubmission:
:param file_format:
:param args:
:param kwargs:
:return: display_error or send_file depending on success of conversion
"""
if file_format == 'json':
return redirect('/record/data/{0}/{1}/{2}'.format(
datasubmission.publication_recid, datasubmission.id,
datasubmission.version))
elif file_format not in CFG_SUPPORTED_FORMATS:
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
dataresource = DataResource.query.filter_by(
id=datasubmission.data_file).one()
record_path, table_name = os.path.split(dataresource.file_location)
filename = 'HEPData-{0}-v{1}'.format(kwargs.pop('submission_id'),
datasubmission.version)
if 'table_name' in kwargs:
filename += '-' + kwargs.pop('table_name').replace(' ', '_').replace(
'/', '_').replace('$', '').replace('\\', '')
output_path = os.path.join(current_app.config['CFG_TMPDIR'], filename)
if file_format == 'yaml':
return send_file(dataresource.file_location,
as_attachment=True,
attachment_filename=filename + '.yaml')
options = {
'input_format': 'yaml',
'output_format': file_format,
'table': table_name,
'filename': table_name.split('.')[0],
'validator_schema_version': '0.1.0',
}
hepsubmission = HEPSubmission.query.filter_by(
publication_recid=datasubmission.publication_recid,
version=datasubmission.version).first()
if datasubmission.doi and hepsubmission.overall_status != 'sandbox':
options['hepdata_doi'] = datasubmission.doi.rsplit(
'/', 1)[0].encode('ascii')
if file_format == 'yoda':
rivet_analysis_name = kwargs.pop('rivet_analysis_name', '')
if rivet_analysis_name:
options['rivet_analysis_name'] = rivet_analysis_name
elif datasubmission.publication_inspire_id:
record = get_record_contents(datasubmission.publication_recid)
if record:
# Check if this record has a Rivet analysis, then extract the Rivet analysis name from the URL.
if 'analyses' in record:
for analysis in record['analyses']:
if analysis['type'] == 'rivet':
options['rivet_analysis_name'] = analysis[
'analysis'].split('/')[-1]
# Otherwise guess the Rivet analysis name using the collaboration name,
# the creation year of the INSPIRE record, and the INSPIRE ID.
if 'rivet_analysis_name' not in options:
try:
year = parse(record['creation_date']).year
except:
year = record['year'] # publication year
options['rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), year,
datasubmission.publication_inspire_id)
successful = convert(
CFG_CONVERTER_URL,
record_path,
output=output_path + '-dir',
options=options,
extract=False,
)
if successful:
new_path = output_path + "." + file_format
new_path = extract(output_path + '-dir', new_path)
os.remove(output_path + '-dir')
file_to_send = get_file_in_directory(new_path, file_format)
else:
# Error occurred, the output is a HTML file
file_to_send = output_path + '-dir'
file_format = 'html'
return send_file(file_to_send,
as_attachment=True,
attachment_filename=filename + '.' + file_format)
def download_submission(submission,
file_format,
offline=False,
force=False,
rivet_analysis_name=''):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param submission: HEPSubmission
:param file_format: yaml, csv, root, or yoda
:param offline: offline creation of the conversion when a record is finalised
:param force: force recreation of the conversion
:param rivet_analysis_name: Rivet analysis name to override default written in YODA export
:return: display_error or send_file depending on success of conversion
"""
version = submission.version
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
if file_format == 'json':
return redirect('/record/{0}?version={1}&format=json'.format(
file_identifier, version))
elif file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
path = os.path.join(current_app.config['CFG_DATADIR'],
str(submission.publication_recid))
data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
submission.publication_recid, version)
output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier,
submission.version,
file_format)
converted_dir = os.path.join(current_app.config['CFG_DATADIR'],
'converted')
if not os.path.exists(converted_dir):
os.mkdir(converted_dir)
if file_format == 'yoda' and rivet_analysis_name:
# Don't store in converted_dir since rivet_analysis_name might possibly change between calls.
output_path = os.path.join(current_app.config['CFG_TMPDIR'],
output_file)
else:
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file or the submission is not finished.
if os.path.exists(
output_path
) and not force and submission.overall_status == 'finished':
if not offline:
return send_file(output_path, as_attachment=True)
else:
print('File already downloaded at {0}'.format(output_path))
return
converter_options = {
'input_format':
'yaml',
'output_format':
file_format,
'filename':
'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version,
file_format),
'validator_schema_version':
'0.1.0',
}
if submission.doi and submission.overall_status != 'sandbox':
converter_options['hepdata_doi'] = '{0}.v{1}'.format(
submission.doi, version)
if file_format == 'yoda':
if rivet_analysis_name:
converter_options['rivet_analysis_name'] = rivet_analysis_name
elif submission.inspire_id:
record = get_record_contents(submission.publication_recid)
if record:
# Check if this record has a Rivet analysis, then extract the Rivet analysis name from the URL.
if 'analyses' in record:
for analysis in record['analyses']:
if analysis['type'] == 'rivet':
converter_options[
'rivet_analysis_name'] = analysis[
'analysis'].split('/')[-1]
# Otherwise guess the Rivet analysis name using the collaboration name,
# the creation year of the INSPIRE record, and the INSPIRE ID.
if 'rivet_analysis_name' not in converter_options:
try:
year = parse(record['creation_date']).year
except:
year = record['year'] # publication year
converter_options[
'rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), year,
submission.inspire_id)
data_filepath = os.path.join(path, data_filename)
converted_file = convert_zip_archive(data_filepath, output_path,
converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} created successfully at {1}'.format(
file_identifier, output_path))
def download_submission(submission, file_format, offline=False, force=False):
"""
Gets the submission file and either serves it back directly from YAML, or converts it
for other formats.
:param submission: HEPSubmission
:param file_format: yaml, csv, root, or yoda
:return:
"""
version = submission.version
file_identifier = submission.publication_recid
if submission.inspire_id:
file_identifier = 'ins{0}'.format(submission.inspire_id)
if file_format == 'json':
return redirect('/record/{0}?version={1}&format=json'.format(
file_identifier, version))
elif file_format not in CFG_SUPPORTED_FORMATS:
if offline:
log.error('Format not supported')
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
path = os.path.join(current_app.config['CFG_DATADIR'],
str(submission.publication_recid))
data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
submission.publication_recid, version)
output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier,
submission.version,
file_format)
converted_dir = os.path.join(current_app.config['CFG_DATADIR'],
'converted')
if not os.path.exists(converted_dir):
os.mkdir(converted_dir)
output_path = os.path.join(converted_dir, output_file)
# If the file is already available in the dir, send it back
# unless we are forcing recreation of the file.
if os.path.exists(output_path) and not force:
if not offline:
return send_file(
output_path,
as_attachment=True,
)
else:
print('File already downloaded at {0}'.format(output_path))
return
converter_options = {
'input_format':
'yaml',
'output_format':
file_format,
'filename':
'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version,
file_format),
}
if submission.doi:
converter_options['hepdata_doi'] = '{0}.v{1}'.format(
submission.doi, version)
if submission.inspire_id and file_format == 'yoda':
record = get_record_contents(submission.publication_recid)
if record:
converter_options['rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), record['year'],
submission.inspire_id)
data_filepath = os.path.join(path, data_filename)
converted_file = convert_zip_archive(data_filepath, output_path,
converter_options)
if not offline:
return send_file(converted_file, as_attachment=True)
else:
print('File for {0} created successfully at {1}'.format(
file_identifier, output_path))
def render_record(recid, record, version, output_format, light_mode=False):
# Count number of all versions and number of finished versions of a publication record.
version_count_all = HEPSubmission.query.filter(
HEPSubmission.publication_recid == recid,
HEPSubmission.overall_status != 'sandbox').count()
version_count_finished = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status='finished').count()
# Number of versions that a user is allowed to access based on their permissions.
version_count = version_count_all if user_allowed_to_perform_action(
recid) else version_count_finished
# If version not given explicitly, take to be latest allowed version (or 1 if there are no allowed versions).
if version == -1:
version = version_count if version_count else 1
# Check for a user trying to access a version of a publication record where they don't have permissions.
if version_count < version_count_all and version == version_count_all:
# Prompt the user to login if they are not authenticated then redirect, otherwise return a 403 error.
if not current_user.is_authenticated:
redirect_url_after_login = '******'.format(
recid, version, output_format)
if 'table' in request.args:
redirect_url_after_login += '%26table%3D{0}'.format(
request.args['table'])
return redirect(
'/login/?next={0}'.format(redirect_url_after_login))
else:
abort(403)
hepdata_submission = get_latest_hepsubmission(publication_recid=recid,
version=version)
if hepdata_submission is not None:
if hepdata_submission.overall_status != 'sandbox':
ctx = format_submission(recid, record, version, version_count,
hepdata_submission)
increment(recid)
if output_format == 'html':
return render_template(
'hepdata_records/publication_record.html', ctx=ctx)
elif 'table' not in request.args:
if output_format == 'json':
ctx = process_ctx(ctx, light_mode)
return jsonify(ctx)
elif output_format == 'yoda' and 'rivet' in request.args:
return redirect(
'/download/submission/{0}/{1}/{2}/{3}'.format(
recid, version, output_format,
request.args['rivet']))
else:
return redirect('/download/submission/{0}/{1}/{2}'.format(
recid, version, output_format))
else:
file_identifier = 'ins{}'.format(
hepdata_submission.inspire_id
) if hepdata_submission.inspire_id else recid
if output_format == 'yoda' and 'rivet' in request.args:
return redirect(
'/download/table/{0}/{1}/{2}/{3}/{4}'.format(
file_identifier, request.args['table'].replace(
'%', '%25').replace('\\', '%5C'), version,
output_format, request.args['rivet']))
else:
return redirect('/download/table/{0}/{1}/{2}/{3}'.format(
file_identifier, request.args['table'].replace(
'%', '%25').replace('\\', '%5C'), version,
output_format))
else:
abort(404)
elif record is not None: # this happens when we access an id of a data record
# in which case, we find the related publication, and
# make the front end focus on the relevant data table.
try:
publication_recid = int(record['related_publication'])
publication_record = get_record_contents(publication_recid)
hepdata_submission = get_latest_hepsubmission(
publication_recid=publication_recid)
ctx = format_submission(publication_recid,
publication_record,
hepdata_submission.version,
1,
hepdata_submission,
data_table=record['title'])
ctx['related_publication_id'] = publication_recid
ctx['table_name'] = record['title']
if output_format == 'html':
return render_template('hepdata_records/data_record.html',
ctx=ctx)
elif output_format == 'yoda' and 'rivet' in request.args:
return redirect('/download/table/{0}/{1}/{2}/{3}/{4}'.format(
publication_recid,
ctx['table_name'].replace('%', '%25').replace('\\', '%5C'),
hepdata_submission.version, output_format,
request.args['rivet']))
else:
return redirect('/download/table/{0}/{1}/{2}/{3}'.format(
publication_recid,
ctx['table_name'].replace('%', '%25').replace('\\', '%5C'),
hepdata_submission.version, output_format))
except Exception as e:
abort(404)
else:
abort(404)
def download_datatable(datasubmission, file_format, *args, **kwargs):
if file_format == 'json':
return redirect('/record/data/{0}/{1}/{2}'.format(
datasubmission.publication_recid, datasubmission.id,
datasubmission.version))
elif file_format not in CFG_SUPPORTED_FORMATS:
return display_error(
title="The " + file_format + " output format is not supported",
description="This output format is not supported. " +
"Currently supported formats: " + str(CFG_SUPPORTED_FORMATS),
)
dataresource = DataResource.query.filter_by(
id=datasubmission.data_file).one()
record_path, table_name = os.path.split(dataresource.file_location)
filename = 'HEPData-{0}-v{1}'.format(kwargs.pop('submission_id'),
datasubmission.version)
if 'table_name' in kwargs:
filename += '-' + kwargs.pop('table_name').replace(' ', '')
output_path = os.path.join(current_app.config['CFG_TMPDIR'], filename)
if file_format == 'yaml':
return send_file(dataresource.file_location,
as_attachment=True,
attachment_filename=filename + '.yaml')
options = {
'input_format': 'yaml',
'output_format': file_format,
'table': table_name,
'filename': table_name.split('.')[0],
}
if datasubmission.doi:
options['hepdata_doi'] = datasubmission.doi.rsplit(
'/', 1)[0].encode('ascii')
if datasubmission.publication_inspire_id and file_format == 'yoda':
record = get_record_contents(datasubmission.publication_recid)
if record:
options['rivet_analysis_name'] = '{0}_{1}_I{2}'.format(
''.join(record['collaborations']).upper(), record['year'],
datasubmission.publication_inspire_id)
if not os.path.exists(output_path):
successful = convert(
CFG_CONVERTER_URL,
record_path,
output=output_path + '-dir',
options=options,
extract=False,
)
else:
successful = True
# Error occurred, the output is a HTML file
if successful:
new_path = output_path + "." + file_format
new_path = extract(filename + ".tar.gz", output_path + '-dir',
new_path)
file_to_send = get_file_in_directory(new_path, file_format)
else:
file_to_send = output_path + '-dir'
file_format = 'html'
return send_file(file_to_send,
as_attachment=True,
attachment_filename=filename + '.' + file_format)
def test_create_submission(app, admin_idx):
"""
Test the whole submission pipeline in loading a file, ensuring the HEPSubmission object is created,
all the files have been added, and the record has been indexed.
:return:
"""
with app.app_context():
admin_idx.recreate_index()
# test submission part works
record = {
'inspire_id': '19999999',
'title': 'HEPData Testing 1',
'reviewer': {
'name': 'Testy McTester',
'email': '*****@*****.**'
},
'uploader': {
'name': 'Testy McTester',
'email': '*****@*****.**'
},
'message': 'This is ready',
'user_id': 1
}
hepdata_submission = process_submission_payload(**record)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
# test upload works
base_dir = os.path.dirname(os.path.realpath(__file__))
test_directory = os.path.join(base_dir, 'test_data/test_submission')
time_stamp = str(int(round(time.time())))
directory = get_data_path_for_record(
hepdata_submission.publication_recid, time_stamp)
shutil.copytree(test_directory, directory)
assert (os.path.exists(directory))
process_submission_directory(
directory, os.path.join(directory, 'submission.yaml'),
hepdata_submission.publication_recid)
admin_idx_results = admin_idx.search(
term=hepdata_submission.publication_recid, fields=['recid'])
assert (admin_idx_results is not None)
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepdata_submission.publication_recid).count()
assert (data_submissions == 8)
assert (len(hepdata_submission.resources) == 4)
assert (len(hepdata_submission.participants) == 4)
do_finalise(hepdata_submission.publication_recid,
force_finalise=True,
convert=False)
assert (record_exists(inspire_id=record['inspire_id']))
# Test record is in index...
index_records = get_records_matching_field('inspire_id',
record['inspire_id'],
doc_type='publication')
assert (len(index_records['hits']['hits']) == 1)
publication_record = get_record_contents(
hepdata_submission.publication_recid)
assert (publication_record is not None)
ctx = format_submission(hepdata_submission.publication_recid,
publication_record, hepdata_submission.version,
1, hepdata_submission)
assert (ctx is not None)
assert (ctx['version'] == 1)
assert (ctx['recid'] == hepdata_submission.publication_recid)
# remove the submission and test that all is remove
unload_submission(hepdata_submission.publication_recid)
assert (not record_exists(inspire_id=record['inspire_id']))
data_submissions = DataSubmission.query.filter_by(
publication_recid=hepdata_submission.publication_recid).count()
assert (data_submissions == 0)
sleep(2)
admin_idx_results = admin_idx.search(
term=hepdata_submission.publication_recid, fields=['recid'])
assert (len(admin_idx_results) == 0)
# Check file dir has been deleted
assert (not os.path.exists(directory))
def render_record(recid, record, version, output_format, light_mode=False):
if user_allowed_to_perform_action(recid):
version_count = HEPSubmission.query.filter_by(
publication_recid=recid).count()
else:
version_count = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status='finished').count()
if version == -1:
version = version_count
hepdata_submission = get_latest_hepsubmission(publication_recid=recid,
version=version)
if hepdata_submission is not None:
ctx = format_submission(recid, record, version, version_count,
hepdata_submission)
increment(recid)
if output_format == 'html':
return render_template('hepdata_records/publication_record.html',
ctx=ctx)
elif 'table' not in request.args:
if output_format == 'json':
ctx = process_ctx(ctx, light_mode)
return jsonify(ctx)
else:
return redirect('/download/submission/{0}/{1}/{2}'.format(
recid, version, output_format))
else:
file_identifier = 'ins{}'.format(
hepdata_submission.inspire_id
) if hepdata_submission.inspire_id else recid
return redirect('/download/table/{0}/{1}/{2}/{3}'.format(
file_identifier, request.args['table'], version,
output_format))
elif record is not None: # this happens when we access an id of a data record
# in which case, we find the related publication, and
# make the front end focus on the relevant data table.
try:
publication_recid = int(record['related_publication'])
publication_record = get_record_contents(publication_recid)
hepdata_submission = get_latest_hepsubmission(
publication_recid=publication_recid)
ctx = format_submission(publication_recid,
publication_record,
hepdata_submission.version,
1,
hepdata_submission,
data_table=record['title'])
ctx['related_publication_id'] = publication_recid
ctx['table_name'] = record['title']
if output_format == 'html':
return render_template('hepdata_records/data_record.html',
ctx=ctx)
else:
return redirect('/download/table/{0}/{1}/{2}/{3}'.format(
publication_recid, ctx['table_name'],
hepdata_submission.version, output_format))
except Exception as e:
raise e
return render_template('hepdata_theme/404.html')
else:
return render_template('hepdata_theme/404.html')
