def attach_information_to_record(recid):
"""
Given an INSPIRE data representation, this will process the data, and update information
for a given record id with the contents.
:return:
"""
inspire_id = request.form['inspire_id']
content, status = get_inspire_record_information(inspire_id)
content["inspire_id"] = inspire_id
record = get_record_by_id(recid)
if record is not None and status == 'success':
content['recid'] = recid
record.update(content)
record.commit()
hep_submission = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status="todo").first()
hep_submission.inspire_id = inspire_id
db.session.add(hep_submission)
db.session.commit()
return jsonify({'status': 'success'})
elif status != 'success':
return jsonify({'status': status,
'message': 'Request for INSPIRE record {} failed.'.format(inspire_id)})
else:
return jsonify({'status': 'failed',
'message': 'No record with recid {} was found.'.format(str(recid))})
def get_data_review_status():
data_id = request.args.get('data_recid')
record_sql = DataReview.query.filter_by(data_recid=data_id)
try:
record = record_sql.one()
return jsonify(
{"publication_recid": record.publication_recid,
"data_recid": record.data_recid, "status": record.status})
except:
return jsonify({"error": "no review found."})
def notify_coordinator(recid, version):
message = request.form['message']
try:
current_user_obj = get_user_from_id(current_user.get_id())
send_coordinator_notification_email(
recid, version, current_user_obj,
message=message
)
return jsonify({"status": "success"})
except Exception as e:
db.session.rollback()
return jsonify({"status": "error", "message": e.__str__()})
def get_latest():
"""
Returns the N latest records from the database.
:param n:
:return:
"""
n = int(request.args.get('n', 3))
latest_records = get_n_latest_records(n)
result = {"latest": []}
for record in latest_records:
record_information = record['_source']
if 'recid' in record_information:
last_updated = record_information['creation_date']
if "last_updated" in record_information:
last_updated = record_information["last_updated"]
last_updated = parser.parse(last_updated).strftime("%Y-%m-%d")
extract_journal_info(record_information)
record_information['author_count'] = len(record_information.get('summary_authors', []))
record_information['last_updated'] = last_updated
result['latest'].append(record_information)
return jsonify(result)
def get_count_stats():
pub_count = get_count_for_collection(CFG_PUB_TYPE)
data_count = get_count_for_collection(CFG_DATA_TYPE)
return jsonify({
"data": data_count['count'],
"publications": pub_count["count"]
})
def set_data_review_status():
recid = int(request.form['publication_recid'])
status = request.form['status']
version = int(request.form['version'])
all_tables = request.form.get('all_tables')
if user_allowed_to_perform_action(recid):
if all_tables:
data_ids = db.session.query(DataSubmission.id) \
.filter_by(publication_recid=recid, version=version).distinct()
else:
data_ids = [int(request.form['data_recid'])]
for data_id in data_ids:
record_sql = DataReview.query.filter_by(data_recid=data_id,
version=version)
record = record_sql.first()
if not record:
record = create_data_review(data_id, recid, version)
record_sql.update({"status": status}, synchronize_session='fetch')
try:
db.session.commit()
success = True
except Exception:
db.session.rollback()
success = False
if all_tables:
return jsonify({"recid": recid, "success": success})
else:
return jsonify({
"recid": record.publication_recid,
"data_id": record.data_recid,
"status": record.status
})
return jsonify({
"recid":
recid,
'message':
'You are not authorised to update the review status for '
'this data record.'
})
def submit_question(recid):
question = request.form['question']
try:
question = Question(user=int(current_user.get_id()), publication_recid=recid, question=str(question))
db.session.add(question)
db.session.commit()
send_question_email(question)
except Exception as e:
log.error(e)
db.session.rollback()
return jsonify({'status': 'queued', 'message': 'Your question has been posted.'})
def notify_participants(recid, version):
message = request.form['message']
show_detail = request.form.get('show_detail', 'false').lower() == 'true'
submission = HEPSubmission.query.filter_by(publication_recid=recid, version=version).first()
try:
current_user_obj = get_user_from_id(current_user.get_id())
send_notification_email(
recid, version, current_user_obj, submission.reviewers_notified,
message=message, show_detail=show_detail
)
submission.reviewers_notified = True
db.session.add(submission)
db.session.commit()
return jsonify({"status": "success"})
except NoParticipantsException:
return jsonify({"status": "error", "message": "There are no uploaders or reviewers for this submission."})
except Exception as e:
db.session.rollback()
return jsonify({"status": "error", "message": e.__str__()})
def notify_reviewers(recid, version):
message = request.form['message']
submission = HEPSubmission.query.filter_by(publication_recid=recid,
version=version).first()
try:
current_user_obj = get_user_from_id(current_user.get_id())
send_new_upload_email(recid, current_user_obj, message=message)
submission.reviewers_notified = True
db.session.add(submission)
db.session.commit()
return jsonify({"status": "success"})
except NoReviewersException:
return jsonify({
"status": "error",
"message": "There are no reviewers for this submission."
})
except Exception as e:
db.session.rollback()
return jsonify({"status": "error", "message": e.__str__()})
def revise_submission(recid):
"""
This method creates a new version of a submission.
:param recid: record id to attach the data to
:return: For POST requests, returns JSONResponse either containing 'url'
(for success cases) or 'message' (for error cases, which will
give a 400 error). For GET requests, redirects to the record.
"""
if not has_coordinator_permissions(recid, current_user):
return jsonify({"message": "Current user is not a coordinator for this record."}), 403
notify_uploader = request.values['notify-uploader'] == 'true'
uploader_message = request.values['notify-uploader-message']
return create_new_version(recid, current_user, notify_uploader, uploader_message)
def get_resource(resource_id):
"""
Attempts to find any HTML resources to be displayed for a record in the event that it
does not have proper data records included.
:param recid: publication record id
:return: json dictionary containing any HTML files to show.
"""
resource = DataResource.query.filter_by(id=resource_id)
view_mode = bool(request.args.get('view', False))
if resource.count() > 0:
resource_obj = resource.first()
if view_mode:
return send_file(resource_obj.file_location, as_attachment=True)
elif 'html' in resource_obj.file_location and 'http' not in resource_obj.file_location.lower(
):
with open(resource_obj.file_location, 'r') as resource_file:
html = resource_file.read()
return html
else:
contents = ''
if resource_obj.file_type.lower() not in IMAGE_TYPES:
print("Resource is at: " + resource_obj.file_location)
with open(resource_obj.file_location, 'r') as resource_file:
contents = resource_file.read()
# Don't return file contents if they contain a null byte.
if '\0' in contents:
contents = 'Binary'
return jsonify({
"location":
'/record/resource/{0}?view=true'.format(resource_obj.id),
'type':
resource_obj.file_type,
'description':
resource_obj.file_description,
'file_contents':
decode_string(contents)
})
else:
log.error("Unable to find resource %d.", resource_id)
return abort(404)
def update_sandbox_payload(recid):
"""
Updates the Sandbox submission with a new file upload.
:param recid:
"""
if request.method == 'GET':
return redirect('/record/sandbox/' + str(recid))
if not has_upload_permissions(recid, current_user, is_sandbox=True):
return jsonify({
"message": "Current user does not correspond to a confirmed uploader for this record."
}), 403
file = request.files['hep_archive']
redirect_url = request.url_root + "record/sandbox/{}"
return process_payload(recid, file, redirect_url)
def get_data_reviews_for_record():
"""
Get the data reviews for a record.
:return: json response with reviews (or a json with an error key if not)
"""
recid = int(request.args.get('publication_recid'))
record_sql = DataReview.query.filter_by(publication_recid=recid)
if user_allowed_to_perform_action(recid):
try:
records = record_sql.all()
record_result = []
for record in records:
record_result.append(
{"data_recid": record.data_recid, "status": record.status,
"last_updated": record.modification_date})
return json.dumps(record_result, default=default_time)
except:
return jsonify({"error": "no reviews found"})
def consume_data_payload(recid):
"""
This method persists, then presents the loaded data back to the user.
:param recid: record id to attach the data to
:return: For POST requests, returns JSONResponse either containing 'url'
(for success cases) or 'message' (for error cases, which will
give a 400 error). For GET requests, redirects to the record.
"""
if request.method == 'POST':
if not has_upload_permissions(recid, current_user):
return jsonify({
"message": "Current user does not correspond to a confirmed uploader for this record."
}), 403
file = request.files['hep_archive']
redirect_url = request.url_root + "record/{}"
return process_payload(recid, file, redirect_url)
else:
return redirect('/record/' + str(recid))
def cli_upload():
"""
Used by the hepdata-cli tool to upload a submission.
:return:
"""
if request.method == 'GET':
return redirect('/')
# email must be provided
if 'email' not in request.form.keys():
return jsonify({"message": "User email is required: specify one using -e user-email."}), 400
user_email = request.form['email']
# password must be provided
if 'pswd' not in request.form.keys():
return jsonify({"message": "User password is required."}), 400
user_pswd = request.form['pswd']
# check user associated with this email exists and is active with a confirmed email address
user = User.query.filter(func.lower(User.email) == user_email.lower(), User.active.is_(True), User.confirmed_at.isnot(None)).first()
if user is None:
return jsonify({"message": "Email {} does not match an active confirmed user.".format(user_email)}), 404
elif user.password is None:
return jsonify({"message": "Set HEPData password from {} first.".format(SITE_URL + '/lost-password/')}), 403
elif verify_password(user_pswd, user.password) is False:
return jsonify({"message": "Wrong password, please try again."}), 403
else:
login_user(user)
# sandbox must be provided
if 'sandbox' not in request.form.keys():
return jsonify({"message": "sandbox (True or False) is required."}), 400
str_sandbox = request.form['sandbox']
is_sandbox = False if str_sandbox == 'False' else True if str_sandbox == 'True' else None
recid = request.form['recid'] if 'recid' in request.form.keys() else None
invitation_cookie = request.form['invitation_cookie'] if 'invitation_cookie' in request.form.keys() else None
# Check the user has upload permissions for this record
if not has_upload_permissions(recid, user, is_sandbox):
return jsonify({
"message": "Email {} does not correspond to a confirmed uploader for this record.".format(str(user_email))
}), 403
if is_sandbox is True:
if recid is None:
return consume_sandbox_payload() # '/sandbox/consume'
else:
# check that sandbox record exists and belongs to this user
hepsubmission_record = get_latest_hepsubmission(publication_recid=recid, overall_status='sandbox')
if hepsubmission_record is None:
return jsonify({"message": "Sandbox record {} not found.".format(str(recid))}), 404
else:
return update_sandbox_payload(recid) # '/sandbox/<int:recid>/consume'
elif is_sandbox is False:
# check that record exists and has 'todo' status
hepsubmission_record = get_latest_hepsubmission(publication_recid=recid, overall_status='todo')
if hepsubmission_record is None:
return jsonify({"message": "Record {} not found.".format(str(recid))}), 404
# check user is allowed to upload to this record and supplies the correct invitation cookie
participant = SubmissionParticipant.query.filter_by(user_account=user.id, role='uploader', publication_recid=recid, status='primary').first()
if participant and str(participant.invitation_cookie) != invitation_cookie:
return jsonify({"message": "Invitation cookie did not match."}), 403
return consume_data_payload(recid) # '/<int:recid>/consume'
def get_table_details(recid, data_recid, version):
"""
Get the table details.
:param recid:
:param data_recid:
:param version:
:return:
"""
datasub_query = DataSubmission.query.filter_by(id=data_recid,
version=version)
table_contents = {}
if datasub_query.count() > 0:
datasub_record = datasub_query.one()
data_query = db.session.query(DataResource).filter(
DataResource.id == datasub_record.data_file)
if data_query.count() > 0:
data_record = data_query.one()
file_location = data_record.file_location
attempts = 0
while True:
try:
with open(file_location, 'r') as table_file:
table_contents = yaml.load(table_file, Loader=Loader)
except:
attempts += 1
# allow multiple attempts to read file in case of temporary disk problems
if (table_contents and table_contents is not None) or attempts > 5:
break
table_contents["name"] = datasub_record.name
table_contents["title"] = datasub_record.description
table_contents["keywords"] = datasub_record.keywords
table_contents["doi"] = datasub_record.doi
table_contents["location"] = datasub_record.location_in_publication
# we create a map of files mainly to accommodate the use of thumbnails for images where possible.
tmp_assoc_files = {}
for associated_data_file in datasub_record.resources:
alt_location = associated_data_file.file_location
location_parts = alt_location.split('/')
key = location_parts[-1].replace("thumb_", "")
if key not in tmp_assoc_files:
tmp_assoc_files[key] = {}
if "thumb_" in alt_location and associated_data_file.file_type.lower() in IMAGE_TYPES:
tmp_assoc_files[key]['preview_location'] = '/record/resource/{0}?view=true'.format(
associated_data_file.id)
else:
tmp_assoc_files[key].update({'description': associated_data_file.file_description,
'type': associated_data_file.file_type,
'id': associated_data_file.id,
'alt_location': alt_location})
# add associated files to the table contents
table_contents['associated_files'] = list(tmp_assoc_files.values())
table_contents["review"] = {}
data_review_record = create_data_review(data_recid, recid, version)
table_contents["review"]["review_flag"] = data_review_record.status if data_review_record else "todo"
table_contents["review"]["messages"] = len(data_review_record.messages) > 0 if data_review_record else False
# translate the table_contents to an easy to render format of the qualifiers (with colspan),
# x and y headers (should not require a colspan)
# values, that also encompass the errors
return jsonify(generate_table_structure(table_contents))