def test_record_update(app):
"""___test_record_update___"""
with app.app_context():
record_information = create_record({'journal_info': 'Phys. Letts', 'title': 'My Journal Paper'})
record = get_record_by_id(record_information['recid'])
assert (record['title'] == 'My Journal Paper')
assert (record['journal_info'] == 'Phys. Letts')
update_record(record_information['recid'], {'journal_info': 'test'})
updated_record = get_record_by_id(record_information['recid'])
assert (updated_record['journal_info'] == 'test')
def test_record_update(app):
"""___test_record_update___"""
with app.app_context():
record_information = create_record({'journal_info': 'Phys. Letts', 'title': 'My Journal Paper'})
record = get_record_by_id(record_information['recid'])
assert (record['title'] == 'My Journal Paper')
assert (record['journal_info'] == 'Phys. Letts')
update_record(record_information['recid'], {'journal_info': 'test'})
updated_record = get_record_by_id(record_information['recid'])
assert (updated_record['journal_info'] == 'test')
def attach_information_to_record(recid):
"""
Given an INSPIRE data representation, this will process the data, and update information
for a given record id with the contents.
:return:
"""
inspire_id = request.form['inspire_id']
content, status = get_inspire_record_information(inspire_id)
content["inspire_id"] = inspire_id
record = get_record_by_id(recid)
if record is not None:
content['recid'] = recid
patch = jsonpatch.JsonPatch.from_diff(record, content)
record = record.patch(patch=patch)
record.commit()
db.session.commit()
return jsonify({'status': 'success'})
else:
return jsonify({'status': 'failed',
'message': 'No record with that recid was found.'})
def generate_doi_for_submission(recid, version):
data_submissions = DataSubmission.query.filter_by(publication_recid=recid, version=version).order_by(
DataSubmission.id.asc())
hep_submission = HEPSubmission.query.filter_by(publication_recid=recid).first()
publication_info = get_record_by_id(recid)
version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version)
hep_submission.data_abstract = decode_string(hep_submission.data_abstract)
publication_info['title'] = decode_string(publication_info['title'])
publication_info['abstract'] = decode_string(publication_info['abstract'])
base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml',
doi=hep_submission.doi,
overall_submission=hep_submission,
data_submissions=data_submissions,
publication_info=publication_info)
version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml',
doi=version_doi,
overall_submission=hep_submission,
data_submissions=data_submissions,
publication_info=publication_info)
# Register DOI for the version, and update the base DOI to resolve to the latest submission version.
register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']),
base_xml, publication_info['uuid'])
register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format(
publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def promote_or_demote_participant(recid, action, demote_or_promote,
participant_id):
"""
Can promote or demote a participant to/from primary reviewer/uploader.
:param recid: record id that the user will be promoted or demoted for
:param action: upload or review
:param demote_or_promote: demote or promote
:param participant_id: id of user from the SubmissionParticipant table.
:return:
"""
try:
participant = SubmissionParticipant.query.filter_by(
id=participant_id).one()
status = 'reserve'
if demote_or_promote == 'promote':
status = 'primary'
participant.status = status
db.session.add(participant)
db.session.commit()
record = get_record_by_id(recid)
# now send the email telling the user of their new status!
if status == 'primary':
send_cookie_email(participant, record)
return json.dumps({"success": True, "recid": recid})
except Exception as e:
return json.dumps({"success": False, "recid": recid, "error": str(e)})
def generate_dois_for_submission(*args, **kwargs):
"""
Generate DOIs for all the submission components.
:param args:
:param kwargs:
:return:
"""
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
hep_submissions = HEPSubmission.query.filter_by(**kwargs).order_by(
HEPSubmission.publication_recid.asc()).all()
for hep_submission in hep_submissions:
if args:
start_recid, end_recid = args
if hep_submission.publication_recid < start_recid or hep_submission.publication_recid > end_recid:
continue
if hep_submission.overall_status != 'finished':
continue
data_submissions = DataSubmission.query.filter_by(
publication_inspire_id=hep_submission.inspire_id,
version=hep_submission.version).order_by(DataSubmission.id.asc())
publication_info = get_record_by_id(hep_submission.publication_recid)
create_container_doi(hep_submission, data_submissions,
publication_info, site_url)
for data_submission in data_submissions:
create_data_doi(hep_submission, data_submission, publication_info,
site_url)
def create_new_version(recid, user, notify_uploader=True, uploader_message=None):
hepsubmission = get_latest_hepsubmission(publication_recid=recid)
if hepsubmission.overall_status == 'finished':
# Reopen the submission to allow for revisions,
# by creating a new HEPSubmission object.
_rev_hepsubmission = HEPSubmission(publication_recid=recid,
overall_status='todo',
inspire_id=hepsubmission.inspire_id,
coordinator=hepsubmission.coordinator,
version=hepsubmission.version + 1)
db.session.add(_rev_hepsubmission)
db.session.commit()
if notify_uploader:
uploaders = SubmissionParticipant.query.filter_by(
role='uploader', publication_recid=recid, status='primary'
)
record_information = get_record_by_id(recid)
for uploader in uploaders:
send_cookie_email(uploader,
record_information,
message=uploader_message,
version=_rev_hepsubmission.version)
return jsonify({'success': True, 'version': _rev_hepsubmission.version})
else:
return jsonify({"message": f"Rec id {recid} is not finished so cannot create a new version"}), 400
def finalise_datasubmission(current_time, existing_submissions,
generated_record_ids, publication_record, recid,
submission, version):
# we now create a 'payload' for each data submission
# by creating a record json and uploading it via a bibupload task.
# add in key for associated publication...
keywords = []
for keyword in submission.keywords:
keywords.append({
"name": keyword.name,
"value": keyword.value,
"synonyms": ""
})
# we want to retrieve back the authors of the paper
# and assign them as authors of the data too
if not publication_record:
publication_record = get_record_by_id(recid)
submission_info = {
"title": submission.name,
"abstract": submission.description,
"inspire_id": publication_record['inspire_id'],
"doi": submission.doi,
"authors": publication_record['authors'],
"first_author": publication_record.get('first_author', None),
"related_publication": submission.publication_recid,
"creation_date": publication_record["creation_date"],
"last_updated": current_time,
"journal_info": publication_record.get("journal_info", ""),
"keywords": keywords,
"version": version,
"collaborations": publication_record.get("collaborations", []),
}
if submission_info["title"] in existing_submissions:
# in the event that we're performing an update operation, we need
# to get the data record information
# from the index, and use the same record id. This way, we'll just
# update the submission instead of recreating
# a completely new record.
recid = existing_submissions[submission_info["title"]]
submission_info["control_number"] = submission_info["recid"] = recid
else:
submission_info = create_record(submission_info)
submission_info["control_number"] = submission_info["recid"]
submission.associated_recid = submission_info['recid']
submission.publication_inspire_id = publication_record['inspire_id']
generated_record_ids.append(submission_info["recid"])
submission.version = version
data_review = DataReview.query.filter_by(data_recid=submission.id).first()
if data_review:
data_review.version = version
db.session.add(data_review)
db.session.add(submission)
def promote_or_demote_participant(recid, action, demote_or_promote,
participant_id):
"""
Can promote or demote a participant to/from primary reviewer/uploader
:param recid: record id that the user will be promoted or demoted
:param action: upload or review
:param demote_or_promote: demote or promote
:param participant_id: id of user from the SubmissionParticipant table.
:return:
"""
try:
participant = SubmissionParticipant.query.filter_by(
id=participant_id).one()
status = 'reserve'
if demote_or_promote == 'promote':
status = 'primary'
participant.status = status
db.session.add(participant)
db.session.commit()
record = get_record_by_id(recid)
# now send the email telling the user of their new status!
if status == 'primary':
send_cookie_email(participant, record)
return json.dumps({"success": True, "recid": recid})
except Exception as e:
return json.dumps(
{"success": False, "recid": recid, "error": e.message})
def generate_doi_for_data_submission(data_submission_id, version):
data_submission = DataSubmission.query.filter_by(id=data_submission_id).first()
hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first()
publication_info = get_record_by_id(data_submission.publication_recid)
data_file = DataResource.query.filter_by(id=data_submission.data_file).first()
license = None
if data_file:
if data_file.file_license:
license = License.query.filter_by(id=data_file.file_license).first()
xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml',
doi=data_submission.doi,
table_name=decode_string(data_submission.name),
table_description=decode_string(data_submission.description),
overall_submission=hep_submission,
data_submission=data_submission,
license=license,
publication_info=publication_info)
register_doi(data_submission.doi, 'http://www.hepdata.net/record/ins{0}?version={1}&table={2}'.format(
publication_info['inspire_id'], data_submission.version, data_submission.name),
xml, publication_info['uuid'])
def generate_doi_for_data_submission(data_submission_id, version):
data_submission = DataSubmission.query.filter_by(id=data_submission_id).first()
hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first()
publication_info = get_record_by_id(data_submission.publication_recid)
data_file = DataResource.query.filter_by(id=data_submission.data_file).first()
license = None
if data_file:
if data_file.file_license:
license = License.query.filter_by(id=data_file.file_license).first()
xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml',
doi=data_submission.doi,
table_name=decode_string(data_submission.name),
table_description=decode_string(data_submission.description),
overall_submission=hep_submission,
data_submission=data_submission,
license=license,
publication_info=publication_info)
register_doi(data_submission.doi, 'http://www.hepdata.net/record/{0}?version={1}&table={2}'.format(
hep_submission.publication_recid, data_submission.version, data_submission.name),
xml, publication_info['uuid'])
def get_pending_invitations_for_user(user):
"""
Returns pending invites for upload or review of records.
:param user: User object
:return: array of pending invites
"""
pending_invites = SubmissionParticipant.query.filter(
func.lower(SubmissionParticipant.email) == func.lower(user.email),
or_(SubmissionParticipant.role == 'reviewer',
SubmissionParticipant.role == 'uploader'),
SubmissionParticipant.status == 'primary',
SubmissionParticipant.user_account == None).all()
result = []
for invite in pending_invites:
publication_record = get_record_by_id(invite.publication_recid)
hepsubmission = get_latest_hepsubmission(
publication_recid=invite.publication_recid)
coordinator = get_user_from_id(hepsubmission.coordinator)
result.append({
'title': decode_string(publication_record['title']),
'invitation_cookie': invite.invitation_cookie,
'role': invite.role,
'coordinator': coordinator
})
return result
def generate_dois_for_submission(*args, **kwargs):
"""
Generate DOIs for all the submission components
:param inspire_id:
:param version:
:param kwargs:
:return:
"""
hep_submissions = HEPSubmission.query.filter_by(**kwargs).all()
for hep_submission in hep_submissions:
if hep_submission.overall_status != 'finished':
continue
data_submissions = DataSubmission.query.filter_by(publication_inspire_id=hep_submission.inspire_id,
version=hep_submission.version).order_by(
DataSubmission.id.asc())
publication_info = get_record_by_id(hep_submission.publication_recid)
create_container_doi(hep_submission, data_submissions, publication_info)
for data_submission in data_submissions:
create_data_doi(hep_submission, data_submission, publication_info)
def generate_doi_for_submission(recid, version):
data_submissions = DataSubmission.query.filter_by(publication_recid=recid, version=version).order_by(
DataSubmission.id.asc())
hep_submission = HEPSubmission.query.filter_by(publication_recid=recid).first()
publication_info = get_record_by_id(recid)
version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version)
hep_submission.data_abstract = decode_string(hep_submission.data_abstract)
publication_info['title'] = decode_string(publication_info['title'])
publication_info['abstract'] = decode_string(publication_info['abstract'])
base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml',
doi=hep_submission.doi,
overall_submission=hep_submission,
data_submissions=data_submissions,
publication_info=publication_info)
version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml',
doi=version_doi,
overall_submission=hep_submission,
data_submissions=data_submissions,
publication_info=publication_info)
# Register DOI for the version, and update the base DOI to resolve to the latest submission version.
register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']),
base_xml, publication_info['uuid'])
register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format(
publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def attach_information_to_record(recid):
"""
Given an INSPIRE data representation, this will process the data, and update information
for a given record id with the contents.
:return:
"""
inspire_id = request.form['inspire_id']
content, status = get_inspire_record_information(inspire_id)
content["inspire_id"] = inspire_id
record = get_record_by_id(recid)
if record is not None and status == 'success':
content['recid'] = recid
record.update(content)
record.commit()
hep_submission = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status="todo").first()
hep_submission.inspire_id = inspire_id
db.session.add(hep_submission)
db.session.commit()
return jsonify({'status': 'success'})
elif status != 'success':
return jsonify({'status': status,
'message': 'Request for INSPIRE record {} failed.'.format(inspire_id)})
else:
return jsonify({'status': 'failed',
'message': 'No record with recid {} was found.'.format(str(recid))})
def send_new_review_message_email(review, message, user):
"""
Sends a message to all uploaders to tell them that a
comment has been made on a record
:param hepsubmission:
:param review:
:param user:
:return:
"""
submission_participants = SubmissionParticipant.query.filter_by(
publication_recid=review.publication_recid,
status="primary", role="uploader")
table_information = DataSubmission.query \
.filter_by(id=review.data_recid).one()
record = get_record_by_id(review.publication_recid)
for participant in submission_participants:
message_body = render_template(
'hepdata_theme/email/review-message.html',
name=participant.full_name,
actor=user.email,
table_name=table_information.name,
table_message=message.message,
article=review.publication_recid,
title=record['title'],
link="http://hepdata.net/record/{0}"
.format(review.publication_recid))
create_send_email_task(
participant.email,
'[HEPData] Submission {0} has a new review message.' \
.format(review.publication_recid), message_body)
def generate_doi_for_table(doi):
"""
Generate DOI for a specific table given by its doi.
:param doi:
:return:
"""
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
try:
data_submission = DataSubmission.query.filter_by(doi=doi).one()
except NoResultFound:
print('Table DOI {} not found in database'.format(doi))
return
hep_submission = HEPSubmission.query.filter_by(
inspire_id=data_submission.publication_inspire_id,
version=data_submission.version,
overall_status='finished').one()
if hep_submission:
publication_info = get_record_by_id(hep_submission.publication_recid)
create_data_doi(hep_submission, data_submission, publication_info,
site_url)
else:
print(
'Finished submission with INSPIRE ID {} and version {} not found in database'
.format(data_submission.publication_inspire_id,
data_submission.version))
def create_record_for_dashboard(record_id,
submissions,
current_user,
coordinator=None,
user_role=None,
status="todo"):
if user_role is None:
user_role = ["coordinator"]
publication_record = get_record_by_id(int(record_id))
if publication_record is not None:
if record_id not in submissions:
hepdata_submission_record = get_latest_hepsubmission(
publication_recid=record_id)
submissions[record_id] = {}
submissions[record_id]["metadata"] = {
"recid": record_id,
"role": user_role,
"start_date": publication_record.created
}
submissions[record_id]["metadata"][
"versions"] = hepdata_submission_record.version
submissions[record_id]["status"] = status
submissions[record_id]["stats"] = {
"passed": 0,
"attention": 0,
"todo": 0
}
if coordinator:
submissions[record_id]["metadata"]["coordinator"] = {
'id': coordinator.id,
'name': coordinator.email,
'email': coordinator.email
}
submissions[record_id]["metadata"]["show_coord_view"] = int(
current_user.get_id()) == coordinator.id
else:
submissions[record_id]["metadata"]["coordinator"] = {
'name': 'No coordinator'
}
if "title" in publication_record:
submissions[record_id]["metadata"]["title"] = \
publication_record['title']
if "inspire_id" not in publication_record or publication_record[
"inspire_id"] is None:
submissions[record_id]["metadata"][
"requires_inspire_id"] = True
else:
# if it is, it's because the user has two roles for that
# submission. So we should show them!
if user_role not in submissions[record_id]["metadata"]["role"]:
submissions[record_id]["metadata"]["role"].append(user_role)
def send_finalised_email(hepsubmission):
submission_participants = get_submission_participants_for_record(
hepsubmission.publication_recid)
record = get_record_by_id(hepsubmission.publication_recid)
notify_participants(hepsubmission, record, submission_participants)
notify_subscribers(hepsubmission, record)
def finalise_datasubmission(current_time, existing_submissions,
generated_record_ids, publication_record, recid,
submission, version):
# we now create a 'payload' for each data submission
# by creating a record json and uploading it via a bibupload task.
# add in key for associated publication...
keywords = []
for keyword in submission.keywords:
keywords.append({"name": keyword.name,
"value": keyword.value,
"synonyms": ""})
# we want to retrieve back the authors of the paper
# and assign them as authors of the data too
if not publication_record:
publication_record = get_record_by_id(recid)
submission_info = {
"title": submission.name,
"abstract": submission.description,
"inspire_id": publication_record['inspire_id'],
"doi": submission.doi,
"authors": publication_record['authors'],
"first_author": publication_record.get('first_author', None),
"related_publication": submission.publication_recid,
"creation_date": publication_record["creation_date"],
"last_updated": current_time,
"journal_info": publication_record.get("journal_info", ""),
"keywords": keywords,
"version": version,
"collaborations": publication_record.get("collaborations", []),
}
if submission_info["title"] in existing_submissions:
# in the event that we're performing an update operation, we need
# to get the data record information
# from the index, and use the same record id. This way, we'll just
# update the submission instead of recreating
# a completely new record.
recid = existing_submissions[submission_info["title"]]
submission_info["control_number"] = submission_info["recid"] = recid
else:
submission_info = create_record(submission_info)
submission_info["control_number"] = submission_info["recid"]
submission.associated_recid = submission_info['recid']
submission.publication_inspire_id = publication_record['inspire_id']
generated_record_ids.append(submission_info["recid"])
submission.version = version
data_review = DataReview.query.filter_by(data_recid=submission.id).first()
if data_review:
data_review.version = version
db.session.add(data_review)
db.session.add(submission)
def test_create_record_for_dashboard(app):
with app.app_context():
record_information = create_record({
'journal_info': 'Phys. Letts',
'title': 'My Journal Paper',
'inspire_id': '1487726'
})
hepsubmission = get_or_create_hepsubmission(
record_information['recid'])
record = get_record_by_id(record_information['recid'])
user = User(email='*****@*****.**',
password='******',
active=True,
id=101)
test_submissions = {}
create_record_for_dashboard(record['recid'], test_submissions, user)
assert (test_submissions == {
record_information['recid']: {
'metadata': {
'coordinator': {
'name': 'No coordinator'
},
'recid': record_information['recid'],
'role': [],
'start_date': hepsubmission.created,
'last_updated': hepsubmission.last_updated,
'title': u'My Journal Paper',
'versions': 1
},
'stats': {
'attention': 0,
'passed': 0,
'todo': 0
},
'status': 'todo'
}
})
test_submissions = {
record_information['recid']: {
"metadata": {
"role": []
}
}
}
create_record_for_dashboard(record['recid'], test_submissions, user)
assert (test_submissions == {
record_information['recid']: {
"metadata": {
"role": [[]]
}
}
})
def index_record_ids(record_ids, index=None):
""" Index records given in the argument.
:param record_ids: [list of ints] list of record ids e.g. [1, 5, 2, 3]
:param index: [string] name of the index. If None a default is used
:return: list of indexed publication and data recids
"""
from hepdata.modules.records.utils.common import get_record_by_id
docs = filter(None, [get_record_by_id(recid) for recid in record_ids])
to_index = []
indexed_result = {CFG_DATA_TYPE: [], CFG_PUB_TYPE: []}
for doc in docs:
if "related_publication" in doc:
# Remove unnecessary fields if it's a data record
for field in ["authors", "_additional_authors", "_first_author"]:
if field in doc:
del doc[field]
enhance_data_document(doc)
op_dict = {
"index": {
"_index": index,
"_type": CFG_DATA_TYPE,
"_id": doc["recid"],
"_parent": str(doc["related_publication"]),
}
}
indexed_result[CFG_DATA_TYPE].append(doc["recid"])
to_index.append(op_dict)
else:
author_docs = prepare_author_for_indexing(es, doc)
to_index += author_docs
enhance_publication_document(doc)
op_dict = {"index": {"_index": index, "_type": CFG_PUB_TYPE, "_id": doc["recid"]}}
indexed_result[CFG_PUB_TYPE].append(doc["recid"])
to_index.append(op_dict)
if doc["last_updated"] is not None:
doc["last_updated"] = parse(doc["last_updated"]).isoformat()
to_index.append(doc)
es.bulk(index=index, body=to_index, refresh=True)
return indexed_result
def finalise(recid, publication_record=None, force_finalise=False):
commit_message = request.form.get('message')
# Update publication information from INSPIRE record before finalising.
if not publication_record:
record = get_record_by_id(recid)
content, status = get_inspire_record_information(record['inspire_id'])
if status == 'success':
publication_record = update_record(recid, content)
return do_finalise(recid, publication_record=publication_record, force_finalise=force_finalise,
commit_message=commit_message, send_tweet=True)
def send_tweet(inspireids):
"""Send tweet announcing that records have been added or revised (in case it wasn't done automatically)."""
processed_inspireids = parse_inspireids_from_string(inspireids)
for inspireid in processed_inspireids:
_cleaned_id = inspireid.replace("ins", "")
submission = get_latest_hepsubmission(inspire_id=_cleaned_id, overall_status='finished')
if submission:
record = get_record_by_id(submission.publication_recid)
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
url = site_url + '/record/ins{0}'.format(record['inspire_id'])
tweet(record['title'], record['collaborations'], url, record['version'])
else:
print("No records found for Inspire ID {}".format(inspireid))
def create_missing_datasubmission_records():
# Get submissions with missing IDs
missing_submissions = DataSubmission.query \
.join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid) \
.filter(
DataSubmission.associated_recid == None,
DataSubmission.publication_inspire_id == None,
DataSubmission.version == HEPSubmission.version,
HEPSubmission.overall_status == 'finished')
missing_submissions = missing_submissions.all()
if not missing_submissions:
print("No datasubmissions found with missing record or inspire ids.")
return
# Organise missing submissions by publication
submissions_by_publication = {}
for submission in missing_submissions:
if submission.publication_recid in submissions_by_publication:
submissions_by_publication[submission.publication_recid].append(
submission)
else:
submissions_by_publication[submission.publication_recid] = [
submission
]
# Loop through each publication
for publication_recid, submissions in submissions_by_publication.items():
publication_record = get_record_by_id(publication_recid)
current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.utcnow())
generated_record_ids = []
for submission in submissions:
# Finalise each data submission that does not have a record
finalise_datasubmission(current_time, {}, generated_record_ids,
publication_record, publication_recid,
submission, submission.version)
# Register the datasubmission's DOI
if not current_app.config.get('TESTING', False):
generate_doi_for_table.delay(submission.doi)
print(f"Generated DOI {submission.doi}")
else:
print(f"Would generate DOI {submission.doi}")
# finalise_datasubmission does not commit, so commit once for each publication
db.session.commit()
# Reindex the publication and its updated datasubmissions
index_record_ids([publication_recid] + generated_record_ids)
push_data_keywords(pub_ids=[publication_recid])
def update_record(recid, ctx):
"""
Updates a record given a new dictionary.
:param recid:
:param ctx:
:return:
"""
print('Recid is {}'.format(recid))
record = get_record_by_id(recid)
for key, value in ctx.iteritems():
record[key] = value
record.commit()
db.session.commit()
return record
def create_record_for_dashboard(record_id, submissions, primary_uploader=None,
primary_reviewer=None, coordinator=None,
user_role=None,
status="todo"):
if user_role is None:
user_role = ["coordinator"]
publication_record = get_record_by_id(int(record_id))
if publication_record is not None:
if record_id not in submissions:
hepdata_submission_record = get_latest_hepsubmission(publication_recid=record_id)
submissions[record_id] = {}
submissions[record_id]["metadata"] = {"recid": record_id,
"role": user_role,
"start_date": publication_record.created}
submissions[record_id]["metadata"][
"versions"] = hepdata_submission_record.version
submissions[record_id]["status"] = status
submissions[record_id]["stats"] = {"passed": 0, "attention": 0,
"todo": 0}
if coordinator:
submissions[record_id]["metadata"]["coordinator"] = {
'id': coordinator.id, 'name': coordinator.email,
'email': coordinator.email}
submissions[record_id]["metadata"][
"show_coord_view"] = int(current_user.get_id()) == coordinator.id
else:
submissions[record_id]["metadata"]["coordinator"] = {
'name': 'No coordinator'}
if "title" in publication_record:
submissions[record_id]["metadata"]["title"] = \
publication_record['title']
if "inspire_id" not in publication_record or publication_record["inspire_id"] is None:
submissions[record_id]["metadata"][
"requires_inspire_id"] = True
else:
# if it is, it's because the user has two roles for that
# submission. So we should show them!
if user_role not in submissions[record_id]["metadata"]["role"]:
submissions[record_id]["metadata"]["role"].append(user_role)
def update_record(recid, ctx):
"""
Updates a record given a new dictionary.
:param recid:
:param ctx:
:return:
"""
print('Updating record {}'.format(recid))
record = get_record_by_id(recid)
for key, value in ctx.iteritems():
record[key] = value
record["recid"] = recid
record.commit()
db.session.commit()
return record
def list_submission_titles(current_user):
query = _prepare_submission_query(current_user)
hepdata_submission_records = query.order_by(
HEPSubmission.last_updated.desc()).all()
titles = []
for hepsubmission in hepdata_submission_records:
publication_record = get_record_by_id(
int(hepsubmission.publication_recid))
if publication_record:
titles.append({
'id': int(hepsubmission.publication_recid),
'title': publication_record['title']
})
return titles
def send_new_review_message_email(review, message, user):
"""
Sends a message to all uploaders and reviewers to tell them that a
comment has been made on a record.
:param review:
:param message:
:param user:
:return:
"""
submission_participants = get_submission_participants_for_record(
review.publication_recid)
table_information = DataSubmission.query.filter_by(
id=review.data_recid).one()
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
record = get_record_by_id(review.publication_recid)
destinations = [
participant.email for participant in submission_participants
]
full_names = [
participant.full_name for participant in submission_participants
]
message_body = render_template(
'hepdata_theme/email/review-message.html',
name=', '.join(set(full_names)),
actor=user.email,
table_name=table_information.name,
table_message=decode_string(message.message),
article=review.publication_recid,
title=record['title'],
site_url=site_url,
link=site_url + "/record/{0}".format(review.publication_recid),
table_link=site_url + "/record/{0}?table={1}".format(
review.publication_recid, table_information.name.replace(
'+', '%2B')))
create_send_email_task(
','.join(set(destinations)),
'[HEPData] Submission {0} ({1}) has a new review message'.format(
review.publication_recid, table_information.name), message_body)
def send_tweet(inspireids):
"""
Send tweet announcing that records have been added or revised (in case it wasn't done automatically).
:param inspireids: list of record IDs to tweet
"""
processed_inspireids = parse_inspireids_from_string(inspireids)
for inspireid in processed_inspireids:
_cleaned_id = inspireid.replace("ins", "")
submission = get_latest_hepsubmission(inspire_id=_cleaned_id)
if submission:
record = get_record_by_id(submission.publication_recid)
url = "http://www.hepdata.net/record/ins{0}".format(
record['inspire_id'])
tweet(record['title'], record['collaborations'], url,
record['version'])
else:
print("No records found for Inspire ID {}".format(inspireid))
def send_coordinator_notification_email(recid, version, user, message=None):
"""
:param recid:
:param user: user object
:param message: message to send
:return:
"""
hepsubmission = get_latest_hepsubmission(publication_recid=recid)
coordinator = get_user_from_id(hepsubmission.coordinator)
if not coordinator:
raise NoParticipantsException()
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
record = get_record_by_id(recid)
name = coordinator.email
coordinator_profile = UserProfile.get_by_userid(hepsubmission.coordinator)
if coordinator_profile:
name = coordinator_profile.full_name
collaboration = _get_collaboration(hepsubmission.coordinator)
message_body = render_template('hepdata_theme/email/passed_review.html',
name=name,
actor=user.email,
collaboration=collaboration,
article=recid,
version=version,
message=message,
title=record['title'],
site_url=site_url,
link=site_url + "/record/{0}".format(recid),
dashboard_link=site_url + "/dashboard")
create_send_email_task(
coordinator.email,
'[HEPData] Submission {0} is ready to be finalised'.format(recid),
message_body)
def get_pending_invitations_for_user(user):
pending_invites = SubmissionParticipant.query.filter(
SubmissionParticipant.email == user.email,
or_(SubmissionParticipant.role == 'reviewer',
SubmissionParticipant.role == 'uploader'),
SubmissionParticipant.user_account == None
).all()
result = []
for invite in pending_invites:
publication_record = get_record_by_id(invite.publication_recid)
hepsubmission = get_latest_hepsubmission(publication_recid=invite.publication_recid)
coordinator = get_user_from_id(hepsubmission.coordinator)
result.append(
{'title': encode_string(publication_record['title'], 'utf-8'),
'invitation_cookie': invite.invitation_cookie,
'role': invite.role, 'coordinator': coordinator})
return result
def test_create_new_version(app, load_default_data, identifiers, mocker):
hepsubmission = get_latest_hepsubmission(publication_recid=1)
assert hepsubmission.version == 1
# Add an uploader
uploader = SubmissionParticipant(publication_recid=1,
email='*****@*****.**',
role='uploader',
status='primary')
db.session.add(uploader)
db.session.commit()
user = User.query.first()
# Mock `send_cookie_email` method
send_cookie_mock = mocker.patch(
'hepdata.modules.records.api.send_cookie_email')
# Create new version of valid finished record
result = create_new_version(1, user, uploader_message="Hello!")
assert result.status_code == 200
assert result.json == {'success': True, 'version': 2}
# get_latest_hepsubmission should now return version 2
hepsubmission = get_latest_hepsubmission(publication_recid=1)
assert hepsubmission.version == 2
assert hepsubmission.overall_status == 'todo'
# Should have attempted to send uploader email
send_cookie_mock.assert_called_with(uploader,
get_record_by_id(1),
message="Hello!",
version=2)
# Try creating a new version - should not work as status of most recent is 'todo'
result, status_code = create_new_version(1, user)
assert status_code == 400
assert result.json == {
'message': 'Rec id 1 is not finished so cannot create a new version'
}
def send_new_upload_email(recid, user, message=None):
"""
:param recid:
:param user: user object
:param message:
:return:
"""
submission_participants = SubmissionParticipant.query.filter_by(
publication_recid=recid, status="primary", role="reviewer")
if submission_participants.count() == 0:
raise NoReviewersException()
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
record = get_record_by_id(recid)
for participant in submission_participants:
invite_token = None
if not participant.user_account:
invite_token = participant.invitation_cookie
message_body = render_template('hepdata_theme/email/upload.html',
name=participant.full_name,
actor=user.email,
article=recid,
message=message,
invite_token=invite_token,
role=participant.role,
title=record['title'],
site_url=site_url,
link=site_url +
"/record/{0}".format(recid))
create_send_email_task(
participant.email,
'[HEPData] Submission {0} has a new upload available for you to review'
.format(recid), message_body)
def test_prepare_submissions_admin(app, load_submission):
with app.app_context():
record_information = create_record({
'journal_info': 'Phys. Letts',
'title': 'My Journal Paper',
'inspire_id': '1487726'
})
get_or_create_hepsubmission(record_information['recid'])
record = get_record_by_id(record_information['recid'])
role = Role(name='admin')
user = User(email='*****@*****.**',
password='******',
active=True,
id=101,
roles=[role])
submissions = prepare_submissions(user)
assert (len(submissions) == 1)
assert (submissions[str(record_information['recid'])] == {
'metadata': {
'coordinator': {
'email': u'*****@*****.**',
'name': u'*****@*****.**',
'id': 1
},
'recid': str(record_information['recid']),
'role': ['coordinator'],
'show_coord_view': False,
'start_date': record.created,
'title': u'My Journal Paper',
'versions': 1
},
'stats': {
'attention': 0L,
'passed': 0L,
'todo': 0L
},
'status': u'todo'
})
def send_new_upload_email(recid, user, message=None):
"""
:param action: e.g. upload or review_message
:param hepsubmission: submission information
:param user: user object
:return:
"""
submission_participants = SubmissionParticipant.query.filter_by(
publication_recid=recid, status="primary", role="reviewer")
if submission_participants.count() == 0:
raise NoReviewersException()
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
record = get_record_by_id(recid)
for participant in submission_participants:
invite_token = None
if not participant.user_account:
invite_token = participant.invitation_cookie
message_body = render_template('hepdata_theme/email/upload.html',
name=participant.full_name,
actor=user.email,
article=recid,
message=message,
invite_token=invite_token,
role=participant.role,
title=record['title'],
site_url=site_url,
link=site_url + "/record/{0}"
.format(recid))
create_send_email_task(participant.email,
'[HEPData] Submission {0} has a new upload available for you to review.'.format(recid),
message_body)
def send_new_review_message_email(review, message, user):
"""
Sends a message to all uploaders to tell them that a
comment has been made on a record
:param hepsubmission:
:param review:
:param user:
:return:
"""
submission_participants = SubmissionParticipant.query.filter_by(
publication_recid=review.publication_recid,
status="primary",
role="uploader")
table_information = DataSubmission.query \
.filter_by(id=review.data_recid).one()
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
record = get_record_by_id(review.publication_recid)
for participant in submission_participants:
message_body = render_template(
'hepdata_theme/email/review-message.html',
name=participant.full_name,
actor=user.email,
table_name=table_information.name,
table_message=message.message,
article=review.publication_recid,
title=record['title'],
site_url=site_url,
link=site_url + "/record/{0}".format(review.publication_recid))
create_send_email_task(
participant.email,
'[HEPData] Submission {0} has a new review message.' \
.format(review.publication_recid), message_body)
def attach_information_to_record(recid):
"""
Given an INSPIRE data representation, this will process the data, and update information
for a given record id with the contents.
:return:
"""
inspire_id = request.form["inspire_id"]
content, status = get_inspire_record_information(inspire_id)
content["inspire_id"] = inspire_id
record = get_record_by_id(recid)
if record is not None:
content["recid"] = recid
patch = jsonpatch.JsonPatch.from_diff(record, content)
record = record.patch(patch=patch)
record.commit()
db.session.commit()
return jsonify({"status": "success"})
else:
return jsonify({"status": "failed", "message": "No record with that recid was found."})
def send_finalised_email(hepsubmission):
record = get_record_by_id(hepsubmission.publication_recid)
notify_participants(hepsubmission, record)
notify_subscribers(hepsubmission, record)
def do_finalise(recid, publication_record=None, force_finalise=False,
commit_message=None, send_tweet=False, update=False):
"""
Creates record SIP for each data record with a link to the associated
publication
:param synchronous: if true then workflow execution and creation is
waited on, then everything is indexed in one go.
If False, object creation is asynchronous, however reindexing is not
performed. This is only really useful for the full migration of
content.
"""
hep_submission = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status="todo").first()
print('Finalising record {}'.format(recid))
generated_record_ids = []
if hep_submission \
and (force_finalise or hep_submission.coordinator == int(current_user.get_id())):
submissions = DataSubmission.query.filter_by(
publication_recid=recid,
version=hep_submission.version).all()
version = hep_submission.version
existing_submissions = {}
if hep_submission.version > 1 or update:
# we need to determine which are the existing record ids.
existing_data_records = get_records_matching_field(
'related_publication', recid, doc_type=CFG_DATA_TYPE)
for record in existing_data_records["hits"]["hits"]:
if "recid" in record["_source"]:
existing_submissions[record["_source"]["title"]] = \
record["_source"]["recid"]
delete_item_from_index(record["_id"],
doc_type=CFG_DATA_TYPE, parent=record["_source"]["related_publication"])
current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.now())
for submission in submissions:
finalise_datasubmission(current_time, existing_submissions,
generated_record_ids,
publication_record, recid, submission,
version)
try:
record = get_record_by_id(recid)
# If we have a commit message, then we have a record update.
# We will store the commit message and also update the
# last_updated flag for the record.
record['hepdata_doi'] = hep_submission.doi
if commit_message:
# On a revision, the last updated date will
# be the current date.
hep_submission.last_updated = datetime.now()
commit_record = RecordVersionCommitMessage(
recid=recid,
version=version,
message=str(commit_message))
db.session.add(commit_record)
record['last_updated'] = datetime.strftime(
hep_submission.last_updated, '%Y-%m-%d %H:%M:%S')
record['version'] = version
record.commit()
hep_submission.inspire_id = record['inspire_id']
hep_submission.overall_status = "finished"
db.session.add(hep_submission)
db.session.commit()
create_celery_app(current_app)
# only mint DOIs if not testing.
if not current_app.config.get('TESTING', False) and not current_app.config.get('NO_DOI_MINTING', False):
for submission in submissions:
generate_doi_for_data_submission.delay(submission.id, submission.version)
generate_doi_for_submission.delay(recid, version)
# Reindex everything.
index_record_ids([recid] + generated_record_ids)
push_data_keywords(pub_ids=[recid])
admin_indexer = AdminIndexer()
admin_indexer.index_submission(hep_submission)
send_finalised_email(hep_submission)
for file_format in ['csv', 'yoda', 'root']:
convert_and_store.delay(hep_submission.inspire_id, file_format, force=True)
if send_tweet:
tweet(record.get('title'), record.get('collaborations'),
"http://www.hepdata.net/record/ins{0}".format(record.get('inspire_id')))
return json.dumps({"success": True, "recid": recid,
"data_count": len(submissions),
"generated_records": generated_record_ids})
except NoResultFound:
print('No record found to update. Which is super strange.')
else:
return json.dumps(
{"success": False, "recid": recid,
"errors": ["You do not have permission to finalise this "
"submission. Only coordinators can do that."]})
def update_record_info(inspire_id, send_email=False):
"""Update publication information from INSPIRE for a specific record."""
if inspire_id is None:
log.error("Inspire ID is None")
return 'Inspire ID is None'
inspire_id = inspire_id.replace("ins", "")
hep_submission = get_latest_hepsubmission(inspire_id=inspire_id)
if hep_submission is None:
log.warning("Failed to retrieve HEPData submission for Inspire ID {0}".format(inspire_id))
return 'No HEPData submission'
publication_recid = hep_submission.publication_recid
log.info("Updating recid {} with information from Inspire record {}".format(publication_recid, inspire_id))
updated_inspire_record_information, status = get_inspire_record_information(inspire_id)
if status == 'success':
# Also need to update publication information for data records.
data_submissions = DataSubmission.query.filter_by(
publication_recid=publication_recid, version=hep_submission.version
).order_by(DataSubmission.id.asc())
record_ids = [publication_recid] # list of record IDs
for data_submission in data_submissions:
record_ids.append(data_submission.associated_recid)
same_information = {}
for index, recid in enumerate(record_ids):
if index == 0:
updated_record_information = updated_inspire_record_information
else:
# Only update selected keys for data records.
updated_record_information = {
key: updated_inspire_record_information[key] for key in (
'authors', 'creation_date', 'journal_info', 'collaborations'
)
}
record_information = get_record_by_id(recid)
same_information[recid] = True
for key, value in updated_record_information.items():
if key not in record_information or record_information[key] != value:
log.debug('For recid {}, key {} has new value {}'.format(recid, key, value))
same_information[recid] = False
update_record(recid, updated_record_information)
break
log.info('For recid {}, information needs to be updated: {}'.format(recid, str(not(same_information[recid]))))
if all(same for same in same_information.values()):
return 'No update needed'
else:
log.warning("Failed to retrieve publication information for Inspire record {0}".format(inspire_id))
return 'Invalid Inspire ID'
if hep_submission.overall_status == 'finished':
index_record_ids(record_ids) # index for Elasticsearch
push_data_keywords(pub_ids=[recid])
if not TESTING:
generate_dois_for_submission.delay(inspire_id=inspire_id) # update metadata stored in DataCite
if send_email:
record_information = get_record_by_id(publication_recid)
notify_publication_update(hep_submission, record_information) # send email to all participants
return 'Success'
def test_import_record(app):
all_submissions = HEPSubmission.query.all()
assert len(all_submissions) == 0
data_submissions = DataSubmission.query.all()
assert len(data_submissions) == 0
# Import a record and check it's been added to the db
inspire_id = '1824424'
result = _import_record(inspire_id)
assert result is True
all_submissions = HEPSubmission.query.all()
assert len(all_submissions) == 1
hep_submission = all_submissions[0]
assert hep_submission.publication_recid == 1
assert hep_submission.inspire_id == inspire_id
assert hep_submission.overall_status == 'finished'
last_updated = hep_submission.last_updated
record = get_record_by_id(1)
assert record['title'] == 'Measurement of the production cross section of 31 GeV/$c$ protons on carbon via beam attenuation in a 90-cm-long target'
assert record['inspire_id'] == inspire_id
assert record['abstract'].startswith('The production cross section of 30.92 GeV/$c$ protons on carbon is measured by')
data_submissions = DataSubmission.query.all()
assert len(data_submissions) == 2
# Try the import again - should not update
result = _import_record(inspire_id)
assert result is False
# Retry with update_existing=True
result = _import_record(inspire_id, update_existing=True)
assert result is True
updated_submission = get_latest_hepsubmission(publication_recid=1)
assert updated_submission.last_updated > last_updated
# Try an old inspire id which uses the old schema
old_inspire_id = '944937'
result = _import_record(old_inspire_id)
assert result is True
all_submissions = HEPSubmission.query.all()
assert len(all_submissions) == 2
data_submissions = DataSubmission.query.all()
assert len(data_submissions) == 3
record = get_record_by_id(all_submissions[1].publication_recid)
assert record['title'] == 'The Production of Charged Photomesons from Deuterium and Hydrogen. I'
assert record['inspire_id'] == old_inspire_id
assert record['abstract'].startswith('Hydrogen and deuterium gases have been bombarded in a gas target at a temperature of 77°K')
# Try an invalid inspire id
assert _import_record('thisisinvalid') is False
# Mock errors with download
with requests_mock.Mocker(real_http=True) as mock:
# 404
mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id),
status_code=404)
result = _import_record(inspire_id, update_existing=True)
assert result is False
# Text not zip
mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id),
text='This is not a zip')
result = _import_record(inspire_id, update_existing=True)
assert result is False
# SocketError
mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id),
exc=requests.exceptions.ConnectTimeout("mock error message"))
result = _import_record(inspire_id, update_existing=True)
assert result is False
# Send an invalid zip
base_dir = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(base_dir, 'test_data/submission_invalid_symlink.tgz')
with open(file_path, 'rb') as f:
mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id),
headers={'content-type': 'application/zip'}, body=f)
result = _import_record(inspire_id, update_existing=True)
assert result is False
def index_record_ids(record_ids, index=None):
""" Index records given in the argument.
:param record_ids: [list of ints] list of record ids e.g. [1, 5, 2, 3]
:param index: [string] name of the index. If None a default is used
:return: list of indexed publication and data recids
"""
from hepdata.modules.records.utils.common import get_record_by_id
docs = filter(None, [get_record_by_id(recid) for recid in record_ids])
existing_record_ids = [doc['recid'] for doc in docs]
print('Indexing existing record IDs:', existing_record_ids)
to_index = []
indexed_result = {CFG_DATA_TYPE: [], CFG_PUB_TYPE: []}
for doc in docs:
if 'related_publication' in doc:
# Remove unnecessary fields if it's a data record
for field in ['authors', '_additional_authors', '_first_author']:
if field in doc:
del doc[field]
enhance_data_document(doc)
op_dict = {
"index": {
"_index": index,
"_type": CFG_DATA_TYPE,
"_id": doc['recid'],
"_parent": str(doc['related_publication'])
}
}
indexed_result[CFG_DATA_TYPE].append(doc['recid'])
to_index.append(op_dict)
else:
if 'version' not in doc:
print('Skipping unfinished record ID {}'.format(doc['recid']))
continue
author_docs = prepare_author_for_indexing(doc)
to_index += author_docs
enhance_publication_document(doc)
op_dict = {
"index": {
"_index": index,
"_type": CFG_PUB_TYPE,
"_id": doc['recid']
}
}
indexed_result[CFG_PUB_TYPE].append(doc['recid'])
to_index.append(op_dict)
if doc["last_updated"] is not None:
doc["last_updated"] = parse(doc["last_updated"]).isoformat()
to_index.append(doc)
if to_index:
es.bulk(index=index, body=to_index, refresh=True)
return indexed_result
def remove_submission(record_id):
"""
Removes the database entries related to a record.
:param record_id:
:return: True if Successful, False if the record does not exist.
"""
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=record_id).all()
try:
try:
for hepdata_submission in hepdata_submissions:
db.session.delete(hepdata_submission)
except NoResultFound as nrf:
print(nrf.args)
submissions = DataSubmission.query.filter_by(
publication_recid=record_id).all()
reviews = DataReview.query.filter_by(
publication_recid=record_id).all()
for review in reviews:
db.session.delete(review)
for submission in submissions:
resource = DataResource.query.filter_by(
id=submission.data_file).first()
db.session.delete(submission)
if resource:
db.session.delete(resource)
try:
SubmissionParticipant.query.filter_by(
publication_recid=record_id).delete()
except Exception:
print("Unable to find a submission participant for {0}".format(record_id))
try:
record = get_record_by_id(record_id)
data_records = get_records_matching_field(
'related_publication', record_id, doc_type=CFG_DATA_TYPE)
if 'hits' in data_records:
for data_record in data_records['hits']['hits']:
data_record_obj = get_record_by_id(data_record['_source']['recid'])
if data_record_obj:
data_record_obj.delete()
if record:
record.delete()
except PIDDoesNotExistError as e:
print('No record entry exists for {0}. Proceeding to delete other files.'.format(record_id))
db.session.commit()
db.session.flush()
return True
except Exception as e:
db.session.rollback()
raise e
