def send_tweet(inspireids):
"""Send tweet announcing that records have been added or revised (in case it wasn't done automatically)."""
processed_inspireids = parse_inspireids_from_string(inspireids)
for inspireid in processed_inspireids:
_cleaned_id = inspireid.replace("ins", "")
submission = get_latest_hepsubmission(inspire_id=_cleaned_id, overall_status='finished')
if submission:
record = get_record_by_id(submission.publication_recid)
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
url = site_url + '/record/ins{0}'.format(record['inspire_id'])
tweet(record['title'], record['collaborations'], url, record['version'])
else:
print("No records found for Inspire ID {}".format(inspireid))
def send_tweet(inspireids):
"""
Send tweet announcing that records have been added or revised (in case it wasn't done automatically).
:param inspireids: list of record IDs to tweet
"""
processed_inspireids = parse_inspireids_from_string(inspireids)
for inspireid in processed_inspireids:
_cleaned_id = inspireid.replace("ins", "")
submission = get_latest_hepsubmission(inspire_id=_cleaned_id)
if submission:
record = get_record_by_id(submission.publication_recid)
url = "http://www.hepdata.net/record/ins{0}".format(
record['inspire_id'])
tweet(record['title'], record['collaborations'], url,
record['version'])
else:
print("No records found for Inspire ID {}".format(inspireid))
def do_finalise(recid,
publication_record=None,
force_finalise=False,
commit_message=None,
send_tweet=False,
update=False,
convert=True):
"""
Creates record SIP for each data record with a link to the associated
publication
:param synchronous: if true then workflow execution and creation is
waited on, then everything is indexed in one go.
If False, object creation is asynchronous, however reindexing is not
performed. This is only really useful for the full migration of
content.
"""
print('Finalising record {}'.format(recid))
hep_submission = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status="todo").first()
generated_record_ids = []
if hep_submission \
and (force_finalise or hep_submission.coordinator == int(current_user.get_id())):
submissions = DataSubmission.query.filter_by(
publication_recid=recid, version=hep_submission.version).all()
version = hep_submission.version
existing_submissions = {}
if hep_submission.version > 1 or update:
# we need to determine which are the existing record ids.
existing_data_records = get_records_matching_field(
'related_publication', recid, doc_type=CFG_DATA_TYPE)
for record in existing_data_records["hits"]["hits"]:
if "recid" in record["_source"]:
existing_submissions[record["_source"]["title"]] = \
record["_source"]["recid"]
delete_item_from_index(
record["_id"],
doc_type=CFG_DATA_TYPE,
parent=record["_source"]["related_publication"])
current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.now())
for submission in submissions:
finalise_datasubmission(current_time, existing_submissions,
generated_record_ids, publication_record,
recid, submission, version)
try:
record = get_record_by_id(recid)
# If we have a commit message, then we have a record update.
# We will store the commit message and also update the
# last_updated flag for the record.
record['hepdata_doi'] = hep_submission.doi
if commit_message:
# On a revision, the last updated date will
# be the current date.
hep_submission.last_updated = datetime.now()
commit_record = RecordVersionCommitMessage(
recid=recid, version=version, message=str(commit_message))
db.session.add(commit_record)
record['last_updated'] = datetime.strftime(
hep_submission.last_updated, '%Y-%m-%d %H:%M:%S')
record['version'] = version
record.commit()
hep_submission.inspire_id = record['inspire_id']
hep_submission.overall_status = "finished"
db.session.add(hep_submission)
db.session.commit()
create_celery_app(current_app)
# only mint DOIs if not testing.
if not current_app.config.get(
'TESTING', False) and not current_app.config.get(
'NO_DOI_MINTING', False):
for submission in submissions:
generate_doi_for_data_submission.delay(
submission.id, submission.version)
log.info("Generating DOIs for ins{0}".format(
hep_submission.inspire_id))
generate_doi_for_submission.delay(recid, version)
# Reindex everything.
index_record_ids([recid] + generated_record_ids)
push_data_keywords(pub_ids=[recid])
try:
admin_indexer = AdminIndexer()
admin_indexer.index_submission(hep_submission)
except ConnectionTimeout as ct:
log.error('Unable to add ins{0} to admin index.\n{1}'.format(
hep_submission.inspire_id, ct))
send_finalised_email(hep_submission)
if convert:
for file_format in ['yaml', 'csv', 'yoda', 'root']:
convert_and_store.delay(hep_submission.inspire_id,
file_format,
force=True)
if send_tweet:
tweet(
record.get('title'), record.get('collaborations'),
"http://www.hepdata.net/record/ins{0}".format(
record.get('inspire_id')), version)
return json.dumps({
"success": True,
"recid": recid,
"data_count": len(submissions),
"generated_records": generated_record_ids
})
except NoResultFound:
print('No record found to update. Which is super strange.')
else:
return json.dumps({
"success":
False,
"recid":
recid,
"errors": [
"You do not have permission to finalise this "
"submission. Only coordinators can do that."
]
})
def do_finalise(recid, publication_record=None, force_finalise=False,
commit_message=None, send_tweet=False, update=False):
"""
Creates record SIP for each data record with a link to the associated
publication
:param synchronous: if true then workflow execution and creation is
waited on, then everything is indexed in one go.
If False, object creation is asynchronous, however reindexing is not
performed. This is only really useful for the full migration of
content.
"""
hep_submission = HEPSubmission.query.filter_by(
publication_recid=recid, overall_status="todo").first()
print('Finalising record {}'.format(recid))
generated_record_ids = []
if hep_submission \
and (force_finalise or hep_submission.coordinator == int(current_user.get_id())):
submissions = DataSubmission.query.filter_by(
publication_recid=recid,
version=hep_submission.version).all()
version = hep_submission.version
existing_submissions = {}
if hep_submission.version > 1 or update:
# we need to determine which are the existing record ids.
existing_data_records = get_records_matching_field(
'related_publication', recid, doc_type=CFG_DATA_TYPE)
for record in existing_data_records["hits"]["hits"]:
if "recid" in record["_source"]:
existing_submissions[record["_source"]["title"]] = \
record["_source"]["recid"]
delete_item_from_index(record["_id"],
doc_type=CFG_DATA_TYPE, parent=record["_source"]["related_publication"])
current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.now())
for submission in submissions:
finalise_datasubmission(current_time, existing_submissions,
generated_record_ids,
publication_record, recid, submission,
version)
try:
record = get_record_by_id(recid)
# If we have a commit message, then we have a record update.
# We will store the commit message and also update the
# last_updated flag for the record.
record['hepdata_doi'] = hep_submission.doi
if commit_message:
# On a revision, the last updated date will
# be the current date.
hep_submission.last_updated = datetime.now()
commit_record = RecordVersionCommitMessage(
recid=recid,
version=version,
message=str(commit_message))
db.session.add(commit_record)
record['last_updated'] = datetime.strftime(
hep_submission.last_updated, '%Y-%m-%d %H:%M:%S')
record['version'] = version
record.commit()
hep_submission.inspire_id = record['inspire_id']
hep_submission.overall_status = "finished"
db.session.add(hep_submission)
db.session.commit()
create_celery_app(current_app)
# only mint DOIs if not testing.
if not current_app.config.get('TESTING', False) and not current_app.config.get('NO_DOI_MINTING', False):
for submission in submissions:
generate_doi_for_data_submission.delay(submission.id, submission.version)
generate_doi_for_submission.delay(recid, version)
# Reindex everything.
index_record_ids([recid] + generated_record_ids)
push_data_keywords(pub_ids=[recid])
admin_indexer = AdminIndexer()
admin_indexer.index_submission(hep_submission)
send_finalised_email(hep_submission)
for file_format in ['csv', 'yoda', 'root']:
convert_and_store.delay(hep_submission.inspire_id, file_format, force=True)
if send_tweet:
tweet(record.get('title'), record.get('collaborations'),
"http://www.hepdata.net/record/ins{0}".format(record.get('inspire_id')))
return json.dumps({"success": True, "recid": recid,
"data_count": len(submissions),
"generated_records": generated_record_ids})
except NoResultFound:
print('No record found to update. Which is super strange.')
else:
return json.dumps(
{"success": False, "recid": recid,
"errors": ["You do not have permission to finalise this "
"submission. Only coordinators can do that."]})