def test_record_update(app): """___test_record_update___""" with app.app_context(): record_information = create_record({'journal_info': 'Phys. Letts', 'title': 'My Journal Paper'}) record = get_record_by_id(record_information['recid']) assert (record['title'] == 'My Journal Paper') assert (record['journal_info'] == 'Phys. Letts') update_record(record_information['recid'], {'journal_info': 'test'}) updated_record = get_record_by_id(record_information['recid']) assert (updated_record['journal_info'] == 'test')
def test_record_update(app): """___test_record_update___""" with app.app_context(): record_information = create_record({'journal_info': 'Phys. Letts', 'title': 'My Journal Paper'}) record = get_record_by_id(record_information['recid']) assert (record['title'] == 'My Journal Paper') assert (record['journal_info'] == 'Phys. Letts') update_record(record_information['recid'], {'journal_info': 'test'}) updated_record = get_record_by_id(record_information['recid']) assert (updated_record['journal_info'] == 'test')
def attach_information_to_record(recid): """ Given an INSPIRE data representation, this will process the data, and update information for a given record id with the contents. :return: """ inspire_id = request.form['inspire_id'] content, status = get_inspire_record_information(inspire_id) content["inspire_id"] = inspire_id record = get_record_by_id(recid) if record is not None: content['recid'] = recid patch = jsonpatch.JsonPatch.from_diff(record, content) record = record.patch(patch=patch) record.commit() db.session.commit() return jsonify({'status': 'success'}) else: return jsonify({'status': 'failed', 'message': 'No record with that recid was found.'})
def generate_doi_for_submission(recid, version): data_submissions = DataSubmission.query.filter_by(publication_recid=recid, version=version).order_by( DataSubmission.id.asc()) hep_submission = HEPSubmission.query.filter_by(publication_recid=recid).first() publication_info = get_record_by_id(recid) version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version) hep_submission.data_abstract = decode_string(hep_submission.data_abstract) publication_info['title'] = decode_string(publication_info['title']) publication_info['abstract'] = decode_string(publication_info['abstract']) base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=hep_submission.doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=version_doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) # Register DOI for the version, and update the base DOI to resolve to the latest submission version. register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']), base_xml, publication_info['uuid']) register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format( publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def promote_or_demote_participant(recid, action, demote_or_promote, participant_id): """ Can promote or demote a participant to/from primary reviewer/uploader. :param recid: record id that the user will be promoted or demoted for :param action: upload or review :param demote_or_promote: demote or promote :param participant_id: id of user from the SubmissionParticipant table. :return: """ try: participant = SubmissionParticipant.query.filter_by( id=participant_id).one() status = 'reserve' if demote_or_promote == 'promote': status = 'primary' participant.status = status db.session.add(participant) db.session.commit() record = get_record_by_id(recid) # now send the email telling the user of their new status! if status == 'primary': send_cookie_email(participant, record) return json.dumps({"success": True, "recid": recid}) except Exception as e: return json.dumps({"success": False, "recid": recid, "error": str(e)})
def generate_dois_for_submission(*args, **kwargs): """ Generate DOIs for all the submission components. :param args: :param kwargs: :return: """ site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') hep_submissions = HEPSubmission.query.filter_by(**kwargs).order_by( HEPSubmission.publication_recid.asc()).all() for hep_submission in hep_submissions: if args: start_recid, end_recid = args if hep_submission.publication_recid < start_recid or hep_submission.publication_recid > end_recid: continue if hep_submission.overall_status != 'finished': continue data_submissions = DataSubmission.query.filter_by( publication_inspire_id=hep_submission.inspire_id, version=hep_submission.version).order_by(DataSubmission.id.asc()) publication_info = get_record_by_id(hep_submission.publication_recid) create_container_doi(hep_submission, data_submissions, publication_info, site_url) for data_submission in data_submissions: create_data_doi(hep_submission, data_submission, publication_info, site_url)
def create_new_version(recid, user, notify_uploader=True, uploader_message=None): hepsubmission = get_latest_hepsubmission(publication_recid=recid) if hepsubmission.overall_status == 'finished': # Reopen the submission to allow for revisions, # by creating a new HEPSubmission object. _rev_hepsubmission = HEPSubmission(publication_recid=recid, overall_status='todo', inspire_id=hepsubmission.inspire_id, coordinator=hepsubmission.coordinator, version=hepsubmission.version + 1) db.session.add(_rev_hepsubmission) db.session.commit() if notify_uploader: uploaders = SubmissionParticipant.query.filter_by( role='uploader', publication_recid=recid, status='primary' ) record_information = get_record_by_id(recid) for uploader in uploaders: send_cookie_email(uploader, record_information, message=uploader_message, version=_rev_hepsubmission.version) return jsonify({'success': True, 'version': _rev_hepsubmission.version}) else: return jsonify({"message": f"Rec id {recid} is not finished so cannot create a new version"}), 400
def finalise_datasubmission(current_time, existing_submissions, generated_record_ids, publication_record, recid, submission, version): # we now create a 'payload' for each data submission # by creating a record json and uploading it via a bibupload task. # add in key for associated publication... keywords = [] for keyword in submission.keywords: keywords.append({ "name": keyword.name, "value": keyword.value, "synonyms": "" }) # we want to retrieve back the authors of the paper # and assign them as authors of the data too if not publication_record: publication_record = get_record_by_id(recid) submission_info = { "title": submission.name, "abstract": submission.description, "inspire_id": publication_record['inspire_id'], "doi": submission.doi, "authors": publication_record['authors'], "first_author": publication_record.get('first_author', None), "related_publication": submission.publication_recid, "creation_date": publication_record["creation_date"], "last_updated": current_time, "journal_info": publication_record.get("journal_info", ""), "keywords": keywords, "version": version, "collaborations": publication_record.get("collaborations", []), } if submission_info["title"] in existing_submissions: # in the event that we're performing an update operation, we need # to get the data record information # from the index, and use the same record id. This way, we'll just # update the submission instead of recreating # a completely new record. recid = existing_submissions[submission_info["title"]] submission_info["control_number"] = submission_info["recid"] = recid else: submission_info = create_record(submission_info) submission_info["control_number"] = submission_info["recid"] submission.associated_recid = submission_info['recid'] submission.publication_inspire_id = publication_record['inspire_id'] generated_record_ids.append(submission_info["recid"]) submission.version = version data_review = DataReview.query.filter_by(data_recid=submission.id).first() if data_review: data_review.version = version db.session.add(data_review) db.session.add(submission)
def promote_or_demote_participant(recid, action, demote_or_promote, participant_id): """ Can promote or demote a participant to/from primary reviewer/uploader :param recid: record id that the user will be promoted or demoted :param action: upload or review :param demote_or_promote: demote or promote :param participant_id: id of user from the SubmissionParticipant table. :return: """ try: participant = SubmissionParticipant.query.filter_by( id=participant_id).one() status = 'reserve' if demote_or_promote == 'promote': status = 'primary' participant.status = status db.session.add(participant) db.session.commit() record = get_record_by_id(recid) # now send the email telling the user of their new status! if status == 'primary': send_cookie_email(participant, record) return json.dumps({"success": True, "recid": recid}) except Exception as e: return json.dumps( {"success": False, "recid": recid, "error": e.message})
def generate_doi_for_data_submission(data_submission_id, version): data_submission = DataSubmission.query.filter_by(id=data_submission_id).first() hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first() publication_info = get_record_by_id(data_submission.publication_recid) data_file = DataResource.query.filter_by(id=data_submission.data_file).first() license = None if data_file: if data_file.file_license: license = License.query.filter_by(id=data_file.file_license).first() xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml', doi=data_submission.doi, table_name=decode_string(data_submission.name), table_description=decode_string(data_submission.description), overall_submission=hep_submission, data_submission=data_submission, license=license, publication_info=publication_info) register_doi(data_submission.doi, 'http://www.hepdata.net/record/ins{0}?version={1}&table={2}'.format( publication_info['inspire_id'], data_submission.version, data_submission.name), xml, publication_info['uuid'])
def generate_doi_for_data_submission(data_submission_id, version): data_submission = DataSubmission.query.filter_by(id=data_submission_id).first() hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first() publication_info = get_record_by_id(data_submission.publication_recid) data_file = DataResource.query.filter_by(id=data_submission.data_file).first() license = None if data_file: if data_file.file_license: license = License.query.filter_by(id=data_file.file_license).first() xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml', doi=data_submission.doi, table_name=decode_string(data_submission.name), table_description=decode_string(data_submission.description), overall_submission=hep_submission, data_submission=data_submission, license=license, publication_info=publication_info) register_doi(data_submission.doi, 'http://www.hepdata.net/record/{0}?version={1}&table={2}'.format( hep_submission.publication_recid, data_submission.version, data_submission.name), xml, publication_info['uuid'])
def get_pending_invitations_for_user(user): """ Returns pending invites for upload or review of records. :param user: User object :return: array of pending invites """ pending_invites = SubmissionParticipant.query.filter( func.lower(SubmissionParticipant.email) == func.lower(user.email), or_(SubmissionParticipant.role == 'reviewer', SubmissionParticipant.role == 'uploader'), SubmissionParticipant.status == 'primary', SubmissionParticipant.user_account == None).all() result = [] for invite in pending_invites: publication_record = get_record_by_id(invite.publication_recid) hepsubmission = get_latest_hepsubmission( publication_recid=invite.publication_recid) coordinator = get_user_from_id(hepsubmission.coordinator) result.append({ 'title': decode_string(publication_record['title']), 'invitation_cookie': invite.invitation_cookie, 'role': invite.role, 'coordinator': coordinator }) return result
def generate_dois_for_submission(*args, **kwargs): """ Generate DOIs for all the submission components :param inspire_id: :param version: :param kwargs: :return: """ hep_submissions = HEPSubmission.query.filter_by(**kwargs).all() for hep_submission in hep_submissions: if hep_submission.overall_status != 'finished': continue data_submissions = DataSubmission.query.filter_by(publication_inspire_id=hep_submission.inspire_id, version=hep_submission.version).order_by( DataSubmission.id.asc()) publication_info = get_record_by_id(hep_submission.publication_recid) create_container_doi(hep_submission, data_submissions, publication_info) for data_submission in data_submissions: create_data_doi(hep_submission, data_submission, publication_info)
def generate_doi_for_submission(recid, version): data_submissions = DataSubmission.query.filter_by(publication_recid=recid, version=version).order_by( DataSubmission.id.asc()) hep_submission = HEPSubmission.query.filter_by(publication_recid=recid).first() publication_info = get_record_by_id(recid) version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version) hep_submission.data_abstract = decode_string(hep_submission.data_abstract) publication_info['title'] = decode_string(publication_info['title']) publication_info['abstract'] = decode_string(publication_info['abstract']) base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=hep_submission.doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=version_doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) # Register DOI for the version, and update the base DOI to resolve to the latest submission version. register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']), base_xml, publication_info['uuid']) register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format( publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def attach_information_to_record(recid): """ Given an INSPIRE data representation, this will process the data, and update information for a given record id with the contents. :return: """ inspire_id = request.form['inspire_id'] content, status = get_inspire_record_information(inspire_id) content["inspire_id"] = inspire_id record = get_record_by_id(recid) if record is not None and status == 'success': content['recid'] = recid record.update(content) record.commit() hep_submission = HEPSubmission.query.filter_by( publication_recid=recid, overall_status="todo").first() hep_submission.inspire_id = inspire_id db.session.add(hep_submission) db.session.commit() return jsonify({'status': 'success'}) elif status != 'success': return jsonify({'status': status, 'message': 'Request for INSPIRE record {} failed.'.format(inspire_id)}) else: return jsonify({'status': 'failed', 'message': 'No record with recid {} was found.'.format(str(recid))})
def send_new_review_message_email(review, message, user): """ Sends a message to all uploaders to tell them that a comment has been made on a record :param hepsubmission: :param review: :param user: :return: """ submission_participants = SubmissionParticipant.query.filter_by( publication_recid=review.publication_recid, status="primary", role="uploader") table_information = DataSubmission.query \ .filter_by(id=review.data_recid).one() record = get_record_by_id(review.publication_recid) for participant in submission_participants: message_body = render_template( 'hepdata_theme/email/review-message.html', name=participant.full_name, actor=user.email, table_name=table_information.name, table_message=message.message, article=review.publication_recid, title=record['title'], link="http://hepdata.net/record/{0}" .format(review.publication_recid)) create_send_email_task( participant.email, '[HEPData] Submission {0} has a new review message.' \ .format(review.publication_recid), message_body)
def generate_doi_for_table(doi): """ Generate DOI for a specific table given by its doi. :param doi: :return: """ site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') try: data_submission = DataSubmission.query.filter_by(doi=doi).one() except NoResultFound: print('Table DOI {} not found in database'.format(doi)) return hep_submission = HEPSubmission.query.filter_by( inspire_id=data_submission.publication_inspire_id, version=data_submission.version, overall_status='finished').one() if hep_submission: publication_info = get_record_by_id(hep_submission.publication_recid) create_data_doi(hep_submission, data_submission, publication_info, site_url) else: print( 'Finished submission with INSPIRE ID {} and version {} not found in database' .format(data_submission.publication_inspire_id, data_submission.version))
def create_record_for_dashboard(record_id, submissions, current_user, coordinator=None, user_role=None, status="todo"): if user_role is None: user_role = ["coordinator"] publication_record = get_record_by_id(int(record_id)) if publication_record is not None: if record_id not in submissions: hepdata_submission_record = get_latest_hepsubmission( publication_recid=record_id) submissions[record_id] = {} submissions[record_id]["metadata"] = { "recid": record_id, "role": user_role, "start_date": publication_record.created } submissions[record_id]["metadata"][ "versions"] = hepdata_submission_record.version submissions[record_id]["status"] = status submissions[record_id]["stats"] = { "passed": 0, "attention": 0, "todo": 0 } if coordinator: submissions[record_id]["metadata"]["coordinator"] = { 'id': coordinator.id, 'name': coordinator.email, 'email': coordinator.email } submissions[record_id]["metadata"]["show_coord_view"] = int( current_user.get_id()) == coordinator.id else: submissions[record_id]["metadata"]["coordinator"] = { 'name': 'No coordinator' } if "title" in publication_record: submissions[record_id]["metadata"]["title"] = \ publication_record['title'] if "inspire_id" not in publication_record or publication_record[ "inspire_id"] is None: submissions[record_id]["metadata"][ "requires_inspire_id"] = True else: # if it is, it's because the user has two roles for that # submission. So we should show them! if user_role not in submissions[record_id]["metadata"]["role"]: submissions[record_id]["metadata"]["role"].append(user_role)
def send_finalised_email(hepsubmission): submission_participants = get_submission_participants_for_record( hepsubmission.publication_recid) record = get_record_by_id(hepsubmission.publication_recid) notify_participants(hepsubmission, record, submission_participants) notify_subscribers(hepsubmission, record)
def finalise_datasubmission(current_time, existing_submissions, generated_record_ids, publication_record, recid, submission, version): # we now create a 'payload' for each data submission # by creating a record json and uploading it via a bibupload task. # add in key for associated publication... keywords = [] for keyword in submission.keywords: keywords.append({"name": keyword.name, "value": keyword.value, "synonyms": ""}) # we want to retrieve back the authors of the paper # and assign them as authors of the data too if not publication_record: publication_record = get_record_by_id(recid) submission_info = { "title": submission.name, "abstract": submission.description, "inspire_id": publication_record['inspire_id'], "doi": submission.doi, "authors": publication_record['authors'], "first_author": publication_record.get('first_author', None), "related_publication": submission.publication_recid, "creation_date": publication_record["creation_date"], "last_updated": current_time, "journal_info": publication_record.get("journal_info", ""), "keywords": keywords, "version": version, "collaborations": publication_record.get("collaborations", []), } if submission_info["title"] in existing_submissions: # in the event that we're performing an update operation, we need # to get the data record information # from the index, and use the same record id. This way, we'll just # update the submission instead of recreating # a completely new record. recid = existing_submissions[submission_info["title"]] submission_info["control_number"] = submission_info["recid"] = recid else: submission_info = create_record(submission_info) submission_info["control_number"] = submission_info["recid"] submission.associated_recid = submission_info['recid'] submission.publication_inspire_id = publication_record['inspire_id'] generated_record_ids.append(submission_info["recid"]) submission.version = version data_review = DataReview.query.filter_by(data_recid=submission.id).first() if data_review: data_review.version = version db.session.add(data_review) db.session.add(submission)
def test_create_record_for_dashboard(app): with app.app_context(): record_information = create_record({ 'journal_info': 'Phys. Letts', 'title': 'My Journal Paper', 'inspire_id': '1487726' }) hepsubmission = get_or_create_hepsubmission( record_information['recid']) record = get_record_by_id(record_information['recid']) user = User(email='*****@*****.**', password='******', active=True, id=101) test_submissions = {} create_record_for_dashboard(record['recid'], test_submissions, user) assert (test_submissions == { record_information['recid']: { 'metadata': { 'coordinator': { 'name': 'No coordinator' }, 'recid': record_information['recid'], 'role': [], 'start_date': hepsubmission.created, 'last_updated': hepsubmission.last_updated, 'title': u'My Journal Paper', 'versions': 1 }, 'stats': { 'attention': 0, 'passed': 0, 'todo': 0 }, 'status': 'todo' } }) test_submissions = { record_information['recid']: { "metadata": { "role": [] } } } create_record_for_dashboard(record['recid'], test_submissions, user) assert (test_submissions == { record_information['recid']: { "metadata": { "role": [[]] } } })
def index_record_ids(record_ids, index=None): """ Index records given in the argument. :param record_ids: [list of ints] list of record ids e.g. [1, 5, 2, 3] :param index: [string] name of the index. If None a default is used :return: list of indexed publication and data recids """ from hepdata.modules.records.utils.common import get_record_by_id docs = filter(None, [get_record_by_id(recid) for recid in record_ids]) to_index = [] indexed_result = {CFG_DATA_TYPE: [], CFG_PUB_TYPE: []} for doc in docs: if "related_publication" in doc: # Remove unnecessary fields if it's a data record for field in ["authors", "_additional_authors", "_first_author"]: if field in doc: del doc[field] enhance_data_document(doc) op_dict = { "index": { "_index": index, "_type": CFG_DATA_TYPE, "_id": doc["recid"], "_parent": str(doc["related_publication"]), } } indexed_result[CFG_DATA_TYPE].append(doc["recid"]) to_index.append(op_dict) else: author_docs = prepare_author_for_indexing(es, doc) to_index += author_docs enhance_publication_document(doc) op_dict = {"index": {"_index": index, "_type": CFG_PUB_TYPE, "_id": doc["recid"]}} indexed_result[CFG_PUB_TYPE].append(doc["recid"]) to_index.append(op_dict) if doc["last_updated"] is not None: doc["last_updated"] = parse(doc["last_updated"]).isoformat() to_index.append(doc) es.bulk(index=index, body=to_index, refresh=True) return indexed_result
def finalise(recid, publication_record=None, force_finalise=False): commit_message = request.form.get('message') # Update publication information from INSPIRE record before finalising. if not publication_record: record = get_record_by_id(recid) content, status = get_inspire_record_information(record['inspire_id']) if status == 'success': publication_record = update_record(recid, content) return do_finalise(recid, publication_record=publication_record, force_finalise=force_finalise, commit_message=commit_message, send_tweet=True)
def send_tweet(inspireids): """Send tweet announcing that records have been added or revised (in case it wasn't done automatically).""" processed_inspireids = parse_inspireids_from_string(inspireids) for inspireid in processed_inspireids: _cleaned_id = inspireid.replace("ins", "") submission = get_latest_hepsubmission(inspire_id=_cleaned_id, overall_status='finished') if submission: record = get_record_by_id(submission.publication_recid) site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') url = site_url + '/record/ins{0}'.format(record['inspire_id']) tweet(record['title'], record['collaborations'], url, record['version']) else: print("No records found for Inspire ID {}".format(inspireid))
def create_missing_datasubmission_records(): # Get submissions with missing IDs missing_submissions = DataSubmission.query \ .join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid) \ .filter( DataSubmission.associated_recid == None, DataSubmission.publication_inspire_id == None, DataSubmission.version == HEPSubmission.version, HEPSubmission.overall_status == 'finished') missing_submissions = missing_submissions.all() if not missing_submissions: print("No datasubmissions found with missing record or inspire ids.") return # Organise missing submissions by publication submissions_by_publication = {} for submission in missing_submissions: if submission.publication_recid in submissions_by_publication: submissions_by_publication[submission.publication_recid].append( submission) else: submissions_by_publication[submission.publication_recid] = [ submission ] # Loop through each publication for publication_recid, submissions in submissions_by_publication.items(): publication_record = get_record_by_id(publication_recid) current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.utcnow()) generated_record_ids = [] for submission in submissions: # Finalise each data submission that does not have a record finalise_datasubmission(current_time, {}, generated_record_ids, publication_record, publication_recid, submission, submission.version) # Register the datasubmission's DOI if not current_app.config.get('TESTING', False): generate_doi_for_table.delay(submission.doi) print(f"Generated DOI {submission.doi}") else: print(f"Would generate DOI {submission.doi}") # finalise_datasubmission does not commit, so commit once for each publication db.session.commit() # Reindex the publication and its updated datasubmissions index_record_ids([publication_recid] + generated_record_ids) push_data_keywords(pub_ids=[publication_recid])
def update_record(recid, ctx): """ Updates a record given a new dictionary. :param recid: :param ctx: :return: """ print('Recid is {}'.format(recid)) record = get_record_by_id(recid) for key, value in ctx.iteritems(): record[key] = value record.commit() db.session.commit() return record
def create_record_for_dashboard(record_id, submissions, primary_uploader=None, primary_reviewer=None, coordinator=None, user_role=None, status="todo"): if user_role is None: user_role = ["coordinator"] publication_record = get_record_by_id(int(record_id)) if publication_record is not None: if record_id not in submissions: hepdata_submission_record = get_latest_hepsubmission(publication_recid=record_id) submissions[record_id] = {} submissions[record_id]["metadata"] = {"recid": record_id, "role": user_role, "start_date": publication_record.created} submissions[record_id]["metadata"][ "versions"] = hepdata_submission_record.version submissions[record_id]["status"] = status submissions[record_id]["stats"] = {"passed": 0, "attention": 0, "todo": 0} if coordinator: submissions[record_id]["metadata"]["coordinator"] = { 'id': coordinator.id, 'name': coordinator.email, 'email': coordinator.email} submissions[record_id]["metadata"][ "show_coord_view"] = int(current_user.get_id()) == coordinator.id else: submissions[record_id]["metadata"]["coordinator"] = { 'name': 'No coordinator'} if "title" in publication_record: submissions[record_id]["metadata"]["title"] = \ publication_record['title'] if "inspire_id" not in publication_record or publication_record["inspire_id"] is None: submissions[record_id]["metadata"][ "requires_inspire_id"] = True else: # if it is, it's because the user has two roles for that # submission. So we should show them! if user_role not in submissions[record_id]["metadata"]["role"]: submissions[record_id]["metadata"]["role"].append(user_role)
def update_record(recid, ctx): """ Updates a record given a new dictionary. :param recid: :param ctx: :return: """ print('Updating record {}'.format(recid)) record = get_record_by_id(recid) for key, value in ctx.iteritems(): record[key] = value record["recid"] = recid record.commit() db.session.commit() return record
def list_submission_titles(current_user): query = _prepare_submission_query(current_user) hepdata_submission_records = query.order_by( HEPSubmission.last_updated.desc()).all() titles = [] for hepsubmission in hepdata_submission_records: publication_record = get_record_by_id( int(hepsubmission.publication_recid)) if publication_record: titles.append({ 'id': int(hepsubmission.publication_recid), 'title': publication_record['title'] }) return titles
def send_new_review_message_email(review, message, user): """ Sends a message to all uploaders and reviewers to tell them that a comment has been made on a record. :param review: :param message: :param user: :return: """ submission_participants = get_submission_participants_for_record( review.publication_recid) table_information = DataSubmission.query.filter_by( id=review.data_recid).one() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(review.publication_recid) destinations = [ participant.email for participant in submission_participants ] full_names = [ participant.full_name for participant in submission_participants ] message_body = render_template( 'hepdata_theme/email/review-message.html', name=', '.join(set(full_names)), actor=user.email, table_name=table_information.name, table_message=decode_string(message.message), article=review.publication_recid, title=record['title'], site_url=site_url, link=site_url + "/record/{0}".format(review.publication_recid), table_link=site_url + "/record/{0}?table={1}".format( review.publication_recid, table_information.name.replace( '+', '%2B'))) create_send_email_task( ','.join(set(destinations)), '[HEPData] Submission {0} ({1}) has a new review message'.format( review.publication_recid, table_information.name), message_body)
def send_tweet(inspireids): """ Send tweet announcing that records have been added or revised (in case it wasn't done automatically). :param inspireids: list of record IDs to tweet """ processed_inspireids = parse_inspireids_from_string(inspireids) for inspireid in processed_inspireids: _cleaned_id = inspireid.replace("ins", "") submission = get_latest_hepsubmission(inspire_id=_cleaned_id) if submission: record = get_record_by_id(submission.publication_recid) url = "http://www.hepdata.net/record/ins{0}".format( record['inspire_id']) tweet(record['title'], record['collaborations'], url, record['version']) else: print("No records found for Inspire ID {}".format(inspireid))
def send_coordinator_notification_email(recid, version, user, message=None): """ :param recid: :param user: user object :param message: message to send :return: """ hepsubmission = get_latest_hepsubmission(publication_recid=recid) coordinator = get_user_from_id(hepsubmission.coordinator) if not coordinator: raise NoParticipantsException() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(recid) name = coordinator.email coordinator_profile = UserProfile.get_by_userid(hepsubmission.coordinator) if coordinator_profile: name = coordinator_profile.full_name collaboration = _get_collaboration(hepsubmission.coordinator) message_body = render_template('hepdata_theme/email/passed_review.html', name=name, actor=user.email, collaboration=collaboration, article=recid, version=version, message=message, title=record['title'], site_url=site_url, link=site_url + "/record/{0}".format(recid), dashboard_link=site_url + "/dashboard") create_send_email_task( coordinator.email, '[HEPData] Submission {0} is ready to be finalised'.format(recid), message_body)
def get_pending_invitations_for_user(user): pending_invites = SubmissionParticipant.query.filter( SubmissionParticipant.email == user.email, or_(SubmissionParticipant.role == 'reviewer', SubmissionParticipant.role == 'uploader'), SubmissionParticipant.user_account == None ).all() result = [] for invite in pending_invites: publication_record = get_record_by_id(invite.publication_recid) hepsubmission = get_latest_hepsubmission(publication_recid=invite.publication_recid) coordinator = get_user_from_id(hepsubmission.coordinator) result.append( {'title': encode_string(publication_record['title'], 'utf-8'), 'invitation_cookie': invite.invitation_cookie, 'role': invite.role, 'coordinator': coordinator}) return result
def test_create_new_version(app, load_default_data, identifiers, mocker): hepsubmission = get_latest_hepsubmission(publication_recid=1) assert hepsubmission.version == 1 # Add an uploader uploader = SubmissionParticipant(publication_recid=1, email='*****@*****.**', role='uploader', status='primary') db.session.add(uploader) db.session.commit() user = User.query.first() # Mock `send_cookie_email` method send_cookie_mock = mocker.patch( 'hepdata.modules.records.api.send_cookie_email') # Create new version of valid finished record result = create_new_version(1, user, uploader_message="Hello!") assert result.status_code == 200 assert result.json == {'success': True, 'version': 2} # get_latest_hepsubmission should now return version 2 hepsubmission = get_latest_hepsubmission(publication_recid=1) assert hepsubmission.version == 2 assert hepsubmission.overall_status == 'todo' # Should have attempted to send uploader email send_cookie_mock.assert_called_with(uploader, get_record_by_id(1), message="Hello!", version=2) # Try creating a new version - should not work as status of most recent is 'todo' result, status_code = create_new_version(1, user) assert status_code == 400 assert result.json == { 'message': 'Rec id 1 is not finished so cannot create a new version' }
def send_new_upload_email(recid, user, message=None): """ :param recid: :param user: user object :param message: :return: """ submission_participants = SubmissionParticipant.query.filter_by( publication_recid=recid, status="primary", role="reviewer") if submission_participants.count() == 0: raise NoReviewersException() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(recid) for participant in submission_participants: invite_token = None if not participant.user_account: invite_token = participant.invitation_cookie message_body = render_template('hepdata_theme/email/upload.html', name=participant.full_name, actor=user.email, article=recid, message=message, invite_token=invite_token, role=participant.role, title=record['title'], site_url=site_url, link=site_url + "/record/{0}".format(recid)) create_send_email_task( participant.email, '[HEPData] Submission {0} has a new upload available for you to review' .format(recid), message_body)
def test_prepare_submissions_admin(app, load_submission): with app.app_context(): record_information = create_record({ 'journal_info': 'Phys. Letts', 'title': 'My Journal Paper', 'inspire_id': '1487726' }) get_or_create_hepsubmission(record_information['recid']) record = get_record_by_id(record_information['recid']) role = Role(name='admin') user = User(email='*****@*****.**', password='******', active=True, id=101, roles=[role]) submissions = prepare_submissions(user) assert (len(submissions) == 1) assert (submissions[str(record_information['recid'])] == { 'metadata': { 'coordinator': { 'email': u'*****@*****.**', 'name': u'*****@*****.**', 'id': 1 }, 'recid': str(record_information['recid']), 'role': ['coordinator'], 'show_coord_view': False, 'start_date': record.created, 'title': u'My Journal Paper', 'versions': 1 }, 'stats': { 'attention': 0L, 'passed': 0L, 'todo': 0L }, 'status': u'todo' })
def send_new_upload_email(recid, user, message=None): """ :param action: e.g. upload or review_message :param hepsubmission: submission information :param user: user object :return: """ submission_participants = SubmissionParticipant.query.filter_by( publication_recid=recid, status="primary", role="reviewer") if submission_participants.count() == 0: raise NoReviewersException() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(recid) for participant in submission_participants: invite_token = None if not participant.user_account: invite_token = participant.invitation_cookie message_body = render_template('hepdata_theme/email/upload.html', name=participant.full_name, actor=user.email, article=recid, message=message, invite_token=invite_token, role=participant.role, title=record['title'], site_url=site_url, link=site_url + "/record/{0}" .format(recid)) create_send_email_task(participant.email, '[HEPData] Submission {0} has a new upload available for you to review.'.format(recid), message_body)
def send_new_review_message_email(review, message, user): """ Sends a message to all uploaders to tell them that a comment has been made on a record :param hepsubmission: :param review: :param user: :return: """ submission_participants = SubmissionParticipant.query.filter_by( publication_recid=review.publication_recid, status="primary", role="uploader") table_information = DataSubmission.query \ .filter_by(id=review.data_recid).one() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(review.publication_recid) for participant in submission_participants: message_body = render_template( 'hepdata_theme/email/review-message.html', name=participant.full_name, actor=user.email, table_name=table_information.name, table_message=message.message, article=review.publication_recid, title=record['title'], site_url=site_url, link=site_url + "/record/{0}".format(review.publication_recid)) create_send_email_task( participant.email, '[HEPData] Submission {0} has a new review message.' \ .format(review.publication_recid), message_body)
def attach_information_to_record(recid): """ Given an INSPIRE data representation, this will process the data, and update information for a given record id with the contents. :return: """ inspire_id = request.form["inspire_id"] content, status = get_inspire_record_information(inspire_id) content["inspire_id"] = inspire_id record = get_record_by_id(recid) if record is not None: content["recid"] = recid patch = jsonpatch.JsonPatch.from_diff(record, content) record = record.patch(patch=patch) record.commit() db.session.commit() return jsonify({"status": "success"}) else: return jsonify({"status": "failed", "message": "No record with that recid was found."})
def send_finalised_email(hepsubmission): record = get_record_by_id(hepsubmission.publication_recid) notify_participants(hepsubmission, record) notify_subscribers(hepsubmission, record)
def do_finalise(recid, publication_record=None, force_finalise=False, commit_message=None, send_tweet=False, update=False): """ Creates record SIP for each data record with a link to the associated publication :param synchronous: if true then workflow execution and creation is waited on, then everything is indexed in one go. If False, object creation is asynchronous, however reindexing is not performed. This is only really useful for the full migration of content. """ hep_submission = HEPSubmission.query.filter_by( publication_recid=recid, overall_status="todo").first() print('Finalising record {}'.format(recid)) generated_record_ids = [] if hep_submission \ and (force_finalise or hep_submission.coordinator == int(current_user.get_id())): submissions = DataSubmission.query.filter_by( publication_recid=recid, version=hep_submission.version).all() version = hep_submission.version existing_submissions = {} if hep_submission.version > 1 or update: # we need to determine which are the existing record ids. existing_data_records = get_records_matching_field( 'related_publication', recid, doc_type=CFG_DATA_TYPE) for record in existing_data_records["hits"]["hits"]: if "recid" in record["_source"]: existing_submissions[record["_source"]["title"]] = \ record["_source"]["recid"] delete_item_from_index(record["_id"], doc_type=CFG_DATA_TYPE, parent=record["_source"]["related_publication"]) current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.now()) for submission in submissions: finalise_datasubmission(current_time, existing_submissions, generated_record_ids, publication_record, recid, submission, version) try: record = get_record_by_id(recid) # If we have a commit message, then we have a record update. # We will store the commit message and also update the # last_updated flag for the record. record['hepdata_doi'] = hep_submission.doi if commit_message: # On a revision, the last updated date will # be the current date. hep_submission.last_updated = datetime.now() commit_record = RecordVersionCommitMessage( recid=recid, version=version, message=str(commit_message)) db.session.add(commit_record) record['last_updated'] = datetime.strftime( hep_submission.last_updated, '%Y-%m-%d %H:%M:%S') record['version'] = version record.commit() hep_submission.inspire_id = record['inspire_id'] hep_submission.overall_status = "finished" db.session.add(hep_submission) db.session.commit() create_celery_app(current_app) # only mint DOIs if not testing. if not current_app.config.get('TESTING', False) and not current_app.config.get('NO_DOI_MINTING', False): for submission in submissions: generate_doi_for_data_submission.delay(submission.id, submission.version) generate_doi_for_submission.delay(recid, version) # Reindex everything. index_record_ids([recid] + generated_record_ids) push_data_keywords(pub_ids=[recid]) admin_indexer = AdminIndexer() admin_indexer.index_submission(hep_submission) send_finalised_email(hep_submission) for file_format in ['csv', 'yoda', 'root']: convert_and_store.delay(hep_submission.inspire_id, file_format, force=True) if send_tweet: tweet(record.get('title'), record.get('collaborations'), "http://www.hepdata.net/record/ins{0}".format(record.get('inspire_id'))) return json.dumps({"success": True, "recid": recid, "data_count": len(submissions), "generated_records": generated_record_ids}) except NoResultFound: print('No record found to update. Which is super strange.') else: return json.dumps( {"success": False, "recid": recid, "errors": ["You do not have permission to finalise this " "submission. Only coordinators can do that."]})
def update_record_info(inspire_id, send_email=False): """Update publication information from INSPIRE for a specific record.""" if inspire_id is None: log.error("Inspire ID is None") return 'Inspire ID is None' inspire_id = inspire_id.replace("ins", "") hep_submission = get_latest_hepsubmission(inspire_id=inspire_id) if hep_submission is None: log.warning("Failed to retrieve HEPData submission for Inspire ID {0}".format(inspire_id)) return 'No HEPData submission' publication_recid = hep_submission.publication_recid log.info("Updating recid {} with information from Inspire record {}".format(publication_recid, inspire_id)) updated_inspire_record_information, status = get_inspire_record_information(inspire_id) if status == 'success': # Also need to update publication information for data records. data_submissions = DataSubmission.query.filter_by( publication_recid=publication_recid, version=hep_submission.version ).order_by(DataSubmission.id.asc()) record_ids = [publication_recid] # list of record IDs for data_submission in data_submissions: record_ids.append(data_submission.associated_recid) same_information = {} for index, recid in enumerate(record_ids): if index == 0: updated_record_information = updated_inspire_record_information else: # Only update selected keys for data records. updated_record_information = { key: updated_inspire_record_information[key] for key in ( 'authors', 'creation_date', 'journal_info', 'collaborations' ) } record_information = get_record_by_id(recid) same_information[recid] = True for key, value in updated_record_information.items(): if key not in record_information or record_information[key] != value: log.debug('For recid {}, key {} has new value {}'.format(recid, key, value)) same_information[recid] = False update_record(recid, updated_record_information) break log.info('For recid {}, information needs to be updated: {}'.format(recid, str(not(same_information[recid])))) if all(same for same in same_information.values()): return 'No update needed' else: log.warning("Failed to retrieve publication information for Inspire record {0}".format(inspire_id)) return 'Invalid Inspire ID' if hep_submission.overall_status == 'finished': index_record_ids(record_ids) # index for Elasticsearch push_data_keywords(pub_ids=[recid]) if not TESTING: generate_dois_for_submission.delay(inspire_id=inspire_id) # update metadata stored in DataCite if send_email: record_information = get_record_by_id(publication_recid) notify_publication_update(hep_submission, record_information) # send email to all participants return 'Success'
def test_import_record(app): all_submissions = HEPSubmission.query.all() assert len(all_submissions) == 0 data_submissions = DataSubmission.query.all() assert len(data_submissions) == 0 # Import a record and check it's been added to the db inspire_id = '1824424' result = _import_record(inspire_id) assert result is True all_submissions = HEPSubmission.query.all() assert len(all_submissions) == 1 hep_submission = all_submissions[0] assert hep_submission.publication_recid == 1 assert hep_submission.inspire_id == inspire_id assert hep_submission.overall_status == 'finished' last_updated = hep_submission.last_updated record = get_record_by_id(1) assert record['title'] == 'Measurement of the production cross section of 31 GeV/$c$ protons on carbon via beam attenuation in a 90-cm-long target' assert record['inspire_id'] == inspire_id assert record['abstract'].startswith('The production cross section of 30.92 GeV/$c$ protons on carbon is measured by') data_submissions = DataSubmission.query.all() assert len(data_submissions) == 2 # Try the import again - should not update result = _import_record(inspire_id) assert result is False # Retry with update_existing=True result = _import_record(inspire_id, update_existing=True) assert result is True updated_submission = get_latest_hepsubmission(publication_recid=1) assert updated_submission.last_updated > last_updated # Try an old inspire id which uses the old schema old_inspire_id = '944937' result = _import_record(old_inspire_id) assert result is True all_submissions = HEPSubmission.query.all() assert len(all_submissions) == 2 data_submissions = DataSubmission.query.all() assert len(data_submissions) == 3 record = get_record_by_id(all_submissions[1].publication_recid) assert record['title'] == 'The Production of Charged Photomesons from Deuterium and Hydrogen. I' assert record['inspire_id'] == old_inspire_id assert record['abstract'].startswith('Hydrogen and deuterium gases have been bombarded in a gas target at a temperature of 77°K') # Try an invalid inspire id assert _import_record('thisisinvalid') is False # Mock errors with download with requests_mock.Mocker(real_http=True) as mock: # 404 mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id), status_code=404) result = _import_record(inspire_id, update_existing=True) assert result is False # Text not zip mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id), text='This is not a zip') result = _import_record(inspire_id, update_existing=True) assert result is False # SocketError mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id), exc=requests.exceptions.ConnectTimeout("mock error message")) result = _import_record(inspire_id, update_existing=True) assert result is False # Send an invalid zip base_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(base_dir, 'test_data/submission_invalid_symlink.tgz') with open(file_path, 'rb') as f: mock.get('https://hepdata.net/download/submission/ins{0}/original'.format(inspire_id), headers={'content-type': 'application/zip'}, body=f) result = _import_record(inspire_id, update_existing=True) assert result is False
def index_record_ids(record_ids, index=None): """ Index records given in the argument. :param record_ids: [list of ints] list of record ids e.g. [1, 5, 2, 3] :param index: [string] name of the index. If None a default is used :return: list of indexed publication and data recids """ from hepdata.modules.records.utils.common import get_record_by_id docs = filter(None, [get_record_by_id(recid) for recid in record_ids]) existing_record_ids = [doc['recid'] for doc in docs] print('Indexing existing record IDs:', existing_record_ids) to_index = [] indexed_result = {CFG_DATA_TYPE: [], CFG_PUB_TYPE: []} for doc in docs: if 'related_publication' in doc: # Remove unnecessary fields if it's a data record for field in ['authors', '_additional_authors', '_first_author']: if field in doc: del doc[field] enhance_data_document(doc) op_dict = { "index": { "_index": index, "_type": CFG_DATA_TYPE, "_id": doc['recid'], "_parent": str(doc['related_publication']) } } indexed_result[CFG_DATA_TYPE].append(doc['recid']) to_index.append(op_dict) else: if 'version' not in doc: print('Skipping unfinished record ID {}'.format(doc['recid'])) continue author_docs = prepare_author_for_indexing(doc) to_index += author_docs enhance_publication_document(doc) op_dict = { "index": { "_index": index, "_type": CFG_PUB_TYPE, "_id": doc['recid'] } } indexed_result[CFG_PUB_TYPE].append(doc['recid']) to_index.append(op_dict) if doc["last_updated"] is not None: doc["last_updated"] = parse(doc["last_updated"]).isoformat() to_index.append(doc) if to_index: es.bulk(index=index, body=to_index, refresh=True) return indexed_result
def remove_submission(record_id): """ Removes the database entries related to a record. :param record_id: :return: True if Successful, False if the record does not exist. """ hepdata_submissions = HEPSubmission.query.filter_by( publication_recid=record_id).all() try: try: for hepdata_submission in hepdata_submissions: db.session.delete(hepdata_submission) except NoResultFound as nrf: print(nrf.args) submissions = DataSubmission.query.filter_by( publication_recid=record_id).all() reviews = DataReview.query.filter_by( publication_recid=record_id).all() for review in reviews: db.session.delete(review) for submission in submissions: resource = DataResource.query.filter_by( id=submission.data_file).first() db.session.delete(submission) if resource: db.session.delete(resource) try: SubmissionParticipant.query.filter_by( publication_recid=record_id).delete() except Exception: print("Unable to find a submission participant for {0}".format(record_id)) try: record = get_record_by_id(record_id) data_records = get_records_matching_field( 'related_publication', record_id, doc_type=CFG_DATA_TYPE) if 'hits' in data_records: for data_record in data_records['hits']['hits']: data_record_obj = get_record_by_id(data_record['_source']['recid']) if data_record_obj: data_record_obj.delete() if record: record.delete() except PIDDoesNotExistError as e: print('No record entry exists for {0}. Proceeding to delete other files.'.format(record_id)) db.session.commit() db.session.flush() return True except Exception as e: db.session.rollback() raise e