def create_container_doi(hep_submission, data_submissions, publication_info): """ Creates the payload to wrap the whole submission :param hep_submission: :param data_submissions: :param publication_info: :return: """ if hep_submission.doi is None: reserve_doi_for_hepsubmission(hep_submission) reserve_dois_for_data_submissions(data_submissions) version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version) hep_submission.data_abstract = decode_string(hep_submission.data_abstract) publication_info['title'] = decode_string(publication_info['title']) publication_info['abstract'] = decode_string(publication_info['abstract']) base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=hep_submission.doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=version_doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) # Register DOI for the version, and update the base DOI to resolve to the latest submission version. register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']), base_xml, publication_info['uuid']) register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format( publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def generate_doi_for_submission(recid, version): data_submissions = DataSubmission.query.filter_by(publication_recid=recid, version=version).order_by( DataSubmission.id.asc()) hep_submission = HEPSubmission.query.filter_by(publication_recid=recid).first() publication_info = get_record_by_id(recid) version_doi = hep_submission.doi + ".v{0}".format(hep_submission.version) hep_submission.data_abstract = decode_string(hep_submission.data_abstract) publication_info['title'] = decode_string(publication_info['title']) publication_info['abstract'] = decode_string(publication_info['abstract']) base_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=hep_submission.doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) version_xml = render_template('hepdata_records/formats/datacite/datacite_container_submission.xml', doi=version_doi, overall_submission=hep_submission, data_submissions=data_submissions, publication_info=publication_info) # Register DOI for the version, and update the base DOI to resolve to the latest submission version. register_doi(hep_submission.doi, 'http://www.hepdata.net/record/ins{0}'.format(publication_info['inspire_id']), base_xml, publication_info['uuid']) register_doi(version_doi, 'http://www.hepdata.net/record/ins{0}?version={1}'.format( publication_info['inspire_id'], hep_submission.version), version_xml, publication_info['uuid'])
def create_data_doi(hep_submission, data_submission, publication_info): """ Generate DOI record for a data record :param data_submission_id: :param version: :return: """ data_file = DataResource.query.filter_by(id=data_submission.data_file).first() license = None if data_file: if data_file.file_license: license = License.query.filter_by(id=data_file.file_license).first() xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml', doi=data_submission.doi, table_name=decode_string(data_submission.name), table_description=decode_string(data_submission.description), overall_submission=hep_submission, data_submission=data_submission, license=license, publication_info=publication_info) register_doi(data_submission.doi, 'http://www.hepdata.net/record/ins{0}?version={1}&table={2}'.format( publication_info['inspire_id'], data_submission.version, data_submission.name), xml, publication_info['uuid'])
def generate_doi_for_data_submission(data_submission_id, version): data_submission = DataSubmission.query.filter_by(id=data_submission_id).first() hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first() publication_info = get_record_by_id(data_submission.publication_recid) data_file = DataResource.query.filter_by(id=data_submission.data_file).first() license = None if data_file: if data_file.file_license: license = License.query.filter_by(id=data_file.file_license).first() xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml', doi=data_submission.doi, table_name=decode_string(data_submission.name), table_description=decode_string(data_submission.description), overall_submission=hep_submission, data_submission=data_submission, license=license, publication_info=publication_info) register_doi(data_submission.doi, 'http://www.hepdata.net/record/{0}?version={1}&table={2}'.format( hep_submission.publication_recid, data_submission.version, data_submission.name), xml, publication_info['uuid'])
def generate_doi_for_data_submission(data_submission_id, version): data_submission = DataSubmission.query.filter_by(id=data_submission_id).first() hep_submission = HEPSubmission.query.filter_by(publication_recid=data_submission.publication_recid).first() publication_info = get_record_by_id(data_submission.publication_recid) data_file = DataResource.query.filter_by(id=data_submission.data_file).first() license = None if data_file: if data_file.file_license: license = License.query.filter_by(id=data_file.file_license).first() xml = render_template('hepdata_records/formats/datacite/datacite_data_record.xml', doi=data_submission.doi, table_name=decode_string(data_submission.name), table_description=decode_string(data_submission.description), overall_submission=hep_submission, data_submission=data_submission, license=license, publication_info=publication_info) register_doi(data_submission.doi, 'http://www.hepdata.net/record/ins{0}?version={1}&table={2}'.format( publication_info['inspire_id'], data_submission.version, data_submission.name), xml, publication_info['uuid'])
def test_parser(inspire_id, title, creation_date, year, subject_area): content, status = get_inspire_record_information(inspire_id) assert decode_string(content["title"]) == decode_string(title) assert content["creation_date"] == creation_date assert int(content["year"]) == year if subject_area is not None: assert content["subject_area"] == subject_area
def test_parser(): test_data = [ { "inspire_id": "1245023", "title": "High-statistics study of $K^0_S$ pair " "production in two-photon collisions", "creation_date": "2013-07-29", "year": 2013, }, { "inspire_id": "1183818", "title": "Measurements of the pseudorapidity dependence " "of the total transverse energy " "in proton-proton " "collisions at $\sqrt{s}=7$ TeV with ATLAS", "creation_date": "2012-08-01", "year": 2012, }, { "inspire_id": "1407276", "title": "Elastic scattering of negative pions by protons at 2 BeV/c", "creation_date": "1963-01-01", "year": 1963, }, { "inspire_id": "44234", "title": "DIFFERENTIAL ELASTIC PION-PROTON SCATTERING AT 600-MEV, 650-MEV and 750-MEV", "creation_date": "2006-04-11", "year": 2006, }, { "inspire_id": "1187688", "title": "Mesure de la polarisation du proton de recul dans la diffusion élastique " "pi+- p entre 550 et 1025 MeV", "creation_date": "1970-01-01", "year": 1970, }, { "inspire_id": "67677", "title": "INELASTIC ELECTRON - DEUTERON SCATTERING AT HIGH-ENERGIES", "creation_date": "1971-01-01", "year": 1971, }, ] for test in test_data: content, status = get_inspire_record_information(test["inspire_id"]) assert decode_string(content["title"]) == decode_string(test["title"]) assert content["creation_date"] == test["creation_date"] assert int(content["year"]) == test["year"]
def process_data_tables(ctx, data_record_query, first_data_id, data_table=None): data_table_metadata = OrderedDict() ctx['show_upload_area'] = False if ctx['show_upload_widget'] and data_record_query.count() == 0: ctx['show_upload_area'] = True elif data_record_query.count() > 0: record_submissions = data_record_query.all() for submission_record in record_submissions: processed_name = "".join(submission_record.name.split()) data_table_metadata[submission_record.id] = { "id": submission_record.id, "processed_name": processed_name, "name": submission_record.name, "location": submission_record.location_in_publication, "doi": submission_record.doi, "description": decode_string( truncate_string(submission_record.description, 20))} if first_data_id == -1: first_data_id = submission_record.id if data_table: if submission_record.name == data_table: first_data_id = submission_record.id return data_table_metadata, first_data_id
def get_pending_invitations_for_user(user): """ Returns pending invites for upload or review of records. :param user: User object :return: array of pending invites """ pending_invites = SubmissionParticipant.query.filter( func.lower(SubmissionParticipant.email) == func.lower(user.email), or_(SubmissionParticipant.role == 'reviewer', SubmissionParticipant.role == 'uploader'), SubmissionParticipant.status == 'primary', SubmissionParticipant.user_account == None).all() result = [] for invite in pending_invites: publication_record = get_record_by_id(invite.publication_recid) hepsubmission = get_latest_hepsubmission( publication_recid=invite.publication_recid) coordinator = get_user_from_id(hepsubmission.coordinator) result.append({ 'title': decode_string(publication_record['title']), 'invitation_cookie': invite.invitation_cookie, 'role': invite.role, 'coordinator': coordinator }) return result
def sandbox(): current_id = current_user.get_id() submissions = HEPSubmission.query.filter_by(coordinator=current_id, overall_status='sandbox').order_by( HEPSubmission.id.desc()).all() for submission in submissions: submission.data_abstract = decode_string(submission.data_abstract) return render_template('hepdata_records/sandbox.html', ctx={"submissions": submissions})
def test_parser(): test_data = [{"inspire_id": "1245023", "title": "High-statistics study of $K^0_S$ pair " "production in two-photon collisions", "creation_date": "2013-07-29", "year": 2013, "subject_area": ['HEP Experiment']}, {"inspire_id": "1183818", "title": "Measurements of the pseudorapidity dependence " "of the total transverse energy " "in proton-proton " "collisions at $\sqrt{s}=7$ TeV with ATLAS", "creation_date": "2012-08-01", "year": 2012, "subject_area": ["HEP Experiment"]}, {"inspire_id": "1407276", "title": "Elastic scattering of negative pions by protons at 2 BeV/c", "creation_date": "1963-01-01", "year": 1963}, {"inspire_id": "44234", "title": "DIFFERENTIAL ELASTIC PION-PROTON SCATTERING AT 600-MEV, 650-MEV and 750-MEV", "creation_date": "2006-04-11", "year": 2006}, {"inspire_id": "1187688", "title": "Mesure de la polarisation du proton de recul dans la diffusion élastique " "pi+- p entre 550 et 1025 MeV", "creation_date": "1970-01-01", "year": 1970}, {"inspire_id": "67677", "title": "INELASTIC ELECTRON - DEUTERON SCATTERING AT HIGH-ENERGIES", "creation_date": "1971-01-01", "year": 1971 } ] for test in test_data: content, status = get_inspire_record_information( test["inspire_id"]) assert (decode_string(content["title"]) == decode_string(test["title"])) assert (content["creation_date"] == test["creation_date"]) assert (int(content["year"]) == test["year"]) if 'subject_area' in test: assert (content["subject_area"] == test["subject_area"])
def add_data_review_messsage(publication_recid, data_recid): """ Adds a new review message for a data submission. :param publication_recid: :param data_recid: """ trace = [] message = encode_string(request.form.get('message', '')) version = request.form['version'] userid = current_user.get_id() try: datareview_query = DataReview.query.filter_by(data_recid=data_recid, version=version) # if the data review is not already created, create one. try: data_review_record = datareview_query.one() trace.append("adding data review record") except: data_review_record = create_data_review(data_recid, publication_recid) trace.append("created a new data review record") data_review_message = Message(user=userid, message=message) data_review_record.messages.append(data_review_message) db.session.commit() current_user_obj = get_user_from_id(userid) update_action_for_submission_participant(publication_recid, userid, 'reviewer') send_new_review_message_email(data_review_record, data_review_message, current_user_obj) return json.dumps( { "publication_recid": data_review_record.publication_recid, "data_recid": data_review_record.data_recid, "status": data_review_record.status, "message": decode_string(data_review_message.message), "post_time": data_review_message.creation_date, 'user': current_user_obj.email }, default=default_time) except Exception as e: db.session.rollback() raise e
def process_year_facet(request, facets): url_path = modify_query('.search', **{'date': None}) year_facet = get_session_item(url_path) if len(year_facet) == 0 or (request.full_path[:-1] == url_path or request.full_path == url_path): # we update the facet if there is no value stored in the session, # or if the base url is the same as the stripped url year_facet = get_facet(facets, 'Date') if year_facet: year_facet = {decode_string(json.dumps(year_facet))} set_session_item(url_path, year_facet) if year_facet and len(year_facet) > 0: year_facet = list(year_facet)[0] return year_facet
def get_resource(resource_id): """ Attempts to find any HTML resources to be displayed for a record in the event that it does not have proper data records included. :param recid: publication record id :return: json dictionary containing any HTML files to show. """ resource = DataResource.query.filter_by(id=resource_id) view_mode = bool(request.args.get('view', False)) if resource.count() > 0: resource_obj = resource.first() if view_mode: return send_file(resource_obj.file_location, as_attachment=True) elif 'html' in resource_obj.file_location and 'http' not in resource_obj.file_location.lower( ): with open(resource_obj.file_location, 'r') as resource_file: html = resource_file.read() return html else: contents = '' if resource_obj.file_type.lower() not in IMAGE_TYPES: print("Resource is at: " + resource_obj.file_location) with open(resource_obj.file_location, 'r') as resource_file: contents = resource_file.read() # Don't return file contents if they contain a null byte. if '\0' in contents: contents = 'Binary' return jsonify({ "location": '/record/resource/{0}?view=true'.format(resource_obj.id), 'type': resource_obj.file_type, 'description': resource_obj.file_description, 'file_contents': decode_string(contents) }) else: log.error("Unable to find resource %d.", resource_id) return abort(404)
def send_new_review_message_email(review, message, user): """ Sends a message to all uploaders and reviewers to tell them that a comment has been made on a record. :param review: :param message: :param user: :return: """ submission_participants = get_submission_participants_for_record( review.publication_recid) table_information = DataSubmission.query.filter_by( id=review.data_recid).one() site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net') record = get_record_by_id(review.publication_recid) destinations = [ participant.email for participant in submission_participants ] full_names = [ participant.full_name for participant in submission_participants ] message_body = render_template( 'hepdata_theme/email/review-message.html', name=', '.join(set(full_names)), actor=user.email, table_name=table_information.name, table_message=decode_string(message.message), article=review.publication_recid, title=record['title'], site_url=site_url, link=site_url + "/record/{0}".format(review.publication_recid), table_link=site_url + "/record/{0}?table={1}".format( review.publication_recid, table_information.name.replace( '+', '%2B'))) create_send_email_task( ','.join(set(destinations)), '[HEPData] Submission {0} ({1}) has a new review message'.format( review.publication_recid, table_information.name), message_body)
def tweet(title, collaborations, url, version=1): """ Announce addition or revision of a HEPData record on Twitter. :param title: :param collaborations: :param url: :param version: :return: """ if USE_TWITTER: OAUTH_TOKEN = current_app.config['OAUTH_TOKEN'] OAUTH_SECRET = current_app.config['OAUTH_SECRET'] CONSUMER_KEY = current_app.config['CONSUMER_KEY'] CONSUMER_SECRET = current_app.config['CONSUMER_SECRET'] if not OAUTH_TOKEN or not OAUTH_SECRET or not CONSUMER_KEY or not CONSUMER_SECRET: # log this error print("Twitter credentials must be supplied!") else: twitter = Twitter(auth=OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY, CONSUMER_SECRET)) cleaned_title = decode_string( encode_string(title)) # in case of binary characters in title cleaned_title = replace( cleaned_title) # use UnicodeIt to replace LaTeX expressions cleaned_title = cleanup_latex( cleaned_title) # remove some remaining LaTeX encodings words = len(cleaned_title.split()) # Try to tweet with complete paper title. # If tweet exceeds 280 characters, keep trying with one less word each time. tweeted = False while words and not tweeted: try: if version == 1: status = "Added{0} data on \"{1}\" to {2}".format( get_collaboration_string(collaborations), truncate_string(cleaned_title, words), url) else: status = "Revised{0} data on \"{1}\" at {2}?version={3}".format( get_collaboration_string(collaborations), truncate_string(cleaned_title, words), url, version) twitter.statuses.update(status=status) tweeted = True print("Tweeted: {}".format(status)) except Exception as e: # It would be nice to get a stack trace here if e.e.code == 403: error = json.loads(e.response_data.decode('utf8')) if error["errors"][0][ "code"] == 186: # Status is over 140 characters. words = words - 1 # Try again with one less word. else: break else: break if not tweeted: print(e.__str__()) print("(P) Failed to post tweet for record {0}".format(url))
def format_submission(recid, record, version, version_count, hepdata_submission, data_table=None): """ Performs all the processing of the record to be displayed. :param recid: :param record: :param version: :param version_count: :param hepdata_submission: :param data_table: :return: """ ctx = {} if hepdata_submission is not None: ctx['site_url'] = current_app.config.get('SITE_URL', 'https://www.hepdata.net') ctx['record'] = record ctx["version_count"] = version_count if version is not -1: ctx["version"] = version else: # we get the latest version by default ctx["version"] = version_count if record is not None: if "collaborations" in record and type( record['collaborations']) is not list: collaborations = [ x.strip() for x in record["collaborations"].split(",") ] ctx['record']['collaborations'] = collaborations authors = record.get('authors', None) create_breadcrumb_text(authors, ctx, record) get_commit_message(ctx, recid) if authors: truncate_author_list(record) determine_user_privileges(recid, ctx) else: ctx['record'] = {} determine_user_privileges(recid, ctx) ctx['show_upload_widget'] = True ctx['show_review_widget'] = False ctx['reviewer_count'] = SubmissionParticipant.query.filter_by( publication_recid=recid, status="primary", role="reviewer").count() ctx['reviewers_notified'] = hepdata_submission.reviewers_notified ctx['record']['last_updated'] = hepdata_submission.last_updated ctx['record']['hepdata_doi'] = "{0}".format(hepdata_submission.doi) if ctx['version'] > 1: ctx['record']['hepdata_doi'] += ".v{0}".format(ctx['version']) ctx['recid'] = recid ctx["status"] = hepdata_submission.overall_status ctx['record']['data_abstract'] = decode_string( hepdata_submission.data_abstract) extract_journal_info(record) if hepdata_submission.overall_status != 'finished' and ctx[ "version_count"] > 0: if not (ctx['show_review_widget'] or ctx['show_upload_widget'] or ctx['is_submission_coordinator_or_admin']): # we show the latest approved version. ctx["version"] -= 1 ctx["version_count"] -= 1 ctx['additional_resources'] = submission_has_resources( hepdata_submission) # query for a related data submission data_record_query = DataSubmission.query.filter_by( publication_recid=recid, version=ctx["version"]).order_by(DataSubmission.id.asc()) format_tables(ctx, data_record_query, data_table, recid) ctx['access_count'] = get_count(recid) ctx['mode'] = 'record' ctx['coordinator'] = hepdata_submission.coordinator ctx['coordinators'] = get_coordinators_in_system() ctx['record'].pop('authors', None) return ctx
def format_submission(recid, record, version, version_count, hepdata_submission, data_table=None): """ Performs all the processing of the record to be display :param recid: :param record: :param version: :param hepdata_submission: :param data_table: :return: """ ctx = {} if hepdata_submission is not None: ctx['record'] = record ctx["version_count"] = version_count if version is not -1: ctx["version"] = version else: # we get the latest version by default ctx["version"] = version_count if record is not None: if "collaborations" in record and type(record['collaborations']) is not list: collaborations = [x.strip() for x in record["collaborations"].split(",")] ctx['record']['collaborations'] = collaborations authors = record.get('authors', None) create_breadcrumb_text(authors, ctx, record) get_commit_message(ctx, recid) if authors: truncate_author_list(record) determine_user_privileges(recid, ctx) else: ctx['record'] = {} determine_user_privileges(recid, ctx) ctx['show_upload_widget'] = True ctx['show_review_widget'] = False ctx['reviewer_count'] = SubmissionParticipant.query.filter_by( publication_recid=recid, status="primary", role="reviewer").count() ctx['reviewers_notified'] = hepdata_submission.reviewers_notified ctx['record']['last_updated'] = hepdata_submission.last_updated ctx['record']['hepdata_doi'] = "{0}".format(hepdata_submission.doi) if ctx['version'] > 1: ctx['record']['hepdata_doi'] += ".v{0}".format(ctx['version']) ctx['recid'] = recid ctx["status"] = hepdata_submission.overall_status ctx['record']['data_abstract'] = decode_string(hepdata_submission.data_abstract) extract_journal_info(record) if hepdata_submission.overall_status != 'finished' and ctx["version_count"] > 0: if not (ctx['show_review_widget'] or ctx['show_upload_widget'] or ctx['is_submission_coordinator_or_admin']): # we show the latest approved version. ctx["version"] -= 1 ctx["version_count"] -= 1 ctx['additional_resources'] = submission_has_resources(hepdata_submission) # query for a related data submission data_record_query = DataSubmission.query.filter_by( publication_recid=recid, version=ctx["version"]).order_by(DataSubmission.id.asc()) format_tables(ctx, data_record_query, data_table, recid) ctx['access_count'] = get_count(recid) ctx['mode'] = 'record' ctx['coordinator'] = hepdata_submission.coordinator ctx['coordinators'] = get_coordinators_in_system() ctx['record'].pop('authors', None) return ctx