def convert_endpoint(): """ Endpoint for general conversion, the file is passed as a GET parameter and options ('from=' & 'to=') are query string arguments. TO DO: is this function used anywhere? If not, it can probably be removed. :return: display_error or send_file depending on success of conversion """ input_format, output_format = request.args.get( 'from', 'oldhepdata'), request.args.get('to', 'yaml') if input_format not in ['yaml', 'oldhepdata'] or \ output_format not in ['root', 'yoda', 'csv', 'yaml']: return display_error( title="Chosen formats are not supported", description="Supported input formats: oldhepdata, yaml\n" + "Supported output formats: root, yoda, csv") fileobject = request.files.get('file') if not fileobject or not fileobject.filename.endswith('.zip'): print("Fileobject: " + str(fileobject)) return display_error( title="Please send a zip file for conversion", description= "No file has been sent or it does not have a zip extension") filename = secure_filename(fileobject.filename) timestamp = str(int(round(time.time()))) input_archive = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.zip') fileobject.save(input_archive) options = { 'input_format': input_format, 'output_format': output_format, 'filename': filename[:-4], } output_file = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.tar.gz') try: conversion_result = convert_zip_archive(input_archive, output_file, options) except Error as error: # hepdata_converter_ws_client.Error return display_error(title='Report concerns to [email protected]', description=str(error)) finally: os.remove(input_archive) if not conversion_result: return display_error( title="Your archive does not contain necessary files", description= "For YAML conversions a submission.yaml file is necessary," " and for conversions from the oldhepdata format" " a file with .oldhepdata extension is required.") return send_file(conversion_result, as_attachment=True)
def convert_endpoint(): """ Endpoint for general conversion, the file is passed as a GET parameter and options ('from=' & 'to=') are query string arguments. """ input_format, output_format = request.args.get('from', 'oldhepdata'), request.args.get('to', 'yaml') if input_format not in ['yaml', 'oldhepdata'] or \ output_format not in ['root', 'yoda', 'csv', 'yaml']: return display_error( title="Chosen formats are not supported", description="Supported input formats: oldhepdata, yaml\n" + "Supported output formats: root, yoda, csv" ) fileobject = request.files.get('file') if not fileobject or not fileobject.filename.endswith('.zip'): print("Fileobject: " + str(fileobject)) return display_error( title="Please send a zip file for conversion", description="No file has been sent or it does not have a zip extension" ) filename = secure_filename(fileobject.filename) timestamp = str(int(round(time.time()))) input_archive = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.zip') fileobject.save(input_archive) options = { 'input_format': input_format, 'output_format': output_format, 'filename': filename[:-4], } output_file = os.path.join(current_app.config['CFG_TMPDIR'], timestamp + '.tar.gz') conversion_result = convert_zip_archive(input_archive, output_file, options) os.remove(input_archive) if not conversion_result: return display_error( title="Your archive does not contain necessary files", description="For YAML conversions a submission.yaml file is necessary," " and for conversions from the oldhepdata format" " a file with .oldhepdata extension is required." ) return send_file(conversion_result, as_attachment=True)
def download_submission(submission, file_format, offline=False, force=False, rivet_analysis_name=''): """ Gets the submission file and either serves it back directly from YAML, or converts it for other formats. :param submission: HEPSubmission :param file_format: yaml, csv, root, or yoda :param offline: offline creation of the conversion when a record is finalised :param force: force recreation of the conversion :param rivet_analysis_name: Rivet analysis name to override default written in YODA export :return: display_error or send_file depending on success of conversion """ version = submission.version file_identifier = submission.publication_recid if submission.inspire_id: file_identifier = 'ins{0}'.format(submission.inspire_id) if file_format == 'json': return redirect('/record/{0}?version={1}&format=json'.format( file_identifier, version)) elif file_format not in CFG_SUPPORTED_FORMATS: if offline: log.error('Format not supported') return display_error( title="The " + file_format + " output format is not supported", description="This output format is not supported. " + "Currently supported formats: " + str(CFG_SUPPORTED_FORMATS), ) path = os.path.join(current_app.config['CFG_DATADIR'], str(submission.publication_recid)) data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format( submission.publication_recid, version) output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier, submission.version, file_format) converted_dir = os.path.join(current_app.config['CFG_DATADIR'], 'converted') if not os.path.exists(converted_dir): os.mkdir(converted_dir) if file_format == 'yoda' and rivet_analysis_name: # Don't store in converted_dir since rivet_analysis_name might possibly change between calls. output_path = os.path.join(current_app.config['CFG_TMPDIR'], output_file) else: output_path = os.path.join(converted_dir, output_file) # If the file is already available in the dir, send it back # unless we are forcing recreation of the file or the submission is not finished. if os.path.exists( output_path ) and not force and submission.overall_status == 'finished': if not offline: return send_file(output_path, as_attachment=True) else: print('File already downloaded at {0}'.format(output_path)) return converter_options = { 'input_format': 'yaml', 'output_format': file_format, 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), 'validator_schema_version': '0.1.0', } if submission.doi and submission.overall_status != 'sandbox': converter_options['hepdata_doi'] = '{0}.v{1}'.format( submission.doi, version) if file_format == 'yoda': if rivet_analysis_name: converter_options['rivet_analysis_name'] = rivet_analysis_name elif submission.inspire_id: record = get_record_contents(submission.publication_recid) if record: # Check if this record has a Rivet analysis, then extract the Rivet analysis name from the URL. if 'analyses' in record: for analysis in record['analyses']: if analysis['type'] == 'rivet': converter_options[ 'rivet_analysis_name'] = analysis[ 'analysis'].split('/')[-1] # Otherwise guess the Rivet analysis name using the collaboration name, # the creation year of the INSPIRE record, and the INSPIRE ID. if 'rivet_analysis_name' not in converter_options: try: year = parse(record['creation_date']).year except: year = record['year'] # publication year converter_options[ 'rivet_analysis_name'] = '{0}_{1}_I{2}'.format( ''.join(record['collaborations']).upper(), year, submission.inspire_id) data_filepath = os.path.join(path, data_filename) converted_file = convert_zip_archive(data_filepath, output_path, converter_options) if not offline: return send_file(converted_file, as_attachment=True) else: print('File for {0} created successfully at {1}'.format( file_identifier, output_path))
def download_submission(submission, file_format, offline=False, force=False, rivet_analysis_name=''): """ Gets the submission file and either serves it back directly from YAML, or converts it for other formats. :param submission: HEPSubmission :param file_format: json, yaml, csv, root, yoda or original :param offline: offline creation of the conversion when a record is finalised :param force: force recreation of the conversion :param rivet_analysis_name: Rivet analysis name to override default written in YODA export :return: display_error or send_file depending on success of conversion """ version = submission.version file_identifier = submission.publication_recid if submission.inspire_id: file_identifier = 'ins{0}'.format(submission.inspire_id) if file_format == 'json': return redirect('/record/{0}?version={1}&format=json'.format(file_identifier, version)) elif file_format not in CFG_SUPPORTED_FORMATS: if offline: log.error('Format not supported') return display_error( title="The " + file_format + " output format is not supported", description="This output format is not supported. " + "Currently supported formats: " + str(CFG_SUPPORTED_FORMATS), ) data_filepath = find_submission_data_file_path(submission) if file_format == 'original': file_format_and_extension = os.path.splitext(data_filepath)[1] else: file_format_and_extension = '-{0}.tar.gz'.format(file_format) output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension) converted_dir = get_converted_directory_path(submission.publication_recid) if not os.path.exists(converted_dir): os.makedirs(converted_dir, exist_ok=True) if file_format == 'yoda' and rivet_analysis_name: # Don't store in converted_dir since rivet_analysis_name might possibly change between calls. output_path = os.path.join(current_app.config['CFG_TMPDIR'], output_file) else: output_path = os.path.join(converted_dir, output_file) # If the file is already available in the dir, send it back # unless we are forcing recreation of the file or the submission is not finished. if os.path.exists(output_path) and not force and submission.overall_status == 'finished': if not offline: return send_file(output_path, as_attachment=True) else: print('File already downloaded at {0}'.format(output_path)) return if file_format == 'original': create_original_with_resources(submission, data_filepath, output_path) if not offline: return send_file(output_path, as_attachment=True) else: print('File created at {0}'.format(output_path)) return converter_options = { 'input_format': 'yaml', 'output_format': file_format, 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), 'validator_schema_version': '0.1.0', } if submission.doi and not submission.overall_status.startswith('sandbox'): converter_options['hepdata_doi'] = '{0}.v{1}'.format(submission.doi, version) if file_format == 'yoda': if not rivet_analysis_name: rivet_analysis_name = guess_rivet_analysis_name(submission) if rivet_analysis_name: converter_options['rivet_analysis_name'] = rivet_analysis_name try: converted_file = convert_zip_archive(data_filepath, output_path, converter_options) if not offline: return send_file(converted_file, as_attachment=True) else: print('File for {0} created successfully at {1}'.format(file_identifier, output_path)) except Error as error: # hepdata_converter_ws_client.Error if not offline: return display_error(title='Report concerns to [email protected]', description=str(error)) else: print('File conversion for {0} at {1} failed: {2}'.format( file_identifier, output_path, str(error) ))
def download_submission(submission, file_format, offline=False, force=False): """ Gets the submission file and either serves it back directly from YAML, or converts it for other formats. :param recid: submissions recid :param version: version of submission to export. If -1, returns the latest. :param file_format: yaml, csv, ROOT, or YODA :return: """ if file_format not in CFG_SUPPORTED_FORMATS: if offline: log.error('Format not supported') return display_error( title="The " + file_format + " output format is not supported", description="This output format is not supported. " + "Currently supported formats: " + CFG_SUPPORTED_FORMATS, ) version = submission.version path = os.path.join(current_app.config['CFG_DATADIR'], str(submission.publication_recid)) data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(submission.publication_recid, version) # If a YAML file is requested, we just need to send this back. if file_format == 'yaml' and os.path.exists(os.path.join(path, data_filename)): if not offline: return send_file(os.path.join(path, data_filename), as_attachment=True) file_identifier = submission.publication_recid if submission.inspire_id: file_identifier = 'ins{0}'.format(submission.inspire_id) output_file = 'HEPData-{0}-{1}-{2}.tar.gz'.format(file_identifier, submission.version, file_format) converted_dir = os.path.join(current_app.config['CFG_DATADIR'], 'converted') if not os.path.exists(converted_dir): os.mkdir(converted_dir) output_path = os.path.join(converted_dir, output_file) # If the file is already available in the dir, send it back # unless we are forcing recreation of the file. if os.path.exists(output_path) and not force: if not offline: return send_file( output_path, as_attachment=True, ) else: print('File already downloaded at {0}'.format(output_path)) return converter_options = { 'input_format': 'yaml', 'output_format': file_format, 'filename': 'HEPData-{0}-{1}-{2}'.format(file_identifier, submission.version, file_format), } data_filepath = os.path.join(path, data_filename) converted_file = convert_zip_archive(data_filepath, output_path, converter_options) if not offline: return send_file(converted_file, as_attachment=True) else: print('File for {0} create successfully at {1}'.format(file_identifier, output_path))
def download_submission(submission, file_format, offline=False, force=False): """ Gets the submission file and either serves it back directly from YAML, or converts it for other formats. :param submission: HEPSubmission :param file_format: yaml, csv, root, or yoda :return: """ version = submission.version file_identifier = submission.publication_recid if submission.inspire_id: file_identifier = 'ins{0}'.format(submission.inspire_id) if file_format == 'json': return redirect('/record/{0}?version={1}&format=json'.format( file_identifier, version)) elif file_format not in CFG_SUPPORTED_FORMATS: if offline: log.error('Format not supported') return display_error( title="The " + file_format + " output format is not supported", description="This output format is not supported. " + "Currently supported formats: " + str(CFG_SUPPORTED_FORMATS), ) path = os.path.join(current_app.config['CFG_DATADIR'], str(submission.publication_recid)) data_filename = current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format( submission.publication_recid, version) output_file = 'HEPData-{0}-v{1}-{2}.tar.gz'.format(file_identifier, submission.version, file_format) converted_dir = os.path.join(current_app.config['CFG_DATADIR'], 'converted') if not os.path.exists(converted_dir): os.mkdir(converted_dir) output_path = os.path.join(converted_dir, output_file) # If the file is already available in the dir, send it back # unless we are forcing recreation of the file. if os.path.exists(output_path) and not force: if not offline: return send_file( output_path, as_attachment=True, ) else: print('File already downloaded at {0}'.format(output_path)) return converter_options = { 'input_format': 'yaml', 'output_format': file_format, 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), } if submission.doi: converter_options['hepdata_doi'] = '{0}.v{1}'.format( submission.doi, version) if submission.inspire_id and file_format == 'yoda': record = get_record_contents(submission.publication_recid) if record: converter_options['rivet_analysis_name'] = '{0}_{1}_I{2}'.format( ''.join(record['collaborations']).upper(), record['year'], submission.inspire_id) data_filepath = os.path.join(path, data_filename) converted_file = convert_zip_archive(data_filepath, output_path, converter_options) if not offline: return send_file(converted_file, as_attachment=True) else: print('File for {0} created successfully at {1}'.format( file_identifier, output_path))