def parse_additional_resources(basepath, recid, yaml_document): """ Parses out the additional resource section for a full submission. :param basepath: the path the submission has been loaded to :param recid: :param yaml_document: :return: """ resources = [] for reference in yaml_document['additional_resources']: resource_location = reference['location'] file_type = infer_file_type(reference["location"]) contains_pattern, pattern = contains_accepted_url(reference['location']) if ('http' in resource_location.lower() and 'hepdata' not in resource_location) or contains_pattern: if pattern: file_type = pattern else: file_type = 'html' # in case URLs do not have http added. if 'http' not in resource_location.lower(): resource_location = "http://" + resource_location elif 'http' not in resource_location.lower() and 'www' not in resource_location.lower(): if resource_location.startswith('/resource'): # This is an old file migrated from hepdata.cedar.ac.uk. We # should only get here if using mock_import_old_record, in # which case the resources should already be in the 'resources' # directory parent_dir = os.path.dirname(basepath) resource_location = os.path.join( parent_dir, 'resources', os.path.basename(resource_location) ) if not os.path.exists(resource_location): raise ValueError("No such path %s" % resource_location) else: # this is a file local to the submission. try: resource_location = os.path.join(basepath, resource_location) except Exception as e: raise e if resource_location: new_reference = DataResource( file_location=resource_location, file_type=file_type, file_description=reference['description']) if "license" in reference: resource_license = get_license(reference["license"]) new_reference.file_license = resource_license.id resources.append(new_reference) return resources
def parse_additional_resources(basepath, recid, version, yaml_document): """ Parses out the additional resource section for a full submission :param hepsubmission: :param recid: :param submission_info_document: :return: """ resources = [] for reference in yaml_document['additional_resources']: resource_location = reference['location'] file_type = infer_file_type(reference["location"]) contains_pattern, pattern = contains_accepted_url(reference['location']) if ('http' in resource_location and 'hepdata' not in resource_location) or contains_pattern: if pattern: file_type = pattern else: file_type = 'html' # in case URLs do not have http added. if 'http' not in resource_location: resource_location = "http://" + resource_location elif 'http' not in resource_location and 'www' not in resource_location and 'resource' not in resource_location: # this is a file local to the submission. try: resource_location = os.path.join(basepath, resource_location) except Exception as e: raise e else: try: resource_location = download_resource_file(recid, resource_location) print('Downloaded resource location is {0}'.format(resource_location)) except URLError as url_error: log.error("Unable to download {0}. The resource is unavailable.".format(resource_location)) resource_location = None if resource_location: new_reference = DataResource( file_location=resource_location, file_type=file_type, file_description=reference['description']) if "license" in reference: dict = get_prefilled_dictionary( ["name", "url", "description"], reference["license"]) resource_license = get_or_create( db.session, License, name=dict['name'], url=dict['url'], description=dict['description']) new_reference.file_license = resource_license.id resources.append(new_reference) return resources
def parse_additional_resources(basepath, recid, version, yaml_document): """ Parses out the additional resource section for a full submission :param hepsubmission: :param recid: :param submission_info_document: :return: """ resources = [] for reference in yaml_document['additional_resources']: resource_location = reference['location'] file_type = infer_file_type(reference["location"]) contains_pattern, pattern = contains_accepted_url(reference['location']) if ('http' in resource_location and 'hepdata' not in resource_location) or contains_pattern: if pattern: file_type = pattern else: file_type = 'html' # in case URLs do not have http added. if 'http' not in resource_location: resource_location = "http://" + resource_location elif 'http' not in resource_location and 'www' not in resource_location and 'resource' not in resource_location: # this is a file local to the submission. try: resource_location = os.path.join(basepath, resource_location) except Exception as e: raise e else: try: resource_location = download_resource_file(recid, resource_location) print('Downloaded resource location is {0}'.format(resource_location)) except URLError as url_error: log.error("Unable to download {0}. The resource is unavailable.".format(resource_location)) resource_location = None if resource_location: new_reference = DataResource( file_location=resource_location, file_type=file_type, file_description=reference['description']) if "license" in reference: dict = get_prefilled_dictionary( ["name", "url", "description"], reference["license"]) resource_license = get_or_create( db.session, License, name=dict['name'], url=dict['url'], description=dict['description']) new_reference.file_license = resource_license.id resources.append(new_reference) return resources
def process_data_file(recid, version, basepath, data_obj, datasubmission, main_file_path): """ Takes a data file and any supplementary files and persists their metadata to the database whilst recording their upload path. :param recid: the record id :param version: version of the resource to be stored :param basepath: the path the submission has been loaded to :param data_obj: Object representation of loaded YAML file :param datasubmission: the DataSubmission object representing this file in the DB :param main_file_path: the data file path :return: """ main_data_file = DataResource(file_location=main_file_path, file_type="data") if "data_license" in data_obj: dict = get_prefilled_dictionary(["name", "url", "description"], data_obj["data_license"]) license = get_or_create(db.session, License, name=dict['name'], url=dict['url'], description=dict['description']) main_data_file.file_license = license.id db.session.add(main_data_file) # I have to do the commit here, otherwise I have no ID to reference in the data submission table. db.session.commit() datasubmission.data_file = main_data_file.id if "location" in data_obj: datasubmission.location_in_publication = data_obj["location"] cleanup_data_keywords(datasubmission) if "keywords" in data_obj: for keyword in data_obj["keywords"]: keyword_name = keyword['name'] for value in keyword['values']: keyword = Keyword(name=keyword_name, value=value) datasubmission.keywords.append(keyword) cleanup_data_resources(datasubmission) if "additional_resources" in data_obj: resources = parse_additional_resources(basepath, recid, version, data_obj) for resource in resources: datasubmission.resources.append(resource) db.session.commit()
def process_data_file(recid, version, basepath, data_obj, datasubmission, main_file_path): """ Takes a data file and any supplementary files and persists their metadata to the database whilst recording their upload path. :param recid: the record id :param version: version of the resource to be stored :param basepath: the path the submission has been loaded to :param data_obj: Object representation of loaded YAML file :param datasubmission: the DataSubmission object representing this file in the DB :param main_file_path: the data file path :return: """ main_data_file = DataResource( file_location=main_file_path, file_type="data") if "data_license" in data_obj: dict = get_prefilled_dictionary( ["name", "url", "description"], data_obj["data_license"]) license = get_or_create( db.session, License, name=dict['name'], url=dict['url'], description=dict['description']) main_data_file.file_license = license.id db.session.add(main_data_file) # I have to do the commit here, otherwise I have no ID to reference in the data submission table. db.session.commit() datasubmission.data_file = main_data_file.id if "location" in data_obj: datasubmission.location_in_publication = data_obj["location"] if "keywords" in data_obj: for keyword in data_obj["keywords"]: keyword_name = keyword['name'] for value in keyword['values']: keyword = Keyword(name=keyword_name, value=value) datasubmission.keywords.append(keyword) cleanup_data_resources(datasubmission) if "additional_resources" in data_obj: resources = parse_additional_resources(basepath, recid, version, data_obj) for resource in resources: datasubmission.resources.append(resource) db.session.commit()