def netcdf_pre_create_resource(sender, **kwargs): files = kwargs['files'] metadata = kwargs['metadata'] validate_files_dict = kwargs['validate_files'] source_names = kwargs['source_names'] if __debug__: assert(isinstance(source_names, list)) file_selected = False in_file_name = '' nc_file_name = '' if files: file_selected = True in_file_name = files[0].file.name nc_file_name = os.path.splitext(files[0].name)[0] elif source_names: nc_file_name = os.path.splitext(os.path.basename(source_names[0]))[0] ref_tmpfiles = utils.get_fed_zone_files(source_names) if ref_tmpfiles: in_file_name = ref_tmpfiles[0] file_selected = True if file_selected and in_file_name: # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(in_file_name) if isinstance(nc_dataset, netCDF4.Dataset): # Extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(in_file_name) # populate metadata list with extracted metadata add_metadata_to_list(metadata, res_dublin_core_meta, res_type_specific_meta) # create the ncdump text file dump_file = create_header_info_txt_file(in_file_name, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file), name=dump_file_name) files.append(uploaded_file) else: validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Please check if the uploaded file ' \ 'is in valid NetCDF format.' if source_names and in_file_name: shutil.rmtree(os.path.dirname(in_file_name))
def set_file_type(cls, resource, file_id, user): """ Sets a tif or zip raster resource file to GeoRasterFile type :param resource: an instance of resource type CompositeResource :param file_id: id of the resource file to be set as GeoRasterFile type :param user: user who is setting the file type :return: """ # had to import it here to avoid import loop from hs_core.views.utils import create_folder log = logging.getLogger() # get the file from irods res_file = utils.get_resource_file_by_id(resource, file_id) if res_file is None: raise ValidationError("File not found.") if res_file.extension != '.nc': raise ValidationError("Not a NetCDF file.") # base file name (no path included) file_name = res_file.file_name # file name without the extension nc_file_name = file_name.split(".")[0] resource_metadata = [] file_type_metadata = [] files_to_add_to_resource = [] if res_file.has_generic_logical_file: # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) temp_dir = os.path.dirname(temp_file) files_to_add_to_resource.append(temp_file) # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): # Extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file( temp_file, nc_file_name) files_to_add_to_resource.append(dump_file) file_folder = res_file.file_folder with transaction.atomic(): # first delete the netcdf file that we retrieved from irods # for setting it to netcdf file type delete_resource_file(resource.short_id, res_file.id, user) # create a netcdf logical file object to be associated with # resource files logical_file = cls.create() # by default set the dataset_name attribute of the logical file to the # name of the file selected to set file type unless the extracted metadata # has a value for title dataset_title = res_dublin_core_meta.get('title', None) if dataset_title is not None: logical_file.dataset_name = dataset_title else: logical_file.dataset_name = nc_file_name logical_file.save() try: # create a folder for the netcdf file type using the base file # name as the name for the new folder new_folder_path = cls.compute_file_type_folder( resource, file_folder, nc_file_name) fed_file_full_path = '' if resource.resource_federation_path: fed_file_full_path = os.path.join( resource.root_path, new_folder_path) create_folder(resource.short_id, new_folder_path) log.info("Folder created:{}".format(new_folder_path)) new_folder_name = new_folder_path.split('/')[-1] if file_folder is None: upload_folder = new_folder_name else: upload_folder = os.path.join( file_folder, new_folder_name) # add all new files to the resource for f in files_to_add_to_resource: uploaded_file = UploadedFile( file=open(f, 'rb'), name=os.path.basename(f)) new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder, fed_res_file_name_or_path=fed_file_full_path) # make each resource file we added as part of the logical file logical_file.add_resource_file(new_res_file) log.info( "NetCDF file type - new files were added to the resource." ) except Exception as ex: msg = "NetCDF file type. Error when setting file type. Error:{}" msg = msg.format(ex.message) log.exception(msg) # TODO: in case of any error put the original file back and # delete the folder that was created raise ValidationError(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) log.info("NetCDF file type was created.") # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element( 'title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info("Resource - metadata was saved to DB") # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() else: logical_file.metadata.create_element(k, **v) log.info("NetCDF file type - metadata was saved to DB") else: err_msg = "Not a valid NetCDF file. File type file validation failed." log.error(err_msg) # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) raise ValidationError(err_msg)
def netcdf_post_create_resource(sender, **kwargs): log = logging.getLogger() resource = kwargs['resource'] validate_files_dict = kwargs['validate_files'] res_file = resource.files.all().first() if res_file: temp_file = utils.get_file_from_irods(res_file) nc_dataset = nc_utils.get_nc_dataset(temp_file) nc_file_name = res_file.file_name if isinstance(nc_dataset, netCDF4.Dataset): # Extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( temp_file) # populate metadata list with extracted metadata metadata = [] add_metadata_to_list(metadata, res_dublin_core_meta, res_type_specific_meta) for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = list(element.items())[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element('title', title_element.id, **v) elif k == 'rights': rights_element = resource.metadata.rights resource.metadata.update_element('rights', rights_element.id, **v) elif k == 'creator': resource.metadata.creators.all().delete() resource.metadata.create_element('creator', **v) else: resource.metadata.create_element(k, **v) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file, mode="rb"), name=dump_file_name) utils.add_file_to_resource(resource, uploaded_file) else: delete_resource_file_only(resource, res_file) validate_files_dict['are_files_valid'] = False err_msg = "Uploaded file was not added to the resource." \ " Please provide a valid NetCDF file. " validate_files_dict['message'] = err_msg log_msg = "File validation failed for netcdf resource (ID:{})." log_msg = log_msg.format(resource.short_id) log.error(log_msg) # cleanup the temp file directory if os.path.exists(temp_file): shutil.rmtree(os.path.dirname(temp_file)) # set metadata is dirty flag as false for resource creation metadata = resource.metadata metadata.is_dirty = False metadata.save() # since we are extracting metadata after resource creation # metadata xml files need to be regenerated - so need to set the # dirty bag flags if resource.files.all().count() > 0: utils.set_dirty_bag_flag(resource)
def netcdf_pre_add_files_to_resource(sender, **kwargs): nc_res = kwargs['resource'] files = kwargs['files'] validate_files_dict = kwargs['validate_files'] source_names = kwargs['source_names'] if __debug__: assert (isinstance(source_names, list)) if len(files) > 1: # file number validation validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Only one file can be uploaded.' file_selected = False in_file_name = '' nc_file_name = '' if files: file_selected = True in_file_name = files[0].file.name nc_file_name = os.path.splitext(files[0].name)[0] elif source_names: nc_file_name = os.path.splitext(os.path.basename(source_names[0]))[0] ref_tmpfiles = utils.get_fed_zone_files(source_names) if ref_tmpfiles: in_file_name = ref_tmpfiles[0] file_selected = True if file_selected and in_file_name: # file type validation and existing metadata update and create new ncdump text file nc_dataset = nc_utils.get_nc_dataset(in_file_name) if isinstance(nc_dataset, netCDF4.Dataset): # delete all existing resource files and metadata related for f in ResourceFile.objects.filter(object_id=nc_res.id): delete_resource_file_only(nc_res, f) # update resource modification info user = kwargs['user'] utils.resource_modified(nc_res, user, overwrite_bag=False) # extract metadata res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( in_file_name) # update title info if res_dublin_core_meta.get('title'): if nc_res.metadata.title: nc_res.metadata.title.delete() nc_res.metadata.create_element( 'title', value=res_dublin_core_meta['title']) # update description info if res_dublin_core_meta.get('description'): if nc_res.metadata.description: nc_res.metadata.description.delete() nc_res.metadata.create_element( 'description', abstract=res_dublin_core_meta.get('description')) # update creator info if res_dublin_core_meta.get('creator_name'): name = res_dublin_core_meta.get('creator_name') email = res_dublin_core_meta.get('creator_email', '') url = res_dublin_core_meta.get('creator_url', '') arguments = dict(name=name, email=email, homepage=url) creator = nc_res.metadata.creators.all().filter( name=name).first() if creator: order = creator.order if order != 1: creator.delete() arguments['order'] = order nc_res.metadata.create_element('creator', **arguments) else: nc_res.metadata.create_element('creator', **arguments) # update contributor info if res_dublin_core_meta.get('contributor_name'): name_list = res_dublin_core_meta['contributor_name'].split(',') existing_contributor_names = [ contributor.name for contributor in nc_res.metadata.contributors.all() ] for name in name_list: if name not in existing_contributor_names: nc_res.metadata.create_element('contributor', name=name) # update subject info if res_dublin_core_meta.get('subject'): keywords = res_dublin_core_meta['subject'].split(',') existing_keywords = [ subject.value for subject in nc_res.metadata.subjects.all() ] for keyword in keywords: if keyword not in existing_keywords: nc_res.metadata.create_element('subject', value=keyword) # update source if res_dublin_core_meta.get('source'): for source in nc_res.metadata.sources.all(): source.delete() nc_res.metadata.create_element( 'source', derived_from=res_dublin_core_meta.get('source')) # update license element: if res_dublin_core_meta.get('rights'): raw_info = res_dublin_core_meta.get('rights') b = re.search("(?P<url>https?://[^\s]+)", raw_info) url = b.group('url') if b else '' statement = raw_info.replace(url, '') if url else raw_info if nc_res.metadata.rights: nc_res.metadata.rights.delete() nc_res.metadata.create_element('rights', statement=statement, url=url) # update relation if res_dublin_core_meta.get('references'): nc_res.metadata.relations.filter(type='cites').all().delete() nc_res.metadata.create_element( 'relation', type='cites', value=res_dublin_core_meta['references']) # update box info nc_res.metadata.coverages.all().delete() if res_dublin_core_meta.get('box'): nc_res.metadata.create_element( 'coverage', type='box', value=res_dublin_core_meta['box']) # update period info if res_dublin_core_meta.get('period'): nc_res.metadata.create_element( 'coverage', type='period', value=res_dublin_core_meta['period']) # update variable info nc_res.metadata.variables.all().delete() for var_info in list(res_type_specific_meta.values()): nc_res.metadata.create_element( 'variable', name=var_info['name'], unit=var_info['unit'], type=var_info['type'], shape=var_info['shape'], missing_value=var_info['missing_value'], descriptive_name=var_info['descriptive_name'], method=var_info['method']) # update the original spatial coverage meta nc_res.metadata.ori_coverage.all().delete() if res_dublin_core_meta.get('original-box'): if res_dublin_core_meta.get('projection-info'): nc_res.metadata.create_element( 'originalcoverage', value=res_dublin_core_meta['original-box'], projection_string_type=res_dublin_core_meta[ 'projection-info']['type'], projection_string_text=res_dublin_core_meta[ 'projection-info']['text'], datum=res_dublin_core_meta['projection-info']['datum']) else: nc_res.metadata.create_element( 'originalcoverage', value=res_dublin_core_meta['original-box']) # create the ncdump text file dump_file = create_header_info_txt_file(in_file_name, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file, 'rb'), name=dump_file_name) files.append(uploaded_file) else: validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Please check if the uploaded file is ' \ 'invalid NetCDF format.' if source_names and in_file_name: shutil.rmtree(os.path.dirname(in_file_name))
def netcdf_post_create_resource(sender, **kwargs): log = logging.getLogger() resource = kwargs['resource'] validate_files_dict = kwargs['validate_files'] res_file = resource.files.all().first() if res_file: temp_file = utils.get_file_from_irods(res_file) nc_dataset = nc_utils.get_nc_dataset(temp_file) nc_file_name = res_file.file_name if isinstance(nc_dataset, netCDF4.Dataset): # Extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(temp_file) # populate metadata list with extracted metadata metadata = [] add_metadata_to_list(metadata, res_dublin_core_meta, res_type_specific_meta) for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element('title', title_element.id, **v) elif k == 'rights': rights_element = resource.metadata.rights resource.metadata.update_element('rights', rights_element.id, **v) elif k == 'creator': resource.metadata.creators.all().delete() resource.metadata.create_element('creator', **v) else: resource.metadata.create_element(k, **v) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file), name=dump_file_name) utils.add_file_to_resource(resource, uploaded_file) else: delete_resource_file_only(resource, res_file) validate_files_dict['are_files_valid'] = False err_msg = "Uploaded file was not added to the resource." \ " Please provide a valid NetCDF file. " validate_files_dict['message'] = err_msg log_msg = "File validation failed for netcdf resource (ID:{})." log_msg = log_msg.format(resource.short_id) log.error(log_msg) # cleanup the temp file directory if os.path.exists(temp_file): shutil.rmtree(os.path.dirname(temp_file)) # set metadata is dirty flag as false for resource creation metadata = resource.metadata metadata.is_dirty = False metadata.save() # since we are extracting metadata after resource creation # metadata xml files need to be regenerated - so need to set the # dirty bag flags if resource.files.all().count() > 0: utils.set_dirty_bag_flag(resource)
def netcdf_pre_add_files_to_resource(sender, **kwargs): nc_res = kwargs['resource'] files = kwargs['files'] validate_files_dict = kwargs['validate_files'] source_names = kwargs['source_names'] if __debug__: assert(isinstance(source_names, list)) if len(files) > 1: # file number validation validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Only one file can be uploaded.' file_selected = False in_file_name = '' nc_file_name = '' if files: file_selected = True in_file_name = files[0].file.name nc_file_name = os.path.splitext(files[0].name)[0] elif source_names: nc_file_name = os.path.splitext(os.path.basename(source_names[0]))[0] ref_tmpfiles = utils.get_fed_zone_files(source_names) if ref_tmpfiles: in_file_name = ref_tmpfiles[0] file_selected = True if file_selected and in_file_name: # file type validation and existing metadata update and create new ncdump text file nc_dataset = nc_utils.get_nc_dataset(in_file_name) if isinstance(nc_dataset, netCDF4.Dataset): # delete all existing resource files and metadata related for f in ResourceFile.objects.filter(object_id=nc_res.id): delete_resource_file_only(nc_res, f) # update resource modification info user = kwargs['user'] utils.resource_modified(nc_res, user, overwrite_bag=False) # extract metadata res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(in_file_name) # update title info if res_dublin_core_meta.get('title'): if nc_res.metadata.title: nc_res.metadata.title.delete() nc_res.metadata.create_element('title', value=res_dublin_core_meta['title']) # update description info if res_dublin_core_meta.get('description'): if nc_res.metadata.description: nc_res.metadata.description.delete() nc_res.metadata.create_element('description', abstract=res_dublin_core_meta.get('description')) # update creator info if res_dublin_core_meta.get('creator_name'): name = res_dublin_core_meta.get('creator_name') email = res_dublin_core_meta.get('creator_email', '') url = res_dublin_core_meta.get('creator_url', '') arguments = dict(name=name, email=email, homepage=url) creator = nc_res.metadata.creators.all().filter(name=name).first() if creator: order = creator.order if order != 1: creator.delete() arguments['order'] = order nc_res.metadata.create_element('creator', **arguments) else: nc_res.metadata.create_element('creator', **arguments) # update contributor info if res_dublin_core_meta.get('contributor_name'): name_list = res_dublin_core_meta['contributor_name'].split(',') existing_contributor_names = [contributor.name for contributor in nc_res.metadata.contributors.all()] for name in name_list: if name not in existing_contributor_names: nc_res.metadata.create_element('contributor', name=name) # update subject info if res_dublin_core_meta.get('subject'): keywords = res_dublin_core_meta['subject'].split(',') existing_keywords = [subject.value for subject in nc_res.metadata.subjects.all()] for keyword in keywords: if keyword not in existing_keywords: nc_res.metadata.create_element('subject', value=keyword) # update source if res_dublin_core_meta.get('source'): for source in nc_res.metadata.sources.all(): source.delete() nc_res.metadata.create_element('source', derived_from=res_dublin_core_meta.get('source')) # update license element: if res_dublin_core_meta.get('rights'): raw_info = res_dublin_core_meta.get('rights') b = re.search("(?P<url>https?://[^\s]+)", raw_info) url = b.group('url') if b else '' statement = raw_info.replace(url, '') if url else raw_info if nc_res.metadata.rights: nc_res.metadata.rights.delete() nc_res.metadata.create_element('rights', statement=statement, url=url) # update relation if res_dublin_core_meta.get('references'): nc_res.metadata.relations.filter(type='cites').all().delete() nc_res.metadata.create_element('relation', type='cites', value=res_dublin_core_meta['references']) # update box info nc_res.metadata.coverages.all().delete() if res_dublin_core_meta.get('box'): nc_res.metadata.create_element('coverage', type='box', value=res_dublin_core_meta['box']) # update period info if res_dublin_core_meta.get('period'): nc_res.metadata.create_element('coverage', type='period', value=res_dublin_core_meta['period']) # update variable info nc_res.metadata.variables.all().delete() for var_info in res_type_specific_meta.values(): nc_res.metadata.create_element('variable', name=var_info['name'], unit=var_info['unit'], type=var_info['type'], shape=var_info['shape'], missing_value=var_info['missing_value'], descriptive_name=var_info['descriptive_name'], method=var_info['method']) # update the original spatial coverage meta nc_res.metadata.ori_coverage.all().delete() if res_dublin_core_meta.get('original-box'): if res_dublin_core_meta.get('projection-info'): nc_res.metadata.create_element( 'originalcoverage', value=res_dublin_core_meta['original-box'], projection_string_type=res_dublin_core_meta['projection-info']['type'], projection_string_text=res_dublin_core_meta['projection-info']['text'], datum=res_dublin_core_meta['projection-info']['datum']) else: nc_res.metadata.create_element('originalcoverage', value=res_dublin_core_meta['original-box']) # create the ncdump text file dump_file = create_header_info_txt_file(in_file_name, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file), name=dump_file_name) files.append(uploaded_file) else: validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Please check if the uploaded file is ' \ 'invalid NetCDF format.' if source_names and in_file_name: shutil.rmtree(os.path.dirname(in_file_name))
def set_file_type(cls, resource, user, file_id=None, folder_path=None): """ Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file or a folder """ log = logging.getLogger() res_file, folder_path = cls._validate_set_file_type_inputs( resource, file_id, folder_path) # base file name (no path included) file_name = res_file.file_name # file name without the extension - needed for naming the new aggregation folder nc_file_name = file_name[:-len(res_file.extension)] resource_metadata = [] file_type_metadata = [] upload_folder = '' res_files_to_delete = [] # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) temp_dir = os.path.dirname(temp_file) # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): msg = "NetCDF aggregation. Error when creating aggregation. Error:{}" file_type_success = False # extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) file_folder = res_file.file_folder aggregation_folder_created = False create_new_folder = cls._check_create_aggregation_folder( selected_res_file=res_file, selected_folder=folder_path, aggregation_file_count=1) with transaction.atomic(): # create a netcdf logical file object to be associated with # resource files dataset_title = res_dublin_core_meta.get('title', nc_file_name) logical_file = cls.initialize(dataset_title, resource) try: if folder_path is None: # we are here means aggregation is being created by selecting a file # create a folder for the netcdf file type using the base file # name as the name for the new folder if the file is not already in a folder if create_new_folder: upload_folder = cls._create_aggregation_folder( resource, file_folder, nc_file_name) aggregation_folder_created = True log.info( "NetCDF Aggregation creation - folder created:{}" .format(upload_folder)) else: # selected nc file is already in a folder upload_folder = file_folder # create logical file record in DB logical_file.save() if aggregation_folder_created: # copy the nc file to the new aggregation folder and make it part # of the logical file tgt_folder = upload_folder files_to_copy = [res_file] logical_file.copy_resource_files( resource, files_to_copy, tgt_folder) res_files_to_delete.append(res_file) else: # make the selected nc file as part of the aggregation/file type logical_file.add_resource_file(res_file) else: # logical file record gets created in DB logical_file.save() # folder has been selected to create aggregation upload_folder = folder_path # make the .nc file part of the aggregation logical_file.add_resource_file(res_file) # add the new dump txt file to the resource uploaded_file = UploadedFile( file=open(dump_file, 'rb'), name=os.path.basename(dump_file)) new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder, add_to_aggregation=False) # make this new resource file we added part of the logical file logical_file.add_resource_file(new_res_file) log.info( "NetCDF aggregation creation - a new file was added to the resource." ) # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element( 'title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info( "NetCDF Aggregation creation - Resource metadata was saved to DB" ) # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() # update resource level keywords resource_keywords = [ subject.value.lower() for subject in resource.metadata.subjects.all() ] for kw in logical_file.metadata.keywords: if kw.lower() not in resource_keywords: resource.metadata.create_element('subject', value=kw) else: logical_file.metadata.create_element(k, **v) log.info( "NetCDF aggregation - metadata was saved in aggregation" ) logical_file._finalize( user, resource, folder_created=aggregation_folder_created, res_files_to_delete=res_files_to_delete) file_type_success = True except Exception as ex: msg = msg.format(ex.message) log.exception(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) if not file_type_success: aggregation_from_folder = folder_path is not None cls._cleanup_on_fail_to_create_aggregation( user, resource, upload_folder, file_folder, aggregation_from_folder) raise ValidationError(msg) else: err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed." log.error(err_msg) # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) raise ValidationError(err_msg)
def set_file_type(cls, resource, user, file_id=None, folder_path=None): """ Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file or a folder """ log = logging.getLogger() res_file, folder_path = cls._validate_set_file_type_inputs(resource, file_id, folder_path) # base file name (no path included) file_name = res_file.file_name # file name without the extension - needed for naming the new aggregation folder nc_file_name = file_name[:-len(res_file.extension)] resource_metadata = [] file_type_metadata = [] upload_folder = '' res_files_to_delete = [] # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) temp_dir = os.path.dirname(temp_file) # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): msg = "NetCDF aggregation. Error when creating aggregation. Error:{}" file_type_success = False # extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) file_folder = res_file.file_folder aggregation_folder_created = False create_new_folder = cls._check_create_aggregation_folder( selected_res_file=res_file, selected_folder=folder_path, aggregation_file_count=1) with transaction.atomic(): # create a netcdf logical file object to be associated with # resource files dataset_title = res_dublin_core_meta.get('title', nc_file_name) logical_file = cls.initialize(dataset_title, resource) try: if folder_path is None: # we are here means aggregation is being created by selecting a file # create a folder for the netcdf file type using the base file # name as the name for the new folder if the file is not already in a folder if create_new_folder: upload_folder = cls._create_aggregation_folder(resource, file_folder, nc_file_name) aggregation_folder_created = True log.info("NetCDF Aggregation creation - folder created:{}".format( upload_folder)) else: # selected nc file is already in a folder upload_folder = file_folder # create logical file record in DB logical_file.save() if aggregation_folder_created: # copy the nc file to the new aggregation folder and make it part # of the logical file tgt_folder = upload_folder files_to_copy = [res_file] logical_file.copy_resource_files(resource, files_to_copy, tgt_folder) res_files_to_delete.append(res_file) else: # make the selected nc file as part of the aggregation/file type logical_file.add_resource_file(res_file) else: # logical file record gets created in DB logical_file.save() # folder has been selected to create aggregation upload_folder = folder_path # make the .nc file part of the aggregation logical_file.add_resource_file(res_file) # add the new dump txt file to the resource uploaded_file = UploadedFile(file=open(dump_file, 'rb'), name=os.path.basename(dump_file)) new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder, add_to_aggregation=False ) # make this new resource file we added part of the logical file logical_file.add_resource_file(new_res_file) log.info("NetCDF aggregation creation - a new file was added to the resource.") # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element('title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info("NetCDF Aggregation creation - Resource metadata was saved to DB") # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() # update resource level keywords resource_keywords = [subject.value.lower() for subject in resource.metadata.subjects.all()] for kw in logical_file.metadata.keywords: if kw.lower() not in resource_keywords: resource.metadata.create_element('subject', value=kw) else: logical_file.metadata.create_element(k, **v) log.info("NetCDF aggregation - metadata was saved in aggregation") logical_file._finalize(user, resource, folder_created=aggregation_folder_created, res_files_to_delete=res_files_to_delete) file_type_success = True post_add_netcdf_aggregation.send( sender=AbstractLogicalFile, resource=resource, file=logical_file ) except Exception as ex: msg = msg.format(ex.message) log.exception(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) if not file_type_success: aggregation_from_folder = folder_path is not None cls._cleanup_on_fail_to_create_aggregation(user, resource, upload_folder, file_folder, aggregation_from_folder) raise ValidationError(msg) else: err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed." log.error(err_msg) # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) raise ValidationError(err_msg)
def set_file_type(cls, resource, user, file_id=None, folder_path=None): """ Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file """ log = logging.getLogger() with FileTypeContext(aggr_cls=cls, user=user, resource=resource, file_id=file_id, folder_path=folder_path, post_aggr_signal=post_add_netcdf_aggregation, is_temp_file=True) as ft_ctx: # base file name (no path included) res_file = ft_ctx.res_file file_name = res_file.file_name # file name without the extension - needed for naming the new aggregation folder nc_file_name = file_name[:-len(res_file.extension)] resource_metadata = [] file_type_metadata = [] # file validation and metadata extraction temp_file = ft_ctx.temp_file nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): msg = "NetCDF aggregation. Error when creating aggregation. Error:{}" file_type_success = False # extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) file_folder = res_file.file_folder upload_folder = file_folder dataset_title = res_dublin_core_meta.get('title', nc_file_name) with transaction.atomic(): try: # create a netcdf logical file object logical_file = cls.create_aggregation(dataset_name=dataset_title, resource=resource, res_files=[res_file], new_files_to_upload=[dump_file], folder_path=upload_folder) log.info("NetCDF aggregation creation - a new file was added to the " "resource.") # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = list(element.items())[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element('title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info("NetCDF Aggregation creation - Resource metadata was saved to DB") # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = list(element.items())[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() # update resource level keywords resource_keywords = [subject.value.lower() for subject in resource.metadata.subjects.all()] for kw in logical_file.metadata.keywords: if kw.lower() not in resource_keywords: resource.metadata.create_element('subject', value=kw) else: logical_file.metadata.create_element(k, **v) log.info("NetCDF aggregation - metadata was saved in aggregation") file_type_success = True ft_ctx.logical_file = logical_file except Exception as ex: msg = msg.format(str(ex)) log.exception(msg) if not file_type_success: raise ValidationError(msg) else: err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed." log.error(err_msg) raise ValidationError(err_msg)