class GeographicFeatureResource(BaseResource): objects = ResourceManager("GeographicFeatureResource") @property def metadata(self): md = GeographicFeatureMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # See Shapefile format: # http://resources.arcgis.com/en/help/main/10.2/index.html#//005600000003000000 return (".zip", ".shp", ".shx", ".dbf", ".prj", ".sbx", ".sbn", ".cpg", ".xml", ".fbn", ".fbx", ".ain", ".aih", ".atx", ".ixs", ".mxs") # add resource-specific HS terms def get_hs_term_dict(self): # get existing hs_term_dict from base class hs_term_dict = super(GeographicFeatureResource, self).get_hs_term_dict() geometryinformation = self.metadata.geometryinformation.all().first() if geometryinformation is not None: hs_term_dict[ "HS_GFR_FEATURE_COUNT"] = geometryinformation.featureCount else: hs_term_dict["HS_GFR_FEATURE_COUNT"] = 0 return hs_term_dict class Meta: verbose_name = 'Geographic Feature (ESRI Shapefiles)' proxy = True
class ToolResource(BaseResource): objects = ResourceManager('ToolResource') class Meta: proxy = True verbose_name = 'Web App Resource' @classmethod def get_supported_upload_file_types(cls): # no file types are supported return () @classmethod def allow_multiple_file_upload(cls): # no file can be uploaded return False @classmethod def can_have_multiple_files(cls): # resource can't have any files return False @property def metadata(self): md = ToolMetaData() return self._get_metadata(md) @property def can_be_published(self): return False
class RefTimeSeriesResource(BaseResource): objects = ResourceManager("RefTimeSeriesResource") class Meta: verbose_name = "HIS Referenced Time Series" proxy = True @property def metadata(self): md = RefTSMetadata() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # no file types are supported return () @classmethod def allow_multiple_file_upload(cls): # no file can be uploaded return False @classmethod def can_have_multiple_files(cls): # resource can't have any files return False
class RefTimeSeriesResource(BaseResource): objects = ResourceManager("RefTimeSeriesResource") discovery_content_type = "Reference to HIS Time Series" # used during discovery class Meta: verbose_name = "HIS Referenced Time Series" proxy = True @classmethod def get_metadata_class(cls): return RefTSMetadata @classmethod def get_supported_upload_file_types(cls): # no file types are supported return () @classmethod def allow_multiple_file_upload(cls): # no file can be uploaded return False @classmethod def can_have_multiple_files(cls): # resource can't have any files return False
class RasterResource(BaseResource): objects = ResourceManager("RasterResource") class Meta: verbose_name = 'Geographic Raster' proxy = True @property def metadata(self): md = RasterMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # only tif file type is supported return (".tif", ".zip") @classmethod def allow_multiple_file_upload(cls): # can upload only 1 file return False @classmethod def can_have_multiple_files(cls): # can have only 1 file return False
class ModelInstanceResource(BaseResource): objects = ResourceManager("ModelInstanceResource") class Meta: verbose_name = 'Model Instance Resource' proxy = True @property def metadata(self): md = ModelInstanceMetaData() return self._get_metadata(md)
class CollectionResource(BaseResource): objects = ResourceManager('CollectionResource') discovery_content_type = 'Collection' # used during discovery class Meta: proxy = True verbose_name = 'Collection Resource' @classmethod def get_supported_upload_file_types(cls): # no file types are supported return () @classmethod def allow_multiple_file_upload(cls): # cannot upload any file return False @classmethod def can_have_multiple_files(cls): # resource can't have any files return False @property def can_be_public_or_discoverable(self): return self.metadata.has_all_required_elements() and ( self.resources.count() > 0) @property def deleted_resources(self): return CollectionDeletedResource.objects.filter(collection=self) @property def has_resources(self): return self.resources.count() > 0 @property def are_all_contained_resources_published(self): if not self.has_resources: return False return not self.resources.all().filter( raccess__published=False).exists() @property def can_be_published(self): return self.can_be_public_or_discoverable and \ self.are_all_contained_resources_published @property def update_text_file(self): return self.extra_data.get('update_text_file', 'True')
class NetcdfResource(BaseResource): objects = ResourceManager("NetcdfResource") @property def metadata(self): md = NetcdfMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # only file with extension .nc is supported for uploading return (".nc", ) @classmethod def allow_multiple_file_upload(cls): # can upload only 1 file return False @classmethod def can_have_multiple_files(cls): # can have only 1 file return False # add resource-specific HS terms def get_hs_term_dict(self): # get existing hs_term_dict from base class hs_term_dict = super(NetcdfResource, self).get_hs_term_dict() # add new terms for NetCDF res hs_term_dict["HS_FILE_NAME"] = "" for res_file in self.files.all(): _, f_fullname, f_ext = get_resource_file_name_and_extension( res_file) if f_ext.lower() == '.nc': hs_term_dict["HS_FILE_NAME"] = f_fullname break return hs_term_dict def update_netcdf_file(self, user): if not self.metadata.is_dirty: return nc_res_file = get_resource_files_by_extension(self, ".nc") txt_res_file = get_resource_files_by_extension(self, ".txt") from hs_file_types.models.netcdf import netcdf_file_update # avoid recursive import if nc_res_file and txt_res_file: netcdf_file_update(self, nc_res_file[0], txt_res_file[0], user) discovery_content_type = 'Multidimensional (NetCDF)' # used during discovery class Meta: verbose_name = 'Multidimensional (NetCDF)' proxy = True
class ModelInstanceResource(BaseResource): objects = ResourceManager("ModelInstanceResource") discovery_content_type = 'Model Instance' # used during discovery class Meta: verbose_name = 'Model Instance Resource' proxy = True @classmethod def get_metadata_class(cls): return ModelInstanceMetaData
class ModelInstanceResource(BaseResource): objects = ResourceManager("ModelInstanceResource") discovery_content_type = 'Model Instance' # used during discovery class Meta: verbose_name = 'Model Instance Resource' proxy = True @property def metadata(self): md = ModelInstanceMetaData() return self._get_metadata(md)
class ScriptResource(BaseResource): objects = ResourceManager('ScriptResource') class Meta: proxy = True verbose_name = 'Script Resource' @classmethod def get_supported_upload_file_types(cls): # one file type is supported return ".r", ".py", ".m" @property def metadata(self): md = ScriptMetaData() return self._get_metadata(md)
class SWATModelInstanceResource(BaseResource): objects = ResourceManager("SWATModelInstanceResource") class Meta: verbose_name = 'SWAT Model Instance Resource' proxy = True @property def metadata(self): md = SWATModelInstanceMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # all file types are supported return '.*'
class ScriptResource(BaseResource): objects = ResourceManager('ScriptResource') discovery_content_type = 'Script' # used during discovery class Meta: proxy = True verbose_name = 'Script Resource' @classmethod def get_supported_upload_file_types(cls): # one file type is supported return ".r", ".py", ".m" @classmethod def get_metadata_class(cls): return ScriptMetaData
class ModelProgramResource(BaseResource): objects = ResourceManager("ModelProgramResource") class Meta: verbose_name = 'Model Program Resource' proxy = True @property def metadata(self): md = ModelProgramMetaData() meta = self._get_metadata(md) return meta @classmethod def get_supported_upload_file_types(cls): # all file types are supported return ('.*')
class SWATModelInstanceResource(BaseResource): objects = ResourceManager("SWATModelInstanceResource") discovery_content_type = 'SWAT Model Instance' # used during discovery class Meta: verbose_name = 'SWAT Model Instance Resource' proxy = True @classmethod def get_metadata_class(cls): return SWATModelInstanceMetaData @classmethod def get_supported_upload_file_types(cls): # all file types are supported return '.*'
class ModelProgramResource(BaseResource): objects = ResourceManager("ModelProgramResource") discovery_content_type = 'Model Program' # used during discovery class Meta: verbose_name = 'Model Program Resource' proxy = True @classmethod def get_metadata_class(cls): return ModelProgramMetaData @classmethod def get_supported_upload_file_types(cls): # all file types are supported return ('.*')
class ToolResource(BaseResource): objects = ResourceManager('ToolResource') discovery_content_type = 'Web App' # used during discovery class Meta: proxy = True verbose_name = 'Web App Resource' @classmethod def get_approved_apps(cls): webapp_resources = cls.objects.all() final_resource_list = [] for resource in webapp_resources: if resource.metadata.approved: final_resource_list.append(resource) return final_resource_list @classmethod def get_supported_upload_file_types(cls): # no file types are supported return () @classmethod def allow_multiple_file_upload(cls): # no file can be uploaded return False @classmethod def can_have_multiple_files(cls): # resource can't have any files return False @property def metadata(self): md = ToolMetaData() return self._get_metadata(md) @property def can_be_published(self): return False
class RasterResource(BaseResource): objects = ResourceManager("RasterResource") discovery_content_type = 'Geographic Raster' # used during discovery class Meta: verbose_name = 'Geographic Raster' proxy = True @property def metadata(self): md = RasterMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # only tif file type is supported return (".tiff", ".tif", ".vrt", ".zip") @classmethod def allow_multiple_file_upload(cls): # can upload multiple files return True @classmethod def can_have_multiple_files(cls): # can have only 1 file return False # add resource-specific HS terms def get_hs_term_dict(self): # get existing hs_term_dict from base class hs_term_dict = super(RasterResource, self).get_hs_term_dict() # add new terms for Raster res hs_term_dict["HS_FILE_NAME"] = "" for res_file in self.files.all(): _, f_fullname, f_ext = get_resource_file_name_and_extension( res_file) if f_ext.lower() == '.vrt': hs_term_dict["HS_FILE_NAME"] = f_fullname break return hs_term_dict
class NetcdfResource(BaseResource): objects = ResourceManager("NetcdfResource") @property def metadata(self): md = NetcdfMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # only file with extension .nc is supported for uploading return (".nc", ) @classmethod def allow_multiple_file_upload(cls): # can upload only 1 file return False @classmethod def can_have_multiple_files(cls): # can have only 1 file return False # add resource-specific HS terms def get_hs_term_dict(self): # get existing hs_term_dict from base class hs_term_dict = super(NetcdfResource, self).get_hs_term_dict() # add new terms for NetCDF res hs_term_dict["HS_NETCDF_FILE_NAME"] = "" for res_file in self.files.all(): _, f_fullname, f_ext = get_resource_file_name_and_extension( res_file) if f_ext.lower() == '.nc': hs_term_dict["HS_NETCDF_FILE_NAME"] = f_fullname break return hs_term_dict class Meta: verbose_name = 'Multidimensional (NetCDF)' proxy = True
class GeographicFeatureResource(BaseResource): objects = ResourceManager("GeographicFeatureResource") @property def metadata(self): md = GeographicFeatureMetaData() return self._get_metadata(md) @classmethod def get_supported_upload_file_types(cls): # See Shapefile format: # http://resources.arcgis.com/en/help/main/10.2/index.html#//005600000003000000 return (".zip", ".shp", ".shx", ".dbf", ".prj", ".sbx", ".sbn", ".cpg", ".xml", ".fbn", ".fbx", ".ain", ".aih", ".atx", ".ixs", ".mxs") def has_required_content_files(self): if self.files.all().count < 3: return False file_extensions = [f.extension for f in self.files.all()] return all(ext in file_extensions for ext in ['.shp', '.shx', '.dbf']) def get_hs_term_dict(self): # get existing hs_term_dict from base class hs_term_dict = super(GeographicFeatureResource, self).get_hs_term_dict() geometryinformation = self.metadata.geometryinformation if geometryinformation is not None: hs_term_dict[ "HS_GFR_FEATURE_COUNT"] = geometryinformation.featureCount else: hs_term_dict["HS_GFR_FEATURE_COUNT"] = 0 return hs_term_dict discovery_content_type = 'Geographic Feature (ESRI Shapefiles)' # used during discovery class Meta: verbose_name = 'Geographic Feature (ESRI Shapefiles)' proxy = True
class CompositeResource(BaseResource): objects = ResourceManager("CompositeResource") discovery_content_type = 'Composite' # used during discovery class Meta: verbose_name = 'Composite Resource' proxy = True @property def can_be_public_or_discoverable(self): # resource level metadata check if not super(CompositeResource, self).can_be_public_or_discoverable: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False return True @property def logical_files(self): """Returns a list of all logical file type objects associated with this resource """ lf_list = [] lf_list.extend(self.filesetlogicalfile_set.all()) lf_list.extend(self.genericlogicalfile_set.all()) lf_list.extend(self.geofeaturelogicalfile_set.all()) lf_list.extend(self.netcdflogicalfile_set.all()) lf_list.extend(self.georasterlogicalfile_set.all()) lf_list.extend(self.reftimeserieslogicalfile_set.all()) lf_list.extend(self.timeserieslogicalfile_set.all()) return lf_list @property def can_be_published(self): # resource level metadata check if not super(CompositeResource, self).can_be_published: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False # url file cannot be published if 'url' in lf.extra_data: return False return True def set_default_logical_file(self): """sets an instance of GenericLogicalFile to any resource file objects of this instance of the resource that is not already associated with a logical file. """ for res_file in self.files.all(): if not res_file.has_logical_file: logical_file = GenericLogicalFile.create() res_file.logical_file_content_object = logical_file res_file.save() def get_folder_aggregation_object(self, dir_path): """Returns an aggregation (file type) object if the specified folder *dir_path* represents a file type aggregation (logical file), otherwise None. :param dir_path: Resource file directory path (full folder path starting with resource id) for which the aggregation object to be retrieved """ aggregation_path = dir_path[len(self.file_path) + 1:] try: return self.get_aggregation_by_name(aggregation_path) except ObjectDoesNotExist: return None def get_file_aggregation_object(self, file_path): """Returns an aggregation (file type) object if the specified file *file_path* represents a file type aggregation (logical file), otherwise None. :param file_path: Resource file path (full file path starting with resource id) for which the aggregation object to be retrieved """ for res_file in self.files.all(): if res_file.full_path == file_path: if res_file.has_logical_file: return res_file.logical_file return None def get_folder_aggregation_type_to_set(self, dir_path): """Returns an aggregation (file type) type that the specified folder *dir_path* can possibly be set to. :param dir_path: Resource file directory path (full folder path starting with resource id) for which the possible aggregation type that can be set needs to be determined :return If the specified folder is already represents an aggregation or does not contain suitable file(s) then returns "" (empty string). If the specified folder contains only the files that meet the requirements of a supported aggregation, and does not contain other folders or does not have a parent folder then return the class name of that matching aggregation type. """ if self.get_folder_aggregation_object(dir_path) is not None: # target folder is already an aggregation return None istorage = self.get_irods_storage() irods_path = dir_path if self.is_federated: irods_path = os.path.join(self.resource_federation_path, irods_path) store = istorage.listdir(irods_path) files_in_folder = ResourceFile.list_folder(self, folder=irods_path, sub_folders=False) if not files_in_folder: # folder is empty # check sub folders for files - if file exist we can set FileSet aggregation files_in_sub_folders = ResourceFile.list_folder(self, folder=irods_path, sub_folders=True) if files_in_sub_folders: return FileSetLogicalFile.__name__ return None if store[0]: # there are folders under dir_path as well as files - only FileSet can bet set return FileSetLogicalFile.__name__ if len(files_in_folder) > 1: # check for geo feature aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for raster aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set return FileSetLogicalFile.__name__ else: # check for raster aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for NetCDF aggregation type aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for TimeSeries aggregation type aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set return FileSetLogicalFile.__name__ @property def supports_folders(self): """ allow folders for CompositeResources """ return True @property def supports_logical_file(self): """ if this resource allows associating resource file objects with logical file""" return True def get_metadata_xml(self, pretty_print=True, include_format_elements=True): from lxml import etree # get resource level core metadata as xml string # for composite resource we don't want the format elements at the resource level # as they are included at the aggregation map xml document xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False, include_format_elements=False) # create an etree xml object RDF_ROOT = etree.fromstring(xml_string) return etree.tostring(RDF_ROOT, pretty_print=pretty_print) def create_aggregation_xml_documents(self, aggregation_name=None): """Creates aggregation map and metadata xml files for each of the contained aggregations :param aggregation_name: (optional) name of the the specific aggregation for which xml documents need to be created """ if aggregation_name is None: for aggregation in self.logical_files: if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() else: try: aggregation = self.get_aggregation_by_name(aggregation_name) if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() except ObjectDoesNotExist: # aggregation_name must be a folder path that doesn't represent an aggregation # there may be single file aggregation in that folder for which xml documents # need to be created self._recreate_xml_docs_for_folder(aggregation_name, check_metadata_dirty=True) def _recreate_xml_docs_for_folder(self, folder, check_metadata_dirty=False): """Re-creates xml metadata and map documents associated with the specified folder. If the *folder* represents an aggregation then map and metadata xml documents are recreated only for that aggregation. Otherwise, xml documents are created for any aggregation that may exist in the specified folder and its sub-folders. :param folder: folder for which xml documents need to be re-created :param check_metadata_dirty: if true, then xml files will be created only if the aggregation metadata is dirty """ # first check if the the folder represents an aggregation try: aggregation = self.get_aggregation_by_name(folder) if check_metadata_dirty: if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() else: aggregation.create_aggregation_xml_documents() # if we found an aggregation by the folder name that means this folder doesn't # have any sub folders as multi-file aggregation folder can't have sub folders except ObjectDoesNotExist: # create xml map and metadata xml documents for all aggregations that exist # in *folder* and its sub-folders if not folder.startswith(self.file_path): folder = os.path.join(self.file_path, folder) res_file_objects = ResourceFile.list_folder(self, folder) aggregations = [] for res_file in res_file_objects: if res_file.has_logical_file and res_file.logical_file not in aggregations: aggregations.append(res_file.logical_file) if check_metadata_dirty: aggregations = [aggr for aggr in aggregations if aggr.metadata.is_dirty] for aggregation in aggregations: aggregation.create_aggregation_xml_documents() def get_aggregation_by_name(self, name): """Get an aggregation that matches the aggregation name specified by *name* :param name: name (aggregation path) of the aggregation to find :return an aggregation object if found :raises ObjectDoesNotExist if no matching aggregation is found """ for aggregation in self.logical_files: # remove the last slash in aggregation_name if any if aggregation.aggregation_name.rstrip('/') == name: return aggregation raise ObjectDoesNotExist("No matching aggregation was found for name:{}".format(name)) def get_fileset_aggregation_in_path(self, path): """Get the first fileset aggregation in the path moving up (towards the root)in the path :param path: directory path in which to search for a fileset aggregation :return a fileset aggregation object if found, otherwise None """ def get_fileset(path): try: aggregation = self.get_aggregation_by_name(path) if aggregation.is_fileset: return aggregation except ObjectDoesNotExist: return None while '/' in path: fileset = get_fileset(path) if fileset is not None: return fileset path = os.path.dirname(path) else: return get_fileset(path) def recreate_aggregation_xml_docs(self, orig_aggr_name, new_aggr_name): """ When a folder or file representing an aggregation is renamed or moved, the associated map and metadata xml documents are deleted and then regenerated :param orig_aggr_name: original aggregation name - used for deleting existing xml documents :param new_aggr_name: new aggregation name - used for finding a matching aggregation so that new xml documents can be recreated """ def delete_old_xml_files(folder=''): istorage = self.get_irods_storage() meta_xml_file_name = orig_aggr_name + "_meta.xml" map_xml_file_name = orig_aggr_name + "_resmap.xml" if not folder: # case if file rename/move for single file aggregation meta_xml_file_full_path = os.path.join(self.file_path, meta_xml_file_name) map_xml_file_full_path = os.path.join(self.file_path, map_xml_file_name) else: # case of folder rename - multi-file aggregation _, meta_xml_file_name = os.path.split(meta_xml_file_name) _, map_xml_file_name = os.path.split(map_xml_file_name) meta_xml_file_full_path = os.path.join(self.file_path, folder, meta_xml_file_name) map_xml_file_full_path = os.path.join(self.file_path, folder, map_xml_file_name) if istorage.exists(meta_xml_file_full_path): istorage.delete(meta_xml_file_full_path) if istorage.exists(map_xml_file_full_path): istorage.delete(map_xml_file_full_path) # first check if the new_aggr_name is a folder path or file path name, ext = os.path.splitext(new_aggr_name) is_new_aggr_a_folder = ext == '' if is_new_aggr_a_folder: delete_old_xml_files(folder=new_aggr_name) try: # in case of fileset aggregation need to update aggregation folder attribute to the # new folder name aggregation = self.get_aggregation_by_name(orig_aggr_name) if aggregation.is_fileset: # update folder attribute of this fileset aggregation and all nested # fileset aggregations of this aggregation aggregation.update_folder(new_folder=new_aggr_name) except ObjectDoesNotExist: # not renaming a fileset aggregation folder pass self._recreate_xml_docs_for_folder(new_aggr_name) else: # check if there is a matching single file aggregation try: aggregation = self.get_aggregation_by_name(new_aggr_name) delete_old_xml_files() aggregation.create_aggregation_xml_documents() except ObjectDoesNotExist: # the file path *new_aggr_name* is not a single file aggregation - no more # action is needed pass def is_aggregation_xml_file(self, file_path): """ determine whether a given file in the file hierarchy is metadata. This is true if it is listed as metadata in any logical file. """ if not (file_path.endswith('_meta.xml') or file_path.endswith('_resmap.xml')): return False for logical_file in self.logical_files: if logical_file.metadata_file_path == file_path or \ logical_file.map_file_path == file_path: return True return False def supports_folder_creation(self, folder_full_path): """this checks if it is allowed to create a folder at the specified path :param folder_full_path: the target path where the new folder needs to be created :return True or False """ if __debug__: assert(folder_full_path.startswith(self.file_path)) # determine containing folder if "/" in folder_full_path: path_to_check, _ = os.path.split(folder_full_path) else: path_to_check = folder_full_path # find if the path represents a multi-file aggregation if path_to_check.startswith(self.file_path): aggregation_path = path_to_check[len(self.file_path) + 1:] else: aggregation_path = path_to_check try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.can_contain_folders except ObjectDoesNotExist: # target path doesn't represent an aggregation - so it is OK to create a folder pass return True def supports_rename_path(self, src_full_path, tgt_full_path): """checks if file/folder rename/move is allowed :param src_full_path: name of the file/folder path to be renamed :param tgt_full_path: new name for file/folder path :return True or False """ if __debug__: assert(src_full_path.startswith(self.file_path)) assert(tgt_full_path.startswith(self.file_path)) istorage = self.get_irods_storage() # need to find out which of the following actions the user is trying to do: # renaming a file # renaming a folder # moving a file # moving a folder is_renaming_file = False is_moving_file = False is_moving_folder = False tgt_folder, tgt_file_name = os.path.split(tgt_full_path) _, tgt_ext = os.path.splitext(tgt_file_name) if tgt_ext: tgt_file_dir = os.path.dirname(tgt_full_path) else: tgt_file_dir = tgt_full_path src_folder, src_file_name = os.path.split(src_full_path) _, src_ext = os.path.splitext(src_file_name) if src_ext: src_file_dir = os.path.dirname(src_full_path) else: src_file_dir = src_full_path if src_ext and tgt_ext: is_renaming_file = True elif src_ext: is_moving_file = True elif not istorage.exists(tgt_file_dir): # renaming folder - no restriction return True else: is_moving_folder = True def check_file_rename_or_move(): # see if the folder containing the file represents an aggregation if src_file_dir != self.file_path: aggregation_path = src_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.supports_resource_file_rename except ObjectDoesNotExist: # check if the source file represents an aggregation aggregation_path = os.path.join(aggregation_path, src_file_name) aggregation = self.get_aggregation_by_name(aggregation_path) if is_renaming_file: return aggregation.supports_resource_file_rename else: return aggregation.supports_resource_file_move else: # check if the source file represents an aggregation aggregation_path = src_file_name aggregation = self.get_aggregation_by_name(aggregation_path) if is_renaming_file: return aggregation.supports_resource_file_rename else: return aggregation.supports_resource_file_move if is_renaming_file: # see if the folder containing the file represents an aggregation try: can_rename = check_file_rename_or_move() return can_rename except ObjectDoesNotExist: return True elif is_moving_file: # check source - see if the folder containing the file represents an aggregation try: can_move = check_file_rename_or_move() if not can_move: return can_move except ObjectDoesNotExist: pass # check target folder only if it is not the root if tgt_file_dir != self.file_path: aggregation_path = tgt_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.supports_resource_file_add except ObjectDoesNotExist: # target folder is not an aggregation - no restriction return True return True elif is_moving_folder: # no check on source is needed in this case # check target - only if it is not the root if tgt_file_dir != self.file_path: aggregation_path = tgt_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.can_contain_folders except ObjectDoesNotExist: # target folder doesn't represent an aggregation - no restriction return True return True def can_add_files(self, target_full_path): """ checks if file(s) can be uploaded to the specified *target_full_path* :param target_full_path: full folder path name where file needs to be uploaded to :return: True or False """ istorage = self.get_irods_storage() if istorage.exists(target_full_path): path_to_check = target_full_path else: return False if not path_to_check.endswith("data/contents"): # it is not the base directory - it must be a directory under base dir if path_to_check.startswith(self.file_path): aggregation_path = path_to_check[len(self.file_path) + 1:] else: aggregation_path = path_to_check try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.supports_resource_file_add except ObjectDoesNotExist: # target path doesn't represent an aggregation - so it is OK to add a file pass return True def supports_zip(self, folder_to_zip): """check if the given folder can be zipped or not""" # find all the resource files in the folder to be zipped # this is being passed both qualified and unqualified paths! full_path = folder_to_zip if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) # get all resource files at full_path and its sub-folders res_file_objects = ResourceFile.list_folder(self, full_path) # check any logical file associated with the resource file supports zip functionality for res_file in res_file_objects: if res_file.has_logical_file: if not res_file.logical_file.supports_zip: return False return True def supports_delete_folder_on_zip(self, original_folder): """check if the specified folder can be deleted at the end of zipping that folder""" # find all the resource files in the folder to be deleted # this is being passed both qualified and unqualified paths! full_path = original_folder if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) # get all resource files at full_path and its sub-folders res_file_objects = ResourceFile.list_folder(self, full_path) # check any logical file associated with the resource file supports deleting the folder # after its zipped for res_file in res_file_objects: if res_file.has_logical_file: if not res_file.logical_file.supports_delete_folder_on_zip: return False return True def get_missing_file_type_metadata_info(self): # this is used in page pre-processor to build the context # so that the landing page can show what metadata items are missing for each # logical file/aggregation metadata_missing_info = [] for lfo in self.logical_files: if not lfo.metadata.has_all_required_elements(): missing_elements = lfo.metadata.get_required_missing_elements() metadata_missing_info.append({'file_path': lfo.aggregation_name, 'missing_elements': missing_elements}) return metadata_missing_info def delete_coverage(self, coverage_type): """Deletes coverage data for the resource :param coverage_type: A value of either 'spatial' or 'temporal :return: """ if coverage_type.lower() == 'spatial' and self.metadata.spatial_coverage: self.metadata.spatial_coverage.delete() self.metadata.is_dirty = True self.metadata.save() elif coverage_type.lower() == 'temporal' and self.metadata.temporal_coverage: self.metadata.temporal_coverage.delete() self.metadata.is_dirty = True self.metadata.save() def update_coverage(self): """Update resource spatial and temporal coverage based on the corresponding coverages from all the contained aggregations (logical file) only if the resource coverage is not already set""" # update resource spatial coverage only if there is no spatial coverage already if self.metadata.spatial_coverage is None: self.update_spatial_coverage() # update resource temporal coverage only if there is no temporal coverage already if self.metadata.temporal_coverage is None: self.update_temporal_coverage() def update_spatial_coverage(self): """Updates resource spatial coverage based on the contained spatial coverages of aggregations (file type). Note: This action will overwrite any existing resource spatial coverage data. """ update_target_spatial_coverage(self) def update_temporal_coverage(self): """Updates resource temporal coverage based on the contained temporal coverages of aggregations (file type). Note: This action will overwrite any existing resource temporal coverage data. """ update_target_temporal_coverage(self)
class CompositeResource(BaseResource): objects = ResourceManager("CompositeResource") class Meta: verbose_name = 'Composite Resource' proxy = True @property def can_be_public_or_discoverable(self): # resource level metadata check if not super(CompositeResource, self).can_be_public_or_discoverable: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False return True def set_default_logical_file(self): """sets an instance of GenericLogicalFile to any resource file objects of this instance of the resource that is not already associated with a logical file. """ for res_file in self.files.all(): if not res_file.has_logical_file: logical_file = GenericLogicalFile.create() res_file.logical_file_content_object = logical_file res_file.save() @property def supports_logical_file(self): """ if this resource allows associating resource file objects with logical file""" return True def get_metadata_xml(self, pretty_print=True): from lxml import etree # get resource level core metadata as xml string xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False) # add file type metadata xml # create an etree xml object RDF_ROOT = etree.fromstring(xml_string) # get root 'Description' element that contains all other elements container = RDF_ROOT.find('rdf:Description', namespaces=self.metadata.NAMESPACES) for lf in self.logical_files: lf.metadata.add_to_xml_container(container) return etree.tostring(RDF_ROOT, pretty_print=pretty_print) def supports_folder_creation(self, folder_full_path): """this checks if it is allowed to create a folder at the specified path""" if "/" in folder_full_path: path_to_check = folder_full_path[:folder_full_path.rfind("/")] else: path_to_check = folder_full_path if not path_to_check.endswith("/data/contents"): res_file_objs = [ res_file_obj for res_file_obj in self.files.all() if res_file_obj.dir_path == path_to_check ] for res_file_obj in res_file_objs: if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False return True def supports_rename_path(self, src_full_path, tgt_full_path): """checks if file/folder rename/move is allowed""" istorage = self.get_irods_storage() tgt_file_dir = os.path.dirname(tgt_full_path) src_file_dir = os.path.dirname(src_full_path) def check_directory(): path_to_check = '' if istorage.exists(tgt_file_dir): path_to_check = tgt_file_dir else: if tgt_file_dir.startswith(src_file_dir): path_to_check = src_file_dir if path_to_check and not path_to_check.endswith("data/contents"): # it is not the base directory - it must be a directory under base dir res_file_objs = [ res_file_obj for res_file_obj in self.files.all() if res_file_obj.dir_path == path_to_check ] for res_file_obj in res_file_objs: if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False return True res_file_objs = [ res_file_obj for res_file_obj in self.files.all() if res_file_obj.full_path == src_full_path ] if res_file_objs: res_file_obj = res_file_objs[0] # src_full_path contains file name if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False # check if the target directory allows stuff to be moved there return check_directory() else: # src_full_path is a folder path without file name # tgt_full_path also must be a folder path without file name # check that if the target folder contains any files and if any of those files # allow moving stuff there return check_directory() def supports_zip(self, folder_to_zip): """check if the given folder can be zipped or not""" # find all the resource files in the folder to be zipped if self.resource_federation_path: res_file_objects = self.files.filter( object_id=self.id, fed_resource_file_name_or_path__contains=folder_to_zip).all() else: res_file_objects = self.files.filter( object_id=self.id, resource_file__contains=folder_to_zip).all() # check any logical file associated with the resource file supports zip functionality for res_file in res_file_objects: if not res_file.logical_file.supports_zip: return False return True def supports_delete_folder_on_zip(self, original_folder): """check if the specified folder can be deleted at the end of zipping that folder""" # find all the resource files in the folder to be deleted if self.resource_federation_path: res_file_objects = self.files.filter( object_id=self.id, fed_resource_file_name_or_path__contains=original_folder).all( ) else: res_file_objects = self.files.filter( object_id=self.id, resource_file__contains=original_folder).all() # check any logical file associated with the resource file supports deleting the folder # after its zipped for res_file in res_file_objects: if not res_file.logical_file.supports_delete_folder_on_zip: return False return True
class MODFLOWModelInstanceResource(BaseResource): objects = ResourceManager("MODFLOWModelInstanceResource") discovery_content_type = 'MODFLOW Model Instance' # used during discovery class Meta: verbose_name = 'MODFLOW Model Instance Resource' proxy = True @classmethod def get_metadata_class(cls): return MODFLOWModelInstanceMetaData def check_content_files(self): """ like 'has_required_content_files()' this method checks that one and only of .nam or .mfn exists, but unlike 'has_required_content_files()' this method returns which files are missing so that more information can be returned to the user via the interface :return: -['.nam or .mfn'] if there are no files or if there are files but no .nam or .mfn file -'multiple_nam' if more than one .nam file has been uploaded -missing_files, a list of file names that are included in the .nam file but have not been uploaded by the user """ missing_files = [] if self.files.all().count() >= 1: nam_files, reqd_files, existing_files, model_packages = self.find_content_files( ) if not nam_files: return ['.nam or .mfn'] else: if nam_files > 1: return 'multiple_nam' else: for f in reqd_files: if f not in existing_files: missing_files.append(f) return missing_files else: return ['.nam or .mfn '] def find_content_files(self): """ loops through uploaded files to count the .nam and/or .mfn files, creates a list of required files (file names listed in the .nam or .mfn file needed to run the model), and a list of existing file names :return: -nam_file_count, (int), the number of .nam files uploaded -reqd_files, (list of strings), the files listed in the .nam file that should be included for the model to run -existing_files, (list of strings), the names of files that have been uploaded -model packages (list of strings), the names of the packages for the uploaded model found in the .nam or .mfn file. These are the entries in the far left column of the .nam or .mfn file. """ nam_file_count = 0 existing_files = [] reqd_files = [] model_packages = [] for res_file in self.files.all(): ext = res_file.extension existing_files.append(res_file.file_name) if ext == '.nam' or ext == '.mfn': nam_file_count += 1 name_file = res_file.resource_file.file for row in name_file: row = row.strip() row = row.replace("'", "") row = row.replace('"', "") row = row.split(" ") r = row[0].strip() if not r.startswith('#') and r != '' and r.lower() != 'list' \ and r.lower() != 'data' and r.lower() != 'data(binary)': reqd_file = row[-1].strip() reqd_files.append(reqd_file) model_package_name = row[0].strip() model_package_ext = reqd_file.split('.')[-1].upper() model_packages.extend( set([model_package_name, model_package_ext])) return nam_file_count, reqd_files, existing_files, model_packages
class CompositeResource(BaseResource): objects = ResourceManager("CompositeResource") class Meta: verbose_name = 'Composite Resource' proxy = True @property def can_be_public_or_discoverable(self): # resource level metadata check if not super(CompositeResource, self).can_be_public_or_discoverable: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False return True def set_default_logical_file(self): """sets an instance of GenericLogicalFile to any resource file objects of this instance of the resource that is not already associated with a logical file. """ for res_file in self.files.all(): if not res_file.has_logical_file: logical_file = GenericLogicalFile.create() res_file.logical_file_content_object = logical_file res_file.save() @property def supports_folders(self): """ allow folders for CompositeResources """ return True @property def supports_logical_file(self): """ if this resource allows associating resource file objects with logical file""" return True def get_metadata_xml(self, pretty_print=True, include_format_elements=True): from lxml import etree # get resource level core metadata as xml string # for composite resource we don't want the format elements at the resource level # as they are included at the file level xml node xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False, include_format_elements=False) # add file type metadata xml # create an etree xml object RDF_ROOT = etree.fromstring(xml_string) # get root 'Description' element that contains all other elements container = RDF_ROOT.find('rdf:Description', namespaces=self.metadata.NAMESPACES) for lf in self.logical_files: lf.metadata.add_to_xml_container(container) return etree.tostring(RDF_ROOT, pretty_print=pretty_print) def supports_folder_creation(self, folder_full_path): """this checks if it is allowed to create a folder at the specified path""" if __debug__: assert(folder_full_path.startswith(self.file_path)) # determine containing folder if "/" in folder_full_path: path_to_check, _ = os.path.split(folder_full_path) else: path_to_check = folder_full_path if path_to_check != self.file_path: res_file_objs = [res_file_obj for res_file_obj in self.files.all() if res_file_obj.dir_path == path_to_check] for res_file_obj in res_file_objs: if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False return True def supports_rename_path(self, src_full_path, tgt_full_path): """checks if file/folder rename/move is allowed""" if __debug__: assert(src_full_path.startswith(self.file_path)) assert(tgt_full_path.startswith(self.file_path)) istorage = self.get_irods_storage() tgt_file_dir = os.path.dirname(tgt_full_path) src_file_dir = os.path.dirname(src_full_path) def check_directory(): path_to_check = '' if istorage.exists(tgt_file_dir): path_to_check = tgt_file_dir else: if tgt_file_dir.startswith(src_file_dir): path_to_check = src_file_dir if path_to_check and not path_to_check.endswith("data/contents"): # it is not the base directory - it must be a directory under base dir res_file_objs = [res_file_obj for res_file_obj in self.files.all() if res_file_obj.dir_path == path_to_check] for res_file_obj in res_file_objs: if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False return True res_file_objs = [res_file_obj for res_file_obj in self.files.all() if res_file_obj.full_path == src_full_path] if res_file_objs: res_file_obj = res_file_objs[0] # src_full_path contains file name if not res_file_obj.logical_file.supports_resource_file_rename or \ not res_file_obj.logical_file.supports_resource_file_move: return False # check if the target directory allows stuff to be moved there return check_directory() else: # src_full_path is a folder path without file name # tgt_full_path also must be a folder path without file name # check that if the target folder contains any files and if any of those files # allow moving stuff there return check_directory() def can_add_files(self, target_full_path): """ checks if file(s) can be uploaded to the specified *target_full_path* :param target_full_path: full folder path name where file needs to be uploaded to :return: True or False """ istorage = self.get_irods_storage() if istorage.exists(target_full_path): path_to_check = target_full_path else: return False if not path_to_check.endswith("data/contents"): # it is not the base directory - it must be a directory under base dir res_file_objs = [res_file_obj for res_file_obj in self.files.all() if res_file_obj.dir_path == path_to_check] for res_file_obj in res_file_objs: if not res_file_obj.logical_file.supports_resource_file_add: return False return True def supports_zip(self, folder_to_zip): """check if the given folder can be zipped or not""" # find all the resource files in the folder to be zipped # this is being passed both qualified and unqualified paths! full_path = folder_to_zip if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) if self.is_federated: res_file_objects = self.files.filter( object_id=self.id, fed_resource_file__startswith=full_path).all() else: res_file_objects = self.files.filter(object_id=self.id, resource_file__startswith=full_path).all() # check any logical file associated with the resource file supports zip functionality for res_file in res_file_objects: if not res_file.logical_file.supports_zip: return False return True def supports_delete_folder_on_zip(self, original_folder): """check if the specified folder can be deleted at the end of zipping that folder""" # find all the resource files in the folder to be deleted # this is being passed both qualified and unqualified paths! full_path = original_folder if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) if self.is_federated: res_file_objects = self.files.filter( object_id=self.id, fed_resource_file__startswith=full_path).all() else: res_file_objects = self.files.filter( object_id=self.id, resource_file__startswith=full_path).all() # check any logical file associated with the resource file supports deleting the folder # after its zipped for res_file in res_file_objects: if not res_file.logical_file.supports_delete_folder_on_zip: return False return True def get_missing_file_type_metadata_info(self): # this is used in page pre-processor to build the context # so that the landing page can show what metadata items are missing for each logical file metadata_missing_info = [] for lfo in self.logical_files: if not lfo.metadata.has_all_required_elements(): file_path = lfo.files.first().short_path missing_elements = lfo.metadata.get_required_missing_elements() metadata_missing_info.append({'file_path': file_path, 'missing_elements': missing_elements}) return metadata_missing_info
class CompositeResource(BaseResource): objects = ResourceManager("CompositeResource") discovery_content_type = 'Composite' # used during discovery class Meta: verbose_name = 'Composite Resource' proxy = True @property def can_be_public_or_discoverable(self): # resource level metadata check if not super(CompositeResource, self).can_be_public_or_discoverable: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False return True @property def can_be_published(self): # resource level metadata check if not super(CompositeResource, self).can_be_published: return False # filetype level metadata check for lf in self.logical_files: if not lf.metadata.has_all_required_elements(): return False # url file cannot be published if 'url' in lf.extra_data: return False return True def set_default_logical_file(self): """sets an instance of GenericLogicalFile to any resource file objects of this instance of the resource that is not already associated with a logical file. """ for res_file in self.files.all(): if not res_file.has_logical_file: logical_file = GenericLogicalFile.create() res_file.logical_file_content_object = logical_file res_file.save() def get_folder_aggregation_object(self, dir_path): """Returns an aggregation (file type) object if the specified folder *dir_path* represents a file type aggregation (logical file), otherwise None. :param dir_path: Resource file directory path (full folder path starting with resource id) for which the aggregation object to be retrieved """ files_in_folder = [ res_file for res_file in self.files.all() if res_file.dir_path == dir_path ] for fl in files_in_folder: if fl.has_logical_file: return fl.logical_file return None def get_file_aggregation_object(self, file_path): """Returns an aggregation (file type) object if the specified file *file_path* represents a file type aggregation (logical file), otherwise None. :param file_path: Resource file path (full file path starting with resource id) for which the aggregation object to be retrieved """ for res_file in self.files.all(): if res_file.full_path == file_path: if res_file.has_logical_file: return res_file.logical_file return None def get_folder_aggregation_type_to_set(self, dir_path): """Returns an aggregation (file type) type that the specified folder *dir_path* can possibly be set to. :param dir_path: Resource file directory path (full folder path starting with resource id) for which the possible aggregation type that can be set needs to be determined :return If the specified folder is already represents an aggregation or does not contain suitable file(s) then returns "" (empty string). If the specified folder contains only the files that meet the requirements of a supported aggregation, and does not contain other folders or does not have a parent folder then return the class name of that matching aggregation type. """ aggregation_type_to_set = "" if self.get_folder_aggregation_object(dir_path) is not None: # target folder is already an aggregation return None istorage = self.get_irods_storage() irods_path = dir_path if self.is_federated: irods_path = os.path.join(self.resource_federation_path, irods_path) store = istorage.listdir(irods_path) if store[0]: # seems there are folders under dir_path - no aggregation type can be set if the target # folder contains other folders return None files_in_folder = [ res_file for res_file in self.files.all() if res_file.dir_path == dir_path ] if not files_in_folder: # folder is empty return None if len(files_in_folder) > 1: # check for geo feature aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for raster aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set else: # check for raster aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for NetCDF aggregation type aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set # check for TimeSeries aggregation type aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type( files_in_folder) if aggregation_type_to_set: return aggregation_type_to_set return None @property def supports_folders(self): """ allow folders for CompositeResources """ return True @property def supports_logical_file(self): """ if this resource allows associating resource file objects with logical file""" return True def get_metadata_xml(self, pretty_print=True, include_format_elements=True): from lxml import etree # get resource level core metadata as xml string # for composite resource we don't want the format elements at the resource level # as they are included at the aggregation map xml document xml_string = super(CompositeResource, self).get_metadata_xml( pretty_print=False, include_format_elements=False) # create an etree xml object RDF_ROOT = etree.fromstring(xml_string) return etree.tostring(RDF_ROOT, pretty_print=pretty_print) def create_aggregation_xml_documents(self, aggregation_name=None): """Creates aggregation map and metadata xml files for each of the contained aggregations :param aggregation_name: (optional) name of the the specific aggregation for which xml documents need to be created """ if aggregation_name is None: for aggregation in self.logical_files: if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() else: try: aggregation = self.get_aggregation_by_name(aggregation_name) if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() except ObjectDoesNotExist: # aggregation_name must be a folder path that doesn't represent an aggregation # there may be single file aggregation in that folder for which xml documents # need to be created self._recreate_xml_docs_for_folder(aggregation_name, check_metadata_dirty=True) def _recreate_xml_docs_for_folder(self, folder, check_metadata_dirty=False): """Re-creates xml metadata and map documents associated with the specified folder. If the *folder* represents an aggregation then map and metadata xml documents are recreated only for that aggregation. Otherwise, xml documents are created for any aggregation that may exist in the specified folder and its sub-folders. :param folder: folder for which xml documents need to be re-created :param check_metadata_dirty: if true, then xml files will be created only if the aggregation metadata is dirty """ # first check if the the folder represents an aggregation try: aggregation = self.get_aggregation_by_name(folder) if check_metadata_dirty: if aggregation.metadata.is_dirty: aggregation.create_aggregation_xml_documents() else: aggregation.create_aggregation_xml_documents() # if we found an aggregation by the folder name that means this folder doesn't # have any sub folders as multi-file aggregation folder can't have sub folders except ObjectDoesNotExist: # create xml map and metadata xml documents for all aggregations that exist # in *folder* and its sub-folders if not folder.startswith(self.file_path): folder = os.path.join(self.file_path, folder) res_file_objects = ResourceFile.list_folder(self, folder) aggregations = [] for res_file in res_file_objects: if res_file.has_logical_file and res_file.logical_file not in aggregations: aggregations.append(res_file.logical_file) if check_metadata_dirty: aggregations = [ aggr for aggr in aggregations if aggr.metadata.is_dirty ] for aggregation in aggregations: aggregation.create_aggregation_xml_documents() def get_aggregation_by_name(self, name): """Get an aggregation that matches the aggregation name specified by *name* :param name: name (aggregation path) of the aggregation to find :return an aggregation object if found :raises ObjectDoesNotExist if no matching aggregation is found """ for aggregation in self.logical_files: # remove the last slash in aggregation_name if any if aggregation.aggregation_name.rstrip('/') == name: return aggregation raise ObjectDoesNotExist( "No matching aggregation was found for name:{}".format(name)) def recreate_aggregation_xml_docs(self, orig_aggr_name, new_aggr_name): """ When a folder or file representing an aggregation is renamed or moved, the associated map and metadata xml documents are deleted and then regenerated :param orig_aggr_name: original aggregation name - used for deleting existing xml documents :param new_aggr_name: new aggregation name - used for finding a matching aggregation so that new xml documents can be recreated """ def delete_old_xml_files(folder=''): istorage = self.get_irods_storage() meta_xml_file_name = orig_aggr_name + "_meta.xml" map_xml_file_name = orig_aggr_name + "_resmap.xml" if not folder: # case if file rename/move for single file aggregation meta_xml_file_full_path = os.path.join(self.file_path, meta_xml_file_name) map_xml_file_full_path = os.path.join(self.file_path, map_xml_file_name) else: # case of folder rename - multi-file aggregation _, meta_xml_file_name = os.path.split(meta_xml_file_name) _, map_xml_file_name = os.path.split(map_xml_file_name) meta_xml_file_full_path = os.path.join(self.file_path, folder, meta_xml_file_name) map_xml_file_full_path = os.path.join(self.file_path, folder, map_xml_file_name) if istorage.exists(meta_xml_file_full_path): istorage.delete(meta_xml_file_full_path) if istorage.exists(map_xml_file_full_path): istorage.delete(map_xml_file_full_path) # first check if the new_aggr_name is a folder path or file path name, ext = os.path.splitext(new_aggr_name) is_new_aggr_a_folder = ext == '' if is_new_aggr_a_folder: delete_old_xml_files(folder=new_aggr_name) self._recreate_xml_docs_for_folder(new_aggr_name) else: # check if there is a matching single file aggregation try: aggregation = self.get_aggregation_by_name(new_aggr_name) delete_old_xml_files() aggregation.create_aggregation_xml_documents() except ObjectDoesNotExist: # the file path *new_aggr_name* is not a single file aggregation - no more # action is needed pass def is_aggregation_xml_file(self, file_path): """ determine whether a given file in the file hierarchy is metadata. This is true if it is listed as metadata in any logical file. """ if not (file_path.endswith('_meta.xml') or file_path.endswith('_resmap.xml')): return False for logical_file in self.logical_files: if logical_file.metadata_file_path == file_path or \ logical_file.map_file_path == file_path: return True return False def supports_folder_creation(self, folder_full_path): """this checks if it is allowed to create a folder at the specified path :param folder_full_path: the target path where the new folder needs to be created :return True or False """ if __debug__: assert (folder_full_path.startswith(self.file_path)) # determine containing folder if "/" in folder_full_path: path_to_check, _ = os.path.split(folder_full_path) else: path_to_check = folder_full_path # find if the path represents a multi-file aggregation if path_to_check.startswith(self.file_path): aggregation_path = path_to_check[len(self.file_path) + 1:] else: aggregation_path = path_to_check try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.can_contain_folders except ObjectDoesNotExist: # target path doesn't represent an aggregation - so it is OK to create a folder pass return True def supports_rename_path(self, src_full_path, tgt_full_path): """checks if file/folder rename/move is allowed :param src_full_path: name of the file/folder path to be renamed :param tgt_full_path: new name for file/folder path :return True or False """ if __debug__: assert (src_full_path.startswith(self.file_path)) assert (tgt_full_path.startswith(self.file_path)) istorage = self.get_irods_storage() # need to find out which of the following actions the user is trying to do: # renaming a file # renaming a folder # moving a file # moving a folder is_renaming_file = False is_moving_file = False is_moving_folder = False tgt_folder, tgt_file_name = os.path.split(tgt_full_path) _, tgt_ext = os.path.splitext(tgt_file_name) if tgt_ext: tgt_file_dir = os.path.dirname(tgt_full_path) else: tgt_file_dir = tgt_full_path src_folder, src_file_name = os.path.split(src_full_path) _, src_ext = os.path.splitext(src_file_name) if src_ext: src_file_dir = os.path.dirname(src_full_path) else: src_file_dir = src_full_path if src_ext and tgt_ext: is_renaming_file = True elif src_ext: is_moving_file = True elif not istorage.exists(tgt_file_dir): # renaming folder - no restriction return True else: is_moving_folder = True def check_file_rename_or_move(): # see if the folder containing the file represents an aggregation if src_file_dir != self.file_path: aggregation_path = src_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name( aggregation_path) return aggregation.supports_resource_file_rename except ObjectDoesNotExist: # check if the source file represents an aggregation aggregation_path = os.path.join(aggregation_path, src_file_name) aggregation = self.get_aggregation_by_name( aggregation_path) if is_renaming_file: return aggregation.supports_resource_file_rename else: return aggregation.supports_resource_file_move else: # check if the source file represents an aggregation aggregation_path = src_file_name aggregation = self.get_aggregation_by_name(aggregation_path) if is_renaming_file: return aggregation.supports_resource_file_rename else: return aggregation.supports_resource_file_move if is_renaming_file: # see if the folder containing the file represents an aggregation try: can_rename = check_file_rename_or_move() return can_rename except ObjectDoesNotExist: return True elif is_moving_file: # check source - see if the folder containing the file represents an aggregation try: can_move = check_file_rename_or_move() if not can_move: return can_move except ObjectDoesNotExist: pass # check target folder only if it is not the root if tgt_file_dir != self.file_path: aggregation_path = tgt_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name( aggregation_path) return aggregation.supports_resource_file_add except ObjectDoesNotExist: # target folder is not an aggregation - no restriction return True return True elif is_moving_folder: # no check on source is needed in this case # check target - only if it is not the root if tgt_file_dir != self.file_path: aggregation_path = tgt_file_dir[len(self.file_path) + 1:] try: aggregation = self.get_aggregation_by_name( aggregation_path) return aggregation.can_contain_folders except ObjectDoesNotExist: # target folder doesn't represent an aggrgation - no restriction return True return True def can_add_files(self, target_full_path): """ checks if file(s) can be uploaded to the specified *target_full_path* :param target_full_path: full folder path name where file needs to be uploaded to :return: True or False """ istorage = self.get_irods_storage() if istorage.exists(target_full_path): path_to_check = target_full_path else: return False if not path_to_check.endswith("data/contents"): # it is not the base directory - it must be a directory under base dir if path_to_check.startswith(self.file_path): aggregation_path = path_to_check[len(self.file_path) + 1:] else: aggregation_path = path_to_check try: aggregation = self.get_aggregation_by_name(aggregation_path) return aggregation.supports_resource_file_add except ObjectDoesNotExist: # target path doesn't represent an aggregation - so it is OK to add a file pass return True def supports_zip(self, folder_to_zip): """check if the given folder can be zipped or not""" # find all the resource files in the folder to be zipped # this is being passed both qualified and unqualified paths! full_path = folder_to_zip if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) # get all resource files at full_path and its sub-folders res_file_objects = ResourceFile.list_folder(self, full_path) # check any logical file associated with the resource file supports zip functionality for res_file in res_file_objects: if res_file.has_logical_file: if not res_file.logical_file.supports_zip: return False return True def supports_delete_folder_on_zip(self, original_folder): """check if the specified folder can be deleted at the end of zipping that folder""" # find all the resource files in the folder to be deleted # this is being passed both qualified and unqualified paths! full_path = original_folder if not full_path.startswith(self.file_path): full_path = os.path.join(self.file_path, full_path) # get all resource files at full_path and its sub-folders res_file_objects = ResourceFile.list_folder(self, full_path) # check any logical file associated with the resource file supports deleting the folder # after its zipped for res_file in res_file_objects: if res_file.has_logical_file: if not res_file.logical_file.supports_delete_folder_on_zip: return False return True def get_missing_file_type_metadata_info(self): # this is used in page pre-processor to build the context # so that the landing page can show what metadata items are missing for each # logical file/aggregation metadata_missing_info = [] for lfo in self.logical_files: if not lfo.metadata.has_all_required_elements(): missing_elements = lfo.metadata.get_required_missing_elements() metadata_missing_info.append({ 'file_path': lfo.aggregation_name, 'missing_elements': missing_elements }) return metadata_missing_info def update_coverage(self): """Update resource spatial and temporal coverage based on the corresponding coverages from all the contained aggregations (logical file) only if the resource coverage is not already set""" # update resource spatial coverage only if there is no spatial coverage already if self.metadata.spatial_coverage is None: self.update_spatial_coverage() # update resource temporal coverage only if there is no temporal coverage already if self.metadata.temporal_coverage is None: self.update_temporal_coverage() def update_spatial_coverage(self): """Updates resource spatial coverage based on the contained spatial coverages of aggregations (file type). Note: This action will overwrite any existing resource spatial coverage data. """ spatial_coverages = [ lf.metadata.spatial_coverage for lf in self.logical_files if lf.metadata.spatial_coverage is not None ] if not spatial_coverages: # no aggregation level spatial coverage data exist - no need to update resource # spatial coverage return bbox_limits = { 'box': { 'northlimit': 'northlimit', 'southlimit': 'southlimit', 'eastlimit': 'eastlimit', 'westlimit': 'westlimit' }, 'point': { 'northlimit': 'north', 'southlimit': 'north', 'eastlimit': 'east', 'westlimit': 'east' } } def set_coverage_data(res_coverage_value, lfo_coverage_element, box_limits): comparison_operator = { 'northlimit': lt, 'southlimit': gt, 'eastlimit': lt, 'westlimit': gt } for key in comparison_operator.keys(): if comparison_operator[key]( res_coverage_value[key], lfo_coverage_element.value[box_limits[key]]): res_coverage_value[key] = lfo_coverage_element.value[ box_limits[key]] cov_type = "point" bbox_value = { 'northlimit': -90, 'southlimit': 90, 'eastlimit': -180, 'westlimit': 180, 'projection': 'WGS 84 EPSG:4326', 'units': "Decimal degrees" } if len(spatial_coverages) > 1: # check if one of the coverage is of type box if any(sp_cov.type == 'box' for sp_cov in spatial_coverages): cov_type = 'box' else: # check if the coverages represent different locations unique_lats = set( [sp_cov.value['north'] for sp_cov in spatial_coverages]) unique_lons = set( [sp_cov.value['east'] for sp_cov in spatial_coverages]) if len(unique_lats) == 1 and len(unique_lons) == 1: cov_type = 'point' else: cov_type = 'box' if cov_type == 'point': sp_cov = spatial_coverages[0] bbox_value = dict() bbox_value['projection'] = 'WGS 84 EPSG:4326' bbox_value['units'] = 'Decimal degrees' bbox_value['north'] = sp_cov.value['north'] bbox_value['east'] = sp_cov.value['east'] else: for sp_cov in spatial_coverages: if sp_cov.type == "box": box_limits = bbox_limits['box'] set_coverage_data(bbox_value, sp_cov, box_limits) else: # point type coverage box_limits = bbox_limits['point'] set_coverage_data(bbox_value, sp_cov, box_limits) elif len(spatial_coverages) == 1: sp_cov = spatial_coverages[0] if sp_cov.type == "box": cov_type = 'box' bbox_value['projection'] = 'WGS 84 EPSG:4326' bbox_value['units'] = 'Decimal degrees' bbox_value['northlimit'] = sp_cov.value['northlimit'] bbox_value['eastlimit'] = sp_cov.value['eastlimit'] bbox_value['southlimit'] = sp_cov.value['southlimit'] bbox_value['westlimit'] = sp_cov.value['westlimit'] else: # point type coverage cov_type = "point" bbox_value = dict() bbox_value['projection'] = 'WGS 84 EPSG:4326' bbox_value['units'] = 'Decimal degrees' bbox_value['north'] = sp_cov.value['north'] bbox_value['east'] = sp_cov.value['east'] spatial_cov = self.metadata.spatial_coverage if spatial_cov: spatial_cov.type = cov_type place_name = spatial_cov.value.get('name', None) if place_name is not None: bbox_value['name'] = place_name spatial_cov._value = json.dumps(bbox_value) spatial_cov.save() else: self.metadata.create_element("coverage", type=cov_type, value=bbox_value) def update_temporal_coverage(self): """Updates resource temporal coverage based on the contained temporal coverages of aggregations (file type). Note: This action will overwrite any existing resource temporal coverage data. """ temporal_coverages = [ lf.metadata.temporal_coverage for lf in self.logical_files if lf.metadata.temporal_coverage is not None ] if not temporal_coverages: # no aggregation level temporal coverage data - no update at resource level is needed return date_data = {'start': None, 'end': None} def set_date_value(date_data, coverage_element, key): comparison_operator = gt if key == 'start' else lt if date_data[key] is None: date_data[key] = coverage_element.value[key] else: if comparison_operator( parser.parse(date_data[key]), parser.parse(coverage_element.value[key])): date_data[key] = coverage_element.value[key] for temp_cov in temporal_coverages: start_date = parser.parse(temp_cov.value['start']) end_date = parser.parse(temp_cov.value['end']) temp_cov.value['start'] = start_date.strftime('%m/%d/%Y') temp_cov.value['end'] = end_date.strftime('%m/%d/%Y') set_date_value(date_data, temp_cov, 'start') set_date_value(date_data, temp_cov, 'end') temp_cov = self.metadata.temporal_coverage if date_data['start'] is not None and date_data['end'] is not None: if temp_cov: temp_cov._value = json.dumps(date_data) temp_cov.save() else: self.metadata.create_element("coverage", type='period', value=date_data)