class GeographicFeatureResource(BaseResource):
objects = ResourceManager("GeographicFeatureResource")
@property
def metadata(self):
md = GeographicFeatureMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# See Shapefile format:
# http://resources.arcgis.com/en/help/main/10.2/index.html#//005600000003000000
return (".zip", ".shp", ".shx", ".dbf", ".prj", ".sbx", ".sbn", ".cpg",
".xml", ".fbn", ".fbx", ".ain", ".aih", ".atx", ".ixs", ".mxs")
# add resource-specific HS terms
def get_hs_term_dict(self):
# get existing hs_term_dict from base class
hs_term_dict = super(GeographicFeatureResource,
self).get_hs_term_dict()
geometryinformation = self.metadata.geometryinformation.all().first()
if geometryinformation is not None:
hs_term_dict[
"HS_GFR_FEATURE_COUNT"] = geometryinformation.featureCount
else:
hs_term_dict["HS_GFR_FEATURE_COUNT"] = 0
return hs_term_dict
class Meta:
verbose_name = 'Geographic Feature (ESRI Shapefiles)'
proxy = True
class ToolResource(BaseResource):
objects = ResourceManager('ToolResource')
class Meta:
proxy = True
verbose_name = 'Web App Resource'
@classmethod
def get_supported_upload_file_types(cls):
# no file types are supported
return ()
@classmethod
def allow_multiple_file_upload(cls):
# no file can be uploaded
return False
@classmethod
def can_have_multiple_files(cls):
# resource can't have any files
return False
@property
def metadata(self):
md = ToolMetaData()
return self._get_metadata(md)
@property
def can_be_published(self):
return False
class RefTimeSeriesResource(BaseResource):
objects = ResourceManager("RefTimeSeriesResource")
class Meta:
verbose_name = "HIS Referenced Time Series"
proxy = True
@property
def metadata(self):
md = RefTSMetadata()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# no file types are supported
return ()
@classmethod
def allow_multiple_file_upload(cls):
# no file can be uploaded
return False
@classmethod
def can_have_multiple_files(cls):
# resource can't have any files
return False
class RefTimeSeriesResource(BaseResource):
objects = ResourceManager("RefTimeSeriesResource")
discovery_content_type = "Reference to HIS Time Series" # used during discovery
class Meta:
verbose_name = "HIS Referenced Time Series"
proxy = True
@classmethod
def get_metadata_class(cls):
return RefTSMetadata
@classmethod
def get_supported_upload_file_types(cls):
# no file types are supported
return ()
@classmethod
def allow_multiple_file_upload(cls):
# no file can be uploaded
return False
@classmethod
def can_have_multiple_files(cls):
# resource can't have any files
return False
class RasterResource(BaseResource):
objects = ResourceManager("RasterResource")
class Meta:
verbose_name = 'Geographic Raster'
proxy = True
@property
def metadata(self):
md = RasterMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# only tif file type is supported
return (".tif", ".zip")
@classmethod
def allow_multiple_file_upload(cls):
# can upload only 1 file
return False
@classmethod
def can_have_multiple_files(cls):
# can have only 1 file
return False
class ModelInstanceResource(BaseResource):
objects = ResourceManager("ModelInstanceResource")
class Meta:
verbose_name = 'Model Instance Resource'
proxy = True
@property
def metadata(self):
md = ModelInstanceMetaData()
return self._get_metadata(md)
class CollectionResource(BaseResource):
objects = ResourceManager('CollectionResource')
discovery_content_type = 'Collection' # used during discovery
class Meta:
proxy = True
verbose_name = 'Collection Resource'
@classmethod
def get_supported_upload_file_types(cls):
# no file types are supported
return ()
@classmethod
def allow_multiple_file_upload(cls):
# cannot upload any file
return False
@classmethod
def can_have_multiple_files(cls):
# resource can't have any files
return False
@property
def can_be_public_or_discoverable(self):
return self.metadata.has_all_required_elements() and (
self.resources.count() > 0)
@property
def deleted_resources(self):
return CollectionDeletedResource.objects.filter(collection=self)
@property
def has_resources(self):
return self.resources.count() > 0
@property
def are_all_contained_resources_published(self):
if not self.has_resources:
return False
return not self.resources.all().filter(
raccess__published=False).exists()
@property
def can_be_published(self):
return self.can_be_public_or_discoverable and \
self.are_all_contained_resources_published
@property
def update_text_file(self):
return self.extra_data.get('update_text_file', 'True')
class NetcdfResource(BaseResource):
objects = ResourceManager("NetcdfResource")
@property
def metadata(self):
md = NetcdfMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# only file with extension .nc is supported for uploading
return (".nc", )
@classmethod
def allow_multiple_file_upload(cls):
# can upload only 1 file
return False
@classmethod
def can_have_multiple_files(cls):
# can have only 1 file
return False
# add resource-specific HS terms
def get_hs_term_dict(self):
# get existing hs_term_dict from base class
hs_term_dict = super(NetcdfResource, self).get_hs_term_dict()
# add new terms for NetCDF res
hs_term_dict["HS_FILE_NAME"] = ""
for res_file in self.files.all():
_, f_fullname, f_ext = get_resource_file_name_and_extension(
res_file)
if f_ext.lower() == '.nc':
hs_term_dict["HS_FILE_NAME"] = f_fullname
break
return hs_term_dict
def update_netcdf_file(self, user):
if not self.metadata.is_dirty:
return
nc_res_file = get_resource_files_by_extension(self, ".nc")
txt_res_file = get_resource_files_by_extension(self, ".txt")
from hs_file_types.models.netcdf import netcdf_file_update # avoid recursive import
if nc_res_file and txt_res_file:
netcdf_file_update(self, nc_res_file[0], txt_res_file[0], user)
discovery_content_type = 'Multidimensional (NetCDF)' # used during discovery
class Meta:
verbose_name = 'Multidimensional (NetCDF)'
proxy = True
class ModelInstanceResource(BaseResource):
objects = ResourceManager("ModelInstanceResource")
discovery_content_type = 'Model Instance' # used during discovery
class Meta:
verbose_name = 'Model Instance Resource'
proxy = True
@classmethod
def get_metadata_class(cls):
return ModelInstanceMetaData
class ModelInstanceResource(BaseResource):
objects = ResourceManager("ModelInstanceResource")
discovery_content_type = 'Model Instance' # used during discovery
class Meta:
verbose_name = 'Model Instance Resource'
proxy = True
@property
def metadata(self):
md = ModelInstanceMetaData()
return self._get_metadata(md)
class ScriptResource(BaseResource):
objects = ResourceManager('ScriptResource')
class Meta:
proxy = True
verbose_name = 'Script Resource'
@classmethod
def get_supported_upload_file_types(cls):
# one file type is supported
return ".r", ".py", ".m"
@property
def metadata(self):
md = ScriptMetaData()
return self._get_metadata(md)
class SWATModelInstanceResource(BaseResource):
objects = ResourceManager("SWATModelInstanceResource")
class Meta:
verbose_name = 'SWAT Model Instance Resource'
proxy = True
@property
def metadata(self):
md = SWATModelInstanceMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# all file types are supported
return '.*'
class ScriptResource(BaseResource):
objects = ResourceManager('ScriptResource')
discovery_content_type = 'Script' # used during discovery
class Meta:
proxy = True
verbose_name = 'Script Resource'
@classmethod
def get_supported_upload_file_types(cls):
# one file type is supported
return ".r", ".py", ".m"
@classmethod
def get_metadata_class(cls):
return ScriptMetaData
class ModelProgramResource(BaseResource):
objects = ResourceManager("ModelProgramResource")
class Meta:
verbose_name = 'Model Program Resource'
proxy = True
@property
def metadata(self):
md = ModelProgramMetaData()
meta = self._get_metadata(md)
return meta
@classmethod
def get_supported_upload_file_types(cls):
# all file types are supported
return ('.*')
class SWATModelInstanceResource(BaseResource):
objects = ResourceManager("SWATModelInstanceResource")
discovery_content_type = 'SWAT Model Instance' # used during discovery
class Meta:
verbose_name = 'SWAT Model Instance Resource'
proxy = True
@classmethod
def get_metadata_class(cls):
return SWATModelInstanceMetaData
@classmethod
def get_supported_upload_file_types(cls):
# all file types are supported
return '.*'
class ModelProgramResource(BaseResource):
objects = ResourceManager("ModelProgramResource")
discovery_content_type = 'Model Program' # used during discovery
class Meta:
verbose_name = 'Model Program Resource'
proxy = True
@classmethod
def get_metadata_class(cls):
return ModelProgramMetaData
@classmethod
def get_supported_upload_file_types(cls):
# all file types are supported
return ('.*')
class ToolResource(BaseResource):
objects = ResourceManager('ToolResource')
discovery_content_type = 'Web App' # used during discovery
class Meta:
proxy = True
verbose_name = 'Web App Resource'
@classmethod
def get_approved_apps(cls):
webapp_resources = cls.objects.all()
final_resource_list = []
for resource in webapp_resources:
if resource.metadata.approved:
final_resource_list.append(resource)
return final_resource_list
@classmethod
def get_supported_upload_file_types(cls):
# no file types are supported
return ()
@classmethod
def allow_multiple_file_upload(cls):
# no file can be uploaded
return False
@classmethod
def can_have_multiple_files(cls):
# resource can't have any files
return False
@property
def metadata(self):
md = ToolMetaData()
return self._get_metadata(md)
@property
def can_be_published(self):
return False
class RasterResource(BaseResource):
objects = ResourceManager("RasterResource")
discovery_content_type = 'Geographic Raster' # used during discovery
class Meta:
verbose_name = 'Geographic Raster'
proxy = True
@property
def metadata(self):
md = RasterMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# only tif file type is supported
return (".tiff", ".tif", ".vrt", ".zip")
@classmethod
def allow_multiple_file_upload(cls):
# can upload multiple files
return True
@classmethod
def can_have_multiple_files(cls):
# can have only 1 file
return False
# add resource-specific HS terms
def get_hs_term_dict(self):
# get existing hs_term_dict from base class
hs_term_dict = super(RasterResource, self).get_hs_term_dict()
# add new terms for Raster res
hs_term_dict["HS_FILE_NAME"] = ""
for res_file in self.files.all():
_, f_fullname, f_ext = get_resource_file_name_and_extension(
res_file)
if f_ext.lower() == '.vrt':
hs_term_dict["HS_FILE_NAME"] = f_fullname
break
return hs_term_dict
class NetcdfResource(BaseResource):
objects = ResourceManager("NetcdfResource")
@property
def metadata(self):
md = NetcdfMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# only file with extension .nc is supported for uploading
return (".nc", )
@classmethod
def allow_multiple_file_upload(cls):
# can upload only 1 file
return False
@classmethod
def can_have_multiple_files(cls):
# can have only 1 file
return False
# add resource-specific HS terms
def get_hs_term_dict(self):
# get existing hs_term_dict from base class
hs_term_dict = super(NetcdfResource, self).get_hs_term_dict()
# add new terms for NetCDF res
hs_term_dict["HS_NETCDF_FILE_NAME"] = ""
for res_file in self.files.all():
_, f_fullname, f_ext = get_resource_file_name_and_extension(
res_file)
if f_ext.lower() == '.nc':
hs_term_dict["HS_NETCDF_FILE_NAME"] = f_fullname
break
return hs_term_dict
class Meta:
verbose_name = 'Multidimensional (NetCDF)'
proxy = True
class GeographicFeatureResource(BaseResource):
objects = ResourceManager("GeographicFeatureResource")
@property
def metadata(self):
md = GeographicFeatureMetaData()
return self._get_metadata(md)
@classmethod
def get_supported_upload_file_types(cls):
# See Shapefile format:
# http://resources.arcgis.com/en/help/main/10.2/index.html#//005600000003000000
return (".zip", ".shp", ".shx", ".dbf", ".prj", ".sbx", ".sbn", ".cpg",
".xml", ".fbn", ".fbx", ".ain", ".aih", ".atx", ".ixs", ".mxs")
def has_required_content_files(self):
if self.files.all().count < 3:
return False
file_extensions = [f.extension for f in self.files.all()]
return all(ext in file_extensions for ext in ['.shp', '.shx', '.dbf'])
def get_hs_term_dict(self):
# get existing hs_term_dict from base class
hs_term_dict = super(GeographicFeatureResource,
self).get_hs_term_dict()
geometryinformation = self.metadata.geometryinformation
if geometryinformation is not None:
hs_term_dict[
"HS_GFR_FEATURE_COUNT"] = geometryinformation.featureCount
else:
hs_term_dict["HS_GFR_FEATURE_COUNT"] = 0
return hs_term_dict
discovery_content_type = 'Geographic Feature (ESRI Shapefiles)' # used during discovery
class Meta:
verbose_name = 'Geographic Feature (ESRI Shapefiles)'
proxy = True
class CompositeResource(BaseResource):
objects = ResourceManager("CompositeResource")
discovery_content_type = 'Composite' # used during discovery
class Meta:
verbose_name = 'Composite Resource'
proxy = True
@property
def can_be_public_or_discoverable(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_public_or_discoverable:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
return True
@property
def logical_files(self):
"""Returns a list of all logical file type objects associated with this resource """
lf_list = []
lf_list.extend(self.filesetlogicalfile_set.all())
lf_list.extend(self.genericlogicalfile_set.all())
lf_list.extend(self.geofeaturelogicalfile_set.all())
lf_list.extend(self.netcdflogicalfile_set.all())
lf_list.extend(self.georasterlogicalfile_set.all())
lf_list.extend(self.reftimeserieslogicalfile_set.all())
lf_list.extend(self.timeserieslogicalfile_set.all())
return lf_list
@property
def can_be_published(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_published:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
# url file cannot be published
if 'url' in lf.extra_data:
return False
return True
def set_default_logical_file(self):
"""sets an instance of GenericLogicalFile to any resource file objects of this instance
of the resource that is not already associated with a logical file. """
for res_file in self.files.all():
if not res_file.has_logical_file:
logical_file = GenericLogicalFile.create()
res_file.logical_file_content_object = logical_file
res_file.save()
def get_folder_aggregation_object(self, dir_path):
"""Returns an aggregation (file type) object if the specified folder *dir_path* represents a
file type aggregation (logical file), otherwise None.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the aggregation object to be retrieved
"""
aggregation_path = dir_path[len(self.file_path) + 1:]
try:
return self.get_aggregation_by_name(aggregation_path)
except ObjectDoesNotExist:
return None
def get_file_aggregation_object(self, file_path):
"""Returns an aggregation (file type) object if the specified file *file_path* represents a
file type aggregation (logical file), otherwise None.
:param file_path: Resource file path (full file path starting with resource id)
for which the aggregation object to be retrieved
"""
for res_file in self.files.all():
if res_file.full_path == file_path:
if res_file.has_logical_file:
return res_file.logical_file
return None
def get_folder_aggregation_type_to_set(self, dir_path):
"""Returns an aggregation (file type) type that the specified folder *dir_path* can
possibly be set to.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the possible aggregation type that can be set needs to be determined
:return If the specified folder is already represents an aggregation or does
not contain suitable file(s) then returns "" (empty string). If the specified folder
contains only the files that meet the requirements of a supported aggregation, and
does not contain other folders or does not have a parent folder then return the
class name of that matching aggregation type.
"""
if self.get_folder_aggregation_object(dir_path) is not None:
# target folder is already an aggregation
return None
istorage = self.get_irods_storage()
irods_path = dir_path
if self.is_federated:
irods_path = os.path.join(self.resource_federation_path, irods_path)
store = istorage.listdir(irods_path)
files_in_folder = ResourceFile.list_folder(self, folder=irods_path, sub_folders=False)
if not files_in_folder:
# folder is empty
# check sub folders for files - if file exist we can set FileSet aggregation
files_in_sub_folders = ResourceFile.list_folder(self, folder=irods_path,
sub_folders=True)
if files_in_sub_folders:
return FileSetLogicalFile.__name__
return None
if store[0]:
# there are folders under dir_path as well as files - only FileSet can bet set
return FileSetLogicalFile.__name__
if len(files_in_folder) > 1:
# check for geo feature
aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
return FileSetLogicalFile.__name__
else:
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for NetCDF aggregation type
aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for TimeSeries aggregation type
aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
return FileSetLogicalFile.__name__
@property
def supports_folders(self):
""" allow folders for CompositeResources """
return True
@property
def supports_logical_file(self):
""" if this resource allows associating resource file objects with logical file"""
return True
def get_metadata_xml(self, pretty_print=True, include_format_elements=True):
from lxml import etree
# get resource level core metadata as xml string
# for composite resource we don't want the format elements at the resource level
# as they are included at the aggregation map xml document
xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False,
include_format_elements=False)
# create an etree xml object
RDF_ROOT = etree.fromstring(xml_string)
return etree.tostring(RDF_ROOT, pretty_print=pretty_print)
def create_aggregation_xml_documents(self, aggregation_name=None):
"""Creates aggregation map and metadata xml files for each of the contained aggregations
:param aggregation_name: (optional) name of the the specific aggregation for which xml
documents need to be created
"""
if aggregation_name is None:
for aggregation in self.logical_files:
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
else:
try:
aggregation = self.get_aggregation_by_name(aggregation_name)
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
except ObjectDoesNotExist:
# aggregation_name must be a folder path that doesn't represent an aggregation
# there may be single file aggregation in that folder for which xml documents
# need to be created
self._recreate_xml_docs_for_folder(aggregation_name, check_metadata_dirty=True)
def _recreate_xml_docs_for_folder(self, folder, check_metadata_dirty=False):
"""Re-creates xml metadata and map documents associated with the specified folder.
If the *folder* represents an aggregation then map and metadata xml documents are
recreated only for that aggregation. Otherwise, xml documents are created for any
aggregation that may exist in the specified folder and its sub-folders.
:param folder: folder for which xml documents need to be re-created
:param check_metadata_dirty: if true, then xml files will be created only if the
aggregation metadata is dirty
"""
# first check if the the folder represents an aggregation
try:
aggregation = self.get_aggregation_by_name(folder)
if check_metadata_dirty:
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
else:
aggregation.create_aggregation_xml_documents()
# if we found an aggregation by the folder name that means this folder doesn't
# have any sub folders as multi-file aggregation folder can't have sub folders
except ObjectDoesNotExist:
# create xml map and metadata xml documents for all aggregations that exist
# in *folder* and its sub-folders
if not folder.startswith(self.file_path):
folder = os.path.join(self.file_path, folder)
res_file_objects = ResourceFile.list_folder(self, folder)
aggregations = []
for res_file in res_file_objects:
if res_file.has_logical_file and res_file.logical_file not in aggregations:
aggregations.append(res_file.logical_file)
if check_metadata_dirty:
aggregations = [aggr for aggr in aggregations if aggr.metadata.is_dirty]
for aggregation in aggregations:
aggregation.create_aggregation_xml_documents()
def get_aggregation_by_name(self, name):
"""Get an aggregation that matches the aggregation name specified by *name*
:param name: name (aggregation path) of the aggregation to find
:return an aggregation object if found
:raises ObjectDoesNotExist if no matching aggregation is found
"""
for aggregation in self.logical_files:
# remove the last slash in aggregation_name if any
if aggregation.aggregation_name.rstrip('/') == name:
return aggregation
raise ObjectDoesNotExist("No matching aggregation was found for name:{}".format(name))
def get_fileset_aggregation_in_path(self, path):
"""Get the first fileset aggregation in the path moving up (towards the root)in the path
:param path: directory path in which to search for a fileset aggregation
:return a fileset aggregation object if found, otherwise None
"""
def get_fileset(path):
try:
aggregation = self.get_aggregation_by_name(path)
if aggregation.is_fileset:
return aggregation
except ObjectDoesNotExist:
return None
while '/' in path:
fileset = get_fileset(path)
if fileset is not None:
return fileset
path = os.path.dirname(path)
else:
return get_fileset(path)
def recreate_aggregation_xml_docs(self, orig_aggr_name, new_aggr_name):
"""
When a folder or file representing an aggregation is renamed or moved,
the associated map and metadata xml documents are deleted
and then regenerated
:param orig_aggr_name: original aggregation name - used for deleting existing
xml documents
:param new_aggr_name: new aggregation name - used for finding a matching
aggregation so that new xml documents can be recreated
"""
def delete_old_xml_files(folder=''):
istorage = self.get_irods_storage()
meta_xml_file_name = orig_aggr_name + "_meta.xml"
map_xml_file_name = orig_aggr_name + "_resmap.xml"
if not folder:
# case if file rename/move for single file aggregation
meta_xml_file_full_path = os.path.join(self.file_path, meta_xml_file_name)
map_xml_file_full_path = os.path.join(self.file_path, map_xml_file_name)
else:
# case of folder rename - multi-file aggregation
_, meta_xml_file_name = os.path.split(meta_xml_file_name)
_, map_xml_file_name = os.path.split(map_xml_file_name)
meta_xml_file_full_path = os.path.join(self.file_path, folder, meta_xml_file_name)
map_xml_file_full_path = os.path.join(self.file_path, folder, map_xml_file_name)
if istorage.exists(meta_xml_file_full_path):
istorage.delete(meta_xml_file_full_path)
if istorage.exists(map_xml_file_full_path):
istorage.delete(map_xml_file_full_path)
# first check if the new_aggr_name is a folder path or file path
name, ext = os.path.splitext(new_aggr_name)
is_new_aggr_a_folder = ext == ''
if is_new_aggr_a_folder:
delete_old_xml_files(folder=new_aggr_name)
try:
# in case of fileset aggregation need to update aggregation folder attribute to the
# new folder name
aggregation = self.get_aggregation_by_name(orig_aggr_name)
if aggregation.is_fileset:
# update folder attribute of this fileset aggregation and all nested
# fileset aggregations of this aggregation
aggregation.update_folder(new_folder=new_aggr_name)
except ObjectDoesNotExist:
# not renaming a fileset aggregation folder
pass
self._recreate_xml_docs_for_folder(new_aggr_name)
else:
# check if there is a matching single file aggregation
try:
aggregation = self.get_aggregation_by_name(new_aggr_name)
delete_old_xml_files()
aggregation.create_aggregation_xml_documents()
except ObjectDoesNotExist:
# the file path *new_aggr_name* is not a single file aggregation - no more
# action is needed
pass
def is_aggregation_xml_file(self, file_path):
""" determine whether a given file in the file hierarchy is metadata.
This is true if it is listed as metadata in any logical file.
"""
if not (file_path.endswith('_meta.xml') or file_path.endswith('_resmap.xml')):
return False
for logical_file in self.logical_files:
if logical_file.metadata_file_path == file_path or \
logical_file.map_file_path == file_path:
return True
return False
def supports_folder_creation(self, folder_full_path):
"""this checks if it is allowed to create a folder at the specified path
:param folder_full_path: the target path where the new folder needs to be created
:return True or False
"""
if __debug__:
assert(folder_full_path.startswith(self.file_path))
# determine containing folder
if "/" in folder_full_path:
path_to_check, _ = os.path.split(folder_full_path)
else:
path_to_check = folder_full_path
# find if the path represents a multi-file aggregation
if path_to_check.startswith(self.file_path):
aggregation_path = path_to_check[len(self.file_path) + 1:]
else:
aggregation_path = path_to_check
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.can_contain_folders
except ObjectDoesNotExist:
# target path doesn't represent an aggregation - so it is OK to create a folder
pass
return True
def supports_rename_path(self, src_full_path, tgt_full_path):
"""checks if file/folder rename/move is allowed
:param src_full_path: name of the file/folder path to be renamed
:param tgt_full_path: new name for file/folder path
:return True or False
"""
if __debug__:
assert(src_full_path.startswith(self.file_path))
assert(tgt_full_path.startswith(self.file_path))
istorage = self.get_irods_storage()
# need to find out which of the following actions the user is trying to do:
# renaming a file
# renaming a folder
# moving a file
# moving a folder
is_renaming_file = False
is_moving_file = False
is_moving_folder = False
tgt_folder, tgt_file_name = os.path.split(tgt_full_path)
_, tgt_ext = os.path.splitext(tgt_file_name)
if tgt_ext:
tgt_file_dir = os.path.dirname(tgt_full_path)
else:
tgt_file_dir = tgt_full_path
src_folder, src_file_name = os.path.split(src_full_path)
_, src_ext = os.path.splitext(src_file_name)
if src_ext:
src_file_dir = os.path.dirname(src_full_path)
else:
src_file_dir = src_full_path
if src_ext and tgt_ext:
is_renaming_file = True
elif src_ext:
is_moving_file = True
elif not istorage.exists(tgt_file_dir):
# renaming folder - no restriction
return True
else:
is_moving_folder = True
def check_file_rename_or_move():
# see if the folder containing the file represents an aggregation
if src_file_dir != self.file_path:
aggregation_path = src_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.supports_resource_file_rename
except ObjectDoesNotExist:
# check if the source file represents an aggregation
aggregation_path = os.path.join(aggregation_path, src_file_name)
aggregation = self.get_aggregation_by_name(aggregation_path)
if is_renaming_file:
return aggregation.supports_resource_file_rename
else:
return aggregation.supports_resource_file_move
else:
# check if the source file represents an aggregation
aggregation_path = src_file_name
aggregation = self.get_aggregation_by_name(aggregation_path)
if is_renaming_file:
return aggregation.supports_resource_file_rename
else:
return aggregation.supports_resource_file_move
if is_renaming_file:
# see if the folder containing the file represents an aggregation
try:
can_rename = check_file_rename_or_move()
return can_rename
except ObjectDoesNotExist:
return True
elif is_moving_file:
# check source - see if the folder containing the file represents an aggregation
try:
can_move = check_file_rename_or_move()
if not can_move:
return can_move
except ObjectDoesNotExist:
pass
# check target folder only if it is not the root
if tgt_file_dir != self.file_path:
aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.supports_resource_file_add
except ObjectDoesNotExist:
# target folder is not an aggregation - no restriction
return True
return True
elif is_moving_folder:
# no check on source is needed in this case
# check target - only if it is not the root
if tgt_file_dir != self.file_path:
aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.can_contain_folders
except ObjectDoesNotExist:
# target folder doesn't represent an aggregation - no restriction
return True
return True
def can_add_files(self, target_full_path):
"""
checks if file(s) can be uploaded to the specified *target_full_path*
:param target_full_path: full folder path name where file needs to be uploaded to
:return: True or False
"""
istorage = self.get_irods_storage()
if istorage.exists(target_full_path):
path_to_check = target_full_path
else:
return False
if not path_to_check.endswith("data/contents"):
# it is not the base directory - it must be a directory under base dir
if path_to_check.startswith(self.file_path):
aggregation_path = path_to_check[len(self.file_path) + 1:]
else:
aggregation_path = path_to_check
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.supports_resource_file_add
except ObjectDoesNotExist:
# target path doesn't represent an aggregation - so it is OK to add a file
pass
return True
def supports_zip(self, folder_to_zip):
"""check if the given folder can be zipped or not"""
# find all the resource files in the folder to be zipped
# this is being passed both qualified and unqualified paths!
full_path = folder_to_zip
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
# get all resource files at full_path and its sub-folders
res_file_objects = ResourceFile.list_folder(self, full_path)
# check any logical file associated with the resource file supports zip functionality
for res_file in res_file_objects:
if res_file.has_logical_file:
if not res_file.logical_file.supports_zip:
return False
return True
def supports_delete_folder_on_zip(self, original_folder):
"""check if the specified folder can be deleted at the end of zipping that folder"""
# find all the resource files in the folder to be deleted
# this is being passed both qualified and unqualified paths!
full_path = original_folder
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
# get all resource files at full_path and its sub-folders
res_file_objects = ResourceFile.list_folder(self, full_path)
# check any logical file associated with the resource file supports deleting the folder
# after its zipped
for res_file in res_file_objects:
if res_file.has_logical_file:
if not res_file.logical_file.supports_delete_folder_on_zip:
return False
return True
def get_missing_file_type_metadata_info(self):
# this is used in page pre-processor to build the context
# so that the landing page can show what metadata items are missing for each
# logical file/aggregation
metadata_missing_info = []
for lfo in self.logical_files:
if not lfo.metadata.has_all_required_elements():
missing_elements = lfo.metadata.get_required_missing_elements()
metadata_missing_info.append({'file_path': lfo.aggregation_name,
'missing_elements': missing_elements})
return metadata_missing_info
def delete_coverage(self, coverage_type):
"""Deletes coverage data for the resource
:param coverage_type: A value of either 'spatial' or 'temporal
:return:
"""
if coverage_type.lower() == 'spatial' and self.metadata.spatial_coverage:
self.metadata.spatial_coverage.delete()
self.metadata.is_dirty = True
self.metadata.save()
elif coverage_type.lower() == 'temporal' and self.metadata.temporal_coverage:
self.metadata.temporal_coverage.delete()
self.metadata.is_dirty = True
self.metadata.save()
def update_coverage(self):
"""Update resource spatial and temporal coverage based on the corresponding coverages
from all the contained aggregations (logical file) only if the resource coverage is not
already set"""
# update resource spatial coverage only if there is no spatial coverage already
if self.metadata.spatial_coverage is None:
self.update_spatial_coverage()
# update resource temporal coverage only if there is no temporal coverage already
if self.metadata.temporal_coverage is None:
self.update_temporal_coverage()
def update_spatial_coverage(self):
"""Updates resource spatial coverage based on the contained spatial coverages of
aggregations (file type). Note: This action will overwrite any existing resource spatial
coverage data.
"""
update_target_spatial_coverage(self)
def update_temporal_coverage(self):
"""Updates resource temporal coverage based on the contained temporal coverages of
aggregations (file type). Note: This action will overwrite any existing resource temporal
coverage data.
"""
update_target_temporal_coverage(self)
class CompositeResource(BaseResource):
objects = ResourceManager("CompositeResource")
class Meta:
verbose_name = 'Composite Resource'
proxy = True
@property
def can_be_public_or_discoverable(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_public_or_discoverable:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
return True
def set_default_logical_file(self):
"""sets an instance of GenericLogicalFile to any resource file objects of this instance
of the resource that is not already associated with a logical file. """
for res_file in self.files.all():
if not res_file.has_logical_file:
logical_file = GenericLogicalFile.create()
res_file.logical_file_content_object = logical_file
res_file.save()
@property
def supports_logical_file(self):
""" if this resource allows associating resource file objects with logical file"""
return True
def get_metadata_xml(self, pretty_print=True):
from lxml import etree
# get resource level core metadata as xml string
xml_string = super(CompositeResource,
self).get_metadata_xml(pretty_print=False)
# add file type metadata xml
# create an etree xml object
RDF_ROOT = etree.fromstring(xml_string)
# get root 'Description' element that contains all other elements
container = RDF_ROOT.find('rdf:Description',
namespaces=self.metadata.NAMESPACES)
for lf in self.logical_files:
lf.metadata.add_to_xml_container(container)
return etree.tostring(RDF_ROOT, pretty_print=pretty_print)
def supports_folder_creation(self, folder_full_path):
"""this checks if it is allowed to create a folder at the specified path"""
if "/" in folder_full_path:
path_to_check = folder_full_path[:folder_full_path.rfind("/")]
else:
path_to_check = folder_full_path
if not path_to_check.endswith("/data/contents"):
res_file_objs = [
res_file_obj for res_file_obj in self.files.all()
if res_file_obj.dir_path == path_to_check
]
for res_file_obj in res_file_objs:
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
return True
def supports_rename_path(self, src_full_path, tgt_full_path):
"""checks if file/folder rename/move is allowed"""
istorage = self.get_irods_storage()
tgt_file_dir = os.path.dirname(tgt_full_path)
src_file_dir = os.path.dirname(src_full_path)
def check_directory():
path_to_check = ''
if istorage.exists(tgt_file_dir):
path_to_check = tgt_file_dir
else:
if tgt_file_dir.startswith(src_file_dir):
path_to_check = src_file_dir
if path_to_check and not path_to_check.endswith("data/contents"):
# it is not the base directory - it must be a directory under base dir
res_file_objs = [
res_file_obj for res_file_obj in self.files.all()
if res_file_obj.dir_path == path_to_check
]
for res_file_obj in res_file_objs:
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
return True
res_file_objs = [
res_file_obj for res_file_obj in self.files.all()
if res_file_obj.full_path == src_full_path
]
if res_file_objs:
res_file_obj = res_file_objs[0]
# src_full_path contains file name
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
# check if the target directory allows stuff to be moved there
return check_directory()
else:
# src_full_path is a folder path without file name
# tgt_full_path also must be a folder path without file name
# check that if the target folder contains any files and if any of those files
# allow moving stuff there
return check_directory()
def supports_zip(self, folder_to_zip):
"""check if the given folder can be zipped or not"""
# find all the resource files in the folder to be zipped
if self.resource_federation_path:
res_file_objects = self.files.filter(
object_id=self.id,
fed_resource_file_name_or_path__contains=folder_to_zip).all()
else:
res_file_objects = self.files.filter(
object_id=self.id,
resource_file__contains=folder_to_zip).all()
# check any logical file associated with the resource file supports zip functionality
for res_file in res_file_objects:
if not res_file.logical_file.supports_zip:
return False
return True
def supports_delete_folder_on_zip(self, original_folder):
"""check if the specified folder can be deleted at the end of zipping that folder"""
# find all the resource files in the folder to be deleted
if self.resource_federation_path:
res_file_objects = self.files.filter(
object_id=self.id,
fed_resource_file_name_or_path__contains=original_folder).all(
)
else:
res_file_objects = self.files.filter(
object_id=self.id,
resource_file__contains=original_folder).all()
# check any logical file associated with the resource file supports deleting the folder
# after its zipped
for res_file in res_file_objects:
if not res_file.logical_file.supports_delete_folder_on_zip:
return False
return True
class MODFLOWModelInstanceResource(BaseResource):
objects = ResourceManager("MODFLOWModelInstanceResource")
discovery_content_type = 'MODFLOW Model Instance' # used during discovery
class Meta:
verbose_name = 'MODFLOW Model Instance Resource'
proxy = True
@classmethod
def get_metadata_class(cls):
return MODFLOWModelInstanceMetaData
def check_content_files(self):
"""
like 'has_required_content_files()' this method checks that one and only of .nam or .mfn
exists, but unlike 'has_required_content_files()' this method returns which files are
missing so that more information can be returned to the user via the interface
:return: -['.nam or .mfn'] if there are no files or if there are files but no .nam or .mfn
file
-'multiple_nam' if more than one .nam file has been uploaded
-missing_files, a list of file names that are included in the .nam file but have
not been uploaded by the user
"""
missing_files = []
if self.files.all().count() >= 1:
nam_files, reqd_files, existing_files, model_packages = self.find_content_files(
)
if not nam_files:
return ['.nam or .mfn']
else:
if nam_files > 1:
return 'multiple_nam'
else:
for f in reqd_files:
if f not in existing_files:
missing_files.append(f)
return missing_files
else:
return ['.nam or .mfn ']
def find_content_files(self):
"""
loops through uploaded files to count the .nam and/or .mfn files, creates a list of required
files (file names listed in the .nam or .mfn file needed to run the model), and a list of
existing file names
:return: -nam_file_count, (int), the number of .nam files uploaded
-reqd_files, (list of strings), the files listed in the .nam file that should be
included for the model to run
-existing_files, (list of strings), the names of files that have been uploaded
-model packages (list of strings), the names of the packages for the uploaded model
found in the .nam or .mfn file. These are the entries in the far left column of the
.nam or .mfn file.
"""
nam_file_count = 0
existing_files = []
reqd_files = []
model_packages = []
for res_file in self.files.all():
ext = res_file.extension
existing_files.append(res_file.file_name)
if ext == '.nam' or ext == '.mfn':
nam_file_count += 1
name_file = res_file.resource_file.file
for row in name_file:
row = row.strip()
row = row.replace("'", "")
row = row.replace('"', "")
row = row.split(" ")
r = row[0].strip()
if not r.startswith('#') and r != '' and r.lower() != 'list' \
and r.lower() != 'data' and r.lower() != 'data(binary)':
reqd_file = row[-1].strip()
reqd_files.append(reqd_file)
model_package_name = row[0].strip()
model_package_ext = reqd_file.split('.')[-1].upper()
model_packages.extend(
set([model_package_name, model_package_ext]))
return nam_file_count, reqd_files, existing_files, model_packages
class CompositeResource(BaseResource):
objects = ResourceManager("CompositeResource")
class Meta:
verbose_name = 'Composite Resource'
proxy = True
@property
def can_be_public_or_discoverable(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_public_or_discoverable:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
return True
def set_default_logical_file(self):
"""sets an instance of GenericLogicalFile to any resource file objects of this instance
of the resource that is not already associated with a logical file. """
for res_file in self.files.all():
if not res_file.has_logical_file:
logical_file = GenericLogicalFile.create()
res_file.logical_file_content_object = logical_file
res_file.save()
@property
def supports_folders(self):
""" allow folders for CompositeResources """
return True
@property
def supports_logical_file(self):
""" if this resource allows associating resource file objects with logical file"""
return True
def get_metadata_xml(self, pretty_print=True, include_format_elements=True):
from lxml import etree
# get resource level core metadata as xml string
# for composite resource we don't want the format elements at the resource level
# as they are included at the file level xml node
xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False,
include_format_elements=False)
# add file type metadata xml
# create an etree xml object
RDF_ROOT = etree.fromstring(xml_string)
# get root 'Description' element that contains all other elements
container = RDF_ROOT.find('rdf:Description', namespaces=self.metadata.NAMESPACES)
for lf in self.logical_files:
lf.metadata.add_to_xml_container(container)
return etree.tostring(RDF_ROOT, pretty_print=pretty_print)
def supports_folder_creation(self, folder_full_path):
"""this checks if it is allowed to create a folder at the specified path"""
if __debug__:
assert(folder_full_path.startswith(self.file_path))
# determine containing folder
if "/" in folder_full_path:
path_to_check, _ = os.path.split(folder_full_path)
else:
path_to_check = folder_full_path
if path_to_check != self.file_path:
res_file_objs = [res_file_obj for res_file_obj in self.files.all() if
res_file_obj.dir_path == path_to_check]
for res_file_obj in res_file_objs:
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
return True
def supports_rename_path(self, src_full_path, tgt_full_path):
"""checks if file/folder rename/move is allowed"""
if __debug__:
assert(src_full_path.startswith(self.file_path))
assert(tgt_full_path.startswith(self.file_path))
istorage = self.get_irods_storage()
tgt_file_dir = os.path.dirname(tgt_full_path)
src_file_dir = os.path.dirname(src_full_path)
def check_directory():
path_to_check = ''
if istorage.exists(tgt_file_dir):
path_to_check = tgt_file_dir
else:
if tgt_file_dir.startswith(src_file_dir):
path_to_check = src_file_dir
if path_to_check and not path_to_check.endswith("data/contents"):
# it is not the base directory - it must be a directory under base dir
res_file_objs = [res_file_obj for res_file_obj in self.files.all() if
res_file_obj.dir_path == path_to_check]
for res_file_obj in res_file_objs:
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
return True
res_file_objs = [res_file_obj for res_file_obj in self.files.all() if
res_file_obj.full_path == src_full_path]
if res_file_objs:
res_file_obj = res_file_objs[0]
# src_full_path contains file name
if not res_file_obj.logical_file.supports_resource_file_rename or \
not res_file_obj.logical_file.supports_resource_file_move:
return False
# check if the target directory allows stuff to be moved there
return check_directory()
else:
# src_full_path is a folder path without file name
# tgt_full_path also must be a folder path without file name
# check that if the target folder contains any files and if any of those files
# allow moving stuff there
return check_directory()
def can_add_files(self, target_full_path):
"""
checks if file(s) can be uploaded to the specified *target_full_path*
:param target_full_path: full folder path name where file needs to be uploaded to
:return: True or False
"""
istorage = self.get_irods_storage()
if istorage.exists(target_full_path):
path_to_check = target_full_path
else:
return False
if not path_to_check.endswith("data/contents"):
# it is not the base directory - it must be a directory under base dir
res_file_objs = [res_file_obj for res_file_obj in self.files.all() if
res_file_obj.dir_path == path_to_check]
for res_file_obj in res_file_objs:
if not res_file_obj.logical_file.supports_resource_file_add:
return False
return True
def supports_zip(self, folder_to_zip):
"""check if the given folder can be zipped or not"""
# find all the resource files in the folder to be zipped
# this is being passed both qualified and unqualified paths!
full_path = folder_to_zip
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
if self.is_federated:
res_file_objects = self.files.filter(
object_id=self.id,
fed_resource_file__startswith=full_path).all()
else:
res_file_objects = self.files.filter(object_id=self.id,
resource_file__startswith=full_path).all()
# check any logical file associated with the resource file supports zip functionality
for res_file in res_file_objects:
if not res_file.logical_file.supports_zip:
return False
return True
def supports_delete_folder_on_zip(self, original_folder):
"""check if the specified folder can be deleted at the end of zipping that folder"""
# find all the resource files in the folder to be deleted
# this is being passed both qualified and unqualified paths!
full_path = original_folder
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
if self.is_federated:
res_file_objects = self.files.filter(
object_id=self.id,
fed_resource_file__startswith=full_path).all()
else:
res_file_objects = self.files.filter(
object_id=self.id,
resource_file__startswith=full_path).all()
# check any logical file associated with the resource file supports deleting the folder
# after its zipped
for res_file in res_file_objects:
if not res_file.logical_file.supports_delete_folder_on_zip:
return False
return True
def get_missing_file_type_metadata_info(self):
# this is used in page pre-processor to build the context
# so that the landing page can show what metadata items are missing for each logical file
metadata_missing_info = []
for lfo in self.logical_files:
if not lfo.metadata.has_all_required_elements():
file_path = lfo.files.first().short_path
missing_elements = lfo.metadata.get_required_missing_elements()
metadata_missing_info.append({'file_path': file_path,
'missing_elements': missing_elements})
return metadata_missing_info
class CompositeResource(BaseResource):
objects = ResourceManager("CompositeResource")
discovery_content_type = 'Composite' # used during discovery
class Meta:
verbose_name = 'Composite Resource'
proxy = True
@property
def can_be_public_or_discoverable(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_public_or_discoverable:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
return True
@property
def can_be_published(self):
# resource level metadata check
if not super(CompositeResource, self).can_be_published:
return False
# filetype level metadata check
for lf in self.logical_files:
if not lf.metadata.has_all_required_elements():
return False
# url file cannot be published
if 'url' in lf.extra_data:
return False
return True
def set_default_logical_file(self):
"""sets an instance of GenericLogicalFile to any resource file objects of this instance
of the resource that is not already associated with a logical file. """
for res_file in self.files.all():
if not res_file.has_logical_file:
logical_file = GenericLogicalFile.create()
res_file.logical_file_content_object = logical_file
res_file.save()
def get_folder_aggregation_object(self, dir_path):
"""Returns an aggregation (file type) object if the specified folder *dir_path* represents a
file type aggregation (logical file), otherwise None.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the aggregation object to be retrieved
"""
files_in_folder = [
res_file for res_file in self.files.all()
if res_file.dir_path == dir_path
]
for fl in files_in_folder:
if fl.has_logical_file:
return fl.logical_file
return None
def get_file_aggregation_object(self, file_path):
"""Returns an aggregation (file type) object if the specified file *file_path* represents a
file type aggregation (logical file), otherwise None.
:param file_path: Resource file path (full file path starting with resource id)
for which the aggregation object to be retrieved
"""
for res_file in self.files.all():
if res_file.full_path == file_path:
if res_file.has_logical_file:
return res_file.logical_file
return None
def get_folder_aggregation_type_to_set(self, dir_path):
"""Returns an aggregation (file type) type that the specified folder *dir_path* can
possibly be set to.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the possible aggregation type that can be set needs to be determined
:return If the specified folder is already represents an aggregation or does
not contain suitable file(s) then returns "" (empty string). If the specified folder
contains only the files that meet the requirements of a supported aggregation, and
does not contain other folders or does not have a parent folder then return the
class name of that matching aggregation type.
"""
aggregation_type_to_set = ""
if self.get_folder_aggregation_object(dir_path) is not None:
# target folder is already an aggregation
return None
istorage = self.get_irods_storage()
irods_path = dir_path
if self.is_federated:
irods_path = os.path.join(self.resource_federation_path,
irods_path)
store = istorage.listdir(irods_path)
if store[0]:
# seems there are folders under dir_path - no aggregation type can be set if the target
# folder contains other folders
return None
files_in_folder = [
res_file for res_file in self.files.all()
if res_file.dir_path == dir_path
]
if not files_in_folder:
# folder is empty
return None
if len(files_in_folder) > 1:
# check for geo feature
aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
else:
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for NetCDF aggregation type
aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for TimeSeries aggregation type
aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
return None
@property
def supports_folders(self):
""" allow folders for CompositeResources """
return True
@property
def supports_logical_file(self):
""" if this resource allows associating resource file objects with logical file"""
return True
def get_metadata_xml(self,
pretty_print=True,
include_format_elements=True):
from lxml import etree
# get resource level core metadata as xml string
# for composite resource we don't want the format elements at the resource level
# as they are included at the aggregation map xml document
xml_string = super(CompositeResource, self).get_metadata_xml(
pretty_print=False, include_format_elements=False)
# create an etree xml object
RDF_ROOT = etree.fromstring(xml_string)
return etree.tostring(RDF_ROOT, pretty_print=pretty_print)
def create_aggregation_xml_documents(self, aggregation_name=None):
"""Creates aggregation map and metadata xml files for each of the contained aggregations
:param aggregation_name: (optional) name of the the specific aggregation for which xml
documents need to be created
"""
if aggregation_name is None:
for aggregation in self.logical_files:
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
else:
try:
aggregation = self.get_aggregation_by_name(aggregation_name)
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
except ObjectDoesNotExist:
# aggregation_name must be a folder path that doesn't represent an aggregation
# there may be single file aggregation in that folder for which xml documents
# need to be created
self._recreate_xml_docs_for_folder(aggregation_name,
check_metadata_dirty=True)
def _recreate_xml_docs_for_folder(self,
folder,
check_metadata_dirty=False):
"""Re-creates xml metadata and map documents associated with the specified folder.
If the *folder* represents an aggregation then map and metadata xml documents are
recreated only for that aggregation. Otherwise, xml documents are created for any
aggregation that may exist in the specified folder and its sub-folders.
:param folder: folder for which xml documents need to be re-created
:param check_metadata_dirty: if true, then xml files will be created only if the
aggregation metadata is dirty
"""
# first check if the the folder represents an aggregation
try:
aggregation = self.get_aggregation_by_name(folder)
if check_metadata_dirty:
if aggregation.metadata.is_dirty:
aggregation.create_aggregation_xml_documents()
else:
aggregation.create_aggregation_xml_documents()
# if we found an aggregation by the folder name that means this folder doesn't
# have any sub folders as multi-file aggregation folder can't have sub folders
except ObjectDoesNotExist:
# create xml map and metadata xml documents for all aggregations that exist
# in *folder* and its sub-folders
if not folder.startswith(self.file_path):
folder = os.path.join(self.file_path, folder)
res_file_objects = ResourceFile.list_folder(self, folder)
aggregations = []
for res_file in res_file_objects:
if res_file.has_logical_file and res_file.logical_file not in aggregations:
aggregations.append(res_file.logical_file)
if check_metadata_dirty:
aggregations = [
aggr for aggr in aggregations if aggr.metadata.is_dirty
]
for aggregation in aggregations:
aggregation.create_aggregation_xml_documents()
def get_aggregation_by_name(self, name):
"""Get an aggregation that matches the aggregation name specified by *name*
:param name: name (aggregation path) of the aggregation to find
:return an aggregation object if found
:raises ObjectDoesNotExist if no matching aggregation is found
"""
for aggregation in self.logical_files:
# remove the last slash in aggregation_name if any
if aggregation.aggregation_name.rstrip('/') == name:
return aggregation
raise ObjectDoesNotExist(
"No matching aggregation was found for name:{}".format(name))
def recreate_aggregation_xml_docs(self, orig_aggr_name, new_aggr_name):
"""
When a folder or file representing an aggregation is renamed or moved,
the associated map and metadata xml documents are deleted
and then regenerated
:param orig_aggr_name: original aggregation name - used for deleting existing
xml documents
:param new_aggr_name: new aggregation name - used for finding a matching
aggregation so that new xml documents can be recreated
"""
def delete_old_xml_files(folder=''):
istorage = self.get_irods_storage()
meta_xml_file_name = orig_aggr_name + "_meta.xml"
map_xml_file_name = orig_aggr_name + "_resmap.xml"
if not folder:
# case if file rename/move for single file aggregation
meta_xml_file_full_path = os.path.join(self.file_path,
meta_xml_file_name)
map_xml_file_full_path = os.path.join(self.file_path,
map_xml_file_name)
else:
# case of folder rename - multi-file aggregation
_, meta_xml_file_name = os.path.split(meta_xml_file_name)
_, map_xml_file_name = os.path.split(map_xml_file_name)
meta_xml_file_full_path = os.path.join(self.file_path, folder,
meta_xml_file_name)
map_xml_file_full_path = os.path.join(self.file_path, folder,
map_xml_file_name)
if istorage.exists(meta_xml_file_full_path):
istorage.delete(meta_xml_file_full_path)
if istorage.exists(map_xml_file_full_path):
istorage.delete(map_xml_file_full_path)
# first check if the new_aggr_name is a folder path or file path
name, ext = os.path.splitext(new_aggr_name)
is_new_aggr_a_folder = ext == ''
if is_new_aggr_a_folder:
delete_old_xml_files(folder=new_aggr_name)
self._recreate_xml_docs_for_folder(new_aggr_name)
else:
# check if there is a matching single file aggregation
try:
aggregation = self.get_aggregation_by_name(new_aggr_name)
delete_old_xml_files()
aggregation.create_aggregation_xml_documents()
except ObjectDoesNotExist:
# the file path *new_aggr_name* is not a single file aggregation - no more
# action is needed
pass
def is_aggregation_xml_file(self, file_path):
""" determine whether a given file in the file hierarchy is metadata.
This is true if it is listed as metadata in any logical file.
"""
if not (file_path.endswith('_meta.xml')
or file_path.endswith('_resmap.xml')):
return False
for logical_file in self.logical_files:
if logical_file.metadata_file_path == file_path or \
logical_file.map_file_path == file_path:
return True
return False
def supports_folder_creation(self, folder_full_path):
"""this checks if it is allowed to create a folder at the specified path
:param folder_full_path: the target path where the new folder needs to be created
:return True or False
"""
if __debug__:
assert (folder_full_path.startswith(self.file_path))
# determine containing folder
if "/" in folder_full_path:
path_to_check, _ = os.path.split(folder_full_path)
else:
path_to_check = folder_full_path
# find if the path represents a multi-file aggregation
if path_to_check.startswith(self.file_path):
aggregation_path = path_to_check[len(self.file_path) + 1:]
else:
aggregation_path = path_to_check
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.can_contain_folders
except ObjectDoesNotExist:
# target path doesn't represent an aggregation - so it is OK to create a folder
pass
return True
def supports_rename_path(self, src_full_path, tgt_full_path):
"""checks if file/folder rename/move is allowed
:param src_full_path: name of the file/folder path to be renamed
:param tgt_full_path: new name for file/folder path
:return True or False
"""
if __debug__:
assert (src_full_path.startswith(self.file_path))
assert (tgt_full_path.startswith(self.file_path))
istorage = self.get_irods_storage()
# need to find out which of the following actions the user is trying to do:
# renaming a file
# renaming a folder
# moving a file
# moving a folder
is_renaming_file = False
is_moving_file = False
is_moving_folder = False
tgt_folder, tgt_file_name = os.path.split(tgt_full_path)
_, tgt_ext = os.path.splitext(tgt_file_name)
if tgt_ext:
tgt_file_dir = os.path.dirname(tgt_full_path)
else:
tgt_file_dir = tgt_full_path
src_folder, src_file_name = os.path.split(src_full_path)
_, src_ext = os.path.splitext(src_file_name)
if src_ext:
src_file_dir = os.path.dirname(src_full_path)
else:
src_file_dir = src_full_path
if src_ext and tgt_ext:
is_renaming_file = True
elif src_ext:
is_moving_file = True
elif not istorage.exists(tgt_file_dir):
# renaming folder - no restriction
return True
else:
is_moving_folder = True
def check_file_rename_or_move():
# see if the folder containing the file represents an aggregation
if src_file_dir != self.file_path:
aggregation_path = src_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(
aggregation_path)
return aggregation.supports_resource_file_rename
except ObjectDoesNotExist:
# check if the source file represents an aggregation
aggregation_path = os.path.join(aggregation_path,
src_file_name)
aggregation = self.get_aggregation_by_name(
aggregation_path)
if is_renaming_file:
return aggregation.supports_resource_file_rename
else:
return aggregation.supports_resource_file_move
else:
# check if the source file represents an aggregation
aggregation_path = src_file_name
aggregation = self.get_aggregation_by_name(aggregation_path)
if is_renaming_file:
return aggregation.supports_resource_file_rename
else:
return aggregation.supports_resource_file_move
if is_renaming_file:
# see if the folder containing the file represents an aggregation
try:
can_rename = check_file_rename_or_move()
return can_rename
except ObjectDoesNotExist:
return True
elif is_moving_file:
# check source - see if the folder containing the file represents an aggregation
try:
can_move = check_file_rename_or_move()
if not can_move:
return can_move
except ObjectDoesNotExist:
pass
# check target folder only if it is not the root
if tgt_file_dir != self.file_path:
aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(
aggregation_path)
return aggregation.supports_resource_file_add
except ObjectDoesNotExist:
# target folder is not an aggregation - no restriction
return True
return True
elif is_moving_folder:
# no check on source is needed in this case
# check target - only if it is not the root
if tgt_file_dir != self.file_path:
aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
try:
aggregation = self.get_aggregation_by_name(
aggregation_path)
return aggregation.can_contain_folders
except ObjectDoesNotExist:
# target folder doesn't represent an aggrgation - no restriction
return True
return True
def can_add_files(self, target_full_path):
"""
checks if file(s) can be uploaded to the specified *target_full_path*
:param target_full_path: full folder path name where file needs to be uploaded to
:return: True or False
"""
istorage = self.get_irods_storage()
if istorage.exists(target_full_path):
path_to_check = target_full_path
else:
return False
if not path_to_check.endswith("data/contents"):
# it is not the base directory - it must be a directory under base dir
if path_to_check.startswith(self.file_path):
aggregation_path = path_to_check[len(self.file_path) + 1:]
else:
aggregation_path = path_to_check
try:
aggregation = self.get_aggregation_by_name(aggregation_path)
return aggregation.supports_resource_file_add
except ObjectDoesNotExist:
# target path doesn't represent an aggregation - so it is OK to add a file
pass
return True
def supports_zip(self, folder_to_zip):
"""check if the given folder can be zipped or not"""
# find all the resource files in the folder to be zipped
# this is being passed both qualified and unqualified paths!
full_path = folder_to_zip
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
# get all resource files at full_path and its sub-folders
res_file_objects = ResourceFile.list_folder(self, full_path)
# check any logical file associated with the resource file supports zip functionality
for res_file in res_file_objects:
if res_file.has_logical_file:
if not res_file.logical_file.supports_zip:
return False
return True
def supports_delete_folder_on_zip(self, original_folder):
"""check if the specified folder can be deleted at the end of zipping that folder"""
# find all the resource files in the folder to be deleted
# this is being passed both qualified and unqualified paths!
full_path = original_folder
if not full_path.startswith(self.file_path):
full_path = os.path.join(self.file_path, full_path)
# get all resource files at full_path and its sub-folders
res_file_objects = ResourceFile.list_folder(self, full_path)
# check any logical file associated with the resource file supports deleting the folder
# after its zipped
for res_file in res_file_objects:
if res_file.has_logical_file:
if not res_file.logical_file.supports_delete_folder_on_zip:
return False
return True
def get_missing_file_type_metadata_info(self):
# this is used in page pre-processor to build the context
# so that the landing page can show what metadata items are missing for each
# logical file/aggregation
metadata_missing_info = []
for lfo in self.logical_files:
if not lfo.metadata.has_all_required_elements():
missing_elements = lfo.metadata.get_required_missing_elements()
metadata_missing_info.append({
'file_path':
lfo.aggregation_name,
'missing_elements':
missing_elements
})
return metadata_missing_info
def update_coverage(self):
"""Update resource spatial and temporal coverage based on the corresponding coverages
from all the contained aggregations (logical file) only if the resource coverage is not
already set"""
# update resource spatial coverage only if there is no spatial coverage already
if self.metadata.spatial_coverage is None:
self.update_spatial_coverage()
# update resource temporal coverage only if there is no temporal coverage already
if self.metadata.temporal_coverage is None:
self.update_temporal_coverage()
def update_spatial_coverage(self):
"""Updates resource spatial coverage based on the contained spatial coverages of
aggregations (file type). Note: This action will overwrite any existing resource spatial
coverage data.
"""
spatial_coverages = [
lf.metadata.spatial_coverage for lf in self.logical_files
if lf.metadata.spatial_coverage is not None
]
if not spatial_coverages:
# no aggregation level spatial coverage data exist - no need to update resource
# spatial coverage
return
bbox_limits = {
'box': {
'northlimit': 'northlimit',
'southlimit': 'southlimit',
'eastlimit': 'eastlimit',
'westlimit': 'westlimit'
},
'point': {
'northlimit': 'north',
'southlimit': 'north',
'eastlimit': 'east',
'westlimit': 'east'
}
}
def set_coverage_data(res_coverage_value, lfo_coverage_element,
box_limits):
comparison_operator = {
'northlimit': lt,
'southlimit': gt,
'eastlimit': lt,
'westlimit': gt
}
for key in comparison_operator.keys():
if comparison_operator[key](
res_coverage_value[key],
lfo_coverage_element.value[box_limits[key]]):
res_coverage_value[key] = lfo_coverage_element.value[
box_limits[key]]
cov_type = "point"
bbox_value = {
'northlimit': -90,
'southlimit': 90,
'eastlimit': -180,
'westlimit': 180,
'projection': 'WGS 84 EPSG:4326',
'units': "Decimal degrees"
}
if len(spatial_coverages) > 1:
# check if one of the coverage is of type box
if any(sp_cov.type == 'box' for sp_cov in spatial_coverages):
cov_type = 'box'
else:
# check if the coverages represent different locations
unique_lats = set(
[sp_cov.value['north'] for sp_cov in spatial_coverages])
unique_lons = set(
[sp_cov.value['east'] for sp_cov in spatial_coverages])
if len(unique_lats) == 1 and len(unique_lons) == 1:
cov_type = 'point'
else:
cov_type = 'box'
if cov_type == 'point':
sp_cov = spatial_coverages[0]
bbox_value = dict()
bbox_value['projection'] = 'WGS 84 EPSG:4326'
bbox_value['units'] = 'Decimal degrees'
bbox_value['north'] = sp_cov.value['north']
bbox_value['east'] = sp_cov.value['east']
else:
for sp_cov in spatial_coverages:
if sp_cov.type == "box":
box_limits = bbox_limits['box']
set_coverage_data(bbox_value, sp_cov, box_limits)
else:
# point type coverage
box_limits = bbox_limits['point']
set_coverage_data(bbox_value, sp_cov, box_limits)
elif len(spatial_coverages) == 1:
sp_cov = spatial_coverages[0]
if sp_cov.type == "box":
cov_type = 'box'
bbox_value['projection'] = 'WGS 84 EPSG:4326'
bbox_value['units'] = 'Decimal degrees'
bbox_value['northlimit'] = sp_cov.value['northlimit']
bbox_value['eastlimit'] = sp_cov.value['eastlimit']
bbox_value['southlimit'] = sp_cov.value['southlimit']
bbox_value['westlimit'] = sp_cov.value['westlimit']
else:
# point type coverage
cov_type = "point"
bbox_value = dict()
bbox_value['projection'] = 'WGS 84 EPSG:4326'
bbox_value['units'] = 'Decimal degrees'
bbox_value['north'] = sp_cov.value['north']
bbox_value['east'] = sp_cov.value['east']
spatial_cov = self.metadata.spatial_coverage
if spatial_cov:
spatial_cov.type = cov_type
place_name = spatial_cov.value.get('name', None)
if place_name is not None:
bbox_value['name'] = place_name
spatial_cov._value = json.dumps(bbox_value)
spatial_cov.save()
else:
self.metadata.create_element("coverage",
type=cov_type,
value=bbox_value)
def update_temporal_coverage(self):
"""Updates resource temporal coverage based on the contained temporal coverages of
aggregations (file type). Note: This action will overwrite any existing resource temporal
coverage data.
"""
temporal_coverages = [
lf.metadata.temporal_coverage for lf in self.logical_files
if lf.metadata.temporal_coverage is not None
]
if not temporal_coverages:
# no aggregation level temporal coverage data - no update at resource level is needed
return
date_data = {'start': None, 'end': None}
def set_date_value(date_data, coverage_element, key):
comparison_operator = gt if key == 'start' else lt
if date_data[key] is None:
date_data[key] = coverage_element.value[key]
else:
if comparison_operator(
parser.parse(date_data[key]),
parser.parse(coverage_element.value[key])):
date_data[key] = coverage_element.value[key]
for temp_cov in temporal_coverages:
start_date = parser.parse(temp_cov.value['start'])
end_date = parser.parse(temp_cov.value['end'])
temp_cov.value['start'] = start_date.strftime('%m/%d/%Y')
temp_cov.value['end'] = end_date.strftime('%m/%d/%Y')
set_date_value(date_data, temp_cov, 'start')
set_date_value(date_data, temp_cov, 'end')
temp_cov = self.metadata.temporal_coverage
if date_data['start'] is not None and date_data['end'] is not None:
if temp_cov:
temp_cov._value = json.dumps(date_data)
temp_cov.save()
else:
self.metadata.create_element("coverage",
type='period',
value=date_data)