def test_tif_set_file_type_to_geo_raster(self):
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
# there should not be any file level keywords at this point
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertTrue(isinstance(logical_file, GeoRasterLogicalFile))
self.assertTrue(logical_file.metadata, GeoRasterFileMetaData)
# TODO: not sure why there would be file level keywords - commented out as the test is
# failing
# self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
def test_set_file_type_to_geo_raster_invalid_file_2(self):
# here we are using a raster tif file for setting it
# to Geo Raster file type which already been previously set to this file type - should fail
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# set tif file to GeoRasterFileType
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# check that the resource file is associated with a logical file
res_file = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.tif')[0]
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name,
"GeoRasterLogicalFile")
# trying to set this tif file again to geo raster file type should raise
# ValidationError
with self.assertRaises(ValidationError):
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
self.composite_resource.delete()
def test_logical_file_delete(self):
# test that when an instance GeoRasterFileType is deleted
# all files associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
logical_file = GeoRasterLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 2)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the logical file using the custom delete function - logical_delete()
logical_file.logical_delete(self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(RasterFileTypeMetaDataTest,
self).is_federated_irods_available():
return
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
fed_res_file_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
fed_copy_or_move='copy',
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
# set the logical file -which get sets as part of the post resource creation signal
resource_post_create_actions(resource=self.composite_resource,
user=self.user,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
fed_file_path = "data/contents/{}".format(self.raster_file_name)
self.assertEqual(res_file.fed_resource_file_name_or_path,
fed_file_path)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
def test_add_update_metadata_to_raster_file_type(self):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource(self.raster_file_obj)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")
# no temporal coverage for the raster file type yet
self.assertEqual(logical_file.metadata.temporal_coverage, None)
# add temporal coverage
url_params = {'hs_file_type': 'GeoRasterLogicalFile',
'file_type_id': logical_file.id,
'element_name': 'coverage'
}
url = reverse('add_file_metadata', kwargs=url_params)
request = self.factory.post(url, data={'start': '1/1/2010', 'end': '12/12/2015'})
request.user = self.user
# this is the view function we are testing
response = add_metadata_element(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id, element_name='coverage')
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# now the raster file should have temporal coverage element
self.assertNotEqual(logical_file.metadata.temporal_coverage, None)
# test updating temporal coverage
url_params['element_id'] = logical_file.metadata.temporal_coverage.id
url = reverse('update_file_metadata', kwargs=url_params)
request = self.factory.post(url, data={'start': '1/1/2011', 'end': '12/12/2016'})
request.user = self.user
# this is the view function we are testing
response = update_metadata_element(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id, element_name='coverage',
element_id=logical_file.metadata.temporal_coverage.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
temporal_coverage = logical_file.metadata.temporal_coverage
self.assertEqual(temporal_coverage.value['start'], '2011-01-01')
self.assertEqual(temporal_coverage.value['end'], '2016-12-12')
self.composite_resource.delete()
def test_raster_file_type_folder_delete(self):
# when a file is set to georasterlogical file type
# system automatically creates folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
# should have one GeoRasterFileMetadata object
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
# there should be 2 content files
self.assertEqual(self.composite_resource.files.count(), 2)
# test that there are metadata associated with the logical file
self.assertNotEqual(Coverage.objects.count(), 0)
self.assertNotEqual(OriginalCoverage.objects.count(), 0)
self.assertNotEqual(CellInformation.objects.count(), 0)
self.assertNotEqual(BandInformation.objects.count(), 0)
# delete the folder for the logical file
folder_path = "data/contents/small_logan"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any GeoRaster logical file or metadata file
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata associated with the logical file got deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self):
super(RasterFileTypeMetaDataTest,
self).assert_federated_irods_available()
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[],
auto_aggregate=False)
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path,
self.raster_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the tif file to GeoRasterLogicalFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self,
aggr_folder_path=expected_folder_name)
def test_bag_ingestion(self):
from hs_core.views.utils import unzip_file
def normalize_metadata(metadata_str):
"""Prepares metadata string to match resource id and hydroshare url of original"""
return metadata_str\
.replace(current_site_url(), "http://www.hydroshare.org")\
.replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1")
# create empty resource
res = resource.create_resource(
'CompositeResource',
self.user,
'My Test Resource'
)
full_paths = {}
files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'),
name="test_resource_metadata_files.zip")]
add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths)
unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True,
overwrite=True, auto_aggregate=True, ingest_metadata=True)
def compare_metadatas(new_metadata_str, original_metadata_file):
original_graph = Graph()
with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f:
original_graph = original_graph.parse(data=f.read())
new_graph = Graph()
new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str))
# remove modified date, they'll never match
subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified)
new_graph.remove((subject, None, None))
subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified)
original_graph.remove((subject, None, None))
for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph):
self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original")
res.refresh_from_db()
compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml")
compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(),
"test_meta.xml")
compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(),
"asdf/asdf_meta.xml")
compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(),
"watersheds_meta.xml")
compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(),
"logan_meta.xml")
compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(),
"SWE_time_meta.xml")
compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(),
"msf_version.refts_meta.xml")
compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(),
"ODM2_Multi_Site_One_Variable_meta.xml")
def _test_file_metadata_on_file_delete(self, ext):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# there should be 1 coverage element of type spatial
self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
self.assertNotEqual(logical_file.metadata.spatial_coverage, None)
self.assertNotEqual(logical_file.metadata.originalCoverage, None)
self.assertNotEqual(logical_file.metadata.cellInformation, None)
self.assertNotEqual(logical_file.metadata.bandInformations, None)
# there should be 2 coverage objects - one at the resource level
# and the other one at the file type level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(CellInformation.objects.count(), 1)
self.assertEqual(BandInformation.objects.count(), 1)
# delete content file specified by extension (ext parameter)
res_file_tif = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file_tif.id, self.user)
# test that we don't have logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
def test_file_metadata_on_logical_file_delete(self):
# test that when the GeoRasterFileType is deleted
# all metadata associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# test that we have the metadata elements
# there should be 2 Coverage objects - one at the resource level and
# the other one at the file type level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(
self.composite_resource.metadata.coverages.all().count(), 1)
self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(CellInformation.objects.count(), 1)
self.assertEqual(BandInformation.objects.count(), 1)
# delete the logical file
logical_file.logical_delete(self.user)
# test that we have no logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
self.composite_resource.delete()
def _test_invalid_file(self):
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with the generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# trying to set this invalid tif file to geo raster file type should raise
# ValidationError
with self.assertRaises(ValidationError):
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that the invalid file did not get deleted
self.assertEqual(self.composite_resource.files.all().count(), 1)
# check that the resource file is not associated with generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
def test_tif_set_file_type_to_geo_raster(self):
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
def _content_file_delete(self, ext):
# test that when any file that is part of an instance GeoRasterFileType is deleted
# all files associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
# delete the content file specified by the ext (file extension param)
res_file_tif = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file_tif.id, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
def test_file_rename_or_move(self):
# test that file can't be moved or renamed for any resource file
# that's part of the GeoRaster logical file object (LFO)
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test renaming of files that are associated with raster LFO - which should raise exception
self.assertEqual(self.composite_resource.files.count(), 2)
src_path = 'data/contents/small_logan/small_logan.tif'
tgt_path = "data/contents/small_logan/small_logan_1.tif"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/small_logan/small_logan.vrt'
tgt_path = "data/contents/small_logan/small_logan_1.vrt"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
# test moving the files associated with geo raster LFO
src_path = 'data/contents/small_logan/small_logan.tif'
tgt_path = "data/contents/big_logan/small_logan.tif"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/small_logan/small_logan.vrt'
tgt_path = "data/contents/big_logan/small_logan.vrt"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_update_resource_spatial_coverage(self):
"""Here we are testing the update of resource spatial coverage based on the
spatial coverages from all the aggregations in the resource using the view function
update_resource_coverage"""
self.create_composite_resource(file_to_upload=self.raster_file)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterLogicalFile type (aggregation)
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
url_params = {'resource_id': self.composite_resource.short_id,
'coverage_type': 'spatial'
}
url = reverse('update_resource_coverage', kwargs=url_params)
request = self.factory.post(url)
request.user = self.user
# this is the view function we are testing
response = update_resource_coverage(request, resource_id=self.composite_resource.short_id,
coverage_type='spatial')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.composite_resource.delete()
def get_folder_aggregation_type_to_set(self, dir_path):
"""Returns an aggregation (file type) type that the specified folder *dir_path* can
possibly be set to.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the possible aggregation type that can be set needs to be determined
:return If the specified folder is already represents an aggregation or does
not contain suitable file(s) then returns "" (empty string). If the specified folder
contains only the files that meet the requirements of a supported aggregation, and
does not contain other folders or does not have a parent folder then return the
class name of that matching aggregation type.
"""
aggregation_type_to_set = ""
if self.get_folder_aggregation_object(dir_path) is not None:
# target folder is already an aggregation
return None
istorage = self.get_irods_storage()
irods_path = dir_path
if self.is_federated:
irods_path = os.path.join(self.resource_federation_path,
irods_path)
store = istorage.listdir(irods_path)
if store[0]:
# seems there are folders under dir_path - no aggregation type can be set if the target
# folder contains other folders
return None
files_in_folder = [
res_file for res_file in self.files.all()
if res_file.dir_path == dir_path
]
if not files_in_folder:
# folder is empty
return None
if len(files_in_folder) > 1:
# check for geo feature
aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
else:
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for NetCDF aggregation type
aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for TimeSeries aggregation type
aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
return None
def test_CRUD_key_value_metadata_raster_file_type(self):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource(self.raster_file_obj)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")
# no key/value metadata for the raster file type yet
self.assertEqual(logical_file.metadata.extra_metadata, {})
url_params = {'hs_file_type': 'GeoRasterLogicalFile',
'file_type_id': logical_file.id
}
url = reverse('update_file_keyvalue_metadata', kwargs=url_params)
request = self.factory.post(url, data={'key': 'key-1', 'value': 'value-1'})
request.user = self.user
# this is the view function we are testing
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# there should be key/value metadata for the raster file type yet
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertNotEqual(logical_file.metadata.extra_metadata, {})
self.assertEqual(logical_file.metadata.extra_metadata['key-1'], 'value-1')
# update existing key value metadata - updating both key and value
request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-2',
'key_original': 'key-1'})
request.user = self.user
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-2')
self.assertNotIn('key-1', logical_file.metadata.extra_metadata.keys())
# update existing key value metadata - updating value only
request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-1',
'key_original': 'key-2'})
request.user = self.user
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-1')
# delete key/value data using the view function
request = self.factory.post(url, data={'key': 'key-2'})
request.user = self.user
# this the view function we are testing
response = delete_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# at this point there should not be any key/value metadata
self.assertEqual(logical_file.metadata.extra_metadata, {})
self.composite_resource.delete()
def _add_delete_keywords_file_type(self, file_obj, file_type):
self._create_composite_resource(file_obj)
res_file = self.composite_resource.files.first()
# set specific file type
if file_type == "GeoRasterLogicalFile":
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
else:
NetCDFLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, file_type)
if file_type != "NetCDFLogicalFile":
# no keyword metadata for the raster file type yet
self.assertEqual(len(logical_file.metadata.keywords), 0)
else:
# one keyword metadata for the netcdf file type
self.assertEqual(len(logical_file.metadata.keywords), 1)
# at this point resource should have all the keywords that we have for the file type
res_keywords = [subject.value for subject in
self.composite_resource.metadata.subjects.all()]
for kw in logical_file.metadata.keywords:
self.assertIn(kw, res_keywords)
# add keywords at the file level
url_params = {'hs_file_type': file_type,
'file_type_id': logical_file.id
}
url = reverse('add_file_keyword_metadata', kwargs=url_params)
request = self.factory.post(url, data={'keywords': 'keyword-1,keyword-2'})
request.user = self.user
# this is the view function we are testing
response = add_keyword_metadata(request, hs_file_type=file_type,
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# there should be 2 keywords for the raster file type yet
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
if file_type != "NetCDFLogicalFile":
self.assertEqual(len(logical_file.metadata.keywords), 2)
else:
self.assertEqual(len(logical_file.metadata.keywords), 3)
self.assertIn('keyword-1', logical_file.metadata.keywords)
self.assertIn('keyword-2', logical_file.metadata.keywords)
# resource level keywords must have been updated with the keywords we added
# to file level
res_keywords = [subject.value for subject in
self.composite_resource.metadata.subjects.all()]
for kw in logical_file.metadata.keywords:
self.assertIn(kw, res_keywords)
# delete keyword
url = reverse('delete_file_keyword_metadata', kwargs=url_params)
request = self.factory.post(url, data={'keyword': 'keyword-1'})
request.user = self.user
# this is the view function we are testing
response = delete_keyword_metadata(request, hs_file_type=file_type,
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
if file_type != "NetCDFLogicalFile":
self.assertEqual(len(logical_file.metadata.keywords), 1)
else:
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.assertIn('keyword-2', logical_file.metadata.keywords)
# test that deleting a file level keyword doesn't delete the same keyword from
# resource level
self.assertIn('keyword-1', res_keywords)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_raster(self):
# here we are using a valid raster zip file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_zip_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that the resource file is not associated with any logical file
# self.assertEqual(res_file.has_logical_file, False)
# set the zip file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test the resource now has 3 files (one vrt file and 2 tif files)
self.assertEqual(self.composite_resource.files.all().count(), 3)
tif_files = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.tif')
self.assertEqual(len(tif_files), 2)
vrt_files = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.vrt')
self.assertEqual(len(vrt_files), 1)
# check that the logicalfile is associated with 3 files
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'logan_vrt_small')
self.assertEqual(logical_file.has_metadata, True)
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# check that we put the 3 files in a new folder (small_logan)
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/logan_vrt_small/{}"
expected_file_path = expected_file_path.format(
self.composite_resource.short_id, base_file_name)
self.assertEqual(file_path, expected_file_path)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# test that size property of the logical file is equal to sun of size of all files
# that are part of the logical file
self.assertEqual(logical_file.size,
sum([f.size for f in logical_file.files.all()]))
# test extracted metadata for the file type
# geo raster file type should have all the metadata elements
self.assertEqual(logical_file.metadata.has_all_required_elements(),
True)
# there should be 1 coverage element - box type
self.assertNotEqual(logical_file.metadata.spatial_coverage, None)
self.assertEqual(logical_file.metadata.spatial_coverage.type, 'box')
box_coverage = logical_file.metadata.spatial_coverage
self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326')
self.assertEqual(box_coverage.value['units'], 'Decimal degrees')
self.assertEqual(box_coverage.value['northlimit'], 42.050028785767275)
self.assertEqual(box_coverage.value['eastlimit'], -111.5773750264389)
self.assertEqual(box_coverage.value['southlimit'], 41.98745777902698)
self.assertEqual(box_coverage.value['westlimit'], -111.65768822411239)
# testing extended metadata element: original coverage
ori_coverage = logical_file.metadata.originalCoverage
self.assertNotEqual(ori_coverage, None)
self.assertEqual(ori_coverage.value['northlimit'], 4655492.446916306)
self.assertEqual(ori_coverage.value['eastlimit'], 452174.01909127034)
self.assertEqual(ori_coverage.value['southlimit'], 4648592.446916306)
self.assertEqual(ori_coverage.value['westlimit'], 445574.01909127034)
self.assertEqual(ori_coverage.value['units'], 'meter')
self.assertEqual(ori_coverage.value['projection'],
'NAD83 / UTM zone 12N')
# testing extended metadata element: cell information
cell_info = logical_file.metadata.cellInformation
self.assertEqual(cell_info.rows, 230)
self.assertEqual(cell_info.columns, 220)
self.assertEqual(cell_info.cellSizeXValue, 30.0)
self.assertEqual(cell_info.cellSizeYValue, 30.0)
self.assertEqual(cell_info.cellDataType, 'Float32')
# testing extended metadata element: band information
self.assertEqual(logical_file.metadata.bandInformations.count(), 1)
band_info = logical_file.metadata.bandInformations.first()
self.assertEqual(band_info.noDataValue, '-3.40282346639e+38')
self.assertEqual(band_info.maximumValue, '2880.00708008')
self.assertEqual(band_info.minimumValue, '2274.95898438')
self.composite_resource.delete()
def test_metadata_CRUD(self):
# this is test metadata related to GeoRasterLogicalFile
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# extract metadata by setting to geo raster file type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
res_file = self.composite_resource.files.first()
# test that we can update raster specific metadata at the file level
# test that we can update dataset_name of the logical file object
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'small_logan')
logical_file.dataset_name = "big_logan"
logical_file.save()
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'big_logan')
# delete default original coverage metadata
self.assertNotEquals(logical_file.metadata.originalCoverage, None)
logical_file.metadata.originalCoverage.delete()
# create new original coverage metadata with meaningful value
value = {
"northlimit": 12,
"projection": "transverse_mercator",
"units": "meter",
"southlimit": 10,
"eastlimit": 23,
"westlimit": 2
}
logical_file.metadata.create_element('originalcoverage', value=value)
self.assertEquals(logical_file.metadata.originalCoverage.value, value)
# multiple original coverage elements are not allowed - should raise exception
with self.assertRaises(IntegrityError):
logical_file.metadata.create_element('originalcoverage',
value=value)
# delete default cell information element
self.assertNotEquals(logical_file.metadata.cellInformation, None)
logical_file.metadata.cellInformation.delete()
# create new cell information metadata with meaningful value
logical_file.metadata.create_element(
'cellinformation',
name='cellinfo',
cellDataType='Float32',
rows=1660,
columns=985,
cellSizeXValue=30.0,
cellSizeYValue=30.0,
)
cell_info = logical_file.metadata.cellInformation
self.assertEquals(cell_info.rows, 1660)
self.assertEquals(cell_info.columns, 985)
self.assertEquals(cell_info.cellSizeXValue, 30.0)
self.assertEquals(cell_info.cellSizeYValue, 30.0)
self.assertEquals(cell_info.cellDataType, 'Float32')
# multiple cell Information elements are not allowed - should raise exception
with self.assertRaises(IntegrityError):
logical_file.metadata.create_element(
'cellinformation',
name='cellinfo',
cellDataType='Float32',
rows=1660,
columns=985,
cellSizeXValue=30.0,
cellSizeYValue=30.0,
)
# delete default band information element
self.assertNotEquals(logical_file.metadata.bandInformations, None)
logical_file.metadata.bandInformations.first().delete()
# create band information element with meaningful value
logical_file.metadata.create_element('bandinformation',
name='bandinfo',
variableName='diginal elevation',
variableUnit='meter',
method='this is method',
comment='this is comment',
maximumValue=1000,
minimumValue=0,
noDataValue=-9999)
band_info = logical_file.metadata.bandInformations.first()
self.assertEquals(band_info.name, 'bandinfo')
self.assertEquals(band_info.variableName, 'diginal elevation')
self.assertEquals(band_info.variableUnit, 'meter')
self.assertEquals(band_info.method, 'this is method')
self.assertEquals(band_info.comment, 'this is comment')
self.assertEquals(band_info.maximumValue, '1000')
self.assertEquals(band_info.minimumValue, '0')
self.assertEquals(band_info.noDataValue, '-9999')
# multiple band information elements are allowed
logical_file.metadata.create_element('bandinformation',
name='bandinfo',
variableName='diginal elevation2',
variableUnit='meter',
method='this is method',
comment='this is comment',
maximumValue=1000,
minimumValue=0,
noDataValue=-9999)
self.assertEquals(logical_file.metadata.bandInformations.all().count(),
2)
# test metadata delete
# original coverage deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'originalcoverage', logical_file.metadata.originalCoverage.id)
# cell information deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'cellinformation', logical_file.metadata.cellInformation.id)
# band information deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'bandinformation',
logical_file.metadata.bandInformations.first().id)
# test metadata update
# update original coverage element
value_2 = {
"northlimit": 12.5,
"projection": "transverse_mercator",
"units": "meter",
"southlimit": 10.5,
"eastlimit": 23.5,
"westlimit": 2.5
}
logical_file.metadata.update_element(
'originalcoverage',
logical_file.metadata.originalCoverage.id,
value=value_2)
self.assertEquals(logical_file.metadata.originalCoverage.value,
value_2)
# update cell info element
logical_file.metadata.update_element(
'cellinformation',
logical_file.metadata.cellInformation.id,
name='cellinfo',
cellDataType='Double',
rows=166,
columns=98,
cellSizeXValue=3.0,
cellSizeYValue=3.0,
)
cell_info = logical_file.metadata.cellInformation
self.assertEquals(cell_info.rows, 166)
self.assertEquals(cell_info.columns, 98)
self.assertEquals(cell_info.cellSizeXValue, 3.0)
self.assertEquals(cell_info.cellSizeYValue, 3.0)
self.assertEquals(cell_info.cellDataType, 'Double')
# update band info element
logical_file.metadata.update_element(
'bandinformation',
logical_file.metadata.bandInformations.first().id,
name='bandinfo',
variableName='precipitation',
variableUnit='mm/h',
method='this is method2',
comment='this is comment2',
maximumValue=1001,
minimumValue=1,
noDataValue=-9998)
band_info = logical_file.metadata.bandInformations.first()
self.assertEquals(band_info.name, 'bandinfo')
self.assertEquals(band_info.variableName, 'precipitation')
self.assertEquals(band_info.variableUnit, 'mm/h')
self.assertEquals(band_info.method, 'this is method2')
self.assertEquals(band_info.comment, 'this is comment2')
self.assertEquals(band_info.maximumValue, '1001')
self.assertEquals(band_info.minimumValue, '1')
self.assertEquals(band_info.noDataValue, '-9998')
# test extra_metadata for the logical file
# there should be no key/value metadata at this point
self.assertEqual(logical_file.metadata.extra_metadata, {})
# create key/vale metadata
logical_file.metadata.extra_metadata = {
'key1': 'value 1',
'key2': 'value 2'
}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {
'key1': 'value 1',
'key2': 'value 2'
})
# update key/value metadata
logical_file.metadata.extra_metadata = {
'key1': 'value 1',
'key2': 'value 2',
'key 3': 'value3'
}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {
'key1': 'value 1',
'key2': 'value 2',
'key 3': 'value3'
})
# delete key/value metadata
logical_file.metadata.extra_metadata = {}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {})
self.composite_resource.delete()
def test_copy_composite_resource(self):
"""Test that logical file type objects gets copied along with the metadata that each
logical file type object contains. Here we are not testing resource level metadata copy
as that has been tested in separate unit tests"""
self.raster_obj = open(self.temp_raster_file, 'r')
files = [UploadedFile(file=self.raster_obj, name='cea.tif')]
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.owner,
title='Test Composite Resource',
files=files,
auto_aggregate=False
)
# run the resource post creation signal
utils.resource_post_create_actions(resource=self.composite_resource, user=self.owner,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with file type
self.assertEqual(res_file.has_logical_file, False)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.owner, res_file.id)
# ensure a nonowner who does not have permission to view a resource cannot copy it
with self.assertRaises(PermissionDenied):
hydroshare.create_empty_resource(self.composite_resource.short_id,
self.nonowner,
action='copy')
# give nonowner view privilege so nonowner can create a new copy of this resource
self.owner.uaccess.share_resource_with_user(self.composite_resource, self.nonowner,
PrivilegeCodes.VIEW)
orig_res_file = self.composite_resource.files.first()
orig_geo_raster_lfo = orig_res_file.logical_file
# add some key value metadata
orig_geo_raster_lfo.metadata.extra_metadata = {'key-1': 'value-1', 'key-2': 'value-2'}
# create a copy of the composite resource
new_composite_resource = hydroshare.create_empty_resource(self.composite_resource.short_id,
self.nonowner,
action='copy')
new_composite_resource = hydroshare.copy_resource(self.composite_resource,
new_composite_resource)
# check that there is 2 GeoRasterLogicalFile objects
self.assertEqual(GeoRasterLogicalFile.objects.count(), 2)
# compare the 2 GeoRasterLogicalFile objects from the original resource and the new one
orig_res_file = self.composite_resource.files.first()
orig_geo_raster_lfo = orig_res_file.logical_file
copy_res_file = new_composite_resource.files.first()
copy_geo_raster_lfo = copy_res_file.logical_file
# check that we put the 2 files in a new folder (cea)
for res_file in self.composite_resource.files.all():
file_path, base_file_name = res_file.full_path, res_file.file_name
expected_file_path = "{}/data/contents/cea/{}"
expected_file_path = expected_file_path.format(self.composite_resource.root_path,
base_file_name)
self.assertEqual(file_path, expected_file_path)
for res_file in new_composite_resource.files.all():
file_path, base_file_name = res_file.full_path, res_file.file_name
expected_file_path = "{}/data/contents/cea/{}"
expected_file_path = expected_file_path.format(new_composite_resource.root_path,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# both logical file objects should have 2 resource files
self.assertEqual(orig_geo_raster_lfo.files.count(), copy_geo_raster_lfo.files.count())
self.assertEqual(orig_geo_raster_lfo.files.count(), 2)
# both logical file objects should have same dataset_name
self.assertEqual(orig_geo_raster_lfo.dataset_name, copy_geo_raster_lfo.dataset_name)
# both should have same key/value metadata
self.assertEqual(orig_geo_raster_lfo.metadata.extra_metadata,
copy_geo_raster_lfo.metadata.extra_metadata)
# both logical file objects should have same coverage metadata
self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(),
copy_geo_raster_lfo.metadata.coverages.count())
self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), 1)
org_spatial_coverage = orig_geo_raster_lfo.metadata.spatial_coverage
copy_spatial_coverage = copy_geo_raster_lfo.metadata.spatial_coverage
self.assertEqual(org_spatial_coverage.type, copy_spatial_coverage.type)
self.assertEqual(org_spatial_coverage.type, 'box')
self.assertEqual(org_spatial_coverage.value['projection'],
copy_spatial_coverage.value['projection'])
self.assertEqual(org_spatial_coverage.value['units'],
copy_spatial_coverage.value['units'])
self.assertEqual(org_spatial_coverage.value['northlimit'],
copy_spatial_coverage.value['northlimit'])
self.assertEqual(org_spatial_coverage.value['eastlimit'],
copy_spatial_coverage.value['eastlimit'])
self.assertEqual(org_spatial_coverage.value['southlimit'],
copy_spatial_coverage.value['southlimit'])
self.assertEqual(org_spatial_coverage.value['westlimit'],
copy_spatial_coverage.value['westlimit'])
# both logical file objects should have same original coverage
org_orig_coverage = orig_geo_raster_lfo.metadata.originalCoverage
copy_orig_coverage = copy_geo_raster_lfo.metadata.originalCoverage
self.assertEqual(org_orig_coverage.value['projection'],
copy_orig_coverage.value['projection'])
self.assertEqual(org_orig_coverage.value['units'],
copy_orig_coverage.value['units'])
self.assertEqual(org_orig_coverage.value['northlimit'],
copy_orig_coverage.value['northlimit'])
self.assertEqual(org_orig_coverage.value['eastlimit'],
copy_orig_coverage.value['eastlimit'])
self.assertEqual(org_orig_coverage.value['southlimit'],
copy_orig_coverage.value['southlimit'])
self.assertEqual(org_orig_coverage.value['westlimit'],
copy_orig_coverage.value['westlimit'])
# both logical file objects should have same cell information metadata
orig_cell_info = orig_geo_raster_lfo.metadata.cellInformation
copy_cell_info = copy_geo_raster_lfo.metadata.cellInformation
self.assertEqual(orig_cell_info.rows, copy_cell_info.rows)
self.assertEqual(orig_cell_info.columns, copy_cell_info.columns)
self.assertEqual(orig_cell_info.cellSizeXValue, copy_cell_info.cellSizeXValue)
self.assertEqual(orig_cell_info.cellSizeYValue, copy_cell_info.cellSizeYValue)
self.assertEqual(orig_cell_info.cellDataType, copy_cell_info.cellDataType)
# both logical file objects should have same band information metadata
self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), 1)
self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(),
copy_geo_raster_lfo.metadata.bandInformations.count())
orig_band_info = orig_geo_raster_lfo.metadata.bandInformations.first()
copy_band_info = copy_geo_raster_lfo.metadata.bandInformations.first()
self.assertEqual(orig_band_info.noDataValue, copy_band_info.noDataValue)
self.assertEqual(orig_band_info.maximumValue, copy_band_info.maximumValue)
self.assertEqual(orig_band_info.minimumValue, copy_band_info.minimumValue)
# make sure to clean up all created resources to clean up iRODS storage
if self.composite_resource:
self.composite_resource.delete()
if new_composite_resource:
new_composite_resource.delete()
def handle(self, *args, **options):
logger = logging.getLogger(__name__)
resource_counter = 0
create_raster_aggregation = False
to_resource_type = 'CompositeResource'
msg = "THERE ARE CURRENTLY {} RASTER RESOURCES PRIOR TO CONVERSION.".format(
RasterResource.objects.all().count())
logger.info(msg)
print(">> {}".format(msg))
for rast_res in RasterResource.objects.all():
# check resource exists on irods
istorage = rast_res.get_irods_storage()
create_raster_aggregation = False
if not istorage.exists(rast_res.root_path):
err_msg = "Raster resource not found in irods (ID: {})".format(rast_res.short_id)
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
# skip this raster resource
continue
if rast_res.metadata.cellInformation is not None:
# get the vrt file name which needs to be used to create a new folder for
# raster aggregation
vrt_file = None
for res_file in rast_res.files.all():
if res_file.extension.lower() == '.vrt':
vrt_file = res_file
break
create_raster_aggregation = vrt_file is not None
if create_raster_aggregation:
# check resource files exist on irods
file_missing = False
for res_file in rast_res.files.all():
file_path = res_file.public_path
if not istorage.exists(file_path):
err_msg = "File path not found in irods:{}".format(file_path)
logger.error(err_msg)
err_msg = "Failed to convert raster resource (ID: {}). " \
"Resource file is missing on irods".format(rast_res.short_id)
print("Error:>> {}".format(err_msg))
file_missing = True
break
if file_missing:
# skip this corrupt raster resource for migration
continue
# change the resource_type
ras_metadata_obj = rast_res.metadata
rast_res.resource_type = to_resource_type
rast_res.content_model = to_resource_type.lower()
rast_res.save()
# get the converted resource object - CompositeResource
comp_res = rast_res.get_content_model()
# set CoreMetaData object for the composite resource
core_meta_obj = CoreMetaData.objects.create()
comp_res.content_object = core_meta_obj
# migrate raster resource core metadata elements to composite resource
migrate_core_meta_elements(ras_metadata_obj, comp_res)
# update url attribute of the metadata 'type' element
type_element = comp_res.metadata.type
type_element.url = '{0}/terms/{1}'.format(current_site_url(), to_resource_type)
type_element.save()
if create_raster_aggregation:
# create a Raster aggregation
ras_aggr = None
try:
ras_aggr = GeoRasterLogicalFile.create(resource=comp_res)
except Exception as ex:
err_msg = 'Failed to create raster aggregation for resource (ID: {})'
err_msg = err_msg.format(rast_res.short_id)
err_msg = err_msg + '\n' + ex.message
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
if ras_aggr is not None:
# set aggregation dataset title
ras_aggr.dataset_name = comp_res.metadata.title.value
ras_aggr.save()
# make the res files part of the aggregation
for res_file in comp_res.files.all():
ras_aggr.add_resource_file(res_file)
# migrate raster specific metadata to aggregation
for bandinfo in ras_metadata_obj.bandInformations:
bandinfo.content_object = ras_aggr.metadata
bandinfo.save()
# create aggregation level spatial coverage element
# note - the resource level spatial coverage which is a core metadata
# element gets populated as part of raster resource creation
spatial_coverage = comp_res.metadata.spatial_coverage
if spatial_coverage:
aggr_coverage = Coverage()
aggr_coverage.type = spatial_coverage.type
aggr_coverage._value = spatial_coverage._value
aggr_coverage.content_object = ras_aggr.metadata
aggr_coverage.save()
org_coverage = ras_metadata_obj.originalCoverage
if org_coverage:
org_coverage.content_object = ras_aggr.metadata
org_coverage.save()
cell_info = ras_metadata_obj.cellInformation
if cell_info:
cell_info.content_object = ras_aggr.metadata
cell_info.save()
# create aggregation level xml files
ras_aggr.create_aggregation_xml_documents()
msg = 'One Raster aggregation was created in resource (ID: {})'
msg = msg.format(comp_res.short_id)
logger.info(msg)
# set resource to dirty so that resource level xml files (resource map and
# metadata xml files) will be re-generated as part of next bag download
comp_res.save()
try:
set_dirty_bag_flag(comp_res)
except Exception as ex:
err_msg = 'Failed to set bag flag dirty for the converted resource (ID: {})'
err_msg = err_msg.format(rast_res.short_id)
err_msg = err_msg + '\n' + ex.message
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
resource_counter += 1
# delete the instance of RasterMetaData that was part of the original raster resource
ras_metadata_obj.delete()
msg = 'Raster resource (ID: {}) was converted to Composite Resource type'
msg = msg.format(comp_res.short_id)
logger.info(msg)
msg = "{} RASTER RESOURCES WERE CONVERTED TO COMPOSITE RESOURCE.".format(
resource_counter)
logger.info(msg)
print(">> {}".format(msg))
msg = "THERE ARE CURRENTLY {} RASTER RESOURCES AFTER CONVERSION.".format(
RasterResource.objects.all().count())
logger.info(msg)
if RasterResource.objects.all().count() > 0:
msg = "NOT ALL RASTER RESOURCES WERE CONVERTED TO COMPOSITE RESOURCE TYPE"
logger.error(msg)
print(">> {}".format(msg))
