def test_tif_set_file_type_to_geo_raster(self):
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
# there should not be any file level keywords at this point
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertTrue(isinstance(logical_file, GeoRasterLogicalFile))
self.assertTrue(logical_file.metadata, GeoRasterFileMetaData)
# TODO: not sure why there would be file level keywords - commented out as the test is
# failing
# self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
def test_logical_file_delete(self):
# test that when an instance GeoRasterFileType is deleted
# all files associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
logical_file = GeoRasterLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 2)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the logical file using the custom delete function - logical_delete()
logical_file.logical_delete(self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.composite_resource.delete()
def test_set_file_type_to_geo_raster_invalid_file_2(self):
# here we are using a raster tif file for setting it
# to Geo Raster file type which already been previously set to this file type - should fail
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# set tif file to GeoRasterFileType
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# check that the resource file is associated with a logical file
res_file = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.tif')[0]
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name,
"GeoRasterLogicalFile")
# trying to set this tif file again to geo raster file type should raise
# ValidationError
with self.assertRaises(ValidationError):
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(RasterFileTypeMetaDataTest,
self).is_federated_irods_available():
return
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
fed_res_file_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
fed_copy_or_move='copy',
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
# set the logical file -which get sets as part of the post resource creation signal
resource_post_create_actions(resource=self.composite_resource,
user=self.user,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
fed_file_path = "data/contents/{}".format(self.raster_file_name)
self.assertEqual(res_file.fed_resource_file_name_or_path,
fed_file_path)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
def test_add_update_metadata_to_raster_file_type(self):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource(self.raster_file_obj)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")
# no temporal coverage for the raster file type yet
self.assertEqual(logical_file.metadata.temporal_coverage, None)
# add temporal coverage
url_params = {'hs_file_type': 'GeoRasterLogicalFile',
'file_type_id': logical_file.id,
'element_name': 'coverage'
}
url = reverse('add_file_metadata', kwargs=url_params)
request = self.factory.post(url, data={'start': '1/1/2010', 'end': '12/12/2015'})
request.user = self.user
# this is the view function we are testing
response = add_metadata_element(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id, element_name='coverage')
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# now the raster file should have temporal coverage element
self.assertNotEqual(logical_file.metadata.temporal_coverage, None)
# test updating temporal coverage
url_params['element_id'] = logical_file.metadata.temporal_coverage.id
url = reverse('update_file_metadata', kwargs=url_params)
request = self.factory.post(url, data={'start': '1/1/2011', 'end': '12/12/2016'})
request.user = self.user
# this is the view function we are testing
response = update_metadata_element(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id, element_name='coverage',
element_id=logical_file.metadata.temporal_coverage.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
temporal_coverage = logical_file.metadata.temporal_coverage
self.assertEqual(temporal_coverage.value['start'], '2011-01-01')
self.assertEqual(temporal_coverage.value['end'], '2016-12-12')
self.composite_resource.delete()
def test_raster_file_type_folder_delete(self):
# when a file is set to georasterlogical file type
# system automatically creates folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
# should have one GeoRasterFileMetadata object
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
# there should be 2 content files
self.assertEqual(self.composite_resource.files.count(), 2)
# test that there are metadata associated with the logical file
self.assertNotEqual(Coverage.objects.count(), 0)
self.assertNotEqual(OriginalCoverage.objects.count(), 0)
self.assertNotEqual(CellInformation.objects.count(), 0)
self.assertNotEqual(BandInformation.objects.count(), 0)
# delete the folder for the logical file
folder_path = "data/contents/small_logan"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any GeoRaster logical file or metadata file
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata associated with the logical file got deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self):
super(RasterFileTypeMetaDataTest,
self).assert_federated_irods_available()
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[],
auto_aggregate=False)
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path,
self.raster_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the tif file to GeoRasterLogicalFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self,
aggr_folder_path=expected_folder_name)
def _test_file_metadata_on_file_delete(self, ext):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# there should be 1 coverage element of type spatial
self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
self.assertNotEqual(logical_file.metadata.spatial_coverage, None)
self.assertNotEqual(logical_file.metadata.originalCoverage, None)
self.assertNotEqual(logical_file.metadata.cellInformation, None)
self.assertNotEqual(logical_file.metadata.bandInformations, None)
# there should be 2 coverage objects - one at the resource level
# and the other one at the file type level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(CellInformation.objects.count(), 1)
self.assertEqual(BandInformation.objects.count(), 1)
# delete content file specified by extension (ext parameter)
res_file_tif = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file_tif.id, self.user)
# test that we don't have logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
def test_file_metadata_on_logical_file_delete(self):
# test that when the GeoRasterFileType is deleted
# all metadata associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# test that we have the metadata elements
# there should be 2 Coverage objects - one at the resource level and
# the other one at the file type level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(
self.composite_resource.metadata.coverages.all().count(), 1)
self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(CellInformation.objects.count(), 1)
self.assertEqual(BandInformation.objects.count(), 1)
# delete the logical file
logical_file.logical_delete(self.user)
# test that we have no logical file of type GeoRasterFileType
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
self.composite_resource.delete()
def _test_invalid_file(self):
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with the generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# trying to set this invalid tif file to geo raster file type should raise
# ValidationError
with self.assertRaises(ValidationError):
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that the invalid file did not get deleted
self.assertEqual(self.composite_resource.files.all().count(), 1)
# check that the resource file is not associated with generic logical file
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
def test_tif_set_file_type_to_geo_raster(self):
# here we are using a valid raster tif file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test extracted raster file type metadata
assert_raster_file_type_metadata(self)
def _content_file_delete(self, ext):
# test that when any file that is part of an instance GeoRasterFileType is deleted
# all files associated with GeoRasterFileType is deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
# delete the content file specified by the ext (file extension param)
res_file_tif = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file_tif.id, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
def test_file_rename_or_move(self):
# test that file can't be moved or renamed for any resource file
# that's part of the GeoRaster logical file object (LFO)
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test renaming of files that are associated with raster LFO - which should raise exception
self.assertEqual(self.composite_resource.files.count(), 2)
src_path = 'data/contents/small_logan/small_logan.tif'
tgt_path = "data/contents/small_logan/small_logan_1.tif"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/small_logan/small_logan.vrt'
tgt_path = "data/contents/small_logan/small_logan_1.vrt"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
# test moving the files associated with geo raster LFO
src_path = 'data/contents/small_logan/small_logan.tif'
tgt_path = "data/contents/big_logan/small_logan.tif"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/small_logan/small_logan.vrt'
tgt_path = "data/contents/big_logan/small_logan.vrt"
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_update_resource_spatial_coverage(self):
"""Here we are testing the update of resource spatial coverage based on the
spatial coverages from all the aggregations in the resource using the view function
update_resource_coverage"""
self.create_composite_resource(file_to_upload=self.raster_file)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterLogicalFile type (aggregation)
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
url_params = {'resource_id': self.composite_resource.short_id,
'coverage_type': 'spatial'
}
url = reverse('update_resource_coverage', kwargs=url_params)
request = self.factory.post(url)
request.user = self.user
# this is the view function we are testing
response = update_resource_coverage(request, resource_id=self.composite_resource.short_id,
coverage_type='spatial')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.composite_resource.delete()
def test_copy_composite_resource(self):
"""Test that logical file type objects gets copied along with the metadata that each
logical file type object contains. Here we are not testing resource level metadata copy
as that has been tested in separate unit tests"""
self.raster_obj = open(self.temp_raster_file, 'r')
files = [UploadedFile(file=self.raster_obj, name='cea.tif')]
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.owner,
title='Test Composite Resource',
files=files,
auto_aggregate=False
)
# run the resource post creation signal
utils.resource_post_create_actions(resource=self.composite_resource, user=self.owner,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with file type
self.assertEqual(res_file.has_logical_file, False)
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, self.owner, res_file.id)
# ensure a nonowner who does not have permission to view a resource cannot copy it
with self.assertRaises(PermissionDenied):
hydroshare.create_empty_resource(self.composite_resource.short_id,
self.nonowner,
action='copy')
# give nonowner view privilege so nonowner can create a new copy of this resource
self.owner.uaccess.share_resource_with_user(self.composite_resource, self.nonowner,
PrivilegeCodes.VIEW)
orig_res_file = self.composite_resource.files.first()
orig_geo_raster_lfo = orig_res_file.logical_file
# add some key value metadata
orig_geo_raster_lfo.metadata.extra_metadata = {'key-1': 'value-1', 'key-2': 'value-2'}
# create a copy of the composite resource
new_composite_resource = hydroshare.create_empty_resource(self.composite_resource.short_id,
self.nonowner,
action='copy')
new_composite_resource = hydroshare.copy_resource(self.composite_resource,
new_composite_resource)
# check that there is 2 GeoRasterLogicalFile objects
self.assertEqual(GeoRasterLogicalFile.objects.count(), 2)
# compare the 2 GeoRasterLogicalFile objects from the original resource and the new one
orig_res_file = self.composite_resource.files.first()
orig_geo_raster_lfo = orig_res_file.logical_file
copy_res_file = new_composite_resource.files.first()
copy_geo_raster_lfo = copy_res_file.logical_file
# check that we put the 2 files in a new folder (cea)
for res_file in self.composite_resource.files.all():
file_path, base_file_name = res_file.full_path, res_file.file_name
expected_file_path = "{}/data/contents/cea/{}"
expected_file_path = expected_file_path.format(self.composite_resource.root_path,
base_file_name)
self.assertEqual(file_path, expected_file_path)
for res_file in new_composite_resource.files.all():
file_path, base_file_name = res_file.full_path, res_file.file_name
expected_file_path = "{}/data/contents/cea/{}"
expected_file_path = expected_file_path.format(new_composite_resource.root_path,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# both logical file objects should have 2 resource files
self.assertEqual(orig_geo_raster_lfo.files.count(), copy_geo_raster_lfo.files.count())
self.assertEqual(orig_geo_raster_lfo.files.count(), 2)
# both logical file objects should have same dataset_name
self.assertEqual(orig_geo_raster_lfo.dataset_name, copy_geo_raster_lfo.dataset_name)
# both should have same key/value metadata
self.assertEqual(orig_geo_raster_lfo.metadata.extra_metadata,
copy_geo_raster_lfo.metadata.extra_metadata)
# both logical file objects should have same coverage metadata
self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(),
copy_geo_raster_lfo.metadata.coverages.count())
self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), 1)
org_spatial_coverage = orig_geo_raster_lfo.metadata.spatial_coverage
copy_spatial_coverage = copy_geo_raster_lfo.metadata.spatial_coverage
self.assertEqual(org_spatial_coverage.type, copy_spatial_coverage.type)
self.assertEqual(org_spatial_coverage.type, 'box')
self.assertEqual(org_spatial_coverage.value['projection'],
copy_spatial_coverage.value['projection'])
self.assertEqual(org_spatial_coverage.value['units'],
copy_spatial_coverage.value['units'])
self.assertEqual(org_spatial_coverage.value['northlimit'],
copy_spatial_coverage.value['northlimit'])
self.assertEqual(org_spatial_coverage.value['eastlimit'],
copy_spatial_coverage.value['eastlimit'])
self.assertEqual(org_spatial_coverage.value['southlimit'],
copy_spatial_coverage.value['southlimit'])
self.assertEqual(org_spatial_coverage.value['westlimit'],
copy_spatial_coverage.value['westlimit'])
# both logical file objects should have same original coverage
org_orig_coverage = orig_geo_raster_lfo.metadata.originalCoverage
copy_orig_coverage = copy_geo_raster_lfo.metadata.originalCoverage
self.assertEqual(org_orig_coverage.value['projection'],
copy_orig_coverage.value['projection'])
self.assertEqual(org_orig_coverage.value['units'],
copy_orig_coverage.value['units'])
self.assertEqual(org_orig_coverage.value['northlimit'],
copy_orig_coverage.value['northlimit'])
self.assertEqual(org_orig_coverage.value['eastlimit'],
copy_orig_coverage.value['eastlimit'])
self.assertEqual(org_orig_coverage.value['southlimit'],
copy_orig_coverage.value['southlimit'])
self.assertEqual(org_orig_coverage.value['westlimit'],
copy_orig_coverage.value['westlimit'])
# both logical file objects should have same cell information metadata
orig_cell_info = orig_geo_raster_lfo.metadata.cellInformation
copy_cell_info = copy_geo_raster_lfo.metadata.cellInformation
self.assertEqual(orig_cell_info.rows, copy_cell_info.rows)
self.assertEqual(orig_cell_info.columns, copy_cell_info.columns)
self.assertEqual(orig_cell_info.cellSizeXValue, copy_cell_info.cellSizeXValue)
self.assertEqual(orig_cell_info.cellSizeYValue, copy_cell_info.cellSizeYValue)
self.assertEqual(orig_cell_info.cellDataType, copy_cell_info.cellDataType)
# both logical file objects should have same band information metadata
self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), 1)
self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(),
copy_geo_raster_lfo.metadata.bandInformations.count())
orig_band_info = orig_geo_raster_lfo.metadata.bandInformations.first()
copy_band_info = copy_geo_raster_lfo.metadata.bandInformations.first()
self.assertEqual(orig_band_info.noDataValue, copy_band_info.noDataValue)
self.assertEqual(orig_band_info.maximumValue, copy_band_info.maximumValue)
self.assertEqual(orig_band_info.minimumValue, copy_band_info.minimumValue)
# make sure to clean up all created resources to clean up iRODS storage
if self.composite_resource:
self.composite_resource.delete()
if new_composite_resource:
new_composite_resource.delete()
def test_CRUD_key_value_metadata_raster_file_type(self):
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource(self.raster_file_obj)
res_file = self.composite_resource.files.first()
# set the tif file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")
# no key/value metadata for the raster file type yet
self.assertEqual(logical_file.metadata.extra_metadata, {})
url_params = {'hs_file_type': 'GeoRasterLogicalFile',
'file_type_id': logical_file.id
}
url = reverse('update_file_keyvalue_metadata', kwargs=url_params)
request = self.factory.post(url, data={'key': 'key-1', 'value': 'value-1'})
request.user = self.user
# this is the view function we are testing
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# there should be key/value metadata for the raster file type yet
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertNotEqual(logical_file.metadata.extra_metadata, {})
self.assertEqual(logical_file.metadata.extra_metadata['key-1'], 'value-1')
# update existing key value metadata - updating both key and value
request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-2',
'key_original': 'key-1'})
request.user = self.user
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-2')
self.assertNotIn('key-1', logical_file.metadata.extra_metadata.keys())
# update existing key value metadata - updating value only
request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-1',
'key_original': 'key-2'})
request.user = self.user
response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-1')
# delete key/value data using the view function
request = self.factory.post(url, data={'key': 'key-2'})
request.user = self.user
# this the view function we are testing
response = delete_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile",
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
# at this point there should not be any key/value metadata
self.assertEqual(logical_file.metadata.extra_metadata, {})
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_raster(self):
# here we are using a valid raster zip file for setting it
# to Geo Raster file type which includes metadata extraction
self.raster_file_obj = open(self.raster_zip_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that the resource file is not associated with any logical file
# self.assertEqual(res_file.has_logical_file, False)
# set the zip file to GeoRasterFile type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test the resource now has 3 files (one vrt file and 2 tif files)
self.assertEqual(self.composite_resource.files.all().count(), 3)
tif_files = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.tif')
self.assertEqual(len(tif_files), 2)
vrt_files = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, '.vrt')
self.assertEqual(len(vrt_files), 1)
# check that the logicalfile is associated with 3 files
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'logan_vrt_small')
self.assertEqual(logical_file.has_metadata, True)
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# check that we put the 3 files in a new folder (small_logan)
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/logan_vrt_small/{}"
expected_file_path = expected_file_path.format(
self.composite_resource.short_id, base_file_name)
self.assertEqual(file_path, expected_file_path)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# test that size property of the logical file is equal to sun of size of all files
# that are part of the logical file
self.assertEqual(logical_file.size,
sum([f.size for f in logical_file.files.all()]))
# test extracted metadata for the file type
# geo raster file type should have all the metadata elements
self.assertEqual(logical_file.metadata.has_all_required_elements(),
True)
# there should be 1 coverage element - box type
self.assertNotEqual(logical_file.metadata.spatial_coverage, None)
self.assertEqual(logical_file.metadata.spatial_coverage.type, 'box')
box_coverage = logical_file.metadata.spatial_coverage
self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326')
self.assertEqual(box_coverage.value['units'], 'Decimal degrees')
self.assertEqual(box_coverage.value['northlimit'], 42.050028785767275)
self.assertEqual(box_coverage.value['eastlimit'], -111.5773750264389)
self.assertEqual(box_coverage.value['southlimit'], 41.98745777902698)
self.assertEqual(box_coverage.value['westlimit'], -111.65768822411239)
# testing extended metadata element: original coverage
ori_coverage = logical_file.metadata.originalCoverage
self.assertNotEqual(ori_coverage, None)
self.assertEqual(ori_coverage.value['northlimit'], 4655492.446916306)
self.assertEqual(ori_coverage.value['eastlimit'], 452174.01909127034)
self.assertEqual(ori_coverage.value['southlimit'], 4648592.446916306)
self.assertEqual(ori_coverage.value['westlimit'], 445574.01909127034)
self.assertEqual(ori_coverage.value['units'], 'meter')
self.assertEqual(ori_coverage.value['projection'],
'NAD83 / UTM zone 12N')
# testing extended metadata element: cell information
cell_info = logical_file.metadata.cellInformation
self.assertEqual(cell_info.rows, 230)
self.assertEqual(cell_info.columns, 220)
self.assertEqual(cell_info.cellSizeXValue, 30.0)
self.assertEqual(cell_info.cellSizeYValue, 30.0)
self.assertEqual(cell_info.cellDataType, 'Float32')
# testing extended metadata element: band information
self.assertEqual(logical_file.metadata.bandInformations.count(), 1)
band_info = logical_file.metadata.bandInformations.first()
self.assertEqual(band_info.noDataValue, '-3.40282346639e+38')
self.assertEqual(band_info.maximumValue, '2880.00708008')
self.assertEqual(band_info.minimumValue, '2274.95898438')
self.composite_resource.delete()
def _add_delete_keywords_file_type(self, file_obj, file_type):
self._create_composite_resource(file_obj)
res_file = self.composite_resource.files.first()
# set specific file type
if file_type == "GeoRasterLogicalFile":
GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
else:
NetCDFLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(res_file.logical_file_type_name, file_type)
if file_type != "NetCDFLogicalFile":
# no keyword metadata for the raster file type yet
self.assertEqual(len(logical_file.metadata.keywords), 0)
else:
# one keyword metadata for the netcdf file type
self.assertEqual(len(logical_file.metadata.keywords), 1)
# at this point resource should have all the keywords that we have for the file type
res_keywords = [subject.value for subject in
self.composite_resource.metadata.subjects.all()]
for kw in logical_file.metadata.keywords:
self.assertIn(kw, res_keywords)
# add keywords at the file level
url_params = {'hs_file_type': file_type,
'file_type_id': logical_file.id
}
url = reverse('add_file_keyword_metadata', kwargs=url_params)
request = self.factory.post(url, data={'keywords': 'keyword-1,keyword-2'})
request.user = self.user
# this is the view function we are testing
response = add_keyword_metadata(request, hs_file_type=file_type,
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
# there should be 2 keywords for the raster file type yet
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
if file_type != "NetCDFLogicalFile":
self.assertEqual(len(logical_file.metadata.keywords), 2)
else:
self.assertEqual(len(logical_file.metadata.keywords), 3)
self.assertIn('keyword-1', logical_file.metadata.keywords)
self.assertIn('keyword-2', logical_file.metadata.keywords)
# resource level keywords must have been updated with the keywords we added
# to file level
res_keywords = [subject.value for subject in
self.composite_resource.metadata.subjects.all()]
for kw in logical_file.metadata.keywords:
self.assertIn(kw, res_keywords)
# delete keyword
url = reverse('delete_file_keyword_metadata', kwargs=url_params)
request = self.factory.post(url, data={'keyword': 'keyword-1'})
request.user = self.user
# this is the view function we are testing
response = delete_keyword_metadata(request, hs_file_type=file_type,
file_type_id=logical_file.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_dict = json.loads(response.content)
self.assertEqual('success', response_dict['status'])
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
if file_type != "NetCDFLogicalFile":
self.assertEqual(len(logical_file.metadata.keywords), 1)
else:
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.assertIn('keyword-2', logical_file.metadata.keywords)
# test that deleting a file level keyword doesn't delete the same keyword from
# resource level
self.assertIn('keyword-1', res_keywords)
self.composite_resource.delete()
def test_metadata_CRUD(self):
# this is test metadata related to GeoRasterLogicalFile
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# extract metadata by setting to geo raster file type
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
res_file = self.composite_resource.files.first()
# test that we can update raster specific metadata at the file level
# test that we can update dataset_name of the logical file object
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'small_logan')
logical_file.dataset_name = "big_logan"
logical_file.save()
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, 'big_logan')
# delete default original coverage metadata
self.assertNotEquals(logical_file.metadata.originalCoverage, None)
logical_file.metadata.originalCoverage.delete()
# create new original coverage metadata with meaningful value
value = {
"northlimit": 12,
"projection": "transverse_mercator",
"units": "meter",
"southlimit": 10,
"eastlimit": 23,
"westlimit": 2
}
logical_file.metadata.create_element('originalcoverage', value=value)
self.assertEquals(logical_file.metadata.originalCoverage.value, value)
# multiple original coverage elements are not allowed - should raise exception
with self.assertRaises(IntegrityError):
logical_file.metadata.create_element('originalcoverage',
value=value)
# delete default cell information element
self.assertNotEquals(logical_file.metadata.cellInformation, None)
logical_file.metadata.cellInformation.delete()
# create new cell information metadata with meaningful value
logical_file.metadata.create_element(
'cellinformation',
name='cellinfo',
cellDataType='Float32',
rows=1660,
columns=985,
cellSizeXValue=30.0,
cellSizeYValue=30.0,
)
cell_info = logical_file.metadata.cellInformation
self.assertEquals(cell_info.rows, 1660)
self.assertEquals(cell_info.columns, 985)
self.assertEquals(cell_info.cellSizeXValue, 30.0)
self.assertEquals(cell_info.cellSizeYValue, 30.0)
self.assertEquals(cell_info.cellDataType, 'Float32')
# multiple cell Information elements are not allowed - should raise exception
with self.assertRaises(IntegrityError):
logical_file.metadata.create_element(
'cellinformation',
name='cellinfo',
cellDataType='Float32',
rows=1660,
columns=985,
cellSizeXValue=30.0,
cellSizeYValue=30.0,
)
# delete default band information element
self.assertNotEquals(logical_file.metadata.bandInformations, None)
logical_file.metadata.bandInformations.first().delete()
# create band information element with meaningful value
logical_file.metadata.create_element('bandinformation',
name='bandinfo',
variableName='diginal elevation',
variableUnit='meter',
method='this is method',
comment='this is comment',
maximumValue=1000,
minimumValue=0,
noDataValue=-9999)
band_info = logical_file.metadata.bandInformations.first()
self.assertEquals(band_info.name, 'bandinfo')
self.assertEquals(band_info.variableName, 'diginal elevation')
self.assertEquals(band_info.variableUnit, 'meter')
self.assertEquals(band_info.method, 'this is method')
self.assertEquals(band_info.comment, 'this is comment')
self.assertEquals(band_info.maximumValue, '1000')
self.assertEquals(band_info.minimumValue, '0')
self.assertEquals(band_info.noDataValue, '-9999')
# multiple band information elements are allowed
logical_file.metadata.create_element('bandinformation',
name='bandinfo',
variableName='diginal elevation2',
variableUnit='meter',
method='this is method',
comment='this is comment',
maximumValue=1000,
minimumValue=0,
noDataValue=-9999)
self.assertEquals(logical_file.metadata.bandInformations.all().count(),
2)
# test metadata delete
# original coverage deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'originalcoverage', logical_file.metadata.originalCoverage.id)
# cell information deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'cellinformation', logical_file.metadata.cellInformation.id)
# band information deletion is not allowed
with self.assertRaises(ValidationError):
logical_file.metadata.delete_element(
'bandinformation',
logical_file.metadata.bandInformations.first().id)
# test metadata update
# update original coverage element
value_2 = {
"northlimit": 12.5,
"projection": "transverse_mercator",
"units": "meter",
"southlimit": 10.5,
"eastlimit": 23.5,
"westlimit": 2.5
}
logical_file.metadata.update_element(
'originalcoverage',
logical_file.metadata.originalCoverage.id,
value=value_2)
self.assertEquals(logical_file.metadata.originalCoverage.value,
value_2)
# update cell info element
logical_file.metadata.update_element(
'cellinformation',
logical_file.metadata.cellInformation.id,
name='cellinfo',
cellDataType='Double',
rows=166,
columns=98,
cellSizeXValue=3.0,
cellSizeYValue=3.0,
)
cell_info = logical_file.metadata.cellInformation
self.assertEquals(cell_info.rows, 166)
self.assertEquals(cell_info.columns, 98)
self.assertEquals(cell_info.cellSizeXValue, 3.0)
self.assertEquals(cell_info.cellSizeYValue, 3.0)
self.assertEquals(cell_info.cellDataType, 'Double')
# update band info element
logical_file.metadata.update_element(
'bandinformation',
logical_file.metadata.bandInformations.first().id,
name='bandinfo',
variableName='precipitation',
variableUnit='mm/h',
method='this is method2',
comment='this is comment2',
maximumValue=1001,
minimumValue=1,
noDataValue=-9998)
band_info = logical_file.metadata.bandInformations.first()
self.assertEquals(band_info.name, 'bandinfo')
self.assertEquals(band_info.variableName, 'precipitation')
self.assertEquals(band_info.variableUnit, 'mm/h')
self.assertEquals(band_info.method, 'this is method2')
self.assertEquals(band_info.comment, 'this is comment2')
self.assertEquals(band_info.maximumValue, '1001')
self.assertEquals(band_info.minimumValue, '1')
self.assertEquals(band_info.noDataValue, '-9998')
# test extra_metadata for the logical file
# there should be no key/value metadata at this point
self.assertEqual(logical_file.metadata.extra_metadata, {})
# create key/vale metadata
logical_file.metadata.extra_metadata = {
'key1': 'value 1',
'key2': 'value 2'
}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {
'key1': 'value 1',
'key2': 'value 2'
})
# update key/value metadata
logical_file.metadata.extra_metadata = {
'key1': 'value 1',
'key2': 'value 2',
'key 3': 'value3'
}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {
'key1': 'value 1',
'key2': 'value 2',
'key 3': 'value3'
})
# delete key/value metadata
logical_file.metadata.extra_metadata = {}
logical_file.metadata.save()
self.assertEqual(logical_file.metadata.extra_metadata, {})
self.composite_resource.delete()
