def test_tif_set_file_type_to_geo_raster(self): # here we are using a valid raster tif file for setting it # to Geo Raster file type which includes metadata extraction self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test extracted raster file type metadata assert_raster_file_type_metadata(self) # there should not be any file level keywords at this point res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertTrue(isinstance(logical_file, GeoRasterLogicalFile)) self.assertTrue(logical_file.metadata, GeoRasterFileMetaData) # TODO: not sure why there would be file level keywords - commented out as the test is # failing # self.assertEqual(logical_file.metadata.keywords, []) self.composite_resource.delete()
def test_logical_file_delete(self): # test that when an instance GeoRasterFileType is deleted # all files associated with GeoRasterFileType is deleted self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that we have one logical file of type GeoRasterFileType as a result # of metadata extraction self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) logical_file = GeoRasterLogicalFile.objects.first() self.assertEqual(logical_file.files.all().count(), 2) self.assertEqual(self.composite_resource.files.all().count(), 2) self.assertEqual(set(self.composite_resource.files.all()), set(logical_file.files.all())) # delete the logical file using the custom delete function - logical_delete() logical_file.logical_delete(self.user) self.assertEqual(self.composite_resource.files.all().count(), 0) self.composite_resource.delete()
def test_set_file_type_to_geo_raster_invalid_file_2(self): # here we are using a raster tif file for setting it # to Geo Raster file type which already been previously set to this file type - should fail self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with generic logical file self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # set tif file to GeoRasterFileType GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # check that the resource file is associated with a logical file res_file = hydroshare.utils.get_resource_files_by_extension( self.composite_resource, '.tif')[0] self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile") # trying to set this tif file again to geo raster file type should raise # ValidationError with self.assertRaises(ValidationError): GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(RasterFileTypeMetaDataTest, self).is_federated_irods_available(): return # here we are using a valid raster tif file for setting it # to Geo Raster file type which includes metadata extraction fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_file_name) res_upload_files = [] fed_res_path = hydroshare.utils.get_federated_zone_home_path( fed_test_file_full_path) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Federated Composite Resource Raster File Type Testing', files=res_upload_files, fed_res_file_names=[fed_test_file_full_path], fed_res_path=fed_res_path, fed_copy_or_move='copy', metadata=[]) # test resource is created on federated zone self.assertNotEqual(self.composite_resource.resource_federation_path, '') # set the logical file -which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) fed_file_path = "data/contents/{}".format(self.raster_file_name) self.assertEqual(res_file.fed_resource_file_name_or_path, fed_file_path) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test extracted raster file type metadata assert_raster_file_type_metadata(self)
def test_add_update_metadata_to_raster_file_type(self): self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource(self.raster_file_obj) res_file = self.composite_resource.files.first() # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile") # no temporal coverage for the raster file type yet self.assertEqual(logical_file.metadata.temporal_coverage, None) # add temporal coverage url_params = {'hs_file_type': 'GeoRasterLogicalFile', 'file_type_id': logical_file.id, 'element_name': 'coverage' } url = reverse('add_file_metadata', kwargs=url_params) request = self.factory.post(url, data={'start': '1/1/2010', 'end': '12/12/2015'}) request.user = self.user # this is the view function we are testing response = add_metadata_element(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id, element_name='coverage') self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) # now the raster file should have temporal coverage element self.assertNotEqual(logical_file.metadata.temporal_coverage, None) # test updating temporal coverage url_params['element_id'] = logical_file.metadata.temporal_coverage.id url = reverse('update_file_metadata', kwargs=url_params) request = self.factory.post(url, data={'start': '1/1/2011', 'end': '12/12/2016'}) request.user = self.user # this is the view function we are testing response = update_metadata_element(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id, element_name='coverage', element_id=logical_file.metadata.temporal_coverage.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) temporal_coverage = logical_file.metadata.temporal_coverage self.assertEqual(temporal_coverage.value['start'], '2011-01-01') self.assertEqual(temporal_coverage.value['end'], '2016-12-12') self.composite_resource.delete()
def test_raster_file_type_folder_delete(self): # when a file is set to georasterlogical file type # system automatically creates folder using the name of the file # that was used to set the file type # Here we need to test that when that folder gets deleted, all files # in that folder gets deleted, the logicalfile object gets deleted and # the associated metadata objects get deleted self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that we have one logical file of type GeoRasterFileType as a result # of metadata extraction self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) # should have one GeoRasterFileMetadata object self.assertEqual(GeoRasterFileMetaData.objects.count(), 1) # there should be 2 content files self.assertEqual(self.composite_resource.files.count(), 2) # test that there are metadata associated with the logical file self.assertNotEqual(Coverage.objects.count(), 0) self.assertNotEqual(OriginalCoverage.objects.count(), 0) self.assertNotEqual(CellInformation.objects.count(), 0) self.assertNotEqual(BandInformation.objects.count(), 0) # delete the folder for the logical file folder_path = "data/contents/small_logan" remove_folder(self.user, self.composite_resource.short_id, folder_path) # there should no content files self.assertEqual(self.composite_resource.files.count(), 0) # there should not be any GeoRaster logical file or metadata file self.assertEqual(GeoRasterLogicalFile.objects.count(), 0) self.assertEqual(GeoRasterFileMetaData.objects.count(), 0) # test that all metadata associated with the logical file got deleted self.assertEqual(Coverage.objects.count(), 0) self.assertEqual(OriginalCoverage.objects.count(), 0) self.assertEqual(CellInformation.objects.count(), 0) self.assertEqual(BandInformation.objects.count(), 0) self.composite_resource.delete()
def test_tif_set_file_type_to_geo_raster(self): super(RasterFileTypeMetaDataTest, self).assert_federated_irods_available() # here we are using a valid raster tif file for setting it # to Geo Raster file type which includes metadata extraction fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_file_name) res_upload_files = [] fed_res_path = hydroshare.utils.get_federated_zone_home_path( fed_test_file_full_path) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Federated Composite Resource Raster File Type Testing', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path, move=False, metadata=[], auto_aggregate=False) # test resource is created on federated zone self.assertNotEqual(self.composite_resource.resource_federation_path, '') self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) expected_folder_name = base_file_name # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) fed_file_path = "{}/{}".format(self.composite_resource.file_path, self.raster_file_name) self.assertEqual(res_file.storage_path, fed_file_path) # set the tif file to GeoRasterLogicalFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test extracted raster file type metadata assert_raster_file_type_metadata(self, aggr_folder_path=expected_folder_name)
def _test_file_metadata_on_file_delete(self, ext): self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that we have one logical file of type GeoRasterFileType self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) self.assertEqual(GeoRasterFileMetaData.objects.count(), 1) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file # there should be 1 coverage element of type spatial self.assertEqual(logical_file.metadata.coverages.all().count(), 1) self.assertNotEqual(logical_file.metadata.spatial_coverage, None) self.assertNotEqual(logical_file.metadata.originalCoverage, None) self.assertNotEqual(logical_file.metadata.cellInformation, None) self.assertNotEqual(logical_file.metadata.bandInformations, None) # there should be 2 coverage objects - one at the resource level # and the other one at the file type level self.assertEqual(Coverage.objects.count(), 2) self.assertEqual(OriginalCoverage.objects.count(), 1) self.assertEqual(CellInformation.objects.count(), 1) self.assertEqual(BandInformation.objects.count(), 1) # delete content file specified by extension (ext parameter) res_file_tif = hydroshare.utils.get_resource_files_by_extension( self.composite_resource, ext)[0] hydroshare.delete_resource_file(self.composite_resource.short_id, res_file_tif.id, self.user) # test that we don't have logical file of type GeoRasterFileType self.assertEqual(GeoRasterLogicalFile.objects.count(), 0) self.assertEqual(GeoRasterFileMetaData.objects.count(), 0) # test that all metadata deleted self.assertEqual(Coverage.objects.count(), 0) self.assertEqual(OriginalCoverage.objects.count(), 0) self.assertEqual(CellInformation.objects.count(), 0) self.assertEqual(BandInformation.objects.count(), 0)
def test_file_metadata_on_logical_file_delete(self): # test that when the GeoRasterFileType is deleted # all metadata associated with GeoRasterFileType is deleted self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that we have one logical file of type GeoRasterFileType as a result # of metadata extraction self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) self.assertEqual(GeoRasterFileMetaData.objects.count(), 1) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file # test that we have the metadata elements # there should be 2 Coverage objects - one at the resource level and # the other one at the file type level self.assertEqual(Coverage.objects.count(), 2) self.assertEqual( self.composite_resource.metadata.coverages.all().count(), 1) self.assertEqual(logical_file.metadata.coverages.all().count(), 1) self.assertEqual(OriginalCoverage.objects.count(), 1) self.assertEqual(CellInformation.objects.count(), 1) self.assertEqual(BandInformation.objects.count(), 1) # delete the logical file logical_file.logical_delete(self.user) # test that we have no logical file of type GeoRasterFileType self.assertEqual(GeoRasterLogicalFile.objects.count(), 0) self.assertEqual(GeoRasterFileMetaData.objects.count(), 0) # test that all metadata deleted self.assertEqual(Coverage.objects.count(), 0) self.assertEqual(OriginalCoverage.objects.count(), 0) self.assertEqual(CellInformation.objects.count(), 0) self.assertEqual(BandInformation.objects.count(), 0) self.composite_resource.delete()
def _test_invalid_file(self): self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with the generic logical file self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # trying to set this invalid tif file to geo raster file type should raise # ValidationError with self.assertRaises(ValidationError): GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that the invalid file did not get deleted self.assertEqual(self.composite_resource.files.all().count(), 1) # check that the resource file is not associated with generic logical file self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
def test_tif_set_file_type_to_geo_raster(self): # here we are using a valid raster tif file for setting it # to Geo Raster file type which includes metadata extraction self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test extracted raster file type metadata assert_raster_file_type_metadata(self)
def _content_file_delete(self, ext): # test that when any file that is part of an instance GeoRasterFileType is deleted # all files associated with GeoRasterFileType is deleted self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) self.assertEqual(self.composite_resource.files.all().count(), 2) self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) # delete the content file specified by the ext (file extension param) res_file_tif = hydroshare.utils.get_resource_files_by_extension( self.composite_resource, ext)[0] hydroshare.delete_resource_file(self.composite_resource.short_id, res_file_tif.id, self.user) self.assertEqual(self.composite_resource.files.all().count(), 0) self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
def test_file_rename_or_move(self): # test that file can't be moved or renamed for any resource file # that's part of the GeoRaster logical file object (LFO) self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test renaming of files that are associated with raster LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 2) src_path = 'data/contents/small_logan/small_logan.tif' tgt_path = "data/contents/small_logan/small_logan_1.tif" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/small_logan/small_logan.vrt' tgt_path = "data/contents/small_logan/small_logan_1.vrt" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # test moving the files associated with geo raster LFO src_path = 'data/contents/small_logan/small_logan.tif' tgt_path = "data/contents/big_logan/small_logan.tif" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/small_logan/small_logan.vrt' tgt_path = "data/contents/big_logan/small_logan.vrt" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_update_resource_spatial_coverage(self): """Here we are testing the update of resource spatial coverage based on the spatial coverages from all the aggregations in the resource using the view function update_resource_coverage""" self.create_composite_resource(file_to_upload=self.raster_file) res_file = self.composite_resource.files.first() # set the tif file to GeoRasterLogicalFile type (aggregation) GeoRasterLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) url_params = {'resource_id': self.composite_resource.short_id, 'coverage_type': 'spatial' } url = reverse('update_resource_coverage', kwargs=url_params) request = self.factory.post(url) request.user = self.user # this is the view function we are testing response = update_resource_coverage(request, resource_id=self.composite_resource.short_id, coverage_type='spatial') self.assertEqual(response.status_code, status.HTTP_200_OK) self.composite_resource.delete()
def test_copy_composite_resource(self): """Test that logical file type objects gets copied along with the metadata that each logical file type object contains. Here we are not testing resource level metadata copy as that has been tested in separate unit tests""" self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.owner, title='Test Composite Resource', files=files, auto_aggregate=False ) # run the resource post creation signal utils.resource_post_create_actions(resource=self.composite_resource, user=self.owner, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not associated with file type self.assertEqual(res_file.has_logical_file, False) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, self.owner, res_file.id) # ensure a nonowner who does not have permission to view a resource cannot copy it with self.assertRaises(PermissionDenied): hydroshare.create_empty_resource(self.composite_resource.short_id, self.nonowner, action='copy') # give nonowner view privilege so nonowner can create a new copy of this resource self.owner.uaccess.share_resource_with_user(self.composite_resource, self.nonowner, PrivilegeCodes.VIEW) orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file # add some key value metadata orig_geo_raster_lfo.metadata.extra_metadata = {'key-1': 'value-1', 'key-2': 'value-2'} # create a copy of the composite resource new_composite_resource = hydroshare.create_empty_resource(self.composite_resource.short_id, self.nonowner, action='copy') new_composite_resource = hydroshare.copy_resource(self.composite_resource, new_composite_resource) # check that there is 2 GeoRasterLogicalFile objects self.assertEqual(GeoRasterLogicalFile.objects.count(), 2) # compare the 2 GeoRasterLogicalFile objects from the original resource and the new one orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file copy_res_file = new_composite_resource.files.first() copy_geo_raster_lfo = copy_res_file.logical_file # check that we put the 2 files in a new folder (cea) for res_file in self.composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format(self.composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) for res_file in new_composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format(new_composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) # both logical file objects should have 2 resource files self.assertEqual(orig_geo_raster_lfo.files.count(), copy_geo_raster_lfo.files.count()) self.assertEqual(orig_geo_raster_lfo.files.count(), 2) # both logical file objects should have same dataset_name self.assertEqual(orig_geo_raster_lfo.dataset_name, copy_geo_raster_lfo.dataset_name) # both should have same key/value metadata self.assertEqual(orig_geo_raster_lfo.metadata.extra_metadata, copy_geo_raster_lfo.metadata.extra_metadata) # both logical file objects should have same coverage metadata self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), copy_geo_raster_lfo.metadata.coverages.count()) self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), 1) org_spatial_coverage = orig_geo_raster_lfo.metadata.spatial_coverage copy_spatial_coverage = copy_geo_raster_lfo.metadata.spatial_coverage self.assertEqual(org_spatial_coverage.type, copy_spatial_coverage.type) self.assertEqual(org_spatial_coverage.type, 'box') self.assertEqual(org_spatial_coverage.value['projection'], copy_spatial_coverage.value['projection']) self.assertEqual(org_spatial_coverage.value['units'], copy_spatial_coverage.value['units']) self.assertEqual(org_spatial_coverage.value['northlimit'], copy_spatial_coverage.value['northlimit']) self.assertEqual(org_spatial_coverage.value['eastlimit'], copy_spatial_coverage.value['eastlimit']) self.assertEqual(org_spatial_coverage.value['southlimit'], copy_spatial_coverage.value['southlimit']) self.assertEqual(org_spatial_coverage.value['westlimit'], copy_spatial_coverage.value['westlimit']) # both logical file objects should have same original coverage org_orig_coverage = orig_geo_raster_lfo.metadata.originalCoverage copy_orig_coverage = copy_geo_raster_lfo.metadata.originalCoverage self.assertEqual(org_orig_coverage.value['projection'], copy_orig_coverage.value['projection']) self.assertEqual(org_orig_coverage.value['units'], copy_orig_coverage.value['units']) self.assertEqual(org_orig_coverage.value['northlimit'], copy_orig_coverage.value['northlimit']) self.assertEqual(org_orig_coverage.value['eastlimit'], copy_orig_coverage.value['eastlimit']) self.assertEqual(org_orig_coverage.value['southlimit'], copy_orig_coverage.value['southlimit']) self.assertEqual(org_orig_coverage.value['westlimit'], copy_orig_coverage.value['westlimit']) # both logical file objects should have same cell information metadata orig_cell_info = orig_geo_raster_lfo.metadata.cellInformation copy_cell_info = copy_geo_raster_lfo.metadata.cellInformation self.assertEqual(orig_cell_info.rows, copy_cell_info.rows) self.assertEqual(orig_cell_info.columns, copy_cell_info.columns) self.assertEqual(orig_cell_info.cellSizeXValue, copy_cell_info.cellSizeXValue) self.assertEqual(orig_cell_info.cellSizeYValue, copy_cell_info.cellSizeYValue) self.assertEqual(orig_cell_info.cellDataType, copy_cell_info.cellDataType) # both logical file objects should have same band information metadata self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), 1) self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), copy_geo_raster_lfo.metadata.bandInformations.count()) orig_band_info = orig_geo_raster_lfo.metadata.bandInformations.first() copy_band_info = copy_geo_raster_lfo.metadata.bandInformations.first() self.assertEqual(orig_band_info.noDataValue, copy_band_info.noDataValue) self.assertEqual(orig_band_info.maximumValue, copy_band_info.maximumValue) self.assertEqual(orig_band_info.minimumValue, copy_band_info.minimumValue) # make sure to clean up all created resources to clean up iRODS storage if self.composite_resource: self.composite_resource.delete() if new_composite_resource: new_composite_resource.delete()
def test_CRUD_key_value_metadata_raster_file_type(self): self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource(self.raster_file_obj) res_file = self.composite_resource.files.first() # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile") # no key/value metadata for the raster file type yet self.assertEqual(logical_file.metadata.extra_metadata, {}) url_params = {'hs_file_type': 'GeoRasterLogicalFile', 'file_type_id': logical_file.id } url = reverse('update_file_keyvalue_metadata', kwargs=url_params) request = self.factory.post(url, data={'key': 'key-1', 'value': 'value-1'}) request.user = self.user # this is the view function we are testing response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) # there should be key/value metadata for the raster file type yet res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertNotEqual(logical_file.metadata.extra_metadata, {}) self.assertEqual(logical_file.metadata.extra_metadata['key-1'], 'value-1') # update existing key value metadata - updating both key and value request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-2', 'key_original': 'key-1'}) request.user = self.user response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-2') self.assertNotIn('key-1', logical_file.metadata.extra_metadata.keys()) # update existing key value metadata - updating value only request = self.factory.post(url, data={'key': 'key-2', 'value': 'value-1', 'key_original': 'key-2'}) request.user = self.user response = update_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.extra_metadata['key-2'], 'value-1') # delete key/value data using the view function request = self.factory.post(url, data={'key': 'key-2'}) request.user = self.user # this the view function we are testing response = delete_key_value_metadata(request, hs_file_type="GeoRasterLogicalFile", file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file # at this point there should not be any key/value metadata self.assertEqual(logical_file.metadata.extra_metadata, {}) self.composite_resource.delete()
def test_zip_set_file_type_to_geo_raster(self): # here we are using a valid raster zip file for setting it # to Geo Raster file type which includes metadata extraction self.raster_file_obj = open(self.raster_zip_file, 'r') self._create_composite_resource() self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that the resource file is not associated with any logical file # self.assertEqual(res_file.has_logical_file, False) # set the zip file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test the resource now has 3 files (one vrt file and 2 tif files) self.assertEqual(self.composite_resource.files.all().count(), 3) tif_files = hydroshare.utils.get_resource_files_by_extension( self.composite_resource, '.tif') self.assertEqual(len(tif_files), 2) vrt_files = hydroshare.utils.get_resource_files_by_extension( self.composite_resource, '.vrt') self.assertEqual(len(vrt_files), 1) # check that the logicalfile is associated with 3 files self.assertEqual(GeoRasterLogicalFile.objects.count(), 1) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.dataset_name, 'logan_vrt_small') self.assertEqual(logical_file.has_metadata, True) self.assertEqual(logical_file.files.all().count(), 3) self.assertEqual(set(self.composite_resource.files.all()), set(logical_file.files.all())) # check that we put the 3 files in a new folder (small_logan) for res_file in self.composite_resource.files.all(): file_path, base_file_name, _ = get_resource_file_name_and_extension( res_file) expected_file_path = "{}/data/contents/logan_vrt_small/{}" expected_file_path = expected_file_path.format( self.composite_resource.short_id, base_file_name) self.assertEqual(file_path, expected_file_path) # check that there is no GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 0) # test that size property of the logical file is equal to sun of size of all files # that are part of the logical file self.assertEqual(logical_file.size, sum([f.size for f in logical_file.files.all()])) # test extracted metadata for the file type # geo raster file type should have all the metadata elements self.assertEqual(logical_file.metadata.has_all_required_elements(), True) # there should be 1 coverage element - box type self.assertNotEqual(logical_file.metadata.spatial_coverage, None) self.assertEqual(logical_file.metadata.spatial_coverage.type, 'box') box_coverage = logical_file.metadata.spatial_coverage self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 42.050028785767275) self.assertEqual(box_coverage.value['eastlimit'], -111.5773750264389) self.assertEqual(box_coverage.value['southlimit'], 41.98745777902698) self.assertEqual(box_coverage.value['westlimit'], -111.65768822411239) # testing extended metadata element: original coverage ori_coverage = logical_file.metadata.originalCoverage self.assertNotEqual(ori_coverage, None) self.assertEqual(ori_coverage.value['northlimit'], 4655492.446916306) self.assertEqual(ori_coverage.value['eastlimit'], 452174.01909127034) self.assertEqual(ori_coverage.value['southlimit'], 4648592.446916306) self.assertEqual(ori_coverage.value['westlimit'], 445574.01909127034) self.assertEqual(ori_coverage.value['units'], 'meter') self.assertEqual(ori_coverage.value['projection'], 'NAD83 / UTM zone 12N') # testing extended metadata element: cell information cell_info = logical_file.metadata.cellInformation self.assertEqual(cell_info.rows, 230) self.assertEqual(cell_info.columns, 220) self.assertEqual(cell_info.cellSizeXValue, 30.0) self.assertEqual(cell_info.cellSizeYValue, 30.0) self.assertEqual(cell_info.cellDataType, 'Float32') # testing extended metadata element: band information self.assertEqual(logical_file.metadata.bandInformations.count(), 1) band_info = logical_file.metadata.bandInformations.first() self.assertEqual(band_info.noDataValue, '-3.40282346639e+38') self.assertEqual(band_info.maximumValue, '2880.00708008') self.assertEqual(band_info.minimumValue, '2274.95898438') self.composite_resource.delete()
def _add_delete_keywords_file_type(self, file_obj, file_type): self._create_composite_resource(file_obj) res_file = self.composite_resource.files.first() # set specific file type if file_type == "GeoRasterLogicalFile": GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) else: NetCDFLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(res_file.logical_file_type_name, file_type) if file_type != "NetCDFLogicalFile": # no keyword metadata for the raster file type yet self.assertEqual(len(logical_file.metadata.keywords), 0) else: # one keyword metadata for the netcdf file type self.assertEqual(len(logical_file.metadata.keywords), 1) # at this point resource should have all the keywords that we have for the file type res_keywords = [subject.value for subject in self.composite_resource.metadata.subjects.all()] for kw in logical_file.metadata.keywords: self.assertIn(kw, res_keywords) # add keywords at the file level url_params = {'hs_file_type': file_type, 'file_type_id': logical_file.id } url = reverse('add_file_keyword_metadata', kwargs=url_params) request = self.factory.post(url, data={'keywords': 'keyword-1,keyword-2'}) request.user = self.user # this is the view function we are testing response = add_keyword_metadata(request, hs_file_type=file_type, file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) # there should be 2 keywords for the raster file type yet res_file = self.composite_resource.files.first() logical_file = res_file.logical_file if file_type != "NetCDFLogicalFile": self.assertEqual(len(logical_file.metadata.keywords), 2) else: self.assertEqual(len(logical_file.metadata.keywords), 3) self.assertIn('keyword-1', logical_file.metadata.keywords) self.assertIn('keyword-2', logical_file.metadata.keywords) # resource level keywords must have been updated with the keywords we added # to file level res_keywords = [subject.value for subject in self.composite_resource.metadata.subjects.all()] for kw in logical_file.metadata.keywords: self.assertIn(kw, res_keywords) # delete keyword url = reverse('delete_file_keyword_metadata', kwargs=url_params) request = self.factory.post(url, data={'keyword': 'keyword-1'}) request.user = self.user # this is the view function we are testing response = delete_keyword_metadata(request, hs_file_type=file_type, file_type_id=logical_file.id) self.assertEqual(response.status_code, status.HTTP_200_OK) response_dict = json.loads(response.content) self.assertEqual('success', response_dict['status']) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file if file_type != "NetCDFLogicalFile": self.assertEqual(len(logical_file.metadata.keywords), 1) else: self.assertEqual(len(logical_file.metadata.keywords), 2) self.assertIn('keyword-2', logical_file.metadata.keywords) # test that deleting a file level keyword doesn't delete the same keyword from # resource level self.assertIn('keyword-1', res_keywords) self.composite_resource.delete()
def test_metadata_CRUD(self): # this is test metadata related to GeoRasterLogicalFile self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # extract metadata by setting to geo raster file type GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) res_file = self.composite_resource.files.first() # test that we can update raster specific metadata at the file level # test that we can update dataset_name of the logical file object logical_file = res_file.logical_file self.assertEqual(logical_file.dataset_name, 'small_logan') logical_file.dataset_name = "big_logan" logical_file.save() logical_file = res_file.logical_file self.assertEqual(logical_file.dataset_name, 'big_logan') # delete default original coverage metadata self.assertNotEquals(logical_file.metadata.originalCoverage, None) logical_file.metadata.originalCoverage.delete() # create new original coverage metadata with meaningful value value = { "northlimit": 12, "projection": "transverse_mercator", "units": "meter", "southlimit": 10, "eastlimit": 23, "westlimit": 2 } logical_file.metadata.create_element('originalcoverage', value=value) self.assertEquals(logical_file.metadata.originalCoverage.value, value) # multiple original coverage elements are not allowed - should raise exception with self.assertRaises(IntegrityError): logical_file.metadata.create_element('originalcoverage', value=value) # delete default cell information element self.assertNotEquals(logical_file.metadata.cellInformation, None) logical_file.metadata.cellInformation.delete() # create new cell information metadata with meaningful value logical_file.metadata.create_element( 'cellinformation', name='cellinfo', cellDataType='Float32', rows=1660, columns=985, cellSizeXValue=30.0, cellSizeYValue=30.0, ) cell_info = logical_file.metadata.cellInformation self.assertEquals(cell_info.rows, 1660) self.assertEquals(cell_info.columns, 985) self.assertEquals(cell_info.cellSizeXValue, 30.0) self.assertEquals(cell_info.cellSizeYValue, 30.0) self.assertEquals(cell_info.cellDataType, 'Float32') # multiple cell Information elements are not allowed - should raise exception with self.assertRaises(IntegrityError): logical_file.metadata.create_element( 'cellinformation', name='cellinfo', cellDataType='Float32', rows=1660, columns=985, cellSizeXValue=30.0, cellSizeYValue=30.0, ) # delete default band information element self.assertNotEquals(logical_file.metadata.bandInformations, None) logical_file.metadata.bandInformations.first().delete() # create band information element with meaningful value logical_file.metadata.create_element('bandinformation', name='bandinfo', variableName='diginal elevation', variableUnit='meter', method='this is method', comment='this is comment', maximumValue=1000, minimumValue=0, noDataValue=-9999) band_info = logical_file.metadata.bandInformations.first() self.assertEquals(band_info.name, 'bandinfo') self.assertEquals(band_info.variableName, 'diginal elevation') self.assertEquals(band_info.variableUnit, 'meter') self.assertEquals(band_info.method, 'this is method') self.assertEquals(band_info.comment, 'this is comment') self.assertEquals(band_info.maximumValue, '1000') self.assertEquals(band_info.minimumValue, '0') self.assertEquals(band_info.noDataValue, '-9999') # multiple band information elements are allowed logical_file.metadata.create_element('bandinformation', name='bandinfo', variableName='diginal elevation2', variableUnit='meter', method='this is method', comment='this is comment', maximumValue=1000, minimumValue=0, noDataValue=-9999) self.assertEquals(logical_file.metadata.bandInformations.all().count(), 2) # test metadata delete # original coverage deletion is not allowed with self.assertRaises(ValidationError): logical_file.metadata.delete_element( 'originalcoverage', logical_file.metadata.originalCoverage.id) # cell information deletion is not allowed with self.assertRaises(ValidationError): logical_file.metadata.delete_element( 'cellinformation', logical_file.metadata.cellInformation.id) # band information deletion is not allowed with self.assertRaises(ValidationError): logical_file.metadata.delete_element( 'bandinformation', logical_file.metadata.bandInformations.first().id) # test metadata update # update original coverage element value_2 = { "northlimit": 12.5, "projection": "transverse_mercator", "units": "meter", "southlimit": 10.5, "eastlimit": 23.5, "westlimit": 2.5 } logical_file.metadata.update_element( 'originalcoverage', logical_file.metadata.originalCoverage.id, value=value_2) self.assertEquals(logical_file.metadata.originalCoverage.value, value_2) # update cell info element logical_file.metadata.update_element( 'cellinformation', logical_file.metadata.cellInformation.id, name='cellinfo', cellDataType='Double', rows=166, columns=98, cellSizeXValue=3.0, cellSizeYValue=3.0, ) cell_info = logical_file.metadata.cellInformation self.assertEquals(cell_info.rows, 166) self.assertEquals(cell_info.columns, 98) self.assertEquals(cell_info.cellSizeXValue, 3.0) self.assertEquals(cell_info.cellSizeYValue, 3.0) self.assertEquals(cell_info.cellDataType, 'Double') # update band info element logical_file.metadata.update_element( 'bandinformation', logical_file.metadata.bandInformations.first().id, name='bandinfo', variableName='precipitation', variableUnit='mm/h', method='this is method2', comment='this is comment2', maximumValue=1001, minimumValue=1, noDataValue=-9998) band_info = logical_file.metadata.bandInformations.first() self.assertEquals(band_info.name, 'bandinfo') self.assertEquals(band_info.variableName, 'precipitation') self.assertEquals(band_info.variableUnit, 'mm/h') self.assertEquals(band_info.method, 'this is method2') self.assertEquals(band_info.comment, 'this is comment2') self.assertEquals(band_info.maximumValue, '1001') self.assertEquals(band_info.minimumValue, '1') self.assertEquals(band_info.noDataValue, '-9998') # test extra_metadata for the logical file # there should be no key/value metadata at this point self.assertEqual(logical_file.metadata.extra_metadata, {}) # create key/vale metadata logical_file.metadata.extra_metadata = { 'key1': 'value 1', 'key2': 'value 2' } logical_file.metadata.save() self.assertEqual(logical_file.metadata.extra_metadata, { 'key1': 'value 1', 'key2': 'value 2' }) # update key/value metadata logical_file.metadata.extra_metadata = { 'key1': 'value 1', 'key2': 'value 2', 'key 3': 'value3' } logical_file.metadata.save() self.assertEqual(logical_file.metadata.extra_metadata, { 'key1': 'value 1', 'key2': 'value 2', 'key 3': 'value3' }) # delete key/value metadata logical_file.metadata.extra_metadata = {} logical_file.metadata.save() self.assertEqual(logical_file.metadata.extra_metadata, {}) self.composite_resource.delete()