def test_public_or_discoverable(self): self.assertFalse(self.resModelProgram.has_required_content_files()) self.assertFalse( self.resModelProgram.metadata.has_all_required_elements()) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Subject', value="test subject") self.assertTrue(self.resModelProgram.has_required_content_files()) self.assertTrue( self.resModelProgram.metadata.has_all_required_elements()) self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_set_file_type_success_1(self): # here we will set the tif file to GeoRaster file type # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "rb") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file, ), user=self.user, auto_aggregate=False) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) # test the set file type endpoint url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" set_file_type_url = url_template.format( res_id=self.resource.short_id, file_path=self.raster_file_name, file_type="GeoRaster") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_allowed_file_types(self): # test allowed file type is '.*' self.assertEquals(self.resModelProgram.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # Upload any file type should pass both the file pre add check and post add check files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelProgram.metadata.program, None) # Uploading any other file type should pass both the file pre add check post add check files = [UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=True) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=True) # there should two content files self.assertEquals(self.resModelProgram.files.all().count(), 2) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelProgram.metadata.program, None)
def test_auto_aggregate_files_add_child_folder(self): """test adding files in different folders""" self.create_composite_resource() self.assertEqual(0, GeoFeatureLogicalFile.objects.count()) # test add a file that auto-aggregates dbf_file = open(self.test_file_path.format("watersheds.dbf"), 'r') shp_file = open(self.test_file_path.format("watersheds.shp"), 'r') shx_file = open(self.test_file_path.format("watersheds.shx"), 'r') resource_file_add_process(resource=self.composite_resource, folder="parentfolder", files=(dbf_file, shp_file, shx_file), user=self.user, full_paths={dbf_file: "folder/watersheds.dbf", shp_file: "folder/watersheds.shp", shx_file: "folder/watersheds.shx"}) # because the files are in different folders, auto aggreate won't work self.assertEqual(1, GeoFeatureLogicalFile.objects.count()) storage_paths = ["parentfolder/folder/watersheds.dbf", "parentfolder/folder/watersheds.shp", "parentfolder/folder/watersheds.shx"] for res_file in self.composite_resource.files.all(): index = -1 for i, name in enumerate(storage_paths): if name == res_file.storage_path: index = i break del storage_paths[index] self.assertEquals(0, len(storage_paths))
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual(self.resRaster.metadata.title.value, 'My Test Raster Resource') # there shouldn't any abstract element self.assertEqual(self.resRaster.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should not be any extended metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) # adding a valid tiff file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) super(TestRasterMetaData, self).raster_metadata_extraction()
def test_set_file_type_success_2(self): # here we will set the tif file (the file being not in root dir)to GeoRaster file type # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "r") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user, auto_aggregate=False) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) create_folder(self.resource.short_id, 'data/contents/sub_test_dir') # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder(self.user, self.resource.short_id, 'data/contents/' + self.raster_file_name, 'data/contents/sub_test_dir/' + self.raster_file_name) res_file = self.resource.files.all().first() self.assertEqual(res_file.short_path, "sub_test_dir/" + self.raster_file_name) # test the set file type endpoint url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" set_file_type_url = url_template.format(res_id=self.resource.short_id, file_path=res_file.short_path, file_type="GeoRaster") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_metadata_extraction_DIS_file(self): # Extract Metadata from DIS File files = [UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name)] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadeta=False) # ---Tests for Grid Dimensions # Number of Layers self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4)) # Type of Rows self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular') # Number of Rows self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20)) # Type of Columns self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular') # Number of Columns self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15)) # ---Tests for Stress Period # Stress Period Type self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient') # ---Tests for Study Area # Total Length self.assertEqual(self.res.metadata.study_area.totalLength, str(20*2000.0)) # double # Total Width self.assertEqual(self.res.metadata.study_area.totalWidth, str(15*2000.0)) # double
def test_set_file_type_failure_2(self): # here we will set the tif file to GeoRaster file type with an invalid file path # which should fail # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "r") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user, auto_aggregate=False) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) # test the set file type endpoint using a wrong file path url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" file_path = os.path.join("no-such-folder", self.raster_file_name) set_file_type_url = url_template.format(res_id=self.resource.short_id, file_path=file_path, file_type="GeoRaster") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_file_extensions(self): # set url launching pattern for aggregations metadata = [{ 'requesturlbasefile': { 'value': 'https://www.google.com?agg_path=${HS_FILE_PATH}' } }] self.resWebApp.metadata.update(metadata, self.user) self.assertEqual(RequestUrlBaseFile.objects.all().count(), 1) # set web app to launch for geo raster metadata = [{'supportedfileextensions': {'value': '.tif'}}] self.resWebApp.metadata.update(metadata, self.user) self.assertEqual(SupportedFileExtensions.objects.all().count(), 1) # add a geo raster aggregation to the resource open_file = open(self.test_file_path.format("small_logan.tif"), 'r') resource_file_add_process(resource=self.resComposite, files=(open_file, ), user=self.user) # setup the web app to be launched by the resource url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user self.assertEqual(self.resWebApp.extra_metadata, {}) self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'} self.resWebApp.save() self.assertNotEqual(self.resWebApp.extra_metadata, {}) self.assertEqual(self.resWebApp.extra_metadata[tool_app_key], 'test-app-value') self.assertEqual(self.resComposite.extra_metadata, {}) self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'} self.resComposite.save() self.assertNotEqual(self.resComposite.extra_metadata, {}) self.assertEqual(self.resComposite.extra_metadata[tool_app_key], 'test-app-value') url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user # get the web tools relevant_tools = resource_level_tool_urls(self.resComposite, request) self.assertIsNotNone( relevant_tools, msg='relevant_tools should not be None with appkey ' 'matching') tc = relevant_tools['resource_level_app_counter'] self.assertEqual(0, tc) tl = relevant_tools['tool_list'][0] self.assertEqual(self.resWebApp.short_id, tl['res_id']) self.assertFalse(tl['approved']) self.assertTrue(tl['openwithlist']) self.assertEqual('', tl['agg_types']) self.assertEqual('.tif', tl['file_extensions'])
def test_public_or_discoverable(self): self.assertFalse(self.resRaster.has_required_content_files()) self.assertFalse(self.resRaster.metadata.has_all_required_elements()) self.assertFalse(self.resRaster.can_be_public_or_discoverable) # adding a valid raster file files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # adding required metadata self.resRaster.metadata.create_element('description', abstract='example abstract') self.resRaster.metadata.create_element('subject', value='logan') self.assertTrue(self.resRaster.has_required_content_files()) self.assertTrue(self.resRaster.metadata.has_all_required_elements()) self.assertTrue(self.resRaster.can_be_public_or_discoverable) self.resRaster.delete()
def test_set_file_type_failure_1(self): # here we will set the tif file to NetCDF file type which should fail # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "r") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file, ), user=self.user) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) # test the set file type endpoint using a wrong file type (NetCDF) url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" set_file_type_url = url_template.format( res_id=self.resource.short_id, file_path=self.raster_file_name, file_type="NetCDF") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_agg_types(self): # set url launching pattern for aggregations metadata = [{'requesturlbaseaggregation': { 'value': 'https://www.google.com?agg_path=${HS_AGG_PATH}'}}] self.resWebApp.metadata.update(metadata, self.user) self.assertEqual(RequestUrlBaseAggregation.objects.all().count(), 1) # set web app to launch for geo raster self.resWebApp.metadata.update(metadata, self.user) resource.create_metadata_element(self.resWebApp.short_id, 'SupportedAggTypes', supported_agg_types=['GeoRasterLogicalFile']) self.assertEqual(1, SupportedAggTypes.objects.all().count()) # add a geo raster aggregation to the resource open_file = open(self.test_file_path.format("small_logan.tif"), 'rb') resource_file_add_process(resource=self.resComposite, files=(open_file,), user=self.user) # setup the web app to be launched by the resource url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user self.assertEqual(self.resWebApp.extra_metadata, {}) self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'} self.resWebApp.save() self.assertNotEqual(self.resWebApp.extra_metadata, {}) self.assertEqual(self.resWebApp.extra_metadata[tool_app_key], 'test-app-value') self.assertEqual(self.resComposite.extra_metadata, {}) self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'} self.resComposite.save() self.assertNotEqual(self.resComposite.extra_metadata, {}) self.assertEqual(self.resComposite.extra_metadata[tool_app_key], 'test-app-value') url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user # get the web tools and ensure agg_types is there relevant_tools = resource_level_tool_urls(self.resComposite, request) self.assertIsNotNone(relevant_tools, msg='relevant_tools should not be None with appkey ' 'matching') tc = relevant_tools['resource_level_app_counter'] self.assertEqual(0, tc) tl = relevant_tools['tool_list'][0] self.assertEqual(self.resWebApp.short_id, tl['res_id']) self.assertEqual('GeoRasterLogicalFile', tl['agg_types']) self.assertEqual('', tl['file_extensions']) # Remove appkey to turn off openwithlist for this resource self.resWebApp.extra_metadata = {} self.resWebApp.save() relevant_tools = resource_level_tool_urls(self.resComposite, request) self.assertIsNone(relevant_tools, msg='relevant_tools should have no approved resources')
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self): files = [UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) self._test_metadata_extraction_wkt_crs()
def test_metadata_on_content_file_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelProgram.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelProgram.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_agg_types(self): # set url launching pattern for aggregations metadata = [{'requesturlbaseaggregation': { 'value': 'https://www.google.com?agg_path=${HS_AGG_PATH}'}}] self.resWebApp.metadata.update(metadata, self.user) self.assertEqual(RequestUrlBaseAggregation.objects.all().count(), 1) # set web app to launch for geo raster self.resWebApp.metadata.update(metadata, self.user) resource.create_metadata_element(self.resWebApp.short_id, 'SupportedAggTypes', supported_agg_types=['GeoRasterLogicalFile']) self.assertEqual(1, SupportedAggTypes.objects.all().count()) # add a geo raster aggregation to the resource open_file = open(self.test_file_path.format("small_logan.tif"), 'r') resource_file_add_process(resource=self.resComposite, files=(open_file,), user=self.user) # setup the web app to be launched by the resource url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user self.assertEqual(self.resWebApp.extra_metadata, {}) self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'} self.resWebApp.save() self.assertNotEqual(self.resWebApp.extra_metadata, {}) self.assertEqual(self.resWebApp.extra_metadata[tool_app_key], 'test-app-value') self.assertEqual(self.resComposite.extra_metadata, {}) self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'} self.resComposite.save() self.assertNotEqual(self.resComposite.extra_metadata, {}) self.assertEqual(self.resComposite.extra_metadata[tool_app_key], 'test-app-value') url = '/resource/' + self.resComposite.short_id + '/' request = self.factory.get(url) request.user = self.user # get the web tools and ensure agg_types is there relevant_tools = resource_level_tool_urls(self.resComposite, request) self.assertIsNotNone(relevant_tools, msg='relevant_tools should not be None with appkey ' 'matching') tc = relevant_tools['resource_level_app_counter'] self.assertEqual(0, tc) tl = relevant_tools['tool_list'][0] self.assertEqual(self.resWebApp.short_id, tl['res_id']) self.assertFalse(tl['approved']) self.assertTrue(tl['openwithlist']) self.assertEqual('GeoRasterLogicalFile', tl['agg_types']) self.assertEqual('', tl['file_extensions'])
def test_metadata_extraction(self): # test allowed file type is '.*' self.assertEqual(self.res.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEqual(self.res.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name) ] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadata=False) # here are the things that the modflow modelinstance should include from the .nam file # GroundWaterFlow # flowPackage: 'LPF', 'UZF'(?) # # GeneralElements # modelSolver: 'SIP' # outputcontrolpackage: 'OC', 'GAGE' # # BoundaryCondition # head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?) # specified_flux_boundary_packages: 'WEL' self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP') self.assertTrue( self.res.metadata.ground_water_flow.unsaturatedZonePackage) self.assertIn( 'GHB', self.res.metadata.boundary_condition. get_head_dependent_flux_boundary_packages()) self.assertIn( 'SFR', self.res.metadata.boundary_condition. get_head_dependent_flux_boundary_packages()) self.assertIn( 'WEL', self.res.metadata.boundary_condition. get_specified_flux_boundary_packages()) self.assertIn( 'OC', self.res.metadata.general_elements.get_output_control_package()) self.assertIn( 'GAGE', self.res.metadata.general_elements.get_output_control_package())
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # there should be 2 content files self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should not be any resource specific metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) self.resRaster.delete()
def test_allowed_file_types(self): # test allowed file type is '.nc' self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types()) self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1) # there should not be any content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add bad .nc file should raise file validation error files = [ UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add valid .nc file should pass the file check files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with ncdump file created by system self.assertEqual(self.resNetcdf.files.all().count(), 2) # file pre add process should raise validation error if we try to add a 2nd file # when the resource has already 2 content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False)
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # there should be 1 content files self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 1 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file that we added above hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 2 creator element self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual( self.resNetcdf.metadata.title.value, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010' ) # there shouldn't any abstract element self.assertEqual(self.resNetcdf.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0) # there shouldn't any source element self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0) # there shouldn't any relation element self.assertEqual( self.resNetcdf.metadata.relations.all().filter( type='cites').count(), 0) # there should be 1 creator self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there shouldn't any extended metadata self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_auto_aggregate_file_add_tif(self): """test that auto-aggregate works on tif file add""" self.create_composite_resource() self.assertEqual(0, GeoRasterLogicalFile.objects.count()) # test add a file that auto-aggregates open_file = open(self.test_file_path.format("small_logan.tif"), 'r') resource_file_add_process(resource=self.composite_resource, files=(open_file,), user=self.user) self.assertEqual(1, GeoRasterLogicalFile.objects.count())
def test_get_xml(self): # add a valid raster file to generate metadata files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # test if xml from get_xml() is well formed ET.fromstring(self.resRaster.metadata.get_xml()) self.resRaster.delete()
def test_auto_aggregate_file_add_nc(self): """test that auto-aggregate works on nc file add""" self.create_composite_resource() self.assertEqual(0, NetCDFLogicalFile.objects.count()) # test add a file that auto-aggregates open_file = open(self.test_file_path.format("netcdf_valid.nc"), 'r') resource_file_add_process(resource=self.composite_resource, files=(open_file,), user=self.user) # because of auto aggregation, there should be 2 files self.assertEqual(1, NetCDFLogicalFile.objects.count())
def test_auto_aggregate_file_add_refts(self): """test that auto-aggregate works on refts file add""" self.create_composite_resource() self.assertEqual(0, RefTimeseriesLogicalFile.objects.count()) # test add a file that auto-aggregates open_file = open(self.test_file_path.format("multi_sites_formatted_version1.0.refts.json"), 'r') resource_file_add_process(resource=self.composite_resource, files=(open_file,), user=self.user) self.assertEqual(1, RefTimeseriesLogicalFile.objects.count())
def test_auto_aggregate_file_add_sqlite(self): """test that auto-aggregate works on sqlite file add""" self.create_composite_resource() self.assertEqual(0, TimeSeriesLogicalFile.objects.count()) # test add a file that auto-aggregates open_file = open(self.test_file_path.format("ODM2.sqlite"), 'r') resource_file_add_process(resource=self.composite_resource, files=(open_file,), user=self.user) self.assertEqual(1, TimeSeriesLogicalFile.objects.count())
def test_allowed_file_types(self): # test allowed file type is '.*' self.assertEquals( self.resModelInstance.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelInstance.metadata.model_output, None) self.assertEquals(self.resModelInstance.metadata.executed_by, None) # Upload any other file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=True) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=True) # there should two content files self.assertEquals(self.resModelInstance.files.all().count(), 2) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelInstance.metadata.model_output, None) self.assertEquals(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element( 'ExecutedBy', model_name=self.resModelProgram.short_id) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelInstance.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelInstance.metadata.description, None) # there should be one creator element self.assertEquals( self.resModelInstance.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelInstance.metadata.model_output, None) self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element('MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1) # the short path should just consist of the file name. self.assertEquals(self.resModelProgram.files.all()[0].short_path, self.file_name) # delete content file that we added above; note that file name is a short_path hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelProgram.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelProgram.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self): files = [ UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) self._test_metadata_extraction_wkt_crs()
def test_auto_aggregate_file_add_geo_feature(self): """test that auto-aggregate works on geo feature file add""" self.create_composite_resource() self.assertEqual(0, GeoFeatureLogicalFile.objects.count()) # test add a file that auto-aggregates dbf_file = open(self.test_file_path.format("watersheds.dbf"), 'r') shp_file = open(self.test_file_path.format("watersheds.shp"), 'r') shx_file = open(self.test_file_path.format("watersheds.shx"), 'r') resource_file_add_process(resource=self.composite_resource, files=(dbf_file, shp_file, shx_file), user=self.user) self.assertEqual(1, GeoFeatureLogicalFile.objects.count())
def test_metadata_extraction(self): # test allowed file type is '.*' self.assertEqual(self.res.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEqual(self.res.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name)] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadata=False) # here are the things that the modflow modelinstance should include from the .nam file # GroundWaterFlow # flowPackage: 'LPF', 'UZF'(?) # # GeneralElements # modelSolver: 'SIP' # outputcontrolpackage: 'OC', 'GAGE' # # BoundaryCondition # head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?) # specified_flux_boundary_packages: 'WEL' self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP') self.assertTrue(self.res.metadata.ground_water_flow.unsaturatedZonePackage) self.assertIn( 'GHB', self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages() ) self.assertIn( 'SFR', self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages() ) self.assertIn( 'WEL', self.res.metadata.boundary_condition.get_specified_flux_boundary_packages() ) self.assertIn( 'OC', self.res.metadata.general_elements.get_output_control_package() ) self.assertIn( 'GAGE', self.res.metadata.general_elements.get_output_control_package() )
def test_public_or_discoverable(self): self.assertFalse(self.resNetcdf.has_required_content_files()) self.assertFalse(self.resNetcdf.metadata.has_all_required_elements()) self.assertFalse(self.resNetcdf.can_be_public_or_discoverable) # adding a valid netcdf file files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) self.assertTrue(self.resNetcdf.has_required_content_files()) self.assertTrue(self.resNetcdf.metadata.has_all_required_elements()) self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion # adding a valid netcdf file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # there should be 1 content files self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 1 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file that we added above hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEquals(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEquals(self.resNetcdf.metadata.description, None) # there should be 2 creator element self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEquals(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_DIS_file(self): # Extract Metadata from DIS File files = [ UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name) ] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadeta=False) # ---Tests for Grid Dimensions # Number of Layers self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4)) # Type of Rows self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular') # Number of Rows self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20)) # Type of Columns self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular') # Number of Columns self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15)) # ---Tests for Stress Period # Stress Period Type self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient') # ---Tests for Study Area # Total Length self.assertEqual(self.res.metadata.study_area.totalLength, str(20 * 2000.0)) # double # Total Width self.assertEqual(self.res.metadata.study_area.totalWidth, str(15 * 2000.0)) # double
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual( self.resNetcdf.metadata.title.value, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010') # there shouldn't any abstract element self.assertEqual(self.resNetcdf.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0) # there shouldn't any source element self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0) # there shouldn't any relation element self.assertEqual(self.resNetcdf.metadata.relations.all().filter(type='cites').count(), 0) # there should be 1 creator self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there shouldn't any extended metadata self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # adding a valid netcdf file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_file_add_to_composite_resource(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(CompositeResourceTest, self).is_federated_irods_available(): return # test that when we add file to an existing composite resource, the added file # automatically set to genericlogicalfile type self.assertEqual(BaseResource.objects.count(), 0) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test Composite Resource Federated') # there should not be any GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 0) # add a file to the resource fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_file_name) res_upload_files = [] resource_file_add_process(resource=self.composite_resource, files=res_upload_files, fed_res_file_names=[fed_test_file_full_path], user=self.user) # there should be one resource at this point self.assertEqual(BaseResource.objects.count(), 1) self.assertEqual(self.composite_resource.resource_type, "CompositeResource") self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # there should be 1 GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 1)
def test_get_xml(self): # add a valid raster file to generate metadata files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # test if xml from get_xml() is well formed ET.fromstring(self.resRaster.metadata.get_xml()) self.resRaster.delete()
def _test_csv_file_add(self, new_folder=None): self.create_composite_resource() # no timeseries aggregation prior to adding the csv file to a folder self.assertEqual(0, TimeSeriesLogicalFile.objects.count()) if new_folder is not None: # create a folder hydroshare.ResourceFile.create_folder(self.composite_resource, folder=new_folder) # upload the csv file to the folder (new-folder) csv_file_name = "ODM2_One_Site_One_Series_Test.csv" csv_file = open(self.test_file_path.format(csv_file_name), 'r') files = [UploadedFile(file=csv_file, name=csv_file_name)] resource_file_add_process(resource=self.composite_resource, files=files, user=self.user, folder=new_folder) # check that the resource has one file self.assertEqual(self.composite_resource.files.all().count(), 1) csv_res_file = self.composite_resource.files.first() self.assertEqual(csv_res_file.file_folder, new_folder) # no timeseries aggregation after adding the scv file to a folder self.assertEqual(0, TimeSeriesLogicalFile.objects.count())
def test_public_or_discoverable(self): self.assertFalse(self.resRaster.has_required_content_files()) self.assertFalse(self.resRaster.metadata.has_all_required_elements()) self.assertFalse(self.resRaster.can_be_public_or_discoverable) # adding a valid raster file files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # adding required metadata self.resRaster.metadata.create_element('description', abstract='example abstract') self.resRaster.metadata.create_element('subject', value='logan') self.assertTrue(self.resRaster.has_required_content_files()) self.assertTrue(self.resRaster.metadata.has_all_required_elements()) self.assertTrue(self.resRaster.can_be_public_or_discoverable) self.resRaster.delete()
def test_metadata_on_content_file_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1) # created file should have a short path equal to the declared name self.assertEquals(self.resModelInstance.files.all()[0].short_path, self.file_name) # delete content file that we added above hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelInstance.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelInstance.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelInstance.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelInstance.metadata.model_output, None) self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_public_or_discoverable(self): self.assertFalse(self.resNetcdf.has_required_content_files()) self.assertFalse(self.resNetcdf.metadata.has_all_required_elements()) self.assertFalse(self.resNetcdf.can_be_public_or_discoverable) # adding a valid netcdf file files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) self.assertTrue(self.resNetcdf.has_required_content_files()) self.assertTrue(self.resNetcdf.metadata.has_all_required_elements()) self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_allowed_file_types(self): # test allowed file type is '.nc' self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types()) self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1) # there should not be any content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add bad .nc file should raise file validation error files = [UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add valid .nc file should pass the file check files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with ncdump file created by system self.assertEqual(self.resNetcdf.files.all().count(), 2) # file pre add process should raise validation error if we try to add a 2nd file # when the resource has already 2 content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False)
def test_public_or_discoverable(self): self.assertFalse(self.resModelProgram.has_required_content_files()) self.assertFalse(self.resModelProgram.metadata.has_all_required_elements()) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Subject', value="test subject") self.assertTrue(self.resModelProgram.has_required_content_files()) self.assertTrue(self.resModelProgram.metadata.has_all_required_elements()) self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_set_file_type_success_2(self): # here we will set the tif file (the file being not in root dir)to GeoRaster file type # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "r") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file, ), user=self.user) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) create_folder(self.resource.short_id, 'data/contents/sub_test_dir') # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder( self.user, self.resource.short_id, 'data/contents/' + self.raster_file_name, 'data/contents/sub_test_dir/' + self.raster_file_name) res_file = self.resource.files.all().first() self.assertEqual(res_file.short_path, "sub_test_dir/" + self.raster_file_name) # test the set file type endpoint url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" set_file_type_url = url_template.format(res_id=self.resource.short_id, file_path=res_file.short_path, file_type="GeoRaster") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual(self.resRaster.metadata.title.value, 'My Test Raster Resource') # there shouldn't any abstract element self.assertEqual(self.resRaster.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should not be any extended metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) # adding a valid tiff file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) super(TestRasterMetaData, self).raster_metadata_extraction()
def test_bulk_metadata_update(self): # here we are testing the update() method of the RasterMetaData class # update of resource specific metadata should fail when the resource does not have content # files self.assertEqual(self.resRaster.files.all().count(), 0) self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 0) band_data = {'original_band_name': 'bandinfo', 'name': 'Band_1', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } metadata = [] metadata.append({'bandinformation': band_data}) with self.assertRaises(ValidationError): self.resRaster.metadata.update(metadata, self.user) self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 0) del metadata[:] # adding a valid tiff file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # testing extended metadata element: band information self.assertEqual(self.resRaster.metadata.bandInformations.count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.noDataValue, '-3.40282346639e+38') self.assertEqual(band_info.maximumValue, '3031.44311523') self.assertEqual(band_info.minimumValue, '1358.33459473') self.assertEqual(band_info.name, 'Band_1') # updating of bandinformation using a name that does not exist (band-name) should fail band_data = {'original_band_name': 'band-name', 'name': 'Band_1', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } metadata.append({'bandinformation': band_data}) with self.assertRaises(ValidationError): self.resRaster.metadata.update(metadata, self.user) self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1) # updating of bandinformation using a valid band lookup name (Band_1) should be successful band_data = {'original_band_name': 'Band_1', 'name': 'Band_2', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } del metadata[:] metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata, self.user) self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_2') self.assertEqual(band_info.variableName, 'digital elevation') self.assertEqual(band_info.variableUnit, 'meter') self.assertEqual(band_info.method, 'this is method') self.assertEqual(band_info.comment, 'this is comment') self.assertEqual(band_info.maximumValue, '1000') self.assertEqual(band_info.minimumValue, '0') self.assertEqual(band_info.noDataValue, '-9999') # test updating only one attribute of bandinformation band_data = {'original_band_name': 'Band_2', 'name': 'Band_1' } del metadata[:] metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata, self.user) self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_1') # test updating both core and resource specific metadata # there should be 1 creator self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) del metadata[:] metadata.append({'creator': {'name': 'creator one'}}) metadata.append({'creator': {'name': 'creator two'}}) metadata.append({'contributor': {'name': 'contributor one'}}) metadata.append({'contributor': {'name': 'contributor two'}}) band_data = {'original_band_name': 'Band_1', 'name': 'Band_3' } metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata, self.user) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_3') # there should be 2 creators self.assertEqual(self.resRaster.metadata.creators.all().count(), 2) # there should be 2 contributor self.assertEqual(self.resRaster.metadata.contributors.all().count(), 2) self.resRaster.delete()
def test_metadata_delete_on_resource_delete(self): # adding a valid raster tif file should generate some core metadata and all extended # metadata files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # before resource delete # resource core metadata raster_metadata_obj = self.resRaster.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Coverage metadata objects self.assertTrue(Coverage.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=raster_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertTrue(OriginalCoverage.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be CellInformation metadata objects self.assertTrue(CellInformation.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be BandInformation metadata objects self.assertTrue(BandInformation.objects.filter(object_id=raster_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resRaster.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=raster_metadata_obj.id).exists()) # resource specific metadata # there should be no original coverage metadata objects self.assertFalse(OriginalCoverage.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be CellInformation metadata objects self.assertFalse(CellInformation.objects.filter(object_id=raster_metadata_obj.id).exists()) # there should be bandInformation metadata objects self.assertFalse(BandInformation.objects.filter(object_id=raster_metadata_obj.id).exists())
def test_allowed_file_types(self): # test allowed file types are: '.tiff', '.tif,', '.vrt' and '.zip' self.assertIn('.tiff', RasterResource.get_supported_upload_file_types()) self.assertIn('.tif', RasterResource.get_supported_upload_file_types()) self.assertIn('.vrt', RasterResource.get_supported_upload_file_types()) self.assertIn('.zip', RasterResource.get_supported_upload_file_types()) self.assertEqual(len(RasterResource.get_supported_upload_file_types()), 4) # there should not be any content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add bad .tif file should raise file validation error files = [UploadedFile(file=self.raster_bad_tif_file_obj, name=self.raster_bad_tif_file_name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add good .tif file should pass the file check files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with .vrt file created by system self.assertEqual(self.resRaster.files.all().count(), 2) file_names = [os.path.basename(f.resource_file.name) for f in self.resRaster.files.all()] self.assertIn('raster_tif_valid.vrt', file_names) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add good .zip file should pass the file check files = [UploadedFile(file=self.raster_zip_file_obj, name=self.raster_zip_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 10 content file: self.assertEqual(self.resRaster.files.all().count(), 10) # file pre add process should raise validation error if we try to add a 2nd file when # the resource has already content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False)
def test_metadata_delete_on_resource_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') self.resModelProgram.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelProgram.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Program metadata objects self.assertTrue( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelProgram.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestNetcdfMetaData, self).is_federated_irods_available(): return # test metadata extraction with resource creation with nc file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title= 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June ' '2010', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content files at this point self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file now hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 1 creator element (based on the extracted metadata) self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # test metadata extraction with a valid nc file being added coming from user zone space res_add_files = [] utils.resource_file_add_pre_process( resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_in_user_zone(self): super(TestRasterMetaData, self).assert_federated_irods_available() # test metadata extraction with resource creation with tif file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_tif_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='RasterResource', resource_title='My Test Raster Resource', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) # raster file validation and metadata extraction in post resource creation signal handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file now hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should be no extended metadata elements # at this point self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) # test metadata extraction with a valid tif file being added coming from user zone space res_add_files = [] # now necessary in order to test add_process utils.resource_file_add_pre_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata deletion when deleting a resource in user zone space self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0) # test adding file from user zone to existing empty resource in hydroshare zone # even there is no file uploaded to resource initially, there are default extended # automatically metadata created self.resRaster = hydroshare.create_resource( resource_type='RasterResource', owner=self.user, title='My Test Raster Resource' ) # test metadata extraction with a valid tif file being added coming from user zone space # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=[], user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element('MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') self.resModelProgram.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelProgram.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Program metadata objects self.assertTrue(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelProgram.short_id) self.assertEquals(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())