def test_allowed_file_types(self): # test allowed file type is '.*' self.assertEquals(self.resModelProgram.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # Upload any file type should pass both the file pre add check and post add check files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelProgram.metadata.program, None) # Uploading any other file type should pass both the file pre add check post add check files = [UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=True) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=True) # there should two content files self.assertEquals(self.resModelProgram.files.all().count(), 2) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelProgram.metadata.program, None)
def test_metadata_extraction_DIS_file(self): # Extract Metadata from DIS File files = [UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name)] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadeta=False) # ---Tests for Grid Dimensions # Number of Layers self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4)) # Type of Rows self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular') # Number of Rows self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20)) # Type of Columns self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular') # Number of Columns self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15)) # ---Tests for Stress Period # Stress Period Type self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient') # ---Tests for Study Area # Total Length self.assertEqual(self.res.metadata.study_area.totalLength, str(20*2000.0)) # double # Total Width self.assertEqual(self.res.metadata.study_area.totalWidth, str(15*2000.0)) # double
def test_public_or_discoverable(self): self.assertFalse(self.resRaster.has_required_content_files()) self.assertFalse(self.resRaster.metadata.has_all_required_elements()) self.assertFalse(self.resRaster.can_be_public_or_discoverable) # adding a valid raster file files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # adding required metadata self.resRaster.metadata.create_element('description', abstract='example abstract') self.resRaster.metadata.create_element('subject', value='logan') self.assertTrue(self.resRaster.has_required_content_files()) self.assertTrue(self.resRaster.metadata.has_all_required_elements()) self.assertTrue(self.resRaster.can_be_public_or_discoverable) self.resRaster.delete()
def test_public_or_discoverable(self): self.assertFalse(self.resModelProgram.has_required_content_files()) self.assertFalse( self.resModelProgram.metadata.has_all_required_elements()) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Subject', value="test subject") self.assertTrue(self.resModelProgram.has_required_content_files()) self.assertTrue( self.resModelProgram.metadata.has_all_required_elements()) self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def add_file_to_resource(request, shortkey, *args, **kwargs): resource, _, _ = authorize(request, shortkey, edit=True, full=True, superuser=True) res_files = request.FILES.getlist('files') extract_metadata = request.REQUEST.get('extract-metadata', 'No') extract_metadata = True if extract_metadata.lower() == 'yes' else False try: utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata) except hydroshare.utils.ResourceFileSizeException as ex: request.session['file_size_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: request.session['file_validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) try: hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: request.session['file_validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self): files = [UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) self._test_metadata_extraction_wkt_crs()
def test_metadata_on_content_file_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelProgram.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelProgram.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_extraction(self): # test allowed file type is '.*' self.assertEqual(self.res.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEqual(self.res.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name) ] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadata=False) # here are the things that the modflow modelinstance should include from the .nam file # GroundWaterFlow # flowPackage: 'LPF', 'UZF'(?) # # GeneralElements # modelSolver: 'SIP' # outputcontrolpackage: 'OC', 'GAGE' # # BoundaryCondition # head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?) # specified_flux_boundary_packages: 'WEL' self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP') self.assertTrue( self.res.metadata.ground_water_flow.unsaturatedZonePackage) self.assertIn( 'GHB', self.res.metadata.boundary_condition. get_head_dependent_flux_boundary_packages()) self.assertIn( 'SFR', self.res.metadata.boundary_condition. get_head_dependent_flux_boundary_packages()) self.assertIn( 'WEL', self.res.metadata.boundary_condition. get_specified_flux_boundary_packages()) self.assertIn( 'OC', self.res.metadata.general_elements.get_output_control_package()) self.assertIn( 'GAGE', self.res.metadata.general_elements.get_output_control_package())
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # there should be 2 content files self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should not be any resource specific metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) self.resRaster.delete()
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # there should be 1 content files self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 1 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file that we added above hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 2 creator element self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual( self.resNetcdf.metadata.title.value, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010' ) # there shouldn't any abstract element self.assertEqual(self.resNetcdf.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0) # there shouldn't any source element self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0) # there shouldn't any relation element self.assertEqual( self.resNetcdf.metadata.relations.all().filter( type='cites').count(), 0) # there should be 1 creator self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there shouldn't any extended metadata self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_get_xml(self): # add a valid raster file to generate metadata files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # test if xml from get_xml() is well formed ET.fromstring(self.resRaster.metadata.get_xml()) self.resRaster.delete()
def test_allowed_file_types(self): # test allowed file type is '.*' self.assertEquals( self.resModelInstance.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelInstance.metadata.model_output, None) self.assertEquals(self.resModelInstance.metadata.executed_by, None) # Upload any other file type should pass both the file pre add check post add check files = [ UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=True) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=True) # there should two content files self.assertEquals(self.resModelInstance.files.all().count(), 2) # check that there are no extended metadata elements at this point self.assertEquals(self.resModelInstance.metadata.model_output, None) self.assertEquals(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element('MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1) # the short path should just consist of the file name. self.assertEquals(self.resModelProgram.files.all()[0].short_path, self.file_name) # delete content file that we added above; note that file name is a short_path hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelProgram.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelProgram.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_on_content_file_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element( 'ExecutedBy', model_name=self.resModelProgram.short_id) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelInstance.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelInstance.metadata.description, None) # there should be one creator element self.assertEquals( self.resModelInstance.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelInstance.metadata.model_output, None) self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self): files = [ UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) self._test_metadata_extraction_wkt_crs()
def test_metadata_extraction(self): # test allowed file type is '.*' self.assertEqual(self.res.get_supported_upload_file_types(), '.*') # there should not be any content file self.assertEqual(self.res.files.all().count(), 0) # Upload any file type should pass both the file pre add check post add check files = [UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name)] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadata=False) # here are the things that the modflow modelinstance should include from the .nam file # GroundWaterFlow # flowPackage: 'LPF', 'UZF'(?) # # GeneralElements # modelSolver: 'SIP' # outputcontrolpackage: 'OC', 'GAGE' # # BoundaryCondition # head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?) # specified_flux_boundary_packages: 'WEL' self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP') self.assertTrue(self.res.metadata.ground_water_flow.unsaturatedZonePackage) self.assertIn( 'GHB', self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages() ) self.assertIn( 'SFR', self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages() ) self.assertIn( 'WEL', self.res.metadata.boundary_condition.get_specified_flux_boundary_packages() ) self.assertIn( 'OC', self.res.metadata.general_elements.get_output_control_package() ) self.assertIn( 'GAGE', self.res.metadata.general_elements.get_output_control_package() )
def test_public_or_discoverable(self): self.assertFalse(self.resNetcdf.has_required_content_files()) self.assertFalse(self.resNetcdf.metadata.has_all_required_elements()) self.assertFalse(self.resNetcdf.can_be_public_or_discoverable) # adding a valid netcdf file files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) self.assertTrue(self.resNetcdf.has_required_content_files()) self.assertTrue(self.resNetcdf.metadata.has_all_required_elements()) self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion # adding a valid netcdf file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # there should be 1 content files self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 1 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file that we added above hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEquals(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEquals(self.resNetcdf.metadata.description, None) # there should be 2 creator element self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEquals(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_add_files_over_quota(self): # create a resource res = create_resource( resource_type='GenericResource', owner=self.user, title='Test Resource', metadata=[], ) if not QuotaMessage.objects.exists(): QuotaMessage.objects.create() qmsg = QuotaMessage.objects.first() qmsg.enforce_quota = True qmsg.save() uquota = self.user.quotas.first() # make user's quota over hard limit 125% uquota.used_value = uquota.allocated_value * 1.3 uquota.save() # add files should raise quota exception now that the quota holder is over hard limit # and quota enforce flag is set to True files = [self.myfile1, self.myfile2, self.myfile3] with self.assertRaises(QuotaException): resource_file_add_pre_process(resource=res, files=files, user=self.user, extract_metadata=False) qmsg.enforce_quota = False qmsg.save() # add files should not raise quota exception since enforce_quota flag is set to False try: resource_file_add_pre_process(resource=res, files=files, user=self.user, extract_metadata=False) except QuotaException as ex: self.fail( "add resource file action should not raise QuotaException for " "over quota cases if quota is not enforced - Quota Exception: " + ex.message) res.delete()
def test_metadata_extraction_DIS_file(self): # Extract Metadata from DIS File files = [ UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name) ] utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.res, files=files, user=self.user, extract_metadeta=False) # ---Tests for Grid Dimensions # Number of Layers self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4)) # Type of Rows self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular') # Number of Rows self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20)) # Type of Columns self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular') # Number of Columns self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15)) # ---Tests for Stress Period # Stress Period Type self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient') # ---Tests for Study Area # Total Length self.assertEqual(self.res.metadata.study_area.totalLength, str(20 * 2000.0)) # double # Total Width self.assertEqual(self.res.metadata.study_area.totalWidth, str(15 * 2000.0)) # double
def test_allowed_file_types(self): # test allowed file type is '.nc' self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types()) self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1) # there should not be any content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add bad .nc file should raise file validation error files = [ UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add valid .nc file should pass the file check files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with ncdump file created by system self.assertEqual(self.resNetcdf.files.all().count(), 2) # file pre add process should raise validation error if we try to add a 2nd file # when the resource has already 2 content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False)
def test_metadata_extraction_on_content_file_add(self): # test the core metadata at this point self.assertEqual( self.resNetcdf.metadata.title.value, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010') # there shouldn't any abstract element self.assertEqual(self.resNetcdf.metadata.description, None) # there shouldn't any coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there shouldn't any format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there shouldn't any subject element self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # there shouldn't any contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0) # there shouldn't any source element self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0) # there shouldn't any relation element self.assertEqual(self.resNetcdf.metadata.relations.all().filter(type='cites').count(), 0) # there should be 1 creator self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there shouldn't any extended metadata self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # adding a valid netcdf file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_get_xml(self): # add a valid raster file to generate metadata files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # test if xml from get_xml() is well formed ET.fromstring(self.resRaster.metadata.get_xml()) self.resRaster.delete()
def test_public_or_discoverable(self): self.assertFalse(self.resRaster.has_required_content_files()) self.assertFalse(self.resRaster.metadata.has_all_required_elements()) self.assertFalse(self.resRaster.can_be_public_or_discoverable) # adding a valid raster file files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # adding required metadata self.resRaster.metadata.create_element('description', abstract='example abstract') self.resRaster.metadata.create_element('subject', value='logan') self.assertTrue(self.resRaster.has_required_content_files()) self.assertTrue(self.resRaster.metadata.has_all_required_elements()) self.assertTrue(self.resRaster.can_be_public_or_discoverable) self.resRaster.delete()
def test_metadata_on_content_file_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id) # there should one content file self.assertEquals(self.resModelInstance.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1) # created file should have a short path equal to the declared name self.assertEquals(self.resModelInstance.files.all()[0].short_path, self.file_name) # delete content file that we added above hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelInstance.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelInstance.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelInstance.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelInstance.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelInstance.metadata.model_output, None) self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_public_or_discoverable(self): self.assertFalse(self.resNetcdf.has_required_content_files()) self.assertFalse(self.resNetcdf.metadata.has_all_required_elements()) self.assertFalse(self.resNetcdf.can_be_public_or_discoverable) # adding a valid netcdf file files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) self.assertTrue(self.resNetcdf.has_required_content_files()) self.assertTrue(self.resNetcdf.metadata.has_all_required_elements()) self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_public_or_discoverable(self): self.assertFalse(self.resModelProgram.has_required_content_files()) self.assertFalse(self.resModelProgram.metadata.has_all_required_elements()) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.assertFalse(self.resModelProgram.can_be_public_or_discoverable) self.resModelProgram.metadata.create_element('Subject', value="test subject") self.assertTrue(self.resModelProgram.has_required_content_files()) self.assertTrue(self.resModelProgram.metadata.has_all_required_elements()) self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_allowed_file_types(self): # test allowed file type is '.nc' self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types()) self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1) # there should not be any content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add bad .nc file should raise file validation error files = [UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # trying to add valid .nc file should pass the file check files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with ncdump file created by system self.assertEqual(self.resNetcdf.files.all().count(), 2) # file pre add process should raise validation error if we try to add a 2nd file # when the resource has already 2 content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False)
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestNetcdfMetaData, self).is_federated_irods_available(): return # test metadata extraction with resource creation with nc file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title= 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June ' '2010', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content files at this point self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file now hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 1 creator element (based on the extracted metadata) self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # test metadata extraction with a valid nc file being added coming from user zone space res_add_files = [] utils.resource_file_add_pre_process( resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self): # adding a valid netcdf file should generate some core metadata and all extended metadata files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # before resource delete # resource core metadata core_metadata_obj = self.resNetcdf.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Source metadata objects self.assertTrue(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Relation metadata objects self.assertTrue(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Coverage metadata objects self.assertTrue(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertTrue(OriginalCoverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertTrue(Variable.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertFalse(OriginalCoverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertFalse(Variable.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_delete_on_resource_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element('MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') self.resModelProgram.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelProgram.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Program metadata objects self.assertTrue(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelProgram.short_id) self.assertEquals(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_delete_on_resource_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id) self.resModelInstance.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelInstance.metadata self.assertEqual(CoreMetaData.objects.all().count(), 3) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Output metadata objects self.assertTrue(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be ExecutedBy metadata objects self.assertTrue(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelInstance.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 2) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no ExecutedBy metadata objects self.assertFalse(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
def test_allowed_file_types(self): # test allowed file types are: '.tiff', '.tif,', '.vrt' and '.zip' self.assertIn('.tiff', RasterResource.get_supported_upload_file_types()) self.assertIn('.tif', RasterResource.get_supported_upload_file_types()) self.assertIn('.vrt', RasterResource.get_supported_upload_file_types()) self.assertIn('.zip', RasterResource.get_supported_upload_file_types()) self.assertEqual(len(RasterResource.get_supported_upload_file_types()), 4) # there should not be any content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add bad .tif file should raise file validation error files = [UploadedFile(file=self.raster_bad_tif_file_obj, name=self.raster_bad_tif_file_name)] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add good .tif file should pass the file check files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with .vrt file created by system self.assertEqual(self.resRaster.files.all().count(), 2) file_names = [os.path.basename(f.resource_file.name) for f in self.resRaster.files.all()] self.assertIn('raster_tif_valid.vrt', file_names) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add good .zip file should pass the file check files = [UploadedFile(file=self.raster_zip_file_obj, name=self.raster_zip_file_name)] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 10 content file: self.assertEqual(self.resRaster.files.all().count(), 10) # file pre add process should raise validation error if we try to add a 2nd file when # the resource has already content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False)
def upload_add(request): # add irods file into an existing resource res_id = request.POST['res_id'] resource, _, _ = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) res_files = request.FILES.getlist('files') extract_metadata = request.POST.get('extract-metadata', 'No') extract_metadata = True if extract_metadata.lower() == 'yes' else False irods_fnames = request.POST.get('irods_file_names', '') irods_fnames_list = string.split(irods_fnames, ',') res_cls = resource.__class__ # TODO: read resource type from resource, not from input file valid, ext = check_upload_files(res_cls, irods_fnames_list) source_names = [] irods_federated = False if not valid: request.session['file_type_error'] = "Invalid file type: {ext}".format(ext=ext) return HttpResponseRedirect(request.META['HTTP_REFERER']) else: homepath = irods_fnames_list[0] # TODO: this should happen whether resource is federated or not irods_federated = utils.is_federated(homepath) if irods_federated: source_names = irods_fnames.split(',') else: user = request.POST.get('irods-username') password = request.POST.get("irods-password") port = request.POST.get("irods-port") host = request.POST.get("irods-host") zone = request.POST.get("irods-zone") try: upload_from_irods(username=user, password=password, host=host, port=port, zone=zone, irods_fnames=irods_fnames, res_files=res_files) except SessionException as ex: request.session['validation_error'] = ex.stderr return HttpResponseRedirect(request.META['HTTP_REFERER']) try: utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata, source_names=source_names, folder=None) except hydroshare.utils.ResourceFileSizeException as ex: request.session['file_size_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: request.session['validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) try: hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata, source_names=source_names, folder=None) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: if ex.message: request.session['validation_error'] = ex.message elif ex.stderr: request.session['validation_error'] = ex.stderr except SessionException as ex: request.session['validation_error'] = ex.stderr request.session['resource-mode'] = 'edit' return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_allowed_file_types(self): # test allowed file type is '.tif, .zip' self.assertIn('.tif', RasterResource.get_supported_upload_file_types()) self.assertIn('.zip', RasterResource.get_supported_upload_file_types()) self.assertEqual(len(RasterResource.get_supported_upload_file_types()), 2) # there should not be any content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add a text file to this resource should raise exception files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add bad .tif file should raise file validation error files = [ UploadedFile(file=self.raster_bad_tif_file_obj, name=self.raster_bad_tif_file_name) ] with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # trying to add good .tif file should pass the file check files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 2 content file: with .vrt file created by system self.assertEqual(self.resRaster.files.all().count(), 2) file_names = [ os.path.basename(f.resource_file.name) for f in self.resRaster.files.all() ] self.assertIn('raster_tif_valid.vrt', file_names) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # trying to add good .zip file should pass the file check files = [ UploadedFile(file=self.raster_zip_file_obj, name=self.raster_zip_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # there should be 10 content file: self.assertEqual(self.resRaster.files.all().count(), 10) # file pre add process should raise validation error if we try to add a 2nd file when # the resource has already content files with self.assertRaises(utils.ResourceFileValidationException): utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False)
def test_metadata_delete_on_resource_delete(self): # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # before resource delete # resource core metadata core_metadata_obj = self.resNetcdf.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Source metadata objects self.assertTrue( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Relation metadata objects self.assertTrue( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Coverage metadata objects self.assertTrue( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertTrue( OriginalCoverage.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertTrue( Variable.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertFalse( OriginalCoverage.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertFalse( Variable.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # there should be 2 content files self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual( self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual( self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should not be any resource specific metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) self.resRaster.delete()
def upload_add(request): # add irods file into an existing resource res_id = request.POST['res_id'] resource, _, _ = authorize(request, res_id, edit=True, full=True, superuser=True) res_files = request.FILES.getlist('files') extract_metadata = request.REQUEST.get('extract-metadata', 'No') extract_metadata = True if extract_metadata.lower() == 'yes' else False irods_fname = request.POST.get('irods_file_name', '') res_cls = resource.__class__ file_types = res_cls.get_supported_upload_file_types() valid = False if file_types == ".*": valid = True else: ext = os.path.splitext(irods_fname)[1] if ext == file_types: valid = True else: for index in range(len(file_types)): file_type_str = file_types[index].strip() if file_type_str == ".*" or ext == file_type_str: valid = True break if not valid: request.session['file_type_error'] = "Invalid file type: {ext}".format(ext=ext) return HttpResponseRedirect(request.META['HTTP_REFERER']) else: user = request.POST.get('irods-username') password = request.POST.get("irods-password") port = request.POST.get("irods-port") host = request.POST.get("irods-host") zone = request.POST.get("irods-zone") try: upload_from_irods(username=user, password=password, host=host, port=port, zone=zone, irods_fname=irods_fname, res_files=res_files) except Exception as ex: request.session['file_validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) try: utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata) except hydroshare.utils.ResourceFileSizeException as ex: request.session['file_size_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: request.session['file_validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER']) try: hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: request.session['file_validation_error'] = ex.message return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly super(TestNetcdfMetaData, self).assert_federated_irods_available() # test metadata extraction with resource creation with nc file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Snow water equivalent estimation at TWDEF site from Oct 2009 to June ' '2010', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content files at this point self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file now hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 1 creator element (based on the extracted metadata) self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # test metadata extraction with a valid nc file being added coming from user zone space res_add_files = [] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') self.resModelProgram.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelProgram.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Program metadata objects self.assertTrue( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelProgram.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestTimeSeriesMetaData, self).is_federated_irods_available(): return # test metadata extraction with resource creation with file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.odm2_sqlite_file_name) self.resTimeSeries = hydroshare.create_resource( resource_type='TimeSeriesResource', owner=self.user, title='My Test TimeSeries Resource', source_names=[fed_test_file_full_path], move=False) utils.resource_post_create_actions(resource=self.resTimeSeries, user=self.user, metadata=[]) super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction() # test that metadata is NOT deleted (except format element) on content file deletion # there should be one content file at this point self.assertEqual(self.resTimeSeries.files.all().count(), 1) # there should be one format element self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 1) hydroshare.delete_resource_file(self.resTimeSeries.short_id, self.odm2_sqlite_file_name, self.user) # there should be no content file self.assertEqual(self.resTimeSeries.files.all().count(), 0) # test the core metadata at this point self.assertNotEqual(self.resTimeSeries.metadata.title, None) # there should be an abstract element self.assertNotEqual(self.resTimeSeries.metadata.description, None) # there should be one creator element self.assertEqual(self.resTimeSeries.metadata.creators.all().count(), 1) # there should be one contributor element self.assertEqual( self.resTimeSeries.metadata.contributors.all().count(), 1) # there should be 2 coverage element - point type and period type self.assertEqual(self.resTimeSeries.metadata.coverages.all().count(), 2) self.assertEqual( self.resTimeSeries.metadata.coverages.all().filter( type='box').count(), 1) self.assertEqual( self.resTimeSeries.metadata.coverages.all().filter( type='period').count(), 1) # there should be no format element self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 0) # there should be one subject element self.assertEqual(self.resTimeSeries.metadata.subjects.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resTimeSeries.metadata.sites.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.variables.all().count(), 0) self.assertNotEqual(self.resTimeSeries.metadata.methods.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.processing_levels.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.time_series_results.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_variable_names.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_variable_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_speciations.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_site_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_elevation_datums.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_method_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_statuses.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_mediums.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_aggregation_statistics.all().count( ), 0) # test metadata extraction with a valid ODM2 sqlite file being added coming from user zone # space res_add_files = [] utils.resource_file_add_pre_process( resource=self.resTimeSeries, files=res_add_files, user=self.user, extract_metadata=False, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resTimeSeries, files=res_add_files, user=self.user, extract_metadata=True, source_names=[fed_test_file_full_path]) super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction() # test metadata deletion when deleting a resource in user zone space # all metadata should get deleted when the resource get deleted self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resTimeSeries.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self): files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)] utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelInstance, files=files, user=self.user, extract_metadata=False) self.resModelInstance.metadata.create_element('Description', abstract="test abstract") self.resModelInstance.metadata.create_element('Subject', value="test subject") self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True) self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id) self.resModelInstance.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelInstance.metadata self.assertEquals(CoreMetaData.objects.all().count(), 3) # there should be Creator metadata objects self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Output metadata objects self.assertTrue(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be ExecutedBy metadata objects self.assertTrue(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelInstance.short_id) self.assertEquals(CoreMetaData.objects.all().count(), 2) # there should be no Creator metadata objects self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no ExecutedBy metadata objects self.assertFalse(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self): super(TestRasterMetaData, self).assert_federated_irods_available() # test metadata extraction with resource creation with tif file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_tif_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='RasterResource', resource_title='My Test Raster Resource', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) # raster file validation and metadata extraction in post resource creation signal handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file now hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should be no extended metadata elements # at this point self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) # test metadata extraction with a valid tif file being added coming from user zone space res_add_files = [] # now necessary in order to test add_process utils.resource_file_add_pre_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata deletion when deleting a resource in user zone space self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0) # test adding file from user zone to existing empty resource in hydroshare zone # even there is no file uploaded to resource initially, there are default extended # automatically metadata created self.resRaster = hydroshare.create_resource( resource_type='RasterResource', owner=self.user, title='My Test Raster Resource' ) # test metadata extraction with a valid tif file being added coming from user zone space # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=[], user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0)
def upload_add(request): # add irods file into an existing resource res_id = request.POST.get('res_id', '') resource, _, _ = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) res_files = request.FILES.getlist('files') extract_metadata = request.POST.get('extract-metadata', 'No') extract_metadata = True if extract_metadata.lower() == 'yes' else False irods_fnames = request.POST.get('upload', '') irods_fnames_list = irods_fnames.split(',') res_cls = resource.__class__ # TODO: read resource type from resource, not from input file valid, ext = check_upload_files(res_cls, irods_fnames_list) source_names = [] if not valid: return JsonResponse({'error': "Invalid file type: {ext}".format(ext=ext)}, status=status.HTTP_400_BAD_REQUEST) else: homepath = irods_fnames_list[0] # TODO: this should happen whether resource is federated or not irods_federated = utils.is_federated(homepath) if irods_federated: source_names = irods_fnames.split(',') else: user = request.POST.get('irods_username') password = request.POST.get("irods_password") port = request.POST.get("irods_port") host = request.POST.get("irods_host") zone = request.POST.get("irods_zone") try: upload_from_irods(username=user, password=password, host=host, port=port, zone=zone, irods_fnames=irods_fnames, res_files=res_files) except SessionException as ex: return JsonResponse( {"error": ex.stderr}, status=status.HTTP_500_INTERNAL_SERVER_ERROR ) try: utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata, source_names=source_names, folder='') except hydroshare.utils.ResourceFileSizeException as ex: return JsonResponse({'error': str(ex)}, status=status.HTTP_400_BAD_REQUEST) except (hydroshare.utils.ResourceFileValidationException, Exception) as ex: return JsonResponse({'error': str(ex)}, status=status.HTTP_400_BAD_REQUEST) try: hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user, extract_metadata=extract_metadata, source_names=source_names, folder='') except (hydroshare.utils.ResourceFileValidationException, SessionException) as ex: if str(ex): return JsonResponse({'error': str(ex)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) elif ex.stderr: return JsonResponse({'error': ex.stderr}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return JsonResponse({}, status=status.HTTP_200_OK)
def test_file_add_to_composite_resource(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly super(CompositeResourceTest, self).assert_federated_irods_available() # test that when we add file to an existing composite resource, the added file # automatically set to genericlogicalfile type self.assertEqual(BaseResource.objects.count(), 0) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test Composite Resource With Files Added From Federated Zone', auto_aggregate=False ) # there should not be any GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 0) # add a file to the resource fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_file_name) res_upload_files = [] resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=None) resource_file_add_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, auto_aggregate=False) # there should be one resource at this point self.assertEqual(BaseResource.objects.count(), 1) self.assertEqual(self.composite_resource.resource_type, "CompositeResource") self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # create the generic aggregation (logical file) GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # check that the resource file is associated with GenericLogicalFile res_file = self.composite_resource.files.first() self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # there should be 1 GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 1) # test adding a file to a folder (Note the UI does not support uploading a iRODS file # to a specific folder) # create the folder new_folder = "my-new-folder" new_folder_path = os.path.join("data", "contents", new_folder) create_folder(self.composite_resource.short_id, new_folder_path) resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=new_folder) resource_file_add_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=new_folder, auto_aggregate=False) self.assertEqual(self.composite_resource.files.all().count(), 2) self.composite_resource.delete()