def test_multiple_tif_file_with_vrt_upload(self): # test that multiple tif files along with a vrt file can be uploaded and # be successful with file validation tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'r'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'r'), name=self.logan_tif_2_file_name) vrt = UploadedFile(file=open(self.logan_vrt_file, 'r'), name=self.logan_vrt_file_name) files = [tif_1, tif_2, vrt] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # if the 3 files are in resource, assume file validation successful self.assertEqual(self.resRaster.files.all().count(), 3)
def test_extra_tif_with_vrt_upload(self): # test that when a vrt file is uploaded, all tif files referenced in vrt must be uploaded # and no extra file can be uploaded, otherwise file validation fails tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'r'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'r'), name=self.logan_tif_2_file_name) extra_tif = UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) vrt = UploadedFile(file=open(self.logan_vrt_file, 'r'), name=self.logan_vrt_file_name) files = [tif_1, tif_2, extra_tif, vrt] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler - which should raise validation exception with self.assertRaises(utils.ResourceFileValidationException): utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # resource should not have any files self.assertEqual(self.resRaster.files.all().count(), 0)
def test_extra_tif_with_vrt_upload(self): # test that when a vrt file is uploaded, all tif files referenced in vrt must be uploaded # and no extra file can be uploaded, otherwise file validation fails tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'rb'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'rb'), name=self.logan_tif_2_file_name) extra_tif = UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) vrt = UploadedFile(file=open(self.logan_vrt_file, 'rb'), name=self.logan_vrt_file_name) files = [tif_1, tif_2, extra_tif, vrt] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler - which should raise validation exception with self.assertRaises(utils.ResourceFileValidationException): utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # resource should not have any files self.assertEqual(self.resRaster.files.all().count(), 0)
def test_multiple_tif_file_without_vrt_upload(self): # test that multiple tif files without a vrt file uploaded will fail file validation tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'rb'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'rb'), name=self.logan_tif_2_file_name) files = [tif_1, tif_2] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler - which should raise validation exception with self.assertRaises(utils.ResourceFileValidationException): utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # resource should not have any files self.assertEqual(self.resRaster.files.all().count(), 0)
def test_multiple_tif_file_with_vrt_upload(self): # test that multiple tif files along with a vrt file can be uploaded and # be successful with file validation tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'rb'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'rb'), name=self.logan_tif_2_file_name) vrt = UploadedFile(file=open(self.logan_vrt_file, 'rb'), name=self.logan_vrt_file_name) files = [tif_1, tif_2, vrt] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # if the 3 files are in resource, assume file validation successful self.assertEqual(self.resRaster.files.all().count(), 3)
def setUp(self): super(TestNewVersionResource, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be versioned self.owner = hydroshare.create_account( '*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[] ) # create a user who is NOT the owner of the resource to be versioned self.nonowner = hydroshare.create_account( '*****@*****.**', username='******', first_name='nonowner_firstname', last_name='nonowner_lastname', superuser=False, groups=[] ) # create a generic resource self.res_generic = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource' ) test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'r') self.test_file2 = open('test2.txt', 'r') hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2) # create a raster resource that represents a specific resource type raster_file = 'hs_core/tests/data/cea.tif' temp_dir = tempfile.mkdtemp() temp_raster_file = os.path.join(temp_dir, 'cea.tif') shutil.copy(raster_file, temp_raster_file) self.raster_obj = open(temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.res_raster = hydroshare.create_resource( resource_type='RasterResource', owner=self.owner, title='Test Raster Resource', files=files, metadata=[] ) # call the post creation process here for the metadata to be # extracted utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_zip_set_file_type_to_geo_feature(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(GeoFeatureFileTypeMetaDataTest, self).is_federated_irods_available(): return # here we are using a valid zip file for setting it # to Geo Feature file type which includes metadata extraction fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.zip_file_name) res_upload_files = [] fed_res_path = hydroshare.utils.get_federated_zone_home_path( fed_test_file_full_path) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Federated Composite Resource Raster File Type Testing', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path, move=False, metadata=[]) # test resource is created on federated zone self.assertNotEqual(self.composite_resource.resource_federation_path, '') # set the logical file -which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() expected_folder_name = res_file.file_name[:-4] # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) fed_file_path = "{}/data/contents/{}".format( self.composite_resource.root_path, self.zip_file_name) self.assertEqual(res_file.storage_path, fed_file_path) # set the zip file to GeoFeatureFile type GeoFeatureLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test file type and file type extracted metadata assert_geofeature_file_type_metadata(self, expected_folder_name) self.composite_resource.delete() # there should be no GeoFeatureLogicalFile object at this point self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) # there should be no GenericFileMetaData object at this point self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_refts_set_file_type_to_reftimeseries(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(RefTimeSeriesFileTypeMetaDataTest, self).is_federated_irods_available(): return # here we are using a valid ref time series for setting it # to RefTimeseries file type which includes metadata extraction fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.refts_file_name) res_upload_files = [] fed_res_path = hydroshare.utils.get_federated_zone_home_path(fed_test_file_full_path) res_title = 'Untitled resource' self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title=res_title, files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path, move=False, metadata=[] ) # test resource is created on federated zone self.assertNotEqual(self.composite_resource.resource_federation_path, '') # set the logical file -which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) fed_file_path = "data/contents/{}".format(self.refts_file_name) self.assertEqual(os.path.join('data', 'contents', res_file.short_path), fed_file_path) # set the tif file to RefTimeseries file type RefTimeseriesLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test that the content of the json file is same is what we have # saved in json_file_content field of the file metadata object res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.json_file_content, res_file.fed_resource_file.read()) # test extracted ref time series file type metadata assert_ref_time_series_file_type_metadata(self) self.composite_resource.delete()
def test_nc_set_file_type_to_netcdf(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(NetCDFFileTypeMetaDataTest, self).is_federated_irods_available(): return # here we are using a valid netcdf file for setting it # to NetCDF file type which includes metadata extraction fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] fed_res_path = hydroshare.utils.get_federated_zone_home_path( fed_test_file_full_path) res_title = 'Federated Composite Resource NetCDF File Type Testing' self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title=res_title, files=res_upload_files, fed_res_file_names=[fed_test_file_full_path], fed_res_path=fed_res_path, fed_copy_or_move='copy', metadata=[]) # test resource is created on federated zone self.assertNotEqual(self.composite_resource.resource_federation_path, '') # set the logical file -which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is one GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 1) fed_file_path = "data/contents/{}".format(self.netcdf_file_name) self.assertEqual(res_file.fed_resource_file_name_or_path, fed_file_path) # set the tif file to NetCDF file type NetCDFLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test extracted netcdf file type metadata assert_netcdf_file_type_metadata(self, res_title)
def _create_composite_resource(self, title='Test Ref Time Series File Type Metadata'): uploaded_file = UploadedFile(file=self.refts_file_obj, name=os.path.basename(self.refts_file_obj.name)) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title=title, files=(uploaded_file,) ) # set the generic logical file as part of resource post create signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata)
def _create_composite_resource(self, file_obj): uploaded_file = UploadedFile(file=file_obj, name=os.path.basename(file_obj.name)) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test Raster File Type Metadata', files=(uploaded_file,) ) # set the generic logical file type resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata)
def test_delete_composite_resource(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(CompositeResourceTest, self).is_federated_irods_available(): return self.assertEqual(BaseResource.objects.count(), 0) self._create_composite_resource() # set the logical file - which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) self.assertEqual(BaseResource.objects.count(), 1) self.composite_resource.delete() self.assertEqual(BaseResource.objects.count(), 0)
def test_metadata_extraction_on_resource_creation(self): # passing the file object that points to the temp dir doesn't work - create_resource throws # error. open the file from the fixed file location files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) super(TestRasterMetaData, self).raster_metadata_extraction()
def _create_composite_resource(self, title='Test Time series File Type Metadata', file_to_upload=None): if file_to_upload is None: file_to_upload = UploadedFile(file=self.sqlite_file_obj, name=os.path.basename( self.sqlite_file_obj.name)) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title=title, files=(file_to_upload, )) # set the generic logical file as part of resource post create signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata)
def _create_netcdf_resource(self): files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] _, _, metadata, _ = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Snow water equivalent estimation at TWDEF site ' 'from Oct 2009 to June 2010', page_redirect_url_key=None, files=files, metadata=None) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=files, metadata=metadata ) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata)
def test_metadata_extraction_on_resource_creation(self): # passing the file object that points to the temp dir doesn't work - create_resource throws # error. open the file from the fixed file location files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] self.resRaster = hydroshare.create_resource('RasterResource', self.user, 'My Test Raster Resource', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) super(TestRasterMetaData, self).raster_metadata_extraction()
def _create_netcdf_resource(self): files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] _, _, metadata = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Snow water equivalent estimation at TWDEF site ' 'from Oct 2009 to June 2010', page_redirect_url_key=None, files=files, metadata=None) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=files, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata)
def _create_composite_resource(self, file_to_upload=None): files = [] if file_to_upload is not None: file_name = os.path.basename(file_to_upload) target_temp_file = os.path.join(self.temp_dir, file_name) shutil.copy(file_to_upload, target_temp_file) file_obj = open(target_temp_file, 'r') uploaded_file = UploadedFile(file=file_obj, name=os.path.basename(file_obj.name)) files.append(uploaded_file) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test Raster File Type Metadata', files=files) # set the generic logical file resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata)
def test_create_composite_resource_with_file_upload(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(CompositeResourceTest, self).is_federated_irods_available(): return # test that when we create composite resource with an uploaded file, then the uploaded file # is automatically set to genericlogicalfile type self.assertEqual(BaseResource.objects.count(), 0) self._create_composite_resource() # make sure composite_resource is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(self.composite_resource.resource_federation_path, fed_path) # Deprecated: there should not be any GenericLogicalFile object at this point # Issue 2456 Create composite with uploaded file now part of logical file self.assertEqual(GenericLogicalFile.objects.count(), 1) # set the logical file - which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) # there should be one resource at this point self.assertEqual(BaseResource.objects.count(), 1) self.assertEqual(self.composite_resource.resource_type, "CompositeResource") self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # there should be 1 GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 1) self.composite_resource.delete()
def test_file_delete_composite_resource(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(CompositeResourceTest, self).is_federated_irods_available(): return self._create_composite_resource() # set the logical file - which get sets as part of the post resource creation signal resource_post_create_actions(resource=self.composite_resource, user=self.user, metadata=self.composite_resource.metadata) # there should be one GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 1) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() hydroshare.delete_resource_file(self.composite_resource.short_id, res_file.id, self.user) self.assertEqual(self.composite_resource.files.all().count(), 0) # there should not be any GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 0)
def test_multiple_tif_file_without_vrt_upload(self): # test that multiple tif files without a vrt file uploaded will fail file validation tif_1 = UploadedFile(file=open(self.logan_tif_1_file, 'r'), name=self.logan_tif_1_file_name) tif_2 = UploadedFile(file=open(self.logan_tif_2_file, 'r'), name=self.logan_tif_2_file_name) files = [tif_1, tif_2] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler - which should raise validation exception with self.assertRaises(utils.ResourceFileValidationException): utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) # resource should not have any files self.assertEqual(self.resRaster.files.all().count(), 0)
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly super(TestNetcdfMetaData, self).assert_federated_irods_available() # test metadata extraction with resource creation with nc file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Snow water equivalent estimation at TWDEF site from Oct 2009 to June ' '2010', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content files at this point self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file now hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 1 creator element (based on the extracted metadata) self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # test metadata extraction with a valid nc file being added coming from user zone space res_add_files = [] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def setUp(self): super(TestCopyResource, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be copied self.owner = hydroshare.create_account( '*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[] ) # create a user who is NOT the owner of the resource to be copied self.nonowner = hydroshare.create_account( '*****@*****.**', username='******', first_name='nonowner_firstname', last_name='nonowner_lastname', superuser=False, groups=[] ) # create a generic resource self.res_generic = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource' ) test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'r') self.test_file2 = open('test2.txt', 'r') hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2) # create a generic empty resource with one license that prohibits derivation statement = 'This resource is shared under the Creative Commons Attribution-NoDerivs CC ' \ 'BY-ND.' url = 'http://creativecommons.org/licenses/by-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata ) # create a generic empty resource with another license that prohibits derivation statement = 'This resource is shared under the Creative Commons ' \ 'Attribution-NoCommercial-NoDerivs CC BY-NC-ND.' url = 'http://creativecommons.org/licenses/by-nc-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nc_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata ) # create a raster resource that represents a specific resource type raster_file = 'hs_core/tests/data/cea.tif' temp_dir = tempfile.mkdtemp() self.temp_raster_file = os.path.join(temp_dir, 'cea.tif') shutil.copy(raster_file, self.temp_raster_file) self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.res_raster = hydroshare.create_resource( resource_type='RasterResource', owner=self.owner, title='Test Raster Resource', files=files, metadata=[] ) # call the post creation process here for the metadata to be # extracted utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestNetcdfMetaData, self).is_federated_irods_available(): return # test metadata extraction with resource creation with nc file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.netcdf_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title= 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June ' '2010', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.user, 'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.user, metadata) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content files at this point self.assertEqual(self.resNetcdf.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2) # delete content file now hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user) # there should no content file self.assertEqual(self.resNetcdf.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resNetcdf.metadata.title, None) # there should be abstract element self.assertNotEqual(self.resNetcdf.metadata.description, None) # there should be 1 creator element (based on the extracted metadata) self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1) # there should be 1 contributor element self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1) # there should be no coverage element self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0) # there should be subject element self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0) # testing extended metadata elements self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0) self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0) # test metadata extraction with a valid nc file being added coming from user zone space res_add_files = [] utils.resource_file_add_pre_process( resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resNetcdf, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestNetcdfMetaData, self).netcdf_metadata_extraction() self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_wgs84_coverage_extraction(self): # this is test that the resource level coverage (wsg84) is being created correctly # test for BearCr.tif files = [ UploadedFile(file=self.raster_bearcr_tif_file_obj, name=self.raster_bearcr_tif_file_name) ] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter( type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 30.214583003567654) self.assertEqual(box_coverage.value['eastlimit'], -97.92170777387547) self.assertEqual(box_coverage.value['southlimit'], 30.127513332692264) self.assertEqual(box_coverage.value['westlimit'], -98.01556648306897) self.resRaster.delete() # test for Honduras.tif files = [ UploadedFile(file=self.raster_honduras_tif_file_obj, name=self.raster_honduras_tif_file_name) ] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) self.assertEqual(self.resRaster.files.all().count(), 2) # there should not be any coverage extracted in the case of Honduras self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) self.resRaster.delete() # test for mawhefel.tif files = [ UploadedFile(file=self.raster_mawhefel_tif_file_obj, name=self.raster_mawhefel_tif_file_name) ] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) self.assertEqual(self.resRaster.files.all().count(), 2) # there should not be any coverage extracted in the case of Honduras self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) self.resRaster.delete() # test for lidarDEM.tif files = [ UploadedFile(file=self.raster_lidardem_tif_file_obj, name=self.raster_lidardem_tif_file_name) ] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter( type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 40.678328957336106) self.assertEqual(box_coverage.value['eastlimit'], -77.88875113570533) self.assertEqual(box_coverage.value['southlimit'], 40.650831557543725) self.assertEqual(box_coverage.value['westlimit'], -77.92486166705822) self.resRaster.delete() # test for HT_Elevation.tif files = [ UploadedFile(file=self.raster_htelevation_tif_file_obj, name=self.raster_htelevation_tif_file_name) ] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[]) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter( type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 40.75042571735219) self.assertEqual(box_coverage.value['eastlimit'], -77.84406750606215) self.assertEqual(box_coverage.value['southlimit'], 40.58088597175731) self.assertEqual(box_coverage.value['westlimit'], -78.01781620659817) self.resRaster.delete()
def test_update_netcdf_file(self): # create a netcdf resource files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)] _, _, metadata, _ = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Untitled resource', page_redirect_url_key=None, files=files, metadata=None,) self.resNetcdf = hydroshare.create_resource( 'NetcdfResource', self.john, 'Untitled resource', files=files, metadata=metadata ) utils.resource_post_create_actions(self.resNetcdf, self.john, metadata) res_metadata = self.resNetcdf.metadata # update title for setting flag self.assertFalse(res_metadata.is_dirty) element = res_metadata.title res_metadata.update_element('title', element.id, value='new title') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update temporal coverage for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('coverage', type='period', value={'start': '01/01/2000', 'end': '12/12/2010'}) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('coverage', element.id, type='period', value={'start': '01/02/2000', 'end': '01/03/2000'}) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update spatial coverage for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('coverage', type='box', value={'northlimit': 12, 'eastlimit': 12, 'southlimit': 1, 'westlimit': 1, 'units': 'Decimal degree', 'projection': 'WGS84'} ) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('coverage', element.id, type='box', value={'northlimit': '13', 'eastlimit': '13', 'southlimit': '2', 'westlimit': '2', 'units': 'Decimal degree', 'projection': 'WGS84'} ) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update description for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('description', abstract='new abstract') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('description', element.id, abstract='update abstract') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create subject for setting flag res_metadata.is_dirty = False res_metadata.save() res_metadata.create_element('subject', value='new keyword2') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete source for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('source', derived_from='new source') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('source', element.id, abstract='update source') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('source', element.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete relation for setting flag res_metadata.is_dirty = False res_metadata.save() element_1 = res_metadata.create_element('relation', type='isHostedBy', value='new host') res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) element_2 = res_metadata.create_element('relation', type='cites', value='new reference') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('relation', element_1.id, type='isHostedBy', value='update host') res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) res_metadata.update_element('relation', element_2.id, type='cites', value='update reference') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('relation', element_2.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # update creator res_metadata.is_dirty = False res_metadata.save() element = res_metadata.creators.all().first() res_metadata.update_element('creator', element.id, name='update name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete contributor res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('contributor', name='new name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('contributor', element.id, name='update name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('contributor', element.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # update variable res_metadata.is_dirty = False res_metadata.save() element = res_metadata.variables.all().first() res_metadata.update_element('variable', element.id, name=element.name, unit='update unit', type=element.type, shape=element.shape, missing_value='1', descriptive_name='update long name', method='update method') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # check file content update request url = reverse("update_netcdf_resfile", kwargs={'resource_id': self.resNetcdf.short_id}) request = self.factory.post(url) request.user = self.john self._set_request_message_attributes(request) request.META['HTTP_REFERER'] = "/some_url/" response = update_netcdf_file(request, resource_id=self.resNetcdf.short_id) self.assertEqual(response.status_code, status.HTTP_302_FOUND) # check file update content nc_dump_res_file = None for f in self.resNetcdf.files.all(): if f.extension == ".txt": nc_dump_res_file = f break self.assertNotEqual(nc_dump_res_file, None) self.assertIn('title = "new title"', nc_dump_res_file.resource_file.read()) res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) # test extra metadata update for setting flag self.assertEqual(self.resNetcdf.extra_metadata, {}) url_params = {'shortkey': self.resNetcdf.short_id} post_data = {'key1': 'key-1', 'value1': 'value-1'} url = reverse('update_key_value_metadata', kwargs=url_params) request = self.factory.post(url, data=post_data) request.user = self.john # make it a ajax request request.META['HTTP_X_REQUESTED_WITH'] = 'XMLHttpRequest' self._set_request_message_attributes(request) response = update_key_value_metadata(request, shortkey=self.resNetcdf.short_id) self.assertEqual(response.status_code, status.HTTP_200_OK) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty)
def test_metadata_in_user_zone(self): super(TestRasterMetaData, self).assert_federated_irods_available() # test metadata extraction with resource creation with tif file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_tif_file_name) res_upload_files = [] _, _, metadata, fed_res_path = utils.resource_pre_create_actions( resource_type='RasterResource', resource_title='My Test Raster Resource', page_redirect_url_key=None, files=res_upload_files, source_names=[fed_test_file_full_path]) self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource', files=res_upload_files, source_names=[fed_test_file_full_path], fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '', move=False, metadata=metadata) # raster file validation and metadata extraction in post resource creation signal handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata is deleted after content file is deleted in user zone space # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual(self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file now hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should be no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should be no extended metadata elements # at this point self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) # test metadata extraction with a valid tif file being added coming from user zone space res_add_files = [] # now necessary in order to test add_process utils.resource_file_add_pre_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=res_add_files, user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # test metadata deletion when deleting a resource in user zone space self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0) # test adding file from user zone to existing empty resource in hydroshare zone # even there is no file uploaded to resource initially, there are default extended # automatically metadata created self.resRaster = hydroshare.create_resource( resource_type='RasterResource', owner=self.user, title='My Test Raster Resource' ) # test metadata extraction with a valid tif file being added coming from user zone space # file validation and metadata extraction happen during post file add signal handler utils.resource_file_add_process(resource=self.resRaster, files=[], user=self.user, source_names=[fed_test_file_full_path]) super(TestRasterMetaData, self).raster_metadata_extraction() # there should be 2 content file: tif file and vrt file at this point self.assertEqual(self.resRaster.files.all().count(), 2) # delete resource hydroshare.delete_resource(self.resRaster.short_id) # resource core metadata is deleted after resource deletion self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestTimeSeriesMetaData, self).is_federated_irods_available(): return # test metadata extraction with resource creation with file coming from user zone space fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.odm2_sqlite_file_name) self.resTimeSeries = hydroshare.create_resource( resource_type='TimeSeriesResource', owner=self.user, title='My Test TimeSeries Resource', source_names=[fed_test_file_full_path], move=False) utils.resource_post_create_actions(resource=self.resTimeSeries, user=self.user, metadata=[]) super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction() # test that metadata is NOT deleted (except format element) on content file deletion # there should be one content file at this point self.assertEqual(self.resTimeSeries.files.all().count(), 1) # there should be one format element self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 1) hydroshare.delete_resource_file(self.resTimeSeries.short_id, self.odm2_sqlite_file_name, self.user) # there should be no content file self.assertEqual(self.resTimeSeries.files.all().count(), 0) # test the core metadata at this point self.assertNotEqual(self.resTimeSeries.metadata.title, None) # there should be an abstract element self.assertNotEqual(self.resTimeSeries.metadata.description, None) # there should be one creator element self.assertEqual(self.resTimeSeries.metadata.creators.all().count(), 1) # there should be one contributor element self.assertEqual( self.resTimeSeries.metadata.contributors.all().count(), 1) # there should be 2 coverage element - point type and period type self.assertEqual(self.resTimeSeries.metadata.coverages.all().count(), 2) self.assertEqual( self.resTimeSeries.metadata.coverages.all().filter( type='box').count(), 1) self.assertEqual( self.resTimeSeries.metadata.coverages.all().filter( type='period').count(), 1) # there should be no format element self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 0) # there should be one subject element self.assertEqual(self.resTimeSeries.metadata.subjects.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resTimeSeries.metadata.sites.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.variables.all().count(), 0) self.assertNotEqual(self.resTimeSeries.metadata.methods.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.processing_levels.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.time_series_results.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_variable_names.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_variable_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_speciations.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_site_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_elevation_datums.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_method_types.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_statuses.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_mediums.all().count(), 0) self.assertNotEqual( self.resTimeSeries.metadata.cv_aggregation_statistics.all().count( ), 0) # test metadata extraction with a valid ODM2 sqlite file being added coming from user zone # space res_add_files = [] utils.resource_file_add_pre_process( resource=self.resTimeSeries, files=res_add_files, user=self.user, extract_metadata=False, source_names=[fed_test_file_full_path]) utils.resource_file_add_process(resource=self.resTimeSeries, files=res_add_files, user=self.user, extract_metadata=True, source_names=[fed_test_file_full_path]) super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction() # test metadata deletion when deleting a resource in user zone space # all metadata should get deleted when the resource get deleted self.assertEqual(CoreMetaData.objects.all().count(), 1) # delete resource hydroshare.delete_resource(self.resTimeSeries.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0)
def create_resource(request, *args, **kwargs): resource_type = request.POST['resource-type'] res_title = request.POST['title'] resource_files = request.FILES.getlist('files') irods_fname = request.POST.get('irods_file_name') if irods_fname: user = request.POST.get('irods-username') password = request.POST.get("irods-password") port = request.POST.get("irods-port") host = request.POST.get("irods-host") zone = request.POST.get("irods-zone") try: upload_from_irods(username=user, password=password, host=host, port=port, zone=zone, irods_fname=irods_fname, res_files=resource_files) except Exception as ex: context = {'resource_creation_error': ex.message} return render_to_response('pages/create-resource.html', context, context_instance=RequestContext(request)) url_key = "page_redirect_url" try: page_url_dict, res_title, metadata = hydroshare.utils.resource_pre_create_actions(resource_type=resource_type, files=resource_files, resource_title=res_title, page_redirect_url_key=url_key, **kwargs) except utils.ResourceFileSizeException as ex: context = {'file_size_error': ex.message} return render_to_response('pages/create-resource.html', context, context_instance=RequestContext(request)) except utils.ResourceFileValidationException as ex: context = {'file_validation_error': ex.message} return render_to_response('pages/create-resource.html', context, context_instance=RequestContext(request)) except Exception as ex: context = {'resource_creation_error': ex.message} return render_to_response('pages/create-resource.html', context, context_instance=RequestContext(request)) if url_key in page_url_dict: return render(request, page_url_dict[url_key], {'title': res_title, 'metadata': metadata}) try: resource = hydroshare.create_resource( resource_type=request.POST['resource-type'], owner=request.user, title=res_title, keywords=None, metadata=metadata, files=resource_files, content=res_title ) except Exception as ex: context = {'resource_creation_error': ex.message } return render_to_response('pages/create-resource.html', context, context_instance=RequestContext(request)) try: utils.resource_post_create_actions(resource=resource, user=request.user, metadata=metadata, **kwargs) except (utils.ResourceFileValidationException, Exception) as ex: request.session['file_validation_error'] = ex.message # go to resource landing page request.session['just_created'] = True return HttpResponseRedirect(resource.get_absolute_url())
def test_update_netcdf_file(self): # create a netcdf resource files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] _, _, metadata, _ = utils.resource_pre_create_actions( resource_type='NetcdfResource', resource_title='Untitled resource', page_redirect_url_key=None, files=files, metadata=None, ) self.resNetcdf = hydroshare.create_resource('NetcdfResource', self.john, 'Untitled resource', files=files, metadata=metadata) utils.resource_post_create_actions(self.resNetcdf, self.john, metadata) res_metadata = self.resNetcdf.metadata # update title for setting flag self.assertFalse(res_metadata.is_dirty) element = res_metadata.title res_metadata.update_element('title', element.id, value='new title') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update temporal coverage for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('coverage', type='period', value={ 'start': '01/01/2000', 'end': '12/12/2010' }) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('coverage', element.id, type='period', value={ 'start': '01/02/2000', 'end': '01/03/2000' }) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update spatial coverage for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('coverage', type='box', value={ 'northlimit': 12, 'eastlimit': 12, 'southlimit': 1, 'westlimit': 1, 'units': 'Decimal degree', 'projection': 'WGS84' }) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('coverage', element.id, type='box', value={ 'northlimit': '13', 'eastlimit': '13', 'southlimit': '2', 'westlimit': '2', 'units': 'Decimal degree', 'projection': 'WGS84' }) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update description for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('description', abstract='new abstract') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('description', element.id, abstract='update abstract') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create subject for setting flag res_metadata.is_dirty = False res_metadata.save() res_metadata.create_element('subject', value='new keyword2') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete source for setting flag res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('source', derived_from='new source') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('source', element.id, abstract='update source') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('source', element.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete relation for setting flag res_metadata.is_dirty = False res_metadata.save() element_1 = res_metadata.create_element('relation', type='isHostedBy', value='new host') res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) element_2 = res_metadata.create_element('relation', type='cites', value='new reference') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('relation', element_1.id, type='isHostedBy', value='update host') res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) res_metadata.update_element('relation', element_2.id, type='cites', value='update reference') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('relation', element_2.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # update creator res_metadata.is_dirty = False res_metadata.save() element = res_metadata.creators.all().first() res_metadata.update_element('creator', element.id, name='update name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # create/update/delete contributor res_metadata.is_dirty = False res_metadata.save() element = res_metadata.create_element('contributor', name='new name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.update_element('contributor', element.id, name='update name') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) res_metadata.is_dirty = False res_metadata.save() res_metadata.delete_element('contributor', element.id) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # update variable res_metadata.is_dirty = False res_metadata.save() element = res_metadata.variables.all().first() res_metadata.update_element('variable', element.id, name=element.name, unit='update unit', type=element.type, shape=element.shape, missing_value='1', descriptive_name='update long name', method='update method') res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty) # check file content update request url = reverse("update_netcdf_resfile", kwargs={'resource_id': self.resNetcdf.short_id}) request = self.factory.post(url) request.user = self.john self._set_request_message_attributes(request) request.META['HTTP_REFERER'] = "/some_url/" response = update_netcdf_file(request, resource_id=self.resNetcdf.short_id) self.assertEqual(response.status_code, status.HTTP_302_FOUND) # check file update content nc_dump_res_file = None for f in self.resNetcdf.files.all(): if f.extension == ".txt": nc_dump_res_file = f break self.assertNotEqual(nc_dump_res_file, None) self.assertIn('title = "new title"', nc_dump_res_file.resource_file.read()) res_metadata.refresh_from_db() self.assertFalse(res_metadata.is_dirty) # test extra metadata update for setting flag self.assertEqual(self.resNetcdf.extra_metadata, {}) url_params = {'shortkey': self.resNetcdf.short_id} post_data = {'key1': 'key-1', 'value1': 'value-1'} url = reverse('update_key_value_metadata', kwargs=url_params) request = self.factory.post(url, data=post_data) request.user = self.john # make it a ajax request request.META['HTTP_X_REQUESTED_WITH'] = 'XMLHttpRequest' self._set_request_message_attributes(request) response = update_key_value_metadata(request, shortkey=self.resNetcdf.short_id) self.assertEqual(response.status_code, status.HTTP_200_OK) res_metadata.refresh_from_db() self.assertTrue(res_metadata.is_dirty)
def test_wgs84_coverage_extraction(self): # this is test that the resource level coverage (wsg84) is being created correctly # test for BearCr.tif files = [UploadedFile(file=self.raster_bearcr_tif_file_obj, name=self.raster_bearcr_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter(type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 30.214583003567654) self.assertEqual(box_coverage.value['eastlimit'], -97.92170777387547) self.assertEqual(box_coverage.value['southlimit'], 30.127513332692264) self.assertEqual(box_coverage.value['westlimit'], -98.01556648306897) self.resRaster.delete() # test for Honduras.tif files = [UploadedFile(file=self.raster_honduras_tif_file_obj, name=self.raster_honduras_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) self.assertEqual(self.resRaster.files.all().count(), 2) # there should not be any coverage extracted in the case of Honduras self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) self.resRaster.delete() # test for mawhefel.tif files = [UploadedFile(file=self.raster_mawhefel_tif_file_obj, name=self.raster_mawhefel_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) self.assertEqual(self.resRaster.files.all().count(), 2) # there should not be any coverage extracted in the case of Honduras self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) self.resRaster.delete() # test for lidarDEM.tif files = [UploadedFile(file=self.raster_lidardem_tif_file_obj, name=self.raster_lidardem_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter(type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 40.678328957336106) self.assertEqual(box_coverage.value['eastlimit'], -77.88875113570533) self.assertEqual(box_coverage.value['southlimit'], 40.650831557543725) self.assertEqual(box_coverage.value['westlimit'], -77.92486166705822) self.resRaster.delete() # test for HT_Elevation.tif files = [UploadedFile(file=self.raster_htelevation_tif_file_obj, name=self.raster_htelevation_tif_file_name)] self.resRaster = hydroshare.create_resource( 'RasterResource', self.user, 'My Test Raster Resource Coverage', files=files, metadata=[] ) # uploaded file validation and metadata extraction happens in post resource # creation handler utils.resource_post_create_actions(resource=self.resRaster, user=self.user, metadata=[]) box_coverage = self.resRaster.metadata.coverages.all().filter(type='box').first() self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(box_coverage.value['units'], 'Decimal degrees') self.assertEqual(box_coverage.value['northlimit'], 40.75042571735219) self.assertEqual(box_coverage.value['eastlimit'], -77.84406750606215) self.assertEqual(box_coverage.value['southlimit'], 40.58088597175731) self.assertEqual(box_coverage.value['westlimit'], -78.01781620659817) self.resRaster.delete()
def test_copy_composite_resource(self): """Test that logical file type objects gets copied along with the metadata that each logical file type object contains. Here we are not testing resource level metadata copy as that has been tested in separate unit tests""" self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.owner, title='Test Composite Resource', files=files, auto_aggregate=False ) # run the resource post creation signal utils.resource_post_create_actions(resource=self.composite_resource, user=self.owner, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not associated with file type self.assertEqual(res_file.has_logical_file, False) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, self.owner, res_file.id) # ensure a nonowner who does not have permission to view a resource cannot copy it with self.assertRaises(PermissionDenied): hydroshare.create_empty_resource(self.composite_resource.short_id, self.nonowner, action='copy') # give nonowner view privilege so nonowner can create a new copy of this resource self.owner.uaccess.share_resource_with_user(self.composite_resource, self.nonowner, PrivilegeCodes.VIEW) orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file # add some key value metadata orig_geo_raster_lfo.metadata.extra_metadata = {'key-1': 'value-1', 'key-2': 'value-2'} # create a copy of the composite resource new_composite_resource = hydroshare.create_empty_resource(self.composite_resource.short_id, self.nonowner, action='copy') new_composite_resource = hydroshare.copy_resource(self.composite_resource, new_composite_resource) # check that there is 2 GeoRasterLogicalFile objects self.assertEqual(GeoRasterLogicalFile.objects.count(), 2) # compare the 2 GeoRasterLogicalFile objects from the original resource and the new one orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file copy_res_file = new_composite_resource.files.first() copy_geo_raster_lfo = copy_res_file.logical_file # check that we put the 2 files in a new folder (cea) for res_file in self.composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format(self.composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) for res_file in new_composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format(new_composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) # both logical file objects should have 2 resource files self.assertEqual(orig_geo_raster_lfo.files.count(), copy_geo_raster_lfo.files.count()) self.assertEqual(orig_geo_raster_lfo.files.count(), 2) # both logical file objects should have same dataset_name self.assertEqual(orig_geo_raster_lfo.dataset_name, copy_geo_raster_lfo.dataset_name) # both should have same key/value metadata self.assertEqual(orig_geo_raster_lfo.metadata.extra_metadata, copy_geo_raster_lfo.metadata.extra_metadata) # both logical file objects should have same coverage metadata self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), copy_geo_raster_lfo.metadata.coverages.count()) self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), 1) org_spatial_coverage = orig_geo_raster_lfo.metadata.spatial_coverage copy_spatial_coverage = copy_geo_raster_lfo.metadata.spatial_coverage self.assertEqual(org_spatial_coverage.type, copy_spatial_coverage.type) self.assertEqual(org_spatial_coverage.type, 'box') self.assertEqual(org_spatial_coverage.value['projection'], copy_spatial_coverage.value['projection']) self.assertEqual(org_spatial_coverage.value['units'], copy_spatial_coverage.value['units']) self.assertEqual(org_spatial_coverage.value['northlimit'], copy_spatial_coverage.value['northlimit']) self.assertEqual(org_spatial_coverage.value['eastlimit'], copy_spatial_coverage.value['eastlimit']) self.assertEqual(org_spatial_coverage.value['southlimit'], copy_spatial_coverage.value['southlimit']) self.assertEqual(org_spatial_coverage.value['westlimit'], copy_spatial_coverage.value['westlimit']) # both logical file objects should have same original coverage org_orig_coverage = orig_geo_raster_lfo.metadata.originalCoverage copy_orig_coverage = copy_geo_raster_lfo.metadata.originalCoverage self.assertEqual(org_orig_coverage.value['projection'], copy_orig_coverage.value['projection']) self.assertEqual(org_orig_coverage.value['units'], copy_orig_coverage.value['units']) self.assertEqual(org_orig_coverage.value['northlimit'], copy_orig_coverage.value['northlimit']) self.assertEqual(org_orig_coverage.value['eastlimit'], copy_orig_coverage.value['eastlimit']) self.assertEqual(org_orig_coverage.value['southlimit'], copy_orig_coverage.value['southlimit']) self.assertEqual(org_orig_coverage.value['westlimit'], copy_orig_coverage.value['westlimit']) # both logical file objects should have same cell information metadata orig_cell_info = orig_geo_raster_lfo.metadata.cellInformation copy_cell_info = copy_geo_raster_lfo.metadata.cellInformation self.assertEqual(orig_cell_info.rows, copy_cell_info.rows) self.assertEqual(orig_cell_info.columns, copy_cell_info.columns) self.assertEqual(orig_cell_info.cellSizeXValue, copy_cell_info.cellSizeXValue) self.assertEqual(orig_cell_info.cellSizeYValue, copy_cell_info.cellSizeYValue) self.assertEqual(orig_cell_info.cellDataType, copy_cell_info.cellDataType) # both logical file objects should have same band information metadata self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), 1) self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), copy_geo_raster_lfo.metadata.bandInformations.count()) orig_band_info = orig_geo_raster_lfo.metadata.bandInformations.first() copy_band_info = copy_geo_raster_lfo.metadata.bandInformations.first() self.assertEqual(orig_band_info.noDataValue, copy_band_info.noDataValue) self.assertEqual(orig_band_info.maximumValue, copy_band_info.maximumValue) self.assertEqual(orig_band_info.minimumValue, copy_band_info.minimumValue) # make sure to clean up all created resources to clean up iRODS storage if self.composite_resource: self.composite_resource.delete() if new_composite_resource: new_composite_resource.delete()