def test_create_aggregation_2(self): """Test that we can create a generic aggregation from a resource file that exists in a folder """ self.create_composite_resource() new_folder = 'generic_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the txt file to the resource at the above folder self.add_file_to_resource(file_to_add=self.generic_file, upload_folder=new_folder) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # file has a folder self.assertEqual(res_file.file_folder, new_folder) # check that the resource file is not part of an aggregation self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GenericLogicalFile.objects.count(), 0) # set file to generic logical file type (aggregation) GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() # file has the same folder self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) self.assertEqual(GenericLogicalFile.objects.count(), 1) self.composite_resource.delete()
def _test_create_aggregation_from_folder(self, foldet_to_test): self.create_composite_resource() self.assertEqual(self.composite_resource.files.count(), 0) # create a folder to upload the nc file there new_folder = foldet_to_test ResourceFile.create_folder(self.composite_resource, new_folder) # add the nc file to the resource at the above folder res_file = self.add_file_to_resource(file_to_add=self.netcdf_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) # resource should have 1 file now self.assertEqual(self.composite_resource.files.count(), 1) for res_file in self.composite_resource.files.all(): self.assertFalse(res_file.has_logical_file) self.assertEqual(NetCDFLogicalFile.objects.count(), 0) # create the aggregation from the folder NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, folder_path=new_folder) self.assertEqual(NetCDFLogicalFile.objects.count(), 1) for res_file in self.composite_resource.files.all(): # test that each resource file is part of an aggregation (logical file) self.assertTrue(res_file.has_logical_file) # test that the each resource file has the same folder - no new folder created self.assertEqual(res_file.file_folder, new_folder) self.composite_resource.delete()
def test_create_aggregation_from_nc_file_2(self): # here we are using a valid nc file for setting it # to NetCDF file type which includes metadata extraction # the nc file in this case is not at the root of the folder hierarchy but in a folder self.create_composite_resource() new_folder = 'netcdf_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the nc file to the resource at the above folder self.add_file_to_resource(file_to_add=self.netcdf_file, upload_folder=new_folder) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) # check that there is no NetCDFLogicalFile object self.assertEqual(NetCDFLogicalFile.objects.count(), 0) # set the nc file to NetCDF file type NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test extracted metadata assert_netcdf_file_type_metadata(self, self.res_title, aggr_folder=new_folder) # test file level keywords res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(len(logical_file.metadata.keywords), 1) self.assertEqual(logical_file.metadata.keywords[0], 'Snow water equivalent') self.composite_resource.delete()
def setUp(self): super(TestFolderDownloadZip, self).setUp() self.output_path = "zips/rand/foo.zip" self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = create_account( '*****@*****.**', username='******', first_name='Shaun', last_name='Livingston', superuser=False, groups=[] ) # create files self.n1 = "test1.txt" test_file = open(self.n1, 'w') test_file.write("Test text file in test1.txt") test_file.close() test_file = open(self.n1, "r") self.res = create_resource(resource_type='GenericResource', owner=self.user, title='Test Resource', metadata=[], ) ResourceFile.create_folder(self.res, 'foo') # add one file to the resource add_resource_files(self.res.short_id, test_file, folder='foo')
def test_aggregation_file_move(self): # test any resource file that's part of the NetCDF logical file can't be moved self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() # create the aggregation using the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with raster LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 2) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) expected_folder_name = base_file_name self.assertEqual(res_file.file_folder, expected_folder_name) new_folder = 'netcdf_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # moving any of the resource files to this new folder should raise exception tgt_path = 'data/contents/{}'.format(new_folder) for res_file in self.composite_resource.files.all(): with self.assertRaises(DRF_ValidationError): src_path = os.path.join('data', 'contents', res_file.short_path) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_unfederated_folder_path_setting(self): """ an unfederated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live shortpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath)) # non-existent files should raise error otherpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file2.txt") with self.assertRaises(ValidationError): resfile.path_is_acceptable(otherpath) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt") # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object with self.assertRaises(ValidationError): resfile.set_storage_path(otherpath) # should not change self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # TODO: how to eliminate this particular error. # dumbpath = 'x' + shortpath # dumbpath = self.res.short_id + "file1.txt" # clean up after folder test # ResourceFile.remove_folder(self.res, 'foo', self.user) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_unfederated_folder_path_setting(self): """ an unfederated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live shortpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath)) # non-existent files should raise error otherpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file2.txt") with self.assertRaises(ValidationError): resfile.path_is_acceptable(otherpath) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt") # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object with self.assertRaises(ValidationError): resfile.set_storage_path(otherpath) # should not change self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # TODO: how to eliminate this particular error. # dumbpath = 'x' + shortpath # dumbpath = self.res.short_id + "file1.txt" # clean up after folder test # ResourceFile.remove_folder(self.res, 'foo', self.user) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_aggregation_file_move(self): # test any resource file that's part of the GeoFeature logical file can't be moved self.create_composite_resource(self.states_required_zip_file) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) # extract metadata from the tif file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with geo feature LFO - which should # raise exception self.assertEqual(self.composite_resource.files.count(), 3) new_folder = 'geofeature_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # moving any of the resource files to this new folder should raise exception tgt_path = 'data/contents/geofeature_aggr' for res_file in self.composite_resource.files.all(): with self.assertRaises(DRF_ValidationError): src_path = os.path.join('data', 'contents', res_file.short_path) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_create_aggregation_from_shp_file_required_3(self): # here we are using a shp file that exists in a folder # for setting it to Geo Feature file type which includes metadata extraction # The same folder contains another file that is not going to be part of the # geofeature aggregation a new folder should be created in this case to represent the # geofeature aggregation # location shp file before aggregation is created: my_folder/states.shp # location of another file before aggregation is created: my_folder/states_invalid.zip # location of shp file after aggregation is created: my_folder/states/states.shp # location of another file after aggregation is created: my_folder/states_invalid.zip self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder res_file = self.add_file_to_resource(file_to_add=self.states_shp_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) res_file = self.add_file_to_resource(file_to_add=self.states_shx_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(self.composite_resource.files.all().count(), 3) # add a file that is not related to aggregation res_file = self.add_file_to_resource( file_to_add=self.states_zip_invalid_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(self.composite_resource.files.all().count(), 4) # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) # set the shp file to GeoFeatureLogicalFile type shp_res_file = [ f for f in self.composite_resource.files.all() if f.extension == '.shp' ][0] GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, shp_res_file.id) self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1) base_shp_file_base_name, _ = os.path.splitext(shp_res_file.file_name) shp_res_file = [ f for f in self.composite_resource.files.all() if f.extension == '.shp' ][0] logical_file = shp_res_file.logical_file self.assertEqual(logical_file.files.count(), 3) for res_file in logical_file.files.all(): # test that the each resource file has the same folder - no new folder created self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(self.composite_resource.files.all().count(), 4) self.composite_resource.delete()
def test_create_aggregation_from_nc_file_3(self): # here we are using a valid nc file for setting it # to NetCDF file type which includes metadata extraction # the nc file in this case is not at the root of the folder hierarchy but in a folder. The # same folder contains another file that's not going part of the aggregation # location nc file before aggregation is created: /my_folder/netcdf_valid.nc # location of another file before aggregation is created: /my_folder/netcdf_invalid.nc # location of nc file after aggregation is created: # /my_folder/netcdf_valid/netcdf_valid.nc # location of another file after aggregation is created: /my_folder/netcdf_invalid.nc self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the nc file to the resource at the above folder self.add_file_to_resource(file_to_add=self.netcdf_file, upload_folder=new_folder) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) # check that there is no NetCDFLogicalFile object self.assertEqual(NetCDFLogicalFile.objects.count(), 0) # add another file to the same folder self.add_file_to_resource(file_to_add=self.netcdf_invalid_file, upload_folder=new_folder) self.assertEqual(self.composite_resource.files.all().count(), 2) # set the nc file to NetCDF file type NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(self.composite_resource.files.all().count(), 3) # test logical file/aggregation self.assertEqual(len(self.composite_resource.logical_files), 1) logical_file = self.composite_resource.logical_files[0] self.assertEqual(logical_file.files.count(), 2) base_nc_file_name, _ = os.path.splitext(self.netcdf_file_name) expected_file_folder = '{0}/{1}'.format(new_folder, base_nc_file_name) for res_file in logical_file.files.all(): self.assertEqual(res_file.file_folder, expected_file_folder) self.assertTrue(isinstance(logical_file, NetCDFLogicalFile)) self.assertTrue(logical_file.metadata, NetCDFLogicalFile) # test the location of the file that's not part of the netcdf aggregation other_res_file = None for res_file in self.composite_resource.files.all(): if not res_file.has_logical_file: other_res_file = res_file break self.assertEqual(other_res_file.file_folder, new_folder) self.composite_resource.delete()
def test_aggregation_folder_move(self): # test changes to aggregation name, aggregation metadata xml file path, and aggregation # resource map xml file path on aggregation folder move self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) aggregation_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(self.composite_resource.files.count(), 2) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, aggregation_folder_name) # create a folder to move the aggregation folder there parent_folder = 'parent_folder' ResourceFile.create_folder(self.composite_resource, parent_folder) # move the aggregation folder to the parent folder src_path = 'data/contents/{}'.format(aggregation_folder_name) tgt_path = 'data/contents/{0}/{1}'.format(parent_folder, aggregation_folder_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) file_folder = '{0}/{1}'.format(parent_folder, aggregation_folder_name) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, file_folder) # test aggregation name update res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_folder) # test aggregation xml file paths expected_meta_file_path = '{0}/{1}/{2}_meta.xml'.format(parent_folder, aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}/{1}/{2}_resmap.xml'.format(parent_folder, aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) self.composite_resource.delete()
def test_create_aggregation_from_zip_file_required_2(self): # here we are using a zip file that has only the 3 required files for setting it # to Geo Feature file type which includes metadata extraction # the zip file that we are using to create an aggregation here is not at the root of the # folder hierarchy but in a folder - no new folder should be created as part of creating # this aggregation self.create_composite_resource() new_folder = 'geofeature_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the zip file to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_folder, new_folder) # check that the resource file is not associated with any logical file type self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) res_file = self.composite_resource.files.first() self.assertEqual(res_file.extension, '.zip') # set the zip file to GeoFeatureLogicalFile type GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test file type and file type metadata assert_geofeature_file_type_metadata(self, new_folder) for res_file in self.composite_resource.files.all(): # test that each resource file is part of an aggregation (logical file) self.assertTrue(res_file.has_logical_file) # test that the each resource file has the same folder - no new folder was created self.assertEqual(res_file.file_folder, new_folder) # there should not be any file level keywords res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.keywords, []) self.composite_resource.delete() # there should be no GeoFeatureLogicalFile object at this point self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) # there should be no GeoFeatureFileMetaData object at this point self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_create_aggregation_2(self): # here we are using a valid time series json file for setting it # to RefTimeSeries file type which includes metadata extraction # this resource file is in a folder self.res_title = "Untitled resource" self.create_composite_resource() new_folder = 'refts_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the json file to the resource at the above folder self.add_file_to_resource(file_to_add=self.refts_file, upload_folder=new_folder) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # test resource file is in a folder self.assertEqual(res_file.file_folder, new_folder) # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) # check that there is no RefTimeseriesLogicalFile object self.assertEqual(RefTimeseriesLogicalFile.objects.count(), 0) # set the json file to RefTimeseries file type RefTimeseriesLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() # check that there is one RefTimeseriesLogicalFile object self.assertEqual(RefTimeseriesLogicalFile.objects.count(), 1) # test resource file is in the same folder self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) # test extracted ref time series file type metadata assert_ref_time_series_file_type_metadata(self) # test that the content of the json file is same is what we have # saved in json_file_content field of the file metadata object res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.json_file_content, res_file.resource_file.read()) # test resource file is in a folder self.assertEqual(res_file.file_folder, new_folder) self.composite_resource.delete()
def test_aggregation_folder_sub_folder_not_allowed(self): # test a folder can't be created inside a folder that represents an aggregation self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_folder, None) # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertNotEqual(res_file.file_folder, None) # create a folder inside the aggregation folder new_folder = '{}/sub_folder'.format(res_file.file_folder) with self.assertRaises(DRF_ValidationError): ResourceFile.create_folder(self.composite_resource, new_folder) self.composite_resource.delete()
def test_create_aggregation_from_shp_file_required_2(self): # here we are using a shp file that exists in a folder # for setting it to Geo Feature file type which includes metadata extraction # no new folder should be created as part o creating this aggregation self.create_composite_resource() new_folder = 'geofeature_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder res_file = self.add_file_to_resource(file_to_add=self.states_shp_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) res_file = self.add_file_to_resource(file_to_add=self.states_shx_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file, upload_folder=new_folder) self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(self.composite_resource.files.all().count(), 3) res_file = self.composite_resource.files.first() # check that the resource file is not associated with any logical file self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) # set the shp file to GeoFeatureLogicalFile type shp_res_file = [ f for f in self.composite_resource.files.all() if f.extension == '.shp' ][0] GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, shp_res_file.id) self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1) for res_file in self.composite_resource.files.all(): # test that each resource file is part of an aggregation (logical file) self.assertTrue(res_file.has_logical_file) # test that the each resource file has the same folder - no new folder created self.assertEqual(res_file.file_folder, new_folder) self.composite_resource.delete()
def test_aggregation_file_rename(self): # test that a file can't renamed for any resource file # that's part of the GeoFeature logical file self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) # create aggregation from the zip file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with aggregation raises exception self.assertEqual(self.composite_resource.files.count(), 3) src_path = 'data/contents/{}/states.shp'.format(new_folder) tgt_path = 'data/contents/{}/states-1.shp'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/{}/states.dbf'.format(new_folder) tgt_path = 'data/contents/{}/states-1.dbf'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/{}/states.shx'.format(new_folder) tgt_path = 'data/contents/{}/states-1.shx'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def setUp(self): super(TestFolderDownloadZip, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = create_account('*****@*****.**', username='******', first_name='Shaun', last_name='Livingston', superuser=False, groups=[]) self.res = create_resource(resource_type='CompositeResource', owner=self.user, title='Test Resource', metadata=[]) ResourceFile.create_folder(self.res, 'foo') # create files self.n1 = "test1.txt" test_file = open(self.n1, 'w') test_file.write("Test text file in test1.txt") test_file.close() self.test_file = open(self.n1, "rb") add_resource_files(self.res.short_id, self.test_file, folder='foo') # copy refts file into new file to be added to the resource as an aggregation reft_data_file = open( 'hs_core/tests/data/multi_sites_formatted_version1.0.refts.json', 'rb') refts_file = open('multi_sites_formatted_version1.0.refts.json', 'wb') refts_file.writelines(reft_data_file.readlines()) refts_file.close() self.refts_file = open('multi_sites_formatted_version1.0.refts.json', 'rb') add_resource_files(self.res.short_id, self.refts_file) self.res.create_aggregation_xml_documents() self.istorage = IrodsStorage()
def test_aggregation_folder_move_not_allowed(self): # test a folder is not allowed to be moved into a folder that represents an aggregation self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) aggregation_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # create a folder to move the aggregation folder there new_folder = 'folder_to_move' ResourceFile.create_folder(self.composite_resource, new_folder) # move the new folder into the aggregation folder src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(aggregation_folder_name) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_fileset_create_delete(self): """Test that we can create a fileset aggregation from a folder that contains one file and delete the aggregation through the api""" self.create_composite_resource() new_folder = 'fileset_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the txt file to the resource at the above folder self.add_file_to_resource(file_to_add=self.generic_file, upload_folder=new_folder) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # file has a folder self.assertEqual(res_file.file_folder, new_folder) # check that the resource file is not part of an aggregation self.assertEqual(res_file.has_logical_file, False) self.assertEqual(FileSetLogicalFile.objects.count(), 0) # set folder to fileset logical file type (aggregation) set_type_url = reverse('set_file_type_public', kwargs={"pk": self.composite_resource.short_id, "file_path": "", "hs_file_type": "FileSet"}) self.client.post(set_type_url, data={"folder_path": new_folder}) res_file = self.composite_resource.files.first() # file has the same folder self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) self.assertEqual(FileSetLogicalFile.objects.count(), 1) # aggregation dataset name should be same as the folder name self.assertEqual(res_file.logical_file.dataset_name, new_folder) delete_agg_url = reverse('delete_aggregation_public', kwargs={"resource_id": self.composite_resource.short_id, "file_path": new_folder, "hs_file_type": "FileSetLogicalFile"}) self.client.delete(delete_agg_url) self.assertEqual(FileSetLogicalFile.objects.count(), 0) self.assertEqual(self.composite_resource.files.all().count(), 0) self.composite_resource.delete()
def test_geofeature_file_type_folder_delete(self): # when a file is set to geofeaturelogical file type # system automatically creates a folder using the name of the file # that was used to set the file type # Here we need to test that when that folder gets deleted, all files # in that folder gets deleted, the logicalfile object gets deleted and # the associated metadata objects get deleted self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) # extract metadata from the zip file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test that we have one logical file of type GeoFeatureLogicalFile type as a result # of metadata extraction self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1) # should have one GeoFeatureFileMetadata object self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1) # there should be 3 content files self.assertEqual(self.composite_resource.files.count(), 3) # delete the folder for the logical file folder_path = "data/contents/{}".format(new_folder) remove_folder(self.user, self.composite_resource.short_id, folder_path) # there should be no content files self.assertEqual(self.composite_resource.files.count(), 0) # there should not be any logical file or file metadata object as a result # of folder deletion self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0) self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0) self.composite_resource.delete()
def setUp(self): super(TestFolderDownloadZip, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = create_account( '*****@*****.**', username='******', first_name='Shaun', last_name='Livingston', superuser=False, groups=[] ) self.res = create_resource(resource_type='CompositeResource', owner=self.user, title='Test Resource', metadata=[]) ResourceFile.create_folder(self.res, 'foo') # create files self.n1 = "test1.txt" test_file = open(self.n1, 'w') test_file.write("Test text file in test1.txt") test_file.close() self.test_file = open(self.n1, "r") add_resource_files(self.res.short_id, self.test_file, folder='foo') # copy refts file into new file to be added to the resource as an aggregation reft_data_file = open('hs_core/tests/data/multi_sites_formatted_version1.0.refts.json', 'r') refts_file = open('multi_sites_formatted_version1.0.refts.json', 'w') refts_file.writelines(reft_data_file.readlines()) refts_file.close() self.refts_file = open('multi_sites_formatted_version1.0.refts.json', 'r') add_resource_files(self.res.short_id, self.refts_file) self.res.create_aggregation_xml_documents()
def test_unfederated_folder_path_checks(self): """ an unfederated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # should succeed without errors check_irods_files(self.res, stop_on_error=True) # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # should succeed without errors check_irods_files(self.res, stop_on_error=True) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live fullpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, fullpath) # now try to intentionally corrupt it resfile.set_short_path("fuzz.txt") # should raise exception with self.assertRaises(ValidationError): check_irods_files(self.res, stop_on_error=True) # now don't raise exception and read error errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS')) self.assertTrue(errors[1].endswith( 'data/contents/foo/file1.txt in iRODs does not exist in Django')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # now try to clean it up errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False, clean_irods=True, clean_django=True) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)')) self.assertTrue(errors[1].endswith( 'data/contents/foo/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # now check should succeed errors, ecount = check_irods_files(self.res, stop_on_error=True, log_errors=False) self.assertEqual(ecount, 0) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_federated_folder_path_logic(self): """ a federated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] self.assertEqual(resfile.resource_file.name, os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt")) self.assertEqual(resfile.file_folder, "foo") # cheat: set a fake federated path to test path logic fedpath = "/myzone/home/myuser" self.res.resource_federation_path = fedpath self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path('foo/file1.txt', test_exists=False) # determine where that file should live shortpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(shortpath, resfile.storage_path) # this should result in an exact path resfile.set_storage_path(shortpath, test_exists=False) self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False)) # non-existent files should raise error otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file2.txt") resfile.path_is_acceptable(otherpath, test_exists=False) # This won't work because federation path is fake. # with self.assertRaises(ValidationError): # resfile.path_is_acceptable(otherpath, test_exists=True) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt", test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath, test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object resfile.set_storage_path(otherpath, test_exists=False) # conclusion: unfederate the resource self.res.resource_federation_path = "" self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path("foo/file1.txt", test_exists=False) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_aggregation_metadata_CRUD(self): # here we are using a valid nc file for creating a NetCDF file type (aggregation) # then testing with metadata CRUD actions for the aggregation self.create_composite_resource() new_folder = 'nc_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the nc file to the resource at the above folder self.add_file_to_resource(file_to_add=self.netcdf_file, upload_folder=new_folder) # make the netcdf file part of the NetCDFLogicalFile res_file = self.composite_resource.files.first() self.assertEqual(NetCDFFileMetaData.objects.count(), 0) netcdf_logical_file = NetCDFLogicalFile.create(self.composite_resource) netcdf_logical_file.save() self.assertEqual(NetCDFFileMetaData.objects.count(), 1) netcdf_logical_file.add_resource_file(res_file) res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, 'NetCDFLogicalFile') self.assertEqual(netcdf_logical_file.files.count(), 1) # create keywords - note it is possible to have duplicate keywords # appropriate view functions need to disallow duplicate keywords keywords = ['key-1', 'key-1', 'key-2'] netcdf_logical_file.metadata.keywords = keywords netcdf_logical_file.metadata.save() self.assertEqual(len(keywords), len(netcdf_logical_file.metadata.keywords)) for keyword in keywords: self.assertIn(keyword, netcdf_logical_file.metadata.keywords) # create OriginalCoverage element self.assertEqual(netcdf_logical_file.metadata.original_coverage, None) coverage_data = {'northlimit': 121.345, 'southlimit': 42.678, 'eastlimit': 123.789, 'westlimit': 40.789, 'units': 'meters'} netcdf_logical_file.metadata.create_element('OriginalCoverage', value=coverage_data) self.assertNotEqual(netcdf_logical_file.metadata.original_coverage, None) self.assertEqual(float(netcdf_logical_file.metadata.original_coverage.value['northlimit']), 121.345) # test updating OriginalCoverage element orig_coverage = netcdf_logical_file.metadata.original_coverage coverage_data = {'northlimit': 111.333, 'southlimit': 42.678, 'eastlimit': 123.789, 'westlimit': 40.789, 'units': 'meters'} netcdf_logical_file.metadata.update_element('OriginalCoverage', orig_coverage.id, value=coverage_data) self.assertEqual(float(netcdf_logical_file.metadata.original_coverage.value['northlimit']), 111.333) # trying to create a 2nd OriginalCoverage element should raise exception with self.assertRaises(Exception): netcdf_logical_file.metadata.create_element('OriginalCoverage', value=coverage_data) # trying to update bounding box values with non-numeric values # (e.g., 'north_limit' key with a non-numeric value) should raise exception coverage_data = {'northlimit': '121.345a', 'southlimit': 42.678, 'eastlimit': 123.789, 'westlimit': 40.789, 'units': 'meters'} with self.assertRaises(ValidationError): netcdf_logical_file.metadata.update_element('OriginalCoverage', orig_coverage.id, value=coverage_data) # test creating spatial coverage # there should not be any spatial coverage for the netcdf file type self.assertEqual(netcdf_logical_file.metadata.spatial_coverage, None) coverage_data = {'projection': 'WGS 84 EPSG:4326', 'northlimit': 41.87, 'southlimit': 41.863, 'eastlimit': -111.505, 'westlimit': -111.511, 'units': 'meters'} # create spatial coverage netcdf_logical_file.metadata.create_element('Coverage', type="box", value=coverage_data) spatial_coverage = netcdf_logical_file.metadata.spatial_coverage self.assertEqual(float(spatial_coverage.value['northlimit']), 41.87) # test updating spatial coverage coverage_data = {'projection': 'WGS 84 EPSG:4326', 'northlimit': 41.87706, 'southlimit': 41.863, 'eastlimit': -111.505, 'westlimit': -111.511, 'units': 'meters'} netcdf_logical_file.metadata.update_element('Coverage', element_id=spatial_coverage.id, type="box", value=coverage_data) spatial_coverage = netcdf_logical_file.metadata.spatial_coverage self.assertEqual(float(spatial_coverage.value['northlimit']), 41.87706) # create Variable element self.assertEqual(netcdf_logical_file.metadata.variables.count(), 0) variable_data = {'name': 'variable_name', 'type': 'Int', 'unit': 'deg F', 'shape': 'variable_shape'} netcdf_logical_file.metadata.create_element('Variable', **variable_data) self.assertEqual(netcdf_logical_file.metadata.variables.count(), 1) self.assertEqual(netcdf_logical_file.metadata.variables.first().name, 'variable_name') # test that multiple Variable elements can be created variable_data = {'name': 'variable_name_2', 'type': 'Int', 'unit': 'deg F', 'shape': 'variable_shape_2'} netcdf_logical_file.metadata.create_element('Variable', **variable_data) self.assertEqual(netcdf_logical_file.metadata.variables.count(), 2) # test update Variable element variable = netcdf_logical_file.metadata.variables.first() variable_data = {'name': 'variable_name_updated', 'type': 'Int', 'unit': 'deg F', 'shape': 'variable_shape'} netcdf_logical_file.metadata.update_element('Variable', variable.id, **variable_data) variable = netcdf_logical_file.metadata.variables.get(id=variable.id) self.assertEqual(variable.name, 'variable_name_updated') self.composite_resource.delete()
def test_federated_folder_path_logic(self): """ a federated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] self.assertEqual(resfile.resource_file.name, os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt")) self.assertEqual(resfile.file_folder, "foo") # cheat: set a fake federated path to test path logic fedpath = "/myzone/home/myuser" self.res.resource_federation_path = fedpath self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path('foo/file1.txt', test_exists=False) # determine where that file should live shortpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(shortpath, resfile.storage_path) # this should result in an exact path resfile.set_storage_path(shortpath, test_exists=False) self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False)) # non-existent files should raise error otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file2.txt") resfile.path_is_acceptable(otherpath, test_exists=False) # This won't work because federation path is fake. # with self.assertRaises(ValidationError): # resfile.path_is_acceptable(otherpath, test_exists=True) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt", test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath, test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object resfile.set_storage_path(otherpath, test_exists=False) # conclusion: unfederate the resource self.res.resource_federation_path = "" self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path("foo/file1.txt", test_exists=False) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_unfederated_folder_path_checks(self): """ an unfederated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # should succeed without errors self.res.check_irods_files(stop_on_error=True) # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # should succeed without errors self.res.check_irods_files(stop_on_error=True) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live fullpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, fullpath) # now try to intentionally corrupt it resfile.set_short_path("fuzz.txt") # should raise exception with self.assertRaises(ValidationError): self.res.check_irods_files(stop_on_error=True) # now don't raise exception and read error errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS')) self.assertTrue(errors[1].endswith( 'data/contents/foo/file1.txt in iRODs does not exist in Django')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # now try to clean it up errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False, clean_irods=True, clean_django=True) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)')) self.assertTrue(errors[1].endswith( 'data/contents/foo/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # now check should succeed errors, ecount = self.res.check_irods_files(stop_on_error=True, log_errors=False) self.assertEqual(ecount, 0) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_aggregation_folder_move(self): # test changes to aggregation name, aggregation metadata xml file path, and aggregation # resource map xml file path on aggregation folder move self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) res_file = self.composite_resource.files.first() # create aggregation from the zip file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(self.composite_resource.files.count(), 3) base_file_name, _ = os.path.splitext(res_file.file_name) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, new_folder) # test aggregation name res_file = self.composite_resource.files.first() logical_file = res_file.logical_file shp_file_path = get_path_with_no_file_extension( logical_file.aggregation_name) # test aggregation xml file paths expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) # create a folder to move the aggregation folder there parent_folder = 'parent_folder' ResourceFile.create_folder(self.composite_resource, parent_folder) # move the aggregation folder to the parent folder src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}/{}'.format(parent_folder, new_folder) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) file_folder = '{}/{}'.format(parent_folder, new_folder) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, file_folder) # test aggregation name update res_file = self.composite_resource.files.first() logical_file = res_file.logical_file shp_file_path = get_path_with_no_file_extension( logical_file.aggregation_name) # test aggregation xml file paths expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) self.composite_resource.delete()