def test_create_aggregation_from_zip_file_required_1(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
# the zip file that we are using to create an aggregation here is at the root of the
# folder hierarchy
self.create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, expected_folder_name='')
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GeoFeatureFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_file_metadata_on_resource_delete(self):
# test that when the composite resource is deleted
# all metadata associated with GeoFeatureLogicalFile Type is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file of type GeoFeatureLogicalFile as a result
# of metadata extraction
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# test that we have the metadata elements
# there should be no Coverage objects
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(FieldInformation.objects.count(), 5)
# delete resource
hydroshare.delete_resource(self.composite_resource.short_id)
# test that we have no logical file of type GeoFeatureLogicalFileType
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(FieldInformation.objects.count(), 0)
def test_zip_invalid_set_file_type_to_geo_feature(self):
# here we are using a invalid zip file that is missing the shx file
# to set Geo Feature file type which should fail
self.create_composite_resource(self.states_zip_invalid_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
# set the tif file to GeoFeatureFile type
with self.assertRaises(ValidationError):
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
self.user, res_file.id)
# test file was rolled back
self.assertEqual(self.composite_resource.files.count(), 1)
# check that there is no GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.composite_resource.delete()
def test_aggregation_file_move(self):
# test any resource file that's part of the GeoFeature logical file can't be moved
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test renaming of files that are associated with geo feature LFO - which should
# raise exception
self.assertEqual(self.composite_resource.files.count(), 3)
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# moving any of the resource files to this new folder should raise exception
tgt_path = 'data/contents/geofeature_aggr'
for res_file in self.composite_resource.files.all():
with self.assertRaises(DRF_ValidationError):
src_path = os.path.join('data', 'contents',
res_file.short_path)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_geofeature_file_type_folder_delete(self):
# when a file is set to geofeaturelogical file type
# system automatically creates a folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self._create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoFeatureLogicalFile type as a result
# of metadata extraction
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# should have one GeoFeatureFileMetadata object
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# there should be 3 content files
self.assertEqual(self.composite_resource.files.count(), 3)
# delete the folder for the logical file
folder_path = "data/contents/states_required_files"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should be no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any logical file or file metadata object as a result
# of folder deletion
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.composite_resource.delete()
def test_remove_aggregation(self):
# test that when an instance GeoFeatureLogicalFile (aggregation) is deleted
# all resource files associated with that aggregation is not deleted but the associated
# metadata is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
file_folder = res_file.file_folder
base_file_name, _ = os.path.splitext(res_file.file_name)
# set the zip file to GeoFeatureLogicalFile (aggregation) type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file (aggregation) of type GeoFeatureLogicalFile
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(self.composite_resource.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the aggregation (logical file) object using the remove_aggregation function
logical_file.remove_aggregation()
# test there is no GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# test there is no GeoFeatureFileMetaData object
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# check the files associated with the aggregation not deleted
self.assertEqual(self.composite_resource.files.all().count(), 3)
# check the file folder is not deleted
for f in self.composite_resource.files.all():
self.assertEqual(f.file_folder, file_folder)
self.composite_resource.delete()
def test_logical_file_delete(self):
# test that when an instance GeoFeatureLogicalFile Type is deleted
# all files associated with GeoFeatureLogicalFile is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(self.composite_resource.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the logical file using the custom delete function - logical_delete()
logical_file.logical_delete(self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.composite_resource.delete()
def test_create_aggregation_from_shp_file_required_3(self):
# here we are using a shp file that exists in a folder
# for setting it to Geo Feature file type which includes metadata extraction
# The same folder contains another file that is not going to be part of the
# geofeature aggregation a new folder should be created in this case to represent the
# geofeature aggregation
# location shp file before aggregation is created: my_folder/states.shp
# location of another file before aggregation is created: my_folder/states_invalid.zip
# location of shp file after aggregation is created: my_folder/states/states.shp
# location of another file after aggregation is created: my_folder/states_invalid.zip
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 3)
# add a file that is not related to aggregation
res_file = self.add_file_to_resource(
file_to_add=self.states_zip_invalid_file, upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 4)
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
base_shp_file_base_name, _ = os.path.splitext(shp_res_file.file_name)
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
logical_file = shp_res_file.logical_file
self.assertEqual(logical_file.files.count(), 3)
for res_file in logical_file.files.all():
# test that the each resource file has the same folder - no new folder created
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 4)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_feature(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(GeoFeatureFileTypeMetaDataTest,
self).is_federated_irods_available():
return
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
# set the logical file -which get sets as part of the post resource creation signal
resource_post_create_actions(resource=self.composite_resource,
user=self.user,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
fed_file_path = "{}/data/contents/{}".format(
self.composite_resource.root_path, self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def _test_file_metadata_on_file_delete(self, ext):
self.create_composite_resource(self.osm_all_files_zip_file)
res_file = self.composite_resource.files.first()
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
# check that there is one GeoFeatureFileMetaData
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# one at the file level and one at the resource level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(FieldInformation.objects.count(), 7)
self.assertEqual(GeometryInformation.objects.count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(len(logical_file.metadata.keywords), 2)
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
# delete content file specified by extension (ext parameter)
res_file = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file.id, self.user)
# test that we don't have logical file of type GeoFeatureLogicalFile type
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# test that all metadata got deleted - there should be still1 coverage at the resource level
self.assertEqual(
self.composite_resource.metadata.coverages.all().count(), 1)
self.assertEqual(Coverage.objects.count(), 1)
self.assertEqual(FieldInformation.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
# there should not be any files left
self.assertEqual(self.composite_resource.files.count(), 0)
self.composite_resource.delete()
def test_create_aggregation_from_zip_file_required_2(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
# the zip file that we are using to create an aggregation here is not at the root of the
# folder hierarchy but in a folder - no new folder should be created as part of creating
# this aggregation
self.create_composite_resource()
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the the zip file to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.file_folder, new_folder)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, new_folder)
for res_file in self.composite_resource.files.all():
# test that each resource file is part of an aggregation (logical file)
self.assertTrue(res_file.has_logical_file)
# test that the each resource file has the same folder - no new folder was created
self.assertEqual(res_file.file_folder, new_folder)
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GeoFeatureFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_main_file(self):
self.create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
self.assertEqual(1, GeoFeatureLogicalFile.objects.count())
self.assertEqual(
".shp",
GeoFeatureLogicalFile.objects.first().get_main_file_type())
self.assertEqual(
self.states_shp_file_name,
GeoFeatureLogicalFile.objects.first().get_main_file.file_name)
def test_zip_set_file_type_to_geo_feature(self):
super(GeoFeatureFileTypeMetaDataTest,
self).assert_federated_irods_available()
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path,
self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_file_rename_or_move(self):
# test that file can't be moved or renamed for any resource file
# that's part of the GeoFeature logical file object (LFO)
self._create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test renaming of files that are associated with raster LFO - which should raise exception
self.assertEqual(self.composite_resource.files.count(), 3)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.file_folder, 'states_required_files')
src_path = 'data/contents/states_required_files/states.shp'
tgt_path = 'data/contents/states_required_files/states-1.shp'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/states_required_files/states.dbf'
tgt_path = 'data/contents/states_required_files/states-1.dbf'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
# test moving the files associated with geo raster LFO
src_path = 'data/contents/states_required_files/states.shx'
tgt_path = 'data/contents/states_required_files/states-1.shx'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/states_required_files/states.shp'
tgt_path = 'data/contents/states_required_files-1/states.shp'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_feature(self):
super(GeoFeatureFileTypeMetaDataTest, self).assert_federated_irods_available()
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[]
)
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path, '')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path, self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_create_aggregation_from_shp_file_required_2(self):
# here we are using a shp file that exists in a folder
# for setting it to Geo Feature file type which includes metadata extraction
# no new folder should be created as part o creating this aggregation
self.create_composite_resource()
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
for res_file in self.composite_resource.files.all():
# test that each resource file is part of an aggregation (logical file)
self.assertTrue(res_file.has_logical_file)
# test that the each resource file has the same folder - no new folder created
self.assertEqual(res_file.file_folder, new_folder)
self.composite_resource.delete()
def test_aggregation_file_rename(self):
# test that a file can't renamed for any resource file
# that's part of the GeoFeature logical file
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# create aggregation from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test renaming of files that are associated with aggregation raises exception
self.assertEqual(self.composite_resource.files.count(), 3)
src_path = 'data/contents/{}/states.shp'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.shp'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/{}/states.dbf'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.dbf'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/{}/states.shx'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.shx'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_feature_required(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
logical_file = res_file.logical_file
self.assertTrue(isinstance(logical_file.metadata, GenericFileMetaData))
self.assertTrue(isinstance(logical_file, GenericLogicalFile))
# no keywords at this point for file
self.assertEqual(logical_file.metadata.keywords, [])
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_zip_set_file_type_to_geo_feature_all(self):
# here we are using a zip file that has all the 15 files for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource(self.osm_all_files_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is no GenericFileMetaData object
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# check that we put the 3 files in a new folder
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/{}/{}"
res_file.file_folder = expected_folder_name
expected_file_path = expected_file_path.format(
self.composite_resource.root_path, expected_folder_name,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# test extracted raster file type metadata
# there should one resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 1)
self.assertEqual(logical_file.metadata.fieldinformations.all().count(),
7)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 87)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType, "POLYGON")
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'WGS_1984')
self.assertTrue(
abs(logical_file.metadata.originalcoverage.eastlimit -
3.4520493) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.northlimit -
45.0466382) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.southlimit -
42.5732416) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.westlimit -
(-0.3263017)) < self.allowance)
self.assertEqual(logical_file.metadata.originalcoverage.unit, 'Degree')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name,
'GCS_WGS_1984')
# there should be file level keywords
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_create_aggregation_from_zip_file_all(self):
# here we are using a zip file that has all the 15 files for setting it
# to Geo Feature file type which includes metadata extraction
# this zip file for creating aggregation is at the root of the folder hierarchy
self.create_composite_resource(self.osm_all_files_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is no GenericFileMetaData object
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# test extracted raster file type metadata
# there should one resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 1)
self.assertEqual(logical_file.metadata.fieldinformations.all().count(),
7)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 87)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType, "POLYGON")
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'WGS_1984')
self.assertTrue(
abs(logical_file.metadata.originalcoverage.eastlimit -
3.4520493) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.northlimit -
45.0466382) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.southlimit -
42.5732416) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.westlimit -
(-0.3263017)) < self.allowance)
self.assertEqual(logical_file.metadata.originalcoverage.unit, 'degree')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'WGS 84')
# there should be file level keywords
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_create_aggregation_from_shp_file_required_1(self):
# here we are using a shp file that exists at the root of the folder hierarchy
# for setting it to Geo Feature file type which includes metadata extraction
self.create_composite_resource()
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file)
self.assertEqual(res_file.file_folder, '')
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file)
self.assertEqual(res_file.file_folder, '')
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file)
self.assertEqual(res_file.file_folder, '')
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 3)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 3)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# test extracted raster file type metadata
# there should not be any resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 0)
self.assertNotEqual(logical_file.metadata.geometryinformation, None)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 51)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType,
"MULTIPOLYGON")
self.assertNotEqual(logical_file.metadata.originalcoverage, None)
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'unknown')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'unknown')
self.assertGreater(
len(logical_file.metadata.originalcoverage.projection_string), 0)
self.assertEqual(logical_file.metadata.originalcoverage.unit,
'unknown')
self.assertEqual(logical_file.metadata.originalcoverage.eastlimit,
-66.9692712587578)
self.assertEqual(logical_file.metadata.originalcoverage.northlimit,
71.406235393967)
self.assertEqual(logical_file.metadata.originalcoverage.southlimit,
18.921786345087)
self.assertEqual(logical_file.metadata.originalcoverage.westlimit,
-178.217598362366)
# there should not be any file level keywords
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_aggregation_folder_move(self):
# test changes to aggregation name, aggregation metadata xml file path, and aggregation
# resource map xml file path on aggregation folder move
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
res_file = self.composite_resource.files.first()
# create aggregation from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
self.assertEqual(self.composite_resource.files.count(), 3)
base_file_name, _ = os.path.splitext(res_file.file_name)
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.file_folder, new_folder)
# test aggregation name
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
shp_file_path = get_path_with_no_file_extension(
logical_file.aggregation_name)
# test aggregation xml file paths
expected_meta_file_path = '{0}{1}'.format(shp_file_path,
METADATA_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_file_path)
expected_map_file_path = '{0}{1}'.format(shp_file_path,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.map_short_file_path,
expected_map_file_path)
# create a folder to move the aggregation folder there
parent_folder = 'parent_folder'
ResourceFile.create_folder(self.composite_resource, parent_folder)
# move the aggregation folder to the parent folder
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}/{}'.format(parent_folder, new_folder)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
file_folder = '{}/{}'.format(parent_folder, new_folder)
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.file_folder, file_folder)
# test aggregation name update
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
shp_file_path = get_path_with_no_file_extension(
logical_file.aggregation_name)
# test aggregation xml file paths
expected_meta_file_path = '{0}{1}'.format(shp_file_path,
METADATA_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_file_path)
expected_map_file_path = '{0}{1}'.format(shp_file_path,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.map_short_file_path,
expected_map_file_path)
self.composite_resource.delete()
def test_shp_set_file_type_to_geo_feature_required(self):
# here we are using a shp file for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource()
# add the 3 required files to the resource
files = []
shp_temp_file = os.path.join(self.temp_dir, self.states_shp_file_name)
shutil.copy(self.states_shp_file, shp_temp_file)
shx_temp_file = os.path.join(self.temp_dir, self.states_shx_file_name)
shutil.copy(self.states_shx_file, shx_temp_file)
dbf_temp_file = os.path.join(self.temp_dir, self.states_dbf_file_name)
shutil.copy(self.states_dbf_file, dbf_temp_file)
files.append(
UploadedFile(file=open(shp_temp_file, 'r'),
name=self.states_shp_file_name))
files.append(
UploadedFile(file=open(shx_temp_file, 'r'),
name=self.states_shx_file_name))
files.append(
UploadedFile(file=open(dbf_temp_file, 'r'),
name=self.states_dbf_file_name))
hydroshare.utils.resource_file_add_process(self.composite_resource,
files, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is 3 GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 3)
# set the shp file to GeoFeatureFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
shp_res_file.id, self.user)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 3)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 3)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# check that we put the 3 files in a new folder
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/{}/{}"
res_file.file_folder = expected_folder_name
expected_file_path = expected_file_path.format(
self.composite_resource.root_path, expected_folder_name,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# test extracted raster file type metadata
# there should not be any resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 0)
self.assertNotEqual(logical_file.metadata.geometryinformation, None)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 51)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType,
"MULTIPOLYGON")
self.assertNotEqual(logical_file.metadata.originalcoverage, None)
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'unknown')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'unknown')
self.assertGreater(
len(logical_file.metadata.originalcoverage.projection_string), 0)
self.assertEqual(logical_file.metadata.originalcoverage.unit,
'unknown')
self.assertEqual(logical_file.metadata.originalcoverage.eastlimit,
-66.9692712587578)
self.assertEqual(logical_file.metadata.originalcoverage.northlimit,
71.406235393967)
self.assertEqual(logical_file.metadata.originalcoverage.southlimit,
18.921786345087)
self.assertEqual(logical_file.metadata.originalcoverage.westlimit,
-178.217598362366)
# there should not be any file level keywords
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
