def test_remove_aggregation(self):
# test that when an instance GeoFeatureLogicalFile (aggregation) is deleted
# all resource files associated with that aggregation is not deleted but the associated
# metadata is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
file_folder = res_file.file_folder
base_file_name, _ = os.path.splitext(res_file.file_name)
# set the zip file to GeoFeatureLogicalFile (aggregation) type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file (aggregation) of type GeoFeatureLogicalFile
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(self.composite_resource.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the aggregation (logical file) object using the remove_aggregation function
logical_file.remove_aggregation()
# test there is no GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# test there is no GeoFeatureFileMetaData object
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# check the files associated with the aggregation not deleted
self.assertEqual(self.composite_resource.files.all().count(), 3)
# check the file folder is not deleted
for f in self.composite_resource.files.all():
self.assertEqual(f.file_folder, file_folder)
self.composite_resource.delete()
def test_create_aggregation_from_zip_file_required_1(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
# the zip file that we are using to create an aggregation here is at the root of the
# folder hierarchy
self.create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, expected_folder_name='')
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GeoFeatureFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_geofeature_file_type_folder_delete(self):
# when a file is set to geofeaturelogical file type
# system automatically creates a folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self._create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoFeatureLogicalFile type as a result
# of metadata extraction
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# should have one GeoFeatureFileMetadata object
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# there should be 3 content files
self.assertEqual(self.composite_resource.files.count(), 3)
# delete the folder for the logical file
folder_path = "data/contents/states_required_files"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should be no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any logical file or file metadata object as a result
# of folder deletion
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.composite_resource.delete()
def test_aggregation_file_move(self):
# test any resource file that's part of the GeoFeature logical file can't be moved
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test renaming of files that are associated with geo feature LFO - which should
# raise exception
self.assertEqual(self.composite_resource.files.count(), 3)
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# moving any of the resource files to this new folder should raise exception
tgt_path = 'data/contents/geofeature_aggr'
for res_file in self.composite_resource.files.all():
with self.assertRaises(DRF_ValidationError):
src_path = os.path.join('data', 'contents',
res_file.short_path)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_file_metadata_on_resource_delete(self):
# test that when the composite resource is deleted
# all metadata associated with GeoFeatureLogicalFile Type is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file of type GeoFeatureLogicalFile as a result
# of metadata extraction
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# test that we have the metadata elements
# there should be no Coverage objects
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(FieldInformation.objects.count(), 5)
# delete resource
hydroshare.delete_resource(self.composite_resource.short_id)
# test that we have no logical file of type GeoFeatureLogicalFileType
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# test that all metadata deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(FieldInformation.objects.count(), 0)
def test_logical_file_delete(self):
# test that when an instance GeoFeatureLogicalFile Type is deleted
# all files associated with GeoFeatureLogicalFile is deleted
self.create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 3)
self.assertEqual(self.composite_resource.files.all().count(), 3)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the logical file using the custom delete function - logical_delete()
logical_file.logical_delete(self.user)
self.assertEqual(self.composite_resource.files.all().count(), 0)
self.composite_resource.delete()
def test_zip_invalid_set_file_type_to_geo_feature(self):
# here we are using a invalid zip file that is missing the shx file
# to set Geo Feature file type which should fail
self.create_composite_resource(self.states_zip_invalid_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
# set the tif file to GeoFeatureFile type
with self.assertRaises(ValidationError):
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
self.user, res_file.id)
# test file was rolled back
self.assertEqual(self.composite_resource.files.count(), 1)
# check that there is no GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.composite_resource.delete()
def test_create_aggregation_from_shp_file_required_3(self):
# here we are using a shp file that exists in a folder
# for setting it to Geo Feature file type which includes metadata extraction
# The same folder contains another file that is not going to be part of the
# geofeature aggregation a new folder should be created in this case to represent the
# geofeature aggregation
# location shp file before aggregation is created: my_folder/states.shp
# location of another file before aggregation is created: my_folder/states_invalid.zip
# location of shp file after aggregation is created: my_folder/states/states.shp
# location of another file after aggregation is created: my_folder/states_invalid.zip
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 3)
# add a file that is not related to aggregation
res_file = self.add_file_to_resource(
file_to_add=self.states_zip_invalid_file, upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 4)
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
base_shp_file_base_name, _ = os.path.splitext(shp_res_file.file_name)
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
logical_file = shp_res_file.logical_file
self.assertEqual(logical_file.files.count(), 3)
for res_file in logical_file.files.all():
# test that the each resource file has the same folder - no new folder created
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 4)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_feature(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(GeoFeatureFileTypeMetaDataTest,
self).is_federated_irods_available():
return
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
# set the logical file -which get sets as part of the post resource creation signal
resource_post_create_actions(resource=self.composite_resource,
user=self.user,
metadata=self.composite_resource.metadata)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
fed_file_path = "{}/data/contents/{}".format(
self.composite_resource.root_path, self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def _test_file_metadata_on_file_delete(self, ext):
self.create_composite_resource(self.osm_all_files_zip_file)
res_file = self.composite_resource.files.first()
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
# check that there is one GeoFeatureFileMetaData
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# one at the file level and one at the resource level
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(FieldInformation.objects.count(), 7)
self.assertEqual(GeometryInformation.objects.count(), 1)
self.assertEqual(OriginalCoverage.objects.count(), 1)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(len(logical_file.metadata.keywords), 2)
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
# delete content file specified by extension (ext parameter)
res_file = hydroshare.utils.get_resource_files_by_extension(
self.composite_resource, ext)[0]
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file.id, self.user)
# test that we don't have logical file of type GeoFeatureLogicalFile type
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
# test that all metadata got deleted - there should be still1 coverage at the resource level
self.assertEqual(
self.composite_resource.metadata.coverages.all().count(), 1)
self.assertEqual(Coverage.objects.count(), 1)
self.assertEqual(FieldInformation.objects.count(), 0)
self.assertEqual(GeometryInformation.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
# there should not be any files left
self.assertEqual(self.composite_resource.files.count(), 0)
self.composite_resource.delete()
def test_create_aggregation_from_zip_file_required_2(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
# the zip file that we are using to create an aggregation here is not at the root of the
# folder hierarchy but in a folder - no new folder should be created as part of creating
# this aggregation
self.create_composite_resource()
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the the zip file to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.file_folder, new_folder)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, new_folder)
for res_file in self.composite_resource.files.all():
# test that each resource file is part of an aggregation (logical file)
self.assertTrue(res_file.has_logical_file)
# test that the each resource file has the same folder - no new folder was created
self.assertEqual(res_file.file_folder, new_folder)
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GeoFeatureFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_content_file_delete(self):
# test that when any file in GeoFeatureLogicalFile type is deleted
# all metadata associated with GeoFeatureLogicalFile is deleted
# test for all 15 file extensions
for ext in GeoFeatureLogicalFile.get_allowed_storage_file_types():
self._test_file_metadata_on_file_delete(ext=ext)
def test_main_file(self):
self.create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
self.assertEqual(1, GeoFeatureLogicalFile.objects.count())
self.assertEqual(
".shp",
GeoFeatureLogicalFile.objects.first().get_main_file_type())
self.assertEqual(
self.states_shp_file_name,
GeoFeatureLogicalFile.objects.first().get_main_file.file_name)
def test_zip_set_file_type_to_geo_feature(self):
super(GeoFeatureFileTypeMetaDataTest,
self).assert_federated_irods_available()
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(
fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[])
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path,
'')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path,
self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_bag_ingestion(self):
from hs_core.views.utils import unzip_file
def normalize_metadata(metadata_str):
"""Prepares metadata string to match resource id and hydroshare url of original"""
return metadata_str\
.replace(current_site_url(), "http://www.hydroshare.org")\
.replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1")
# create empty resource
res = resource.create_resource(
'CompositeResource',
self.user,
'My Test Resource'
)
full_paths = {}
files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'),
name="test_resource_metadata_files.zip")]
add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths)
unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True,
overwrite=True, auto_aggregate=True, ingest_metadata=True)
def compare_metadatas(new_metadata_str, original_metadata_file):
original_graph = Graph()
with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f:
original_graph = original_graph.parse(data=f.read())
new_graph = Graph()
new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str))
# remove modified date, they'll never match
subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified)
new_graph.remove((subject, None, None))
subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified)
original_graph.remove((subject, None, None))
for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph):
self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original")
res.refresh_from_db()
compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml")
compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(),
"test_meta.xml")
compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(),
"asdf/asdf_meta.xml")
compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(),
"watersheds_meta.xml")
compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(),
"logan_meta.xml")
compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(),
"SWE_time_meta.xml")
compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(),
"msf_version.refts_meta.xml")
compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(),
"ODM2_Multi_Site_One_Variable_meta.xml")
def test_zip_set_file_type_to_geo_feature(self):
super(GeoFeatureFileTypeMetaDataTest, self).assert_federated_irods_available()
# here we are using a valid zip file for setting it
# to Geo Feature file type which includes metadata extraction
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.zip_file_name)
res_upload_files = []
fed_res_path = hydroshare.utils.get_federated_zone_home_path(fed_test_file_full_path)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Federated Composite Resource Raster File Type Testing',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path,
move=False,
metadata=[]
)
# test resource is created on federated zone
self.assertNotEqual(self.composite_resource.resource_federation_path, '')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# check that the resource file is not associated with logical file
self.assertEqual(res_file.has_logical_file, False)
fed_file_path = "{}/{}".format(self.composite_resource.file_path, self.zip_file_name)
self.assertEqual(res_file.storage_path, fed_file_path)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# test file type and file type extracted metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_file_rename_or_move(self):
# test that file can't be moved or renamed for any resource file
# that's part of the GeoFeature logical file object (LFO)
self._create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test renaming of files that are associated with raster LFO - which should raise exception
self.assertEqual(self.composite_resource.files.count(), 3)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.file_folder, 'states_required_files')
src_path = 'data/contents/states_required_files/states.shp'
tgt_path = 'data/contents/states_required_files/states-1.shp'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/states_required_files/states.dbf'
tgt_path = 'data/contents/states_required_files/states-1.dbf'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
# test moving the files associated with geo raster LFO
src_path = 'data/contents/states_required_files/states.shx'
tgt_path = 'data/contents/states_required_files/states-1.shx'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/states_required_files/states.shp'
tgt_path = 'data/contents/states_required_files-1/states.shp'
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_create_aggregation_from_shp_file_required_2(self):
# here we are using a shp file that exists in a folder
# for setting it to Geo Feature file type which includes metadata extraction
# no new folder should be created as part o creating this aggregation
self.create_composite_resource()
new_folder = 'geofeature_aggr'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file,
upload_folder=new_folder)
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
for res_file in self.composite_resource.files.all():
# test that each resource file is part of an aggregation (logical file)
self.assertTrue(res_file.has_logical_file)
# test that the each resource file has the same folder - no new folder created
self.assertEqual(res_file.file_folder, new_folder)
self.composite_resource.delete()
def test_aggregation_file_rename(self):
# test that a file can't renamed for any resource file
# that's part of the GeoFeature logical file
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# create aggregation from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test renaming of files that are associated with aggregation raises exception
self.assertEqual(self.composite_resource.files.count(), 3)
src_path = 'data/contents/{}/states.shp'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.shp'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/{}/states.dbf'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.dbf'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
src_path = 'data/contents/{}/states.shx'.format(new_folder)
tgt_path = 'data/contents/{}/states-1.shx'.format(new_folder)
with self.assertRaises(DRF_ValidationError):
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
self.composite_resource.delete()
def test_zip_set_file_type_to_geo_feature_required(self):
# here we are using a zip file that has only the 3 required files for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource(self.states_required_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.extension, '.zip')
logical_file = res_file.logical_file
self.assertTrue(isinstance(logical_file.metadata, GenericFileMetaData))
self.assertTrue(isinstance(logical_file, GenericLogicalFile))
# no keywords at this point for file
self.assertEqual(logical_file.metadata.keywords, [])
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test file type and file type metadata
assert_geofeature_file_type_metadata(self, expected_folder_name)
# there should not be any file level keywords
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_shp_set_file_type_to_geo_feature_required(self):
# here we are using a shp file for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource()
# add the 3 required files to the resource
files = []
shp_temp_file = os.path.join(self.temp_dir, self.states_shp_file_name)
shutil.copy(self.states_shp_file, shp_temp_file)
shx_temp_file = os.path.join(self.temp_dir, self.states_shx_file_name)
shutil.copy(self.states_shx_file, shx_temp_file)
dbf_temp_file = os.path.join(self.temp_dir, self.states_dbf_file_name)
shutil.copy(self.states_dbf_file, dbf_temp_file)
files.append(
UploadedFile(file=open(shp_temp_file, 'r'),
name=self.states_shp_file_name))
files.append(
UploadedFile(file=open(shx_temp_file, 'r'),
name=self.states_shx_file_name))
files.append(
UploadedFile(file=open(dbf_temp_file, 'r'),
name=self.states_dbf_file_name))
hydroshare.utils.resource_file_add_process(self.composite_resource,
files, self.user)
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is 3 GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 3)
# set the shp file to GeoFeatureFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
shp_res_file.id, self.user)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 3)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 3)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# check that we put the 3 files in a new folder
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/{}/{}"
res_file.file_folder = expected_folder_name
expected_file_path = expected_file_path.format(
self.composite_resource.root_path, expected_folder_name,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# test extracted raster file type metadata
# there should not be any resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 0)
self.assertNotEqual(logical_file.metadata.geometryinformation, None)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 51)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType,
"MULTIPOLYGON")
self.assertNotEqual(logical_file.metadata.originalcoverage, None)
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'unknown')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'unknown')
self.assertGreater(
len(logical_file.metadata.originalcoverage.projection_string), 0)
self.assertEqual(logical_file.metadata.originalcoverage.unit,
'unknown')
self.assertEqual(logical_file.metadata.originalcoverage.eastlimit,
-66.9692712587578)
self.assertEqual(logical_file.metadata.originalcoverage.northlimit,
71.406235393967)
self.assertEqual(logical_file.metadata.originalcoverage.southlimit,
18.921786345087)
self.assertEqual(logical_file.metadata.originalcoverage.westlimit,
-178.217598362366)
# there should not be any file level keywords
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def handle(self, *args, **options):
logger = logging.getLogger(__name__)
resource_counter = 0
to_resource_type = 'CompositeResource'
msg = "THERE ARE CURRENTLY {} GEOFEATURE RESOURCES PRIOR TO CONVERSION.".format(
GeographicFeatureResource.objects.all().count())
logger.info(msg)
print(">> {}".format(msg))
for gf_res in GeographicFeatureResource.objects.all():
# check resource exists on irods
istorage = gf_res.get_irods_storage()
if not istorage.exists(gf_res.root_path):
err_msg = "Geofeature resource not found in irods (ID: {})".format(
gf_res.short_id)
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
# skip this geofeature resource for migration
continue
create_gf_aggregation = False
if gf_res.has_required_content_files() and \
gf_res.metadata.geometryinformation is not None:
create_gf_aggregation = True
if create_gf_aggregation:
# check resource files exist on irods
file_missing = False
for res_file in gf_res.files.all():
file_path = res_file.public_path
if not istorage.exists(file_path):
err_msg = "File path not found in irods:{}".format(
file_path)
logger.error(err_msg)
err_msg = "Failed to convert geofeature resource (ID: {}). " \
"Resource file is missing on irods"
err_msg = err_msg.format(gf_res.short_id)
print("Error:>> {}".format(err_msg))
file_missing = True
break
if file_missing:
# skip this corrupt geofeature resource for migration
continue
# change the resource_type
gf_metadata_obj = gf_res.metadata
gf_res.resource_type = to_resource_type
gf_res.content_model = to_resource_type.lower()
gf_res.save()
# get the converted resource object - CompositeResource
comp_res = gf_res.get_content_model()
# set CoreMetaData object for the composite resource
core_meta_obj = CoreMetaData.objects.create()
comp_res.content_object = core_meta_obj
# migrate geofeature resource core metadata elements to composite resource
migrate_core_meta_elements(gf_metadata_obj, comp_res)
# update url attribute of the metadata 'type' element
type_element = comp_res.metadata.type
type_element.url = '{0}/terms/{1}'.format(current_site_url(),
to_resource_type)
type_element.save()
if create_gf_aggregation:
# create a Geofeature aggregation
gf_aggr = None
try:
gf_aggr = GeoFeatureLogicalFile.create(resource=comp_res)
except Exception as ex:
err_msg = 'Failed to create Geofeature aggregation for resource (ID: {})'
err_msg = err_msg.format(gf_res.short_id)
err_msg = err_msg + '\n' + ex.message
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
if gf_aggr is not None:
# set aggregation dataset title
gf_aggr.dataset_name = comp_res.metadata.title.value
gf_aggr.save()
# make the res files part of the aggregation
for res_file in comp_res.files.all():
gf_aggr.add_resource_file(res_file)
# migrate geofeature specific metadata to aggregation
for fieldinfo in gf_metadata_obj.fieldinformations.all():
fieldinfo.content_object = gf_aggr.metadata
fieldinfo.save()
# create aggregation level coverage elements
for coverage in comp_res.metadata.coverages.all():
aggr_coverage = Coverage()
aggr_coverage.type = coverage.type
aggr_coverage._value = coverage._value
aggr_coverage.content_object = gf_aggr.metadata
aggr_coverage.save()
org_coverage = gf_metadata_obj.originalcoverage
if org_coverage:
org_coverage.content_object = gf_aggr.metadata
org_coverage.save()
geom_info = gf_metadata_obj.geometryinformation
if geom_info:
geom_info.content_object = gf_aggr.metadata
geom_info.save()
# create aggregation level keywords
keywords = [
sub.value for sub in comp_res.metadata.subjects.all()
]
gf_aggr.metadata.keywords = keywords
# create aggregation level xml files
gf_aggr.create_aggregation_xml_documents()
msg = 'One Geofeature aggregation was created in resource (ID: {})'
msg = msg.format(comp_res.short_id)
logger.info(msg)
comp_res.save()
# set resource to dirty so that resource level xml files (resource map and
# metadata xml files) will be re-generated as part of next bag download
try:
set_dirty_bag_flag(comp_res)
except Exception as ex:
err_msg = 'Failed to set bag flag dirty for the converted resource (ID: {})'
err_msg = err_msg.format(gf_res.short_id)
err_msg = err_msg + '\n' + ex.message
logger.error(err_msg)
print("Error:>> {}".format(err_msg))
resource_counter += 1
# delete the instance of GeographicFeatureMetaData that was part of the original
# geofeature resource
gf_metadata_obj.delete()
msg = 'Geofeature resource (ID: {}) was converted to Composite Resource type'
msg = msg.format(comp_res.short_id)
logger.info(msg)
msg = "{} GEOFEATURE RESOURCES WERE CONVERTED TO COMPOSITE RESOURCE.".format(
resource_counter)
logger.info(msg)
print(">> {}".format(msg))
msg = "THERE ARE CURRENTLY {} GEOFEATURE RESOURCES AFTER CONVERSION.".format(
GeographicFeatureResource.objects.all().count())
logger.info(msg)
if GeographicFeatureResource.objects.all().count() > 0:
msg = "NOT ALL GEOFEATURE RESOURCES WERE CONVERTED TO COMPOSITE RESOURCE TYPE"
logger.error(msg)
print(">> {}".format(msg))
def test_aggregation_folder_move(self):
# test changes to aggregation name, aggregation metadata xml file path, and aggregation
# resource map xml file path on aggregation folder move
self.create_composite_resource()
new_folder = 'my_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the 3 required files to the resource at the above folder
self.add_file_to_resource(file_to_add=self.states_required_zip_file,
upload_folder=new_folder)
res_file = self.composite_resource.files.first()
# create aggregation from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
self.assertEqual(self.composite_resource.files.count(), 3)
base_file_name, _ = os.path.splitext(res_file.file_name)
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.file_folder, new_folder)
# test aggregation name
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
shp_file_path = get_path_with_no_file_extension(
logical_file.aggregation_name)
# test aggregation xml file paths
expected_meta_file_path = '{0}{1}'.format(shp_file_path,
METADATA_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_file_path)
expected_map_file_path = '{0}{1}'.format(shp_file_path,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.map_short_file_path,
expected_map_file_path)
# create a folder to move the aggregation folder there
parent_folder = 'parent_folder'
ResourceFile.create_folder(self.composite_resource, parent_folder)
# move the aggregation folder to the parent folder
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}/{}'.format(parent_folder, new_folder)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
file_folder = '{}/{}'.format(parent_folder, new_folder)
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.file_folder, file_folder)
# test aggregation name update
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
shp_file_path = get_path_with_no_file_extension(
logical_file.aggregation_name)
# test aggregation xml file paths
expected_meta_file_path = '{0}{1}'.format(shp_file_path,
METADATA_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_file_path)
expected_map_file_path = '{0}{1}'.format(shp_file_path,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.map_short_file_path,
expected_map_file_path)
self.composite_resource.delete()
def get_folder_aggregation_type_to_set(self, dir_path):
"""Returns an aggregation (file type) type that the specified folder *dir_path* can
possibly be set to.
:param dir_path: Resource file directory path (full folder path starting with resource id)
for which the possible aggregation type that can be set needs to be determined
:return If the specified folder is already represents an aggregation or does
not contain suitable file(s) then returns "" (empty string). If the specified folder
contains only the files that meet the requirements of a supported aggregation, and
does not contain other folders or does not have a parent folder then return the
class name of that matching aggregation type.
"""
aggregation_type_to_set = ""
if self.get_folder_aggregation_object(dir_path) is not None:
# target folder is already an aggregation
return None
istorage = self.get_irods_storage()
irods_path = dir_path
if self.is_federated:
irods_path = os.path.join(self.resource_federation_path,
irods_path)
store = istorage.listdir(irods_path)
if store[0]:
# seems there are folders under dir_path - no aggregation type can be set if the target
# folder contains other folders
return None
files_in_folder = [
res_file for res_file in self.files.all()
if res_file.dir_path == dir_path
]
if not files_in_folder:
# folder is empty
return None
if len(files_in_folder) > 1:
# check for geo feature
aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
else:
# check for raster
aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for NetCDF aggregation type
aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
# check for TimeSeries aggregation type
aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type(
files_in_folder)
if aggregation_type_to_set:
return aggregation_type_to_set
return None
def test_create_aggregation_from_zip_file_all(self):
# here we are using a zip file that has all the 15 files for setting it
# to Geo Feature file type which includes metadata extraction
# this zip file for creating aggregation is at the root of the folder hierarchy
self.create_composite_resource(self.osm_all_files_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file type
self.assertEqual(res_file.has_logical_file, False)
# set the zip file to GeoFeatureLogicalFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is no GenericFileMetaData object
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# test extracted raster file type metadata
# there should one resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 1)
self.assertEqual(logical_file.metadata.fieldinformations.all().count(),
7)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 87)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType, "POLYGON")
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'WGS_1984')
self.assertTrue(
abs(logical_file.metadata.originalcoverage.eastlimit -
3.4520493) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.northlimit -
45.0466382) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.southlimit -
42.5732416) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.westlimit -
(-0.3263017)) < self.allowance)
self.assertEqual(logical_file.metadata.originalcoverage.unit, 'degree')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'WGS 84')
# there should be file level keywords
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_create_aggregation_from_shp_file_required_1(self):
# here we are using a shp file that exists at the root of the folder hierarchy
# for setting it to Geo Feature file type which includes metadata extraction
self.create_composite_resource()
# add the 3 required files to the resource at the above folder
res_file = self.add_file_to_resource(file_to_add=self.states_shp_file)
self.assertEqual(res_file.file_folder, '')
res_file = self.add_file_to_resource(file_to_add=self.states_shx_file)
self.assertEqual(res_file.file_folder, '')
res_file = self.add_file_to_resource(file_to_add=self.states_dbf_file)
self.assertEqual(res_file.file_folder, '')
self.assertEqual(self.composite_resource.files.all().count(), 3)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
# check that the resource file is not associated with any logical file
self.assertEqual(res_file.has_logical_file, False)
# set the shp file to GeoFeatureLogicalFile type
shp_res_file = [
f for f in self.composite_resource.files.all()
if f.extension == '.shp'
][0]
GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user,
shp_res_file.id)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 3)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 3)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# test extracted raster file type metadata
# there should not be any resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 0)
self.assertNotEqual(logical_file.metadata.geometryinformation, None)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 51)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType,
"MULTIPOLYGON")
self.assertNotEqual(logical_file.metadata.originalcoverage, None)
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'unknown')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name, 'unknown')
self.assertGreater(
len(logical_file.metadata.originalcoverage.projection_string), 0)
self.assertEqual(logical_file.metadata.originalcoverage.unit,
'unknown')
self.assertEqual(logical_file.metadata.originalcoverage.eastlimit,
-66.9692712587578)
self.assertEqual(logical_file.metadata.originalcoverage.northlimit,
71.406235393967)
self.assertEqual(logical_file.metadata.originalcoverage.southlimit,
18.921786345087)
self.assertEqual(logical_file.metadata.originalcoverage.westlimit,
-178.217598362366)
# there should not be any file level keywords
self.assertEqual(logical_file.metadata.keywords, [])
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def test_zip_set_file_type_to_geo_feature_all(self):
# here we are using a zip file that has all the 15 files for setting it
# to Geo Feature file type which includes metadata extraction
self._create_composite_resource(self.osm_all_files_zip_file)
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
expected_folder_name = res_file.file_name[:-4]
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# check that there is one GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# set the zip file to GeoFeatureFile type
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test files in the file type
self.assertEqual(self.composite_resource.files.count(), 15)
# check that there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# check that there is no GenericFileMetaData object
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# check that there is one GeoFeatureLogicalFile object
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1)
logical_file = GeoFeatureLogicalFile.objects.first()
self.assertEqual(logical_file.files.count(), 15)
# check that the 3 resource files are now associated with GeoFeatureLogicalFile
for res_file in self.composite_resource.files.all():
self.assertEqual(res_file.logical_file_type_name,
"GeoFeatureLogicalFile")
self.assertEqual(res_file.has_logical_file, True)
self.assertTrue(
isinstance(res_file.logical_file, GeoFeatureLogicalFile))
# check that we put the 3 files in a new folder
for res_file in self.composite_resource.files.all():
file_path, base_file_name, _ = get_resource_file_name_and_extension(
res_file)
expected_file_path = "{}/data/contents/{}/{}"
res_file.file_folder = expected_folder_name
expected_file_path = expected_file_path.format(
self.composite_resource.root_path, expected_folder_name,
base_file_name)
self.assertEqual(file_path, expected_file_path)
# test extracted raster file type metadata
# there should one resource level coverage
self.assertEqual(self.composite_resource.metadata.coverages.count(), 1)
self.assertEqual(logical_file.metadata.fieldinformations.all().count(),
7)
self.assertEqual(
logical_file.metadata.geometryinformation.featureCount, 87)
self.assertEqual(
logical_file.metadata.geometryinformation.geometryType, "POLYGON")
self.assertEqual(logical_file.metadata.originalcoverage.datum,
'WGS_1984')
self.assertTrue(
abs(logical_file.metadata.originalcoverage.eastlimit -
3.4520493) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.northlimit -
45.0466382) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.southlimit -
42.5732416) < self.allowance)
self.assertTrue(
abs(logical_file.metadata.originalcoverage.westlimit -
(-0.3263017)) < self.allowance)
self.assertEqual(logical_file.metadata.originalcoverage.unit, 'Degree')
self.assertEqual(
logical_file.metadata.originalcoverage.projection_name,
'GCS_WGS_1984')
# there should be file level keywords
for key in ('Logan River', 'TauDEM'):
self.assertIn(key, logical_file.metadata.keywords)
self.assertEqual(len(logical_file.metadata.keywords), 2)
self.composite_resource.delete()
# there should be no GeoFeatureLogicalFile object at this point
self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
