def test_file_move(self):
# test that a resource file that is part of a GenericLogicalFile object
# can be moved
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
self.assertEqual(res_file.file_name, 'generic_file.txt')
# create generic aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# file should not be in a folder
self.assertEqual(res_file.file_folder, None)
# test moving the file to a new folder is allowed
new_folder = 'test_folder'
create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = "data/contents/{0}/{1}".format(new_folder, res_file.file_name)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path,
tgt_path)
res_file = self.composite_resource.files.first()
# file should in a folder
self.assertEqual(res_file.file_folder, new_folder)
self.assertTrue(res_file.resource_file.name.endswith(tgt_path))
self.composite_resource.delete()
def test_remove_aggregation(self):
# test that when an instance GenericLogicalFile Type (aggregation) is deleted
# all resource files associated with that aggregation is not deleted but the associated
# metadata is deleted
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
# create generic aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# test that we have one logical file of type GenericLogicalFile Type as a result
# of setting file type (aggregation)
self.assertEqual(GenericLogicalFile.objects.count(), 1)
self.assertEqual(GenericFileMetaData.objects.count(), 1)
logical_file = GenericLogicalFile.objects.first()
self.assertEqual(logical_file.files.all().count(), 1)
self.assertEqual(self.composite_resource.files.all().count(), 1)
self.assertEqual(set(self.composite_resource.files.all()),
set(logical_file.files.all()))
# delete the aggregation (logical file) object using the remove_aggregation function
logical_file.remove_aggregation()
# test there is no GenericLogicalFile object
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# test there is no GenericFileMetaData object
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# check the files previously associated with the generic aggregation not deleted
self.assertEqual(self.composite_resource.files.all().count(), 1)
self.composite_resource.delete()
def test_create_aggregation_2(self):
"""Test that we can create a generic aggregation from a resource file that
exists in a folder """
self.create_composite_resource()
new_folder = 'generic_folder'
ResourceFile.create_folder(self.composite_resource, new_folder)
# add the the txt file to the resource at the above folder
self.add_file_to_resource(file_to_add=self.generic_file, upload_folder=new_folder)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# file has a folder
self.assertEqual(res_file.file_folder, new_folder)
# check that the resource file is not part of an aggregation
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# set file to generic logical file type (aggregation)
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
# file has the same folder
self.assertEqual(res_file.file_folder, new_folder)
self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name)
self.assertEqual(GenericLogicalFile.objects.count(), 1)
self.composite_resource.delete()
def test_aggregation_metadata_on_file_delete(self):
# test that when a file that's part of the GenericLogicalFile object
# is deleted all metadata associated with the file type also get deleted
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
# set the file to generic logical file
GenericLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
res_file = self.composite_resource.files.first()
gen_logical_file = res_file.logical_file
self.assertEqual(GenericLogicalFile.objects.count(), 1)
self.assertEqual(GenericFileMetaData.objects.count(), 1)
# at this point there should not be any coverage elements associated with
# logical file
self.assertEqual(gen_logical_file.metadata.coverages.count(), 0)
# at this point there should not be any key/value metadata associated with
# logical file
self.assertEqual(gen_logical_file.metadata.extra_metadata, {})
# add temporal coverage
value_dict = {
'name': 'Name for period coverage',
'start': '1/1/2000',
'end': '12/12/2012'
}
gen_logical_file.metadata.create_element('coverage',
type='period',
value=value_dict)
# add spatial coverage
value_dict = {
'east': '56.45678',
'north': '12.6789',
'units': 'Decimal degree'
}
gen_logical_file.metadata.create_element('coverage',
type='point',
value=value_dict)
# at this point there should be 2 coverage elements associated with
# logical file
self.assertEqual(gen_logical_file.metadata.coverages.count(), 2)
# at this point we should have 4 coverage elements (2 resource level
# and 2 file type level
self.assertEqual(Coverage.objects.count(), 4)
# add key/value metadata
gen_logical_file.metadata.extra_metadata = {
'key1': 'value 1',
'key2': 'value 2'
}
gen_logical_file.metadata.save()
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file.id, self.user)
# test that we don't have logical file of type GenericLogicalFile
self.assertEqual(GenericLogicalFile.objects.count(), 0)
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# test that resource level coverage element exist - not got deleted
self.assertEqual(Coverage.objects.count(), 2)
def test_main_file(self):
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
self.assertEqual(1, GenericLogicalFile.objects.count())
self.assertEqual(None, GenericLogicalFile.objects.first().get_main_file_type())
self.assertEqual(None, GenericLogicalFile.objects.first().get_main_file)
def test_delete_aggregation_coverage(self):
"""Here we are testing deleting of temporal and spatial coverage for a single file
aggregation
"""
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set file to generic logical file type - single file aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
sf_aggr = GenericLogicalFile.objects.first()
# test deleting spatial coverage
self.assertEqual(sf_aggr.metadata.spatial_coverage, None)
value_dict = {
'east': '56.45678',
'north': '12.6789',
'units': 'Decimal degree'
}
sf_aggr.metadata.create_element('coverage',
type='point',
value=value_dict)
self.assertNotEqual(sf_aggr.metadata.spatial_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
sf_aggr.metadata.is_dirty = False
sf_aggr.metadata.save()
sf_aggr.metadata.delete_element('coverage',
sf_aggr.metadata.spatial_coverage.id)
self.assertEqual(sf_aggr.metadata.spatial_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
# test deleting temporal coverage
self.assertEqual(sf_aggr.metadata.temporal_coverage, None)
value_dict = {
'name': 'Name for period coverage',
'start': '1/1/2000',
'end': '12/12/2012'
}
sf_aggr.metadata.create_element('coverage',
type='period',
value=value_dict)
self.assertTrue(sf_aggr.metadata.is_dirty)
sf_aggr.metadata.is_dirty = False
sf_aggr.metadata.save()
self.assertNotEqual(sf_aggr.metadata.temporal_coverage, None)
sf_aggr.metadata.delete_element('coverage',
sf_aggr.metadata.temporal_coverage.id)
self.assertEqual(sf_aggr.metadata.temporal_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
self.composite_resource.delete()
def test_aggregation_name(self):
# test the aggregation_name property for the generic aggregation (logical file)
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, False)
# set file to generic logical file type
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name)
logical_file = res_file.logical_file
self.assertEqual(logical_file.aggregation_name, res_file.file_name)
# test the aggregation name after moving the file into a folder
new_folder = 'generic_folder'
create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}'.format(new_folder, res_file.file_name)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
# test the aggregation name after renaming the file
src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name)
tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}_1{2}'.format(new_folder, base_file_name, ext)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
# test the aggregation name after renaming the folder
folder_rename = '{}_1'.format(new_folder)
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}'.format(folder_rename)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}'.format(folder_rename, res_file.file_name)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
self.composite_resource.delete()
def test_has_modified_metadata_no_change(self):
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
self.assertEqual(1, GenericLogicalFile.objects.count())
gen_logical_file = GenericLogicalFile.objects.first()
# check expected generated metadata state without modifications
self.assertEqual(0, gen_logical_file.metadata.coverages.count())
self.assertEqual({}, gen_logical_file.metadata.extra_metadata)
self.assertEqual("generic_file", gen_logical_file.dataset_name)
self.assertFalse(gen_logical_file.metadata.has_modified_metadata)
def test_has_modified_metadata_updated_title(self):
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
self.assertEqual(1, GenericLogicalFile.objects.count())
gen_logical_file = GenericLogicalFile.objects.first()
gen_logical_file.dataset_name = "Updated"
gen_logical_file.save()
# check expected generated metadata state updated title only
self.assertEqual(0, gen_logical_file.metadata.coverages.count())
self.assertEqual({}, gen_logical_file.metadata.extra_metadata)
self.assertEqual("Updated", gen_logical_file.dataset_name)
self.assertTrue(gen_logical_file.metadata.has_modified_metadata)
def test_has_modified_metadata_updated_coverages(self):
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
self.assertEqual(1, GenericLogicalFile.objects.count())
gen_logical_file = GenericLogicalFile.objects.first()
value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict)
# check expected generated metadata state updated coverages only
self.assertEqual(1, gen_logical_file.metadata.coverages.count())
self.assertEqual({}, gen_logical_file.metadata.extra_metadata)
self.assertEqual("generic_file", gen_logical_file.dataset_name)
self.assertTrue(gen_logical_file.metadata.has_modified_metadata)
def set_default_logical_file(self):
"""sets an instance of GenericLogicalFile to any resource file objects of this instance
of the resource that is not already associated with a logical file. """
for res_file in self.files.all():
if not res_file.has_logical_file:
logical_file = GenericLogicalFile.create()
res_file.logical_file_content_object = logical_file
res_file.save()
def test_bag_ingestion(self):
from hs_core.views.utils import unzip_file
def normalize_metadata(metadata_str):
"""Prepares metadata string to match resource id and hydroshare url of original"""
return metadata_str\
.replace(current_site_url(), "http://www.hydroshare.org")\
.replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1")
# create empty resource
res = resource.create_resource(
'CompositeResource',
self.user,
'My Test Resource'
)
full_paths = {}
files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'),
name="test_resource_metadata_files.zip")]
add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths)
unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True,
overwrite=True, auto_aggregate=True, ingest_metadata=True)
def compare_metadatas(new_metadata_str, original_metadata_file):
original_graph = Graph()
with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f:
original_graph = original_graph.parse(data=f.read())
new_graph = Graph()
new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str))
# remove modified date, they'll never match
subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified)
new_graph.remove((subject, None, None))
subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified)
original_graph.remove((subject, None, None))
for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph):
self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original")
res.refresh_from_db()
compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml")
compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(),
"test_meta.xml")
compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(),
"asdf/asdf_meta.xml")
compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(),
"watersheds_meta.xml")
compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(),
"logan_meta.xml")
compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(),
"SWE_time_meta.xml")
compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(),
"msf_version.refts_meta.xml")
compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(),
"ODM2_Multi_Site_One_Variable_meta.xml")
def test_create_aggregation_1(self):
"""Test that we can create a generic aggregation from a resource file that
exists at the root of the folder hierarchy """
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is not part of an aggregation
self.assertEqual(res_file.has_logical_file, False)
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# set file to generic logical file type (aggregation)
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
# file has no folder
self.assertEqual(res_file.file_folder, None)
self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name)
self.assertEqual(GenericLogicalFile.objects.count(), 1)
self.composite_resource.delete()
def test_aggregation_metadata_on_file_delete(self):
# test that when a file that's part of the GenericLogicalFile object
# is deleted all metadata associated with the file type also get deleted
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
# set the file to generic logical file
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
gen_logical_file = res_file.logical_file
self.assertEqual(GenericLogicalFile.objects.count(), 1)
self.assertEqual(GenericFileMetaData.objects.count(), 1)
# at this point there should not be any coverage elements associated with
# logical file
self.assertEqual(gen_logical_file.metadata.coverages.count(), 0)
# at this point there should not be any key/value metadata associated with
# logical file
self.assertEqual(gen_logical_file.metadata.extra_metadata, {})
# add temporal coverage
value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'}
gen_logical_file.metadata.create_element('coverage', type='period', value=value_dict)
# add spatial coverage
value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict)
# at this point there should be 2 coverage elements associated with
# logical file
self.assertEqual(gen_logical_file.metadata.coverages.count(), 2)
# at this point we should have 4 coverage elements (2 resource level
# and 2 file type level
self.assertEqual(Coverage.objects.count(), 4)
# add key/value metadata
gen_logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2'}
gen_logical_file.metadata.save()
hydroshare.delete_resource_file(self.composite_resource.short_id,
res_file.id,
self.user)
# test that we don't have logical file of type GenericLogicalFile
self.assertEqual(GenericLogicalFile.objects.count(), 0)
self.assertEqual(GenericFileMetaData.objects.count(), 0)
# test that resource level coverage element exist - not got deleted
self.assertEqual(Coverage.objects.count(), 2)
def test_file_rename(self):
# test that a resource file that is part of a GenericLogicalFile object
# can be renamed
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
self.assertEqual(res_file.file_name, 'generic_file.txt')
# create generic aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# file should not be in a folder
self.assertEqual(res_file.file_folder, None)
# test rename of file is allowed
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = "data/contents/{0}_1{1}".format(base_file_name, ext)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path,
tgt_path)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.file_name, '{0}_1{1}'.format(base_file_name, ext))
self.composite_resource.delete()
def test_delete_aggregation_coverage(self):
"""Here we are testing deleting of temporal and spatial coverage for a single file
aggregation
"""
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set file to generic logical file type - single file aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
sf_aggr = GenericLogicalFile.objects.first()
# test deleting spatial coverage
self.assertEqual(sf_aggr.metadata.spatial_coverage, None)
value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
sf_aggr.metadata.create_element('coverage', type='point', value=value_dict)
self.assertNotEqual(sf_aggr.metadata.spatial_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
sf_aggr.metadata.is_dirty = False
sf_aggr.metadata.save()
sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.spatial_coverage.id)
self.assertEqual(sf_aggr.metadata.spatial_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
# test deleting temporal coverage
self.assertEqual(sf_aggr.metadata.temporal_coverage, None)
value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'}
sf_aggr.metadata.create_element('coverage', type='period', value=value_dict)
self.assertTrue(sf_aggr.metadata.is_dirty)
sf_aggr.metadata.is_dirty = False
sf_aggr.metadata.save()
self.assertNotEqual(sf_aggr.metadata.temporal_coverage, None)
sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.temporal_coverage.id)
self.assertEqual(sf_aggr.metadata.temporal_coverage, None)
self.assertTrue(sf_aggr.metadata.is_dirty)
self.composite_resource.delete()
def add_file_to_resource(resource,
f,
folder=None,
source_name='',
source_size=0,
move=False,
is_file_reference=False):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is
set to 0 and useless.
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:param is_file_reference: indicate whether the file being added is a reference to an external
file stored in an external zone or URL. source_name will hold
the reference file path or url
:return: The identifier of the ResourceFile added.
"""
# importing here to avoid circular import
from hs_file_types.models import GenericLogicalFile
if f:
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None,
move=False)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name,
source_size=source_size,
is_file_reference=is_file_reference,
move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
# if a file gets added successfully to composite resource, then better to set the generic
# logical file here
if resource.resource_type == "CompositeResource":
logical_file = GenericLogicalFile.create()
ret.logical_file_content_object = logical_file
ret.save()
return ret
def test_aggregation_metadata(self):
"""Test that we can create the following metadata for a generic aggregation
coverage (spatial and temporal)
key/value metadata
title (dataset_name)
keywords
"""
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set file to generic logical file type
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
logical_file = res_file.logical_file
# test that the generic logical file datataset_name attribute has the value of the
# the content file name
self.assertEqual(logical_file.dataset_name, base_file_name)
dataset_name = "This is a generic dataset"
logical_file.dataset_name = dataset_name
logical_file.save()
logical_file = res_file.logical_file
self.assertEqual(logical_file.dataset_name, dataset_name)
# check that the logical file has metadata object
self.assertNotEqual(logical_file.metadata, None)
# there should be no coverage element at this point
self.assertEqual(logical_file.metadata.coverages.count(), 0)
# there should not be any extra_metadata (key/value) at this point
self.assertEqual(logical_file.metadata.extra_metadata, {})
# create key/vale metadata
logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2'}
logical_file.metadata.save()
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata,
{'key1': 'value 1', 'key2': 'value 2'})
# update key/value metadata
logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2',
'key 3': 'value3'}
logical_file.metadata.save()
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata,
{'key1': 'value 1', 'key2': 'value 2', 'key 3': 'value3'})
# delete key/value metadata
logical_file.metadata.extra_metadata = {}
logical_file.metadata.save()
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
self.assertEqual(logical_file.metadata.extra_metadata, {})
# test keywords
self.assertEqual(logical_file.metadata.keywords, [])
logical_file.metadata.keywords = ['kw-1', 'kw-2']
logical_file.metadata.save()
self.assertIn('kw-1', logical_file.metadata.keywords)
self.assertIn('kw-2', logical_file.metadata.keywords)
# test coverage element CRUD
res_file = [f for f in self.composite_resource.files.all()
if f.logical_file_type_name == self.logical_file_type_name][0]
gen_logical_file = res_file.logical_file
value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'}
temp_cov = gen_logical_file.metadata.create_element('coverage', type='period',
value=value_dict)
self.assertEqual(temp_cov.value['name'], 'Name for period coverage')
self.assertEqual(temp_cov.value['start'], '1/1/2000')
self.assertEqual(temp_cov.value['end'], '12/12/2012')
# update temporal coverage
value_dict = {'start': '10/1/2010', 'end': '12/1/2016'}
gen_logical_file.metadata.update_element('coverage', temp_cov.id, type='period',
value=value_dict)
temp_cov = gen_logical_file.metadata.temporal_coverage
self.assertEqual(temp_cov.value['name'], 'Name for period coverage')
self.assertEqual(temp_cov.value['start'], '10/1/2010')
self.assertEqual(temp_cov.value['end'], '12/1/2016')
# add spatial coverage
value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
spatial_cov = gen_logical_file.metadata.create_element('coverage', type='point',
value=value_dict)
self.assertEqual(spatial_cov.value['projection'], 'WGS 84 EPSG:4326')
self.assertEqual(spatial_cov.value['units'], 'Decimal degree')
self.assertEqual(spatial_cov.value['north'], 12.6789)
self.assertEqual(spatial_cov.value['east'], 56.45678)
# update spatial coverage
value_dict = {'east': '-156.45678', 'north': '45.6789', 'units': 'Decimal degree'}
gen_logical_file.metadata.update_element('coverage', spatial_cov.id, type='point',
value=value_dict)
spatial_cov = logical_file.metadata.spatial_coverage
self.assertEqual(spatial_cov.value['projection'], 'WGS 84 EPSG:4326')
self.assertEqual(spatial_cov.value['units'], 'Decimal degree')
self.assertEqual(spatial_cov.value['north'], 45.6789)
self.assertEqual(spatial_cov.value['east'], -156.45678)
self.composite_resource.delete()
# there should be no GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# there should be no GenericFileMetaData object at this point
self.assertEqual(GenericFileMetaData.objects.count(), 0)
def test_aggregation_xml_file_paths(self):
# test the aggregation meta and map xml file paths with file name and folder name
# changes
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, False)
# set file to generic logical file type
GenericLogicalFile.set_file_type(self.composite_resource, self.user,
res_file.id)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name,
self.logical_file_type_name)
logical_file = res_file.logical_file
# file name without extension
res_file_name, _ = os.path.splitext(res_file.file_name)
expected_meta_path = '{0}{1}'.format(res_file_name,
METADATA_FILE_ENDSWITH)
expected_map_path = '{0}{1}'.format(res_file_name,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test xml file paths after moving the file into a folder
new_folder = 'test_folder'
create_folder(self.composite_resource.short_id,
'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = 'data/contents/{0}/{1}'.format(new_folder,
res_file.file_name)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
res_file_name, _ = os.path.splitext(res_file.file_name)
expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
METADATA_FILE_ENDSWITH)
expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test xml file paths after renaming the file
src_path = 'data/contents/{0}/{1}'.format(new_folder,
res_file.file_name)
tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder,
base_file_name, ext)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
res_file_name, _ = os.path.splitext(res_file.file_name)
expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
METADATA_FILE_ENDSWITH)
expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test the xml file path after renaming the folder
folder_rename = '{}_1'.format(new_folder)
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}'.format(folder_rename)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
res_file_name, _ = os.path.splitext(res_file.file_name)
expected_meta_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
METADATA_FILE_ENDSWITH)
expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
RESMAP_FILE_ENDSWITH)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
self.composite_resource.delete()
def test_file_add_to_composite_resource(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
super(CompositeResourceTest, self).assert_federated_irods_available()
# test that when we add file to an existing composite resource, the added file
# automatically set to genericlogicalfile type
self.assertEqual(BaseResource.objects.count(), 0)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test Composite Resource With Files Added From Federated Zone',
auto_aggregate=False
)
# there should not be any GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# add a file to the resource
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=None)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
auto_aggregate=False)
# there should be one resource at this point
self.assertEqual(BaseResource.objects.count(), 1)
self.assertEqual(self.composite_resource.resource_type, "CompositeResource")
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# create the generic aggregation (logical file)
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# check that the resource file is associated with GenericLogicalFile
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# there should be 1 GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# test adding a file to a folder (Note the UI does not support uploading a iRODS file
# to a specific folder)
# create the folder
new_folder = "my-new-folder"
new_folder_path = os.path.join("data", "contents", new_folder)
create_folder(self.composite_resource.short_id, new_folder_path)
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder, auto_aggregate=False)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.composite_resource.delete()
