def test_file_move(self): # test that a resource file that is part of a GenericLogicalFile object # can be moved self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) self.assertEqual(res_file.file_name, 'generic_file.txt') # create generic aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # file should not be in a folder self.assertEqual(res_file.file_folder, None) # test moving the file to a new folder is allowed new_folder = 'test_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = "data/contents/{0}/{1}".format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() # file should in a folder self.assertEqual(res_file.file_folder, new_folder) self.assertTrue(res_file.resource_file.name.endswith(tgt_path)) self.composite_resource.delete()
def test_remove_aggregation(self): # test that when an instance GenericLogicalFile Type (aggregation) is deleted # all resource files associated with that aggregation is not deleted but the associated # metadata is deleted self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() # create generic aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test that we have one logical file of type GenericLogicalFile Type as a result # of setting file type (aggregation) self.assertEqual(GenericLogicalFile.objects.count(), 1) self.assertEqual(GenericFileMetaData.objects.count(), 1) logical_file = GenericLogicalFile.objects.first() self.assertEqual(logical_file.files.all().count(), 1) self.assertEqual(self.composite_resource.files.all().count(), 1) self.assertEqual(set(self.composite_resource.files.all()), set(logical_file.files.all())) # delete the aggregation (logical file) object using the remove_aggregation function logical_file.remove_aggregation() # test there is no GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 0) # test there is no GenericFileMetaData object self.assertEqual(GenericFileMetaData.objects.count(), 0) # check the files previously associated with the generic aggregation not deleted self.assertEqual(self.composite_resource.files.all().count(), 1) self.composite_resource.delete()
def test_create_aggregation_2(self): """Test that we can create a generic aggregation from a resource file that exists in a folder """ self.create_composite_resource() new_folder = 'generic_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the the txt file to the resource at the above folder self.add_file_to_resource(file_to_add=self.generic_file, upload_folder=new_folder) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # file has a folder self.assertEqual(res_file.file_folder, new_folder) # check that the resource file is not part of an aggregation self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GenericLogicalFile.objects.count(), 0) # set file to generic logical file type (aggregation) GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() # file has the same folder self.assertEqual(res_file.file_folder, new_folder) self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) self.assertEqual(GenericLogicalFile.objects.count(), 1) self.composite_resource.delete()
def test_aggregation_metadata_on_file_delete(self): # test that when a file that's part of the GenericLogicalFile object # is deleted all metadata associated with the file type also get deleted self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() # set the file to generic logical file GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() gen_logical_file = res_file.logical_file self.assertEqual(GenericLogicalFile.objects.count(), 1) self.assertEqual(GenericFileMetaData.objects.count(), 1) # at this point there should not be any coverage elements associated with # logical file self.assertEqual(gen_logical_file.metadata.coverages.count(), 0) # at this point there should not be any key/value metadata associated with # logical file self.assertEqual(gen_logical_file.metadata.extra_metadata, {}) # add temporal coverage value_dict = { 'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012' } gen_logical_file.metadata.create_element('coverage', type='period', value=value_dict) # add spatial coverage value_dict = { 'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree' } gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict) # at this point there should be 2 coverage elements associated with # logical file self.assertEqual(gen_logical_file.metadata.coverages.count(), 2) # at this point we should have 4 coverage elements (2 resource level # and 2 file type level self.assertEqual(Coverage.objects.count(), 4) # add key/value metadata gen_logical_file.metadata.extra_metadata = { 'key1': 'value 1', 'key2': 'value 2' } gen_logical_file.metadata.save() hydroshare.delete_resource_file(self.composite_resource.short_id, res_file.id, self.user) # test that we don't have logical file of type GenericLogicalFile self.assertEqual(GenericLogicalFile.objects.count(), 0) self.assertEqual(GenericFileMetaData.objects.count(), 0) # test that resource level coverage element exist - not got deleted self.assertEqual(Coverage.objects.count(), 2)
def test_main_file(self): self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertEqual(1, GenericLogicalFile.objects.count()) self.assertEqual(None, GenericLogicalFile.objects.first().get_main_file_type()) self.assertEqual(None, GenericLogicalFile.objects.first().get_main_file)
def test_delete_aggregation_coverage(self): """Here we are testing deleting of temporal and spatial coverage for a single file aggregation """ self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # set file to generic logical file type - single file aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) sf_aggr = GenericLogicalFile.objects.first() # test deleting spatial coverage self.assertEqual(sf_aggr.metadata.spatial_coverage, None) value_dict = { 'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree' } sf_aggr.metadata.create_element('coverage', type='point', value=value_dict) self.assertNotEqual(sf_aggr.metadata.spatial_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) sf_aggr.metadata.is_dirty = False sf_aggr.metadata.save() sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.spatial_coverage.id) self.assertEqual(sf_aggr.metadata.spatial_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) # test deleting temporal coverage self.assertEqual(sf_aggr.metadata.temporal_coverage, None) value_dict = { 'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012' } sf_aggr.metadata.create_element('coverage', type='period', value=value_dict) self.assertTrue(sf_aggr.metadata.is_dirty) sf_aggr.metadata.is_dirty = False sf_aggr.metadata.save() self.assertNotEqual(sf_aggr.metadata.temporal_coverage, None) sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.temporal_coverage.id) self.assertEqual(sf_aggr.metadata.temporal_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) self.composite_resource.delete()
def test_aggregation_name(self): # test the aggregation_name property for the generic aggregation (logical file) self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, False) # set file to generic logical file type GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_name) # test the aggregation name after moving the file into a folder new_folder = 'generic_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}'.format(new_folder, res_file.file_name) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) # test the aggregation name after renaming the file src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) # test the aggregation name after renaming the folder folder_rename = '{}_1'.format(new_folder) src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(folder_rename) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}'.format(folder_rename, res_file.file_name) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) self.composite_resource.delete()
def test_has_modified_metadata_no_change(self): self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(1, GenericLogicalFile.objects.count()) gen_logical_file = GenericLogicalFile.objects.first() # check expected generated metadata state without modifications self.assertEqual(0, gen_logical_file.metadata.coverages.count()) self.assertEqual({}, gen_logical_file.metadata.extra_metadata) self.assertEqual("generic_file", gen_logical_file.dataset_name) self.assertFalse(gen_logical_file.metadata.has_modified_metadata)
def test_has_modified_metadata_updated_title(self): self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(1, GenericLogicalFile.objects.count()) gen_logical_file = GenericLogicalFile.objects.first() gen_logical_file.dataset_name = "Updated" gen_logical_file.save() # check expected generated metadata state updated title only self.assertEqual(0, gen_logical_file.metadata.coverages.count()) self.assertEqual({}, gen_logical_file.metadata.extra_metadata) self.assertEqual("Updated", gen_logical_file.dataset_name) self.assertTrue(gen_logical_file.metadata.has_modified_metadata)
def test_has_modified_metadata_updated_coverages(self): self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(1, GenericLogicalFile.objects.count()) gen_logical_file = GenericLogicalFile.objects.first() value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'} gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict) # check expected generated metadata state updated coverages only self.assertEqual(1, gen_logical_file.metadata.coverages.count()) self.assertEqual({}, gen_logical_file.metadata.extra_metadata) self.assertEqual("generic_file", gen_logical_file.dataset_name) self.assertTrue(gen_logical_file.metadata.has_modified_metadata)
def set_default_logical_file(self): """sets an instance of GenericLogicalFile to any resource file objects of this instance of the resource that is not already associated with a logical file. """ for res_file in self.files.all(): if not res_file.has_logical_file: logical_file = GenericLogicalFile.create() res_file.logical_file_content_object = logical_file res_file.save()
def test_bag_ingestion(self): from hs_core.views.utils import unzip_file def normalize_metadata(metadata_str): """Prepares metadata string to match resource id and hydroshare url of original""" return metadata_str\ .replace(current_site_url(), "http://www.hydroshare.org")\ .replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1") # create empty resource res = resource.create_resource( 'CompositeResource', self.user, 'My Test Resource' ) full_paths = {} files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'), name="test_resource_metadata_files.zip")] add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths) unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True, overwrite=True, auto_aggregate=True, ingest_metadata=True) def compare_metadatas(new_metadata_str, original_metadata_file): original_graph = Graph() with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f: original_graph = original_graph.parse(data=f.read()) new_graph = Graph() new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str)) # remove modified date, they'll never match subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified) new_graph.remove((subject, None, None)) subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified) original_graph.remove((subject, None, None)) for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph): self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original") res.refresh_from_db() compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml") compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(), "test_meta.xml") compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(), "asdf/asdf_meta.xml") compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(), "watersheds_meta.xml") compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(), "logan_meta.xml") compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(), "SWE_time_meta.xml") compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(), "msf_version.refts_meta.xml") compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(), "ODM2_Multi_Site_One_Variable_meta.xml")
def test_create_aggregation_1(self): """Test that we can create a generic aggregation from a resource file that exists at the root of the folder hierarchy """ self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not part of an aggregation self.assertEqual(res_file.has_logical_file, False) self.assertEqual(GenericLogicalFile.objects.count(), 0) # set file to generic logical file type (aggregation) GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() # file has no folder self.assertEqual(res_file.file_folder, None) self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) self.assertEqual(GenericLogicalFile.objects.count(), 1) self.composite_resource.delete()
def test_aggregation_metadata_on_file_delete(self): # test that when a file that's part of the GenericLogicalFile object # is deleted all metadata associated with the file type also get deleted self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() # set the file to generic logical file GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() gen_logical_file = res_file.logical_file self.assertEqual(GenericLogicalFile.objects.count(), 1) self.assertEqual(GenericFileMetaData.objects.count(), 1) # at this point there should not be any coverage elements associated with # logical file self.assertEqual(gen_logical_file.metadata.coverages.count(), 0) # at this point there should not be any key/value metadata associated with # logical file self.assertEqual(gen_logical_file.metadata.extra_metadata, {}) # add temporal coverage value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'} gen_logical_file.metadata.create_element('coverage', type='period', value=value_dict) # add spatial coverage value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'} gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict) # at this point there should be 2 coverage elements associated with # logical file self.assertEqual(gen_logical_file.metadata.coverages.count(), 2) # at this point we should have 4 coverage elements (2 resource level # and 2 file type level self.assertEqual(Coverage.objects.count(), 4) # add key/value metadata gen_logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2'} gen_logical_file.metadata.save() hydroshare.delete_resource_file(self.composite_resource.short_id, res_file.id, self.user) # test that we don't have logical file of type GenericLogicalFile self.assertEqual(GenericLogicalFile.objects.count(), 0) self.assertEqual(GenericFileMetaData.objects.count(), 0) # test that resource level coverage element exist - not got deleted self.assertEqual(Coverage.objects.count(), 2)
def test_file_rename(self): # test that a resource file that is part of a GenericLogicalFile object # can be renamed self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) self.assertEqual(res_file.file_name, 'generic_file.txt') # create generic aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # file should not be in a folder self.assertEqual(res_file.file_folder, None) # test rename of file is allowed src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = "data/contents/{0}_1{1}".format(base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_name, '{0}_1{1}'.format(base_file_name, ext)) self.composite_resource.delete()
def test_delete_aggregation_coverage(self): """Here we are testing deleting of temporal and spatial coverage for a single file aggregation """ self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # set file to generic logical file type - single file aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) sf_aggr = GenericLogicalFile.objects.first() # test deleting spatial coverage self.assertEqual(sf_aggr.metadata.spatial_coverage, None) value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'} sf_aggr.metadata.create_element('coverage', type='point', value=value_dict) self.assertNotEqual(sf_aggr.metadata.spatial_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) sf_aggr.metadata.is_dirty = False sf_aggr.metadata.save() sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.spatial_coverage.id) self.assertEqual(sf_aggr.metadata.spatial_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) # test deleting temporal coverage self.assertEqual(sf_aggr.metadata.temporal_coverage, None) value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'} sf_aggr.metadata.create_element('coverage', type='period', value=value_dict) self.assertTrue(sf_aggr.metadata.is_dirty) sf_aggr.metadata.is_dirty = False sf_aggr.metadata.save() self.assertNotEqual(sf_aggr.metadata.temporal_coverage, None) sf_aggr.metadata.delete_element('coverage', sf_aggr.metadata.temporal_coverage.id) self.assertEqual(sf_aggr.metadata.temporal_coverage, None) self.assertTrue(sf_aggr.metadata.is_dirty) self.composite_resource.delete()
def add_file_to_resource(resource, f, folder=None, source_name='', source_size=0, move=False, is_file_reference=False): """ Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource. :param resource: Resource to which file should be added :param f: File-like object to add to a resource :param source_name: the logical file name of the resource content file for federated iRODS resource or the federated zone name; By default, it is empty. A non-empty value indicates the file needs to be added into the federated zone, either from local disk where f holds the uploaded file from local disk, or from the federated zone directly where f is empty but source_name has the whole data object iRODS path in the federated zone :param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is set to 0 and useless. :param move: indicate whether the file should be copied or moved from private user account to proxy user account in federated zone; A value of False indicates copy is needed, a value of True indicates no copy, but the file will be moved from private user account to proxy user account. The default value is False. :param is_file_reference: indicate whether the file being added is a reference to an external file stored in an external zone or URL. source_name will hold the reference file path or url :return: The identifier of the ResourceFile added. """ # importing here to avoid circular import from hs_file_types.models import GenericLogicalFile if f: openfile = File(f) if not isinstance(f, UploadedFile) else f ret = ResourceFile.create(resource, openfile, folder=folder, source=None, move=False) # add format metadata element if necessary file_format_type = get_file_mime_type(f.name) elif source_name: try: # create from existing iRODS file ret = ResourceFile.create(resource, None, folder=folder, source=source_name, source_size=source_size, is_file_reference=is_file_reference, move=move) except SessionException as ex: try: ret.delete() except Exception: pass # raise the exception for the calling function to inform the error on the page interface raise SessionException(ex.exitcode, ex.stdout, ex.stderr) # add format metadata element if necessary file_format_type = get_file_mime_type(source_name) else: raise ValueError( 'Invalid input parameter is passed into this add_file_to_resource() ' 'function') # TODO: generate this from data in ResourceFile rather than extension if file_format_type not in [ mime.value for mime in resource.metadata.formats.all() ]: resource.metadata.create_element('format', value=file_format_type) # if a file gets added successfully to composite resource, then better to set the generic # logical file here if resource.resource_type == "CompositeResource": logical_file = GenericLogicalFile.create() ret.logical_file_content_object = logical_file ret.save() return ret
def test_aggregation_metadata(self): """Test that we can create the following metadata for a generic aggregation coverage (spatial and temporal) key/value metadata title (dataset_name) keywords """ self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # set file to generic logical file type GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) logical_file = res_file.logical_file # test that the generic logical file datataset_name attribute has the value of the # the content file name self.assertEqual(logical_file.dataset_name, base_file_name) dataset_name = "This is a generic dataset" logical_file.dataset_name = dataset_name logical_file.save() logical_file = res_file.logical_file self.assertEqual(logical_file.dataset_name, dataset_name) # check that the logical file has metadata object self.assertNotEqual(logical_file.metadata, None) # there should be no coverage element at this point self.assertEqual(logical_file.metadata.coverages.count(), 0) # there should not be any extra_metadata (key/value) at this point self.assertEqual(logical_file.metadata.extra_metadata, {}) # create key/vale metadata logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2'} logical_file.metadata.save() res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.extra_metadata, {'key1': 'value 1', 'key2': 'value 2'}) # update key/value metadata logical_file.metadata.extra_metadata = {'key1': 'value 1', 'key2': 'value 2', 'key 3': 'value3'} logical_file.metadata.save() res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.extra_metadata, {'key1': 'value 1', 'key2': 'value 2', 'key 3': 'value3'}) # delete key/value metadata logical_file.metadata.extra_metadata = {} logical_file.metadata.save() res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.metadata.extra_metadata, {}) # test keywords self.assertEqual(logical_file.metadata.keywords, []) logical_file.metadata.keywords = ['kw-1', 'kw-2'] logical_file.metadata.save() self.assertIn('kw-1', logical_file.metadata.keywords) self.assertIn('kw-2', logical_file.metadata.keywords) # test coverage element CRUD res_file = [f for f in self.composite_resource.files.all() if f.logical_file_type_name == self.logical_file_type_name][0] gen_logical_file = res_file.logical_file value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'} temp_cov = gen_logical_file.metadata.create_element('coverage', type='period', value=value_dict) self.assertEqual(temp_cov.value['name'], 'Name for period coverage') self.assertEqual(temp_cov.value['start'], '1/1/2000') self.assertEqual(temp_cov.value['end'], '12/12/2012') # update temporal coverage value_dict = {'start': '10/1/2010', 'end': '12/1/2016'} gen_logical_file.metadata.update_element('coverage', temp_cov.id, type='period', value=value_dict) temp_cov = gen_logical_file.metadata.temporal_coverage self.assertEqual(temp_cov.value['name'], 'Name for period coverage') self.assertEqual(temp_cov.value['start'], '10/1/2010') self.assertEqual(temp_cov.value['end'], '12/1/2016') # add spatial coverage value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'} spatial_cov = gen_logical_file.metadata.create_element('coverage', type='point', value=value_dict) self.assertEqual(spatial_cov.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(spatial_cov.value['units'], 'Decimal degree') self.assertEqual(spatial_cov.value['north'], 12.6789) self.assertEqual(spatial_cov.value['east'], 56.45678) # update spatial coverage value_dict = {'east': '-156.45678', 'north': '45.6789', 'units': 'Decimal degree'} gen_logical_file.metadata.update_element('coverage', spatial_cov.id, type='point', value=value_dict) spatial_cov = logical_file.metadata.spatial_coverage self.assertEqual(spatial_cov.value['projection'], 'WGS 84 EPSG:4326') self.assertEqual(spatial_cov.value['units'], 'Decimal degree') self.assertEqual(spatial_cov.value['north'], 45.6789) self.assertEqual(spatial_cov.value['east'], -156.45678) self.composite_resource.delete() # there should be no GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 0) # there should be no GenericFileMetaData object at this point self.assertEqual(GenericFileMetaData.objects.count(), 0)
def test_aggregation_xml_file_paths(self): # test the aggregation meta and map xml file paths with file name and folder name # changes self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, False) # set file to generic logical file type GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) logical_file = res_file.logical_file # file name without extension res_file_name, _ = os.path.splitext(res_file.file_name) expected_meta_path = '{0}{1}'.format(res_file_name, METADATA_FILE_ENDSWITH) expected_map_path = '{0}{1}'.format(res_file_name, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test xml file paths after moving the file into a folder new_folder = 'test_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file res_file_name, _ = os.path.splitext(res_file.file_name) expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH) expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test xml file paths after renaming the file src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file res_file_name, _ = os.path.splitext(res_file.file_name) expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH) expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test the xml file path after renaming the folder folder_rename = '{}_1'.format(new_folder) src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(folder_rename) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file res_file_name, _ = os.path.splitext(res_file.file_name) expected_meta_path = '{0}/{1}{2}'.format(folder_rename, res_file_name, METADATA_FILE_ENDSWITH) expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) self.composite_resource.delete()
def test_file_add_to_composite_resource(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly super(CompositeResourceTest, self).assert_federated_irods_available() # test that when we add file to an existing composite resource, the added file # automatically set to genericlogicalfile type self.assertEqual(BaseResource.objects.count(), 0) self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test Composite Resource With Files Added From Federated Zone', auto_aggregate=False ) # there should not be any GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 0) # add a file to the resource fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, username=self.user.username, fname=self.raster_file_name) res_upload_files = [] resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=None) resource_file_add_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, auto_aggregate=False) # there should be one resource at this point self.assertEqual(BaseResource.objects.count(), 1) self.assertEqual(self.composite_resource.resource_type, "CompositeResource") self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # create the generic aggregation (logical file) GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # check that the resource file is associated with GenericLogicalFile res_file = self.composite_resource.files.first() self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # there should be 1 GenericLogicalFile object at this point self.assertEqual(GenericLogicalFile.objects.count(), 1) # test adding a file to a folder (Note the UI does not support uploading a iRODS file # to a specific folder) # create the folder new_folder = "my-new-folder" new_folder_path = os.path.join("data", "contents", new_folder) create_folder(self.composite_resource.short_id, new_folder_path) resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=new_folder) resource_file_add_process(resource=self.composite_resource, files=res_upload_files, source_names=[fed_test_file_full_path], user=self.user, folder=new_folder, auto_aggregate=False) self.assertEqual(self.composite_resource.files.all().count(), 2) self.composite_resource.delete()