def test_public_or_discoverable(self):
self.assertFalse(self.resModelProgram.has_required_content_files())
self.assertFalse(
self.resModelProgram.metadata.has_all_required_elements())
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
self.assertTrue(self.resModelProgram.has_required_content_files())
self.assertTrue(
self.resModelProgram.metadata.has_all_required_elements())
self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_set_file_type_success_1(self):
# here we will set the tif file to GeoRaster file type
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "rb")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource,
files=(uploaded_file, ),
user=self.user,
auto_aggregate=False)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
# test the set file type endpoint
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
set_file_type_url = url_template.format(
res_id=self.resource.short_id,
file_path=self.raster_file_name,
file_type="GeoRaster")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_allowed_file_types(self):
# test allowed file type is '.*'
self.assertEquals(self.resModelProgram.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# Upload any file type should pass both the file pre add check and post add check
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelProgram.metadata.program, None)
# Uploading any other file type should pass both the file pre add check post add check
files = [UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=True)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=True)
# there should two content files
self.assertEquals(self.resModelProgram.files.all().count(), 2)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelProgram.metadata.program, None)
def test_auto_aggregate_files_add_child_folder(self):
"""test adding files in different folders"""
self.create_composite_resource()
self.assertEqual(0, GeoFeatureLogicalFile.objects.count())
# test add a file that auto-aggregates
dbf_file = open(self.test_file_path.format("watersheds.dbf"), 'r')
shp_file = open(self.test_file_path.format("watersheds.shp"), 'r')
shx_file = open(self.test_file_path.format("watersheds.shx"), 'r')
resource_file_add_process(resource=self.composite_resource,
folder="parentfolder",
files=(dbf_file, shp_file, shx_file), user=self.user,
full_paths={dbf_file: "folder/watersheds.dbf",
shp_file: "folder/watersheds.shp",
shx_file: "folder/watersheds.shx"})
# because the files are in different folders, auto aggreate won't work
self.assertEqual(1, GeoFeatureLogicalFile.objects.count())
storage_paths = ["parentfolder/folder/watersheds.dbf",
"parentfolder/folder/watersheds.shp",
"parentfolder/folder/watersheds.shx"]
for res_file in self.composite_resource.files.all():
index = -1
for i, name in enumerate(storage_paths):
if name == res_file.storage_path:
index = i
break
del storage_paths[index]
self.assertEquals(0, len(storage_paths))
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(self.resRaster.metadata.title.value, 'My Test Raster Resource')
# there shouldn't any abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should not be any extended metadata
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
# adding a valid tiff file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
super(TestRasterMetaData, self).raster_metadata_extraction()
def test_set_file_type_success_2(self):
# here we will set the tif file (the file being not in root dir)to GeoRaster file type
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "r")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user,
auto_aggregate=False)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
create_folder(self.resource.short_id, 'data/contents/sub_test_dir')
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(self.user, self.resource.short_id,
'data/contents/' + self.raster_file_name,
'data/contents/sub_test_dir/' + self.raster_file_name)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.short_path, "sub_test_dir/" + self.raster_file_name)
# test the set file type endpoint
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
set_file_type_url = url_template.format(res_id=self.resource.short_id,
file_path=res_file.short_path,
file_type="GeoRaster")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_metadata_extraction_DIS_file(self):
# Extract Metadata from DIS File
files = [UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name)]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadeta=False)
# ---Tests for Grid Dimensions
# Number of Layers
self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4))
# Type of Rows
self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular')
# Number of Rows
self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20))
# Type of Columns
self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular')
# Number of Columns
self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15))
# ---Tests for Stress Period
# Stress Period Type
self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient')
# ---Tests for Study Area
# Total Length
self.assertEqual(self.res.metadata.study_area.totalLength, str(20*2000.0)) # double
# Total Width
self.assertEqual(self.res.metadata.study_area.totalWidth, str(15*2000.0)) # double
def test_set_file_type_failure_2(self):
# here we will set the tif file to GeoRaster file type with an invalid file path
# which should fail
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "r")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user,
auto_aggregate=False)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
# test the set file type endpoint using a wrong file path
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
file_path = os.path.join("no-such-folder", self.raster_file_name)
set_file_type_url = url_template.format(res_id=self.resource.short_id,
file_path=file_path,
file_type="GeoRaster")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_file_extensions(self):
# set url launching pattern for aggregations
metadata = [{
'requesturlbasefile': {
'value': 'https://www.google.com?agg_path=${HS_FILE_PATH}'
}
}]
self.resWebApp.metadata.update(metadata, self.user)
self.assertEqual(RequestUrlBaseFile.objects.all().count(), 1)
# set web app to launch for geo raster
metadata = [{'supportedfileextensions': {'value': '.tif'}}]
self.resWebApp.metadata.update(metadata, self.user)
self.assertEqual(SupportedFileExtensions.objects.all().count(), 1)
# add a geo raster aggregation to the resource
open_file = open(self.test_file_path.format("small_logan.tif"), 'r')
resource_file_add_process(resource=self.resComposite,
files=(open_file, ),
user=self.user)
# setup the web app to be launched by the resource
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
self.assertEqual(self.resWebApp.extra_metadata, {})
self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'}
self.resWebApp.save()
self.assertNotEqual(self.resWebApp.extra_metadata, {})
self.assertEqual(self.resWebApp.extra_metadata[tool_app_key],
'test-app-value')
self.assertEqual(self.resComposite.extra_metadata, {})
self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'}
self.resComposite.save()
self.assertNotEqual(self.resComposite.extra_metadata, {})
self.assertEqual(self.resComposite.extra_metadata[tool_app_key],
'test-app-value')
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
# get the web tools
relevant_tools = resource_level_tool_urls(self.resComposite, request)
self.assertIsNotNone(
relevant_tools,
msg='relevant_tools should not be None with appkey '
'matching')
tc = relevant_tools['resource_level_app_counter']
self.assertEqual(0, tc)
tl = relevant_tools['tool_list'][0]
self.assertEqual(self.resWebApp.short_id, tl['res_id'])
self.assertFalse(tl['approved'])
self.assertTrue(tl['openwithlist'])
self.assertEqual('', tl['agg_types'])
self.assertEqual('.tif', tl['file_extensions'])
def test_public_or_discoverable(self):
self.assertFalse(self.resRaster.has_required_content_files())
self.assertFalse(self.resRaster.metadata.has_all_required_elements())
self.assertFalse(self.resRaster.can_be_public_or_discoverable)
# adding a valid raster file
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=True)
# adding required metadata
self.resRaster.metadata.create_element('description',
abstract='example abstract')
self.resRaster.metadata.create_element('subject', value='logan')
self.assertTrue(self.resRaster.has_required_content_files())
self.assertTrue(self.resRaster.metadata.has_all_required_elements())
self.assertTrue(self.resRaster.can_be_public_or_discoverable)
self.resRaster.delete()
def test_set_file_type_failure_1(self):
# here we will set the tif file to NetCDF file type which should fail
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "r")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource,
files=(uploaded_file, ),
user=self.user)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
# test the set file type endpoint using a wrong file type (NetCDF)
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
set_file_type_url = url_template.format(
res_id=self.resource.short_id,
file_path=self.raster_file_name,
file_type="NetCDF")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_agg_types(self):
# set url launching pattern for aggregations
metadata = [{'requesturlbaseaggregation': {
'value': 'https://www.google.com?agg_path=${HS_AGG_PATH}'}}]
self.resWebApp.metadata.update(metadata, self.user)
self.assertEqual(RequestUrlBaseAggregation.objects.all().count(), 1)
# set web app to launch for geo raster
self.resWebApp.metadata.update(metadata, self.user)
resource.create_metadata_element(self.resWebApp.short_id, 'SupportedAggTypes',
supported_agg_types=['GeoRasterLogicalFile'])
self.assertEqual(1, SupportedAggTypes.objects.all().count())
# add a geo raster aggregation to the resource
open_file = open(self.test_file_path.format("small_logan.tif"), 'rb')
resource_file_add_process(resource=self.resComposite,
files=(open_file,), user=self.user)
# setup the web app to be launched by the resource
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
self.assertEqual(self.resWebApp.extra_metadata, {})
self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'}
self.resWebApp.save()
self.assertNotEqual(self.resWebApp.extra_metadata, {})
self.assertEqual(self.resWebApp.extra_metadata[tool_app_key], 'test-app-value')
self.assertEqual(self.resComposite.extra_metadata, {})
self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'}
self.resComposite.save()
self.assertNotEqual(self.resComposite.extra_metadata, {})
self.assertEqual(self.resComposite.extra_metadata[tool_app_key], 'test-app-value')
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
# get the web tools and ensure agg_types is there
relevant_tools = resource_level_tool_urls(self.resComposite, request)
self.assertIsNotNone(relevant_tools, msg='relevant_tools should not be None with appkey '
'matching')
tc = relevant_tools['resource_level_app_counter']
self.assertEqual(0, tc)
tl = relevant_tools['tool_list'][0]
self.assertEqual(self.resWebApp.short_id, tl['res_id'])
self.assertEqual('GeoRasterLogicalFile', tl['agg_types'])
self.assertEqual('', tl['file_extensions'])
# Remove appkey to turn off openwithlist for this resource
self.resWebApp.extra_metadata = {}
self.resWebApp.save()
relevant_tools = resource_level_tool_urls(self.resComposite, request)
self.assertIsNone(relevant_tools, msg='relevant_tools should have no approved resources')
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self):
files = [UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
self._test_metadata_extraction_wkt_crs()
def test_metadata_on_content_file_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element(
'MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelProgram.metadata.formats.all().count(),
1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelProgram.short_id,
self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelProgram.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelProgram.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelProgram.metadata.creators.all().count(),
1)
# testing extended metadata elements
self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_agg_types(self):
# set url launching pattern for aggregations
metadata = [{'requesturlbaseaggregation': {
'value': 'https://www.google.com?agg_path=${HS_AGG_PATH}'}}]
self.resWebApp.metadata.update(metadata, self.user)
self.assertEqual(RequestUrlBaseAggregation.objects.all().count(), 1)
# set web app to launch for geo raster
self.resWebApp.metadata.update(metadata, self.user)
resource.create_metadata_element(self.resWebApp.short_id, 'SupportedAggTypes',
supported_agg_types=['GeoRasterLogicalFile'])
self.assertEqual(1, SupportedAggTypes.objects.all().count())
# add a geo raster aggregation to the resource
open_file = open(self.test_file_path.format("small_logan.tif"), 'r')
resource_file_add_process(resource=self.resComposite,
files=(open_file,), user=self.user)
# setup the web app to be launched by the resource
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
self.assertEqual(self.resWebApp.extra_metadata, {})
self.resWebApp.extra_metadata = {tool_app_key: 'test-app-value'}
self.resWebApp.save()
self.assertNotEqual(self.resWebApp.extra_metadata, {})
self.assertEqual(self.resWebApp.extra_metadata[tool_app_key], 'test-app-value')
self.assertEqual(self.resComposite.extra_metadata, {})
self.resComposite.extra_metadata = {tool_app_key: 'test-app-value'}
self.resComposite.save()
self.assertNotEqual(self.resComposite.extra_metadata, {})
self.assertEqual(self.resComposite.extra_metadata[tool_app_key], 'test-app-value')
url = '/resource/' + self.resComposite.short_id + '/'
request = self.factory.get(url)
request.user = self.user
# get the web tools and ensure agg_types is there
relevant_tools = resource_level_tool_urls(self.resComposite, request)
self.assertIsNotNone(relevant_tools, msg='relevant_tools should not be None with appkey '
'matching')
tc = relevant_tools['resource_level_app_counter']
self.assertEqual(0, tc)
tl = relevant_tools['tool_list'][0]
self.assertEqual(self.resWebApp.short_id, tl['res_id'])
self.assertFalse(tl['approved'])
self.assertTrue(tl['openwithlist'])
self.assertEqual('GeoRasterLogicalFile', tl['agg_types'])
self.assertEqual('', tl['file_extensions'])
def test_metadata_extraction(self):
# test allowed file type is '.*'
self.assertEqual(self.res.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEqual(self.res.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.sample_nam_obj,
name=self.sample_nam_obj.name)
]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
# here are the things that the modflow modelinstance should include from the .nam file
# GroundWaterFlow
# flowPackage: 'LPF', 'UZF'(?)
#
# GeneralElements
# modelSolver: 'SIP'
# outputcontrolpackage: 'OC', 'GAGE'
#
# BoundaryCondition
# head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?)
# specified_flux_boundary_packages: 'WEL'
self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP')
self.assertTrue(
self.res.metadata.ground_water_flow.unsaturatedZonePackage)
self.assertIn(
'GHB',
self.res.metadata.boundary_condition.
get_head_dependent_flux_boundary_packages())
self.assertIn(
'SFR',
self.res.metadata.boundary_condition.
get_head_dependent_flux_boundary_packages())
self.assertIn(
'WEL',
self.res.metadata.boundary_condition.
get_specified_flux_boundary_packages())
self.assertIn(
'OC',
self.res.metadata.general_elements.get_output_control_package())
self.assertIn(
'GAGE',
self.res.metadata.general_elements.get_output_control_package())
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# there should be 2 content files
self.assertEqual(self.resRaster.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resRaster.metadata.formats.all().count(), 2)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='application/vrt').count(), 1)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='image/tiff').count(), 1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resRaster.metadata.title, None)
# there should be no abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there should be 1 creator element
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should be no coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there should be no subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# testing extended metadata elements - there should not be any resource specific metadata
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
self.resRaster.delete()
def test_allowed_file_types(self):
# test allowed file type is '.nc'
self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types())
self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()),
1)
# there should not be any content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .nc file should raise file validation error
files = [
UploadedFile(file=self.netcdf_bad_file_obj,
name=self.netcdf_bad_file_name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# trying to add valid .nc file should pass the file check
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# there should be 2 content file: with ncdump file created by system
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# file pre add process should raise validation error if we try to add a 2nd file
# when the resource has already 2 content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
# there should be 1 content files
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 1 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file that we added above
hydroshare.delete_resource_file(self.resNetcdf.short_id,
self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEqual(self.resNetcdf.metadata.description, None)
# there should be 2 creator element
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(
self.resNetcdf.metadata.title.value,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010'
)
# there shouldn't any abstract element
self.assertEqual(self.resNetcdf.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0)
# there shouldn't any source element
self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0)
# there shouldn't any relation element
self.assertEqual(
self.resNetcdf.metadata.relations.all().filter(
type='cites').count(), 0)
# there should be 1 creator
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there shouldn't any extended metadata
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
super(TestNetcdfMetaData,
self).netcdf_metadata_extraction(expected_creators_count=2)
def test_auto_aggregate_file_add_tif(self):
"""test that auto-aggregate works on tif file add"""
self.create_composite_resource()
self.assertEqual(0, GeoRasterLogicalFile.objects.count())
# test add a file that auto-aggregates
open_file = open(self.test_file_path.format("small_logan.tif"), 'r')
resource_file_add_process(resource=self.composite_resource,
files=(open_file,), user=self.user)
self.assertEqual(1, GeoRasterLogicalFile.objects.count())
def test_get_xml(self):
# add a valid raster file to generate metadata
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# test if xml from get_xml() is well formed
ET.fromstring(self.resRaster.metadata.get_xml())
self.resRaster.delete()
def test_auto_aggregate_file_add_nc(self):
"""test that auto-aggregate works on nc file add"""
self.create_composite_resource()
self.assertEqual(0, NetCDFLogicalFile.objects.count())
# test add a file that auto-aggregates
open_file = open(self.test_file_path.format("netcdf_valid.nc"), 'r')
resource_file_add_process(resource=self.composite_resource,
files=(open_file,), user=self.user)
# because of auto aggregation, there should be 2 files
self.assertEqual(1, NetCDFLogicalFile.objects.count())
def test_auto_aggregate_file_add_refts(self):
"""test that auto-aggregate works on refts file add"""
self.create_composite_resource()
self.assertEqual(0, RefTimeseriesLogicalFile.objects.count())
# test add a file that auto-aggregates
open_file = open(self.test_file_path.format("multi_sites_formatted_version1.0.refts.json"),
'r')
resource_file_add_process(resource=self.composite_resource,
files=(open_file,), user=self.user)
self.assertEqual(1, RefTimeseriesLogicalFile.objects.count())
def test_auto_aggregate_file_add_sqlite(self):
"""test that auto-aggregate works on sqlite file add"""
self.create_composite_resource()
self.assertEqual(0, TimeSeriesLogicalFile.objects.count())
# test add a file that auto-aggregates
open_file = open(self.test_file_path.format("ODM2.sqlite"), 'r')
resource_file_add_process(resource=self.composite_resource,
files=(open_file,), user=self.user)
self.assertEqual(1, TimeSeriesLogicalFile.objects.count())
def test_allowed_file_types(self):
# test allowed file type is '.*'
self.assertEquals(
self.resModelInstance.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelInstance.metadata.model_output, None)
self.assertEquals(self.resModelInstance.metadata.executed_by, None)
# Upload any other file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.text_file_obj_2,
name=self.text_file_obj_2.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=True)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=True)
# there should two content files
self.assertEquals(self.resModelInstance.files.all().count(), 2)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelInstance.metadata.model_output, None)
self.assertEquals(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
self.resModelInstance.metadata.create_element('Description',
abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject',
value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput',
includes_output=True)
self.resModelInstance.metadata.create_element(
'ExecutedBy', model_name=self.resModelProgram.short_id)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelInstance.metadata.formats.all().count(),
1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelInstance.short_id,
self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelInstance.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelInstance.metadata.description, None)
# there should be one creator element
self.assertEquals(
self.resModelInstance.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelInstance.metadata.model_output, None)
self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1)
# the short path should just consist of the file name.
self.assertEquals(self.resModelProgram.files.all()[0].short_path, self.file_name)
# delete content file that we added above; note that file name is a short_path
hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelProgram.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelProgram.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self):
files = [
UploadedFile(file=self.netcdf_file_obj_crs,
name=self.netcdf_file_name_crs)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
self._test_metadata_extraction_wkt_crs()
def test_auto_aggregate_file_add_geo_feature(self):
"""test that auto-aggregate works on geo feature file add"""
self.create_composite_resource()
self.assertEqual(0, GeoFeatureLogicalFile.objects.count())
# test add a file that auto-aggregates
dbf_file = open(self.test_file_path.format("watersheds.dbf"), 'r')
shp_file = open(self.test_file_path.format("watersheds.shp"), 'r')
shx_file = open(self.test_file_path.format("watersheds.shx"), 'r')
resource_file_add_process(resource=self.composite_resource,
files=(dbf_file, shp_file, shx_file), user=self.user)
self.assertEqual(1, GeoFeatureLogicalFile.objects.count())
def test_metadata_extraction(self):
# test allowed file type is '.*'
self.assertEqual(self.res.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEqual(self.res.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name)]
utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res, files=files, user=self.user,
extract_metadata=False)
# here are the things that the modflow modelinstance should include from the .nam file
# GroundWaterFlow
# flowPackage: 'LPF', 'UZF'(?)
#
# GeneralElements
# modelSolver: 'SIP'
# outputcontrolpackage: 'OC', 'GAGE'
#
# BoundaryCondition
# head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?)
# specified_flux_boundary_packages: 'WEL'
self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP')
self.assertTrue(self.res.metadata.ground_water_flow.unsaturatedZonePackage)
self.assertIn(
'GHB',
self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages()
)
self.assertIn(
'SFR',
self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages()
)
self.assertIn(
'WEL',
self.res.metadata.boundary_condition.get_specified_flux_boundary_packages()
)
self.assertIn(
'OC',
self.res.metadata.general_elements.get_output_control_package()
)
self.assertIn(
'GAGE',
self.res.metadata.general_elements.get_output_control_package()
)
def test_public_or_discoverable(self):
self.assertFalse(self.resNetcdf.has_required_content_files())
self.assertFalse(self.resNetcdf.metadata.has_all_required_elements())
self.assertFalse(self.resNetcdf.can_be_public_or_discoverable)
# adding a valid netcdf file
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=True)
self.assertTrue(self.resNetcdf.has_required_content_files())
self.assertTrue(self.resNetcdf.metadata.has_all_required_elements())
self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=True)
# there should be 1 content files
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 1 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file that we added above
hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEquals(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEquals(self.resNetcdf.metadata.description, None)
# there should be 2 creator element
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEquals(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_DIS_file(self):
# Extract Metadata from DIS File
files = [
UploadedFile(file=self.sample_dis_obj,
name=self.sample_dis_obj.name)
]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadeta=False)
# ---Tests for Grid Dimensions
# Number of Layers
self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers,
str(4))
# Type of Rows
self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows,
'Regular')
# Number of Rows
self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows,
str(20))
# Type of Columns
self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns,
'Regular')
# Number of Columns
self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns,
str(15))
# ---Tests for Stress Period
# Stress Period Type
self.assertEqual(self.res.metadata.stress_period.stressPeriodType,
'Steady and Transient')
# ---Tests for Study Area
# Total Length
self.assertEqual(self.res.metadata.study_area.totalLength,
str(20 * 2000.0)) # double
# Total Width
self.assertEqual(self.res.metadata.study_area.totalWidth,
str(15 * 2000.0)) # double
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(
self.resNetcdf.metadata.title.value,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010')
# there shouldn't any abstract element
self.assertEqual(self.resNetcdf.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0)
# there shouldn't any source element
self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0)
# there shouldn't any relation element
self.assertEqual(self.resNetcdf.metadata.relations.all().filter(type='cites').count(), 0)
# there should be 1 creator
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there shouldn't any extended metadata
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_file_add_to_composite_resource(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(CompositeResourceTest,
self).is_federated_irods_available():
return
# test that when we add file to an existing composite resource, the added file
# automatically set to genericlogicalfile type
self.assertEqual(BaseResource.objects.count(), 0)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test Composite Resource Federated')
# there should not be any GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# add a file to the resource
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
resource_file_add_process(resource=self.composite_resource,
files=res_upload_files,
fed_res_file_names=[fed_test_file_full_path],
user=self.user)
# there should be one resource at this point
self.assertEqual(BaseResource.objects.count(), 1)
self.assertEqual(self.composite_resource.resource_type,
"CompositeResource")
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# there should be 1 GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 1)
def test_get_xml(self):
# add a valid raster file to generate metadata
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=True)
# test if xml from get_xml() is well formed
ET.fromstring(self.resRaster.metadata.get_xml())
self.resRaster.delete()
def _test_csv_file_add(self, new_folder=None):
self.create_composite_resource()
# no timeseries aggregation prior to adding the csv file to a folder
self.assertEqual(0, TimeSeriesLogicalFile.objects.count())
if new_folder is not None:
# create a folder
hydroshare.ResourceFile.create_folder(self.composite_resource, folder=new_folder)
# upload the csv file to the folder (new-folder)
csv_file_name = "ODM2_One_Site_One_Series_Test.csv"
csv_file = open(self.test_file_path.format(csv_file_name), 'r')
files = [UploadedFile(file=csv_file, name=csv_file_name)]
resource_file_add_process(resource=self.composite_resource,
files=files, user=self.user, folder=new_folder)
# check that the resource has one file
self.assertEqual(self.composite_resource.files.all().count(), 1)
csv_res_file = self.composite_resource.files.first()
self.assertEqual(csv_res_file.file_folder, new_folder)
# no timeseries aggregation after adding the scv file to a folder
self.assertEqual(0, TimeSeriesLogicalFile.objects.count())
def test_public_or_discoverable(self):
self.assertFalse(self.resRaster.has_required_content_files())
self.assertFalse(self.resRaster.metadata.has_all_required_elements())
self.assertFalse(self.resRaster.can_be_public_or_discoverable)
# adding a valid raster file
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# adding required metadata
self.resRaster.metadata.create_element('description', abstract='example abstract')
self.resRaster.metadata.create_element('subject', value='logan')
self.assertTrue(self.resRaster.has_required_content_files())
self.assertTrue(self.resRaster.metadata.has_all_required_elements())
self.assertTrue(self.resRaster.can_be_public_or_discoverable)
self.resRaster.delete()
def test_metadata_on_content_file_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
self.resModelInstance.metadata.create_element('Description', abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject', value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True)
self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1)
# created file should have a short path equal to the declared name
self.assertEquals(self.resModelInstance.files.all()[0].short_path, self.file_name)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelInstance.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelInstance.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelInstance.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelInstance.metadata.model_output, None)
self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_public_or_discoverable(self):
self.assertFalse(self.resNetcdf.has_required_content_files())
self.assertFalse(self.resNetcdf.metadata.has_all_required_elements())
self.assertFalse(self.resNetcdf.can_be_public_or_discoverable)
# adding a valid netcdf file
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
self.assertTrue(self.resNetcdf.has_required_content_files())
self.assertTrue(self.resNetcdf.metadata.has_all_required_elements())
self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_allowed_file_types(self):
# test allowed file type is '.nc'
self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types())
self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1)
# there should not be any content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .nc file should raise file validation error
files = [UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
# trying to add valid .nc file should pass the file check
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
# there should be 2 content file: with ncdump file created by system
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# file pre add process should raise validation error if we try to add a 2nd file
# when the resource has already 2 content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
def test_public_or_discoverable(self):
self.assertFalse(self.resModelProgram.has_required_content_files())
self.assertFalse(self.resModelProgram.metadata.has_all_required_elements())
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Subject', value="test subject")
self.assertTrue(self.resModelProgram.has_required_content_files())
self.assertTrue(self.resModelProgram.metadata.has_all_required_elements())
self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_set_file_type_success_2(self):
# here we will set the tif file (the file being not in root dir)to GeoRaster file type
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "r")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource,
files=(uploaded_file, ),
user=self.user)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
create_folder(self.resource.short_id, 'data/contents/sub_test_dir')
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(
self.user, self.resource.short_id,
'data/contents/' + self.raster_file_name,
'data/contents/sub_test_dir/' + self.raster_file_name)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.short_path,
"sub_test_dir/" + self.raster_file_name)
# test the set file type endpoint
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
set_file_type_url = url_template.format(res_id=self.resource.short_id,
file_path=res_file.short_path,
file_type="GeoRaster")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(self.resRaster.metadata.title.value,
'My Test Raster Resource')
# there shouldn't any abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should not be any extended metadata
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
# adding a valid tiff file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
super(TestRasterMetaData, self).raster_metadata_extraction()
def test_bulk_metadata_update(self):
# here we are testing the update() method of the RasterMetaData class
# update of resource specific metadata should fail when the resource does not have content
# files
self.assertEqual(self.resRaster.files.all().count(), 0)
self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 0)
band_data = {'original_band_name': 'bandinfo',
'name': 'Band_1',
'variableName': 'digital elevation',
'variableUnit': 'meter',
'method': 'this is method',
'comment': 'this is comment',
'maximumValue': 1000,
'minimumValue': 0,
'noDataValue': -9999
}
metadata = []
metadata.append({'bandinformation': band_data})
with self.assertRaises(ValidationError):
self.resRaster.metadata.update(metadata, self.user)
self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 0)
del metadata[:]
# adding a valid tiff file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
# testing extended metadata element: band information
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 1)
band_info = self.resRaster.metadata.bandInformations.first()
self.assertEqual(band_info.noDataValue, '-3.40282346639e+38')
self.assertEqual(band_info.maximumValue, '3031.44311523')
self.assertEqual(band_info.minimumValue, '1358.33459473')
self.assertEqual(band_info.name, 'Band_1')
# updating of bandinformation using a name that does not exist (band-name) should fail
band_data = {'original_band_name': 'band-name',
'name': 'Band_1',
'variableName': 'digital elevation',
'variableUnit': 'meter',
'method': 'this is method',
'comment': 'this is comment',
'maximumValue': 1000,
'minimumValue': 0,
'noDataValue': -9999
}
metadata.append({'bandinformation': band_data})
with self.assertRaises(ValidationError):
self.resRaster.metadata.update(metadata, self.user)
self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1)
# updating of bandinformation using a valid band lookup name (Band_1) should be successful
band_data = {'original_band_name': 'Band_1',
'name': 'Band_2',
'variableName': 'digital elevation',
'variableUnit': 'meter',
'method': 'this is method',
'comment': 'this is comment',
'maximumValue': 1000,
'minimumValue': 0,
'noDataValue': -9999
}
del metadata[:]
metadata.append({'bandinformation': band_data})
self.resRaster.metadata.update(metadata, self.user)
self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1)
band_info = self.resRaster.metadata.bandInformations.first()
self.assertEqual(band_info.name, 'Band_2')
self.assertEqual(band_info.variableName, 'digital elevation')
self.assertEqual(band_info.variableUnit, 'meter')
self.assertEqual(band_info.method, 'this is method')
self.assertEqual(band_info.comment, 'this is comment')
self.assertEqual(band_info.maximumValue, '1000')
self.assertEqual(band_info.minimumValue, '0')
self.assertEqual(band_info.noDataValue, '-9999')
# test updating only one attribute of bandinformation
band_data = {'original_band_name': 'Band_2',
'name': 'Band_1'
}
del metadata[:]
metadata.append({'bandinformation': band_data})
self.resRaster.metadata.update(metadata, self.user)
self.assertEqual(self.resRaster.metadata.bandInformations.all().count(), 1)
band_info = self.resRaster.metadata.bandInformations.first()
self.assertEqual(band_info.name, 'Band_1')
# test updating both core and resource specific metadata
# there should be 1 creator
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
del metadata[:]
metadata.append({'creator': {'name': 'creator one'}})
metadata.append({'creator': {'name': 'creator two'}})
metadata.append({'contributor': {'name': 'contributor one'}})
metadata.append({'contributor': {'name': 'contributor two'}})
band_data = {'original_band_name': 'Band_1',
'name': 'Band_3'
}
metadata.append({'bandinformation': band_data})
self.resRaster.metadata.update(metadata, self.user)
band_info = self.resRaster.metadata.bandInformations.first()
self.assertEqual(band_info.name, 'Band_3')
# there should be 2 creators
self.assertEqual(self.resRaster.metadata.creators.all().count(), 2)
# there should be 2 contributor
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 2)
self.resRaster.delete()
def test_metadata_delete_on_resource_delete(self):
# adding a valid raster tif file should generate some core metadata and all extended
# metadata
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# before resource delete
# resource core metadata
raster_metadata_obj = self.resRaster.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Coverage metadata objects
self.assertTrue(Coverage.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=raster_metadata_obj.id).exists())
# resource specific metadata
# there should be original coverage metadata objects
self.assertTrue(OriginalCoverage.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be CellInformation metadata objects
self.assertTrue(CellInformation.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be BandInformation metadata objects
self.assertTrue(BandInformation.objects.filter(object_id=raster_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resRaster.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=raster_metadata_obj.id).exists())
# resource specific metadata
# there should be no original coverage metadata objects
self.assertFalse(OriginalCoverage.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be CellInformation metadata objects
self.assertFalse(CellInformation.objects.filter(object_id=raster_metadata_obj.id).exists())
# there should be bandInformation metadata objects
self.assertFalse(BandInformation.objects.filter(object_id=raster_metadata_obj.id).exists())
def test_allowed_file_types(self):
# test allowed file types are: '.tiff', '.tif,', '.vrt' and '.zip'
self.assertIn('.tiff', RasterResource.get_supported_upload_file_types())
self.assertIn('.tif', RasterResource.get_supported_upload_file_types())
self.assertIn('.vrt', RasterResource.get_supported_upload_file_types())
self.assertIn('.zip', RasterResource.get_supported_upload_file_types())
self.assertEqual(len(RasterResource.get_supported_upload_file_types()), 4)
# there should not be any content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .tif file should raise file validation error
files = [UploadedFile(file=self.raster_bad_tif_file_obj,
name=self.raster_bad_tif_file_name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
# trying to add good .tif file should pass the file check
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
# there should be 2 content file: with .vrt file created by system
self.assertEqual(self.resRaster.files.all().count(), 2)
file_names = [os.path.basename(f.resource_file.name) for f in self.resRaster.files.all()]
self.assertIn('raster_tif_valid.vrt', file_names)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should be no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add good .zip file should pass the file check
files = [UploadedFile(file=self.raster_zip_file_obj, name=self.raster_zip_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
# there should be 10 content file:
self.assertEqual(self.resRaster.files.all().count(), 10)
# file pre add process should raise validation error if we try to add a 2nd file when
# the resource has already content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
def test_metadata_delete_on_resource_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element(
'MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
self.resModelProgram.metadata.create_element('Contributor',
name="user2")
# before resource delete
core_metadata_obj = self.resModelProgram.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Program metadata objects
self.assertTrue(
MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelProgram.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(
MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(TestNetcdfMetaData, self).is_federated_irods_available():
return
# test metadata extraction with resource creation with nc file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.netcdf_file_name)
res_upload_files = []
_, _, metadata, fed_res_path = utils.resource_pre_create_actions(
resource_type='NetcdfResource',
resource_title=
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June '
'2010',
page_redirect_url_key=None,
files=res_upload_files,
source_names=[fed_test_file_full_path])
self.resNetcdf = hydroshare.create_resource(
'NetcdfResource',
self.user,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '',
move=False,
metadata=metadata)
utils.resource_post_create_actions(self.resNetcdf, self.user, metadata)
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
# test metadata is deleted after content file is deleted in user zone space
# there should be 2 content files at this point
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file now
hydroshare.delete_resource_file(self.resNetcdf.short_id,
self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEqual(self.resNetcdf.metadata.description, None)
# there should be 1 creator element (based on the extracted metadata)
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# test metadata extraction with a valid nc file being added coming from user zone space
res_add_files = []
utils.resource_file_add_pre_process(
resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
utils.resource_file_add_process(resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resNetcdf.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_in_user_zone(self):
super(TestRasterMetaData, self).assert_federated_irods_available()
# test metadata extraction with resource creation with tif file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.raster_tif_file_name)
res_upload_files = []
_, _, metadata, fed_res_path = utils.resource_pre_create_actions(
resource_type='RasterResource',
resource_title='My Test Raster Resource',
page_redirect_url_key=None,
files=res_upload_files,
source_names=[fed_test_file_full_path])
self.resRaster = hydroshare.create_resource(
'RasterResource',
self.user,
'My Test Raster Resource',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '',
move=False,
metadata=metadata)
# raster file validation and metadata extraction in post resource creation signal handler
utils.resource_post_create_actions(resource=self.resRaster, user=self.user,
metadata=[])
super(TestRasterMetaData, self).raster_metadata_extraction()
# test metadata is deleted after content file is deleted in user zone space
# there should be 2 content file: tif file and vrt file at this point
self.assertEqual(self.resRaster.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resRaster.metadata.formats.all().count(), 2)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='application/vrt').count(), 1)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='image/tiff').count(), 1)
# delete content file now
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should be no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resRaster.metadata.title, None)
# there should be no abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there should be 1 creator element
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should be no coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there should be no subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# testing extended metadata elements - there should be no extended metadata elements
# at this point
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
# test metadata extraction with a valid tif file being added coming from user zone space
res_add_files = []
# now necessary in order to test add_process
utils.resource_file_add_pre_process(resource=self.resRaster,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
# file validation and metadata extraction happen during post file add signal handler
utils.resource_file_add_process(resource=self.resRaster,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
super(TestRasterMetaData, self).raster_metadata_extraction()
# test metadata deletion when deleting a resource in user zone space
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resRaster.short_id)
# resource core metadata is deleted after resource deletion
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# test adding file from user zone to existing empty resource in hydroshare zone
# even there is no file uploaded to resource initially, there are default extended
# automatically metadata created
self.resRaster = hydroshare.create_resource(
resource_type='RasterResource',
owner=self.user,
title='My Test Raster Resource'
)
# test metadata extraction with a valid tif file being added coming from user zone space
# file validation and metadata extraction happen during post file add signal handler
utils.resource_file_add_process(resource=self.resRaster,
files=[],
user=self.user,
source_names=[fed_test_file_full_path])
super(TestRasterMetaData, self).raster_metadata_extraction()
# there should be 2 content file: tif file and vrt file at this point
self.assertEqual(self.resRaster.files.all().count(), 2)
# delete resource
hydroshare.delete_resource(self.resRaster.short_id)
# resource core metadata is deleted after resource deletion
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
self.resModelProgram.metadata.create_element('Contributor', name="user2")
# before resource delete
core_metadata_obj = self.resModelProgram.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Program metadata objects
self.assertTrue(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelProgram.short_id)
self.assertEquals(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
