def test_allowed_file_types(self):
# test allowed file type is '.*'
self.assertEquals(self.resModelProgram.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# Upload any file type should pass both the file pre add check and post add check
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelProgram.metadata.program, None)
# Uploading any other file type should pass both the file pre add check post add check
files = [UploadedFile(file=self.text_file_obj_2, name=self.text_file_obj_2.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=True)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=True)
# there should two content files
self.assertEquals(self.resModelProgram.files.all().count(), 2)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelProgram.metadata.program, None)
def test_metadata_extraction_DIS_file(self):
# Extract Metadata from DIS File
files = [UploadedFile(file=self.sample_dis_obj, name=self.sample_dis_obj.name)]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadeta=False)
# ---Tests for Grid Dimensions
# Number of Layers
self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers, str(4))
# Type of Rows
self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows, 'Regular')
# Number of Rows
self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows, str(20))
# Type of Columns
self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns, 'Regular')
# Number of Columns
self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns, str(15))
# ---Tests for Stress Period
# Stress Period Type
self.assertEqual(self.res.metadata.stress_period.stressPeriodType, 'Steady and Transient')
# ---Tests for Study Area
# Total Length
self.assertEqual(self.res.metadata.study_area.totalLength, str(20*2000.0)) # double
# Total Width
self.assertEqual(self.res.metadata.study_area.totalWidth, str(15*2000.0)) # double
def test_public_or_discoverable(self):
self.assertFalse(self.resRaster.has_required_content_files())
self.assertFalse(self.resRaster.metadata.has_all_required_elements())
self.assertFalse(self.resRaster.can_be_public_or_discoverable)
# adding a valid raster file
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=True)
# adding required metadata
self.resRaster.metadata.create_element('description',
abstract='example abstract')
self.resRaster.metadata.create_element('subject', value='logan')
self.assertTrue(self.resRaster.has_required_content_files())
self.assertTrue(self.resRaster.metadata.has_all_required_elements())
self.assertTrue(self.resRaster.can_be_public_or_discoverable)
self.resRaster.delete()
def test_public_or_discoverable(self):
self.assertFalse(self.resModelProgram.has_required_content_files())
self.assertFalse(
self.resModelProgram.metadata.has_all_required_elements())
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
self.assertTrue(self.resModelProgram.has_required_content_files())
self.assertTrue(
self.resModelProgram.metadata.has_all_required_elements())
self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def add_file_to_resource(request, shortkey, *args, **kwargs):
resource, _, _ = authorize(request, shortkey, edit=True, full=True, superuser=True)
res_files = request.FILES.getlist('files')
extract_metadata = request.REQUEST.get('extract-metadata', 'No')
extract_metadata = True if extract_metadata.lower() == 'yes' else False
try:
utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata)
except hydroshare.utils.ResourceFileSizeException as ex:
request.session['file_size_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
request.session['file_validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
try:
hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata)
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
request.session['file_validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self):
files = [UploadedFile(file=self.netcdf_file_obj_crs, name=self.netcdf_file_name_crs)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
self._test_metadata_extraction_wkt_crs()
def test_metadata_on_content_file_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element(
'MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelProgram.metadata.formats.all().count(),
1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelProgram.short_id,
self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelProgram.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelProgram.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelProgram.metadata.creators.all().count(),
1)
# testing extended metadata elements
self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_extraction(self):
# test allowed file type is '.*'
self.assertEqual(self.res.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEqual(self.res.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.sample_nam_obj,
name=self.sample_nam_obj.name)
]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
# here are the things that the modflow modelinstance should include from the .nam file
# GroundWaterFlow
# flowPackage: 'LPF', 'UZF'(?)
#
# GeneralElements
# modelSolver: 'SIP'
# outputcontrolpackage: 'OC', 'GAGE'
#
# BoundaryCondition
# head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?)
# specified_flux_boundary_packages: 'WEL'
self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP')
self.assertTrue(
self.res.metadata.ground_water_flow.unsaturatedZonePackage)
self.assertIn(
'GHB',
self.res.metadata.boundary_condition.
get_head_dependent_flux_boundary_packages())
self.assertIn(
'SFR',
self.res.metadata.boundary_condition.
get_head_dependent_flux_boundary_packages())
self.assertIn(
'WEL',
self.res.metadata.boundary_condition.
get_specified_flux_boundary_packages())
self.assertIn(
'OC',
self.res.metadata.general_elements.get_output_control_package())
self.assertIn(
'GAGE',
self.res.metadata.general_elements.get_output_control_package())
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# there should be 2 content files
self.assertEqual(self.resRaster.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resRaster.metadata.formats.all().count(), 2)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='application/vrt').count(), 1)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='image/tiff').count(), 1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resRaster.metadata.title, None)
# there should be no abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there should be 1 creator element
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should be no coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there should be no subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# testing extended metadata elements - there should not be any resource specific metadata
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
self.resRaster.delete()
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
# there should be 1 content files
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 1 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file that we added above
hydroshare.delete_resource_file(self.resNetcdf.short_id,
self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEqual(self.resNetcdf.metadata.description, None)
# there should be 2 creator element
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(
self.resNetcdf.metadata.title.value,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010'
)
# there shouldn't any abstract element
self.assertEqual(self.resNetcdf.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0)
# there shouldn't any source element
self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0)
# there shouldn't any relation element
self.assertEqual(
self.resNetcdf.metadata.relations.all().filter(
type='cites').count(), 0)
# there should be 1 creator
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there shouldn't any extended metadata
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
super(TestNetcdfMetaData,
self).netcdf_metadata_extraction(expected_creators_count=2)
def test_get_xml(self):
# add a valid raster file to generate metadata
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# test if xml from get_xml() is well formed
ET.fromstring(self.resRaster.metadata.get_xml())
self.resRaster.delete()
def test_allowed_file_types(self):
# test allowed file type is '.*'
self.assertEquals(
self.resModelInstance.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelInstance.metadata.model_output, None)
self.assertEquals(self.resModelInstance.metadata.executed_by, None)
# Upload any other file type should pass both the file pre add check post add check
files = [
UploadedFile(file=self.text_file_obj_2,
name=self.text_file_obj_2.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=True)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=True)
# there should two content files
self.assertEquals(self.resModelInstance.files.all().count(), 2)
# check that there are no extended metadata elements at this point
self.assertEquals(self.resModelInstance.metadata.model_output, None)
self.assertEquals(self.resModelInstance.metadata.executed_by, None)
def test_metadata_on_content_file_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# there should one content file
self.assertEquals(self.resModelProgram.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1)
# the short path should just consist of the file name.
self.assertEquals(self.resModelProgram.files.all()[0].short_path, self.file_name)
# delete content file that we added above; note that file name is a short_path
hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelProgram.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelProgram.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelProgram.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_on_content_file_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance,
files=files,
user=self.user,
extract_metadata=False)
self.resModelInstance.metadata.create_element('Description',
abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject',
value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput',
includes_output=True)
self.resModelInstance.metadata.create_element(
'ExecutedBy', model_name=self.resModelProgram.short_id)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelInstance.metadata.formats.all().count(),
1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelInstance.short_id,
self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelInstance.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelInstance.metadata.description, None)
# there should be one creator element
self.assertEquals(
self.resModelInstance.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelInstance.metadata.model_output, None)
self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_metadata_extraction_of_wkt_crs_on_content_file_add(self):
files = [
UploadedFile(file=self.netcdf_file_obj_crs,
name=self.netcdf_file_name_crs)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
self._test_metadata_extraction_wkt_crs()
def test_metadata_extraction(self):
# test allowed file type is '.*'
self.assertEqual(self.res.get_supported_upload_file_types(), '.*')
# there should not be any content file
self.assertEqual(self.res.files.all().count(), 0)
# Upload any file type should pass both the file pre add check post add check
files = [UploadedFile(file=self.sample_nam_obj, name=self.sample_nam_obj.name)]
utils.resource_file_add_pre_process(resource=self.res, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res, files=files, user=self.user,
extract_metadata=False)
# here are the things that the modflow modelinstance should include from the .nam file
# GroundWaterFlow
# flowPackage: 'LPF', 'UZF'(?)
#
# GeneralElements
# modelSolver: 'SIP'
# outputcontrolpackage: 'OC', 'GAGE'
#
# BoundaryCondition
# head_dependent_flux_boundary_packages: 'SFR', 'GHB', 'UZF'(?)
# specified_flux_boundary_packages: 'WEL'
self.assertEqual(self.res.metadata.general_elements.modelSolver, 'SIP')
self.assertTrue(self.res.metadata.ground_water_flow.unsaturatedZonePackage)
self.assertIn(
'GHB',
self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages()
)
self.assertIn(
'SFR',
self.res.metadata.boundary_condition.get_head_dependent_flux_boundary_packages()
)
self.assertIn(
'WEL',
self.res.metadata.boundary_condition.get_specified_flux_boundary_packages()
)
self.assertIn(
'OC',
self.res.metadata.general_elements.get_output_control_package()
)
self.assertIn(
'GAGE',
self.res.metadata.general_elements.get_output_control_package()
)
def test_public_or_discoverable(self):
self.assertFalse(self.resNetcdf.has_required_content_files())
self.assertFalse(self.resNetcdf.metadata.has_all_required_elements())
self.assertFalse(self.resNetcdf.can_be_public_or_discoverable)
# adding a valid netcdf file
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=True)
self.assertTrue(self.resNetcdf.has_required_content_files())
self.assertTrue(self.resNetcdf.metadata.has_all_required_elements())
self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=True)
# there should be 1 content files
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 1 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file that we added above
hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEquals(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEquals(self.resNetcdf.metadata.description, None)
# there should be 2 creator element
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEquals(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def test_add_files_over_quota(self):
# create a resource
res = create_resource(
resource_type='GenericResource',
owner=self.user,
title='Test Resource',
metadata=[],
)
if not QuotaMessage.objects.exists():
QuotaMessage.objects.create()
qmsg = QuotaMessage.objects.first()
qmsg.enforce_quota = True
qmsg.save()
uquota = self.user.quotas.first()
# make user's quota over hard limit 125%
uquota.used_value = uquota.allocated_value * 1.3
uquota.save()
# add files should raise quota exception now that the quota holder is over hard limit
# and quota enforce flag is set to True
files = [self.myfile1, self.myfile2, self.myfile3]
with self.assertRaises(QuotaException):
resource_file_add_pre_process(resource=res,
files=files,
user=self.user,
extract_metadata=False)
qmsg.enforce_quota = False
qmsg.save()
# add files should not raise quota exception since enforce_quota flag is set to False
try:
resource_file_add_pre_process(resource=res,
files=files,
user=self.user,
extract_metadata=False)
except QuotaException as ex:
self.fail(
"add resource file action should not raise QuotaException for "
"over quota cases if quota is not enforced - Quota Exception: "
+ ex.message)
res.delete()
def test_metadata_extraction_DIS_file(self):
# Extract Metadata from DIS File
files = [
UploadedFile(file=self.sample_dis_obj,
name=self.sample_dis_obj.name)
]
utils.resource_file_add_pre_process(resource=self.res,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.res,
files=files,
user=self.user,
extract_metadeta=False)
# ---Tests for Grid Dimensions
# Number of Layers
self.assertEqual(self.res.metadata.grid_dimensions.numberOfLayers,
str(4))
# Type of Rows
self.assertEqual(self.res.metadata.grid_dimensions.typeOfRows,
'Regular')
# Number of Rows
self.assertEqual(self.res.metadata.grid_dimensions.numberOfRows,
str(20))
# Type of Columns
self.assertEqual(self.res.metadata.grid_dimensions.typeOfColumns,
'Regular')
# Number of Columns
self.assertEqual(self.res.metadata.grid_dimensions.numberOfColumns,
str(15))
# ---Tests for Stress Period
# Stress Period Type
self.assertEqual(self.res.metadata.stress_period.stressPeriodType,
'Steady and Transient')
# ---Tests for Study Area
# Total Length
self.assertEqual(self.res.metadata.study_area.totalLength,
str(20 * 2000.0)) # double
# Total Width
self.assertEqual(self.res.metadata.study_area.totalWidth,
str(15 * 2000.0)) # double
def test_allowed_file_types(self):
# test allowed file type is '.nc'
self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types())
self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()),
1)
# there should not be any content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .nc file should raise file validation error
files = [
UploadedFile(file=self.netcdf_bad_file_obj,
name=self.netcdf_bad_file_name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# trying to add valid .nc file should pass the file check
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
# there should be 2 content file: with ncdump file created by system
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# file pre add process should raise validation error if we try to add a 2nd file
# when the resource has already 2 content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(
self.resNetcdf.metadata.title.value,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010')
# there shouldn't any abstract element
self.assertEqual(self.resNetcdf.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0)
# there shouldn't any source element
self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0)
# there shouldn't any relation element
self.assertEqual(self.resNetcdf.metadata.relations.all().filter(type='cites').count(), 0)
# there should be 1 creator
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there shouldn't any extended metadata
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
super(TestNetcdfMetaData, self).netcdf_metadata_extraction(expected_creators_count=2)
def test_get_xml(self):
# add a valid raster file to generate metadata
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=True)
# test if xml from get_xml() is well formed
ET.fromstring(self.resRaster.metadata.get_xml())
self.resRaster.delete()
def test_public_or_discoverable(self):
self.assertFalse(self.resRaster.has_required_content_files())
self.assertFalse(self.resRaster.metadata.has_all_required_elements())
self.assertFalse(self.resRaster.can_be_public_or_discoverable)
# adding a valid raster file
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=True)
# adding required metadata
self.resRaster.metadata.create_element('description', abstract='example abstract')
self.resRaster.metadata.create_element('subject', value='logan')
self.assertTrue(self.resRaster.has_required_content_files())
self.assertTrue(self.resRaster.metadata.has_all_required_elements())
self.assertTrue(self.resRaster.can_be_public_or_discoverable)
self.resRaster.delete()
def test_metadata_on_content_file_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
self.resModelInstance.metadata.create_element('Description', abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject', value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True)
self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id)
# there should one content file
self.assertEquals(self.resModelInstance.files.all().count(), 1)
# there should be one format element
self.assertEquals(self.resModelInstance.metadata.formats.all().count(), 1)
# created file should have a short path equal to the declared name
self.assertEquals(self.resModelInstance.files.all()[0].short_path, self.file_name)
# delete content file that we added above
hydroshare.delete_resource_file(self.resModelInstance.short_id, self.file_name, self.user)
# there should no content file
self.assertEquals(self.resModelInstance.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEquals(self.resModelInstance.metadata.title, None)
# there should be an abstract element
self.assertNotEquals(self.resModelInstance.metadata.description, None)
# there should be one creator element
self.assertEquals(self.resModelInstance.metadata.creators.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resModelInstance.metadata.model_output, None)
self.assertNotEqual(self.resModelInstance.metadata.executed_by, None)
def test_public_or_discoverable(self):
self.assertFalse(self.resNetcdf.has_required_content_files())
self.assertFalse(self.resNetcdf.metadata.has_all_required_elements())
self.assertFalse(self.resNetcdf.can_be_public_or_discoverable)
# adding a valid netcdf file
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
self.assertTrue(self.resNetcdf.has_required_content_files())
self.assertTrue(self.resNetcdf.metadata.has_all_required_elements())
self.assertTrue(self.resNetcdf.can_be_public_or_discoverable)
def test_public_or_discoverable(self):
self.assertFalse(self.resModelProgram.has_required_content_files())
self.assertFalse(self.resModelProgram.metadata.has_all_required_elements())
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.assertFalse(self.resModelProgram.can_be_public_or_discoverable)
self.resModelProgram.metadata.create_element('Subject', value="test subject")
self.assertTrue(self.resModelProgram.has_required_content_files())
self.assertTrue(self.resModelProgram.metadata.has_all_required_elements())
self.assertTrue(self.resModelProgram.can_be_public_or_discoverable)
def test_allowed_file_types(self):
# test allowed file type is '.nc'
self.assertIn('.nc', NetcdfResource.get_supported_upload_file_types())
self.assertEqual(len(NetcdfResource.get_supported_upload_file_types()), 1)
# there should not be any content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .nc file should raise file validation error
files = [UploadedFile(file=self.netcdf_bad_file_obj, name=self.netcdf_bad_file_name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
# trying to add valid .nc file should pass the file check
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
# there should be 2 content file: with ncdump file created by system
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# file pre add process should raise validation error if we try to add a 2nd file
# when the resource has already 2 content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files,
user=self.user,
extract_metadata=False)
def test_metadata_in_user_zone(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(TestNetcdfMetaData, self).is_federated_irods_available():
return
# test metadata extraction with resource creation with nc file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.netcdf_file_name)
res_upload_files = []
_, _, metadata, fed_res_path = utils.resource_pre_create_actions(
resource_type='NetcdfResource',
resource_title=
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June '
'2010',
page_redirect_url_key=None,
files=res_upload_files,
source_names=[fed_test_file_full_path])
self.resNetcdf = hydroshare.create_resource(
'NetcdfResource',
self.user,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '',
move=False,
metadata=metadata)
utils.resource_post_create_actions(self.resNetcdf, self.user, metadata)
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
# test metadata is deleted after content file is deleted in user zone space
# there should be 2 content files at this point
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file now
hydroshare.delete_resource_file(self.resNetcdf.short_id,
self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEqual(self.resNetcdf.metadata.description, None)
# there should be 1 creator element (based on the extracted metadata)
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# test metadata extraction with a valid nc file being added coming from user zone space
res_add_files = []
utils.resource_file_add_pre_process(
resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
utils.resource_file_add_process(resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resNetcdf.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self):
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)]
utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user,
extract_metadata=True)
# before resource delete
# resource core metadata
core_metadata_obj = self.resNetcdf.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Source metadata objects
self.assertTrue(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Relation metadata objects
self.assertTrue(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Coverage metadata objects
self.assertTrue(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be original coverage metadata objects
self.assertTrue(OriginalCoverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Variable metadata objects
self.assertTrue(Variable.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resNetcdf.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be original coverage metadata objects
self.assertFalse(OriginalCoverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Variable metadata objects
self.assertFalse(Variable.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_delete_on_resource_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram, files=files,
user=self.user, extract_metadata=False)
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
self.resModelProgram.metadata.create_element('Contributor', name="user2")
# before resource delete
core_metadata_obj = self.resModelProgram.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Program metadata objects
self.assertTrue(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelProgram.short_id)
self.assertEquals(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_delete_on_resource_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
self.resModelInstance.metadata.create_element('Description', abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject', value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True)
self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id)
self.resModelInstance.metadata.create_element('Contributor', name="user2")
# before resource delete
core_metadata_obj = self.resModelInstance.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 3)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Output metadata objects
self.assertTrue(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be ExecutedBy metadata objects
self.assertTrue(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelInstance.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 2)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no ExecutedBy metadata objects
self.assertFalse(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
def test_allowed_file_types(self):
# test allowed file types are: '.tiff', '.tif,', '.vrt' and '.zip'
self.assertIn('.tiff', RasterResource.get_supported_upload_file_types())
self.assertIn('.tif', RasterResource.get_supported_upload_file_types())
self.assertIn('.vrt', RasterResource.get_supported_upload_file_types())
self.assertIn('.zip', RasterResource.get_supported_upload_file_types())
self.assertEqual(len(RasterResource.get_supported_upload_file_types()), 4)
# there should not be any content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .tif file should raise file validation error
files = [UploadedFile(file=self.raster_bad_tif_file_obj,
name=self.raster_bad_tif_file_name)]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
# trying to add good .tif file should pass the file check
files = [UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user,
extract_metadata=False)
# there should be 2 content file: with .vrt file created by system
self.assertEqual(self.resRaster.files.all().count(), 2)
file_names = [os.path.basename(f.resource_file.name) for f in self.resRaster.files.all()]
self.assertIn('raster_tif_valid.vrt', file_names)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should be no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add good .zip file should pass the file check
files = [UploadedFile(file=self.raster_zip_file_obj, name=self.raster_zip_file_name)]
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
# there should be 10 content file:
self.assertEqual(self.resRaster.files.all().count(), 10)
# file pre add process should raise validation error if we try to add a 2nd file when
# the resource has already content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster, files=files,
user=self.user, extract_metadata=False)
def upload_add(request):
# add irods file into an existing resource
res_id = request.POST['res_id']
resource, _, _ = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
res_files = request.FILES.getlist('files')
extract_metadata = request.POST.get('extract-metadata', 'No')
extract_metadata = True if extract_metadata.lower() == 'yes' else False
irods_fnames = request.POST.get('irods_file_names', '')
irods_fnames_list = string.split(irods_fnames, ',')
res_cls = resource.__class__
# TODO: read resource type from resource, not from input file
valid, ext = check_upload_files(res_cls, irods_fnames_list)
source_names = []
irods_federated = False
if not valid:
request.session['file_type_error'] = "Invalid file type: {ext}".format(ext=ext)
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
homepath = irods_fnames_list[0]
# TODO: this should happen whether resource is federated or not
irods_federated = utils.is_federated(homepath)
if irods_federated:
source_names = irods_fnames.split(',')
else:
user = request.POST.get('irods-username')
password = request.POST.get("irods-password")
port = request.POST.get("irods-port")
host = request.POST.get("irods-host")
zone = request.POST.get("irods-zone")
try:
upload_from_irods(username=user, password=password, host=host, port=port,
zone=zone, irods_fnames=irods_fnames, res_files=res_files)
except SessionException as ex:
request.session['validation_error'] = ex.stderr
return HttpResponseRedirect(request.META['HTTP_REFERER'])
try:
utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata,
source_names=source_names, folder=None)
except hydroshare.utils.ResourceFileSizeException as ex:
request.session['file_size_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
request.session['validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
try:
hydroshare.utils.resource_file_add_process(resource=resource, files=res_files,
user=request.user,
extract_metadata=extract_metadata,
source_names=source_names, folder=None)
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
if ex.message:
request.session['validation_error'] = ex.message
elif ex.stderr:
request.session['validation_error'] = ex.stderr
except SessionException as ex:
request.session['validation_error'] = ex.stderr
request.session['resource-mode'] = 'edit'
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_allowed_file_types(self):
# test allowed file type is '.tif, .zip'
self.assertIn('.tif', RasterResource.get_supported_upload_file_types())
self.assertIn('.zip', RasterResource.get_supported_upload_file_types())
self.assertEqual(len(RasterResource.get_supported_upload_file_types()),
2)
# there should not be any content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add a text file to this resource should raise exception
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
# trying to add bad .tif file should raise file validation error
files = [
UploadedFile(file=self.raster_bad_tif_file_obj,
name=self.raster_bad_tif_file_name)
]
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
# trying to add good .tif file should pass the file check
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
# there should be 2 content file: with .vrt file created by system
self.assertEqual(self.resRaster.files.all().count(), 2)
file_names = [
os.path.basename(f.resource_file.name)
for f in self.resRaster.files.all()
]
self.assertIn('raster_tif_valid.vrt', file_names)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id,
self.raster_tif_file_name, self.user)
# there should be no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# trying to add good .zip file should pass the file check
files = [
UploadedFile(file=self.raster_zip_file_obj,
name=self.raster_zip_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
# there should be 10 content file:
self.assertEqual(self.resRaster.files.all().count(), 10)
# file pre add process should raise validation error if we try to add a 2nd file when
# the resource has already content files
with self.assertRaises(utils.ResourceFileValidationException):
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
def test_metadata_delete_on_resource_delete(self):
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
# before resource delete
# resource core metadata
core_metadata_obj = self.resNetcdf.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Source metadata objects
self.assertTrue(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Relation metadata objects
self.assertTrue(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Coverage metadata objects
self.assertTrue(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be original coverage metadata objects
self.assertTrue(
OriginalCoverage.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Variable metadata objects
self.assertTrue(
Variable.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resNetcdf.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be original coverage metadata objects
self.assertFalse(
OriginalCoverage.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Variable metadata objects
self.assertFalse(
Variable.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
files = [
UploadedFile(file=self.raster_tif_file_obj,
name=self.raster_tif_file_name)
]
utils.resource_file_add_pre_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resRaster,
files=files,
user=self.user,
extract_metadata=True)
# there should be 2 content files
self.assertEqual(self.resRaster.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resRaster.metadata.formats.all().count(), 2)
self.assertEqual(
self.resRaster.metadata.formats.all().filter(
value='application/vrt').count(), 1)
self.assertEqual(
self.resRaster.metadata.formats.all().filter(
value='image/tiff').count(), 1)
# delete content file that we added above
hydroshare.delete_resource_file(self.resRaster.short_id,
self.raster_tif_file_name, self.user)
# there should no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resRaster.metadata.title, None)
# there should be no abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there should be 1 creator element
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should be no coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there should be no subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# testing extended metadata elements - there should not be any resource specific metadata
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
self.resRaster.delete()
def upload_add(request):
# add irods file into an existing resource
res_id = request.POST['res_id']
resource, _, _ = authorize(request, res_id, edit=True, full=True, superuser=True)
res_files = request.FILES.getlist('files')
extract_metadata = request.REQUEST.get('extract-metadata', 'No')
extract_metadata = True if extract_metadata.lower() == 'yes' else False
irods_fname = request.POST.get('irods_file_name', '')
res_cls = resource.__class__
file_types = res_cls.get_supported_upload_file_types()
valid = False
if file_types == ".*":
valid = True
else:
ext = os.path.splitext(irods_fname)[1]
if ext == file_types:
valid = True
else:
for index in range(len(file_types)):
file_type_str = file_types[index].strip()
if file_type_str == ".*" or ext == file_type_str:
valid = True
break
if not valid:
request.session['file_type_error'] = "Invalid file type: {ext}".format(ext=ext)
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
user = request.POST.get('irods-username')
password = request.POST.get("irods-password")
port = request.POST.get("irods-port")
host = request.POST.get("irods-host")
zone = request.POST.get("irods-zone")
try:
upload_from_irods(username=user, password=password, host=host, port=port,
zone=zone, irods_fname=irods_fname, res_files=res_files)
except Exception as ex:
request.session['file_validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
try:
utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata)
except hydroshare.utils.ResourceFileSizeException as ex:
request.session['file_size_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
request.session['file_validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
try:
hydroshare.utils.resource_file_add_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata)
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
request.session['file_validation_error'] = ex.message
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def test_metadata_in_user_zone(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
super(TestNetcdfMetaData, self).assert_federated_irods_available()
# test metadata extraction with resource creation with nc file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.netcdf_file_name)
res_upload_files = []
_, _, metadata, fed_res_path = utils.resource_pre_create_actions(
resource_type='NetcdfResource',
resource_title='Snow water equivalent estimation at TWDEF site from Oct 2009 to June '
'2010',
page_redirect_url_key=None,
files=res_upload_files,
source_names=[fed_test_file_full_path])
self.resNetcdf = hydroshare.create_resource(
'NetcdfResource',
self.user,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '',
move=False,
metadata=metadata)
utils.resource_post_create_actions(self.resNetcdf, self.user, metadata)
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
# test metadata is deleted after content file is deleted in user zone space
# there should be 2 content files at this point
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file now
hydroshare.delete_resource_file(self.resNetcdf.short_id, self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEqual(self.resNetcdf.metadata.description, None)
# there should be 1 creator element (based on the extracted metadata)
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# test metadata extraction with a valid nc file being added coming from user zone space
res_add_files = []
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
utils.resource_file_add_process(resource=self.resNetcdf,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
super(TestNetcdfMetaData, self).netcdf_metadata_extraction()
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resNetcdf.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self):
files = [
UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)
]
utils.resource_file_add_pre_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelProgram,
files=files,
user=self.user,
extract_metadata=False)
self.resModelProgram.metadata.create_element('Description',
abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject',
value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element(
'MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
self.resModelProgram.metadata.create_element('Contributor',
name="user2")
# before resource delete
core_metadata_obj = self.resModelProgram.metadata
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# there should be Creator metadata objects
self.assertTrue(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Contributor metadata objects
self.assertTrue(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Program metadata objects
self.assertTrue(
MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelProgram.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# there should be no Creator metadata objects
self.assertFalse(
Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(
Contributor.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(
Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(
Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(
Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(
Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(
Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(
Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(
Description.objects.filter(
object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(
Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(
Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(
Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(
Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(
Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(
Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(
MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(TestTimeSeriesMetaData,
self).is_federated_irods_available():
return
# test metadata extraction with resource creation with file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
username=self.user.username,
fname=self.odm2_sqlite_file_name)
self.resTimeSeries = hydroshare.create_resource(
resource_type='TimeSeriesResource',
owner=self.user,
title='My Test TimeSeries Resource',
source_names=[fed_test_file_full_path],
move=False)
utils.resource_post_create_actions(resource=self.resTimeSeries,
user=self.user,
metadata=[])
super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction()
# test that metadata is NOT deleted (except format element) on content file deletion
# there should be one content file at this point
self.assertEqual(self.resTimeSeries.files.all().count(), 1)
# there should be one format element
self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 1)
hydroshare.delete_resource_file(self.resTimeSeries.short_id,
self.odm2_sqlite_file_name, self.user)
# there should be no content file
self.assertEqual(self.resTimeSeries.files.all().count(), 0)
# test the core metadata at this point
self.assertNotEqual(self.resTimeSeries.metadata.title, None)
# there should be an abstract element
self.assertNotEqual(self.resTimeSeries.metadata.description, None)
# there should be one creator element
self.assertEqual(self.resTimeSeries.metadata.creators.all().count(), 1)
# there should be one contributor element
self.assertEqual(
self.resTimeSeries.metadata.contributors.all().count(), 1)
# there should be 2 coverage element - point type and period type
self.assertEqual(self.resTimeSeries.metadata.coverages.all().count(),
2)
self.assertEqual(
self.resTimeSeries.metadata.coverages.all().filter(
type='box').count(), 1)
self.assertEqual(
self.resTimeSeries.metadata.coverages.all().filter(
type='period').count(), 1)
# there should be no format element
self.assertEqual(self.resTimeSeries.metadata.formats.all().count(), 0)
# there should be one subject element
self.assertEqual(self.resTimeSeries.metadata.subjects.all().count(), 1)
# testing extended metadata elements
self.assertNotEqual(self.resTimeSeries.metadata.sites.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.variables.all().count(), 0)
self.assertNotEqual(self.resTimeSeries.metadata.methods.all().count(),
0)
self.assertNotEqual(
self.resTimeSeries.metadata.processing_levels.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.time_series_results.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_variable_names.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_variable_types.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_speciations.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_site_types.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_elevation_datums.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_method_types.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_statuses.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_mediums.all().count(), 0)
self.assertNotEqual(
self.resTimeSeries.metadata.cv_aggregation_statistics.all().count(
), 0)
# test metadata extraction with a valid ODM2 sqlite file being added coming from user zone
# space
res_add_files = []
utils.resource_file_add_pre_process(
resource=self.resTimeSeries,
files=res_add_files,
user=self.user,
extract_metadata=False,
source_names=[fed_test_file_full_path])
utils.resource_file_add_process(resource=self.resTimeSeries,
files=res_add_files,
user=self.user,
extract_metadata=True,
source_names=[fed_test_file_full_path])
super(TestTimeSeriesMetaData, self).timeseries_metadata_extraction()
# test metadata deletion when deleting a resource in user zone space
# all metadata should get deleted when the resource get deleted
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resTimeSeries.short_id)
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def test_metadata_delete_on_resource_delete(self):
files = [UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name)]
utils.resource_file_add_pre_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
utils.resource_file_add_process(resource=self.resModelInstance, files=files,
user=self.user, extract_metadata=False)
self.resModelInstance.metadata.create_element('Description', abstract="test abstract")
self.resModelInstance.metadata.create_element('Subject', value="test subject")
self.resModelInstance.metadata.create_element('ModelOutput', includes_output=True)
self.resModelInstance.metadata.create_element('ExecutedBy', model_name=self.resModelProgram.short_id)
self.resModelInstance.metadata.create_element('Contributor', name="user2")
# before resource delete
core_metadata_obj = self.resModelInstance.metadata
self.assertEquals(CoreMetaData.objects.all().count(), 3)
# there should be Creator metadata objects
self.assertTrue(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertTrue(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Identifier metadata objects
self.assertTrue(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Type metadata objects
self.assertTrue(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Title metadata objects
self.assertTrue(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Description (Abstract) metadata objects
self.assertTrue(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Date metadata objects
self.assertTrue(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Subject metadata objects
self.assertTrue(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Format metadata objects
self.assertTrue(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Language metadata objects
self.assertTrue(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be Rights metadata objects
self.assertTrue(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be Model Output metadata objects
self.assertTrue(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be ExecutedBy metadata objects
self.assertTrue(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
# delete resource
hydroshare.delete_resource(self.resModelInstance.short_id)
self.assertEquals(CoreMetaData.objects.all().count(), 2)
# there should be no Creator metadata objects
self.assertFalse(Creator.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Contributor metadata objects
self.assertFalse(Contributor.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Identifier metadata objects
self.assertFalse(Identifier.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Type metadata objects
self.assertFalse(Type.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Source metadata objects
self.assertFalse(Source.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Relation metadata objects
self.assertFalse(Relation.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Publisher metadata objects
self.assertFalse(Publisher.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Title metadata objects
self.assertFalse(Title.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Description (Abstract) metadata objects
self.assertFalse(Description.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Date metadata objects
self.assertFalse(Date.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Subject metadata objects
self.assertFalse(Subject.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Coverage metadata objects
self.assertFalse(Coverage.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Format metadata objects
self.assertFalse(Format.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Language metadata objects
self.assertFalse(Language.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no Rights metadata objects
self.assertFalse(Rights.objects.filter(object_id=core_metadata_obj.id).exists())
# resource specific metadata
# there should be no Model Output metadata objects
self.assertFalse(ModelOutput.objects.filter(object_id=core_metadata_obj.id).exists())
# there should be no ExecutedBy metadata objects
self.assertFalse(ExecutedBy.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_in_user_zone(self):
super(TestRasterMetaData, self).assert_federated_irods_available()
# test metadata extraction with resource creation with tif file coming from user zone space
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.raster_tif_file_name)
res_upload_files = []
_, _, metadata, fed_res_path = utils.resource_pre_create_actions(
resource_type='RasterResource',
resource_title='My Test Raster Resource',
page_redirect_url_key=None,
files=res_upload_files,
source_names=[fed_test_file_full_path])
self.resRaster = hydroshare.create_resource(
'RasterResource',
self.user,
'My Test Raster Resource',
files=res_upload_files,
source_names=[fed_test_file_full_path],
fed_res_path=fed_res_path[0] if len(fed_res_path) == 1 else '',
move=False,
metadata=metadata)
# raster file validation and metadata extraction in post resource creation signal handler
utils.resource_post_create_actions(resource=self.resRaster, user=self.user,
metadata=[])
super(TestRasterMetaData, self).raster_metadata_extraction()
# test metadata is deleted after content file is deleted in user zone space
# there should be 2 content file: tif file and vrt file at this point
self.assertEqual(self.resRaster.files.all().count(), 2)
# there should be 2 format elements
self.assertEqual(self.resRaster.metadata.formats.all().count(), 2)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='application/vrt').count(), 1)
self.assertEqual(self.resRaster.metadata.formats.all().filter(
value='image/tiff').count(), 1)
# delete content file now
hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name,
self.user)
# there should be no content file
self.assertEqual(self.resRaster.files.all().count(), 0)
# there should be a title element
self.assertNotEqual(self.resRaster.metadata.title, None)
# there should be no abstract element
self.assertEqual(self.resRaster.metadata.description, None)
# there should be 1 creator element
self.assertEqual(self.resRaster.metadata.creators.all().count(), 1)
# there should be no contributor element
self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0)
# there should be no coverage element
self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resRaster.metadata.formats.all().count(), 0)
# there should be no subject element
self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0)
# testing extended metadata elements - there should be no extended metadata elements
# at this point
self.assertEqual(self.resRaster.metadata.originalCoverage, None)
self.assertEqual(self.resRaster.metadata.cellInformation, None)
self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0)
# test metadata extraction with a valid tif file being added coming from user zone space
res_add_files = []
# now necessary in order to test add_process
utils.resource_file_add_pre_process(resource=self.resRaster,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
# file validation and metadata extraction happen during post file add signal handler
utils.resource_file_add_process(resource=self.resRaster,
files=res_add_files,
user=self.user,
source_names=[fed_test_file_full_path])
super(TestRasterMetaData, self).raster_metadata_extraction()
# test metadata deletion when deleting a resource in user zone space
self.assertEqual(CoreMetaData.objects.all().count(), 1)
# delete resource
hydroshare.delete_resource(self.resRaster.short_id)
# resource core metadata is deleted after resource deletion
self.assertEqual(CoreMetaData.objects.all().count(), 0)
# test adding file from user zone to existing empty resource in hydroshare zone
# even there is no file uploaded to resource initially, there are default extended
# automatically metadata created
self.resRaster = hydroshare.create_resource(
resource_type='RasterResource',
owner=self.user,
title='My Test Raster Resource'
)
# test metadata extraction with a valid tif file being added coming from user zone space
# file validation and metadata extraction happen during post file add signal handler
utils.resource_file_add_process(resource=self.resRaster,
files=[],
user=self.user,
source_names=[fed_test_file_full_path])
super(TestRasterMetaData, self).raster_metadata_extraction()
# there should be 2 content file: tif file and vrt file at this point
self.assertEqual(self.resRaster.files.all().count(), 2)
# delete resource
hydroshare.delete_resource(self.resRaster.short_id)
# resource core metadata is deleted after resource deletion
self.assertEqual(CoreMetaData.objects.all().count(), 0)
def upload_add(request):
# add irods file into an existing resource
res_id = request.POST.get('res_id', '')
resource, _, _ = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
res_files = request.FILES.getlist('files')
extract_metadata = request.POST.get('extract-metadata', 'No')
extract_metadata = True if extract_metadata.lower() == 'yes' else False
irods_fnames = request.POST.get('upload', '')
irods_fnames_list = irods_fnames.split(',')
res_cls = resource.__class__
# TODO: read resource type from resource, not from input file
valid, ext = check_upload_files(res_cls, irods_fnames_list)
source_names = []
if not valid:
return JsonResponse({'error': "Invalid file type: {ext}".format(ext=ext)},
status=status.HTTP_400_BAD_REQUEST)
else:
homepath = irods_fnames_list[0]
# TODO: this should happen whether resource is federated or not
irods_federated = utils.is_federated(homepath)
if irods_federated:
source_names = irods_fnames.split(',')
else:
user = request.POST.get('irods_username')
password = request.POST.get("irods_password")
port = request.POST.get("irods_port")
host = request.POST.get("irods_host")
zone = request.POST.get("irods_zone")
try:
upload_from_irods(username=user, password=password, host=host, port=port,
zone=zone, irods_fnames=irods_fnames, res_files=res_files)
except SessionException as ex:
return JsonResponse(
{"error": ex.stderr}, status=status.HTTP_500_INTERNAL_SERVER_ERROR
)
try:
utils.resource_file_add_pre_process(resource=resource, files=res_files, user=request.user,
extract_metadata=extract_metadata,
source_names=source_names, folder='')
except hydroshare.utils.ResourceFileSizeException as ex:
return JsonResponse({'error': str(ex)}, status=status.HTTP_400_BAD_REQUEST)
except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
return JsonResponse({'error': str(ex)}, status=status.HTTP_400_BAD_REQUEST)
try:
hydroshare.utils.resource_file_add_process(resource=resource, files=res_files,
user=request.user,
extract_metadata=extract_metadata,
source_names=source_names, folder='')
except (hydroshare.utils.ResourceFileValidationException, SessionException) as ex:
if str(ex):
return JsonResponse({'error': str(ex)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
elif ex.stderr:
return JsonResponse({'error': ex.stderr},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JsonResponse({}, status=status.HTTP_200_OK)
def test_file_add_to_composite_resource(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
super(CompositeResourceTest, self).assert_federated_irods_available()
# test that when we add file to an existing composite resource, the added file
# automatically set to genericlogicalfile type
self.assertEqual(BaseResource.objects.count(), 0)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test Composite Resource With Files Added From Federated Zone',
auto_aggregate=False
)
# there should not be any GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# add a file to the resource
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=None)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
auto_aggregate=False)
# there should be one resource at this point
self.assertEqual(BaseResource.objects.count(), 1)
self.assertEqual(self.composite_resource.resource_type, "CompositeResource")
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# create the generic aggregation (logical file)
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# check that the resource file is associated with GenericLogicalFile
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# there should be 1 GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# test adding a file to a folder (Note the UI does not support uploading a iRODS file
# to a specific folder)
# create the folder
new_folder = "my-new-folder"
new_folder_path = os.path.join("data", "contents", new_folder)
create_folder(self.composite_resource.short_id, new_folder_path)
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder, auto_aggregate=False)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.composite_resource.delete()
