def delete(self, request, pk, pathname):
"""
Delete a folder.
"""
try:
resource, authorized, user = view_utils.authorize(
request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE,
raises_exception=False)
except NotFound as ex:
return Response(ex.message, status=status.HTTP_404_NOT_FOUND)
if not authorized:
return Response("Insufficient permission", status=status.HTTP_401_UNAUTHORIZED)
if not resource.supports_folders:
return Response("Resource type does not support subfolders",
status=status.HTTP_403_FORBIDDEN)
try:
view_utils.irods_path_is_allowed(pathname) # check for hacking attempts
except (ValidationError, SuspiciousFileOperation) as ex:
return Response(ex.message, status=status.HTTP_400_BAD_REQUEST)
# relativise the path
relpath = os.path.join('data', 'contents', pathname)
try:
view_utils.remove_folder(request.user, pk, relpath)
except SessionException:
return Response("Cannot remove folder", status=status.HTTP_400_BAD_REQUEST)
return Response(data={'resource_id': pk, 'path': pathname},
status=status.HTTP_200_OK)
def test_geofeature_file_type_folder_delete(self):
# when a file is set to geofeaturelogical file type
# system automatically creates a folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self._create_composite_resource(self.states_required_zip_file)
res_file = self.composite_resource.files.first()
# extract metadata from the zip file
GeoFeatureLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoFeatureLogicalFile type as a result
# of metadata extraction
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# should have one GeoFeatureFileMetadata object
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 1)
# there should be 3 content files
self.assertEqual(self.composite_resource.files.count(), 3)
# delete the folder for the logical file
folder_path = "data/contents/states_required_files"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should be no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any logical file or file metadata object as a result
# of folder deletion
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
self.composite_resource.delete()
def data_store_remove_folder(request):
"""
remove a sub-folder/sub-collection in hydroshareZone or any federated zone used for CommonsShare
resource backend store. It is invoked by an AJAX call and returns json object that include a
status of 'success' if succeeds, and HttpResponse of status code of 403, 400, or 500 if fails.
The AJAX request must be a POST request with input data passed in for res_id and folder_path
where folder_path is the relative path for the folder to be removed under
res_id collection/directory.
"""
res_id = request.POST.get('res_id', None)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, user = authorize(
request,
res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found',
status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied',
status=status.HTTP_401_UNAUTHORIZED)
folder_path = request.POST.get('folder_path', None)
if folder_path is None:
return HttpResponse('Bad request - folder_path is not included',
status=status.HTTP_400_BAD_REQUEST)
folder_path = str(folder_path).strip()
if not folder_path:
return HttpResponse('Bad request - folder_path cannot be empty',
status=status.HTTP_400_BAD_REQUEST)
if not folder_path.startswith('data/contents/'):
return HttpResponse(
'Bad request - folder_path must start with data/contents/',
status=status.HTTP_400_BAD_REQUEST)
if folder_path.find('/../') >= 0 or folder_path.endswith('/..'):
return HttpResponse('Bad request - folder_path must not contain /../',
status=status.HTTP_400_BAD_REQUEST)
try:
remove_folder(user, res_id, folder_path)
except SessionException as ex:
return HttpResponse(ex.stderr,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except Exception as ex:
return HttpResponse(ex.message,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return_object = {'status': 'success'}
return HttpResponse(json.dumps(return_object),
content_type="application/json")
def test_timeseries_file_type_folder_delete(self):
# when a file is set to TimeSeriesLogicalFile type
# system automatically creates folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self.sqlite_file_obj = open(self.sqlite_file, 'r')
self._create_composite_resource(title='Untitled Resource')
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# set the sqlite file to TimeSeries file type
TimeSeriesLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
res_file = self.composite_resource.files.first()
# test that we have one logical file of type TimeSeries
self.assertEqual(TimeSeriesLogicalFile.objects.count(), 1)
self.assertEqual(TimeSeriesFileMetaData.objects.count(), 1)
# delete the folder for the logical file
folder_path = "data/contents/ODM2_Multi_Site_One_Variable"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any timeseries logical file or metadata file
self.assertEqual(TimeSeriesLogicalFile.objects.count(), 0)
self.assertEqual(TimeSeriesFileMetaData.objects.count(), 0)
# test that all file level metadata deleted
# there should be no Site metadata objects
self.assertTrue(Site.objects.count() == 0)
# there should be no Variable metadata objects
self.assertTrue(Variable.objects.count() == 0)
# there should be no Method metadata objects
self.assertTrue(Method.objects.count() == 0)
# there should be no ProcessingLevel metadata objects
self.assertTrue(ProcessingLevel.objects.count() == 0)
# there should be no TimeSeriesResult metadata objects
self.assertTrue(TimeSeriesResult.objects.count() == 0)
# there should not be any CV type records
self.assertEqual(CVVariableType.objects.all().count(), 0)
self.assertEqual(CVVariableName.objects.all().count(), 0)
self.assertEqual(CVSpeciation.objects.all().count(), 0)
self.assertEqual(CVElevationDatum.objects.all().count(), 0)
self.assertEqual(CVSiteType.objects.all().count(), 0)
self.assertEqual(CVMethodType.objects.all().count(), 0)
self.assertEqual(CVUnitsType.objects.all().count(), 0)
self.assertEqual(CVStatus.objects.all().count(), 0)
self.assertEqual(CVMedium.objects.all().count(), 0)
self.assertEqual(CVAggregationStatistic.objects.all().count(), 0)
self.composite_resource.delete()
def test_aggregation_folder_delete(self):
# when a file is set to NetCDFLogicalFile type
# system automatically creates folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self.create_composite_resource(self.netcdf_file)
res_file = self.composite_resource.files.first()
base_file_name, _ = os.path.splitext(res_file.file_name)
expected_folder_name = base_file_name
# extract metadata from the tif file
NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# test that we have one logical file of type NetCDFLogicalFile as a result
# of metadata extraction
self.assertEqual(NetCDFLogicalFile.objects.count(), 1)
# should have one NetCDFFileMetadata object
self.assertEqual(NetCDFFileMetaData.objects.count(), 1)
# there should be 2 content files
self.assertEqual(self.composite_resource.files.count(), 2)
# test that there are metadata associated with the logical file
self.assertEqual(Coverage.objects.count(), 4)
self.assertEqual(OriginalCoverage.objects.count(), 1)
self.assertEqual(Variable.objects.count(), 5)
# delete the folder for the logical file
folder_path = "data/contents/{}".format(expected_folder_name)
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any netCDF logical file or metadata file
self.assertEqual(NetCDFLogicalFile.objects.count(), 0)
self.assertEqual(NetCDFFileMetaData.objects.count(), 0)
# test that all metadata associated with the logical file got deleted - there still be
# 2 resource level coverages
self.assertEqual(self.composite_resource.metadata.coverages.all().count(), 2)
self.assertEqual(Coverage.objects.count(), 2)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(Variable.objects.count(), 0)
self.composite_resource.delete()
def test_raster_file_type_folder_delete(self):
# when a file is set to georasterlogical file type
# system automatically creates folder using the name of the file
# that was used to set the file type
# Here we need to test that when that folder gets deleted, all files
# in that folder gets deleted, the logicalfile object gets deleted and
# the associated metadata objects get deleted
self.raster_file_obj = open(self.raster_file, 'r')
self._create_composite_resource()
res_file = self.composite_resource.files.first()
# extract metadata from the tif file
GeoRasterLogicalFile.set_file_type(self.composite_resource,
res_file.id, self.user)
# test that we have one logical file of type GeoRasterFileType as a result
# of metadata extraction
self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
# should have one GeoRasterFileMetadata object
self.assertEqual(GeoRasterFileMetaData.objects.count(), 1)
# there should be 2 content files
self.assertEqual(self.composite_resource.files.count(), 2)
# test that there are metadata associated with the logical file
self.assertNotEqual(Coverage.objects.count(), 0)
self.assertNotEqual(OriginalCoverage.objects.count(), 0)
self.assertNotEqual(CellInformation.objects.count(), 0)
self.assertNotEqual(BandInformation.objects.count(), 0)
# delete the folder for the logical file
folder_path = "data/contents/small_logan"
remove_folder(self.user, self.composite_resource.short_id, folder_path)
# there should no content files
self.assertEqual(self.composite_resource.files.count(), 0)
# there should not be any GeoRaster logical file or metadata file
self.assertEqual(GeoRasterLogicalFile.objects.count(), 0)
self.assertEqual(GeoRasterFileMetaData.objects.count(), 0)
# test that all metadata associated with the logical file got deleted
self.assertEqual(Coverage.objects.count(), 0)
self.assertEqual(OriginalCoverage.objects.count(), 0)
self.assertEqual(CellInformation.objects.count(), 0)
self.assertEqual(BandInformation.objects.count(), 0)
self.composite_resource.delete()
def data_store_remove_folder(request):
"""
remove a sub-folder/sub-collection in hydroshareZone or any federated zone used for HydroShare
resource backend store. It is invoked by an AJAX call and returns json object that include a
status of 'success' if succeeds, and HttpResponse of status code of 403, 400, or 500 if fails.
The AJAX request must be a POST request with input data passed in for res_id and folder_path
where folder_path is the relative path (relative to res_id/data/contents) for the folder to
be removed under res_id collection/directory.
"""
res_id = request.POST.get('res_id', None)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, user = authorize(request, res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED)
folder_path = request.POST.get('folder_path', None)
try:
folder_path = _validate_path(folder_path, 'folder_path')
except ValidationError as ex:
return HttpResponse(ex.message, status=status.HTTP_400_BAD_REQUEST)
try:
remove_folder(user, res_id, folder_path)
except SessionException as ex:
return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except Exception as ex:
return HttpResponse(ex.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return_object = {'status': 'success'}
return HttpResponse(
json.dumps(return_object),
content_type="application/json"
)
def delete(self, request, pk, pathname):
"""
Delete a folder.
"""
try:
resource, authorized, user = view_utils.authorize(
request,
pk,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE,
raises_exception=False)
except NotFound as ex:
return Response(ex.message, status=status.HTTP_404_NOT_FOUND)
if not authorized:
return Response("Insufficient permission",
status=status.HTTP_401_UNAUTHORIZED)
if not resource.supports_folders:
return Response("Resource type does not support subfolders",
status=status.HTTP_403_FORBIDDEN)
try:
view_utils.irods_path_is_allowed(
pathname) # check for hacking attempts
except (ValidationError, SuspiciousFileOperation) as ex:
return Response(ex.message, status=status.HTTP_400_BAD_REQUEST)
# relativise the path
relpath = os.path.join('data', 'contents', pathname)
try:
view_utils.remove_folder(request.user, pk, relpath)
except SessionException:
return Response("Cannot remove folder",
status=status.HTTP_400_BAD_REQUEST)
return Response(data={
'resource_id': pk,
'path': pathname
},
status=status.HTTP_200_OK)
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir',
store[0],
msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg="resource does not contain the updated file new_" +
file_name_list[2])
self.assertNotIn(file_name_list[2],
updated_res_file_names,
msg='resource still contains the old file ' +
file_name_list[2] + ' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' moved to a folder')
self.assertNotIn(file_name_list[0],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[0] + 'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[1] + ' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(),
2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
uname=user.username,
fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
with self.assertRaises(SessionException):
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip',
False)
# Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
3,
msg="resource file count didn't match - " +
str(file_cnt) + " != 3")
# test unzipping the file succeeds now after deleting the existing folder
# TODO: this causes a multiple delete because the paths are valid now.
istorage = res.get_irods_storage()
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
# Now resource should contain two files: file3_new.txt and sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
2,
msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn(
'sub_test_dir.zip',
updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[0])
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[1])
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg='resource does not contain unzipped file new_' +
file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(),
1,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg="resource does not contain the updated file new_" + file_name_list[2])
self.assertNotIn(file_name_list[2], updated_res_file_names,
msg='resource still contains the old file ' + file_name_list[2] +
' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] + ' moved to a folder')
self.assertNotIn(file_name_list[0], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[0] +
'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[1] +
' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(), 2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id, source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False)
# Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip,
# and file1.txt (2)
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " +
str(file_cnt) + " != 5")
# remove all files except the zippped file
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1')
# Now resource should contain two files: file3_new.txt sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir.zip', updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[0])
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[1])
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg='resource does not contain unzipped file new_' + file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None:
raise ValidationError("File not found.")
if res_file.extension != '.nc':
raise ValidationError("Not a NetCDF file.")
# base file name (no path included)
file_name = res_file.file_name
# file name without the extension
nc_file_name = file_name[:-len(res_file.extension)]
resource_metadata = []
file_type_metadata = []
files_to_add_to_resource = []
upload_folder = ''
if res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
files_to_add_to_resource.append(temp_file)
# file validation and metadata extraction
nc_dataset = nc_utils.get_nc_dataset(temp_file)
if isinstance(nc_dataset, netCDF4.Dataset):
# Extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(
temp_file)
# populate resource_metadata and file_type_metadata lists with extracted metadata
add_metadata_to_list(resource_metadata, res_dublin_core_meta,
res_type_specific_meta,
file_type_metadata, resource)
# create the ncdump text file
dump_file = create_header_info_txt_file(
temp_file, nc_file_name)
files_to_add_to_resource.append(dump_file)
file_folder = res_file.file_folder
with transaction.atomic():
# create a netcdf logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type unless the extracted metadata
# has a value for title
dataset_title = res_dublin_core_meta.get('title', None)
if dataset_title is not None:
logical_file.dataset_name = dataset_title
else:
logical_file.dataset_name = nc_file_name
logical_file.save()
try:
# create a folder for the netcdf file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, nc_file_name)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
# the added resource file will be part of a new generic logical file
# by default
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# delete the generic logical file object
if new_res_file.logical_file is not None:
# deleting the file level metadata object will delete the associated
# logical file object
new_res_file.logical_file.metadata.delete()
# make each resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"NetCDF file type - new files were added to the resource."
)
# use the extracted metadata to populate resource metadata
for element in resource_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element(
'title', title_element.id, **v)
else:
resource.metadata.create_element(k, **v)
log.info("Resource - metadata was saved to DB")
# use the extracted metadata to populate file metadata
for element in file_type_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'subject':
logical_file.metadata.keywords = v
logical_file.metadata.save()
# update resource level keywords
resource_keywords = [
subject.value.lower() for subject in
resource.metadata.subjects.all()
]
for kw in logical_file.metadata.keywords:
if kw.lower() not in resource_keywords:
resource.metadata.create_element(
'subject', value=kw)
else:
logical_file.metadata.create_element(k, **v)
log.info("NetCDF file type - metadata was saved to DB")
# set resource to private if logical file is missing required metadata
resource.update_public_and_discoverable()
# delete the original resource file
delete_resource_file(resource.short_id, res_file.id,
user)
log.info("Deleted original resource file.")
except Exception as ex:
msg = "NetCDF file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
if upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join(
'data', 'contents', upload_folder)
remove_folder(user, resource.short_id,
folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
else:
err_msg = "Not a valid NetCDF file. File type file validation failed."
log.error(err_msg)
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
raise ValidationError(err_msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
# base file name (no path included)
file_name = utils.get_resource_file_name_and_extension(res_file)[1]
# file name without the extension
file_name = file_name[:-len(res_file.extension)]
file_folder = res_file.file_folder
upload_folder = ''
if res_file is not None and res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
error_info, files_to_add_to_resource = raster_file_validation(
raster_file=temp_file)
if not error_info:
log.info("Geo raster file type file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == os.path.splitext(f)[1]
].pop()
metadata = extract_metadata(temp_vrt_file_path)
log.info(
"Geo raster file type metadata extraction was successful.")
with transaction.atomic():
# create a geo raster logical file object to be associated with resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = file_name
logical_file.save()
try:
# create a folder for the raster file type using the base file name as the
# name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, file_name)
log.info("Folder created:{}".format(new_folder_path))
create_folder(resource.short_id, new_folder_path)
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
# the added resource file will be part of a new generic logical file
# by default
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# delete the generic logical file object
if new_res_file.logical_file is not None:
# deleting the file level metadata object will delete the associated
# logical file object
new_res_file.logical_file.metadata.delete()
# make each resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"Geo raster file type - new files were added to the resource."
)
# use the extracted metadata to populate file metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
logical_file.metadata.create_element(k, **v)
log.info(
"Geo raster file type - metadata was saved to DB")
# set resource to private if logical file is missing required metadata
resource.update_public_and_discoverable()
# delete the original resource file
delete_resource_file(resource.short_id, res_file.id,
user)
log.info("Deleted original resource file.")
except Exception as ex:
msg = "Geo raster file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
if upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join(
'data', 'contents', upload_folder)
remove_folder(user, resource.short_id,
folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
else:
err_msg = "Geo raster file type file validation failed.{}".format(
' '.join(error_info))
log.info(err_msg)
raise ValidationError(err_msg)
else:
if res_file is None:
err_msg = "Failed to set Geo raster file type. " \
"Resource doesn't have the specified file."
log.error(err_msg)
raise ValidationError(err_msg)
else:
err_msg = "Failed to set Geo raster file type." \
"The specified file doesn't have a generic logical file type."
log.error(err_msg)
raise ValidationError(err_msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a .shp or .zip resource file to GeoFeatureFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoFeatureFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None or not res_file.exists:
raise ValidationError("File not found.")
if res_file.extension.lower() not in ('.zip', '.shp'):
raise ValidationError("Not a valid geographic feature file.")
if not res_file.has_generic_logical_file:
raise ValidationError(
"Selected file must be part of a generic file type.")
try:
meta_dict, shape_files, shp_res_files = extract_metadata_and_files(
resource, res_file)
except ValidationError as ex:
log.exception(ex.message)
raise ex
# hold on to temp dir for final clean up
temp_dir = os.path.dirname(shape_files[0])
file_name = res_file.file_name
# file name without the extension
base_file_name = file_name[:-len(res_file.extension)]
xml_file = ''
for f in shape_files:
if f.lower().endswith('.shp.xml'):
xml_file = f
break
file_folder = res_file.file_folder
file_type_success = False
upload_folder = ''
msg = "GeoFeature file type. Error when setting file type. Error:{}"
with transaction.atomic():
# create a GeoFeature logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = base_file_name
logical_file.save()
try:
# create a folder for the geofeature file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, base_file_name)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(file_folder, new_folder_name)
# add all new files to the resource
files_to_add_to_resource = shape_files
for fl in files_to_add_to_resource:
uploaded_file = UploadedFile(file=open(fl, 'rb'),
name=os.path.basename(fl))
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# make each resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"GeoFeature file type - files were added to the file type."
)
add_metadata(resource, meta_dict, xml_file, logical_file)
log.info(
"GeoFeature file type and resource level metadata updated."
)
# delete the original resource files used as part of setting file type
for fl in shp_res_files:
delete_resource_file(resource.short_id, fl.id, user)
log.info("Deleted original resource files.")
file_type_success = True
except Exception as ex:
msg = msg.format(ex.message)
log.exception(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
if not file_type_success and upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join('data', 'contents', upload_folder)
remove_folder(user, resource.short_id, folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
