def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir',
store[0],
msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg="resource does not contain the updated file new_" +
file_name_list[2])
self.assertNotIn(file_name_list[2],
updated_res_file_names,
msg='resource still contains the old file ' +
file_name_list[2] + ' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' moved to a folder')
self.assertNotIn(file_name_list[0],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[0] + 'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[1] + ' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(),
2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
uname=user.username,
fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
with self.assertRaises(SessionException):
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip',
False)
# Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
3,
msg="resource file count didn't match - " +
str(file_cnt) + " != 3")
# test unzipping the file succeeds now after deleting the existing folder
# TODO: this causes a multiple delete because the paths are valid now.
istorage = res.get_irods_storage()
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
# Now resource should contain two files: file3_new.txt and sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
2,
msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn(
'sub_test_dir.zip',
updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[0])
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[1])
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg='resource does not contain unzipped file new_' +
file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(),
1,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def data_store_folder_zip(request, res_id=None):
"""
Zip requested files and folders into a zip file in hydroshareZone or any federated zone
used for CommonsShare resource backend store. It is invoked by an AJAX call and returns
json object that holds the created zip file name if it succeeds, and an empty string
if it fails. The AJAX request must be a POST request with input data passed in for
res_id, input_coll_path, output_zip_file_name, and remove_original_after_zip where
input_coll_path is the relative sub-collection path under res_id collection to be zipped,
output_zip_file_name is the file name only with no path of the generated zip file name,
and remove_original_after_zip has a value of "true" or "false" (default is "true") indicating
whether original files will be deleted after zipping.
"""
res_id = request.POST.get('res_id', res_id)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, user = authorize(
request,
res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found',
status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied',
status=status.HTTP_401_UNAUTHORIZED)
input_coll_path = resolve_request(request).get('input_coll_path', None)
if input_coll_path is None:
return HttpResponse('Bad request - input_coll_path is not included',
status=status.HTTP_400_BAD_REQUEST)
input_coll_path = str(input_coll_path).strip()
if not input_coll_path:
return HttpResponse('Bad request - input_coll_path cannot be empty',
status=status.HTTP_400_BAD_REQUEST)
if not input_coll_path.startswith('data/contents/'):
return HttpResponse(
'Bad request - input_coll_path must start with data/contents/',
status=status.HTTP_400_BAD_REQUEST)
if input_coll_path.find('/../') >= 0 or input_coll_path.endswith('/..'):
return HttpResponse(
'Bad request - input_coll_path must not contain /../',
status=status.HTTP_400_BAD_REQUEST)
output_zip_fname = resolve_request(request).get('output_zip_file_name',
None)
if output_zip_fname is None:
return HttpResponse('Bad request - output_zip_fname is not included',
status=status.HTTP_400_BAD_REQUEST)
output_zip_fname = str(output_zip_fname).strip()
if not output_zip_fname:
return HttpResponse('Bad request - output_zip_fname cannot be empty',
status=status.HTTP_400_BAD_REQUEST)
if output_zip_fname.find('/') >= 0:
return HttpResponse('Bad request - output_zip_fname cannot contain /',
status=status.HTTP_400_BAD_REQUEST)
remove_original = resolve_request(request).get('remove_original_after_zip',
None)
bool_remove_original = True
if remove_original:
remove_original = str(remove_original).strip().lower()
if remove_original == 'false':
bool_remove_original = False
try:
output_zip_fname, size = \
zip_folder(user, res_id, input_coll_path, output_zip_fname, bool_remove_original)
except SessionException as ex:
return HttpResponse(ex.stderr,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except DRF_ValidationError as ex:
return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST)
return_object = {'name': output_zip_fname, 'size': size, 'type': 'zip'}
return HttpResponse(json.dumps(return_object),
content_type="application/json")
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg="resource does not contain the updated file new_" + file_name_list[2])
self.assertNotIn(file_name_list[2], updated_res_file_names,
msg='resource still contains the old file ' + file_name_list[2] +
' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] + ' moved to a folder')
self.assertNotIn(file_name_list[0], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[0] +
'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[1] +
' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(), 2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id, source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False)
# Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip,
# and file1.txt (2)
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " +
str(file_cnt) + " != 5")
# remove all files except the zippped file
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1')
# Now resource should contain two files: file3_new.txt sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir.zip', updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[0])
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[1])
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg='resource does not contain unzipped file new_' + file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def data_store_folder_zip(request, res_id=None):
"""
Zip requested files and folders into a zip file in hydroshareZone or any federated zone
used for HydroShare resource backend store. It is invoked by an AJAX call and returns
json object that holds the created zip file name if it succeeds, and an empty string
if it fails. The AJAX request must be a POST request with input data passed in for
res_id, input_coll_path, output_zip_file_name, and remove_original_after_zip where
input_coll_path is the relative path under res_id/data/contents to be zipped,
output_zip_file_name is the file name only with no path of the generated zip file name,
and remove_original_after_zip has a value of "true" or "false" (default is "true") indicating
whether original files will be deleted after zipping.
"""
res_id = request.POST.get('res_id', res_id)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, user = authorize(request, res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED)
input_coll_path = resolve_request(request).get('input_coll_path', None)
try:
input_coll_path = _validate_path(input_coll_path, 'input_coll_path')
except ValidationError as ex:
return HttpResponse(ex.message, status=status.HTTP_400_BAD_REQUEST)
output_zip_fname = resolve_request(request).get('output_zip_file_name', None)
if output_zip_fname is None:
return HttpResponse('Bad request - output_zip_fname is not included',
status=status.HTTP_400_BAD_REQUEST)
output_zip_fname = str(output_zip_fname).strip()
if not output_zip_fname:
return HttpResponse('Bad request - output_zip_fname cannot be empty',
status=status.HTTP_400_BAD_REQUEST)
if output_zip_fname.find('/') >= 0:
return HttpResponse('Bad request - output_zip_fname cannot contain /',
status=status.HTTP_400_BAD_REQUEST)
remove_original = resolve_request(request).get('remove_original_after_zip', None)
bool_remove_original = True
if remove_original:
remove_original = str(remove_original).strip().lower()
if remove_original == 'false':
bool_remove_original = False
try:
output_zip_fname, size = \
zip_folder(user, res_id, input_coll_path, output_zip_fname, bool_remove_original)
except SessionException as ex:
return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except DRF_ValidationError as ex:
return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST)
return_object = {'name': output_zip_fname,
'size': size,
'type': 'zip'}
return HttpResponse(
json.dumps(return_object),
content_type="application/json"
)