def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") with self.assertRaises(SessionException): unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt file_cnt = res.files.all().count() self.assertEqual(file_cnt, 3, msg="resource file count didn't match - " + str(file_cnt) + " != 3") # test unzipping the file succeeds now after deleting the existing folder # TODO: this causes a multiple delete because the paths are valid now. istorage = res.get_irods_storage() remove_folder(user, res.short_id, 'data/contents/sub_test_dir') # Now resource should contain two files: file3_new.txt and sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn( 'sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def data_store_folder_zip(request, res_id=None): """ Zip requested files and folders into a zip file in hydroshareZone or any federated zone used for CommonsShare resource backend store. It is invoked by an AJAX call and returns json object that holds the created zip file name if it succeeds, and an empty string if it fails. The AJAX request must be a POST request with input data passed in for res_id, input_coll_path, output_zip_file_name, and remove_original_after_zip where input_coll_path is the relative sub-collection path under res_id collection to be zipped, output_zip_file_name is the file name only with no path of the generated zip file name, and remove_original_after_zip has a value of "true" or "false" (default is "true") indicating whether original files will be deleted after zipping. """ res_id = request.POST.get('res_id', res_id) if res_id is None: return HttpResponse('Bad request - resource id is not included', status=status.HTTP_400_BAD_REQUEST) res_id = str(res_id).strip() try: resource, _, user = authorize( request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) except NotFound: return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST) except PermissionDenied: return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED) input_coll_path = resolve_request(request).get('input_coll_path', None) if input_coll_path is None: return HttpResponse('Bad request - input_coll_path is not included', status=status.HTTP_400_BAD_REQUEST) input_coll_path = str(input_coll_path).strip() if not input_coll_path: return HttpResponse('Bad request - input_coll_path cannot be empty', status=status.HTTP_400_BAD_REQUEST) if not input_coll_path.startswith('data/contents/'): return HttpResponse( 'Bad request - input_coll_path must start with data/contents/', status=status.HTTP_400_BAD_REQUEST) if input_coll_path.find('/../') >= 0 or input_coll_path.endswith('/..'): return HttpResponse( 'Bad request - input_coll_path must not contain /../', status=status.HTTP_400_BAD_REQUEST) output_zip_fname = resolve_request(request).get('output_zip_file_name', None) if output_zip_fname is None: return HttpResponse('Bad request - output_zip_fname is not included', status=status.HTTP_400_BAD_REQUEST) output_zip_fname = str(output_zip_fname).strip() if not output_zip_fname: return HttpResponse('Bad request - output_zip_fname cannot be empty', status=status.HTTP_400_BAD_REQUEST) if output_zip_fname.find('/') >= 0: return HttpResponse('Bad request - output_zip_fname cannot contain /', status=status.HTTP_400_BAD_REQUEST) remove_original = resolve_request(request).get('remove_original_after_zip', None) bool_remove_original = True if remove_original: remove_original = str(remove_original).strip().lower() if remove_original == 'false': bool_remove_original = False try: output_zip_fname, size = \ zip_folder(user, res_id, input_coll_path, output_zip_fname, bool_remove_original) except SessionException as ex: return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except DRF_ValidationError as ex: return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST) return_object = {'name': output_zip_fname, 'size': size, 'type': 'zip'} return HttpResponse(json.dumps(return_object), content_type="application/json")
def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip, # and file1.txt (2) file_cnt = res.files.all().count() self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " + str(file_cnt) + " != 5") # remove all files except the zippped file remove_folder(user, res.short_id, 'data/contents/sub_test_dir') remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1') # Now resource should contain two files: file3_new.txt sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def data_store_folder_zip(request, res_id=None): """ Zip requested files and folders into a zip file in hydroshareZone or any federated zone used for HydroShare resource backend store. It is invoked by an AJAX call and returns json object that holds the created zip file name if it succeeds, and an empty string if it fails. The AJAX request must be a POST request with input data passed in for res_id, input_coll_path, output_zip_file_name, and remove_original_after_zip where input_coll_path is the relative path under res_id/data/contents to be zipped, output_zip_file_name is the file name only with no path of the generated zip file name, and remove_original_after_zip has a value of "true" or "false" (default is "true") indicating whether original files will be deleted after zipping. """ res_id = request.POST.get('res_id', res_id) if res_id is None: return HttpResponse('Bad request - resource id is not included', status=status.HTTP_400_BAD_REQUEST) res_id = str(res_id).strip() try: resource, _, user = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) except NotFound: return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST) except PermissionDenied: return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED) input_coll_path = resolve_request(request).get('input_coll_path', None) try: input_coll_path = _validate_path(input_coll_path, 'input_coll_path') except ValidationError as ex: return HttpResponse(ex.message, status=status.HTTP_400_BAD_REQUEST) output_zip_fname = resolve_request(request).get('output_zip_file_name', None) if output_zip_fname is None: return HttpResponse('Bad request - output_zip_fname is not included', status=status.HTTP_400_BAD_REQUEST) output_zip_fname = str(output_zip_fname).strip() if not output_zip_fname: return HttpResponse('Bad request - output_zip_fname cannot be empty', status=status.HTTP_400_BAD_REQUEST) if output_zip_fname.find('/') >= 0: return HttpResponse('Bad request - output_zip_fname cannot contain /', status=status.HTTP_400_BAD_REQUEST) remove_original = resolve_request(request).get('remove_original_after_zip', None) bool_remove_original = True if remove_original: remove_original = str(remove_original).strip().lower() if remove_original == 'false': bool_remove_original = False try: output_zip_fname, size = \ zip_folder(user, res_id, input_coll_path, output_zip_fname, bool_remove_original) except SessionException as ex: return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except DRF_ValidationError as ex: return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST) return_object = {'name': output_zip_fname, 'size': size, 'type': 'zip'} return HttpResponse( json.dumps(return_object), content_type="application/json" )