def test_set_file_type_success_2(self): # here we will set the tif file (the file being not in root dir)to GeoRaster file type # resource should have no file at this point self.assertEqual(self.resource.files.count(), 0) # add the tif file to the composite resource tif_file_obj = open(self.raster_file_path, "r") uploaded_file = UploadedFile(file=tif_file_obj, name=os.path.basename(tif_file_obj.name)) resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user, auto_aggregate=False) # resource should have one file at this point self.assertEqual(self.resource.files.count(), 1) res_file = self.resource.files.all().first() self.assertEqual(res_file.file_name, self.raster_file_name) create_folder(self.resource.short_id, 'data/contents/sub_test_dir') # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder(self.user, self.resource.short_id, 'data/contents/' + self.raster_file_name, 'data/contents/sub_test_dir/' + self.raster_file_name) res_file = self.resource.files.all().first() self.assertEqual(res_file.short_path, "sub_test_dir/" + self.raster_file_name) # test the set file type endpoint url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/" set_file_type_url = url_template.format(res_id=self.resource.short_id, file_path=res_file.short_path, file_type="GeoRaster") response = self.client.post(set_file_type_url, {}, format='json') self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_file_rename_or_move(self): # test that resource file that belongs to GenericLogicalFile object # can be moved or renamed self.generic_file_obj = open(self.generic_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() self.assertEqual(os.path.basename(res_file.resource_file.name), 'generic_file.txt') # test rename of file is allowed src_path = 'data/contents/generic_file.txt' tgt_path = "data/contents/generic_file_1.txt" move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() self.assertEqual(os.path.basename(res_file.resource_file.name), 'generic_file_1.txt') # test moving the file to a new folder is allowed src_path = 'data/contents/generic_file_1.txt' tgt_path = "data/contents/test_folder/generic_file_1.txt" move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() self.assertTrue(res_file.resource_file.name.endswith(tgt_path))
def test_aggregation_file_move(self): # test any resource file that's part of the NetCDF logical file can't be moved self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() # create the aggregation using the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with raster LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 2) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) expected_folder_name = base_file_name self.assertEqual(res_file.file_folder, expected_folder_name) new_folder = 'netcdf_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # moving any of the resource files to this new folder should raise exception tgt_path = 'data/contents/{}'.format(new_folder) for res_file in self.composite_resource.files.all(): with self.assertRaises(DRF_ValidationError): src_path = os.path.join('data', 'contents', res_file.short_path) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_aggregation_file_move(self): # test any resource file that's part of the GeoFeature logical file can't be moved self.create_composite_resource(self.states_required_zip_file) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) # extract metadata from the tif file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with geo feature LFO - which should # raise exception self.assertEqual(self.composite_resource.files.count(), 3) new_folder = 'geofeature_aggr' ResourceFile.create_folder(self.composite_resource, new_folder) # moving any of the resource files to this new folder should raise exception tgt_path = 'data/contents/geofeature_aggr' for res_file in self.composite_resource.files.all(): with self.assertRaises(DRF_ValidationError): src_path = os.path.join('data', 'contents', res_file.short_path) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_aggregation_file_rename(self): # test that a file can't renamed for any resource file # that's part of the NetCDF logical file self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) expected_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with aggregation raises exception self.assertEqual(self.composite_resource.files.count(), 2) for res_file in self.composite_resource.files.all(): base_file_name, ext = os.path.splitext(res_file.file_name) self.assertEqual(res_file.file_folder, expected_folder_name) src_path = 'data/contents/{0}/{1}.{2}'.format(expected_folder_name, base_file_name, ext) new_file_name = 'some_netcdf.{}'.format(ext) self.assertNotEqual(res_file.file_name, new_file_name) tgt_path = 'data/contents/{}/{}'.format(expected_folder_name, new_file_name) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_file_move(self): # test that a resource file that is part of a GenericLogicalFile object # can be moved self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) self.assertEqual(res_file.file_name, 'generic_file.txt') # create generic aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # file should not be in a folder self.assertEqual(res_file.file_folder, None) # test moving the file to a new folder is allowed new_folder = 'test_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = "data/contents/{0}/{1}".format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() # file should in a folder self.assertEqual(res_file.file_folder, new_folder) self.assertTrue(res_file.resource_file.name.endswith(tgt_path)) self.composite_resource.delete()
def test_file_rename_or_move(self): # test that file can't be moved or renamed for any resource file # that's part of the TimeSeries logical file object (LFO) self.sqlite_file_obj = open(self.sqlite_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the sqlite file TimeSeriesLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test renaming of files that are associated with timeseries LFO - should raise exception self.assertEqual(self.composite_resource.files.count(), 1) base_path = "data/contents/ODM2_Multi_Site_One_Variable/{}" src_path = base_path.format('ODM2_Multi_Site_One_Variable.sqlite') tgt_path = base_path.format('ODM2_Multi_Site_One_Variable_1.sqlite') with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # TODO: test for renaming csv file when we implement csv file # test moving the files associated with timeseries LFO tgt_path = 'data/contents/new_folder/ODM2_Multi_Site_One_Variable.sqlite' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # TODO: test for moving csv file when we implement csv file self.composite_resource.delete()
def test_aggregation_folder_rename(self): # test changes to aggregation name, aggregation metadata xml file path, and aggregation # resource map xml file path on folder name change self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) expected_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(self.composite_resource.files.count(), 2) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, expected_folder_name) # test aggregation name res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_folder) # test aggregation xml file paths expected_meta_file_path = '{}/{}_meta.xml'.format(base_file_name, base_file_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{}/{}_resmap.xml'.format(base_file_name, base_file_name) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) # test renaming folder src_path = 'data/contents/{}'.format(expected_folder_name) tgt_path = 'data/contents/{}_1'.format(expected_folder_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, '{}_1'.format(expected_folder_name)) # test aggregation name update res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_folder) # test aggregation xml file paths expected_meta_file_path = '{}_1/{}_1_meta.xml'.format(expected_folder_name, expected_folder_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{}_1/{}_1_resmap.xml'.format(expected_folder_name, expected_folder_name) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) self.composite_resource.delete()
def data_store_file_or_folder_move_or_rename(request, res_id=None): """ Move or rename a file or folder in hydroshareZone or any federated zone used for HydroShare resource backend store. It is invoked by an AJAX call and returns json object that has the relative path of the target file or folder being moved to if succeeds, and return empty string if fails. The AJAX request must be a POST request with input data passed in for res_id, source_path, and target_path where source_path and target_path are the relative paths for the source and target file or folder under res_id collection/directory. """ res_id = request.POST.get('res_id', res_id) if res_id is None: return HttpResponse('Bad request - resource id is not included', status=status.HTTP_400_BAD_REQUEST) res_id = str(res_id).strip() try: resource, _, user = authorize( request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) except NotFound: return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST) except PermissionDenied: return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED) src_path = resolve_request(request).get('source_path', None) tgt_path = resolve_request(request).get('target_path', None) if src_path is None or tgt_path is None: return HttpResponse( 'Bad request - src_path or tgt_path is not included', status=status.HTTP_400_BAD_REQUEST) src_path = str(src_path).strip() tgt_path = str(tgt_path).strip() if not src_path or not tgt_path: return HttpResponse( 'Bad request - src_path or tgt_path cannot be empty', status=status.HTTP_400_BAD_REQUEST) try: move_or_rename_file_or_folder(user, res_id, src_path, tgt_path) except SessionException as ex: return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except DRF_ValidationError as ex: return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST) return_object = {'target_rel_path': tgt_path} return HttpResponse(json.dumps(return_object), content_type='application/json')
def data_store_file_or_folder_move_or_rename(request, res_id=None): """ Move or rename a file or folder in hydroshareZone or any federated zone used for HydroShare resource backend store. It is invoked by an AJAX call and returns json object that has the relative path of the target file or folder being moved to if succeeds, and return empty string if fails. The AJAX request must be a POST request with input data passed in for res_id, source_path, and target_path where source_path and target_path are the relative paths (relative to path res_id/data/contents) for the source and target file or folder under res_id collection/directory. """ res_id = request.POST.get('res_id', res_id) if res_id is None: return HttpResponse('Bad request - resource id is not included', status=status.HTTP_400_BAD_REQUEST) res_id = str(res_id).strip() try: resource, _, user = authorize(request, res_id, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) except NotFound: return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST) except PermissionDenied: return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED) src_path = resolve_request(request).get('source_path', None) tgt_path = resolve_request(request).get('target_path', None) try: src_path = _validate_path(src_path, 'src_path') tgt_path = _validate_path(tgt_path, 'tgt_path') except ValidationError as ex: return HttpResponse(ex.message, status=status.HTTP_400_BAD_REQUEST) try: move_or_rename_file_or_folder(user, res_id, src_path, tgt_path) except SessionException as ex: return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except DRF_ValidationError as ex: return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST) return_object = {'target_rel_path': tgt_path} return HttpResponse( json.dumps(return_object), content_type='application/json' )
def test_file_rename(self): # test that a resource file that is part of a GenericLogicalFile object # can be renamed self.create_composite_resource(self.generic_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) self.assertEqual(res_file.file_name, 'generic_file.txt') # create generic aggregation GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # file should not be in a folder self.assertEqual(res_file.file_folder, None) # test rename of file is allowed src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = "data/contents/{0}_1{1}".format(base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_name, '{0}_1{1}'.format(base_file_name, ext)) self.composite_resource.delete()
def test_aggregation_folder_move_not_allowed(self): # test a folder is not allowed to be moved into a folder that represents an aggregation self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) aggregation_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # create a folder to move the aggregation folder there new_folder = 'folder_to_move' ResourceFile.create_folder(self.composite_resource, new_folder) # move the new folder into the aggregation folder src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(aggregation_folder_name) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_file_move_to_aggregation_not_allowed(self): # test no file can be moved into a folder that represents an aggregation self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_folder, None) # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertNotEqual(res_file.file_folder, None) # add a file to the resource which will try to move into the aggregation folder res_file_to_move = self.add_file_to_resource(file_to_add=self.netcdf_invalid_file) src_path = os.path.join('data', 'contents', res_file_to_move.short_path) tgt_path = 'data/contents/{}'.format(res_file.file_folder) # move file to aggregation folder with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_aggregation_name(self): # test the aggregation_name property for the generic aggregation (logical file) self.create_composite_resource(self.generic_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, False) # set file to generic logical file type GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_name) # test the aggregation name after moving the file into a folder new_folder = 'generic_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}'.format(new_folder, res_file.file_name) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) # test the aggregation name after renaming the file src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) # test the aggregation name after renaming the folder folder_rename = '{}_1'.format(new_folder) src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(folder_rename) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) logical_file = res_file.logical_file expected_aggregation_name = '{0}/{1}'.format(folder_rename, res_file.file_name) self.assertEqual(logical_file.aggregation_name, expected_aggregation_name) self.composite_resource.delete()
def test_file_rename_or_move(self): # test that file can't be moved or renamed for any resource file # that's part of the GeoFeature logical file object (LFO) self._create_composite_resource(self.states_required_zip_file) res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoFeatureLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test renaming of files that are associated with raster LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 3) res_file = self.composite_resource.files.first() self.assertEqual(res_file.file_folder, 'states_required_files') src_path = 'data/contents/states_required_files/states.shp' tgt_path = 'data/contents/states_required_files/states-1.shp' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/states_required_files/states.dbf' tgt_path = 'data/contents/states_required_files/states-1.dbf' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # test moving the files associated with geo raster LFO src_path = 'data/contents/states_required_files/states.shx' tgt_path = 'data/contents/states_required_files/states-1.shx' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/states_required_files/states.shp' tgt_path = 'data/contents/states_required_files-1/states.shp' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_file_rename_or_move(self): # test that file can't be moved or renamed for any resource file # that's part of the NetCDF logical file object (LFO) self.netcdf_file_obj = open(self.netcdf_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file NetCDFLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test renaming of files that are associated with netcdf LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 2) src_path = 'data/contents/netcdf_valid/netcdf_valid.nc' tgt_path = "data/contents/netcdf_valid/netcdf_valid_1.nc" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/netcdf_valid/netcdf_valid_header_info.txt' tgt_path = 'data/contents/netcdf_valid/netcdf_valid_header_info_1.txt' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # test moving the files associated with netcdf LFO src_path = 'data/contents/netcdf_valid/netcdf_valid.nc' tgt_path = 'data/contents/netcdf_valid_1/netcdf_valid.nc' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/netcdf_valid/netcdf_valid_header_info.txt' tgt_path = 'data/contents/netcdf_valid_1/netcdf_valid_header_info.txt' with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_file_rename_or_move(self): # test that file can't be moved or renamed for any resource file # that's part of the GeoRaster logical file object (LFO) self.raster_file_obj = open(self.raster_file, 'r') self._create_composite_resource() res_file = self.composite_resource.files.first() # extract metadata from the tif file GeoRasterLogicalFile.set_file_type(self.composite_resource, res_file.id, self.user) # test renaming of files that are associated with raster LFO - which should raise exception self.assertEqual(self.composite_resource.files.count(), 2) src_path = 'data/contents/small_logan/small_logan.tif' tgt_path = "data/contents/small_logan/small_logan_1.tif" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/small_logan/small_logan.vrt' tgt_path = "data/contents/small_logan/small_logan_1.vrt" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) # test moving the files associated with geo raster LFO src_path = 'data/contents/small_logan/small_logan.tif' tgt_path = "data/contents/big_logan/small_logan.tif" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/small_logan/small_logan.vrt' tgt_path = "data/contents/big_logan/small_logan.vrt" with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def test_aggregation_file_rename(self): # test that a file can't renamed for any resource file # that's part of the GeoFeature logical file self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) res_file = self.composite_resource.files.first() base_file_name, _ = os.path.splitext(res_file.file_name) # create aggregation from the zip file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with aggregation raises exception self.assertEqual(self.composite_resource.files.count(), 3) src_path = 'data/contents/{}/states.shp'.format(new_folder) tgt_path = 'data/contents/{}/states-1.shp'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/{}/states.dbf'.format(new_folder) tgt_path = 'data/contents/{}/states-1.dbf'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) src_path = 'data/contents/{}/states.shx'.format(new_folder) tgt_path = 'data/contents/{}/states-1.shx'.format(new_folder) with self.assertRaises(DRF_ValidationError): move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) self.composite_resource.delete()
def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") with self.assertRaises(SessionException): unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt file_cnt = res.files.all().count() self.assertEqual(file_cnt, 3, msg="resource file count didn't match - " + str(file_cnt) + " != 3") # test unzipping the file succeeds now after deleting the existing folder # TODO: this causes a multiple delete because the paths are valid now. istorage = res.get_irods_storage() remove_folder(user, res.short_id, 'data/contents/sub_test_dir') # Now resource should contain two files: file3_new.txt and sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn( 'sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def test_aggregation_folder_move(self): # test changes to aggregation name, aggregation metadata xml file path, and aggregation # resource map xml file path on aggregation folder move self.create_composite_resource() new_folder = 'my_folder' ResourceFile.create_folder(self.composite_resource, new_folder) # add the 3 required files to the resource at the above folder self.add_file_to_resource(file_to_add=self.states_required_zip_file, upload_folder=new_folder) res_file = self.composite_resource.files.first() # create aggregation from the zip file GeoFeatureLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) self.assertEqual(self.composite_resource.files.count(), 3) base_file_name, _ = os.path.splitext(res_file.file_name) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, new_folder) # test aggregation name res_file = self.composite_resource.files.first() logical_file = res_file.logical_file shp_file_path = get_path_with_no_file_extension( logical_file.aggregation_name) # test aggregation xml file paths expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) # create a folder to move the aggregation folder there parent_folder = 'parent_folder' ResourceFile.create_folder(self.composite_resource, parent_folder) # move the aggregation folder to the parent folder src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}/{}'.format(parent_folder, new_folder) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) file_folder = '{}/{}'.format(parent_folder, new_folder) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, file_folder) # test aggregation name update res_file = self.composite_resource.files.first() logical_file = res_file.logical_file shp_file_path = get_path_with_no_file_extension( logical_file.aggregation_name) # test aggregation xml file paths expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) self.composite_resource.delete()
def test_aggregation_parent_folder_rename(self): # test changes to aggregation name, aggregation metadata xml file path, and aggregation # resource map xml file path on aggregation folder parent folder name change self.create_composite_resource() self.add_file_to_resource(file_to_add=self.netcdf_file) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) aggregation_folder_name = base_file_name # create aggregation from the nc file NetCDFLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) # test renaming of files that are associated with aggregation raises exception self.assertEqual(self.composite_resource.files.count(), 2) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, aggregation_folder_name) # test aggregation name res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_folder) # test aggregation xml file paths # test aggregation xml file paths expected_meta_file_path = '{}/{}_meta.xml'.format(aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{}/{}_resmap.xml'.format(aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) # create a folder to be the parent folder of the aggregation folder parent_folder = 'parent_folder' ResourceFile.create_folder(self.composite_resource, parent_folder) # move the aggregation folder to the parent folder src_path = 'data/contents/{}'.format(aggregation_folder_name) tgt_path = 'data/contents/{0}/{1}'.format(parent_folder, aggregation_folder_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) file_folder = '{}/{}'.format(parent_folder, aggregation_folder_name) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, file_folder) # renaming parent folder parent_folder_rename = 'parent_folder_1' src_path = 'data/contents/{}'.format(parent_folder) tgt_path = 'data/contents/{}'.format(parent_folder_rename) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) file_folder = '{}/{}'.format(parent_folder_rename, aggregation_folder_name) for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.file_folder, file_folder) # test aggregation name after folder rename res_file = self.composite_resource.files.first() logical_file = res_file.logical_file self.assertEqual(logical_file.aggregation_name, res_file.file_folder) # test aggregation xml file paths after folder rename expected_meta_file_path = '{0}/{1}/{2}_meta.xml'.format(parent_folder_rename, aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path) expected_map_file_path = '{0}/{1}/{2}_resmap.xml'.format(parent_folder_rename, aggregation_folder_name, aggregation_folder_name) self.assertEqual(logical_file.map_short_file_path, expected_map_file_path) self.composite_resource.delete()
def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip, # and file1.txt (2) file_cnt = res.files.all().count() self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " + str(file_cnt) + " != 5") # remove all files except the zippped file remove_folder(user, res.short_id, 'data/contents/sub_test_dir') remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1') # Now resource should contain two files: file3_new.txt sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def test_aggregation_xml_file_paths(self): # test the aggregation meta and map xml file paths with file name and folder name # changes self.create_composite_resource(self.refts_file) # there should be one resource file self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() base_file_name, ext = os.path.splitext(res_file.file_name) # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, False) # set file to generic logical file type RefTimeseriesLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id) res_file = self.composite_resource.files.first() self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name) logical_file = res_file.logical_file expected_meta_path = '{}_meta.xml'.format(res_file.file_name) expected_map_path = '{}_resmap.xml'.format(res_file.file_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test xml file paths after moving the file into a folder new_folder = 'test_folder' create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder)) src_path = 'data/contents/{}'.format(res_file.file_name) tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_meta_path = '{0}/{1}_meta.xml'.format(new_folder, res_file.file_name) expected_map_path = '{0}/{1}_resmap.xml'.format( new_folder, res_file.file_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test xml file paths after renaming the file src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name) tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_meta_path = '{0}/{1}_meta.xml'.format(new_folder, res_file.file_name) expected_map_path = '{0}/{1}_resmap.xml'.format( new_folder, res_file.file_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) # test the xml file path after renaming the folder folder_rename = '{}_1'.format(new_folder) src_path = 'data/contents/{}'.format(new_folder) tgt_path = 'data/contents/{}'.format(folder_rename) move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path, tgt_path) res_file = self.composite_resource.files.first() logical_file = res_file.logical_file expected_meta_path = '{0}/{1}_meta.xml'.format(folder_rename, res_file.file_name) expected_map_path = '{0}/{1}_resmap.xml'.format( folder_rename, res_file.file_name) self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path) self.assertEqual(logical_file.map_short_file_path, expected_map_path) self.composite_resource.delete()