def test_move_to_folder_basic(self):
group, _ = Group.objects.get_or_create(name='Hydroshare Author')
user = hydroshare.create_account('*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[])
resource = hydroshare.create_resource(
'GenericResource',
user,
'test resource',
)
resource.save()
open('myfile.txt', "w").close()
file = open('myfile.txt', 'rb')
hydroshare.add_resource_files(resource.short_id, file)
create_folder(resource.short_id, "data/contents/test_folder")
move_to_folder(user,
resource.short_id,
src_paths=['data/contents/myfile.txt'],
tgt_path="data/contents/test_folder",
validate_move=True)
folder_contents = list_folder(resource.short_id,
"data/contents/test_folder")
self.assertTrue(['myfile.txt'] in folder_contents)
resource.delete()
def test_file_move(self):
# test that a resource file that is part of a GenericLogicalFile object
# can be moved
self.create_composite_resource(self.generic_file)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
self.assertEqual(res_file.file_name, 'generic_file.txt')
# create generic aggregation
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# file should not be in a folder
self.assertEqual(res_file.file_folder, None)
# test moving the file to a new folder is allowed
new_folder = 'test_folder'
create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = "data/contents/{0}/{1}".format(new_folder, res_file.file_name)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id, src_path,
tgt_path)
res_file = self.composite_resource.files.first()
# file should in a folder
self.assertEqual(res_file.file_folder, new_folder)
self.assertTrue(res_file.resource_file.name.endswith(tgt_path))
self.composite_resource.delete()
def test_readme_file_4(self):
"""Test that when we upload a readme.txt or a readme.md file to a folder,
such a file is NOT considered as the readme file of the resource"""
self._create_composite_resource()
# resource should not have any file at this point
self.assertEqual(self.composite_resource.files.count(), 0)
# create the folder
new_folder_path = os.path.join("data", "contents", "my-new-folder")
create_folder(self.composite_resource.short_id, new_folder_path)
# add the readme.txt file to the resource at the folder 'my-new-folder'
files_to_add = [self.readme_txt]
self._add_files_to_resource(files_to_add, upload_folder=new_folder_path)
# resource should have one file at this point
self.assertEqual(self.composite_resource.files.count(), 1)
# resource has no readme file
self.assertEqual(self.composite_resource.readme_file, None)
# add the readme.md file to the resource at the folder 'my-new-folder'
files_to_add = [self.readme_md]
self._add_files_to_resource(files_to_add, upload_folder=new_folder_path)
# resource should have two files at this point
self.assertEqual(self.composite_resource.files.count(), 2)
# resource has no readme file
self.assertEqual(self.composite_resource.readme_file, None)
def put(self, request, pk, pathname):
""" create a given folder if not present and allowed
"""
try:
resource, authorized, user = view_utils.authorize(
request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE,
raises_exception=False)
except NotFound as ex:
return Response(ex.message, status=status.HTTP_404_NOT_FOUND)
if not authorized:
return Response("Insufficient permission", status=status.HTTP_401_UNAUTHORIZED)
if not resource.supports_folders:
return Response("Resource type does not support subfolders",
status=status.HTTP_403_FORBIDDEN)
try:
view_utils.irods_path_is_allowed(pathname) # check for hacking attempts
except (ValidationError, SuspiciousFileOperation) as ex:
return Response(ex.message, status=status.HTTP_400_BAD_REQUEST)
# relativise the path
relpath = os.path.join('data', 'contents', pathname)
try:
view_utils.create_folder(pk, relpath)
except SessionException:
raise ValidationError("Cannot create folder")
return Response(data={'resource_id': pk, 'path': pathname},
status=status.HTTP_201_CREATED)
def test_set_file_type_success_2(self):
# here we will set the tif file (the file being not in root dir)to GeoRaster file type
# resource should have no file at this point
self.assertEqual(self.resource.files.count(), 0)
# add the tif file to the composite resource
tif_file_obj = open(self.raster_file_path, "r")
uploaded_file = UploadedFile(file=tif_file_obj,
name=os.path.basename(tif_file_obj.name))
resource_file_add_process(resource=self.resource, files=(uploaded_file,), user=self.user,
auto_aggregate=False)
# resource should have one file at this point
self.assertEqual(self.resource.files.count(), 1)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.file_name, self.raster_file_name)
create_folder(self.resource.short_id, 'data/contents/sub_test_dir')
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(self.user, self.resource.short_id,
'data/contents/' + self.raster_file_name,
'data/contents/sub_test_dir/' + self.raster_file_name)
res_file = self.resource.files.all().first()
self.assertEqual(res_file.short_path, "sub_test_dir/" + self.raster_file_name)
# test the set file type endpoint
url_template = "/hsapi/resource/{res_id}/functions/set-file-type/{file_path}/{file_type}/"
set_file_type_url = url_template.format(res_id=self.resource.short_id,
file_path=res_file.short_path,
file_type="GeoRaster")
response = self.client.post(set_file_type_url, {}, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_move_to_folder_basic(self):
group, _ = Group.objects.get_or_create(name='Hydroshare Author')
user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[]
)
resource = hydroshare.create_resource(
'GenericResource',
user,
'test resource',
)
resource.save()
open('myfile.txt', "w").close()
file = open('myfile.txt', 'r')
hydroshare.add_resource_files(resource.short_id, file)
create_folder(resource.short_id, "data/contents/test_folder")
move_to_folder(user, resource.short_id,
src_paths=['data/contents/myfile.txt'],
tgt_path="data/contents/test_folder",
validate_move=True)
folder_contents = list_folder(resource.short_id, "data/contents/test_folder")
self.assertTrue(['myfile.txt'] in folder_contents)
resource.delete()
def data_store_create_folder(request):
"""
create a sub-folder/sub-collection in hydroshareZone or any federated zone used for CommonsShare
resource backend store. It is invoked by an AJAX call and returns json object that has the
relative path of the new folder created if succeeds, and return empty string if fails. The
AJAX request must be a POST request with input data passed in for res_id and folder_path
where folder_path is the relative path for the new folder to be created under
res_id collection/directory.
"""
res_id = request.POST.get('res_id', None)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, _ = authorize(
request,
res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found',
status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied',
status=status.HTTP_401_UNAUTHORIZED)
folder_path = request.POST.get('folder_path', None)
if folder_path is None:
return HttpResponse('Bad request - folder_path is not included',
status=status.HTTP_400_BAD_REQUEST)
folder_path = str(folder_path).strip()
if not folder_path:
return HttpResponse('Bad request - folder_path cannot be empty',
status=status.HTTP_400_BAD_REQUEST)
if not folder_path.startswith('data/contents/'):
return HttpResponse(
'Bad request - folder_path must start with data/contents/',
status=status.HTTP_400_BAD_REQUEST)
if folder_path.find('/../') >= 0 or folder_path.endswith('/..'):
return HttpResponse('Bad request - folder_path must not contain /../',
status=status.HTTP_400_BAD_REQUEST)
try:
create_folder(res_id, folder_path)
except SessionException as ex:
return HttpResponse(ex.stderr,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except DRF_ValidationError as ex:
return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST)
return_object = {'new_folder_rel_path': folder_path}
return HttpResponse(json.dumps(return_object),
content_type="application/json")
def test_aggregation_name(self):
# test the aggregation_name property for the generic aggregation (logical file)
self.create_composite_resource(self.generic_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, False)
# set file to generic logical file type
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name, self.logical_file_type_name)
logical_file = res_file.logical_file
self.assertEqual(logical_file.aggregation_name, res_file.file_name)
# test the aggregation name after moving the file into a folder
new_folder = 'generic_folder'
create_folder(self.composite_resource.short_id, 'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}'.format(new_folder, res_file.file_name)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
# test the aggregation name after renaming the file
src_path = 'data/contents/{0}/{1}'.format(new_folder, res_file.file_name)
tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder, base_file_name, ext)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}_1{2}'.format(new_folder, base_file_name, ext)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
# test the aggregation name after renaming the folder
folder_rename = '{}_1'.format(new_folder)
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}'.format(folder_rename)
move_or_rename_file_or_folder(self.user, self.composite_resource.short_id,
src_path, tgt_path)
logical_file = res_file.logical_file
expected_aggregation_name = '{0}/{1}'.format(folder_rename, res_file.file_name)
self.assertEqual(logical_file.aggregation_name, expected_aggregation_name)
self.composite_resource.delete()
def data_store_create_folder(request):
"""
create a sub-folder/sub-collection in hydroshareZone or any federated zone used for HydroShare
resource backend store. It is invoked by an AJAX call and returns json object that has the
relative path of the new folder created if succeeds, and return empty string if fails. The
AJAX request must be a POST request with input data passed in for res_id and folder_path
where folder_path is the relative path to res_id/data/contents for the new folder to be
created under res_id collection/directory.
"""
res_id = request.POST.get('res_id', None)
if res_id is None:
return HttpResponse('Bad request - resource id is not included',
status=status.HTTP_400_BAD_REQUEST)
res_id = str(res_id).strip()
try:
resource, _, _ = authorize(request, res_id,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
except NotFound:
return HttpResponse('Bad request - resource not found', status=status.HTTP_400_BAD_REQUEST)
except PermissionDenied:
return HttpResponse('Permission denied', status=status.HTTP_401_UNAUTHORIZED)
folder_path = request.POST.get('folder_path', None)
try:
folder_path = _validate_path(folder_path, 'folder_path')
except ValidationError as ex:
return HttpResponse(ex.message, status=status.HTTP_400_BAD_REQUEST)
try:
create_folder(res_id, folder_path)
except SessionException as ex:
return HttpResponse(ex.stderr, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except DRF_ValidationError as ex:
return HttpResponse(ex.detail, status=status.HTTP_400_BAD_REQUEST)
return_object = {'new_folder_rel_path': folder_path}
return HttpResponse(
json.dumps(return_object),
content_type="application/json"
)
def put(self, request, pk, pathname):
""" create a given folder if not present and allowed
"""
try:
resource, authorized, user = view_utils.authorize(
request,
pk,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE,
raises_exception=False)
except NotFound as ex:
return Response(ex.message, status=status.HTTP_404_NOT_FOUND)
if not authorized:
return Response("Insufficient permission",
status=status.HTTP_401_UNAUTHORIZED)
if not resource.supports_folders:
return Response("Resource type does not support subfolders",
status=status.HTTP_403_FORBIDDEN)
try:
view_utils.irods_path_is_allowed(
pathname) # check for hacking attempts
except (ValidationError, SuspiciousFileOperation) as ex:
return Response(ex.message, status=status.HTTP_400_BAD_REQUEST)
# relativise the path
relpath = os.path.join('data', 'contents', pathname)
try:
view_utils.create_folder(pk, relpath)
except SessionException:
raise ValidationError("Cannot create folder")
return Response(data={
'resource_id': pk,
'path': pathname
},
status=status.HTTP_201_CREATED)
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg="resource does not contain the updated file new_" + file_name_list[2])
self.assertNotIn(file_name_list[2], updated_res_file_names,
msg='resource still contains the old file ' + file_name_list[2] +
' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] + ' moved to a folder')
self.assertNotIn(file_name_list[0], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[0] +
'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[1] +
' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(), 2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id, source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False)
# Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip,
# and file1.txt (2)
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " +
str(file_cnt) + " != 5")
# remove all files except the zippped file
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1')
# Now resource should contain two files: file3_new.txt sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir.zip', updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[0])
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[1])
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg='resource does not contain unzipped file new_' + file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def test_file_add_to_composite_resource(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
super(CompositeResourceTest, self).assert_federated_irods_available()
# test that when we add file to an existing composite resource, the added file
# automatically set to genericlogicalfile type
self.assertEqual(BaseResource.objects.count(), 0)
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test Composite Resource With Files Added From Federated Zone',
auto_aggregate=False
)
# there should not be any GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 0)
# add a file to the resource
fed_test_file_full_path = '/{zone}/home/{username}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, username=self.user.username,
fname=self.raster_file_name)
res_upload_files = []
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=None)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
auto_aggregate=False)
# there should be one resource at this point
self.assertEqual(BaseResource.objects.count(), 1)
self.assertEqual(self.composite_resource.resource_type, "CompositeResource")
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
# create the generic aggregation (logical file)
GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
# check that the resource file is associated with GenericLogicalFile
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.has_logical_file, True)
self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
# there should be 1 GenericLogicalFile object at this point
self.assertEqual(GenericLogicalFile.objects.count(), 1)
# test adding a file to a folder (Note the UI does not support uploading a iRODS file
# to a specific folder)
# create the folder
new_folder = "my-new-folder"
new_folder_path = os.path.join("data", "contents", new_folder)
create_folder(self.composite_resource.short_id, new_folder_path)
resource_file_add_pre_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder)
resource_file_add_process(resource=self.composite_resource, files=res_upload_files,
source_names=[fed_test_file_full_path], user=self.user,
folder=new_folder, auto_aggregate=False)
self.assertEqual(self.composite_resource.files.all().count(), 2)
self.composite_resource.delete()
def set_file_type(cls, resource, file_id, user):
"""
Sets a .shp or .zip resource file to GeoFeatureFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoFeatureFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None or not res_file.exists:
raise ValidationError("File not found.")
if res_file.extension.lower() not in ('.zip', '.shp'):
raise ValidationError("Not a valid geographic feature file.")
if not res_file.has_generic_logical_file:
raise ValidationError(
"Selected file must be part of a generic file type.")
try:
meta_dict, shape_files, shp_res_files = extract_metadata_and_files(
resource, res_file)
except ValidationError as ex:
log.exception(ex.message)
raise ex
# hold on to temp dir for final clean up
temp_dir = os.path.dirname(shape_files[0])
file_name = res_file.file_name
# file name without the extension
base_file_name = file_name[:-len(res_file.extension)]
xml_file = ''
for f in shape_files:
if f.lower().endswith('.shp.xml'):
xml_file = f
break
file_folder = res_file.file_folder
file_type_success = False
upload_folder = ''
msg = "GeoFeature file type. Error when setting file type. Error:{}"
with transaction.atomic():
# create a GeoFeature logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = base_file_name
logical_file.save()
try:
# create a folder for the geofeature file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, base_file_name)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(file_folder, new_folder_name)
# add all new files to the resource
files_to_add_to_resource = shape_files
for fl in files_to_add_to_resource:
uploaded_file = UploadedFile(file=open(fl, 'rb'),
name=os.path.basename(fl))
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# make each resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"GeoFeature file type - files were added to the file type."
)
add_metadata(resource, meta_dict, xml_file, logical_file)
log.info(
"GeoFeature file type and resource level metadata updated."
)
# delete the original resource files used as part of setting file type
for fl in shp_res_files:
delete_resource_file(resource.short_id, fl.id, user)
log.info("Deleted original resource files.")
file_type_success = True
except Exception as ex:
msg = msg.format(ex.message)
log.exception(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
if not file_type_success and upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join('data', 'contents', upload_folder)
remove_folder(user, resource.short_id, folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
# base file name (no path included)
file_name = utils.get_resource_file_name_and_extension(res_file)[1]
# file name without the extension
file_name = file_name[:-len(res_file.extension)]
file_folder = res_file.file_folder
upload_folder = ''
if res_file is not None and res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
error_info, files_to_add_to_resource = raster_file_validation(
raster_file=temp_file)
if not error_info:
log.info("Geo raster file type file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == os.path.splitext(f)[1]
].pop()
metadata = extract_metadata(temp_vrt_file_path)
log.info(
"Geo raster file type metadata extraction was successful.")
with transaction.atomic():
# create a geo raster logical file object to be associated with resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = file_name
logical_file.save()
try:
# create a folder for the raster file type using the base file name as the
# name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, file_name)
log.info("Folder created:{}".format(new_folder_path))
create_folder(resource.short_id, new_folder_path)
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
# the added resource file will be part of a new generic logical file
# by default
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# delete the generic logical file object
if new_res_file.logical_file is not None:
# deleting the file level metadata object will delete the associated
# logical file object
new_res_file.logical_file.metadata.delete()
# make each resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"Geo raster file type - new files were added to the resource."
)
# use the extracted metadata to populate file metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
logical_file.metadata.create_element(k, **v)
log.info(
"Geo raster file type - metadata was saved to DB")
# set resource to private if logical file is missing required metadata
resource.update_public_and_discoverable()
# delete the original resource file
delete_resource_file(resource.short_id, res_file.id,
user)
log.info("Deleted original resource file.")
except Exception as ex:
msg = "Geo raster file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
if upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join(
'data', 'contents', upload_folder)
remove_folder(user, resource.short_id,
folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
else:
err_msg = "Geo raster file type file validation failed.{}".format(
' '.join(error_info))
log.info(err_msg)
raise ValidationError(err_msg)
else:
if res_file is None:
err_msg = "Failed to set Geo raster file type. " \
"Resource doesn't have the specified file."
log.error(err_msg)
raise ValidationError(err_msg)
else:
err_msg = "Failed to set Geo raster file type." \
"The specified file doesn't have a generic logical file type."
log.error(err_msg)
raise ValidationError(err_msg)
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir',
store[0],
msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg="resource does not contain the updated file new_" +
file_name_list[2])
self.assertNotIn(file_name_list[2],
updated_res_file_names,
msg='resource still contains the old file ' +
file_name_list[2] + ' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' moved to a folder')
self.assertNotIn(file_name_list[0],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[0] + 'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[1] + ' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(),
2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
uname=user.username,
fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
with self.assertRaises(SessionException):
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip',
False)
# Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
3,
msg="resource file count didn't match - " +
str(file_cnt) + " != 3")
# test unzipping the file succeeds now after deleting the existing folder
# TODO: this causes a multiple delete because the paths are valid now.
istorage = res.get_irods_storage()
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
# Now resource should contain two files: file3_new.txt and sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
2,
msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn(
'sub_test_dir.zip',
updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[0])
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[1])
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg='resource does not contain unzipped file new_' +
file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(),
1,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None:
raise ValidationError("File not found.")
if res_file.extension != '.nc':
raise ValidationError("Not a NetCDF file.")
# base file name (no path included)
file_name = res_file.file_name
# file name without the extension
nc_file_name = file_name.split(".")[0]
resource_metadata = []
file_type_metadata = []
files_to_add_to_resource = []
if res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
files_to_add_to_resource.append(temp_file)
# file validation and metadata extraction
nc_dataset = nc_utils.get_nc_dataset(temp_file)
if isinstance(nc_dataset, netCDF4.Dataset):
# Extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(
temp_file)
# populate resource_metadata and file_type_metadata lists with extracted metadata
add_metadata_to_list(resource_metadata, res_dublin_core_meta,
res_type_specific_meta,
file_type_metadata, resource)
# create the ncdump text file
dump_file = create_header_info_txt_file(
temp_file, nc_file_name)
files_to_add_to_resource.append(dump_file)
file_folder = res_file.file_folder
with transaction.atomic():
# first delete the netcdf file that we retrieved from irods
# for setting it to netcdf file type
delete_resource_file(resource.short_id, res_file.id, user)
# create a netcdf logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type unless the extracted metadata
# has a value for title
dataset_title = res_dublin_core_meta.get('title', None)
if dataset_title is not None:
logical_file.dataset_name = dataset_title
else:
logical_file.dataset_name = nc_file_name
logical_file.save()
try:
# create a folder for the netcdf file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, nc_file_name)
fed_file_full_path = ''
if resource.resource_federation_path:
fed_file_full_path = os.path.join(
resource.root_path, new_folder_path)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
new_res_file = utils.add_file_to_resource(
resource,
uploaded_file,
folder=upload_folder,
fed_res_file_name_or_path=fed_file_full_path)
# make each resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"NetCDF file type - new files were added to the resource."
)
except Exception as ex:
msg = "NetCDF file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
# TODO: in case of any error put the original file back and
# delete the folder that was created
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
log.info("NetCDF file type was created.")
# use the extracted metadata to populate resource metadata
for element in resource_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element(
'title', title_element.id, **v)
else:
resource.metadata.create_element(k, **v)
log.info("Resource - metadata was saved to DB")
# use the extracted metadata to populate file metadata
for element in file_type_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'subject':
logical_file.metadata.keywords = v
logical_file.metadata.save()
else:
logical_file.metadata.create_element(k, **v)
log.info("NetCDF file type - metadata was saved to DB")
else:
err_msg = "Not a valid NetCDF file. File type file validation failed."
log.error(err_msg)
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
raise ValidationError(err_msg)
def test_aggregation_xml_file_paths(self):
# test the aggregation meta and map xml file paths with file name and folder name
# changes
self.create_composite_resource(self.refts_file)
# there should be one resource file
self.assertEqual(self.composite_resource.files.all().count(), 1)
res_file = self.composite_resource.files.first()
base_file_name, ext = os.path.splitext(res_file.file_name)
# check that the resource file is associated with GenericLogicalFile
self.assertEqual(res_file.has_logical_file, False)
# set file to generic logical file type
RefTimeseriesLogicalFile.set_file_type(self.composite_resource,
self.user, res_file.id)
res_file = self.composite_resource.files.first()
self.assertEqual(res_file.logical_file_type_name,
self.logical_file_type_name)
logical_file = res_file.logical_file
expected_meta_path = '{}_meta.xml'.format(res_file.file_name)
expected_map_path = '{}_resmap.xml'.format(res_file.file_name)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test xml file paths after moving the file into a folder
new_folder = 'test_folder'
create_folder(self.composite_resource.short_id,
'data/contents/{}'.format(new_folder))
src_path = 'data/contents/{}'.format(res_file.file_name)
tgt_path = 'data/contents/{0}/{1}'.format(new_folder,
res_file.file_name)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_meta_path = '{0}/{1}_meta.xml'.format(new_folder,
res_file.file_name)
expected_map_path = '{0}/{1}_resmap.xml'.format(
new_folder, res_file.file_name)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test xml file paths after renaming the file
src_path = 'data/contents/{0}/{1}'.format(new_folder,
res_file.file_name)
tgt_path = 'data/contents/{0}/{1}_1{2}'.format(new_folder,
base_file_name, ext)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_meta_path = '{0}/{1}_meta.xml'.format(new_folder,
res_file.file_name)
expected_map_path = '{0}/{1}_resmap.xml'.format(
new_folder, res_file.file_name)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
# test the xml file path after renaming the folder
folder_rename = '{}_1'.format(new_folder)
src_path = 'data/contents/{}'.format(new_folder)
tgt_path = 'data/contents/{}'.format(folder_rename)
move_or_rename_file_or_folder(self.user,
self.composite_resource.short_id,
src_path, tgt_path)
res_file = self.composite_resource.files.first()
logical_file = res_file.logical_file
expected_meta_path = '{0}/{1}_meta.xml'.format(folder_rename,
res_file.file_name)
expected_map_path = '{0}/{1}_resmap.xml'.format(
folder_rename, res_file.file_name)
self.assertEqual(logical_file.metadata_short_file_path,
expected_meta_path)
self.assertEqual(logical_file.map_short_file_path, expected_map_path)
self.composite_resource.delete()
