def link_irods_file_to_django(resource, filepath):
"""
Link a newly created irods file to Django resource model
:param filepath: full path to file
"""
# link the newly created file (**filepath**) to Django resource model
b_add_file = False
# TODO: folder is an abstract concept... utilize short_path for whole API
if resource:
folder, base = ResourceFile.resource_path_is_acceptable(resource, filepath,
test_exists=False)
try:
ResourceFile.get(resource=resource, file=base, folder=folder)
except ObjectDoesNotExist:
# this does not copy the file from anywhere; it must exist already
ResourceFile.create(resource=resource, file=base, folder=folder)
b_add_file = True
if b_add_file:
file_format_type = get_file_mime_type(filepath)
if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]:
resource.metadata.create_element('format', value=file_format_type)
# this should assign a logical file object to this new file
# if this resource supports logical file
resource.set_default_logical_file()
def copy_resource_files_and_AVUs(src_res_id, dest_res_id):
"""
Copy resource files and AVUs from source resource to target resource including both
on iRODS storage and on Django database
:param src_res_id: source resource uuid
:param dest_res_id: target resource uuid
:return:
"""
avu_list = ['bag_modified', 'metadata_dirty', 'isPublic', 'resourceType']
src_res = get_resource_by_shortkey(src_res_id)
tgt_res = get_resource_by_shortkey(dest_res_id)
# This makes the assumption that the destination is in the same exact zone.
# Also, bags and similar attached files are not copied.
istorage = src_res.get_irods_storage()
# This makes an exact copy of all physical files.
src_files = os.path.join(src_res.root_path, 'data')
# This has to be one segment short of the source because it is a target directory.
dest_files = tgt_res.root_path
istorage.copyFiles(src_files, dest_files)
src_coll = src_res.root_path
tgt_coll = tgt_res.root_path
for avu_name in avu_list:
value = istorage.getAVU(src_coll, avu_name)
# make formerly public things private
if avu_name == 'isPublic':
istorage.setAVU(tgt_coll, avu_name, 'false')
# bag_modified AVU needs to be set to true for copied resource
elif avu_name == 'bag_modified':
istorage.setAVU(tgt_coll, avu_name, 'true')
# everything else gets copied literally
else:
istorage.setAVU(tgt_coll, avu_name, value)
# link copied resource files to Django resource model
files = src_res.files.all()
# if resource files are part of logical files, then logical files also need copying
src_logical_files = list(
set([f.logical_file for f in files if f.has_logical_file]))
map_logical_files = {}
for src_logical_file in src_logical_files:
map_logical_files[src_logical_file] = src_logical_file.get_copy()
for n, f in enumerate(files):
folder, base = os.path.split(
f.short_path) # strips object information.
new_resource_file = ResourceFile.create(tgt_res, base, folder=folder)
# if the original file is part of a logical file, then
# add the corresponding new resource file to the copy of that logical file
if f.has_logical_file:
tgt_logical_file = map_logical_files[f.logical_file]
tgt_logical_file.add_resource_file(new_resource_file)
if src_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new collection resource will not contain "deleted resources"
tgt_res.resources = src_res.resources.all()
def add_file_to_resource(resource,
f,
folder=None,
source_name='',
source_size=0,
move=False,
is_file_reference=False):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is
set to 0 and useless.
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:param is_file_reference: indicate whether the file being added is a reference to an external
file stored in an external zone or URL. source_name will hold
the reference file path or url
:return: The identifier of the ResourceFile added.
"""
# importing here to avoid circular import
from hs_file_types.models import GenericLogicalFile
if f:
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None,
move=False)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name,
source_size=source_size,
is_file_reference=is_file_reference,
move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
# if a file gets added successfully to composite resource, then better to set the generic
# logical file here
if resource.resource_type == "CompositeResource":
logical_file = GenericLogicalFile.create()
ret.logical_file_content_object = logical_file
ret.save()
return ret
def add_file_to_resource(resource,
f,
folder=None,
source_name='',
check_target_folder=False,
add_to_aggregation=True):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param folder: folder at which the file will live
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param check_target_folder: if true and the resource is a composite resource then uploading
a file to the specified folder will be validated before adding the file to the resource
:param add_to_aggregation: if true and the resource is a composite resource then the file
being added to the resource also will be added to a fileset aggregation if such an aggregation
exists in the file path
:return: The identifier of the ResourceFile added.
"""
# validate parameters
if check_target_folder and resource.resource_type != 'CompositeResource':
raise ValidationError(
"Resource must be a CompositeResource for validating target folder"
)
if f:
if check_target_folder and folder is not None:
tgt_full_upload_path = os.path.join(resource.file_path, folder)
if not resource.can_add_files(
target_full_path=tgt_full_upload_path):
err_msg = "File can't be added to this folder which represents an aggregation"
raise ValidationError(err_msg)
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None)
if add_to_aggregation:
if folder is not None and resource.resource_type == 'CompositeResource':
aggregation = resource.get_fileset_aggregation_in_path(folder)
if aggregation is not None:
# make the added file part of the fileset aggregation
aggregation.add_resource_file(ret)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
ret.calculate_size()
return ret
def add_file_to_resource(resource, f, folder=None, source_name='',
move=False, check_target_folder=False, add_to_aggregation=True):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param folder: folder at which the file will live
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:param check_target_folder: if true and the resource is a composite resource then uploading
a file to the specified folder will be validated before adding the file to the resource
:param add_to_aggregation: if true and the resource is a composite resource then the file
being added to the resource also will be added to a fileset aggregation if such an aggregation
exists in the file path
:return: The identifier of the ResourceFile added.
"""
# validate parameters
if check_target_folder and resource.resource_type != 'CompositeResource':
raise ValidationError("Resource must be a CompositeResource for validating target folder")
if f:
if check_target_folder and folder is not None:
tgt_full_upload_path = os.path.join(resource.file_path, folder)
if not resource.can_add_files(target_full_path=tgt_full_upload_path):
err_msg = "File can't be added to this folder which represents an aggregation"
raise ValidationError(err_msg)
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource, openfile, folder=folder, source=None, move=False)
if add_to_aggregation:
if folder is not None and resource.resource_type == 'CompositeResource':
aggregation = resource.get_fileset_aggregation_in_path(folder)
if aggregation is not None:
# make the added file part of the fileset aggregation
aggregation.add_resource_file(ret)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource, None, folder=folder, source=source_name, move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError('Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]:
resource.metadata.create_element('format', value=file_format_type)
ret.calculate_size()
return ret
def copy_resource_files_and_AVUs(src_res_id, dest_res_id):
"""
Copy resource files and AVUs from source resource to target resource including both
on iRODS storage and on Django database
:param src_res_id: source resource uuid
:param dest_res_id: target resource uuid
:return:
"""
avu_list = ['bag_modified', 'metadata_dirty', 'isPublic', 'resourceType']
src_res = get_resource_by_shortkey(src_res_id)
tgt_res = get_resource_by_shortkey(dest_res_id)
# This makes the assumption that the destination is in the same exact zone.
# Also, bags and similar attached files are not copied.
istorage = src_res.get_irods_storage()
# This makes an exact copy of all physical files.
src_files = os.path.join(src_res.root_path, 'data')
# This has to be one segment short of the source because it is a target directory.
dest_files = tgt_res.root_path
istorage.copyFiles(src_files, dest_files)
src_coll = src_res.root_path
tgt_coll = tgt_res.root_path
for avu_name in avu_list:
value = istorage.getAVU(src_coll, avu_name)
# make formerly public things private
if avu_name == 'isPublic':
istorage.setAVU(tgt_coll, avu_name, 'false')
# bag_modified AVU needs to be set to true for copied resource
elif avu_name == 'bag_modified':
istorage.setAVU(tgt_coll, avu_name, 'true')
# everything else gets copied literally
else:
istorage.setAVU(tgt_coll, avu_name, value)
# link copied resource files to Django resource model
files = src_res.files.all()
# if resource files are part of logical files, then logical files also need copying
src_logical_files = list(set([f.logical_file for f in files if f.has_logical_file]))
map_logical_files = {}
for src_logical_file in src_logical_files:
map_logical_files[src_logical_file] = src_logical_file.get_copy(tgt_res)
for n, f in enumerate(files):
folder, base = os.path.split(f.short_path) # strips object information.
new_resource_file = ResourceFile.create(tgt_res, base, folder=folder)
# if the original file is part of a logical file, then
# add the corresponding new resource file to the copy of that logical file
if f.has_logical_file:
tgt_logical_file = map_logical_files[f.logical_file]
if f.logical_file.extra_data:
tgt_logical_file.extra_data = copy.deepcopy(f.logical_file.extra_data)
tgt_logical_file.save()
tgt_logical_file.add_resource_file(new_resource_file)
if src_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new collection resource will not contain "deleted resources"
tgt_res.resources = src_res.resources.all()
def add_file_to_resource(resource, f, folder=None, source_name='', move=False):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:return: The identifier of the ResourceFile added.
"""
if f:
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None,
move=False)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name,
move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
return ret
