def replicate_resource_bag_to_user_zone(user, res_id):
"""
Replicate resource bag to iRODS user zone
Args:
user: the requesting user
res_id: the resource id with its bag to be replicated to iRODS user zone
Returns:
None, but exceptions will be raised if there is an issue with iRODS operation
"""
# do on-demand bag creation
res = utils.get_resource_by_shortkey(res_id)
res_coll = res.root_path
istorage = res.get_irods_storage()
bag_modified_flag = True
# needs to check whether res_id collection exists before getting/setting AVU on it to
# accommodate the case where the very same resource gets deleted by another request when
# it is getting downloaded
if istorage.exists(res_coll):
bag_modified = istorage.getAVU(res_coll, 'bag_modified')
# make sure bag_modified_flag is set to False only if bag exists and bag_modified AVU
# is False; otherwise, bag_modified_flag will take the default True value so that the
# bag will be created or recreated
if bag_modified:
if bag_modified.lower() == "false":
bag_file_name = res_id + '.zip'
if res.resource_federation_path:
bag_full_path = os.path.join(res.resource_federation_path, 'bags',
bag_file_name)
else:
bag_full_path = os.path.join('bags', bag_file_name)
if istorage.exists(bag_full_path):
bag_modified_flag = False
if bag_modified_flag:
# import here to avoid circular import issue
from hs_core.tasks import create_bag_by_irods
status = create_bag_by_irods(res_id)
if not status:
# bag fails to be created successfully
raise SessionException(-1, '', 'The resource bag fails to be created '
'before bag replication')
# do replication of the resource bag to irods user zone
if not res.resource_federation_path:
istorage.set_fed_zone_session()
src_file = res.bag_path
tgt_file = '/{userzone}/home/{username}/{resid}.zip'.format(
userzone=settings.HS_USER_IRODS_ZONE, username=user.username, resid=res_id)
fsize = istorage.size(src_file)
utils.validate_user_quota(user, fsize)
istorage.copyFiles(src_file, tgt_file)
update_quota_usage(user=user)
else:
raise ValidationError("Resource {} does not exist in iRODS".format(res.short_id))
def create_irods_user_in_user_zone(self):
"""Create corresponding irods account in user zone."""
try:
exec_cmd = "{0} {1} {2}".format(settings.HS_USER_ZONE_PROXY_USER_CREATE_USER_CMD,
self.user.username, self.user.username)
output = run_ssh_command(host=settings.HS_USER_ZONE_HOST,
uname=settings.HS_USER_ZONE_PROXY_USER,
pwd=settings.HS_USER_ZONE_PROXY_USER_PWD,
exec_cmd=exec_cmd)
if output:
if 'ERROR:' in output.upper():
# irods account failed to create
self.assertRaises(SessionException(-1, output, output))
user_profile = UserProfile.objects.filter(user=self.user).first()
user_profile.create_irods_user_account = True
user_profile.save()
except Exception as ex:
self.assertRaises(SessionException(-1, ex.message, ex.message))
def delete_irods_user_in_user_zone(self):
"""Delete irods test user in user zone."""
try:
exec_cmd = "{0} {1}".format(settings.HS_USER_ZONE_PROXY_USER_DELETE_USER_CMD,
self.user.username)
output = run_ssh_command(host=settings.HS_USER_ZONE_HOST,
uname=settings.HS_USER_ZONE_PROXY_USER,
pwd=settings.HS_USER_ZONE_PROXY_USER_PWD,
exec_cmd=exec_cmd)
if output:
if 'ERROR:' in output.upper():
# there is an error from icommand run, report the error
self.assertRaises(SessionException(-1, output, output))
user_profile = UserProfile.objects.filter(user=self.user).first()
user_profile.create_irods_user_account = False
user_profile.save()
except Exception as ex:
# there is an error from icommand run, report the error
self.assertRaises(SessionException(-1, ex.message, ex.message))
def create_irods_user_in_user_zone(self):
"""Create corresponding irods account in user zone."""
try:
exec_cmd = "{0} {1} {2}".format(
settings.LINUX_ADMIN_USER_CREATE_USER_IN_USER_ZONE_CMD,
self.user.username, self.user.username)
output = run_ssh_command(
host=settings.HS_USER_ZONE_HOST,
uname=settings.LINUX_ADMIN_USER_FOR_HS_USER_ZONE,
pwd=settings.LINUX_ADMIN_USER_PWD_FOR_HS_USER_ZONE,
exec_cmd=exec_cmd)
for out_str in output:
if 'ERROR:' in out_str.upper():
# irods account failed to create
self.assertRaises(SessionException(-1, out_str, out_str))
user_profile = UserProfile.objects.filter(user=self.user).first()
user_profile.create_irods_user_account = True
user_profile.save()
except Exception as ex:
self.assertRaises(SessionException(-1, str(ex), str(ex)))
def create_bag_by_irods(resource_id, create_zip=True):
"""Create a resource bag on iRODS side by running the bagit rule and ibun zip.
This function runs as a celery task, invoked asynchronously so that it does not
block the main web thread when it creates bags for very large files which will take some time.
:param
resource_id: the resource uuid that is used to look for the resource to create the bag for.
:param create_zip: defaults to True, set to false to create bagit files without zipping
:return: bag_url if bag creation operation succeeds or
raise an exception if resource does not exist or any other issues that prevent bags from being created.
"""
res = utils.get_resource_by_shortkey(resource_id)
istorage = res.get_irods_storage()
bag_path = res.bag_path
metadata_dirty = res.getAVU('metadata_dirty')
metadata_dirty = metadata_dirty is None or metadata_dirty
# if metadata has been changed, then regenerate metadata xml files
if metadata_dirty:
create_bag_metadata_files(res)
bag_modified = res.getAVU("bag_modified")
bag_modified = bag_modified is None or bag_modified
if metadata_dirty or bag_modified:
create_bagit_files_by_irods(res, istorage)
res.setAVU("bag_modified", False)
if create_zip:
irods_bagit_input_path = res.get_irods_path(resource_id,
prepend_short_id=False)
# only proceed when the resource is not deleted potentially by another request
# when being downloaded
is_exist = istorage.exists(irods_bagit_input_path)
if is_exist:
try:
if istorage.exists(bag_path):
istorage.delete(bag_path)
istorage.zipup(irods_bagit_input_path, bag_path)
if res.raccess.published:
# compute checksum to meet DataONE distribution requirement
chksum = istorage.checksum(bag_path)
res.bag_checksum = chksum
return res.bag_url
except SessionException as ex:
raise SessionException(-1, '', ex.stderr)
else:
raise ObjectDoesNotExist(
'Resource {} does not exist.'.format(resource_id))
def add_file_to_resource(resource,
f,
folder=None,
fed_res_file_name_or_path='',
fed_copy_or_move=None):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param folder: name of the folder path to which the file needs be uploaded
:param fed_res_file_name_or_path: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but fed_res_file_name_or_path has the whole data object
iRODS path in the federated zone
:param fed_copy_or_move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of 'copy'
indicates copy is needed, a value of 'move' indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is None, which indicates N/A, or not applicable,
since the files do not come from a federated zone, and this copy or
move operation is not applicable, but any value other
than 'copy' or 'move' is regarded as N/A.
:return: The identifier of the ResourceFile added.
"""
if f:
file_format_type = get_file_mime_type(f.name)
if fed_res_file_name_or_path:
ret = ResourceFile.objects.create(
content_object=resource,
file_folder=folder,
resource_file=None,
fed_resource_file=File(f)
if not isinstance(f, UploadedFile) else f)
else:
ret = ResourceFile.objects.create(
content_object=resource,
file_folder=folder,
resource_file=File(f)
if not isinstance(f, UploadedFile) else f,
fed_resource_file=None)
elif fed_res_file_name_or_path and (fed_copy_or_move == 'copy'
or fed_copy_or_move == 'move'):
size = get_fed_zone_file_size(fed_res_file_name_or_path)
file_format_type = get_file_mime_type(fed_res_file_name_or_path)
ret = ResourceFile.objects.create(
content_object=resource,
resource_file=None,
fed_resource_file=None,
fed_resource_file_name_or_path=fed_res_file_name_or_path,
fed_resource_file_size=size)
try:
from_fname = fed_res_file_name_or_path
filename = from_fname.rsplit('/')[-1]
istorage = resource.get_irods_storage()
if resource.resource_federation_path:
if folder:
to_fname = os.path.join(resource.resource_federation_path,
resource.short_id, 'data',
'contents', folder, filename)
else:
to_fname = os.path.join(resource.resource_federation_path,
resource.short_id, 'data',
'contents', filename)
else:
if folder:
to_fname = os.path.join(resource.short_id, 'data',
'contents', folder, filename)
else:
to_fname = os.path.join(resource.short_id, 'data',
'contents', filename)
if fed_copy_or_move == 'copy':
istorage.copyFiles(from_fname, to_fname)
else:
istorage.moveFile(from_fname, to_fname)
# update file path now that file has been copied or moved to HydroShare
# proxy account space
ret.fed_resource_file_name_or_path = 'data/contents/{file_name}'.format(
file_name=filename)
ret.save()
except SessionException as ex:
# delete the file added if there is any exception
ret.delete()
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# add format metadata element if necessary
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
return ret
def add_file_to_resource(resource,
f,
folder=None,
source_name='',
source_size=0,
move=False,
is_file_reference=False):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is
set to 0 and useless.
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:param is_file_reference: indicate whether the file being added is a reference to an external
file stored in an external zone or URL. source_name will hold
the reference file path or url
:return: The identifier of the ResourceFile added.
"""
# importing here to avoid circular import
from hs_file_types.models import GenericLogicalFile
if f:
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None,
move=False)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name,
source_size=source_size,
is_file_reference=is_file_reference,
move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
# if a file gets added successfully to composite resource, then better to set the generic
# logical file here
if resource.resource_type == "CompositeResource":
logical_file = GenericLogicalFile.create()
ret.logical_file_content_object = logical_file
ret.save()
return ret
def add_file_to_resource(resource,
f,
folder=None,
source_name='',
check_target_folder=False,
add_to_aggregation=True):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param folder: folder at which the file will live
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param check_target_folder: if true and the resource is a composite resource then uploading
a file to the specified folder will be validated before adding the file to the resource
:param add_to_aggregation: if true and the resource is a composite resource then the file
being added to the resource also will be added to a fileset aggregation if such an aggregation
exists in the file path
:return: The identifier of the ResourceFile added.
"""
# validate parameters
if check_target_folder and resource.resource_type != 'CompositeResource':
raise ValidationError(
"Resource must be a CompositeResource for validating target folder"
)
if f:
if check_target_folder and folder is not None:
tgt_full_upload_path = os.path.join(resource.file_path, folder)
if not resource.can_add_files(
target_full_path=tgt_full_upload_path):
err_msg = "File can't be added to this folder which represents an aggregation"
raise ValidationError(err_msg)
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None)
if add_to_aggregation:
if folder is not None and resource.resource_type == 'CompositeResource':
aggregation = resource.get_fileset_aggregation_in_path(folder)
if aggregation is not None:
# make the added file part of the fileset aggregation
aggregation.add_resource_file(ret)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
ret.calculate_size()
return ret
def add_file_to_resource(resource, f, folder=None, source_name='', move=False):
"""
Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource.
:param resource: Resource to which file should be added
:param f: File-like object to add to a resource
:param source_name: the logical file name of the resource content file for
federated iRODS resource or the federated zone name;
By default, it is empty. A non-empty value indicates
the file needs to be added into the federated zone, either
from local disk where f holds the uploaded file from local
disk, or from the federated zone directly where f is empty
but source_name has the whole data object
iRODS path in the federated zone
:param move: indicate whether the file should be copied or moved from private user
account to proxy user account in federated zone; A value of False
indicates copy is needed, a value of True indicates no copy, but
the file will be moved from private user account to proxy user account.
The default value is False.
:return: The identifier of the ResourceFile added.
"""
if f:
openfile = File(f) if not isinstance(f, UploadedFile) else f
ret = ResourceFile.create(resource,
openfile,
folder=folder,
source=None,
move=False)
# add format metadata element if necessary
file_format_type = get_file_mime_type(f.name)
elif source_name:
try:
# create from existing iRODS file
ret = ResourceFile.create(resource,
None,
folder=folder,
source=source_name,
move=move)
except SessionException as ex:
try:
ret.delete()
except Exception:
pass
# raise the exception for the calling function to inform the error on the page interface
raise SessionException(ex.exitcode, ex.stdout, ex.stderr)
# add format metadata element if necessary
file_format_type = get_file_mime_type(source_name)
else:
raise ValueError(
'Invalid input parameter is passed into this add_file_to_resource() '
'function')
# TODO: generate this from data in ResourceFile rather than extension
if file_format_type not in [
mime.value for mime in resource.metadata.formats.all()
]:
resource.metadata.create_element('format', value=file_format_type)
return ret