def link_irods_file_to_django(resource, filepath): """ Link a newly created irods file to Django resource model :param filepath: full path to file """ # link the newly created file (**filepath**) to Django resource model b_add_file = False # TODO: folder is an abstract concept... utilize short_path for whole API if resource: folder, base = ResourceFile.resource_path_is_acceptable(resource, filepath, test_exists=False) try: ResourceFile.get(resource=resource, file=base, folder=folder) except ObjectDoesNotExist: # this does not copy the file from anywhere; it must exist already ResourceFile.create(resource=resource, file=base, folder=folder) b_add_file = True if b_add_file: file_format_type = get_file_mime_type(filepath) if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]: resource.metadata.create_element('format', value=file_format_type) # this should assign a logical file object to this new file # if this resource supports logical file resource.set_default_logical_file()
def copy_resource_files_and_AVUs(src_res_id, dest_res_id): """ Copy resource files and AVUs from source resource to target resource including both on iRODS storage and on Django database :param src_res_id: source resource uuid :param dest_res_id: target resource uuid :return: """ avu_list = ['bag_modified', 'metadata_dirty', 'isPublic', 'resourceType'] src_res = get_resource_by_shortkey(src_res_id) tgt_res = get_resource_by_shortkey(dest_res_id) # This makes the assumption that the destination is in the same exact zone. # Also, bags and similar attached files are not copied. istorage = src_res.get_irods_storage() # This makes an exact copy of all physical files. src_files = os.path.join(src_res.root_path, 'data') # This has to be one segment short of the source because it is a target directory. dest_files = tgt_res.root_path istorage.copyFiles(src_files, dest_files) src_coll = src_res.root_path tgt_coll = tgt_res.root_path for avu_name in avu_list: value = istorage.getAVU(src_coll, avu_name) # make formerly public things private if avu_name == 'isPublic': istorage.setAVU(tgt_coll, avu_name, 'false') # bag_modified AVU needs to be set to true for copied resource elif avu_name == 'bag_modified': istorage.setAVU(tgt_coll, avu_name, 'true') # everything else gets copied literally else: istorage.setAVU(tgt_coll, avu_name, value) # link copied resource files to Django resource model files = src_res.files.all() # if resource files are part of logical files, then logical files also need copying src_logical_files = list( set([f.logical_file for f in files if f.has_logical_file])) map_logical_files = {} for src_logical_file in src_logical_files: map_logical_files[src_logical_file] = src_logical_file.get_copy() for n, f in enumerate(files): folder, base = os.path.split( f.short_path) # strips object information. new_resource_file = ResourceFile.create(tgt_res, base, folder=folder) # if the original file is part of a logical file, then # add the corresponding new resource file to the copy of that logical file if f.has_logical_file: tgt_logical_file = map_logical_files[f.logical_file] tgt_logical_file.add_resource_file(new_resource_file) if src_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new collection resource will not contain "deleted resources" tgt_res.resources = src_res.resources.all()
def add_file_to_resource(resource, f, folder=None, source_name='', source_size=0, move=False, is_file_reference=False): """ Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource. :param resource: Resource to which file should be added :param f: File-like object to add to a resource :param source_name: the logical file name of the resource content file for federated iRODS resource or the federated zone name; By default, it is empty. A non-empty value indicates the file needs to be added into the federated zone, either from local disk where f holds the uploaded file from local disk, or from the federated zone directly where f is empty but source_name has the whole data object iRODS path in the federated zone :param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is set to 0 and useless. :param move: indicate whether the file should be copied or moved from private user account to proxy user account in federated zone; A value of False indicates copy is needed, a value of True indicates no copy, but the file will be moved from private user account to proxy user account. The default value is False. :param is_file_reference: indicate whether the file being added is a reference to an external file stored in an external zone or URL. source_name will hold the reference file path or url :return: The identifier of the ResourceFile added. """ # importing here to avoid circular import from hs_file_types.models import GenericLogicalFile if f: openfile = File(f) if not isinstance(f, UploadedFile) else f ret = ResourceFile.create(resource, openfile, folder=folder, source=None, move=False) # add format metadata element if necessary file_format_type = get_file_mime_type(f.name) elif source_name: try: # create from existing iRODS file ret = ResourceFile.create(resource, None, folder=folder, source=source_name, source_size=source_size, is_file_reference=is_file_reference, move=move) except SessionException as ex: try: ret.delete() except Exception: pass # raise the exception for the calling function to inform the error on the page interface raise SessionException(ex.exitcode, ex.stdout, ex.stderr) # add format metadata element if necessary file_format_type = get_file_mime_type(source_name) else: raise ValueError( 'Invalid input parameter is passed into this add_file_to_resource() ' 'function') # TODO: generate this from data in ResourceFile rather than extension if file_format_type not in [ mime.value for mime in resource.metadata.formats.all() ]: resource.metadata.create_element('format', value=file_format_type) # if a file gets added successfully to composite resource, then better to set the generic # logical file here if resource.resource_type == "CompositeResource": logical_file = GenericLogicalFile.create() ret.logical_file_content_object = logical_file ret.save() return ret
def add_file_to_resource(resource, f, folder=None, source_name='', check_target_folder=False, add_to_aggregation=True): """ Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource. :param resource: Resource to which file should be added :param f: File-like object to add to a resource :param folder: folder at which the file will live :param source_name: the logical file name of the resource content file for federated iRODS resource or the federated zone name; By default, it is empty. A non-empty value indicates the file needs to be added into the federated zone, either from local disk where f holds the uploaded file from local disk, or from the federated zone directly where f is empty but source_name has the whole data object iRODS path in the federated zone :param check_target_folder: if true and the resource is a composite resource then uploading a file to the specified folder will be validated before adding the file to the resource :param add_to_aggregation: if true and the resource is a composite resource then the file being added to the resource also will be added to a fileset aggregation if such an aggregation exists in the file path :return: The identifier of the ResourceFile added. """ # validate parameters if check_target_folder and resource.resource_type != 'CompositeResource': raise ValidationError( "Resource must be a CompositeResource for validating target folder" ) if f: if check_target_folder and folder is not None: tgt_full_upload_path = os.path.join(resource.file_path, folder) if not resource.can_add_files( target_full_path=tgt_full_upload_path): err_msg = "File can't be added to this folder which represents an aggregation" raise ValidationError(err_msg) openfile = File(f) if not isinstance(f, UploadedFile) else f ret = ResourceFile.create(resource, openfile, folder=folder, source=None) if add_to_aggregation: if folder is not None and resource.resource_type == 'CompositeResource': aggregation = resource.get_fileset_aggregation_in_path(folder) if aggregation is not None: # make the added file part of the fileset aggregation aggregation.add_resource_file(ret) # add format metadata element if necessary file_format_type = get_file_mime_type(f.name) elif source_name: try: # create from existing iRODS file ret = ResourceFile.create(resource, None, folder=folder, source=source_name) except SessionException as ex: try: ret.delete() except Exception: pass # raise the exception for the calling function to inform the error on the page interface raise SessionException(ex.exitcode, ex.stdout, ex.stderr) # add format metadata element if necessary file_format_type = get_file_mime_type(source_name) else: raise ValueError( 'Invalid input parameter is passed into this add_file_to_resource() ' 'function') # TODO: generate this from data in ResourceFile rather than extension if file_format_type not in [ mime.value for mime in resource.metadata.formats.all() ]: resource.metadata.create_element('format', value=file_format_type) ret.calculate_size() return ret
def add_file_to_resource(resource, f, folder=None, source_name='', move=False, check_target_folder=False, add_to_aggregation=True): """ Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource. :param resource: Resource to which file should be added :param f: File-like object to add to a resource :param folder: folder at which the file will live :param source_name: the logical file name of the resource content file for federated iRODS resource or the federated zone name; By default, it is empty. A non-empty value indicates the file needs to be added into the federated zone, either from local disk where f holds the uploaded file from local disk, or from the federated zone directly where f is empty but source_name has the whole data object iRODS path in the federated zone :param move: indicate whether the file should be copied or moved from private user account to proxy user account in federated zone; A value of False indicates copy is needed, a value of True indicates no copy, but the file will be moved from private user account to proxy user account. The default value is False. :param check_target_folder: if true and the resource is a composite resource then uploading a file to the specified folder will be validated before adding the file to the resource :param add_to_aggregation: if true and the resource is a composite resource then the file being added to the resource also will be added to a fileset aggregation if such an aggregation exists in the file path :return: The identifier of the ResourceFile added. """ # validate parameters if check_target_folder and resource.resource_type != 'CompositeResource': raise ValidationError("Resource must be a CompositeResource for validating target folder") if f: if check_target_folder and folder is not None: tgt_full_upload_path = os.path.join(resource.file_path, folder) if not resource.can_add_files(target_full_path=tgt_full_upload_path): err_msg = "File can't be added to this folder which represents an aggregation" raise ValidationError(err_msg) openfile = File(f) if not isinstance(f, UploadedFile) else f ret = ResourceFile.create(resource, openfile, folder=folder, source=None, move=False) if add_to_aggregation: if folder is not None and resource.resource_type == 'CompositeResource': aggregation = resource.get_fileset_aggregation_in_path(folder) if aggregation is not None: # make the added file part of the fileset aggregation aggregation.add_resource_file(ret) # add format metadata element if necessary file_format_type = get_file_mime_type(f.name) elif source_name: try: # create from existing iRODS file ret = ResourceFile.create(resource, None, folder=folder, source=source_name, move=move) except SessionException as ex: try: ret.delete() except Exception: pass # raise the exception for the calling function to inform the error on the page interface raise SessionException(ex.exitcode, ex.stdout, ex.stderr) # add format metadata element if necessary file_format_type = get_file_mime_type(source_name) else: raise ValueError('Invalid input parameter is passed into this add_file_to_resource() ' 'function') # TODO: generate this from data in ResourceFile rather than extension if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]: resource.metadata.create_element('format', value=file_format_type) ret.calculate_size() return ret
def copy_resource_files_and_AVUs(src_res_id, dest_res_id): """ Copy resource files and AVUs from source resource to target resource including both on iRODS storage and on Django database :param src_res_id: source resource uuid :param dest_res_id: target resource uuid :return: """ avu_list = ['bag_modified', 'metadata_dirty', 'isPublic', 'resourceType'] src_res = get_resource_by_shortkey(src_res_id) tgt_res = get_resource_by_shortkey(dest_res_id) # This makes the assumption that the destination is in the same exact zone. # Also, bags and similar attached files are not copied. istorage = src_res.get_irods_storage() # This makes an exact copy of all physical files. src_files = os.path.join(src_res.root_path, 'data') # This has to be one segment short of the source because it is a target directory. dest_files = tgt_res.root_path istorage.copyFiles(src_files, dest_files) src_coll = src_res.root_path tgt_coll = tgt_res.root_path for avu_name in avu_list: value = istorage.getAVU(src_coll, avu_name) # make formerly public things private if avu_name == 'isPublic': istorage.setAVU(tgt_coll, avu_name, 'false') # bag_modified AVU needs to be set to true for copied resource elif avu_name == 'bag_modified': istorage.setAVU(tgt_coll, avu_name, 'true') # everything else gets copied literally else: istorage.setAVU(tgt_coll, avu_name, value) # link copied resource files to Django resource model files = src_res.files.all() # if resource files are part of logical files, then logical files also need copying src_logical_files = list(set([f.logical_file for f in files if f.has_logical_file])) map_logical_files = {} for src_logical_file in src_logical_files: map_logical_files[src_logical_file] = src_logical_file.get_copy(tgt_res) for n, f in enumerate(files): folder, base = os.path.split(f.short_path) # strips object information. new_resource_file = ResourceFile.create(tgt_res, base, folder=folder) # if the original file is part of a logical file, then # add the corresponding new resource file to the copy of that logical file if f.has_logical_file: tgt_logical_file = map_logical_files[f.logical_file] if f.logical_file.extra_data: tgt_logical_file.extra_data = copy.deepcopy(f.logical_file.extra_data) tgt_logical_file.save() tgt_logical_file.add_resource_file(new_resource_file) if src_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new collection resource will not contain "deleted resources" tgt_res.resources = src_res.resources.all()
def add_file_to_resource(resource, f, folder=None, source_name='', move=False): """ Add a ResourceFile to a Resource. Adds the 'format' metadata element to the resource. :param resource: Resource to which file should be added :param f: File-like object to add to a resource :param source_name: the logical file name of the resource content file for federated iRODS resource or the federated zone name; By default, it is empty. A non-empty value indicates the file needs to be added into the federated zone, either from local disk where f holds the uploaded file from local disk, or from the federated zone directly where f is empty but source_name has the whole data object iRODS path in the federated zone :param move: indicate whether the file should be copied or moved from private user account to proxy user account in federated zone; A value of False indicates copy is needed, a value of True indicates no copy, but the file will be moved from private user account to proxy user account. The default value is False. :return: The identifier of the ResourceFile added. """ if f: openfile = File(f) if not isinstance(f, UploadedFile) else f ret = ResourceFile.create(resource, openfile, folder=folder, source=None, move=False) # add format metadata element if necessary file_format_type = get_file_mime_type(f.name) elif source_name: try: # create from existing iRODS file ret = ResourceFile.create(resource, None, folder=folder, source=source_name, move=move) except SessionException as ex: try: ret.delete() except Exception: pass # raise the exception for the calling function to inform the error on the page interface raise SessionException(ex.exitcode, ex.stdout, ex.stderr) # add format metadata element if necessary file_format_type = get_file_mime_type(source_name) else: raise ValueError( 'Invalid input parameter is passed into this add_file_to_resource() ' 'function') # TODO: generate this from data in ResourceFile rather than extension if file_format_type not in [ mime.value for mime in resource.metadata.formats.all() ]: resource.metadata.create_element('format', value=file_format_type) return ret