def get(self, path):
query = self.sess.query(Collection).filter(Collection.name == path)
try:
result = query.one()
except NoResultFound:
raise CollectionDoesNotExist()
return iRODSCollection(self, result)
def get(self, ipath):
if ipath not in self.collections:
fname = os.path.join(self.session.collection_dir,
ipath.replace('/', '#') + ".json")
if os.path.isfile(fname):
with open(fname) as fp:
coll = iRodsCollection(self.session,
ipath,
json.load(fp))
self.collections[ipath] = coll
else:
raise CollectionDoesNotExist(ipath)
return self.collections[ipath]
def get(self, path):
path = iRODSCollection.normalize_path(path)
filters = [Collection.name == path]
# if a ticket is supplied for this session, try both without and with DataObject join
repeats = (True,False) if hasattr(self.sess,'ticket__') \
else (False,)
for rep in repeats:
query = self.sess.query(Collection).filter(*filters)
try:
result = query.one()
except NoResultFound:
if rep:
filters += [DataObject.id != 0]
continue
raise CollectionDoesNotExist()
return iRODSCollection(self, result)
def move(self, iterator, irods_path, clobber=True, interactive=False,
verbose=False):
""" Moving or renaming iRODS data objects and/or collections,
similar to the UNIX `mv` command.
Raises an CollectionDoesNotExist if the iterator corresponds to more
than one item and the irods_path destination does not correspond to an
existing collection.
Examples:
>>> session.bulk.move('tmpfiles*', '~/tmpdir/', verbose=True)
>>> session.bulk.move('./parent/dirname', './parent/dirname_new')
Arguments:
iterator: iterator or str
Defines which items are subject to the bulk operation.
Can be an iterator (e.g. using search_manager.find())
or a string (which will be used to construct a
search_manager.iglob() iterator). Matching data objects
and collections will be moved to the new path.
irods_path: str (default: '.')
The (absolute or relative) path on the local file system
where the data objects and collections will moved to.
clobber: bool (default: True)
Whether to overwrite existing data objects.
interactive: bool (default: False)
Whether to prompt for permission before overwriting
existing data objects. If True, the value of the 'clobber'
argument is ignored.
verbose: bool (default: False)
Whether to print more output.
"""
def move_one(src, dest):
# Move/renames a single item
src_abs = self.session.path.get_absolute_irods_path(src)
dest_abs = self.session.path.get_absolute_irods_path(dest)
src_is_collection = self.session.collections.exists(src_abs)
dest_is_object = self.session.data_objects.exists(dest_abs)
kind = 'collection' if src_is_collection else 'data object'
ok = True
if not clobber:
ok = not dest_is_object
if interactive:
ok = confirm('move', kind, src_abs + ' to ' + dest_abs)
if ok:
self.log('Moving %s %s to destination %s' % \
(kind, src_abs, dest_abs), verbose)
if src_is_collection:
if dest_is_object:
msg = 'Cannot overwrite obj %s with coll %s'
raise OperationNotSupported(msg % (dest_abs, src_abs))
else:
self.session.collections.move(src_abs, dest_abs)
else:
self.session.data_objects.move(src_abs, dest_abs)
else:
self.log('Skipped moving %s %s to destination %s' % \
(kind, src_abs, dest_abs), verbose)
if isinstance(iterator, str):
iterator = self.session.search.iglob(iterator)
try:
previous_item = next(iterator)
except StopIteration:
raise StopIteration('Iterator yields no objects or collections')
dest = self.session.path.get_absolute_irods_path(irods_path)
item = None
for item in iterator:
# There is more than one item, so irods_path needs
# to be an existing collection
if not self.session.collections.exists(dest):
raise CollectionDoesNotExist(dest)
move_one(previous_item, dest)
previous_item = item
if item is not None:
move_one(item, dest)
else:
# The iterator only held 1 item originally,
# and it hasn't been processed yet
move_one(previous_item, dest)
return
def put(self, iterator, irods_path='.', recurse=False, clobber=True,
interactive=False, verbose=False, create_options={}, **options):
""" Copy local files and/or folders to the iRODS server,
in a manner that resembles the UNIX 'cp' command.
Examples:
>>> session.bulk.put('tmpdir*', recurse=True)
>>> session.bulk.put('~/local_db/*.xyz', irods_path='./irods_db/')
Arguments:
iterator: iterator or str
Defines which items are subject to the bulk operation.
Can be an iterator (e.g. using search_manager.find())
or a string (which will be used to construct a
in search_manager.iglob() iterator). Matching files on the
local machine (and, if used recursively, directories) will
be copied to the iRODS server.
irods_path: str (default: '.')
The (absolute or relative) path on the iRODS file system
where the local files and folders will be copied to.
recurse: bool (default: False)
Whether to use recursion, meaning that also matching folders
and their files and subfolders will be copied to the iRODS server.
clobber: bool (default: True)
Whether to overwrite existing data objects.
interactive: bool (default: False)
Whether to prompt for permission before overwriting
existing data objects. If True, the value of the 'clobber'
argument is ignored.
verbose: bool (default: False)
Whether to print more output.
create_options: dict (default: {})
Additional options to be passed on to PRC's
collections.create() method.
options: (any remaining keywords arguments)
Additional options to be passed on to PRC's
data_objects.put() method.
"""
if type(iterator) is str:
iterator = glob.iglob(iterator)
dest = self.session.path.get_absolute_irods_path(irods_path)
if not self.session.collections.exists(dest):
raise CollectionDoesNotExist(dest)
for item in iterator:
local_path = item.rstrip('/')
path = os.path.join(dest, os.path.basename(local_path))
if os.path.isdir(local_path):
if recurse:
if not self.session.collections.exists(path):
self.log('Creating collection: %s' % path, verbose)
self.session.collections.create(path, recurse=True,
**create_options)
self.put(local_path + '/*', irods_path=path, recurse=True,
clobber=clobber, interactive=interactive,
create_options=create_options, verbose=verbose,
**options)
else:
self.log('Skipping collection %s (no recursion)' % \
local_path, verbose)
elif os.path.isfile(local_path):
object_exists = self.session.data_objects.exists(path)
ok = True
if not clobber:
ok = not object_exists
if interactive:
ok = confirm('put', 'file',
local_path +' in collection ' + dest)
if ok:
self.log('Putting file %s in collection %s' % \
(local_path, dest), verbose)
self.session.data_objects.put(local_path, dest + '/',
**options)
else:
self.log('Skipped putting file %s in collection %s' % \
(local_path, dest), verbose)
def pullFile(self, file_path, custom_name, user_name, galaxy_params):
"""
This function pulls a given file from a iRODS server
@param file_path, the path to the file in iRODS
@param custom_name, a custom name for the file in the Galaxy history
@param user_name, the name for the galaxy user
@param galaxy_params, different params extracted from Galaxy environment
@return True if finished successfully
@throws CollectionDoesNotExist if the file path does not exist
or if it is not readable for the current user
"""
#Step 1. Check if the origin directory exists and is valid for current user
file_path = file_path.split("/")
if len(file_path) < 2:
raise CollectionDoesNotExist("File path is not valid. The path " +
"/".join(file_path) +
" was not found in iRODS.")
fileName = file_path[-1]
file_path = "/".join(file_path[0:len(file_path) - 1]) + "/"
valid = self.checkDestinationPermissions(file_path, user_name,
fileName)
if valid == -1:
raise CollectionDoesNotExist("File path not valid. The path " +
file_path +
" does not exist in iRODS.")
elif valid == 1:
raise CollectionDoesNotExist(
"File path not valid. The directory " + file_path +
" in iRODS is not readable for current user.")
elif valid != 2:
raise CollectionDoesNotExist("Unable to find the file '" +
fileName + "' in directory " +
file_path + " in iRODS.")
print("Copying the file from iRODS...")
#Step 2. Copy the file content to a temporal file
obj = self.session.data_objects.get(file_path + fileName)
with obj.open('r+') as input:
with open(custom_name, "w") as output:
output.write(input.read()) #TODO: write to a temporal file?
output.close()
input.close()
print("Registering the file in Galaxy...")
file_content = {
"uuid": None,
"file_type": galaxy_params["file_type"],
"space_to_tab": False,
"dbkey": "?",
"to_posix_lines": True,
"ext": galaxy_params["file_type"],
"path": os.path.abspath(output.name),
"in_place": True,
"dataset_id": galaxy_params["job_id"],
"type": "file",
"is_binary": False, #TODO: GET FROM METADATA
"link_data_only": "copy_files",
"name": custom_name
}
with open("temporal.json", "w") as fileParams:
fileParams.write(json.dumps(file_content))
fileParams.close()
#Step 3. Call to Galaxy's upload tool
command = [
"python",
galaxy_params["GALAXY_ROOT_DIR"] + "/tools/data_source/upload.py",
galaxy_params["GALAXY_ROOT_DIR"],
galaxy_params["GALAXY_DATATYPES_CONF_FILE"],
os.path.abspath(fileParams.name), galaxy_params["job_id"] + ":" +
galaxy_params["output_dir"] + ":" + galaxy_params["output_file"]
]
# TODO: Check return code from call to subprocess
subprocess.call(command)
return True
def pushFile(self, origin_file, destination_dir, file_name, overwrite,
metadata):
"""
This function push a given file to the iRODS server
@param origin_file, the dataset to save
@param destination_dir, path to destination dir in iRODS
@param file_name, the name for the file in Galaxy
@param overwrite, if true existing files with same name will be replaced
@param metadata, the metadata for the file (including provenance)
@return True if finished successfully
@throws IOError if the file does not exist in the Galaxy server
@throws CollectionDoesNotExist if the destination directory does not exist
or if it is not writable for the current user
"""
#Step 1. Check if the origin file is valid
import os.path
if not os.path.isfile(origin_file):
raise IOError("File not found exception: file " + origin_file +
" was not found in server.")
#Step 2. Check if the destination directory exists and is valid for current user
valid = self.checkDestinationPermissions(destination_dir,
metadata["user_name"],
file_name)
if valid == -1:
raise CollectionDoesNotExist(
"Destination directory not valid. The path " +
destination_dir + " was not found in iRODS.")
if valid == 1:
raise CollectionDoesNotExist(
"Destination directory not valid. The directory " +
destination_dir +
" in iRODS is not writable for current user.")
#Step 3. If the file already exists and the overwrite option is false, get the new name
if overwrite == False and valid == 2:
aux_fileName = file_name
i = 0
while valid == 2:
print("File " + aux_fileName +
" already exits. Generating new name.")
aux_fileName = file_name.split(".")[0]
try:
extension = file_name.split(".")[1]
except Exception:
extension = ""
i += 1
aux_fileName = aux_fileName + "_" + str(i) + "." + extension
print("New name is " + aux_fileName)
valid = self.checkDestinationPermissions(
destination_dir, metadata["user_name"], aux_fileName)
file_name = aux_fileName
#Step 4. Send the file to iRODS
self.copyFileToIRODS(destination_dir, origin_file, file_name, metadata)
#Step 5. Set the metadata for the file
self.setFileMetadata(destination_dir, file_name, metadata)
return True
def get_collection_tree(self, base, path, sessions_cleanup=True):
"""
Lists the folders and files attributes at the input 'path'
Parameters
----------
base : str
The base path to validate ; eg. P000000001/C000000001
path : str
The collection's id; eg. P000000001/C000000001/SubFolder1/Experiment1/
sessions_cleanup: bool
If true, the session will be closed after retrieving the values.
Returns
-------
dict
The folders and files attributes at the requested path
"""
output = []
base_path = "/nlmumc/projects/" + base
absolute_path = "/nlmumc/projects/" + path
if not is_safe_path(base_path, absolute_path):
raise CAT_NO_ACCESS_PERMISSION
collection = self.session.collections.get(absolute_path)
for coll in collection.subcollections:
# query extra collection info: ctime
query = self.session.query(iRODSCollection).filter(
iRODSCollection.id == coll.id)
try:
result = query.one()
except NoResultFound:
raise CollectionDoesNotExist()
name = irods_basename(result[iRODSCollection.name])
ctime = result[iRODSCollection.create_time]
relative_path = path + "/" + name
folder_node = {
"name": name,
"path": relative_path,
"type": "folder",
"size": "--",
"rescname": "--",
"ctime": ctime.strftime("%Y-%m-%d %H:%M:%S"),
}
output.append(folder_node)
for data in collection.data_objects:
# query extra data info: ctime
query = self.session.query(DataObject).filter(
DataObject.id == data.id)
try:
result = query.first()
except NoResultFound:
raise DataObjectDoesNotExist()
ctime = result[DataObject.create_time]
relative_path = path + "/" + data.name
data_node = {
"name": data.name,
"path": relative_path,
"type": "file",
"size": data.size,
"rescname": data.resource_name,
"offlineResource": data.resource_name == "arcRescSURF01",
"ctime": ctime.strftime("%Y-%m-%d %H:%M:%S"),
}
output.append(data_node)
if sessions_cleanup:
self.session.cleanup()
return output