def download_files(self, irods_files, targPath, overwrite=False):
"""Download a list of files from irods to a local directory
irods_files : list of tuples (collection, file, resource)
targPath : local path
If some files already exist in the local directory and if the overwrite
option is not set then a structure is returned, the caller will then
have to define a strategy."""
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.openFlags = irods.O_RDONLY
if overwrite:
irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
already_there = []
for (collection, irods_file, resource) in irods_files:
if resource:
irods.addKeyVal(dataObjOprInp.condInput,
irods.RESC_NAME_KW,
resource)
dataObjOprInp.objPath = collection + '/' + irods_file
status = irods.rcDataObjGet(self.conn,
dataObjOprInp,
targPath + os.sep + irods_file)
# If the file is already present we catch the error
if status == irods.OVERWRITE_WITHOUT_FORCE_FLAG:
already_there.append((collection, irods_file, resource))
return already_there
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__, obj.id)
# For finding all places where get_filename is called...
#log.debug( ''.join( traceback.format_stack() ) )
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if 'dir_only' in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(
os.path.dirname(cached_path),
"__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug('get_filename(): caching %s to %s', doi.objPath,
incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, 'get_filename(): iget %s failed (%s): %s' % (
doi.objPath, status, irods.strerror(status))
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug('get_filename(): cached %s to %s', doi.objPath,
cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug("get_filename(): waiting on incoming '%s' for %s %s",
incoming_path, obj.__class__.__name__, obj.id)
return os.path.abspath(cached_path)
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__,
obj.id,
)
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if "dir_only" in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug("get_filename(): caching %s to %s", doi.objPath, incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, "get_filename(): iget %s failed (%s): %s" % (
doi.objPath,
status,
irods.strerror(status),
)
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug("get_filename(): cached %s to %s", doi.objPath, cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug(
"get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id
)
return os.path.abspath(cached_path)
def download_file(self, irods_path, local_path, resource=None,
overwrite=False):
"""Download a file from irods to a local directory
- irods_files : list of tuples (collection, file, resource)
- local path
- resource
- overwrite the exisiting file if it exists"""
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.openFlags = irods.O_RDONLY
if overwrite:
irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
if resource:
irods.addKeyVal(dataObjOprInp.condInput, irods.RESC_NAME_KW,
resource)
dataObjOprInp.objPath = irods_path
status = irods.rcDataObjGet(self.conn, dataObjOprInp, local_path)
return status == irods.OVERWRITE_WITHOUT_FORCE_FLAG
