def update_from_file(self, obj, file_name=None, create=False, **kwargs):
assert "dir_only" not in kwargs, "update_from_file(): `dir_only` parameter is invalid here"
# do not create if not requested
if create and not self.exists(obj, **kwargs):
raise ObjectNotFound()
if file_name is None:
file_name = self.__get_cache_path(obj, **kwargs)
# put will create if necessary
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.createMode = 0640
doi.dataSize = os.stat(file_name).st_size
doi.numThreads = 0
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource)
irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, "")
# TODO: might want to VERIFY_CHKSUM_KW
log.debug("update_from_file(): updating %s to %s", file_name, doi.objPath)
# do the iput
status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
assert status == 0, "update_from_file(): iput %s failed (%s): %s" % (
doi.objPath,
status,
irods.strerror(status),
)
def update_from_file(self, obj, file_name=None, create=False, **kwargs):
assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here'
# do not create if not requested
if create and not self.exists(obj, **kwargs):
raise ObjectNotFound()
if file_name is None:
file_name = self.__get_cache_path(obj, **kwargs)
# put will create if necessary
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.createMode = 0o640
doi.dataSize = os.stat(file_name).st_size
doi.numThreads = 0
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW,
self.default_resource)
irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, '')
# TODO: might want to VERIFY_CHKSUM_KW
log.debug('update_from_file(): updating %s to %s', file_name,
doi.objPath)
# do the iput
status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % (
doi.objPath, status, irods.strerror(status))
def download_files(self, irods_files, targPath, overwrite=False):
"""Download a list of files from irods to a local directory
irods_files : list of tuples (collection, file, resource)
targPath : local path
If some files already exist in the local directory and if the overwrite
option is not set then a structure is returned, the caller will then
have to define a strategy."""
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.openFlags = irods.O_RDONLY
if overwrite:
irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
already_there = []
for (collection, irods_file, resource) in irods_files:
if resource:
irods.addKeyVal(dataObjOprInp.condInput,
irods.RESC_NAME_KW,
resource)
dataObjOprInp.objPath = collection + '/' + irods_file
status = irods.rcDataObjGet(self.conn,
dataObjOprInp,
targPath + os.sep + irods_file)
# If the file is already present we catch the error
if status == irods.OVERWRITE_WITHOUT_FORCE_FLAG:
already_there.append((collection, irods_file, resource))
return already_there
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__, obj.id)
# For finding all places where get_filename is called...
#log.debug( ''.join( traceback.format_stack() ) )
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if 'dir_only' in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(
os.path.dirname(cached_path),
"__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug('get_filename(): caching %s to %s', doi.objPath,
incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, 'get_filename(): iget %s failed (%s): %s' % (
doi.objPath, status, irods.strerror(status))
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug('get_filename(): cached %s to %s', doi.objPath,
cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug("get_filename(): waiting on incoming '%s' for %s %s",
incoming_path, obj.__class__.__name__, obj.id)
return os.path.abspath(cached_path)
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__,
obj.id,
)
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if "dir_only" in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug("get_filename(): caching %s to %s", doi.objPath, incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, "get_filename(): iget %s failed (%s): %s" % (
doi.objPath,
status,
irods.strerror(status),
)
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug("get_filename(): cached %s to %s", doi.objPath, cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug(
"get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id
)
return os.path.abspath(cached_path)
def replicate_file(self, file_info, dst_resource):
"""Replicate a file in irods
file_info : (collection, file, resource)
dst_resource : str"""
(collection, irods_file, src_resource) = file_info
dataObjInp = irods.dataObjInp_t()
d = irods.getFileInfo(self.conn, collection, irods_file, src_resource)
cond_inp = dataObjInp.condInput
irods.addKeyVal(cond_inp, irods.REPL_NUM_KW, d['data_repl_num'])
irods.addKeyVal(cond_inp, irods.RESC_NAME_KW, src_resource)
irods.addKeyVal(cond_inp, irods.DEST_RESC_NAME_KW, dst_resource)
dataObjInp.objPath = collection + '/' + irods_file
status = irods.rcDataObjRepl(self.conn, dataObjInp)
return status
def delete_file(self, path, resc=None):
"""Delete a file in iRODS
path : irods path (str)
resc : Resource"""
dataObjInp = irods.dataObjInp_t()
dataObjInp.openFlags = irods.O_RDONLY
if resc:
d = irods.getFileInfo(path, resc)
irods.addKeyVal(dataObjInp.condInput,
irods.REPL_NUM_KW,
d['data_repl_num'])
dataObjInp.objPath = path
status = irods.rcDataObjUnlink(self.conn, dataObjInp)
return status
def delete_files(self, irods_files):
"""Delete a list of files from iRODS
irods_files : list of tuples (collection, file, resource)"""
dataObjInp = irods.dataObjInp_t()
dataObjInp.openFlags = irods.O_RDONLY
st = []
for (collection, irods_file, resource) in irods_files:
d = irods.getFileInfo(self.conn, collection, irods_file, resource)
irods.addKeyVal(dataObjInp.condInput,
irods.REPL_NUM_KW,
d['data_repl_num'])
dataObjInp.objPath = collection + '/' + irods_file
status = irods.rcDataObjUnlink(self.conn, dataObjInp)
st.append(status)
return st
def upload_file(self, local_path, irods_path, resource=None,
overwrite=False):
"""Upload a file to irods
return True if the file is already present"""
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.oprType = irods.PUT_OPR
dataObjOprInp.openFlags = irods.O_RDWR
if overwrite:
irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
if resource:
irods.addKeyVal(dataObjOprInp.condInput,
irods.DEST_RESC_NAME_KW,
resource)
dataObjOprInp.objPath = irods_path
status = irods.rcDataObjPut(self.conn, dataObjOprInp, local_path)
return status == irods.OVERWRITE_WITHOUT_FORCE_FLAG
def download_file(self, irods_path, local_path, resource=None,
overwrite=False):
"""Download a file from irods to a local directory
- irods_files : list of tuples (collection, file, resource)
- local path
- resource
- overwrite the exisiting file if it exists"""
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.openFlags = irods.O_RDONLY
if overwrite:
irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
if resource:
irods.addKeyVal(dataObjOprInp.condInput, irods.RESC_NAME_KW,
resource)
dataObjOprInp.objPath = irods_path
status = irods.rcDataObjGet(self.conn, dataObjOprInp, local_path)
return status == irods.OVERWRITE_WITHOUT_FORCE_FLAG
def create(self, obj, **kwargs):
if not self.exists( obj, **kwargs ):
rods_path = self.__get_rods_path( obj, **kwargs )
log.debug( 'create(): %s', rods_path )
dir_only = kwargs.get( 'dir_only', False )
# short circuit collection creation since most of the time it will
# be the root collection which already exists
collection_path = rods_path if dir_only else path_dirname( rods_path )
if collection_path != self.root_collection_path:
self.__mkcolls( collection_path )
if not dir_only:
# rcDataObjCreate is used instead of the irodsOpen wrapper so
# that we can prevent overwriting
doi = irods.dataObjInp_t()
doi.objPath = rods_path
doi.createMode = 0o640
doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable
irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource )
status = irods.rcDataObjCreate( self.rods_conn, doi )
assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) )
def create(self, obj, **kwargs):
if not self.exists( obj, **kwargs ):
rods_path = self.__get_rods_path( obj, **kwargs )
log.debug( 'create(): %s', rods_path )
dir_only = kwargs.get( 'dir_only', False )
# short circuit collection creation since most of the time it will
# be the root collection which already exists
collection_path = rods_path if dir_only else path_dirname( rods_path )
if collection_path != self.root_collection_path:
self.__mkcolls( collection_path )
if not dir_only:
# rcDataObjCreate is used instead of the irodsOpen wrapper so
# that we can prevent overwriting
doi = irods.dataObjInp_t()
doi.objPath = rods_path
doi.createMode = 0o640
doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable
irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource )
status = irods.rcDataObjCreate( self.rods_conn, doi )
assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) )
def close(self):
for (k, c) in self.cxns.iteritems():
c.close()
# upload to irods
srcPath = self.cxnFiles[c]
(dirn, fn) = os.path.split(srcPath)
# use file/C api
dataObjOprInp = irods.dataObjInp_t()
dataObjOprInp.setOprType(irods.PUT_OPR)
dataObjOprInp.setOpenFlags(irods.O_RDWR)
targPath = self.store.coll.getCollName() + "/" + fn
statbuf = os.stat(srcPath)
dataObjOprInp.setCreateMode(statbuf.st_mode)
dataObjOprInp.setObjPath(targPath)
dataObjOprInp.setDataSize(statbuf.st_size)
irods.rcDataObjPut(self.store.cxn, dataObjOprInp, srcPath)
self.cxns[k] = None
return None
def exists( self, obj, **kwargs ):
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path( obj, **kwargs )
log.debug( 'exists(): checking: %s', doi.objPath )
return irods.rcObjStat( self.rods_conn, doi ) is not None
def exists( self, obj, **kwargs ):
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path( obj, **kwargs )
log.debug( 'exists(): checking: %s', doi.objPath )
return irods.rcObjStat( self.rods_conn, doi ) is not None
