def update_from_file(self, obj, file_name=None, create=False, **kwargs):
assert "dir_only" not in kwargs, "update_from_file(): `dir_only` parameter is invalid here"
# do not create if not requested
if create and not self.exists(obj, **kwargs):
raise ObjectNotFound()
if file_name is None:
file_name = self.__get_cache_path(obj, **kwargs)
# put will create if necessary
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.createMode = 0640
doi.dataSize = os.stat(file_name).st_size
doi.numThreads = 0
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource)
irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, "")
# TODO: might want to VERIFY_CHKSUM_KW
log.debug("update_from_file(): updating %s to %s", file_name, doi.objPath)
# do the iput
status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
assert status == 0, "update_from_file(): iput %s failed (%s): %s" % (
doi.objPath,
status,
irods.strerror(status),
)
def delete( self, obj, entire_dir=False, **kwargs ):
assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here'
rods_path = self.__get_rods_path( obj, **kwargs )
# __get_rods_path prepends self.root_collection_path but we are going
# to ensure that it's valid anyway for safety's sake
assert rods_path.startswith( self.root_collection_path + '/' ), 'ERROR: attempt to delete object outside root collection (path was: %s)' % rods_path
if entire_dir:
# TODO
raise NotImplementedError()
h = self.__get_rods_handle( obj, **kwargs )
try:
# note: PyRods' irodsFile.delete() does not set force
status = h.delete()
assert status == 0, '%d: %s' % ( status, irods.strerror( status ) )
return True
except AttributeError:
log.warning( 'delete(): operation failed: object does not exist: %s', rods_path )
except AssertionError as e:
# delete() does not raise on deletion failure
log.error( 'delete(): operation failed: %s', e )
finally:
# remove the cached entry (finally is executed even when the try
# contains a return)
self.__clean_cache_entry( self, obj, **kwargs )
return False
def update_from_file(self, obj, file_name=None, create=False, **kwargs):
assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here'
# do not create if not requested
if create and not self.exists(obj, **kwargs):
raise ObjectNotFound()
if file_name is None:
file_name = self.__get_cache_path(obj, **kwargs)
# put will create if necessary
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.createMode = 0o640
doi.dataSize = os.stat(file_name).st_size
doi.numThreads = 0
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW,
self.default_resource)
irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, '')
# TODO: might want to VERIFY_CHKSUM_KW
log.debug('update_from_file(): updating %s to %s', file_name,
doi.objPath)
# do the iput
status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % (
doi.objPath, status, irods.strerror(status))
def delete( self, obj, entire_dir=False, **kwargs ):
assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here'
rods_path = self.__get_rods_path( obj, **kwargs )
# __get_rods_path prepends self.root_collection_path but we are going
# to ensure that it's valid anyway for safety's sake
assert rods_path.startswith( self.root_collection_path + '/' ), 'ERROR: attempt to delete object outside root collection (path was: %s)' % rods_path
if entire_dir:
# TODO
raise NotImplementedError()
h = self.__get_rods_handle( obj, **kwargs )
try:
# note: PyRods' irodsFile.delete() does not set force
status = h.delete()
assert status == 0, '%d: %s' % ( status, irods.strerror( status ) )
return True
except AttributeError:
log.warning( 'delete(): operation failed: object does not exist: %s', rods_path )
except AssertionError as e:
# delete() does not raise on deletion failure
log.error( 'delete(): operation failed: %s', e )
finally:
# remove the cached entry (finally is executed even when the try
# contains a return)
self.__clean_cache_entry( self, obj, **kwargs )
return False
def rods_connect():
"""
A basic iRODS connection mechanism that connects using the current iRODS
environment
"""
status, env = irods.getRodsEnv()
assert status == 0, "connect(): getRodsEnv() failed (%s): %s" % (status, irods.strerror(status))
conn, err = irods.rcConnect(env.rodsHost, env.rodsPort, env.rodsUserName, env.rodsZone)
assert err.status == 0, "connect(): rcConnect() failed (%s): %s" % (err.status, err.msg)
status, pw = irods.obfGetPw()
assert status == 0, "connect(): getting password with obfGetPw() failed (%s): %s" % (status, irods.strerror(status))
status = irods.clientLoginWithObfPassword(conn, pw)
assert status == 0, "connect(): logging in with clientLoginWithObfPassword() failed (%s): %s" % (
status,
irods.strerror(status),
)
return env, conn
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__, obj.id)
# For finding all places where get_filename is called...
#log.debug( ''.join( traceback.format_stack() ) )
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if 'dir_only' in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(
os.path.dirname(cached_path),
"__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug('get_filename(): caching %s to %s', doi.objPath,
incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, 'get_filename(): iget %s failed (%s): %s' % (
doi.objPath, status, irods.strerror(status))
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug('get_filename(): cached %s to %s', doi.objPath,
cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug("get_filename(): waiting on incoming '%s' for %s %s",
incoming_path, obj.__class__.__name__, obj.id)
return os.path.abspath(cached_path)
def get_filename(self, obj, **kwargs):
log.debug(
"get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
obj.__class__.__name__,
obj.id,
)
cached_path = self.__get_cache_path(obj, **kwargs)
if not self.exists(obj, **kwargs):
raise ObjectNotFound()
# TODO: implement or define whether dir_only is valid
if "dir_only" in kwargs:
raise NotImplementedError()
# cache hit
if os.path.exists(cached_path):
return os.path.abspath(cached_path)
# cache miss
# TODO: thread this
incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path))
doi = irods.dataObjInp_t()
doi.objPath = self.__get_rods_path(obj, **kwargs)
doi.dataSize = 0 # TODO: does this affect performance? should we get size?
doi.numThreads = 0
# TODO: might want to VERIFY_CHKSUM_KW
log.debug("get_filename(): caching %s to %s", doi.objPath, incoming_path)
# do the iget
status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)
# if incoming already exists, we'll wait for another process or thread
# to finish caching
if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
assert status == 0, "get_filename(): iget %s failed (%s): %s" % (
doi.objPath,
status,
irods.strerror(status),
)
# POSIX rename is atomic
# TODO: rename without clobbering
os.rename(incoming_path, cached_path)
log.debug("get_filename(): cached %s to %s", doi.objPath, cached_path)
# another process or thread is caching, wait for it
while not os.path.exists(cached_path):
# TODO: force restart after mod time > some configurable, or
# otherwise deal with this potential deadlock and interrupted
# transfers
time.sleep(5)
log.debug(
"get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id
)
return os.path.abspath(cached_path)
def rods_connect():
"""
A basic iRODS connection mechanism that connects using the current iRODS
environment
"""
status, env = irods.getRodsEnv()
assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % (
status, irods.strerror(status))
conn, err = irods.rcConnect(env.rodsHost, env.rodsPort, env.rodsUserName,
env.rodsZone)
assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % (
err.status, err.msg)
status, pw = irods.obfGetPw()
assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % (
status, irods.strerror(status))
status = irods.clientLoginWithObfPassword(conn, pw)
assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % (
status, irods.strerror(status))
return env, conn
def delete(self, obj, entire_dir=False, **kwargs):
assert "dir_only" not in kwargs, "delete(): `dir_only` parameter is invalid here"
rods_path = self.__get_rods_path(obj, **kwargs)
# __get_rods_path prepends self.root_collection_path but we are going
# to ensure that it's valid anyway for safety's sake
assert rods_path.startswith(self.root_collection_path + "/"), (
"ERROR: attempt to delete object outside root collection (path was: %s)" % rods_path
)
if entire_dir:
# TODO
raise NotImplementedError()
h = self.__get_rods_handle(obj, **kwargs)
try:
# note: PyRods' irodsFile.delete() does not set force
status = h.delete()
assert status == 0, "%d: %s" % (status, irods.strerror(status))
return True
except AttributeError:
log.warning("delete(): operation failed: object does not exist: %s", rods_path)
except AssertionError, e:
# delete() does not raise on deletion failure
log.error("delete(): operation failed: %s", e)
def create(self, obj, **kwargs):
if not self.exists(obj, **kwargs):
rods_path = self.__get_rods_path(obj, **kwargs)
log.debug('create(): %s', rods_path)
dir_only = kwargs.get('dir_only', False)
# short circuit collection creation since most of the time it will
# be the root collection which already exists
collection_path = rods_path if dir_only else path_dirname(
rods_path)
if collection_path != self.root_collection_path:
self.__mkcolls(collection_path)
if not dir_only:
# rcDataObjCreate is used instead of the irodsOpen wrapper so
# that we can prevent overwriting
doi = irods.dataObjInp_t()
doi.objPath = rods_path
doi.createMode = 0o640
doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW,
self.default_resource)
status = irods.rcDataObjCreate(self.rods_conn, doi)
assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % (
rods_path, status, irods.strerror(status))
def create(self, obj, **kwargs):
if not self.exists(obj, **kwargs):
rods_path = self.__get_rods_path(obj, **kwargs)
log.debug("create(): %s", rods_path)
dir_only = kwargs.get("dir_only", False)
# short circuit collection creation since most of the time it will
# be the root collection which already exists
collection_path = rods_path if dir_only else path_dirname(rods_path)
if collection_path != self.root_collection_path:
self.__mkcolls(collection_path)
if not dir_only:
# rcDataObjCreate is used instead of the irodsOpen wrapper so
# that we can prevent overwriting
doi = irods.dataObjInp_t()
doi.objPath = rods_path
doi.createMode = 0640
doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable
irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource)
status = irods.rcDataObjCreate(self.rods_conn, doi)
assert status >= 0, "create(): rcDataObjCreate() failed: %s: %s: %s" % (
rods_path,
status,
irods.strerror(status),
)
