def update_from_file(self, obj, file_name=None, create=False, **kwargs): assert "dir_only" not in kwargs, "update_from_file(): `dir_only` parameter is invalid here" # do not create if not requested if create and not self.exists(obj, **kwargs): raise ObjectNotFound() if file_name is None: file_name = self.__get_cache_path(obj, **kwargs) # put will create if necessary doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path(obj, **kwargs) doi.createMode = 0640 doi.dataSize = os.stat(file_name).st_size doi.numThreads = 0 irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, "") # TODO: might want to VERIFY_CHKSUM_KW log.debug("update_from_file(): updating %s to %s", file_name, doi.objPath) # do the iput status = irods.rcDataObjPut(self.rods_conn, doi, file_name) assert status == 0, "update_from_file(): iput %s failed (%s): %s" % ( doi.objPath, status, irods.strerror(status), )
def delete( self, obj, entire_dir=False, **kwargs ): assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here' rods_path = self.__get_rods_path( obj, **kwargs ) # __get_rods_path prepends self.root_collection_path but we are going # to ensure that it's valid anyway for safety's sake assert rods_path.startswith( self.root_collection_path + '/' ), 'ERROR: attempt to delete object outside root collection (path was: %s)' % rods_path if entire_dir: # TODO raise NotImplementedError() h = self.__get_rods_handle( obj, **kwargs ) try: # note: PyRods' irodsFile.delete() does not set force status = h.delete() assert status == 0, '%d: %s' % ( status, irods.strerror( status ) ) return True except AttributeError: log.warning( 'delete(): operation failed: object does not exist: %s', rods_path ) except AssertionError as e: # delete() does not raise on deletion failure log.error( 'delete(): operation failed: %s', e ) finally: # remove the cached entry (finally is executed even when the try # contains a return) self.__clean_cache_entry( self, obj, **kwargs ) return False
def update_from_file(self, obj, file_name=None, create=False, **kwargs): assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here' # do not create if not requested if create and not self.exists(obj, **kwargs): raise ObjectNotFound() if file_name is None: file_name = self.__get_cache_path(obj, **kwargs) # put will create if necessary doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path(obj, **kwargs) doi.createMode = 0o640 doi.dataSize = os.stat(file_name).st_size doi.numThreads = 0 irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, '') # TODO: might want to VERIFY_CHKSUM_KW log.debug('update_from_file(): updating %s to %s', file_name, doi.objPath) # do the iput status = irods.rcDataObjPut(self.rods_conn, doi, file_name) assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % ( doi.objPath, status, irods.strerror(status))
def rods_connect(): """ A basic iRODS connection mechanism that connects using the current iRODS environment """ status, env = irods.getRodsEnv() assert status == 0, "connect(): getRodsEnv() failed (%s): %s" % (status, irods.strerror(status)) conn, err = irods.rcConnect(env.rodsHost, env.rodsPort, env.rodsUserName, env.rodsZone) assert err.status == 0, "connect(): rcConnect() failed (%s): %s" % (err.status, err.msg) status, pw = irods.obfGetPw() assert status == 0, "connect(): getting password with obfGetPw() failed (%s): %s" % (status, irods.strerror(status)) status = irods.clientLoginWithObfPassword(conn, pw) assert status == 0, "connect(): logging in with clientLoginWithObfPassword() failed (%s): %s" % ( status, irods.strerror(status), ) return env, conn
def get_filename(self, obj, **kwargs): log.debug( "get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.", obj.__class__.__name__, obj.id) # For finding all places where get_filename is called... #log.debug( ''.join( traceback.format_stack() ) ) cached_path = self.__get_cache_path(obj, **kwargs) if not self.exists(obj, **kwargs): raise ObjectNotFound() # TODO: implement or define whether dir_only is valid if 'dir_only' in kwargs: raise NotImplementedError() # cache hit if os.path.exists(cached_path): return os.path.abspath(cached_path) # cache miss # TODO: thread this incoming_path = os.path.join( os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path)) doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path(obj, **kwargs) doi.dataSize = 0 # TODO: does this affect performance? should we get size? doi.numThreads = 0 # TODO: might want to VERIFY_CHKSUM_KW log.debug('get_filename(): caching %s to %s', doi.objPath, incoming_path) # do the iget status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path) # if incoming already exists, we'll wait for another process or thread # to finish caching if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG: assert status == 0, 'get_filename(): iget %s failed (%s): %s' % ( doi.objPath, status, irods.strerror(status)) # POSIX rename is atomic # TODO: rename without clobbering os.rename(incoming_path, cached_path) log.debug('get_filename(): cached %s to %s', doi.objPath, cached_path) # another process or thread is caching, wait for it while not os.path.exists(cached_path): # TODO: force restart after mod time > some configurable, or # otherwise deal with this potential deadlock and interrupted # transfers time.sleep(5) log.debug("get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id) return os.path.abspath(cached_path)
def get_filename(self, obj, **kwargs): log.debug( "get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.", obj.__class__.__name__, obj.id, ) cached_path = self.__get_cache_path(obj, **kwargs) if not self.exists(obj, **kwargs): raise ObjectNotFound() # TODO: implement or define whether dir_only is valid if "dir_only" in kwargs: raise NotImplementedError() # cache hit if os.path.exists(cached_path): return os.path.abspath(cached_path) # cache miss # TODO: thread this incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path)) doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path(obj, **kwargs) doi.dataSize = 0 # TODO: does this affect performance? should we get size? doi.numThreads = 0 # TODO: might want to VERIFY_CHKSUM_KW log.debug("get_filename(): caching %s to %s", doi.objPath, incoming_path) # do the iget status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path) # if incoming already exists, we'll wait for another process or thread # to finish caching if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG: assert status == 0, "get_filename(): iget %s failed (%s): %s" % ( doi.objPath, status, irods.strerror(status), ) # POSIX rename is atomic # TODO: rename without clobbering os.rename(incoming_path, cached_path) log.debug("get_filename(): cached %s to %s", doi.objPath, cached_path) # another process or thread is caching, wait for it while not os.path.exists(cached_path): # TODO: force restart after mod time > some configurable, or # otherwise deal with this potential deadlock and interrupted # transfers time.sleep(5) log.debug( "get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id ) return os.path.abspath(cached_path)
def rods_connect(): """ A basic iRODS connection mechanism that connects using the current iRODS environment """ status, env = irods.getRodsEnv() assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % ( status, irods.strerror(status)) conn, err = irods.rcConnect(env.rodsHost, env.rodsPort, env.rodsUserName, env.rodsZone) assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % ( err.status, err.msg) status, pw = irods.obfGetPw() assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % ( status, irods.strerror(status)) status = irods.clientLoginWithObfPassword(conn, pw) assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % ( status, irods.strerror(status)) return env, conn
def delete(self, obj, entire_dir=False, **kwargs): assert "dir_only" not in kwargs, "delete(): `dir_only` parameter is invalid here" rods_path = self.__get_rods_path(obj, **kwargs) # __get_rods_path prepends self.root_collection_path but we are going # to ensure that it's valid anyway for safety's sake assert rods_path.startswith(self.root_collection_path + "/"), ( "ERROR: attempt to delete object outside root collection (path was: %s)" % rods_path ) if entire_dir: # TODO raise NotImplementedError() h = self.__get_rods_handle(obj, **kwargs) try: # note: PyRods' irodsFile.delete() does not set force status = h.delete() assert status == 0, "%d: %s" % (status, irods.strerror(status)) return True except AttributeError: log.warning("delete(): operation failed: object does not exist: %s", rods_path) except AssertionError, e: # delete() does not raise on deletion failure log.error("delete(): operation failed: %s", e)
def create(self, obj, **kwargs): if not self.exists(obj, **kwargs): rods_path = self.__get_rods_path(obj, **kwargs) log.debug('create(): %s', rods_path) dir_only = kwargs.get('dir_only', False) # short circuit collection creation since most of the time it will # be the root collection which already exists collection_path = rods_path if dir_only else path_dirname( rods_path) if collection_path != self.root_collection_path: self.__mkcolls(collection_path) if not dir_only: # rcDataObjCreate is used instead of the irodsOpen wrapper so # that we can prevent overwriting doi = irods.dataObjInp_t() doi.objPath = rods_path doi.createMode = 0o640 doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) status = irods.rcDataObjCreate(self.rods_conn, doi) assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror(status))
def create(self, obj, **kwargs): if not self.exists(obj, **kwargs): rods_path = self.__get_rods_path(obj, **kwargs) log.debug("create(): %s", rods_path) dir_only = kwargs.get("dir_only", False) # short circuit collection creation since most of the time it will # be the root collection which already exists collection_path = rods_path if dir_only else path_dirname(rods_path) if collection_path != self.root_collection_path: self.__mkcolls(collection_path) if not dir_only: # rcDataObjCreate is used instead of the irodsOpen wrapper so # that we can prevent overwriting doi = irods.dataObjInp_t() doi.objPath = rods_path doi.createMode = 0640 doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) status = irods.rcDataObjCreate(self.rods_conn, doi) assert status >= 0, "create(): rcDataObjCreate() failed: %s: %s: %s" % ( rods_path, status, irods.strerror(status), )