Esempio n. 1
0
    def update_from_file(self, obj, file_name=None, create=False, **kwargs):
        assert "dir_only" not in kwargs, "update_from_file(): `dir_only` parameter is invalid here"

        # do not create if not requested
        if create and not self.exists(obj, **kwargs):
            raise ObjectNotFound()

        if file_name is None:
            file_name = self.__get_cache_path(obj, **kwargs)

        # put will create if necessary
        doi = irods.dataObjInp_t()
        doi.objPath = self.__get_rods_path(obj, **kwargs)
        doi.createMode = 0640
        doi.dataSize = os.stat(file_name).st_size
        doi.numThreads = 0
        irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource)
        irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, "")
        # TODO: might want to VERIFY_CHKSUM_KW
        log.debug("update_from_file(): updating %s to %s", file_name, doi.objPath)

        # do the iput
        status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
        assert status == 0, "update_from_file(): iput %s failed (%s): %s" % (
            doi.objPath,
            status,
            irods.strerror(status),
        )
Esempio n. 2
0
    def update_from_file(self, obj, file_name=None, create=False, **kwargs):
        assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here'

        # do not create if not requested
        if create and not self.exists(obj, **kwargs):
            raise ObjectNotFound()

        if file_name is None:
            file_name = self.__get_cache_path(obj, **kwargs)

        # put will create if necessary
        doi = irods.dataObjInp_t()
        doi.objPath = self.__get_rods_path(obj, **kwargs)
        doi.createMode = 0o640
        doi.dataSize = os.stat(file_name).st_size
        doi.numThreads = 0
        irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW,
                        self.default_resource)
        irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, '')
        # TODO: might want to VERIFY_CHKSUM_KW
        log.debug('update_from_file(): updating %s to %s', file_name,
                  doi.objPath)

        # do the iput
        status = irods.rcDataObjPut(self.rods_conn, doi, file_name)
        assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % (
            doi.objPath, status, irods.strerror(status))
Esempio n. 3
0
 def download_files(self, irods_files, targPath, overwrite=False):
     """Download a list of files from irods to a local directory
     irods_files : list of tuples (collection, file, resource)
     targPath : local path
     If some files already exist in the local directory and if the overwrite
     option is not set then a structure is returned, the caller will then
     have to define a strategy."""
     dataObjOprInp = irods.dataObjInp_t()
     dataObjOprInp.openFlags = irods.O_RDONLY
     if overwrite:
         irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
     already_there = []
     for (collection, irods_file, resource) in irods_files:
         if resource:
             irods.addKeyVal(dataObjOprInp.condInput,
                             irods.RESC_NAME_KW,
                             resource)
         dataObjOprInp.objPath = collection + '/' + irods_file
         status = irods.rcDataObjGet(self.conn,
                                     dataObjOprInp,
                                     targPath + os.sep + irods_file)
         # If the file is already present we catch the error
         if status == irods.OVERWRITE_WITHOUT_FORCE_FLAG:
             already_there.append((collection, irods_file, resource))
     return already_there
Esempio n. 4
0
    def get_filename(self, obj, **kwargs):
        log.debug(
            "get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
            obj.__class__.__name__, obj.id)

        # For finding all places where get_filename is called...
        #log.debug( ''.join( traceback.format_stack() ) )

        cached_path = self.__get_cache_path(obj, **kwargs)

        if not self.exists(obj, **kwargs):
            raise ObjectNotFound()

        # TODO: implement or define whether dir_only is valid
        if 'dir_only' in kwargs:
            raise NotImplementedError()

        # cache hit
        if os.path.exists(cached_path):
            return os.path.abspath(cached_path)

        # cache miss
        # TODO: thread this
        incoming_path = os.path.join(
            os.path.dirname(cached_path),
            "__incoming_%s" % os.path.basename(cached_path))
        doi = irods.dataObjInp_t()
        doi.objPath = self.__get_rods_path(obj, **kwargs)
        doi.dataSize = 0  # TODO: does this affect performance? should we get size?
        doi.numThreads = 0
        # TODO: might want to VERIFY_CHKSUM_KW
        log.debug('get_filename(): caching %s to %s', doi.objPath,
                  incoming_path)

        # do the iget
        status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)

        # if incoming already exists, we'll wait for another process or thread
        # to finish caching
        if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
            assert status == 0, 'get_filename(): iget %s failed (%s): %s' % (
                doi.objPath, status, irods.strerror(status))
            # POSIX rename is atomic
            # TODO: rename without clobbering
            os.rename(incoming_path, cached_path)
            log.debug('get_filename(): cached %s to %s', doi.objPath,
                      cached_path)

        # another process or thread is caching, wait for it
        while not os.path.exists(cached_path):
            # TODO: force restart after mod time > some configurable, or
            # otherwise deal with this potential deadlock and interrupted
            # transfers
            time.sleep(5)
            log.debug("get_filename(): waiting on incoming '%s' for %s %s",
                      incoming_path, obj.__class__.__name__, obj.id)

        return os.path.abspath(cached_path)
Esempio n. 5
0
    def get_filename(self, obj, **kwargs):
        log.debug(
            "get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.",
            obj.__class__.__name__,
            obj.id,
        )
        cached_path = self.__get_cache_path(obj, **kwargs)

        if not self.exists(obj, **kwargs):
            raise ObjectNotFound()

        # TODO: implement or define whether dir_only is valid
        if "dir_only" in kwargs:
            raise NotImplementedError()

        # cache hit
        if os.path.exists(cached_path):
            return os.path.abspath(cached_path)

        # cache miss
        # TODO: thread this
        incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path))
        doi = irods.dataObjInp_t()
        doi.objPath = self.__get_rods_path(obj, **kwargs)
        doi.dataSize = 0  # TODO: does this affect performance? should we get size?
        doi.numThreads = 0
        # TODO: might want to VERIFY_CHKSUM_KW
        log.debug("get_filename(): caching %s to %s", doi.objPath, incoming_path)

        # do the iget
        status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path)

        # if incoming already exists, we'll wait for another process or thread
        # to finish caching
        if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG:
            assert status == 0, "get_filename(): iget %s failed (%s): %s" % (
                doi.objPath,
                status,
                irods.strerror(status),
            )
            # POSIX rename is atomic
            # TODO: rename without clobbering
            os.rename(incoming_path, cached_path)
            log.debug("get_filename(): cached %s to %s", doi.objPath, cached_path)

        # another process or thread is caching, wait for it
        while not os.path.exists(cached_path):
            # TODO: force restart after mod time > some configurable, or
            # otherwise deal with this potential deadlock and interrupted
            # transfers
            time.sleep(5)
            log.debug(
                "get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id
            )

        return os.path.abspath(cached_path)
Esempio n. 6
0
 def replicate_file(self, file_info, dst_resource):
     """Replicate a file in irods
     file_info : (collection, file, resource)
     dst_resource : str"""
     (collection, irods_file, src_resource) = file_info
     dataObjInp = irods.dataObjInp_t()
     d = irods.getFileInfo(self.conn, collection, irods_file, src_resource)
     cond_inp = dataObjInp.condInput
     irods.addKeyVal(cond_inp, irods.REPL_NUM_KW, d['data_repl_num'])
     irods.addKeyVal(cond_inp, irods.RESC_NAME_KW, src_resource)
     irods.addKeyVal(cond_inp, irods.DEST_RESC_NAME_KW, dst_resource)
     dataObjInp.objPath = collection + '/' + irods_file
     status = irods.rcDataObjRepl(self.conn, dataObjInp)
     return status
Esempio n. 7
0
 def delete_file(self, path, resc=None):
     """Delete a file in iRODS
     path : irods path (str)
     resc : Resource"""
     dataObjInp = irods.dataObjInp_t()
     dataObjInp.openFlags = irods.O_RDONLY
     if resc:
         d = irods.getFileInfo(path, resc)
         irods.addKeyVal(dataObjInp.condInput,
                         irods.REPL_NUM_KW,
                         d['data_repl_num'])
     dataObjInp.objPath = path
     status = irods.rcDataObjUnlink(self.conn, dataObjInp)
     return status
Esempio n. 8
0
 def delete_files(self, irods_files):
     """Delete a list of files from iRODS
     irods_files : list of tuples (collection, file, resource)"""
     dataObjInp = irods.dataObjInp_t()
     dataObjInp.openFlags = irods.O_RDONLY
     st = []
     for (collection, irods_file, resource) in irods_files:
         d = irods.getFileInfo(self.conn, collection, irods_file, resource)
         irods.addKeyVal(dataObjInp.condInput,
                         irods.REPL_NUM_KW,
                         d['data_repl_num'])
         dataObjInp.objPath = collection + '/' + irods_file
         status = irods.rcDataObjUnlink(self.conn, dataObjInp)
         st.append(status)
     return st
Esempio n. 9
0
 def upload_file(self, local_path, irods_path, resource=None,
                 overwrite=False):
     """Upload a file to irods
     return True if the file is already present"""
     dataObjOprInp = irods.dataObjInp_t()
     dataObjOprInp.oprType = irods.PUT_OPR
     dataObjOprInp.openFlags = irods.O_RDWR
     if overwrite:
         irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
     if resource:
         irods.addKeyVal(dataObjOprInp.condInput,
                         irods.DEST_RESC_NAME_KW,
                         resource)
     dataObjOprInp.objPath = irods_path
     status = irods.rcDataObjPut(self.conn, dataObjOprInp, local_path)
     return status == irods.OVERWRITE_WITHOUT_FORCE_FLAG
Esempio n. 10
0
 def download_file(self, irods_path, local_path, resource=None,
                 overwrite=False):
     """Download a file from irods to a local directory
     - irods_files : list of tuples (collection, file, resource)
     - local path
     - resource
     - overwrite the exisiting file if it exists"""
     dataObjOprInp = irods.dataObjInp_t()
     dataObjOprInp.openFlags = irods.O_RDONLY
     if overwrite:
         irods.addKeyVal(dataObjOprInp.condInput, irods.FORCE_FLAG_KW, "")
     if resource:
         irods.addKeyVal(dataObjOprInp.condInput, irods.RESC_NAME_KW,
                         resource)
     dataObjOprInp.objPath = irods_path
     status = irods.rcDataObjGet(self.conn, dataObjOprInp, local_path)
     return status == irods.OVERWRITE_WITHOUT_FORCE_FLAG
Esempio n. 11
0
 def create(self, obj, **kwargs):
     if not self.exists( obj, **kwargs ):
         rods_path = self.__get_rods_path( obj, **kwargs )
         log.debug( 'create(): %s', rods_path )
         dir_only = kwargs.get( 'dir_only', False )
         # short circuit collection creation since most of the time it will
         # be the root collection which already exists
         collection_path = rods_path if dir_only else path_dirname( rods_path )
         if collection_path != self.root_collection_path:
             self.__mkcolls( collection_path )
         if not dir_only:
             # rcDataObjCreate is used instead of the irodsOpen wrapper so
             # that we can prevent overwriting
             doi = irods.dataObjInp_t()
             doi.objPath = rods_path
             doi.createMode = 0o640
             doi.dataSize = 0  # 0 actually means "unknown", although literally 0 would be preferable
             irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource )
             status = irods.rcDataObjCreate( self.rods_conn, doi )
             assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) )
Esempio n. 12
0
 def create(self, obj, **kwargs):
     if not self.exists( obj, **kwargs ):
         rods_path = self.__get_rods_path( obj, **kwargs )
         log.debug( 'create(): %s', rods_path )
         dir_only = kwargs.get( 'dir_only', False )
         # short circuit collection creation since most of the time it will
         # be the root collection which already exists
         collection_path = rods_path if dir_only else path_dirname( rods_path )
         if collection_path != self.root_collection_path:
             self.__mkcolls( collection_path )
         if not dir_only:
             # rcDataObjCreate is used instead of the irodsOpen wrapper so
             # that we can prevent overwriting
             doi = irods.dataObjInp_t()
             doi.objPath = rods_path
             doi.createMode = 0o640
             doi.dataSize = 0  # 0 actually means "unknown", although literally 0 would be preferable
             irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource )
             status = irods.rcDataObjCreate( self.rods_conn, doi )
             assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) )
Esempio n. 13
0
    def close(self):
        for (k, c) in self.cxns.iteritems():
            c.close()
            # upload to irods
            srcPath = self.cxnFiles[c]
            (dirn, fn) = os.path.split(srcPath)        

            # use file/C api
            dataObjOprInp = irods.dataObjInp_t()
            dataObjOprInp.setOprType(irods.PUT_OPR)
            dataObjOprInp.setOpenFlags(irods.O_RDWR)
            targPath = self.store.coll.getCollName() + "/" + fn
            statbuf = os.stat(srcPath)            
            dataObjOprInp.setCreateMode(statbuf.st_mode)
            dataObjOprInp.setObjPath(targPath)
            dataObjOprInp.setDataSize(statbuf.st_size)
            irods.rcDataObjPut(self.store.cxn, dataObjOprInp, srcPath)

            self.cxns[k] = None
        return None
Esempio n. 14
0
 def exists( self, obj, **kwargs ):
     doi = irods.dataObjInp_t()
     doi.objPath = self.__get_rods_path( obj, **kwargs )
     log.debug( 'exists(): checking: %s', doi.objPath )
     return irods.rcObjStat( self.rods_conn, doi ) is not None
Esempio n. 15
0
 def exists( self, obj, **kwargs ):
     doi = irods.dataObjInp_t()
     doi.objPath = self.__get_rods_path( obj, **kwargs )
     log.debug( 'exists(): checking: %s', doi.objPath )
     return irods.rcObjStat( self.rods_conn, doi ) is not None