class TestCentera(object): def setup(self): self.pool = FPPool(POOL_ADDRESS) self.pool.setGlobalOption(FPLibrary.FP_OPTION_EMBEDDED_DATA_THRESHOLD, 100 * 1024) self.pool.getPoolInfo() # the application will be attached to the clip id self.pool.registerApplication("python wrapper read example", "1.0") def teardown(self): self.pool.close()
class TestCentera(object): def setup(self): self.pool = FPPool(POOL_ADDRESS) self.pool.setGlobalOption(FPLibrary.FP_OPTION_EMBEDDED_DATA_THRESHOLD, 100 * 1024) self.pool.getPoolInfo() # the application will be attached to the clip id self.pool.registerApplication("python wrapper read example", "1.0") def teardown(self): self.pool.close()
else: print "invalid option" elif audited in ['n','N']: response = raw_input( "Clip will be deleted forever, are you certain? (y/n): " ) if response in ['y','Y']: clip.delete(ca) print "Clip " + ca + " has been deleted" else: print "Deletion aborted" else: print "Deletes capability not enabled." pool.close() except FPClientException, c: print c traceback.print_exc(file=sys.stdout) except FPServerException, s: print s except FPNetException, n: print n except FPException, e: print e
status = 0 while True: res = FPQueryResult(query.fetchResult(0)) status = res.getResultCode() if status in (FPLibrary.FP_QUERY_RESULT_CODE_END, FPLibrary.FP_QUERY_RESULT_CODE_ABORT, FPLibrary.FP_QUERY_RESULT_CODE_ERROR, FPLibrary.FP_QUERY_RESULT_CODE_INCOMPLETE, FPLibrary.FP_QUERY_RESULT_CODE_COMPLETE): break elif status == FPLibrary.FP_QUERY_RESULT_CODE_PROGRESS: continue elif status == FPLibrary.FP_QUERY_RESULT_CODE_OK: print res.getClipId() query.close() pool.close() except FPClientException, c: print c traceback.print_exc(file=sys.stdout) except FPServerException, s: print s except FPNetException, n: print n except FPException, e: print e
class CenteraConnection(object): """ Manage the clips from a pool. :param object: :return: """ def __init__(self, pool_ip, options=POOL_DEFAULT_OPTIONS, application=("caspython-centera-library", _version), replace=None): """ Initialize a pool :param host: :param options: :param application: a couple (appname, version) attached to the clip_id :param replace: a translation map for replacing unsupported characters :return: """ self.log = logging.getLogger(__name__) self.pool = FPPool(pool_ip) for k, v in options.items(): self.pool.setGlobalOption(k, v) # getPoolInfo sets infos in self.pool. self.pool.getPoolInfo() # the application will be attached to the clip id self.pool.registerApplication(*application) def close(self): for i in range(3): try: self.pool.close() return except FPClientException as e: if e.errorString == 'FP_OBJECTINUSE_ERR': self.log.error( "Closing while pool still in use. Sleeping for 1 sec.") sleep(1) continue raise def put(self, clip_name, files, retention_sec): """ Writes a clip and attached files to worm. :param clip_name: :param files: :param retention_sec: :return: """ for f in files: self.validate_tag(f) with closing(FPClip(self.pool, clip_name, close_retries=3)) as clip: clip.setRetentionPeriod(long(retention_sec)) top_handle = clip.getTopTag() for filename in files: tag_name = self.clean_tag(filename) with closing(FPTag(top_handle, tag_name)) as blob_tag: with closing(FPFileInputStream(filename, 16 * 1024)) as fh: blob_tag.blobWrite(fh.stream, 0) clip_id = clip.write() return clip_id @staticmethod def validate_tag(filename): """ Check if a filename is a valid centera tag name. :param filename: :return: None on succes :raises: ValueError if not valid """ for re_m, msg in RE_INVALID_TAGS: if re_m.match(CenteraConnection.clean_tag(filename)): raise ValueError(msg) @staticmethod def clean_tag(filename): return MYTAG_ + basename(filename) # .replace("@", "_") def get(self, clip_id, tag=None): """ Get a (closed) clip from worm. While storing or retrieving tags from worm, you may need to set a close_retry. :param clip: :return: """ with closing(FPClip(self.pool, close_retries=3 if tag else 0)) as clip: self._open_or_not_found(clip, clip_id) clip.attributes = clip.getDescriptionAttributes() if tag: clip.tags = [x for x in clip.getTags()] return clip def get(self, clip_id, tag=None): """ Get a (closed) clip from worm. While storing or retrieving tags from worm, you may need to set a close_retry. :param clip: :return: """ with closing(FPClip(self.pool, close_retries=3 if tag else 0)) as clip: self._open_or_not_found(clip, clip_id) clip.attributes = clip.getDescriptionAttributes() if tag: clip.tags = [x for x in clip.getTags()] return clip def download(self, clip_id, tag_name, outfile): """ :param clip_id: :param tag_name: :param outfile: :return: the output file if present. None if not found. """ outfile = normpath(abspath(outfile)) with closing(FPClip(self.pool, close_retries=3)) as clip: self._open_or_not_found(clip, clip_id) for blob_id in clip.getBlobs(): with closing(FPTag(blob_id)) as blob_tag: blob_tag_name = blob_tag.getTagName() if blob_tag_name != tag_name: self.log.debug("Skipping tag: %s when looking for: %s", blob_tag_name, blob_tag) continue if blob_tag.getBlobSize() < 1: self.log.debug("Empty blob %s" % blob_tag_name) raise ValueError() with closing(FPFileOutputStream(outfile)) as fh: self.log.info("Writing blob %s to %s", blob_tag_name, outfile) blob_tag.blobRead(fh.stream, 0) return outfile def _open_or_not_found(self, clip, clip_id): try: clip.open(clip_id) except FPClientException as e: if e.errorString == 'FP_PARAM_ERR': raise KeyError( "Wrong parameter detected or ClipID not found: %r" % clip_id) raise def list(self, start=None, end=None, limit=0): """ Retrieve clips in the given interval. NOTE: Once iterated, the result is closed, so you cannot access the item any more! @see https://www.emc.com/collateral/TechnicalDocument/docu59169.pdf :param start: :param end: :param limit: max number of entries to retrieve. default 0 (unlimited). :return: """ # 0 means unlimited. if limit == 0: limit = -1 # Open and close FPQuery with closing(self._open_query(start, end)) as query: status = 0 while limit != 0: # Implements point 5-7 opening and closing the FPQueryResult. with closing(FPQueryResult(query.fetchResult(0))) as res: status = res.getResultCode() if status in (FPLibrary.FP_QUERY_RESULT_CODE_END, FPLibrary.FP_QUERY_RESULT_CODE_ABORT, FPLibrary.FP_QUERY_RESULT_CODE_ERROR, FPLibrary.FP_QUERY_RESULT_CODE_INCOMPLETE, FPLibrary.FP_QUERY_RESULT_CODE_COMPLETE): break elif status == FPLibrary.FP_QUERY_RESULT_CODE_PROGRESS: continue elif status == FPLibrary.FP_QUERY_RESULT_CODE_OK: limit -= 1 # Implements point 6 yield res def _open_query(self, start=None, end=None, qtype=FPLibrary.FP_QUERY_TYPE_EXISTING): """ Open a FPQuery and its associated QueryExpression as stated in the doc. The caller is in charge of closing the query. Implements points 1, 2, 3, 4, 10 of https://www.emc.com/collateral/TechnicalDocument/docu59169.pdf :param start: :param end: :param qtype: :return: """ query_expression = FPQueryExpression() query_expression.setType(qtype) if start: query_expression.setStartTime( str_to_seconds(start) * SEC_TO_MILLISEC) if end: query_expression.setEndTime(str_to_seconds(end) * SEC_TO_MILLISEC) try: query = FPQuery(self.pool) query.open(query_expression) return query finally: query_expression.close() def info(self): """ Return a dictionary with essential informations about the centera, like free space, replication options, ... :return: dict """ fields = [ 'clusterid', 'clusterName', 'version', 'infoVersion', 'capacity', 'freeSpace', 'replicaAddress' ] ret = dict((f, getattr(self.pool, f)) for f in fields) ret['clusterTime'] = self.pool.getClusterTime() return ret
def test_work_on_closed(): pool = FPPool(POOL_ADDRESS) log.info("Pool opened with handle: %r", pool.handle) pool.close() log.info("Pool opened with handle: %r", pool.handle) pool.getClusterTime()
def test_double_close(): pool = FPPool(POOL_ADDRESS) log.info("Pool opened with handle: %r", pool.handle) pool.close() log.info("Pool opened with handle: %r", pool.handle) pool.close()
class CenteraConnection(object): """ Manage the clips from a pool. :param object: :return: """ def __init__(self, pool_ip, options=POOL_DEFAULT_OPTIONS, application=("caspython-centera-library", _version), replace=None): """ Initialize a pool :param host: :param options: :param application: a couple (appname, version) attached to the clip_id :param replace: a translation map for replacing unsupported characters :return: """ self.log = logging.getLogger(__name__) self.pool = FPPool(pool_ip) for k, v in options.items(): self.pool.setGlobalOption(k, v) # getPoolInfo sets infos in self.pool. self.pool.getPoolInfo() # the application will be attached to the clip id self.pool.registerApplication(*application) def close(self): for i in range(3): try: self.pool.close() return except FPClientException as e: if e.errorString == 'FP_OBJECTINUSE_ERR': self.log.error("Closing while pool still in use. Sleeping for 1 sec.") sleep(1) continue raise def put(self, clip_name, files, retention_sec): """ Writes a clip and attached files to worm. :param clip_name: :param files: :param retention_sec: :return: """ for f in files: self.validate_tag(f) with closing(FPClip(self.pool, clip_name, close_retries=3)) as clip: clip.setRetentionPeriod(long(retention_sec)) top_handle = clip.getTopTag() for filename in files: tag_name = self.clean_tag(filename) with closing(FPTag(top_handle, tag_name)) as blob_tag: with closing(FPFileInputStream(filename, 16 * 1024)) as fh: blob_tag.blobWrite(fh.stream, 0) clip_id = clip.write() return clip_id @staticmethod def validate_tag(filename): """ Check if a filename is a valid centera tag name. :param filename: :return: None on succes :raises: ValueError if not valid """ for re_m, msg in RE_INVALID_TAGS: if re_m.match(CenteraConnection.clean_tag(filename)): raise ValueError(msg) @staticmethod def clean_tag(filename): return MYTAG_ + basename(filename) # .replace("@", "_") def get(self, clip_id, tag=None): """ Get a (closed) clip from worm. While storing or retrieving tags from worm, you may need to set a close_retry. :param clip: :return: """ with closing(FPClip(self.pool, close_retries=3 if tag else 0)) as clip: self._open_or_not_found(clip, clip_id) clip.attributes = clip.getDescriptionAttributes() if tag: clip.tags = [x for x in clip.getTags()] return clip def get(self, clip_id, tag=None): """ Get a (closed) clip from worm. While storing or retrieving tags from worm, you may need to set a close_retry. :param clip: :return: """ with closing(FPClip(self.pool, close_retries=3 if tag else 0)) as clip: self._open_or_not_found(clip, clip_id) clip.attributes = clip.getDescriptionAttributes() if tag: clip.tags = [x for x in clip.getTags()] return clip def download(self, clip_id, tag_name, outfile): """ :param clip_id: :param tag_name: :param outfile: :return: the output file if present. None if not found. """ outfile = normpath(abspath(outfile)) with closing(FPClip(self.pool, close_retries=3)) as clip: self._open_or_not_found(clip, clip_id) for blob_id in clip.getBlobs(): with closing(FPTag(blob_id)) as blob_tag: blob_tag_name = blob_tag.getTagName() if blob_tag_name != tag_name: self.log.debug( "Skipping tag: %s when looking for: %s", blob_tag_name, blob_tag) continue if blob_tag.getBlobSize() < 1: self.log.debug("Empty blob %s" % blob_tag_name) raise ValueError() with closing(FPFileOutputStream(outfile)) as fh: self.log.info( "Writing blob %s to %s", blob_tag_name, outfile) blob_tag.blobRead(fh.stream, 0) return outfile def _open_or_not_found(self, clip, clip_id): try: clip.open(clip_id) except FPClientException as e: if e.errorString == 'FP_PARAM_ERR': raise KeyError( "Wrong parameter detected or ClipID not found: %r" % clip_id) raise def list(self, start=None, end=None, limit=0): """ Retrieve clips in the given interval. NOTE: Once iterated, the result is closed, so you cannot access the item any more! @see https://www.emc.com/collateral/TechnicalDocument/docu59169.pdf :param start: :param end: :param limit: max number of entries to retrieve. default 0 (unlimited). :return: """ # 0 means unlimited. if limit == 0: limit = -1 # Open and close FPQuery with closing(self._open_query(start, end)) as query: status = 0 while limit != 0: # Implements point 5-7 opening and closing the FPQueryResult. with closing(FPQueryResult(query.fetchResult(0))) as res: status = res.getResultCode() if status in ( FPLibrary.FP_QUERY_RESULT_CODE_END, FPLibrary.FP_QUERY_RESULT_CODE_ABORT, FPLibrary.FP_QUERY_RESULT_CODE_ERROR, FPLibrary.FP_QUERY_RESULT_CODE_INCOMPLETE, FPLibrary.FP_QUERY_RESULT_CODE_COMPLETE ): break elif status == FPLibrary.FP_QUERY_RESULT_CODE_PROGRESS: continue elif status == FPLibrary.FP_QUERY_RESULT_CODE_OK: limit -= 1 # Implements point 6 yield res def _open_query(self, start=None, end=None, qtype=FPLibrary.FP_QUERY_TYPE_EXISTING): """ Open a FPQuery and its associated QueryExpression as stated in the doc. The caller is in charge of closing the query. Implements points 1, 2, 3, 4, 10 of https://www.emc.com/collateral/TechnicalDocument/docu59169.pdf :param start: :param end: :param qtype: :return: """ query_expression = FPQueryExpression() query_expression.setType(qtype) if start: query_expression.setStartTime(str_to_seconds(start) * SEC_TO_MILLISEC) if end: query_expression.setEndTime(str_to_seconds(end) * SEC_TO_MILLISEC) try: query = FPQuery(self.pool) query.open(query_expression) return query finally: query_expression.close() def info(self): """ Return a dictionary with essential informations about the centera, like free space, replication options, ... :return: dict """ fields = [ 'clusterid', 'clusterName', 'version', 'infoVersion', 'capacity', 'freeSpace', 'replicaAddress' ] self.pool.getPoolInfo() ret = dict((f, getattr(self.pool, f)) for f in fields ) ret['clusterTime'] = self.pool.getClusterTime() return ret