def _create_seqLenDict(self, dictpath, seqpath, reader=None): """Create a seqLenDict from 'seqpath' and store in 'dictpath'.""" seqLenDict = classutil.open_shelve(dictpath, 'n') try: logger.debug('Building sequence length index...') _store_seqlen_dict(seqLenDict, seqpath, reader) finally: seqLenDict.close() # close after writing, no matter what! return classutil.open_shelve(dictpath, 'r') # re-open read-only
def __init__(self, filename, mode=None, writeback=False, unpicklingMode=False, verbose=True, **kwargs): '''Wrapper for a shelve object that can be pickled. Ideally, you should specify a TWO letter mode string: the first letter to indicate what mode the shelve should be initially opened in, and the second to indicate the mode to open the shelve during unpickling. e.g. mode='nr': to create an empty shelve (writable), which in future will be re-opened read-only. Also, mode=None makes it first attempt to open read-only, but if the file does not exist will create it using mode 'c'. ''' # Mark this string as a file path. self.filename = classutil.SourceFileName(str(filename)) self.writeback = writeback if unpicklingMode or mode is None or mode == 'r': # Just use mode as given. self.mode = mode elif mode == 'n' or mode == 'c' or mode == 'w': # Ambiguous modes, warn & set default. if verbose: import sys print >> sys.stderr, '''Warning: you opened shelve file %s in mode '%s' but this is ambiguous for how the shelve should be re-opened later during unpickling. By default it will be re-opened in mode 'r' (read-only). To make it be re-opened writable, create it in mode '%sw', or call its method reopen('w'), which will make it be re-opened in mode 'w' now and in later unpickling. To suppress this warning message, use the verbose=False option.''' % (filename, mode, mode) self.mode = 'r' else: # PROCESS UNAMBIGUOUS TWO-LETTER mode STRING try: if len(mode) == 2 and mode[0] in 'ncwr' and mode[1] in 'cwr': self.mode = mode[1] # IN FUTURE OPEN IN THIS MODE mode = mode[0] # OPEN NOW IN THIS MODE else: raise ValueError('invalid mode string: ' + mode) except TypeError: raise ValueError('file mode must be a string!') if unpicklingMode: self.d = classutil.open_shelve(filename, mode, writeback, allowReadOnly=True) else: self.d = classutil.open_shelve(filename, mode, writeback) classutil.apply_itemclass(self, kwargs)
def _init_subclass(cls, db, filepath, **kwargs): '''initialize our indexes if needed, and provide db with a seqInfoDict attribute for looking up length and offset info. Open or build seqLenDict if needed''' cls.db = db # all instances of this class are now bound to this database from dbfile import NoSuchFileError try: # THIS WILL FAIL IF SHELVE NOT ALREADY PRESENT... db.seqLenDict = classutil.open_shelve(filepath+'.seqlen','r') # READ-ONLY except NoSuchFileError: # BUILD: READ ALL SEQ LENGTHS, STORE IN PERSIST DICT db.seqLenDict = classutil.open_shelve(filepath+'.seqlen','n') # NEW EMPTY FILE logger.debug( 'building sequence indices' ) store_seqlen_dict(db.seqLenDict, filepath, **kwargs) db.seqLenDict.close() # FORCE IT TO WRITE DATA TO DISK db.seqLenDict = classutil.open_shelve(filepath+'.seqlen','r') # READ-ONLY db.seqInfoDict = SeqLenDictWrapper(db) # standard interface
def __init__(self, nlmsa, filename, mode, idDictClass=None, maxSequenceCacheSize=_DEFAULT_SEQUENCE_CACHE_SIZE): self._cache = classutil.RecentValueDictionary(maxSequenceCacheSize) self.seqlist = NLMSASeqList(self) self.nlmsa = nlmsa self.filename = filename if mode == "memory": # just use python dictionary idDictClass = dict elif mode == "w": # new database mode = "n" if idDictClass is None: # use persistent id dictionary storage self.seqIDdict = classutil.open_shelve(filename + ".seqIDdict", mode) self.IDdict = classutil.open_shelve(filename + ".idDict", mode) else: # user supplied class for id dictionary storage self.seqIDdict = idDictClass() self.IDdict = idDictClass()
def reopen(self, mode='r'): 're-open shelve in the specified mode, and save mode on self' self.close() self.d = classutil.open_shelve(self.filename, mode, writeback=self.writeback) self.mode = mode
def __init__(self, nlmsa, filename, mode, maxID=1000000, idDictClass=None): dict.__init__(self) self.seqlist = NLMSASeqList(self) self.maxID = maxID self.nlmsa = nlmsa self.filename = filename if mode == "memory": # just use python dictionary idDictClass = dict elif mode == "w": # new database mode = "n" if idDictClass is None: # use persistent id dictionary storage self.seqIDdict = classutil.open_shelve(filename + ".seqIDdict", mode) self.IDdict = classutil.open_shelve(filename + ".idDict", mode) else: # user supplied class for id dictionary storage self.seqIDdict = idDictClass() self.IDdict = idDictClass()
def __init__(self,nlmsa,filename,mode,maxID=1000000,idDictClass=None): dict.__init__(self) self.seqlist=NLMSASeqList(self) self.maxID=maxID self.nlmsa=nlmsa self.filename=filename if mode=='memory': # JUST USE PYTHON DICTIONARY idDictClass = dict elif mode=='w': # NEW DATABASE mode='n' if idDictClass is None: # USE PERSISTENT ID DICTIONARY STORAGE self.seqIDdict = classutil.open_shelve(filename+'.seqIDdict',mode) self.IDdict = classutil.open_shelve(filename+'.idDict',mode) else: # USER SUPPLIED CLASS FOR ID DICTIONARY STORAGE self.seqIDdict=idDictClass() self.IDdict=idDictClass()
def __init__(self, nlmsa, filename, mode, idDictClass=None, maxSequenceCacheSize=_DEFAULT_SEQUENCE_CACHE_SIZE): self._cache = classutil.RecentValueDictionary(maxSequenceCacheSize) self.seqlist = NLMSASeqList(self) self.nlmsa = nlmsa self.filename = filename if mode == 'memory': # just use python dictionary idDictClass = dict elif mode == 'w': # new database mode = 'n' if idDictClass is None: # use persistent id dictionary storage self.seqIDdict = classutil.open_shelve(filename + '.seqIDdict', mode) self.IDdict = classutil.open_shelve(filename + '.idDict', mode) else: # user supplied class for id dictionary storage self.seqIDdict = idDictClass() self.IDdict = idDictClass()
def _init_subclass(cls, db, filepath, **kwargs): '''initialize our indexes if needed, and provide db with a seqInfoDict attribute for looking up length and offset info. Open or build seqLenDict if needed''' cls.db = db # all instances of this class are now bound to this database from dbfile import NoSuchFileError try: # THIS WILL FAIL IF SHELVE NOT ALREADY PRESENT... db.seqLenDict = classutil.open_shelve(filepath + '.seqlen', 'r') # READ-ONLY except NoSuchFileError: # BUILD: READ ALL SEQ LENGTHS, STORE IN PERSIST DICT db.seqLenDict = classutil.open_shelve(filepath + '.seqlen', 'n') # NEW EMPTY FILE logger.debug('building sequence indices') store_seqlen_dict(db.seqLenDict, filepath, **kwargs) db.seqLenDict.close() # FORCE IT TO WRITE DATA TO DISK db.seqLenDict = classutil.open_shelve(filepath + '.seqlen', 'r') # READ-ONLY db.seqInfoDict = SeqLenDictWrapper(db) # standard interface
def __init__(self, filename, mode=None, writeback=False, unpicklingMode=False, verbose=True, **kwargs): '''Wrapper for a shelve object that can be pickled. Ideally, you should specify a TWO letter mode string: the first letter to indicate what mode the shelve should be initially opened in, and the second to indicate the mode to open the shelve during unpickling. e.g. mode='nr': to create an empty shelve (writable), which in future will be re-opened read-only. Also, mode=None makes it first attempt to open read-only, but if the file does not exist will create it using mode 'c'. ''' # Mark this string as a file path. self.filename = classutil.SourceFileName(str(filename)) self.writeback = writeback if unpicklingMode or mode is None or mode == 'r': # Just use mode as given. self.mode = mode elif mode == 'n' or mode == 'c' or mode == 'w': # Ambiguous modes, warn & set default. if verbose: import sys print >>sys.stderr, '''Warning: you opened shelve file %s in mode '%s' but this is ambiguous for how the shelve should be re-opened later during unpickling. By default it will be re-opened in mode 'r' (read-only). To make it be re-opened writable, create it in mode '%sw', or call its method reopen('w'), which will make it be re-opened in mode 'w' now and in later unpickling. To suppress this warning message, use the verbose=False option.''' % (filename, mode, mode) self.mode = 'r' else: # PROCESS UNAMBIGUOUS TWO-LETTER mode STRING try: if len(mode) == 2 and mode[0] in 'ncwr' and mode[1] in 'cwr': self.mode = mode[1] # IN FUTURE OPEN IN THIS MODE mode = mode[0] # OPEN NOW IN THIS MODE else: raise ValueError('invalid mode string: ' + mode) except TypeError: raise ValueError('file mode must be a string!') if unpicklingMode: self.d = classutil.open_shelve(filename, mode, writeback, allowReadOnly=True) else: self.d = classutil.open_shelve(filename, mode, writeback) classutil.apply_itemclass(self, kwargs)
def __init__(self, filepath, reader=None, **kwargs): # make filepath a pickleable attribute. self.filepath = classutil.SourceFileName(str(filepath)) fullpath = self.filepath + '.seqlen' # build the seqLenDict if it doesn't already exist try: seqLenDict = classutil.open_shelve(fullpath, 'r') except NoSuchFileError: seqLenDict = self._create_seqLenDict(fullpath, filepath, reader) self.seqLenDict = seqLenDict self.seqInfoDict = _SeqLenDictWrapper(self) # standard interface # initialize base class. dbname = os.path.basename(filepath) SequenceDB.__init__(self, filepath=filepath, dbname=dbname, **kwargs)
def reopenReadOnly(self, mode="r"): "save existing data and reopen in read-only mode" self.close() self.seqIDdict = classutil.open_shelve(self.filename + ".seqIDdict", mode) self.IDdict = classutil.open_shelve(self.filename + ".idDict", mode)
def reopenReadOnly(self, mode='r'): 'save existing data and reopen in read-only mode' self.close() self.seqIDdict = classutil.open_shelve(self.filename + '.seqIDdict', mode) self.IDdict = classutil.open_shelve(self.filename + '.idDict', mode)
def reopen(self,mode='r'): 're-open shelve in the specified mode, and save mode on self' self.close() self.d = classutil.open_shelve(self.filename,mode,writeback=self.writeback) self.mode = mode
def reopenReadOnly(self,mode='r'): 'save existing data and reopen in read-only mode' self.close() self.seqIDdict = classutil.open_shelve(self.filename+'.seqIDdict',mode) self.IDdict = classutil.open_shelve(self.filename+'.idDict',mode)