def __init__(self, path=None): self.logging = logging.getLogger(self.__class__.__name__) if not path: fd, path = tempfile.mkstemp(".shelve", "octo_", text=False) os.close(fd) self.logging.debug("Shelve DB: %s", path) self._db = bsddb.hashopen(path, 'n') self._shelve = shelve.BsdDbShelf(self._db, protocol=2, writeback=True)
def open_shelf(path): """ Opens a python shelf file, used to store various types of metadata """ shelve_compat.ensure_shelve_compat() # As of Exaile 4, new DBs will only be created as Berkeley DB Hash databases # using either bsddb3 (external) or bsddb (stdlib but sometimes removed). # Existing DBs created with other backends will be migrated to Berkeley DB. # We do this because BDB is generally considered more performant, # and because gdbm currently doesn't work at all in MSYS2. # Some DBM modules don't use the path we give them, but rather they have # multiple filenames. If the specified path doesn't exist, double check # to see if whichdb returns a result before trying to open it with bsddb force_migrate = False if not os.path.exists(path): from dbm import whichdb if whichdb(path) is not None: force_migrate = True if not force_migrate: try: db = bsddb.hashopen(path, 'c') return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL) except bsddb.db.DBInvalidArgError: logger.warning("%s was created with an old backend, migrating it", path) except Exception: raise # special case: zero-length file if not force_migrate and os.path.getsize(path) == 0: os.unlink(path) else: from xl.migrations.database.to_bsddb import migrate migrate(path) db = bsddb.hashopen(path, 'c') return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
def open(self, filename, mode='c'): """Open the given shelf as a BSDDB btree shelf XXX patches bug in Python 2.3.x set_location for bsddb objects as a side-effect """ if isinstance(filename, (str, unicode)): filename = bsddb.btopen(filename, mode) if sys.version >= '2.3': # need to patch bug in 2.3's set_location # XXX need to have a < as well once fixed! bsddb._DBWithCursor.set_location = set_location filename = shelve.BsdDbShelf(filename) return filename
def open(self): file_flag = lambda fname: 'c' if (not os.path.exists(fname)) else 'rw' if (not self.isOpened): if ((self.has_bsddb) and ((self.isPickleMethodUseStrings) or (self.isPickleMethodUseMarshal) or (self.isPickleMethodUseSafeSerializer))): self.__db = bsddb.btopen(self.fileName, file_flag(self.fileName)) self.__isOpened = True elif (self.isPickleMethodUseBsdDbShelf): self.__db = shelve.BsdDbShelf( bsddb.btopen(self.fileName, file_flag(self.fileName)), 0, True) self.__isOpened = True
def __init__(self, name, reactor=reactor): self.others = [] self.conn = [] self.votes = None # self.LeaderFactory = LeaderClientFactory(self) self.name = name self.reactor = reactor ## Persistant state self.current_term = 0 self.voted_for = None self.log = None # Attempt to open them from disk db = bsddb.btopen('%s_log.db' % self.name, 'c') self.log = shelve.BsdDbShelf(db) if len(self.log) == 0: self.log[str(1)] = (0, "START") self.log.sync() try: self.current_term = int(file("%s_term.txt" % self.name).read()) self.voted_for = int(file("%s_voted.txt" % self.name).read()) except: print "Cannot find persistant files" ## Ephemeral state self.state = FOLLOWER self.commit_index = 1 self.last_applied = None ## Leader State self.next_index = None self.match_index = None ## Timers self.heartbeat_timeout = None self.election_timeout = None self.reset_election_timeout(delta=5)
def migrate(path): shelve_compat.ensure_shelve_compat() bak_path = path + os.extsep + 'tmp' try: old_shelf = shelve.open(path, 'r', protocol=common.PICKLE_PROTOCOL) except Exception: logger.warning("%s may be corrupt", path) raise db = common.bsddb.hashopen(bak_path, 'c') new_shelf = shelve.BsdDbShelf(db, protocol=common.PICKLE_PROTOCOL) for k, v in old_shelf.items(): new_shelf[k] = v new_shelf.close() old_shelf.close() try: os.replace(bak_path, path) except Exception: try: os.unlink(bak_path) except Exception: pass raise # Various types of *dbm modules use more than one file to store the data. # Now that we are assured that the migration is complete, delete them if # present for extra in glob.glob(path + os.extsep + '*'): try: os.unlink(extra) except Exception as e: logger.warning("Could not delete %s: %s", extra, e) logger.info("Migration successfully completed!")
def _open_dbs(): global pokervals6_db, pokervals7_db import bsddb pokervals6_db = shelve.BsdDbShelf(bsddb.hashopen('pokervals6.shelf', 'r')) pokervals7_db = shelve.BsdDbShelf(bsddb.hashopen('pokervals7.shelf', 'r'))
import sys, operator import shelve import matplotlib.pyplot as plt import matplotlib.cm as cm import numpy as np import bsddb3 from collections import defaultdict K = int(sys.argv[1]) output_file = sys.argv[2] transit_file = sys.argv[3] bins = shelve.BsdDbShelf(bsddb3.hashopen('bins.data', 'r')) #bins = shelve.open('bins.data','r') out = open(output_file, 'w') keys = [int(key) for key in bins] keys.sort() for key in keys: key = str(key) print "Evaluating ", key, " ..." sorted_top = sorted(bins[key].iteritems(), key=operator.itemgetter(1), reverse=True)[:K] total = sum(v for _, v in sorted_top) sorted_top = map(lambda tup: (tup[0], float(tup[1]) / total), sorted_top) out.write('%10d\t' % (20 * int(key))) out.write('\t'.join('%10s' % i for i, _ in sorted_top) + '\n') out.write('%10s\t' % "") out.write('\t'.join('%10.5f' % i for _, i in sorted_top) + '\n')
def LinuxToDarwin(linux_file_name, darwin_file_name): linux = shelve.open(linux_file_name) darwin = shelve.BsdDbShelf(bsddb.btopen(darwin_file_name, 'c')) CopyAll(linux, darwin) darwin.close() linux.close()
def DarwinToLinux(darwin_file_name, linux_file_name): darwin = shelve.BsdDbShelf(bsddb.btopen(darwin_file_name, 'r')) linux = shelve.open(linux_file_name) CopyAll(darwin, linux) linux.close() darwin.close()