def __init__(self, path=None):
self.logging = logging.getLogger(self.__class__.__name__)
if not path:
fd, path = tempfile.mkstemp(".shelve", "octo_", text=False)
os.close(fd)
self.logging.debug("Shelve DB: %s", path)
self._db = bsddb.hashopen(path, 'n')
self._shelve = shelve.BsdDbShelf(self._db, protocol=2, writeback=True)
def open_shelf(path):
"""
Opens a python shelf file, used to store various types of metadata
"""
shelve_compat.ensure_shelve_compat()
# As of Exaile 4, new DBs will only be created as Berkeley DB Hash databases
# using either bsddb3 (external) or bsddb (stdlib but sometimes removed).
# Existing DBs created with other backends will be migrated to Berkeley DB.
# We do this because BDB is generally considered more performant,
# and because gdbm currently doesn't work at all in MSYS2.
# Some DBM modules don't use the path we give them, but rather they have
# multiple filenames. If the specified path doesn't exist, double check
# to see if whichdb returns a result before trying to open it with bsddb
force_migrate = False
if not os.path.exists(path):
from dbm import whichdb
if whichdb(path) is not None:
force_migrate = True
if not force_migrate:
try:
db = bsddb.hashopen(path, 'c')
return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
except bsddb.db.DBInvalidArgError:
logger.warning("%s was created with an old backend, migrating it", path)
except Exception:
raise
# special case: zero-length file
if not force_migrate and os.path.getsize(path) == 0:
os.unlink(path)
else:
from xl.migrations.database.to_bsddb import migrate
migrate(path)
db = bsddb.hashopen(path, 'c')
return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
def open(self, filename, mode='c'):
"""Open the given shelf as a BSDDB btree shelf
XXX patches bug in Python 2.3.x set_location for
bsddb objects as a side-effect
"""
if isinstance(filename, (str, unicode)):
filename = bsddb.btopen(filename, mode)
if sys.version >= '2.3':
# need to patch bug in 2.3's set_location
# XXX need to have a < as well once fixed!
bsddb._DBWithCursor.set_location = set_location
filename = shelve.BsdDbShelf(filename)
return filename
def open(self):
file_flag = lambda fname: 'c' if (not os.path.exists(fname)) else 'rw'
if (not self.isOpened):
if ((self.has_bsddb) and ((self.isPickleMethodUseStrings) or
(self.isPickleMethodUseMarshal) or
(self.isPickleMethodUseSafeSerializer))):
self.__db = bsddb.btopen(self.fileName,
file_flag(self.fileName))
self.__isOpened = True
elif (self.isPickleMethodUseBsdDbShelf):
self.__db = shelve.BsdDbShelf(
bsddb.btopen(self.fileName, file_flag(self.fileName)), 0,
True)
self.__isOpened = True
def __init__(self, name, reactor=reactor):
self.others = []
self.conn = []
self.votes = None
# self.LeaderFactory = LeaderClientFactory(self)
self.name = name
self.reactor = reactor
## Persistant state
self.current_term = 0
self.voted_for = None
self.log = None
# Attempt to open them from disk
db = bsddb.btopen('%s_log.db' % self.name, 'c')
self.log = shelve.BsdDbShelf(db)
if len(self.log) == 0:
self.log[str(1)] = (0, "START")
self.log.sync()
try:
self.current_term = int(file("%s_term.txt" % self.name).read())
self.voted_for = int(file("%s_voted.txt" % self.name).read())
except:
print "Cannot find persistant files"
## Ephemeral state
self.state = FOLLOWER
self.commit_index = 1
self.last_applied = None
## Leader State
self.next_index = None
self.match_index = None
## Timers
self.heartbeat_timeout = None
self.election_timeout = None
self.reset_election_timeout(delta=5)
def migrate(path):
shelve_compat.ensure_shelve_compat()
bak_path = path + os.extsep + 'tmp'
try:
old_shelf = shelve.open(path, 'r', protocol=common.PICKLE_PROTOCOL)
except Exception:
logger.warning("%s may be corrupt", path)
raise
db = common.bsddb.hashopen(bak_path, 'c')
new_shelf = shelve.BsdDbShelf(db, protocol=common.PICKLE_PROTOCOL)
for k, v in old_shelf.items():
new_shelf[k] = v
new_shelf.close()
old_shelf.close()
try:
os.replace(bak_path, path)
except Exception:
try:
os.unlink(bak_path)
except Exception:
pass
raise
# Various types of *dbm modules use more than one file to store the data.
# Now that we are assured that the migration is complete, delete them if
# present
for extra in glob.glob(path + os.extsep + '*'):
try:
os.unlink(extra)
except Exception as e:
logger.warning("Could not delete %s: %s", extra, e)
logger.info("Migration successfully completed!")
def _open_dbs():
global pokervals6_db, pokervals7_db
import bsddb
pokervals6_db = shelve.BsdDbShelf(bsddb.hashopen('pokervals6.shelf', 'r'))
pokervals7_db = shelve.BsdDbShelf(bsddb.hashopen('pokervals7.shelf', 'r'))
import sys, operator
import shelve
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
import bsddb3
from collections import defaultdict
K = int(sys.argv[1])
output_file = sys.argv[2]
transit_file = sys.argv[3]
bins = shelve.BsdDbShelf(bsddb3.hashopen('bins.data', 'r'))
#bins = shelve.open('bins.data','r')
out = open(output_file, 'w')
keys = [int(key) for key in bins]
keys.sort()
for key in keys:
key = str(key)
print "Evaluating ", key, " ..."
sorted_top = sorted(bins[key].iteritems(),
key=operator.itemgetter(1),
reverse=True)[:K]
total = sum(v for _, v in sorted_top)
sorted_top = map(lambda tup: (tup[0], float(tup[1]) / total), sorted_top)
out.write('%10d\t' % (20 * int(key)))
out.write('\t'.join('%10s' % i for i, _ in sorted_top) + '\n')
out.write('%10s\t' % "")
out.write('\t'.join('%10.5f' % i for _, i in sorted_top) + '\n')
def LinuxToDarwin(linux_file_name, darwin_file_name):
linux = shelve.open(linux_file_name)
darwin = shelve.BsdDbShelf(bsddb.btopen(darwin_file_name, 'c'))
CopyAll(linux, darwin)
darwin.close()
linux.close()
def DarwinToLinux(darwin_file_name, linux_file_name):
darwin = shelve.BsdDbShelf(bsddb.btopen(darwin_file_name, 'r'))
linux = shelve.open(linux_file_name)
CopyAll(darwin, linux)
linux.close()
darwin.close()