def compact_db(self):
"""Compacts the DB by deleting any unused space (e.g., after deletion of files)
"""
assert self._loaded
if be_verbose:
print "[luggage.compact_db] Efficiency before: {:.3f}%".format(100 * self.get_db_usage_ratio())
tmp_optimized_path = self.luggage_path + "_optimized_rnd_{}_{}.raw".format(
random.randint(0, 10000000), random.randint(0, 10000000)
)
tmp_backup_path = self.luggage_path + "_backup_rnd_{}_{}.raw".format(
random.randint(0, 10000000), random.randint(0, 10000000)
)
try:
shutil.copyfile(self.luggage_path, tmp_backup_path)
optimized_db = dbm.open(tmp_optimized_path, "c")
for k, v in self.db.iteritems():
optimized_db[k] = v
optimized_db.sync()
optimized_db.close()
self.db.close()
shutil.copyfile(tmp_optimized_path, self.luggage_path)
self.db = dbm.open(self.luggage_path, "w")
if be_verbose:
print "[luggage.compact_db] Efficiency after: {:.3f}%".format(100 * self.get_db_usage_ratio())
except Exception as ex:
shutil.copyfile(tmp_backup_path, self.luggage_path)
raise Exception("[compact_db] Error: {}".format(str(ex)))
finally:
for p in [tmp_optimized_path, tmp_backup_path]:
if os.path.exists(p):
os.remove(p)
def __init__(self, dbname):
self.__dbname = dbname
try:
import dbhash
dbhash.open(dbname, 'c')
except:
shelve.open(dbname)
def _loadDict(request):
""" Load words from words files or cached dict """
# check for "dbhash" module
try:
import dbhash
except ImportError:
dbhash = None
# load the words
cachename = os.path.join(request.cfg.data_dir, 'cache', 'spellchecker.dict')
if dbhash and os.path.exists(cachename):
wordsdict = dbhash.open(cachename, "r")
else:
wordsfiles = _getWordsFiles(request)
if dbhash:
wordsdict = dbhash.open(cachename, 'n')
else:
wordsdict = {}
for wordsfile in wordsfiles:
_loadWordsFile(request, wordsdict, wordsfile)
if dbhash:
wordsdict.sync()
return wordsdict
def __init__(self, dbname):
self.__dbname=dbname
try:
import dbhash
dbhash.open(dbname, 'c')
except:
shelve.open(dbname)
def _loadDict(request):
""" Load words from words files or cached dict """
# check for "dbhash" module
try:
import dbhash
except ImportError:
dbhash = None
# load the words
cachename = os.path.join(request.cfg.data_dir, 'cache',
'spellchecker.dict')
if dbhash and os.path.exists(cachename):
wordsdict = dbhash.open(cachename, "r")
else:
wordsfiles = _getWordsFiles(request)
if dbhash:
wordsdict = dbhash.open(cachename, 'n')
else:
wordsdict = {}
for wordsfile in wordsfiles:
_loadWordsFile(request, wordsdict, wordsfile)
if dbhash:
wordsdict.sync()
return wordsdict
def cvtdb(ctx, data, dbtype):
'''
Only used for testing purposes
'''
db = ctx.parent.params['db']
newdb = db + '.new'
if dbtype == 'gdbm':
import gdbm
new_d = gdbm.open(newdb, 'n')
elif dbtype == 'dbm':
import dbm
new_d = dbm.open(newdb, 'n')
elif dbtype == 'dbhash':
import dbhash
new_d = dbhash.open(newdb, 'n')
elif dbtype == 'bsddb':
new_d = bsddb.hashopen(newdb, 'n')
elif dbtype == 'dumbdbm':
import dumbdbm
new_d = dumbdbm.open(newdb, 'n')
else:
raise click.ClickException("Invalid type %s" % dbtype)
new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)
for k, v in data.iteritems():
new_data[k] = v
new_data.sync()
new_data.close()
def load_patch(req,user,did,sid,clear_timeout):
""" Periodic patch stack read"""
import time
base='%s/cg'%__BASE__
req.content_type = 'text/plain'
stack = dbhash.open('%s/stack.db'%base,'c')
t = datetime.datetime.now()
d = '%s'%time.mktime(t.timetuple())
stack['_%s'%sid] = '%s:%s'%(d[:10],user)
if stack.has_key(did):
if not re.search(sid,stack[did]):
stack[did] += ':%s'%sid
else:
stack[did] = sid
for other in stack[did].split(':'):
if (other != sid):
if stack.has_key('_%s'%other):
if int(d[:10]) - int(stack['_%s'%other][:10]) > int(clear_timeout):
if stack.has_key(did):
stack[did] = re.sub(':$','',re.sub('%s:?'%other,'',stack[did]))
del stack['_%s'%other]
if stack.has_key(other):
del stack[other]
o = ''
if stack.has_key(sid):
o = stack[sid]
del stack[sid]
stack.close()
return o
def get_doc_list(user):
""" Create a new window id"""
base='%s/cg'%__BASE__
o = '<text class="list" onclick="open_doc(evt);" y="20">'
if os.path.isfile('%s/stack.db'%base) and os.path.isfile('%s/rev.db'%base):
rev = dbhash.open('%s/rev.db'%base)
stack = dbhash.open('%s/stack.db'%base)
if rev.has_key(user):
for did in rev[user].split(':'):
k = '_%s'%did
name = stack[k].split('\n')[0] if stack.has_key(k) else 'Untitled'
o += '<tspan x="10" dy="1.2em"><title>%s</title>%s</tspan>'%(did,name)
o += '<tspan class="small" x="100">1 user</tspan>'
stack.close()
rev.close()
return o + '</text>'
def cvtdb(ctx, data, dbtype):
'''
Only used for testing purposes
'''
db = ctx.parent.params['db']
newdb = db + '.new'
if dbtype == 'gdbm':
import gdbm
new_d = gdbm.open(newdb, 'n')
elif dbtype == 'dbm':
import dbm
new_d = dbm.open(newdb, 'n')
elif dbtype == 'dbhash':
import dbhash
new_d = dbhash.open(newdb, 'n')
elif dbtype == 'bsddb':
new_d = bsddb.hashopen(newdb, 'n')
elif dbtype == 'dumbdbm':
import dumbdbm
new_d = dumbdbm.open(newdb, 'n')
else:
raise click.ClickException("Invalid type %s" % dbtype)
new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)
for k, v in data.iteritems():
new_data[k] = v
new_data.sync()
new_data.close()
def print_db(db):
"""
"""
d = dbhash.open(db, 'r')
for item in d.keys():
print '[%s] %s -> %s' % (db,item,d[item])
d.close()
def print_db(db):
"""
"""
d = dbhash.open(db, 'r')
for item in d.keys():
print '[%s] %s -> %s' % (db, item, d[item])
d.close()
def main():
cache_file = None
want_ids = 0
ids = []
command = None
for arg in sys.argv[1:]:
if arg == "-h" or arg == "--help":
usage()
elif arg == "-C" or arg == "--channel":
if command is not None:
usage_error("Only one command option may be supplied")
command = "channel"
elif arg == "-L" or arg == "--list":
if command is not None:
usage_error("Only one command option may be supplied")
command = "list"
elif arg == "-K" or arg == "--keys":
if command is not None:
usage_error("Only one command option may be supplied")
command = "keys"
elif arg == "-I" or arg == "--item":
if command is not None:
usage_error("Only one command option may be supplied")
command = "item"
want_ids = 1
elif arg == "-H" or arg == "--hide":
if command is not None:
usage_error("Only one command option may be supplied")
command = "hide"
want_ids = 1
elif arg == "-U" or arg == "--unhide":
if command is not None:
usage_error("Only one command option may be supplied")
command = "unhide"
want_ids = 1
elif arg.startswith("-"):
usage_error("Unknown option:", arg)
else:
if cache_file is None:
cache_file = arg
elif want_ids:
ids.append(arg)
else:
usage_error("Unexpected extra argument:", arg)
if cache_file is None:
usage_error("Missing expected cache filename")
elif want_ids and not len(ids):
usage_error("Missing expected entry ids")
# Open the cache file directly to get the URL it represents
try:
db = dbhash.open(cache_file)
url = db["url"]
db.close()
except dbhash.bsddb._db.DBError, e:
print >>sys.stderr, cache_file + ":", e.args[1]
sys.exit(1)
def SetDev(dev):
db = dbhash.open(CFG_FILE, 'c')
db['dev'] = dev
db.close()
return True
def update_tool(req):
""" update the tool from github """
import datetime,time,dbhash,re
from subprocess import Popen, PIPE
(pwd, name,ip) = get_env(req)
t = datetime.datetime.now()
d = time.mktime(t.timetuple())
rev = dbhash.open('%s/cg/rev.db'%ui.__BASE__,'w')
allow,delta = False,0
if rev.has_key('_update_'):
delta = d - float(rev['_update_'])
if delta > 120:
rev['_update_'],allow = '%s'%d,True
if not rev.has_key('_update_'):
rev['_update_'] = '%s'%d
rev.close()
req.content_type = 'text/plain'
if not allow:
return 'Error: Time since last update is %d secondes; should be greater than 2 minutes!'%int(delta)
cmd = 'cd %s/..; rm -rf ConnectedGraph; git clone git://github.com/pelinquin/ConnectedGraph.git; cd ConnectedGraph; git submodule update --init'%pwd
out,err = Popen((cmd), shell=True,stdout=PIPE, stderr=PIPE).communicate()
o = 'Application Updating from %s %s...\n'%ui.get_id_pkg(req)
if err:
o += 'Error:%s\n'%err
else:
o += 'Message:%s\nUpdate OK\n'%out
return o
def __init__(self, planet, url):
if not os.path.isdir(planet.cache_directory):
os.makedirs(planet.cache_directory)
cache_filename = cache.filename(planet.cache_directory, url)
cache_file = dbhash.open(cache_filename, "c", 0666)
cache.CachedInfo.__init__(self, cache_file, url, root=1)
self._items = {}
self._planet = planet
self._expired = []
self.url = url
# retain the original URL for error reporting
self.configured_url = url
self.url_etag = None
self.url_status = None
self.url_modified = None
self.name = None
self.updated = None
self.last_updated = None
self.filter = None
self.exclude = None
self.next_order = "0"
self.relevant_tags = None
self.cache_read()
self.cache_read_entries()
if planet.config.has_section(url):
for option in planet.config.options(url):
value = planet.config.get(url, option)
self.set_as_string(option, value, cached=0)
def main():
print "Pickle is available."
db = dumbdbm.open("dumbdb", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("dumbdb")
if dbstr:
print "Dumbdbm is available."
else:
print "Dumbdbm is not available."
db = dbhash.open("dbhash", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("dbhash")
if dbstr == "dbhash":
print "Dbhash is available."
else:
print "Dbhash is not available."
if bsddb is None:
dbstr = ""
else:
db = bsddb.hashopen("bsddb3", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("bsddb3")
if dbstr == "dbhash":
print "Bsddb[3] is available."
else:
print "Bsddb[3] is not available."
print
hammie = get_pathname_option("Storage", "persistent_storage_file")
use_dbm = options["Storage", "persistent_use_database"]
if not use_dbm:
print "Your storage %s is a: pickle" % (hammie,)
return
if not os.path.exists(hammie):
print "Your storage file does not exist yet."
return
db_type = whichdb.whichdb(hammie)
if db_type == "dbhash":
# could be dbhash or bsddb3
# only bsddb3 has a __version__ attribute - old bsddb module does not
if hasattr(bsddb, '__version__'):
try:
db = bsddb.hashopen(hammie, "r")
except bsddb.error:
pass
else:
db.close()
print "Your storage", hammie, "is a: bsddb[3]"
return
elif db_type is None:
print "Your storage %s is unreadable." % (hammie,)
print "Your storage %s is a: %s" % (hammie, db_type)
def main():
print "Pickle is available."
db = dumbdbm.open("dumbdb", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("dumbdb")
if dbstr:
print "Dumbdbm is available."
else:
print "Dumbdbm is not available."
db = dbhash.open("dbhash", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("dbhash")
if dbstr == "dbhash":
print "Dbhash is available."
else:
print "Dbhash is not available."
if bsddb is None:
dbstr = ""
else:
db = bsddb.hashopen("bsddb3", "c")
db["1"] = "1"
db.close()
dbstr = whichdb.whichdb("bsddb3")
if dbstr == "dbhash":
print "Bsddb[3] is available."
else:
print "Bsddb[3] is not available."
print
hammie = get_pathname_option("Storage", "persistent_storage_file")
use_dbm = options["Storage", "persistent_use_database"]
if not use_dbm:
print "Your storage %s is a: pickle" % (hammie,)
return
if not os.path.exists(hammie):
print "Your storage file does not exist yet."
return
db_type = whichdb.whichdb(hammie)
if db_type == "dbhash":
# could be dbhash or bsddb3
# only bsddb3 has a __version__ attribute - old bsddb module does not
if hasattr(bsddb, '__version__'):
try:
db = bsddb.hashopen(hammie, "r")
except bsddb.error:
pass
else:
db.close()
print "Your storage", hammie, "is a: bsddb[3]"
return
elif db_type is None:
print "Your storage %s is unreadable." % (hammie,)
print "Your storage %s is a: %s" % (hammie, db_type)
def __init__(self, filename, flag='c'):
self.filename = filename
self.db = dbhash.open(self.filename, flag)
self.max_id = int(self.db.get('M_max_id', 0))
self.sort_by = self.sortByPositionAndFrequency
# TODO: consider requirement for loading stopwords here
# or speed it up
self.stopwords = [line.strip() for line in open(dolphy.STOPWORDS)]
def create_window_id():
""" Create a new window id"""
base='%s/cg'%__BASE__
rev = dbhash.open('%s/rev.db'%base,'c')
rev['@'] = '%d'%(long(rev['@'])+1) if rev.has_key('@') else '0'
sid = hashlib.sha1(rev['@']).hexdigest()[:-20]
rev.close()
return sid
def __init__(self, filename, flag='c'):
self.filename = filename
self.db = dbhash.open(self.filename, flag)
self.max_id = int(self.db.get('M_max_id', 0))
self.sort_by = self.sortByPositionAndFrequency
# TODO: consider requirement for loading stopwords here
# or speed it up
self.stopwords = [line.strip() for line in open(dolphy.STOPWORDS)]
def get_new_obj(dbpart):
traceroutetable = config['BDM_BSD_FILE']
try:
obj = db.open('%s-%s.db' % (traceroutetable, dbpart), 'w')
print 'opening %s-%s.db' % (traceroutetable, dbpart)
return obj
except:
return None
def get_new_obj(dbpart):
traceroutetable = config['BDM_BSD_FILE']
try:
obj = db.open('%s-%s.db'%(traceroutetable,dbpart),'w')
print 'opening %s-%s.db'%(traceroutetable,dbpart)
return obj
except:
return None
def GetDev():
db = dbhash.open(CFG_FILE, 'c')
if db.has_key('dev') == False:
db['dev'] = u''
dev = db['dev']
db.close()
return dev
def cacheEntries(cachefile=dnscachefile):
db=dbhash.open(cachefile,"r")
for key, value in dbseq(db):
hostlen=len(value)-12
when, numeric, foo, name = struct.unpack("III%ds"%hostlen, value)
name=name.rstrip('\x00')
if not numeric:
yield key, when, name
db.close()
def save_doc(user,did,title,lout,content):
""" Save the layout and the content of a diagram """
base = __BASE__
stack = dbhash.open('%s/stack.db'%base,'c')
stack['_%s'%did] = title + '\n' + lout + '\n' + content
k = '@%s'%did
if stack.has_key(k):
del stack[k]
stack.close()
return 'Saved'
def check_user(login,pw):
""" check if user registered with good password """
result, base = False, '%s/pw.db'%__BASE__
if login and pw:
if os.path.isfile(base):
db = dbhash.open(base)
if db.has_key(login):
if db[login] == hashlib.sha1(login+pw).hexdigest():
result = True
db.close()
return result
def open():
try:
cache = config.cache_directory()
index=os.path.join(cache,'index')
if not os.path.exists(index): return None
import dbhash
return dbhash.open(filename(index, 'id'),'w')
except Exception, e:
if e.__class__.__name__ == 'DBError': e = e.args[-1]
from planet import logger as log
log.error(str(e))
def change_pw_user(login,pw,pw2):
""" Change password for a registered user"""
result, base = False, '%s/pw.db'%__BASE__
if login and pw and pw2 and os.path.isfile(base):
db = dbhash.open(base,'c')
if db.has_key(login):
if db[login] == hashlib.sha1(login+pw).hexdigest():
if (len(pw2) > 4) and (pw2 != pw) and (pw2 != login):
db[login],result = hashlib.sha1(login+pw2).hexdigest(),True
db.close()
return result
def get_buddyicon(flickr, data, size=48):
"""Lookup the buddyicon from the data in @data using @flickr and resize it
to @size pixels."""
from twisted.web.client import getPage
import dbhash, bsddb
global __buddy_cache
if __buddy_cache is None:
folder = os.path.join(get_cache_path(), "postr")
if not os.path.exists(folder):
os.makedirs(folder)
path = os.path.join(folder, "buddyicons")
try:
__buddy_cache = dbhash.open(path, "c")
except bsddb.db.DBInvalidArgError:
# The database needs upgrading, so delete it
os.remove(path)
__buddy_cache = dbhash.open(path, "c")
def load_thumb(page, size):
loader = gtk.gdk.PixbufLoader()
loader.set_size(size, size)
loader.write(page)
loader.close()
return loader.get_pixbuf()
def got_data(page, url, size):
__buddy_cache[url] = page
return load_thumb(page, size)
if int(data.get("iconfarm")) > 0:
url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (
data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
else:
url = "http://www.flickr.com/images/buddyicon.jpg"
if __buddy_cache.has_key(url):
from twisted.internet import defer
return defer.succeed(load_thumb(__buddy_cache[url], size))
else:
return getPage(url).addCallback(got_data, url, size)
def new_doc(req,user):
""" utiliser user """
base='%s/cg'%__BASE__
req.content_type = 'text/plain'
rev = dbhash.open('%s/rev.db'%base,'c')
gid = create_id(rev)
if rev.has_key(user):
rev[user] += ':%s'%gid
else:
rev[user] = gid
rev.close()
return gid
def check_user(login,pw):
""" """
base='%s/cg/pw.db'%__BASE__
result = False
if login and pw:
if os.path.isfile(base):
db = dbhash.open(base)
if db.has_key(login):
if db[login] == hashlib.sha1(pw).hexdigest():
result = True
db.close()
return result
def get_buddyicon(flickr, data, size=48):
"""Lookup the buddyicon from the data in @data using @flickr and resize it
to @size pixels."""
from twisted.web.client import getPage
import dbhash, bsddb
global __buddy_cache
if __buddy_cache is None:
folder = os.path.join (get_cache_path(), "postr")
if not os.path.exists(folder):
os.makedirs(folder)
path = os.path.join (folder, "buddyicons")
try:
__buddy_cache = dbhash.open(path, "c")
except bsddb.db.DBInvalidArgError:
# The database needs upgrading, so delete it
os.remove(path)
__buddy_cache = dbhash.open(path, "c")
def load_thumb(page, size):
loader = gtk.gdk.PixbufLoader()
loader.set_size (size, size)
loader.write(page)
loader.close()
return loader.get_pixbuf()
def got_data(page, url, size):
__buddy_cache[url] = page
return load_thumb(page, size)
if int(data.get("iconfarm")) > 0:
url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
else:
url = "http://www.flickr.com/images/buddyicon.jpg"
if __buddy_cache.has_key(url):
from twisted.internet import defer
return defer.succeed(load_thumb(__buddy_cache[url], size))
else:
return getPage(url).addCallback(got_data, url, size)
def new_doc(user):
""" utiliser user """
gid = ''
if user:
base = __BASE__
rev = dbhash.open('%s/rev.db'%base,'c')
gid = create_id(rev)
if rev.has_key(user):
rev[user] += ':%s'%gid
else:
rev[user] = gid
rev.close()
return gid
def register(content=''):
""" If the same content is requested, then id does not change """
base = __BASE__
if not os.path.isdir(base):
os.mkdir(base)
rev = dbhash.open('%s/rev.db' % base, 'c')
if rev.has_key(content):
gid = rev[content]
else:
gid = create_id(rev)
rev[content] = gid
rev.close()
return gid
def register(content=''):
""" If the same content is requested, then id does not change """
base = __BASE__
if not os.path.isdir(base):
os.mkdir(base)
rev = dbhash.open('%s/rev.db'%base,'c')
if rev.has_key(content):
gid = rev[content]
else:
gid = create_id(rev)
rev[content] = gid
rev.close()
return gid
def open():
try:
cache = config.cache_directory()
index = os.path.join(cache, "index")
if not os.path.exists(index):
return None
return dbhash.open(filename(index, "id"), "w")
except Exception, e:
if e.__class__.__name__ == "DBError":
e = e.args[-1]
from planet import logger as log
log.error(str(e))
def dbhash_sample():
#打开一个文件，假如不存在则创建一个。
db = dbhash.open('dbhash.dat', 'c')
db['Tom'] = 'Beijing Road'
db['Jerry'] = 'Shanghai Road'
# db['Jim'] = 13 #出错，因为在dbhash模块中要求字典的键和值都必须是字符串。
for k, v in db.iteritems():
print k, v
if db.has_key('Tom'):
del db['Tom']
print db
db.close()
def feed(url):
fn = base64.b64encode(url)
fn = os.path.join(temp, fn)
now = datetime.date.today()
d = kvdb.open(os.path.join(temp, 'update'), 'c')
if d.has_key(url) and d[url] == str(now):
return fn
with open(fn, 'wb') as f:
rep = urllib.urlopen(url)
f.write(rep.read())
d[url] = str(now)
return fn
def create_new_luggage(luggage_path, password, pyluggage_config):
"""Create a new Luggage file and open it, returning an open instance
"""
# TODO: pyluggage_config -> apply config to LuggageParams call
if os.path.exists(luggage_path):
os.remove(luggage_path)
params = LuggageParams()
db = dbm.open(luggage_path, "c")
db[Luggage.params_db_name] = params.dumps()
db.close()
return Luggage(
luggage_path=luggage_path, password=password, pyluggage_config=pyluggage_config, is_creating=True
)
def export_day(day, conn):
db = dbhash.open("traceroutes--2014-01.db", "r")
for keys in db.keys():
key = cp.loads(keys)
value = cp.loads(db[keys])
retVal = run_insert_query_by_day(conn, key, value, day)
# if there is an error or we have gone past the correct day,
# then break out
# if retVal == False:
# break
conn.commit()
db.close()
def main():
"""Main function"""
# WRITE #######
db = dbhash.open('foo_dbhash', 'c')
db['one'] = 'un'
db['two'] = 'dos'
db['three'] = 'tres'
db.close()
# WHICH DBM ###
print "whichdb:", whichdb.whichdb('foo_dbhash')
print
# READ ########
db = dbhash.open('foo_dbhash', 'r')
# Iterate loop: first method (common to any dbm module)
for k in db.keys():
print k, ':', db[k]
# Iterate loop: second method (dbhash supports db.items())
for k, v in db.items():
print k, ':', v
# Iterate loop: third method (the most efficient)
# The following code prints every key in the database db, without having to create a list in memory that contains them all.
print db.first()
for i in xrange(1, len(db)):
print db.next()
db.close()
def main():
"""Main function"""
# WRITE #######
db = dbhash.open('foo_dbhash', 'c')
db['one'] = 'un'
db['two'] = 'dos'
db['three'] = 'tres'
db.close()
# WHICH DBM ###
print "whichdb:", whichdb.whichdb('foo_dbhash')
print
# READ ########
db = dbhash.open('foo_dbhash', 'r')
# Iterate loop: first method (common to any dbm module)
for k in db.keys():
print k, ':', db[k]
# Iterate loop: second method (dbhash supports db.items())
for k, v in db.items():
print k, ':', v
# Iterate loop: third method (the most efficient)
# The following code prints every key in the database db, without having to create a list in memory that contains them all.
print db.first()
for i in xrange(1, len(db)):
print db.next()
db.close()
def export_all(conn):
db = dbhash.open("traceroutes--2014-01.db", "r")
numAdded = 0
for keys in db.keys():
key = cp.loads(keys)
value = cp.loads(db[keys])
retVal = run_insert_query(conn, key, value)
numAdded += 1
if numAdded > 100:
conn.commit()
numAdded = 0
conn.commit()
db.close()
def create():
from planet import logger as log
cache = config.cache_directory()
index=os.path.join(cache,'index')
if not os.path.exists(index): os.makedirs(index)
import dbhash
index = dbhash.open(filename(index, 'id'),'c')
try:
import libxml2
except:
libxml2 = False
from xml.dom import minidom
for file in glob(cache+"/*"):
if os.path.isdir(file):
continue
elif libxml2:
try:
doc = libxml2.parseFile(file)
ctxt = doc.xpathNewContext()
ctxt.xpathRegisterNs('atom','http://www.w3.org/2005/Atom')
entry = ctxt.xpathEval('/atom:entry/atom:id')
source = ctxt.xpathEval('/atom:entry/atom:source/atom:id')
if entry and source:
index[filename('',entry[0].content)] = source[0].content
doc.freeDoc()
except:
log.error(file)
else:
try:
doc = minidom.parse(file)
doc.normalize()
ids = doc.getElementsByTagName('id')
entry = [e for e in ids if e.parentNode.nodeName == 'entry']
source = [e for e in ids if e.parentNode.nodeName == 'source']
if entry and source:
index[filename('',entry[0].childNodes[0].nodeValue)] = \
source[0].childNodes[0].nodeValue
doc.freeDoc()
except:
log.error(file)
log.info(str(len(index.keys())) + " entries indexed")
index.close()
return open()
#!/usr/local/bin/python2.5
import web, dbhash, sys
import simplejson as json
from openlibrary.catalog.load import add_keys
from copy import deepcopy
from openlibrary.catalog.merge.index import *
from urllib import urlopen, urlencode
path = '/1/edward/marc_index/'
#dbm_fields = ('lccn', 'oclc', 'isbn', 'title')
#dbm = dict((i, dbhash.open(path + i + '.dbm', flag='w')) for i in dbm_fields)
store_db = dbhash.open(path + "store.dbm", flag='w')
urls = (
# '/', 'index',
'/store/(.*)', 'store',
'/keys', 'keys',
)
app = web.application(urls, globals())
def build_pool(index_fields): # unused
pool = {}
for field in dbm_fields:
if not field in index_fields:
continue
for v in index_fields[field]:
if field == 'isbn' and len(v) < 10:
continue
try:
import dbhash
db = dbhash.open("dbhash", "c")
db["one"] = "the foot"
db["two"] = "the shoulder"
db["three"] = "the other foot"
db["four"] = "the bridge of the nose"
db["five"] = "the naughty bits"
db["six"] = "just above the elbow"
db["seven"] = "two inches to the right of a very naughty bit indeed"
db["eight"] = "the kneecap"
db.close()
db = dbhash.open("dbhash", "r")
for key in db.keys():
print repr(key), repr(db[key])
import os, re, sys, codecs, dbhash
from catalog.amazon.other_editions import find_others
from catalog.infostore import get_site
from catalog.read_rc import read_rc
from catalog.get_ia import get_data
from catalog.marc.build_record import build_record
from catalog.marc.fast_parse import get_tag_lines, get_all_subfields
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
rc = read_rc()
db = dbhash.open(rc['index_path'] + 'isbn_to_marc.dbm', 'r')
site = get_site()
def get_records_from_marc(isbn):
if isbn not in db:
return
# for loc in db[isbn].split(' '):
# data = get_data(loc)
# print loc
# want = ['100', '110', '111', '240', '245', '260'] + [str(i) for i in range(500,600) if i not in (505, 520)]
# for tag, line in get_tag_lines(data, set(want)):
# sub = list(get_all_subfields(line))
# if tag.startswith('5'):
# assert len(sub) == 1 and sub[0][0] == 'a'
# note = sub[0][1]
# if note.find('ublish') != -1 or note.find('riginal') != -1:
# print ' note:', note
from __future__ import print_function
import web
import dbhash
dbm = dbhash.open("/2/pharos/imagepdfs/oclc_to_marc.dbm", "r")
urls = ('/', 'index')
class index:
def GET(self):
web.header('Content-Type', 'text/html; charset=utf-8', unique=True)
input = web.input()
if 'oclc' in input:
html_oclc = web.htmlquote(input.oclc)
print("<html>\n<head><title>OCLC to MARC: %s</title><body>" %
html_oclc)
else:
print("<html>\n<head><title>OCLC to MARC</title><body>")
print('<form method="get">')
print('OCLC:')
if 'oclc' in input:
print('<input type="text" name="oclc" value="%s">' % html_oclc)
else:
print('<input type="text" name="oclc">')
print('<input type="submit" value="Find MARC">')
print('</form>')
if 'oclc' in input:
print('Searching for OCLC: %s<p>' % html_oclc)
def database(self):
import dbhash
if self.__database is None:
dbname = os.path.abspath(os.path.splitext(sys.argv[0])[0] + '.db')
self.__database = dbhash.open(dbname, 'w')
return self.__database
return ''
def insert_value(id, key, val):
return "INSERT INTO contacts VALUES (" + str(
id) + ",'" + key + "',\"" + val + "\");\n"
file = open("/tmp/contacts.sql", "w")
file.write("drop table contacts;\n")
file.write("drop table contacts_urn;\n")
file.write(
"create table contacts (urn INTEGER NOT NULL, tag TEXT NOT NULL, value TEXT NOT NULL);\n"
)
file.write("create table contacts_urn (urn INTEGER PRIMARY KEY);\n")
db = dbhash.open(
os.environ['HOME'] + "/evolution/local/Contacts/addressbook.db", "r")
x = 0
z = len(db.keys())
db.first()
while x < z - 1:
vcard_key = db.next()
vcard_data = open_vcard(vcard_key[1])
if vcard_data != '':
file.write("insert into contacts_urn values (" + str(x + 1) + ");\n")
file.write(
insert_value(x + 1, "NAME", get_vcard_data(vcard_data, "FN:")))
file.write(
insert_value(x + 1, "HOME.EMAIL",
get_vcard_data(vcard_data, "EMAIL;INTERNET:")))
file.write(
# test performance of serialisers and persisted hashes for dolphy
# TODO: try shelve (should be same as dbhash and cPickle)
# TODO: try low level bsdbb driver
# TODO: use more realistic hash
# TODO: compare index filesizes
import dbhash
import time
import marshal
import cPickle as pickle
DBFILE = 'test.db'
REPITITIONS = 1000
db = dbhash.open(DBFILE, flag='c')
print "=== Starting Test with %s repititions ===" % REPITITIONS
# using eval
d = "{'a':['one','two','three'],'b':[3,2,1], 'c':['this is a longer string']}"
start = time.time()
for i in range(0, REPITITIONS):
db[str(i)] = d
print "populated REPITITIONS dicts as strings in %.3f seconds" % (time.time() -
start)
start = time.time()
for i in range(0, REPITITIONS):
def run(args):
import config
if sys.version_info < (2, 5):
sys.path.append(os.path.join(config.progdir, 'py25modules'))
parser = BakefileOptionParser()
parser.add_option('-f',
'--format',
action="store",
dest='format',
help='format of generated makefile')
parser.add_option('-o',
'--output',
action="store",
dest='outfile',
help='output file')
parser.add_option('-D',
action="append",
dest='defines',
metavar='VAR=VALUE',
help='override variable or option definition')
parser.add_option('-I',
action="append",
dest='includes',
metavar='PATH',
help='search for bakefile rules in PATH')
parser.add_option('-v',
'--verbose',
action="store_true",
dest='verbose',
default=0,
help='display detailed information')
parser.add_option('-q',
'--quiet',
action="store_true",
dest='quiet',
default=0,
help='supress all output except of errors')
parser.add_option('',
'--dry-run',
action="store_true",
dest='dry_run',
default=0,
help="don't write any files, just pretend doing it")
parser.add_option(
'',
'--touch',
action="store_true",
dest='always_touch_output',
default=0,
help="always touch output files, even if their content doesn't change")
parser.add_option(
'',
'--eol',
default="format",
action="store",
dest='eol',
metavar='STYLE',
type='choice',
choices=['format', 'dos', 'unix', 'mac', 'native'],
help=
"line endings type to use in output files (format, dos, unix, mac, native) [default: format]"
)
parser.add_option(
'',
'--wrap-output',
default="75",
action="store",
dest='wrap_lines_at',
metavar='LENGTH',
help=
"don't generate lines longer than LENGTH; set to \"no\" to disable wrapping [default: 75]"
)
parser.add_option('',
'--output-deps',
action="store",
dest='deps_file',
metavar='DEPSFILE',
help="output dependencies information for bakefile_gen")
parser.add_option('',
'--output-changes',
action="store",
dest='changes_file',
metavar='MODSFILE',
help="output list of modified files to a file")
parser.add_option(
'',
'--xml-cache',
action="store",
dest='xml_cache',
metavar='CACHEFILE',
help="cache file where bakefile_gen stores pre-parsed XML files")
parser.add_option('',
'--debug',
action="store_true",
dest='debug',
default=0,
help="show debugging information (you don't want this)")
parser.add_option('',
'--dump',
action="store_true",
dest='dump',
default=0,
help='dump parsed makefile content instead of ' +
'generating makefile')
options, args = parser.parse_args(args)
if len(args) != 1:
parser.error('incorrect number of arguments, exactly 1 .bkl required')
config.dry_run = options.dry_run
config.always_touch_output = options.always_touch_output
config.debug = options.debug
config.quiet = options.quiet
if config.quiet:
config.verbose = 0
else:
config.verbose = options.verbose
if options.includes != None:
addIncludePaths(options.includes)
if options.xml_cache != None:
try:
# don't use shelve.open(), limit ourselves to dbhash version by
# using the Shelf class explicitly; that's because we store large
# chunks of data in the shelve and some dbm implementations can't
# handle it -- see http://www.bakefile.org/ticket/214
import pickle, shelve, dbhash
xmlparser.cache = shelve.Shelf(dbhash.open(options.xml_cache, 'c'),
protocol=pickle.HIGHEST_PROTOCOL)
except ImportError:
sys.stderr.write(
"Warning: disabling XML cache because it's not supported by this version of Python\n"
)
except AttributeError: # python < 2.3 didn't have HIGHEST_PROTOCOL
sys.stderr.write(
"Warning: disabling XML cache because it's not supported by Python 2.2\n"
)
except KeyError: # python < 2.3 didn't have protocol argument
sys.stderr.write(
"Warning: disabling XML cache because it's not supported by Python 2.2\n"
)
except Exception, e:
sys.stderr.write(
"Warning: disabling XML cache because an error occured while loading %s:\n %s\n"
% (options.xml_cache, e))
else:
if cache_file is None:
cache_file = arg
elif want_ids:
ids.append(arg)
else:
usage_error("Unexpected extra argument:", arg)
if cache_file is None:
usage_error("Missing expected cache filename")
elif want_ids and not len(ids):
usage_error("Missing expected entry ids")
# Open the cache file directly to get the URL it represents
try:
db = dbhash.open(cache_file)
url = db["url"]
db.close()
except dbhash.bsddb._db.DBError, e:
print >> sys.stderr, cache_file + ":", e.args[1]
sys.exit(1)
except KeyError:
print >> sys.stderr, cache_file + ": Probably not a cache file"
sys.exit(1)
# Now do it the right way :-)
my_planet = planet.Planet(ConfigParser.ConfigParser())
my_planet.cache_directory = os.path.dirname(cache_file)
channel = planet.Channel(my_planet, url)
for item_id in ids:
import dbhash
db = dbhash.open('dbhashtemp','c')
db['西施'] = '西施浣纱'
db['貂蝉'] = '貂蝉拜月'
db['昭君'] = '昭君出塞'
print "没有进行任何操作："
for k,v in db.iteritems():
print k,v
if db.has_key('西施'):
del db['西施']
print "删除西施对应的数据："
for k,v in db.iteritems():
print k,v
db.close()
# -*- coding:utf-8 -*-
author = 'haibo'
time = '2018-7-28'
"""
数据持久化
dbhash 只支持字符串类型的值
shelve 返回的字典值，支持：字符串、数字、元祖以及列表
"""
import dbhash
db = dbhash.open('tem', 'c')
db['西施'] = '西施浣纱'
db['貂蝉'] = '貂蝉拜月'
db['昭君'] = '昭君出塞'
print '--------------没有进行任何操作的数据---------------'
for k, v in db.iteritems():
print k, v
if db.has_key('西施'):
del db['西施']
print '------------删除键为‘西施’对应的数据---------------'
for k, v in db.iteritems():
print k, v
db.close()
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import dbhash
db = dbhash.open('temp', 'c') # 创建并打开数据库
db['Tom'] = 'Beijing road' # 写数据
db['Jerry'] = 'Shanghai road'
for k, v in db.iteritems(): # 遍历db对象
print k, v
if db.has_key('Tom'):
del db['Tom']
db['Tod'] = 2 # 非字符串的键或值会引发异常
print db
db.close() # 关闭数据库
outname = None
dbs = []
for arg in sys.argv[1:]:
if 'outdb' in arg:
outname = arg.split('=')[1]
else:
if os.path.exists(arg):
dbs.append(arg)
print('FOUND: %s' % arg)
else:
print('WARNING: %s was specified but not found' % arg)
if not outname:
raise Exception('must specify outdb=filename')
ndb = dbhash.open(outname, 'n')
outerdb = {}
for db in dbs:
print('Merging... %s' % db)
opendb = dbhash.open(db, 'r')
keys = opendb.keys()
for key in keys:
print('...Key: %s' % key)
outerdb.setdefault(key, {})
objects = pickle.loads(opendb[key])
for obj in objects.keys():
print('......Subkey: %s' % obj)
if outerdb[key].has_key(obj):
print('WARNING: key collision: %s dbase key: %s ' % (db, obj))
else:
