def hashTableQ2():
answers = open('answers', 'a')
iValue = input("Input Value: ")
iValue = iValue.encode(encoding='UTF-8')
start_time_hDB = time.time()
HT_FILE = "/tmp/aredmond_db/hashtable"
try:
hDB = bsddb.hashopen(HT_FILE, "r")
except:
print("DB doesn't exist.")
return 0
hDB = bsddb.hashopen(HT_FILE, "w")
hNumRecords = 0
currentItem = hDB.first()
for i in range(1, len(hDB)):
if (currentItem[1] == iValue):
answers.write(currentItem[0].decode('UTF-8'))
answers.write("\n")
answers.write(currentItem[1].decode('UTF-8'))
answers.write("\n")
answers.write("\n")
hNumRecords += 1
currentItem = hDB.next()
try:
hDB.close()
answers.close()
except Exception as e:
print (e)
end_time_hDB = time.time()
print("Number of Records Retrieved: {}".format(hNumRecords))
print("--- {} microseconds ---".format((end_time_hDB - start_time_hDB) * 1000000))
def main():
try:
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "w")
print("DB opend without error.")
time.sleep(3)
except:
print("DB doesn't exist, creating a new one")
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "c")
print("DB created.")
time.sleep(1)
random.seed(SEED)
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print (key)
print (value)
print ("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
try:
db.close()
except Exception as e:
print (e)
def openDB(string):
if string == "btree":
db = bsddb.btopen(DA_FILE_B, "r")
elif string == "hash":
db = bsddb.hashopen(DA_FILE_H, "r")
elif string == "indexfile":
db = (bsddb.hashopen(DA_FILE_IH, "r"), bsddb.hashopen(DA_FILE_IRH, "r"), bsddb.btopen(DA_FILE_IB, "r"))
return db
def process(dbo, user, dbfiles, dump=False, doip=False, dousage=False):
count=0
usrlist={}
if type(dbfiles)==type(""):
dbfiles=[dbfiles]
for dbf in dbfiles:
if not user:
print "Scanning ",dbf
try:
db=bsddb3.hashopen(dbf, "r")
except bsddb3._db.DBOldVersionError,msg: # Old v1.85
d=bsddb3.db.DB()
d.upgrade(dbf) # upgrade on the fly
db=bsddb3.hashopen(dbf, "r")
fields=string.split(dbf,'.')
dstr=str(fields[2])
srv=str(fields[1])
try:
record=db.first()
while record:
count=count+1
(key, data) = record;
fields = string.split(data,chr(0))
sid,usr=string.split(key,':',1)
if dump==True:
if doip==True:
print "%d: %s:%s" % (count, key, fields[7])
else:
print "%d: %s" % (count, key)
if not usrlist.has_key(usr):
usrlist[usr]=[]
# Session details for a user may be scattered between
# Radius servers.
nkey=dstr + ':' + srv + ':' + key
#print nkey
usrlist[usr].append(nkey) # Save all keys, incl any dups
#if 0 and usr=="*****@*****.**":
if user and usr==user:
data=db[key]
fields = string.split(data,chr(0))
if dousage==True:
print "%s %s %d %d" % (dbf, user, int(fields[9])/1000000, int(fields[10])/1000000)
else:
print fields
#else:
# print usr,"=",fields
record=db.next()
except KeyError, msg:
pass
def populateDatabase():
global typeOption
global db
global db2
global dbPopFlag
if(dbPopFlag == True):
print("Error: database already populated, returning to menu.")
return
if(typeOption == "btree"):
try:
db = bsddb.btopen(DA_FILE, 'w')
except:
print("Error: Database does not exist, creating one.")
db = bsddb.btopen(DA_FILE, 'c')
elif(typeOption == "hash"):
try:
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "w")
except:
print("Error: Database does not exist, creating one.")
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "c")
elif(typeOption == "indexfile"):
try:
db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "w")
db2 = bsddb.db.DB()
db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
except:
print("Error: Database does not exist, creating one.")
db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "c")
db2 = bsddb.db.DB()
db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
random.seed(SEED)
print("Populating the database...")
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
while True:
for i in range(krng):
key += str(get_random_char())
if(db.has_key(key.encode()) == False):
break
else:
key = ""
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
key = key.encode(encoding="UTF-8")
value = value.encode(encoding="UTF-8")
db[key] = value
if typeOption == "indexfile":
db2[value] = key
# Tracking Database Population Progress
print('\b\b\b'+str(int((index/DB_SIZE)*100))+"%",end="")
dbPopFlag = True
print('\b\b\b\b', end="")
print("100%\nDatabase population complete.")
def createHashTable():
## Create and populate hash Table
print("Creating Hash Table")
try:
db = bsddb.hashopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.hashopen(DA_FILE, "c")
populate(db)
return db
def create_and_populate(data_type):
global db
global inverted_db
try:
if data_type==1:
db = bsddb.btopen(PATH, "w")
elif data_type==2:
db = bsddb.hashopen(PATH, "w")
elif data_type==3:
db = bsddb.btopen(PATH, "w")
inverted_db = bsddb.btopen(INVERTED_PATH, "w")
create_index_file()
except:
print("DB doesn't exist, creating a new one")
if data_type==1:
db = bsddb.btopen(PATH, "c")
elif data_type==2:
db = bsddb.hashopen(PATH, "c")
elif data_type==3:
db = bsddb.btopen(PATH, "c")
inverted_db = bsddb.btopen(INVERTED_PATH, "c")
SEED = 10000000
lib.set_seed(SEED)
index=0
while index<DB_SIZE:
krng = 64 + lib.get_random() % 64
key = ""
for i in range(krng):
key += str(chr(lib.get_random_char()))
vrng = 64 + lib.get_random() % 64
value = ""
for i in range(vrng):
value += str(chr(lib.get_random_char()))
#print(key)
#print(value)
#print("")
if data_type==3:
ALPHA=key[0]
#split_index=alpha.index(ALPHA)
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if db.has_key(key):
pass
else:
db[key] = value
index+=1
if data_type==3:
if inverted_db.has_key(value):
new_key=inverted_db[value].decode(encoding='UTF-8')
new_key+=(","+key.decode(encoding='UTF-8'))
inverted_db[value]=new_key.encode(encoding='UTF-8')
else:
inverted_db[value]=key
ALL_DB[ALPHA][key]=value
def makeHashTable(data):
try:
db = bsddb.hashopen(DA_FILE_H, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.hashopen(DA_FILE_H, "c")
for pair in data:
db[pair[0]] = pair[1]
return db
def indexFileCD():
data = open('data', 'a')
I1_FILE = "/tmp/aredmond_db/indexFile1"
I2_FILE = "/tmp/aredmond_db/indexFile2"
I3_FILE = "/tmp/aredmond_db/indexFile3"
try:
i1DB = bsddb.hashopen(I1_FILE, "w")
i2DB = bsddb.hashopen(I2_FILE, "w")
i3DB = bsddb.btopen(I3_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
i1DB = bsddb.hashopen(I1_File, "c")
i2DB = bsddb.hashopen(I2_FILE, "c")
i3DB = bsddb.btopen(I2_FILE, "c")
random.seed(SEED)
keySet = set()
valueSet = set()
for index in range(DB_SIZE):
while(True):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print ("Key: " + key)
data.write(key)
data.write("\n")
print ("Value: " + value)
data.write(value)
data.write("\n")
print ("")
data.write("\n")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if(not (key in keySet)):
keySet.add(key)
i1DB[key] = value
i3DB[key] = value
if(not (value in valueSet)):
valueSet.add(value)
i2DB[value] = key
else:
i2DB[value] = i2DB[value] + ";;;".encode('UTF-8') + key
break
try:
i1DB.close()
i2DB.close()
data.close()
except Exception as e:
print (e)
def ret_DATA(filetype):
if filetype == 'btree':
db = bsddb.btopen(DB_FILE, 'r')
elif filetype == 'hash':
db = bsddb.hashopen(DB_FILE, 'r')
elif filetype == 'indexfile':
db = bsddb.btopen(DB_FILE, 'r')
indexfile = bsddb.hashopen(SDB_FILE, 'r')
else:
print("Unknown type, function terminated\n")
return
# open answers for writing, appending to the end of the file
answers = open('answers', 'a')
result_lst = []
data = input("Enter the data you want to search > ")
data = data.encode(encoding = 'UTF-8')
start_time = time.time()
for key in db.keys():
if db[key] == data:
result_lst.append(key.decode(encoding = 'UTF-8'))
end_time = time.time()
elapse_time = (end_time - start_time) * 1000000
print("Result:")
data = data.decode(encoding = 'UTF-8')
if result_lst:
for key in result_lst:
print('Key:', key)
answers.write(key)
answers.write('\n')
print('Data:', data)
answers.write(data)
answers.write('\n')
answers.write('\n')
else:
print("Data not found")
print()
print(len(result_lst), "record(s) received")
print("Used", elapse_time, "micro seconds")
print()
answers.close()
db.close()
if filetype == 'indexfile':
indexfile.close()
return
def __init__(self, filename="background_model.counts.bdb"):
import bsddb3
self.all_counts = bsddb3.hashopen(filename, "c")
# self.all_counts.open(filename, pytc.BDBOCREAT|pytc.BDBOWRITER|pytc.BDBOREADER)
self.info = {"big_n": 0}
self.counts = {"unigram": self.all_counts, "bigram": self.all_counts}
def open_db3hash(*args):
"""Open a bsddb3 hash."""
import bsddb3
return bsddb3.hashopen(*args)
def __init__(self, type_option, key):
super(retrieve_1, self).__init__()
start_time = time.time()
self.value = None
if type_option == "btree" or type_option == "indexfile":
file = "/tmp/vanbelle_db/btree.db"
db = bsddb.btopen(file, "r")
try:
self.value = db[key.encode(encoding="UTF-8")]
try:
db.close()
except Exception as e:
print(e)
self.value = self.value.decode(encoding="UTF-8")
print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
except:
print("invalid key")
elif type_option == "hash":
file = "/tmp/vanbelle_db/hash.db"
db = bsddb.hashopen(file, "r")
try:
self.value = db[key.encode(encoding="UTF-8")]
try:
db.close()
except Exception as e:
print(e)
self.value = self.value.decode(encoding="UTF-8")
print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
except:
print("invalid key")
def indexFileQ1():
answers = open('answers', 'a')
iKey = input("Input Key: ")
iKey = iKey.encode(encoding='UTF-8')
start_time_iDB = time.time()
I1_FILE = "/tmp/aredmond_db/indexFile1"
try:
iDB = bsddb.hashopen(I1_FILE, "r")
except:
print("DB doesn't exist.")
return 0
iNumRecords = 0
if(iDB.has_key(iKey)):
location = iDB.set_location(iKey)
answers.write(location[0].decode('UTF-8'))
answers.write("\n")
answers.write(location[1].decode('UTF-8'))
answers.write("\n")
answers.write("\n")
iNumRecords += 1
try:
iDB.close()
answers.close()
except Exception as e:
print (e)
end_time_iDB = time.time()
print("Number of Records Retrieved: {}".format(iNumRecords))
print("--- {} microseconds ---".format((end_time_iDB - start_time_iDB) * 1000000))
def subfunchash2():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
print('Retrieve records with a given key')
Given_key = validity("Please select a number option from the options above: ", "Please select a valid option: ", 300, str)
Retrieved_data = ''
try:
db = bsddb.hashopen(DA_FILE,"r")
except:
print("Open method and file does not match to each other")
break
Given_key = Given_key.encode(encoding = 'UTF-8')
try:
db.has_key(Given_key)
start_time = time.time()
Retrieved_data = db[Given_key]
end_time = time.time()
Given_key = Given_key.decode(encoding = 'UTF-8')
Retrieved_data = Retrieved_data.decode(encoding = 'UTF-8')
time_used = end_time - start_time
time_used *= 1000000
db.close()
except:
print('Key is not found')
break
print('Key: %s\n'%Given_key+'Value: %s\n'%Retrieved_data)
print("The program runs %.6f micro seconds"%time_used)
select = validity("Do you want find another key? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
#os.system('cls' if os.name == 'nt' else 'clear')
#print()
def cvtdb(ctx, data, dbtype):
'''
Only used for testing purposes
'''
db = ctx.parent.params['db']
newdb = db + '.new'
if dbtype == 'gdbm':
import gdbm
new_d = gdbm.open(newdb, 'n')
elif dbtype == 'dbm':
import dbm
new_d = dbm.open(newdb, 'n')
elif dbtype == 'dbhash':
import dbhash
new_d = dbhash.open(newdb, 'n')
elif dbtype == 'bsddb':
new_d = bsddb.hashopen(newdb, 'n')
elif dbtype == 'dumbdbm':
import dumbdbm
new_d = dumbdbm.open(newdb, 'n')
else:
raise click.ClickException("Invalid type %s" % dbtype)
new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)
for k, v in data.iteritems():
new_data[k] = v
new_data.sync()
new_data.close()
def main(argv):
import getopt
def usage():
print 'usage: %s [-k|-v] path ...' % argv[0]
return 100
try:
(opts, args) = getopt.getopt(argv[1:], 'kv')
except getopt.GetoptError:
return usage()
def dispitem(k,v):
print '+%d,%d:%s->%s' % (len(k), len(v), k, v)
return
def dispkey(k,v):
print '+%d:%s' % (len(k), k)
return
def dispvalue(k,v):
print '+%d:%s' % (len(v), v)
return
func = dispitem
for (k,v) in opts:
if k == '-k':
func = dispkey
elif k == '-v':
func = diskvalue
if not args: return usage()
for path in args:
db = bsddb3.hashopen(path, 'r')
for (k,v) in db.iteritems():
func(k, v)
db.close()
return 0
def __init__(self):
self.answers = open("part1keyanswers.txt", "w")
self.db1Path = "part1BT"
self.db2Path = "part1HT"
self.db3Path = "indexfile"
self.db3 = db.DB()
self.db1MicroSeconds = []
self.db1MilliSeconds = []
self.db2MicroSeconds = []
self.db2MilliSeconds = []
self.db3MicroSeconds = []
self.db3MilliSeconds = []
self.db1 = bsddb3.btopen(self.db1Path, "c")
self.db2 = bsddb3.hashopen(self.db2Path, "c")
self.db3.open(self.db3Path, None, db.DB_HASH, db.DB_CREATE)
# self.db1.open(self.db1Path, None, db.DB_BTREE, db.DB_CREATE)
# self.db2.open(self.db2Path, None, db.DB_HASH, db.DB_CREATE)
# self.db3.open(self.db3Path, None, db.DB_BTREE, db.DB_CREATE)
self.mainMenu()
def __init__(self, dbpath, basehref, minlevel, maxlevel):
print >>sys.stderr, 'opening: %r' % dbpath
self.db = bsddb3.hashopen(dbpath)
self.basehref = basehref
self.minlevel = minlevel
self.maxlevel = maxlevel
self.session = requests.session()
return
def __init__(self, path):
self.corpus_path = path
self.chunks = {}
self.idx = rnopen(
os.path.join(self.corpus_path, Corpus.IDX_FILE)
) # FIXME: issue #2 this should be implemented with bsddb3 Queue
self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), "r"))
def __init__(self, fname='/tmp/geocache.bdb'):
try:
self.cache = bsddb3.hashopen(fname)
except:
self.cache = {}
try:
self.ncalls = int(self.cache.get('__ncalls__', '0'))
except:
self.ncalls = 0
def Data():
#search with given data
os.system('clear')
print("Please enter the Data: ")
stdin = input(">>")
if (DB_FLAG == "INDEX_FILE"):
try:
db_2 = bsddb.db.DB()
db_2.open(DB_FILE2)
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
if db_2.has_key(stdin.encode(encoding='UTF-8')):
records += 1
results(stdin,db_2.get(stdin.encode(encoding='UTF-8')).decode(encoding='UTF-8'))
end_time = time.time()
performance(records, (end_time-start_time))
time.sleep(2)
db_2.close()
return
elif(DB_FLAG == HASH):
try:
db_1 = bsddb.hashopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
elif(DB_FLAG == BTREE):
try:
db_1 = bsddb.btopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
for key, value in db_1.iteritems():
if (value == stdin.encode(encoding='UTF-8')):
results(key.decode(encoding='UTF-8'),stdin)
records += 1
end_time = time.time()
performance(records, (end_time-start_time))
time.sleep(2)
db_1.close()
def get_buddyicon(flickr, data, size=48):
"""Lookup the buddyicon from the data in @data using @flickr and resize it
to @size pixels."""
from twisted.web.client import getPage
import bsddb3
global __buddy_cache
if __buddy_cache is None:
folder = os.path.join (get_cache_path(), "postr")
if not os.path.exists(folder):
os.makedirs(folder)
path = os.path.join (folder, "buddyicons")
try:
__buddy_cache = bsddb3.hashopen(path, "c")
except bsddb3.db.DBInvalidArgError:
# The database needs upgrading, so delete it
os.remove(path)
__buddy_cache = bsddb3.hashopen(path, "c")
def load_thumb(page, size):
loader = gtk.gdk.PixbufLoader()
loader.set_size (size, size)
loader.write(page)
loader.close()
return loader.get_pixbuf()
def got_data(page, url, size):
__buddy_cache[url] = page
return load_thumb(page, size)
if int(data.get("iconfarm")) > 0:
url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
else:
url = "http://www.flickr.com/images/buddyicon.jpg"
if __buddy_cache.has_key(url):
from twisted.internet import defer
return defer.succeed(load_thumb(__buddy_cache[url], size))
else:
return getPage(url).addCallback(got_data, url, size)
def __init__(self,type_option,low,high):
super(retrieve_3,self).__init__()
start_time = time.time()
self.keys = []
if type_option == 'btree':
file = '/tmp/vanbelle_db/btree.db'
db = bsddb.btopen(file,'r')
for key, value in db.items():
key = key.decode(encoding='UTF-8')
value = value.decode(encoding='UTF-8')
if key >= low and key <= high:
self.keys.append((key,value))
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'hash':
file = '/tmp/vanbelle_db/hash.db'
db = bsddb.hashopen(file,'r')
for key, value in db.items():
key = key.decode(encoding='UTF-8')
value = value.decode(encoding='UTF-8')
if key >= low and key <= high:
self.keys.append((key,value))
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'indexfile':
file = '/tmp/vanbelle_db/index.db'
dc = bsddb.btopen(file,'r')
datas = dc.keys()
for i in datas:
i = i.decode(encoding='UTF-8')
if i >= low and i <= high:
self.keys.append(dc[i.encode(encoding='UTF-8')])
for i in range(len(self.keys)):
self.keys[i] = self.keys[i].decode(encoding='UTF-8')
try:
dc.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
def open_shelf(path):
'''
Opens a python shelf file, used to store various types of metadata
'''
# As of Exaile 4, new DBs will only be created as Berkeley DB Hash databases
# using either bsddb3 (external) or bsddb (stdlib but sometimes removed).
# Existing DBs created with other backends will be migrated to Berkeley DB.
# We do this because BDB is generally considered more performant,
# and because gdbm currently doesn't work at all in MSYS2.
# Some DBM modules don't use the path we give them, but rather they have
# multiple filenames. If the specified path doesn't exist, double check
# to see if whichdb returns a result before trying to open it with bsddb
force_migrate = False
if not os.path.exists(path):
from whichdb import whichdb
if whichdb(path) is not None:
force_migrate = True
if not force_migrate:
try:
db = bsddb.hashopen(path, 'c')
return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
except bsddb.db.DBInvalidArgError:
logger.warning("%s was created with an old backend, migrating it", path)
except Exception:
raise
# special case: zero-length file
if not force_migrate and os.path.getsize(path) == 0:
os.unlink(path)
else:
from xl.migrations.database.to_bsddb import migrate
migrate(path)
db = bsddb.hashopen(path, 'c')
return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
def __init__(self, type_option, data):
super(retrieve_2,self).__init__()
start_time = time.time()
self.keys = []
if type_option == 'btree':
self.file = '/tmp/vanbelle_db/btree.db'
db = bsddb.btopen(self.file,'r')
for key, value in db.items():
value = value.decode(encoding='UTF-8')
if data == value:
key = key.decode(encoding='UTF-8')
self.keys.append(key)
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'hash':
self.file = '/tmp/vanbelle_db/hash.db'
db = bsddb.hashopen(self.file,'r')
for key, value in db.items():
value = value.decode(encoding='UTF-8')
if data == value:
key = key.decode(encoding='UTF-8')
self.keys.append(key)
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'indexfile':
file = '/tmp/vanbelle_db/index.db'
dc = bsddb.btopen(file,'r')
try:
self.keys.append(dc[data.encode(encoding='UTF-8')])
for i in range(len(self.keys)):
self.keys[i] = self.keys[i].decode(encoding='UTF-8')
try:
dc.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
except:
print('invalid data value')
def create_db(db_type):
# The flag for opening 'w' seems to creating a new database
# even of there isn't one... which isn't what it's supposed to
# https://docs.python.org/2/library/bsddb.html
global db
# Open DB for binary tree
if(db_type == 'btree'):
if(os.path.isfile(db_fileType)):
print("Database already exists")
db = bsddb.btopen(db_fileType, "w")
return False
else:
db = bsddb.btopen(db_fileType, "w")
return True
#Open DB for hash
elif(db_type == 'hash'):
if(os.path.isfile(db_fileType)):
print("Database already exists")
db = bsddb.hashopen(db_fileType, "w")
return False
else:
db = bsddb.hashopen(db_fileType, "w")
return True
#Open Index file
elif(db_type == 'index'):
global db2
if(os.path.isfile(db_fileType) and os.path.isfile(db_fileType_2)):
print("Database already exists")
db = bsddb.btopen(db_fileType, "w")
db2 = bsddb.btopen(db_fileType_2, "w")
return False
else:
db = bsddb.btopen(db_fileType, "w")
db2 = bsddb.btopen(db_fileType_2, "w")
return True
def subfunchash():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
os.system('mkdir /tmp/my_db')
print('Create and Populate the database')
#main code start from here
time.time()
try:
db = bsddb.hashopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.hashopen(DA_FILE, "c")
random.seed(SEED)
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print (key)
print (value)
print ("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
try:
db.close()
print('Database created')
print("The program runs %.6f seconds"%time.clock())
select = validity("Do you want to repopulate the database? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
os.system('cls' if os.name == 'nt' else 'clear')
except Exception as e:
print (e)
def hashTableCD():
data = open('data', 'a')
HT_FILE = "/tmp/aredmond_db/hashtable"
try:
hDB = bsddb.hashopen(HT_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
hDB = bsddb.hashopen(HT_FILE, "c")
random.seed(SEED)
keySet = set()
for index in range(DB_SIZE):
while(True):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
data.write(key)
data.write("\n")
data.write(value)
data.write("\n")
data.write("\n")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if(not (key in keySet)):
keySet.add(key)
hDB[key] = value
break
try:
hDB.close()
data.close()
except Exception as e:
print (e)
def makeIndexFile(data):
try:
dbHash = bsddb.hashopen(DA_FILE_IH, "w")
dbRHash = bsddb.hashopen(DA_FILE_IRH, "w")
dbBTree = bsddb.btopen(DA_FILE_IB, "w")
except:
print("DB doesn't exist, creating a new one")
dbHash = bsddb.hashopen(DA_FILE_IH, "c")
dbRHashTree = bsddb.hashopen(DA_FILE_IRH, "c")
dbBTree = bsddb.btopen(DA_FILE_IB, "c")
for pair in data:
dbHash[pair[0]] = pair[1]
try:
exists = dbRHash.get(pair[1])
exists = exists + " " + pair[0]
except:
dbRHash[pair[1]] = pair[0]
dbBTree[pair[0]] = pair[1]
return (dbHash, dbRHash, dbBTree)
def db_put(key, value):
with open('mapping_db.lock', 'w') as lockfile:
fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX)
with closing(bsddb.hashopen('mapping_db', 'c')) as db:
db[codecs.encode(key, 'ascii')] = codecs.encode(value, 'ascii')
#!/usr/bin/env python
# encoding: utf-8
import json
import bsddb3 as bsddb
db = bsddb.hashopen('mouse.bdb')
for key in db:
j = json.loads(db[key + ""])
print j
def Retrieverecordswithagivendata(mode):
if mode == 1:
DA_FILE = "/tmp/weijie2_db/bt_db"
db = bsddb.btopen(DA_FILE, "r")
invertDA_FILE = "/tmp/weijie2_db/invert_bt_db"
invertdb = bsddb.btopen(invertDA_FILE, "r")
elif mode == 2:
DA_FILE = "/tmp/weijie2_db/hash_db"
db = bsddb.hashopen(DA_FILE, "r")
invertDA_FILE = "/tmp/weijie2_db/invert_hash_db"
invertdb = bsddb.hashopen(invertDA_FILE, "r")
elif mode == 3:
DA_FILE = "/tmp/weijie2_db/indexfile_db"
db = bsddb.btopen(DA_FILE, "r")
invertDA_FILE = "/tmp/weijie2_db/invert_indexfile_db"
invertdb = bsddb.btopen(invertDA_FILE, "r")
while True:
try:
if mode == 1:
value = input("Input a Valid value:").encode(encoding='UTF-8')
keyList = []
start = time.time()
try:
for key in db.keys():
if db[key] == value:
keyList.append(key.decode(encoding='UTF-8'))
except:
print("not found")
break
end = time.time()
print("Time:", 1000000 * (end - start), "micro seconds")
print("Receive", len(keyList), "records")
file = open("answers", "a")
for i in keyList:
print("Key is:", i)
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(value.decode(encoding='UTF-8') + "\n\n")
file.close()
db.close()
break
elif mode == 2:
value = input("Input a Valid value:").encode(encoding='UTF-8')
keyList = []
start = time.time()
try:
for key in db.keys():
if db[key] == value:
keyList.append(key.decode(encoding='UTF-8'))
except:
print("not found")
break
end = time.time()
print("Time :", 1000000 * (end - start), "micro seconds")
print("Receive", len(keyList), "records")
file = open("answers", "a")
for i in keyList:
print("Key is:", i)
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(value.decode(encoding='UTF-8') + "\n\n")
file.close()
db.close()
break
elif mode == 3:
value = input("Input a Valid value:").encode(encoding='UTF-8')
keyList = []
start = time.time()
try:
for key in db.keys():
if db[key] == value:
keyList.append(key.decode(encoding='UTF-8'))
except:
print("not found")
break
end = time.time()
print("Time:", 1000000 * (end - start), "micro seconds")
print("Receive", len(keyList), "records")
file = open("answers", "a")
for i in keyList:
print("Key is:", i)
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(value.decode(encoding='UTF-8') + "\n\n")
file.close()
db.close()
break
except Exception as e:
print(e)
return 0
def updateMovie():
while True:
name = raw_input("Movie name to be updated: ")
flag = True
index = -1
db = bsddb3.hashopen('movies.db', 'c')
for key in db.keys():
index = index + 1
movie = pickle.loads(db[key])
if movie.name == name:
while True:
print "1.Name\n2.Directors\n3.Actors\n4.Genres\n5.Run time\n6.Production House\n7.Country\n8.Release Date:\n9.Languages:\n"
field = int(
raw_input(
"Enter field to be updated(enter choice no. ): "))
if field == 1:
movie.name = raw_input("Enter updated movie name: ")
elif field == 5:
movie.runTime = int(
raw_input("Enter updated run time: "))
elif field == 6:
movie.productionHouse = raw_input(
"Enter updated productionHouse: ")
elif field == 7:
movie.country = raw_input("Enter updated country: ")
elif field == 3:
del movie.actors[:]
act_list = raw_input(
"Enter updated actors list(comma separated list): "
)
while act_list == "":
act_list = raw_input(
"List cannot be empty.Please enter: ")
for actor in act_list.split(','):
movie.actors.append(actor)
elif field == 2:
del movie.directors[:]
dir_list = raw_input(
"Enter updated directors list(comma separated list): "
)
while dir_list == "":
dir_list = raw_input(
"List cannot be empty.Please enter: ")
for director in dir_list.split(','):
movie.directors.append(director)
elif field == 9:
del movie.languages[:]
lan_list = raw_input(
"Enter updated languages list(comma separated list): "
)
while lan_list == "":
lan_list = raw_input(
"List cannot be empty.Please enter: ")
for language in lan_list.split(','):
movie.languages.append(language)
elif field == 4:
genres = raw_input(
"Select genre(comma separated list): adventure/drama/sci_fi "
)
while genres == "":
genres = raw_input(
"List cannot be empty.Plese enter. ")
del movie.genres[:]
for genre in genres.split(','):
if genre.upper() == "ADVENTURE":
movie.genres.append(movie_pb2.ADVENTURE)
elif genre.upper() == "SCI_FI":
movie.genres.append(movie_pb2.SCI_FI)
elif genre.upper() == "DRAMA":
movie.genres.append(movie_pb2.DRAMA)
elif field == 8:
date = raw_input("Release date(DD/MM/YYYY): ")
list_temp = date.split("/")
movie.releaseDate.day = int(list_temp[0])
mon = int(list_temp[1])
if mon == 1:
movie.releaseDate.month = movie_pb2.JANUARY
elif mon == 2:
movie.releaseDate.month = movie_pb2.FEBRUARY
elif mon == 3:
movie.releaseDate.month = movie_pb2.MARCH
elif mon == 4:
movie.releaseDate.month = movie_pb2.APRIL
elif mon == 5:
movie.releaseDate.month = movie_pb2.MAY
elif mon == 6:
movie.releaseDate.month = movie_pb2.JUNE
elif mon == 7:
movie.releaseDate.month = movie_pb2.JULY
elif mon == 8:
movie.releaseDate.month = movie_pb2.AUGUST
elif mon == 9:
movie.releaseDate.month = movie_pb2.SEPTEMBER
elif mon == 10:
movie.releaseDate.month = movie_pb2.OCTOBER
elif mon == 11:
movie.releaseDate.month = movie_pb2.NOVEMBER
elif mon == 12:
movie.releaseDate.month = movie_pb2.DECEMBER
movie.releaseDate.year = int(list_temp[2])
db[key] = pickle.dumps(movie)
flag = False
if raw_input(
"Do you want to update any other fields(or leave blank to finish): "
) == "":
break
break
if index == -1:
print "No movies to show"
break
if flag:
print "Movie not found.Continuing...."
if raw_input(
"Do you want to update another movie(or leave blank to finish): "
) == "":
break
def Receivetracekeyrange(mode):
while True:
try:
while True:
lower = input("Please enter your lower bound key:").encode(
encoding='UTF-8')
upper = input("Please enter your upper bound key:").encode(
encoding='UTF-8')
if lower > upper:
print("invalid input lower should before upper ")
else:
break
if mode == 1:
try:
DA_FILE = "/tmp/weijie2_db/bt_db"
db = bsddb.btopen(DA_FILE, "r")
except:
print("open error")
return 0
file = open("answers", "a")
start = time.time()
count = 0
for key in db.keys():
if key >= lower and key <= upper:
count += 1
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(db[key].decode(encoding='UTF-8') + "\n\n")
end = time.time()
print("There are totall", count, "records in this range.")
print("Time :", 1000000 * (end - start), "micro seconds")
file.close()
elif mode == 2:
try:
DA_FILE = "/tmp/weijie2_db/hash_db"
db = bsddb.hashopen(DA_FILE, "r")
except:
print("open error")
return 0
file = open("answers", "a")
start = time.time()
count = 0
for key in db.keys():
if key >= lower and key <= upper:
count += 1
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(db[key].decode(encoding='UTF-8') + "\n\n")
end = time.time()
print("There are totall", count, "records in this range.")
print("Time :", 1000000 * (end - start), "micro seconds")
file.close()
elif mode == 3:
try:
DA_FILE = "/tmp/weijie2_db/indexfile_db"
db = bsddb.btopen(DA_FILE, "r")
except:
print("open error")
return 0
file = open("answers", "a")
keys = db.keys()
begin = binarysearch(keys, lower)
finish = binarysearch(keys, upper)
count = 0
start = time.time()
while begin <= finish:
key = keys[begin]
count += 1
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(db[key].decode(encoding='UTF-8') + "\n\n")
begin += 1
end = time.time()
print("Time :", 1000000 * (end - start), "micro seconds")
print("There are", count, "records in this range.")
break
except Exception as e:
print(e)
def Createandpopulateadatabase(mode):
DB_SIZE = 100000
#DB_SIZE = 1000
SEED = 10000000
if mode == 1:
DA_FILE = "/tmp/weijie2_db/bt_db"
invertDA_FILE = "/tmp/weijie2_db/invert_bt_db"
try:
db = bsddb.btopen(DA_FILE, "w")
invertdb = bsddb.btopen(invertDA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
invertdb = bsddb.btopen(invertDA_FILE, "c")
random.seed(SEED)
elif mode == 2:
DA_FILE = "/tmp/weijie2_db/hash_db"
invertDA_FILE = "/tmp/weijie2_db/invert_hash_db"
try:
db = bsddb.hashopen(DA_FILE, "w")
invertdb = bsddb.hashopen(invertDA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.hashopen(DA_FILE, "c")
invertdb = bsddb.hashopen(invertDA_FILE, "c")
random.seed(SEED)
elif mode == 3:
DA_FILE = "/tmp/weijie2_db/indexfile_db"
invertDA_FILE = "/tmp/weijie2_db/invert_indexfile_db"
try:
db = bsddb.btopen(DA_FILE, "w")
invertdb = bsddb.btopen(invertDA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
invertdb = bsddb.btopen(invertDA_FILE, "c")
random.seed(SEED)
index = 0
while index < DB_SIZE:
k = 64 + random.randint(0, 63)
key = ""
for i in range(k):
key += str(chr(97 + random.randint(0, 25)))
v = 64 + random.randint(0, 63)
value = ""
for j in range(v):
value += str(chr(97 + random.randint(0, 25)))
print("key:", key, "\nvalue:", value, "\n")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
invertdb[value] = key
index += 1
try:
db.close()
invertdb.close()
except Exception as e:
print(e)
def Retrieverecordswithagivenkey(mode):
if mode == 1:
DA_FILE = "/tmp/weijie2_db/bt_db"
db = bsddb.btopen(DA_FILE, "w")
elif mode == 2:
DA_FILE = "/tmp/weijie2_db/hash_db"
db = bsddb.hashopen(DA_FILE, "w")
elif mode == 3:
DA_FILE = "/tmp/weijie2_db/indexfile_db"
#db = bsddb.btopen(DA_FILE,"w")
else:
print("Mode error")
return 1
while True:
try:
if mode == 1:
DA_FILE = "/tmp/weijie2_db/bt_db"
db = bsddb.btopen(DA_FILE, "r")
key = input("Input a Valid Key :").encode(encoding='UTF-8')
start = time.time()
try:
value = db[key]
print("found")
except:
print("Key does not exist")
end = time.time()
decodevalue = value.decode(encoding='UTF-8')
print("value is :", decodevalue)
print("Time:", 1000000 * (end - start), "micro seconds")
file = open("answers", "a")
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(decodevalue + "\n\n")
file.close()
break
elif mode == 2:
DA_FILE = "/tmp/weijie2_db/hash_db"
db = bsddb.hashopen(DA_FILE, "r")
key = input("Input a Valid Key :").encode(encoding='UTF-8')
start = time.time()
try:
value = db[key]
print("found")
except:
print("Key does not exist")
end = time.time()
decodevalue = value.decode(encoding='UTF-8')
print("value is :", decodevalue)
print("Time:", 1000000 * (end - start), "micro seconds")
file = open("answers", "a")
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(decodevalue + "\n\n")
file.close()
break
elif mode == 3:
DA_FILE = "/tmp/weijie2_db/indexfile_db"
db = bsddb.btopen(DA_FILE, "r")
key = input("Input a Valid Key :").encode(encoding='UTF-8')
start = time.time()
try:
value = db[key]
print("found")
except:
print("Key does not exist")
end = time.time()
decodevalue = value.decode(encoding='UTF-8')
print("value is :", decodevalue)
print("Time:", 1000000 * (end - start), "micro seconds")
file = open("answers", "a")
file.write(key.decode(encoding='UTF-8') + "\n")
file.write(decodevalue + "\n\n")
file.close()
break
db.close()
except Exception as e:
print(e)
def db_get(key):
with closing(bsddb.hashopen('mapping_db', 'c')) as db:
return codecs.decode(db.get(codecs.encode(key, 'ascii')), 'ascii')
def dbopen(path):
if sys.hexversion >= 0x03000000:
return bsddb.hashopen(path, 'c')
else: # pragma: no cover
return bsddb.hashopen(_encode(path), 'c')