def testUnreadableDB(self):
tmp_dir = tempfile.mkdtemp(dir=".", prefix="unreadable")
pckInfo = None
try:
dbFile = os.path.join(tmp_dir, "test.db")
bsddb3.btopen(dbFile).close()
os.chmod(dbFile, 0)
connector = copy.copy(self.connector)
connector.checksumDbPath = dbFile
pckname = "unreadable-db-%.f" % time.time()
pck = connector.Packet(pckname, time.time())
pck.AddJob("cat data.txt")
with tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".txt", mode="w") as f:
for _ in range(100):
print(_, file=f)
f.flush()
pck.AddFiles({"data.txt": f.name})
connector.Queue(TestingQueue.Get()).AddPacket(pck)
pckInfo = connector.PacketInfo(pck.id)
self.assertEqual(WaitForExecution(pckInfo), "SUCCESSFULL")
finally:
if pckInfo:
pckInfo.Delete()
shutil.rmtree(tmp_dir)
def key_search(type_option, created):
global answerfile
if not created:
print("Please Create the Database first!!")
input("Press Enter to Continue....")
return
#Search with given key
if type_option == "btree" :
db = bsddb.btopen(DB_FILE_btree, "r")
elif type_option == 'hash' :
db = bsddb.hashopen(DB_FILE_hash, "r")
else:
db = bsddb.btopen(DB_FILE_index1, "r")
key = input("Please input the key that you want to search: ")
key = key.encode(encoding='UTF-8')
answer = 0
start = time.time()
if db.has_key(key):
answer = 1
answerfile.write("Key: "+str(key.decode(encoding='UTF-8'))+"\n")
answerfile.write("Data: "+str(db.get(key).decode(encoding='UTF-8'))+"\n")
answerfile.write("\n")
answerfile.flush()
end = time.time()
duration = (end - start) * 1000000
print ("Time Used :",duration, "microseconds")
print ("Total number of the searched key is :",answer)
input ("Press Enter to Continue...")
db.close()
return
def load_table(i=conf.FN_I_TABLE, n=conf.FN_N_TABLE):
trunc=conf.TRUNC_LIMIT #This has to match the db
cur_path = os.path.dirname(os.path.abspath(__file__)) + "/../" + conf.DB_DIR + "/"
db_i_table = bsddb.btopen(cur_path + i, 'c')
db_n_table = bsddb.btopen(cur_path + n, 'c')
return db_i_table, db_n_table
def main():
try:
db = bsddb.btopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
random.seed(SEED)
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print (key)
print (value)
print ("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
try:
db.close()
except Exception as e:
print (e)
def __init__(self,
ads: str,
terms: str,
pdates: str,
prices: str,
output: str = "brief"):
"""Initialize a query engine by providing paths to the ``ads``,
``terms``, ``pdates``, and ``prices`` indexes"""
# create a tempdir and make copies of all the indexes
self.temp_dir = tempfile.gettempdir()
ads = create_temporary_copy(self.temp_dir, ads)
self.ads = bsddb3.hashopen(ads)
__log__.debug("loaded ads index: {}".format(self.ads))
terms = create_temporary_copy(self.temp_dir, terms)
self.terms = bsddb3.btopen(terms)
__log__.debug("loaded terms index: {}".format(self.terms))
pdates = create_temporary_copy(self.temp_dir, pdates)
self.pdates = bsddb3.btopen(pdates)
__log__.debug("loaded pdates index: {}".format(self.pdates))
prices = create_temporary_copy(self.temp_dir, prices)
self.prices = bsddb3.btopen(prices)
__log__.debug("loaded prices index: {}".format(self.prices))
if output == "full":
self.full_output = True
elif output == "brief":
self.full_output = False
else:
raise ValueError("Invalid argument for output: {}".format(output))
def main():
try:
db = bsddb.btopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
random.seed(SEED)
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print(key)
print(value)
print("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
try:
db.close()
except Exception as e:
print(e)
def index():
try:
db = bsddb.btopen(DA_FILE, "w")
inverted_db = bsddb.btopen(INVERTED_PATH, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
inverted_db = bsddb.btopen(INVERTED_PATH, "c")
random.seed(SEED)
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print(key)
print(value)
print("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
if inverted_db.has_key(value):
new_key = inverted_db[value].decode(encoding='UTF-8')
new_key += ("," + key.decode(encoding='UTF-8'))
inverted_db[value] = new_key.encode(encoding='UTF-8')
else:
inverted_db[value] = key
try:
db.close()
inverted_db.close()
except Exception as e:
print(e)
def index():
global db
try:
db = bsddb.btopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
lib.set_seed(SEED)
counter = 0
for index in range(DB_SIZE):
krng = 64 + lib.get_random() % 64
key = ""
for i in range(krng):
key += str(chr(lib.get_random_char()))
vrng = 64 + lib.get_random() % 64
value = ""
for i in range(vrng):
value += str(chr(lib.get_random_char()))
print(key)
print(value)
print("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if db.has_key(key):
pass
else:
db[key] = value
counter += 1
print(counter, ' records created------------------------')
try:
db.close()
except Exception as e:
print(e)
def openDB(string):
if string == "btree":
db = bsddb.btopen(DA_FILE_B, "r")
elif string == "hash":
db = bsddb.hashopen(DA_FILE_H, "r")
elif string == "indexfile":
db = (bsddb.btopen(DA_FILE_IB, "r"), bsddb.btopen(DA_FILE_IRB, "r"))
return db
def load_table(i=conf.FN_I_TABLE, n=conf.FN_N_TABLE):
trunc = conf.TRUNC_LIMIT #This has to match the db
cur_path = os.path.dirname(
os.path.abspath(__file__)) + "/../" + conf.DB_DIR + "/"
db_i_table = bsddb.btopen(cur_path + i, 'c')
db_n_table = bsddb.btopen(cur_path + n, 'c')
return db_i_table, db_n_table
def openDB(string):
if string == "btree":
db = bsddb.btopen(DA_FILE_B, "r")
elif string == "hash":
db = bsddb.hashopen(DA_FILE_H, "r")
elif string == "indexfile":
db = (bsddb.btopen(DA_FILE_IB, "r"), bsddb.btopen(DA_FILE_IRB, "r"))
return db
def __init__(self, base_name, policy_module):
self.policy_module = policy_module
self.base_name = base_name
self.found_links_db_name = base_name + self.__class__.FOUND_LINKS_DB + str(uuid.uuid4())
self.found_links = bsddb.btopen(self.found_links_db_name)
self.priority_queue_db_name = base_name + self.__class__.PRIORITY_QUEUE_DB + str(uuid.uuid4())
self.priority_queue = bsddb.btopen(self.priority_queue_db_name)
def populateDatabase():
global typeOption
global db
global db2
global dbPopFlag
if(dbPopFlag == True):
print("Error: database already populated, returning to menu.")
return
if(typeOption == "btree"):
try:
db = bsddb.btopen(DA_FILE, 'w')
except:
print("Error: Database does not exist, creating one.")
db = bsddb.btopen(DA_FILE, 'c')
elif(typeOption == "hash"):
try:
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "w")
except:
print("Error: Database does not exist, creating one.")
db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "c")
elif(typeOption == "indexfile"):
try:
db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "w")
db2 = bsddb.db.DB()
db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
except:
print("Error: Database does not exist, creating one.")
db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "c")
db2 = bsddb.db.DB()
db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
random.seed(SEED)
print("Populating the database...")
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
while True:
for i in range(krng):
key += str(get_random_char())
if(db.has_key(key.encode()) == False):
break
else:
key = ""
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
key = key.encode(encoding="UTF-8")
value = value.encode(encoding="UTF-8")
db[key] = value
if typeOption == "indexfile":
db2[value] = key
# Tracking Database Population Progress
print('\b\b\b'+str(int((index/DB_SIZE)*100))+"%",end="")
dbPopFlag = True
print('\b\b\b\b', end="")
print("100%\nDatabase population complete.")
def createBTree():
## Create and populate B Tree (see initialLabCode.py for original lab code)
print("Creating B Tree")
try:
db = bsddb.btopen(DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
populate(db)
return db
def makeBTree(data):
try:
db = bsddb.btopen(DA_FILE_B, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE_B, "c")
for pair in data:
db[pair[0]] = pair[1]
return db
def makeBTree(data):
try:
db = bsddb.btopen(DA_FILE_B, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE_B, "c")
for pair in data:
db[pair[0]] = pair[1]
return db
def make_graph(dbdir):
childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)
tokens = load_token_map(tnum2tokdb)
digraph, nodes = build_graph(parentdb)
parse_prov(provdb, tokens, nodes)
graph_join(digraph, nodes)
return digraph, nodes
def make_graph(dbdir):
childdb = bsddb.btopen(dbdir + "/" + CHILD_DB)
parentdb = bsddb.btopen(dbdir + "/" + PARENT_DB)
provdb = bsddb.btopen(dbdir + "/" + PROV_DB)
tnum2tokdb = bsddb.rnopen(dbdir + "/" + TNUM2TOK_DB)
tokens = load_token_map(tnum2tokdb)
digraph, nodes = build_graph(parentdb)
parse_prov(provdb, tokens, nodes)
graph_join(digraph, nodes)
return digraph, nodes
def __init__(self, resource_prefix):
"""
Init the knowledge resource
:param resource_prefix - the resource directory and file prefix
"""
# TODO check to make sure files exist
self.term_to_id = bsddb3.btopen(str(resource_prefix + '_term_to_id.db'), 'r')
self.id_to_term = bsddb3.btopen(str(resource_prefix + '_id_to_term.db'), 'r')
self.path_to_id = bsddb3.btopen(str(resource_prefix + '_path_to_id.db'), 'r')
self.id_to_path = bsddb3.btopen(str(resource_prefix + '_id_to_path.db'), 'r')
self.l2r_edges = bsddb3.btopen(str(resource_prefix + '_l2r.db'), 'r')
def indexFileCD():
data = open('data', 'a')
I1_FILE = "/tmp/aredmond_db/indexFile1"
I2_FILE = "/tmp/aredmond_db/indexFile2"
I3_FILE = "/tmp/aredmond_db/indexFile3"
try:
i1DB = bsddb.hashopen(I1_FILE, "w")
i2DB = bsddb.hashopen(I2_FILE, "w")
i3DB = bsddb.btopen(I3_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
i1DB = bsddb.hashopen(I1_File, "c")
i2DB = bsddb.hashopen(I2_FILE, "c")
i3DB = bsddb.btopen(I2_FILE, "c")
random.seed(SEED)
keySet = set()
valueSet = set()
for index in range(DB_SIZE):
while(True):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print ("Key: " + key)
data.write(key)
data.write("\n")
print ("Value: " + value)
data.write(value)
data.write("\n")
print ("")
data.write("\n")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if(not (key in keySet)):
keySet.add(key)
i1DB[key] = value
i3DB[key] = value
if(not (value in valueSet)):
valueSet.add(value)
i2DB[value] = key
else:
i2DB[value] = i2DB[value] + ";;;".encode('UTF-8') + key
break
try:
i1DB.close()
i2DB.close()
data.close()
except Exception as e:
print (e)
def __init__(self,type_option,low,high):
super(retrieve_3,self).__init__()
start_time = time.time()
self.keys = []
if type_option == 'btree':
file = '/tmp/vanbelle_db/btree.db'
db = bsddb.btopen(file,'r')
for key, value in db.items():
key = key.decode(encoding='UTF-8')
value = value.decode(encoding='UTF-8')
if key >= low and key <= high:
self.keys.append((key,value))
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'hash':
file = '/tmp/vanbelle_db/hash.db'
db = bsddb.hashopen(file,'r')
for key, value in db.items():
key = key.decode(encoding='UTF-8')
value = value.decode(encoding='UTF-8')
if key >= low and key <= high:
self.keys.append((key,value))
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'indexfile':
file = '/tmp/vanbelle_db/index.db'
dc = bsddb.btopen(file,'r')
datas = dc.keys()
for i in datas:
i = i.decode(encoding='UTF-8')
if i >= low and i <= high:
self.keys.append(dc[i.encode(encoding='UTF-8')])
for i in range(len(self.keys)):
self.keys[i] = self.keys[i].decode(encoding='UTF-8')
try:
dc.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
def __init__(self, type_option, data):
super(retrieve_2,self).__init__()
start_time = time.time()
self.keys = []
if type_option == 'btree':
self.file = '/tmp/vanbelle_db/btree.db'
db = bsddb.btopen(self.file,'r')
for key, value in db.items():
value = value.decode(encoding='UTF-8')
if data == value:
key = key.decode(encoding='UTF-8')
self.keys.append(key)
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'hash':
self.file = '/tmp/vanbelle_db/hash.db'
db = bsddb.hashopen(self.file,'r')
for key, value in db.items():
value = value.decode(encoding='UTF-8')
if data == value:
key = key.decode(encoding='UTF-8')
self.keys.append(key)
try:
db.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
elif type_option == 'indexfile':
file = '/tmp/vanbelle_db/index.db'
dc = bsddb.btopen(file,'r')
try:
self.keys.append(dc[data.encode(encoding='UTF-8')])
for i in range(len(self.keys)):
self.keys[i] = self.keys[i].decode(encoding='UTF-8')
try:
dc.close()
except Exception as e:
print (e)
print('this function retrived %s records' %len(self.keys))
print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
except:
print('invalid data value')
def ret_DATA(filetype):
if filetype == 'btree':
db = bsddb.btopen(DB_FILE, 'r')
elif filetype == 'hash':
db = bsddb.hashopen(DB_FILE, 'r')
elif filetype == 'indexfile':
db = bsddb.btopen(DB_FILE, 'r')
indexfile = bsddb.hashopen(SDB_FILE, 'r')
else:
print("Unknown type, function terminated\n")
return
# open answers for writing, appending to the end of the file
answers = open('answers', 'a')
result_lst = []
data = input("Enter the data you want to search > ")
data = data.encode(encoding = 'UTF-8')
start_time = time.time()
for key in db.keys():
if db[key] == data:
result_lst.append(key.decode(encoding = 'UTF-8'))
end_time = time.time()
elapse_time = (end_time - start_time) * 1000000
print("Result:")
data = data.decode(encoding = 'UTF-8')
if result_lst:
for key in result_lst:
print('Key:', key)
answers.write(key)
answers.write('\n')
print('Data:', data)
answers.write(data)
answers.write('\n')
answers.write('\n')
else:
print("Data not found")
print()
print(len(result_lst), "record(s) received")
print("Used", elapse_time, "micro seconds")
print()
answers.close()
db.close()
if filetype == 'indexfile':
indexfile.close()
return
def create_index_file():
global ALL_DB
try:
os.system("mkdir /tmp/yufei_db/index_file")
except:
pass
INDEX_PATH = "/tmp/yufei_db/index_file/db"
ALL_DB = dict()
for i in range(26):
try:
db = bsddb.btopen(INDEX_PATH + str(i), "w")
except:
db = bsddb.btopen(INDEX_PATH + str(i), "c")
ALL_DB[alpha[i]] = db
def create_index_file():
global ALL_DB
try:
os.system("mkdir /tmp/yufei_db/index_file")
except:
pass
INDEX_PATH="/tmp/yufei_db/index_file/db"
ALL_DB=dict()
for i in range(26):
try:
db = bsddb.btopen(INDEX_PATH+str(i), "w")
except:
db = bsddb.btopen(INDEX_PATH+str(i), "c")
ALL_DB[alpha[i]]=db
def bt_key_search():
temp_key = str(input("Please enter your key:"))
temp_data = ''
try:
db = bsddb.btopen(DA_FILE, "r")
except:
print(
"Open method and file type does not match,delete current and make a btre then come back!"
)
return main()
temp_key = temp_key.encode(encoding='UTF-8')
try:
start = time.time()
temp_data = db[temp_key]
end = time.time()
temp_key = temp_key.decode(encoding='UTF-8')
temp_data = temp_data.decode(encoding='UTF-8')
time_used = end - start
time_used *= 1000000
print('KEY:')
fin.write(temp_key + '\n')
print(temp_key)
fin.write(temp_data + '\n\n')
print('DATA:')
print(temp_data)
print('')
print(time_used, 'micro seconds used')
db.close()
except:
print('Key does not exist,go to main')
db.close()
return btree_interface()
def index_data_search():
temp_data = str(input("Please enter your data:"))
temp_key = ''
try:
db = bsddb.btopen(DA_FILE, "r")
except:
print(
"Open method and file type does not match,delete current and make a btre then come back!"
)
return main()
temp_data = temp_data.encode(encoding='UTF-8')
try:
start = time.time()
for temp_key in db.keys():
if db[temp_key] == temp_data:
print('KEY:\n' + temp_key.decode(encoding='UTF-8'))
fin.write(temp_key.decode(encoding='UTF-8') + '\n')
print('DATA:\n' + temp_data.decode(encoding='UTF-8') + '\n')
fin.write(temp_data.decode(encoding='UTF-8') + '\n\n')
end = time.time()
time_used = end - start
time_used *= 1000000
print(time_used, 'micro seconds used')
db.close()
except:
print('Data does not exist, go to main')
db.close()
return index_interface()
def testVerifyFailure(self):
# create a map
m = shadow.ShadowMap()
s = shadow.ShadowMapEntry()
s.name = 'foo'
self.assertTrue(m.Add(s))
updater = nssdb.NssDbShadowHandler({
'dir': self.workdir,
'makedb': '/usr/bin/makedb'
})
written = updater.Write(m)
self.assertTrue(os.path.exists(updater.temp_cache_filename),
'updater.Write() did not create a file')
# change the cache
db = btopen(updater.temp_cache_filename)
del db[db.first()[0]]
db.sync()
db.close()
retval = updater.Verify(written)
self.assertEqual(False, retval)
self.assertFalse(
os.path.exists(os.path.join(updater.temp_cache_filename)))
def __init__(self):
self.answers = open("part1keyanswers.txt", "w")
self.db1Path = "part1BT"
self.db2Path = "part1HT"
self.db3Path = "indexfile"
self.db3 = db.DB()
self.db1MicroSeconds = []
self.db1MilliSeconds = []
self.db2MicroSeconds = []
self.db2MilliSeconds = []
self.db3MicroSeconds = []
self.db3MilliSeconds = []
self.db1 = bsddb3.btopen(self.db1Path, "c")
self.db2 = bsddb3.hashopen(self.db2Path, "c")
self.db3.open(self.db3Path, None, db.DB_HASH, db.DB_CREATE)
# self.db1.open(self.db1Path, None, db.DB_BTREE, db.DB_CREATE)
# self.db2.open(self.db2Path, None, db.DB_HASH, db.DB_CREATE)
# self.db3.open(self.db3Path, None, db.DB_BTREE, db.DB_CREATE)
self.mainMenu()
def bt_data_search():
temp_data = str(input("Please enter your data:"))
temp_key = ''
try:
inverse_db = bsddb.btopen(INVERSE_DA_FILE, "r")
except:
print(
"Open method and file type does not match,delete current and make a btre then come back!"
)
return main()
temp_data = temp_data.encode(encoding='UTF-8')
if True:
start = time.time()
temp_key = inverse_db[temp_data]
end = time.time()
temp_key = temp_key.decode(encoding='UTF-8')
temp_data = temp_data.decode(encoding='UTF-8')
time_used = end - start
time_used *= 1000000
print('KEY:')
temp_key = temp_key.split(':')
print(temp_key)
for keys in temp_key:
fin.write(keys + '\n')
fin.write(temp_data + '\n\n')
print('DATA:')
print(temp_data)
print('')
print(time_used, 'micro seconds used')
inverse_db.close()
else:
print('Data does not exist, go to main')
inverse_db.close()
return btree_interface()
def index_range_search():
try:
db = bsddb.btopen(DA_FILE, "r")
except:
print(
"Open method and file type does not match,delete current and make a btre then come back!"
)
return main()
upper = str(input("Enter your upper key:"))
lower = str(input("Enter your lower key:"))
if upper < lower:
print("Upper lessthan lower, go to main")
return main()
elif upper == lower:
temp_key = lower
temp_data = ''
temp_key = temp_key.encode(encoding='UTF-8')
try:
start = time.time()
temp_data = db[temp_key]
end = time.time()
temp_key = temp_key.decode(encoding='UTF-8')
temp_data = temp_data.decode(encoding='UTF-8')
time_used = end - start
time_used *= 1000000
print('Equal bounds\nKEY:')
fin.write(temp_key + '\n')
print(temp_key + '\n')
print('DATA:')
fin.write(temp_data + '\n\n')
print(temp_data)
print('')
print(time_used, 'micro seconds used')
db.close()
except:
print('Key does not exist,go to main')
db.close()
return index_interface()
else:
lower = lower.encode(encoding='UTF-8')
upper = upper.encode(encoding='UTF-8')
counter = 0
try:
start = time.time()
for key, data in db.items():
if key <= upper and key >= lower:
counter += 1
print("KEY:\n" + key.decode(encoding='UTF-8') +
"\nDATA:\n" + data.decode(encoding='UTF-8') + "\n")
fin.write(key.decode(encoding='UTF-8') + '\n')
fin.write(data.decode(encoding='UTF-8') + "\n\n")
end = time.time()
time_used = end - start
print(counter, " records displayed")
time_used *= 1000000
print(time_used, "micro seconds used")
except:
print('Key does not exist,go to main')
db.close()
return index_interface()
def bTreeQ2():
answers = open('answers', 'a')
iValue = input("Input Value: ")
iValue = iValue.encode(encoding='UTF-8')
start_time_bDB = time.time()
BT_FILE = "/tmp/aredmond_db/btree"
try:
bDB = bsddb.btopen(BT_FILE, "r")
except:
print("DB doesn't exist.")
return 0
bNumRecords = 0
currentItem = bDB.first()
for i in range(1, len(bDB)):
if (currentItem[1] == iValue):
answers.write(currentItem[0].decode('UTF-8'))
answers.write("\n")
answers.write(currentItem[1].decode('UTF-8'))
answers.write("\n")
answers.write("\n")
bNumRecords += 1
currentItem = bDB.next()
try:
bDB.close()
answers.close()
except Exception as e:
print (e)
end_time_bDB = time.time()
print("Number of Records Retrieved: {}".format(bNumRecords))
print("--- {} microseconds ---".format((end_time_bDB - start_time_bDB) * 1000000))
def __init__(self, type_option, key):
super(retrieve_1, self).__init__()
start_time = time.time()
self.value = None
if type_option == "btree" or type_option == "indexfile":
file = "/tmp/vanbelle_db/btree.db"
db = bsddb.btopen(file, "r")
try:
self.value = db[key.encode(encoding="UTF-8")]
try:
db.close()
except Exception as e:
print(e)
self.value = self.value.decode(encoding="UTF-8")
print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
except:
print("invalid key")
elif type_option == "hash":
file = "/tmp/vanbelle_db/hash.db"
db = bsddb.hashopen(file, "r")
try:
self.value = db[key.encode(encoding="UTF-8")]
try:
db.close()
except Exception as e:
print(e)
self.value = self.value.decode(encoding="UTF-8")
print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
except:
print("invalid key")
def handle_db(db_file):
"""Load db."""
try:
db = bsddb3.btopen(db_file, "r")
except Exception:
die("Cannon open {0}".format(db_file))
return db
def setPersistConfig(self, persistFile=None):
self.persist = bool(persistFile)
if self.persist:
self.pdb = bsddb3.btopen(persistFile,'c')
for value in self.pdb.values():
task = pickle.loads(value)
heappush(self.queue, task)
def _LoadBdbCacheFile(self, data):
"""Load data from bdb caches into a map.
Args:
data: a map.Map subclass
Returns:
Nothing. Cache data is loaded into the 'data' parameter.
Raises:
CacheNotFound: if the database file does not exist
"""
db_file = os.path.join(self.output_dir, self.CACHE_FILENAME)
if not os.path.exists(db_file):
self.log.debug('cache file does not exist: %r', db_file)
raise error.CacheNotFound('cache file does not exist: %r' %
db_file)
db = bsddb3.btopen(db_file, 'r')
for k in db:
if self.IsMapPrimaryKey(k):
password_entry = self.ConvertValueToMapEntry(db[k])
if not data.Add(password_entry):
self.log.warning('could not add entry built from %r',
db[k])
db.close()
def subfunc2():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
print('Retrieve records with a given key')
Given_key = validity("Please enter your key: ",
"Please select a valid key: ", 300, str)
Retrieved_data = ''
try:
db = bsddb.btopen(DA_FILE, "r")
except:
print("Open method and file does not match to each other")
break
Given_key = Given_key.encode(encoding='UTF-8')
try:
start_time = time.time()
Retrieved_data = db[Given_key]
end_time = time.time()
Given_key = Given_key.decode(encoding='UTF-8')
Retrieved_data = Retrieved_data.decode(encoding='UTF-8')
time_used = end_time - start_time
time_used *= 1000000
db.close()
except:
print('Key is not found')
break
print('Key: %s\n' % Given_key + 'Value: %s\n' % Retrieved_data)
print("The program runs %.6f micro seconds" % time_used)
select = validity("Do you want find another key? y/n: ",
"Please enter a valid option: ", 1, str, ['y', 'n'],
'lower')
def bTreeQ1():
answers = open('answers', 'a')
iKey = input("Input Key: ")
iKey = iKey.encode(encoding='UTF-8')
start_time_bDB = time.time()
BT_FILE = "/tmp/aredmond_db/btree"
try:
bDB = bsddb.btopen(BT_FILE, "r")
except:
print("DB doesn't exist.")
return 0
bNumRecords = 0
if(bDB.has_key(iKey)):
location = bDB.set_location(iKey)
answers.write(location[0].decode('UTF-8'))
answers.write("\n")
answers.write(location[1].decode('UTF-8'))
answers.write("\n")
answers.write("\n")
bNumRecords += 1
try:
bDB.close()
answers.close()
except Exception as e:
print (e)
end_time_bDB = time.time()
print("Number of Records Retrieved: {}".format(bNumRecords))
print("--- {} microseconds ---".format((end_time_bDB - start_time_bDB) * 1000000))
def subfunc2():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
print('Retrieve records with a given key')
Given_key = validity("Please enter your key: ", "Please select a valid key: ", 300, str)
Retrieved_data = ''
try:
db = bsddb.btopen(DA_FILE,"r")
except:
print("Open method and file does not match to each other")
break
Given_key = Given_key.encode(encoding = 'UTF-8')
try:
start_time = time.time()
Retrieved_data = db[Given_key]
end_time = time.time()
Given_key = Given_key.decode(encoding = 'UTF-8')
Retrieved_data = Retrieved_data.decode(encoding = 'UTF-8')
time_used = end_time - start_time
time_used *= 1000000
db.close()
except:
print('Key is not found')
break
print('Key: %s\n'%Given_key+'Value: %s\n'%Retrieved_data)
print("The program runs %.6f micro seconds"%time_used)
select = validity("Do you want find another key? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
#os.system('cls' if os.name == 'nt' else 'clear')
#print()
def chain_bdb_index(inchain, outbdb):
current_chain = 0
chain_data = defaultdict(list)
f = open(inchain, "r")
sys.stderr.write("Making chain_id: chain_data dict...\n")
for line in f:
if line.startswith("#"):
continue
elif line.startswith("c"):
chain_id = line[:-1].split()[-1]
current_chain = chain_id
# sys.stderr.write("Chain: {0}\r".format(current_chain))
elif current_chain == 0:
continue
chain_data[current_chain].append(line)
f.close()
if len(chain_data.keys()) == 0:
sys.stderr.write("(chain_bdb_index.py) Error! Input file {0} is empty! Aborted.\n".format(inchain))
sys.exit(1)
db = bsddb3.btopen(outbdb, "w")
sys.stderr.write("Writing to BDB...\n")
for k, v in chain_data.items():
db_key = k.encode()
chain_data = "".join(v)[:-1].encode()
db[db_key] = chain_data
db.close()
def subfuncid1():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
os.system('mkdir /tmp/my_db/')
print('Create and Populate the database')
#main code start from here
try:
db = bsddb.btopen(DA_FILE, "w")
inverse_db = bsddb.btopen(INVERSE_DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
inverse_db = bsddb.btopen(INVERSE_DA_FILE, "c")
random.seed(SEED)
start_time = time.time()
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print(key)
print(value)
print("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
inverse_db[value] = key
end_time = time.time()
time_used = end_time - start_time
time_used *= 1000000
try:
db.close()
inverse_db.close()
except Exception as e:
print(e)
print('Database created')
print("The program runs %.6f micro seconds" % time_used)
select = validity("Do you want to repopulate the database? y/n: ",
"Please enter a valid option: ", 1, str, ['y', 'n'],
'lower')
os.system('cls' if os.name == 'nt' else 'clear')
def btree():
global db
global inverse_db
try:
db = bsddb.btopen(DA_FILE, "w")
inverse_db = bsddb.btopen(INVERSE_DA_FILE, "w")
except:
print("DB doesn't exist, creating a new one")
inverse_db = bsddb.btopen(DA_FILE, "c")
db = bsddb.btopen(INVERSE_DA_FILE, "c")
counter = 0
lib.set_seed(SEED)
for index in range(DB_SIZE):
krng = 64 + lib.get_random() % 64
key = ""
for i in range(krng):
key += str(chr(lib.get_random_char()))
vrng = 64 + lib.get_random() % 64
value = ""
for i in range(vrng):
value += str(chr(lib.get_random_char()))
print(key)
print(value)
print("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
if db.has_key(key):
pass
else:
db[key] = value
counter += 1
if inverse_db.has_key(value):
original_key = inverse_db[value].decode(encoding='UTF-8')
key = key.decode(encoding='UTF-8')
if key != original_key:
key = original_key + ':' + key
key = key.encode(encoding='UTF-8')
inverse_db[value] = key
else:
inverse_db[value] = key
print(counter, ' records created------------------------')
try:
db.close()
inverse_db.close()
except Exception as e:
print(e)
def __init__(self, path, serializer=msgpack):
''' Initialize a new instance of the BTree class.
:param path: The path to the database on disk
:param serializer: The serializer to use for advanced types
'''
self.database = bsddb3.btopen(path, 'n')
self.serializer = serializer
def check_time(senddb, timelimit, username, to):
with closing(bsddb.btopen(senddb, 'c')) as db:
key = username + ":" + to
if key in db:
timestamp = int(db[key])
if timestamp + timelimit > int(time.time()):
return True
return False
def UpdateContext(self, context):
self.db_file = context.tags_db_file
self.infile_items = bsddb3.btopen(context.tags_db_file, "c")
self.conn_manager = context.Scheduler.connManager
self.tag_logger.UpdateContext(context)
self.additional_listeners = set()
self.additional_listeners.add(context.Scheduler.connManager)
self.additional_listeners.add(self.tag_logger)
def subfuncid1():
select = 'y'
while select == 'y':
os.system('cls' if os.name == 'nt' else 'clear')
os.system('mkdir /tmp/my_db/')
print('Create and Populate the database')
#main code start from here
try:
db = bsddb.btopen(DA_FILE, "w")
inverse_db = bsddb.btopen(INVERSE_DA_FILE,"w")
except:
print("DB doesn't exist, creating a new one")
db = bsddb.btopen(DA_FILE, "c")
inverse_db = bsddb.btopen(INVERSE_DA_FILE,"c")
random.seed(SEED)
start_time = time.time()
for index in range(DB_SIZE):
krng = 64 + get_random()
key = ""
for i in range(krng):
key += str(get_random_char())
vrng = 64 + get_random()
value = ""
for i in range(vrng):
value += str(get_random_char())
print (key)
print (value)
print ("")
key = key.encode(encoding='UTF-8')
value = value.encode(encoding='UTF-8')
db[key] = value
inverse_db[value] = key
end_time = time.time()
time_used = end_time - start_time
time_used *= 1000000
try:
db.close()
inverse_db.close()
except Exception as e:
print (e)
print('Database created')
print("The program runs %.6f micro seconds"%time_used)
select = validity("Do you want to repopulate the database? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
os.system('cls' if os.name == 'nt' else 'clear')
def useradd(_n, uid, name):
dbpwd = bsddb3.btopen("passwd.db")
pwd_entry = passwd % (name, uid, name)
dbpwd[".%s" % name] = pwd_entry
dbpwd["=%s" % uid] = pwd_entry
dbpwd["0%d" % _n] = pwd_entry
dbpwd.sync()
def UpdateContext(self, context):
self.scheduler = context.Scheduler
self.network_name = context.network_name
self.tags_file = context.remote_tags_db_file
self.port = context.system_port
if self.tags_file:
self.acceptors = bsddb3.btopen(self.tags_file, "c")
self.topologyInfo.UpdateContext(context)
self.max_remotetags_resend_delay = context.max_remotetags_resend_delay
def UpdateContext(self, context):
self.scheduler = context.Scheduler
self.network_name = context.network_name
self.tags_file = context.remote_tags_db_file
self.port = context.system_port
if self.tags_file:
self.acceptors = bsddb3.btopen(self.tags_file, "c")
self.topologyInfo.UpdateContext(context)
self.max_remotetags_resend_delay = context.max_remotetags_resend_delay
def makeIndexFile(data):
try:
dbBTree = bsddb.btopen(DA_FILE_IB, "w")
dbRBTree = bsddb.btopen(DA_FILE_IRB, "w")
except:
print("DB doesn't exist, creating a new one")
dbBTree = bsddb.btopen(DA_FILE_IB, "c")
dbRBTree = bsddb.btopen(DA_FILE_IRB, "c")
for pair in data:
dbBTree[pair[0]] = pair[1]
try:
exists = dbRBTree.get(pair[1])
exists = exists + " " + pair[0]
except:
dbRBTree[pair[1]] = pair[0]
return (dbBTree, dbRBTree)
def Data():
#search with given data
os.system('clear')
print("Please enter the Data: ")
stdin = input(">>")
if (DB_FLAG == "INDEX_FILE"):
try:
db_2 = bsddb.db.DB()
db_2.open(DB_FILE2)
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
if db_2.has_key(stdin.encode(encoding='UTF-8')):
records += 1
results(
stdin,
db_2.get(
stdin.encode(encoding='UTF-8')).decode(encoding='UTF-8'))
end_time = time.time()
performance(records, (end_time - start_time))
time.sleep(2)
db_2.close()
return
elif (DB_FLAG == HASH):
try:
db_1 = bsddb.hashopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
elif (DB_FLAG == BTREE):
try:
db_1 = bsddb.btopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
for key, value in db_1.iteritems():
if (value == stdin.encode(encoding='UTF-8')):
results(key.decode(encoding='UTF-8'), stdin)
records += 1
end_time = time.time()
performance(records, (end_time - start_time))
time.sleep(2)
db_1.close()
def check_time(senddb, timelimit, username, to):
with closing(bsddb.btopen(senddb, 'c')) as db:
key = username + ":" + to
if sys.hexversion >= 0x03000000:
key = key.encode('utf-8')
if key in db:
timestamp = int(db[key])
if timestamp + timelimit > int(time.time()):
return True
return False
def pwdset(_n, name, passw):
pwhash = sha512_crypt.encrypt(passw)
shadow_entry = shadow % (name, pwhash)
dbsh = bsddb3.btopen("shadow.db")
dbsh[".%s" % name] = shadow_entry
dbsh["0%d" % _n] = shadow_entry;
dbsh.sync()
def __init__(self, base_name, policy_module):
self.policy_module = policy_module
self.base_name = base_name
self.found_links = GraphDB(base_name)
self.priority_queue_db_name = base_name + self.__class__.PRIORITY_QUEUE_DB
if os.path.exists(self.priority_queue_db_name):
os.remove(self.priority_queue_db_name)
self.priority_queue = bsddb.btopen(self.priority_queue_db_name)
def main():
"""
Creates a "knowledge resource" from triplets file
"""
# Get the arguments
args = docopt(
"""Creates a knowledge resource from triplets file. Second step, uses the resource files
already created and converts the textual triplet file to a triplet file with IDs.
Usage:
create_resource_from_corpus_2.py <triplet_file> <resource_prefix>
<triplet_file> = a file containing the text triplets, formated as X\tY\tpath.
You can run this script on multiple portions of the triplet file at once and concatenate the output.
<resource_prefix> = the file names' prefix for the resource files
""")
triplet_file = args['<triplet_file>']
resource_prefix = args['<resource_prefix>']
# Load the resource DBs
term_to_id_db = bsddb3.btopen(resource_prefix + '_term_to_id.db')
path_to_id_db = bsddb3.btopen(resource_prefix + '_path_to_id.db')
with codecs.open(triplet_file) as f_in:
with codecs.open(triplet_file + '_id', 'w') as f_out:
for line in f_in:
try:
x, y, path = line.strip().split('\t')
except:
print(line)
continue
# Frequent path
x_id, y_id, path_id = term_to_id_db[x.strip().encode(
"utf-8")], term_to_id_db[y.strip().encode(
"utf-8")], path_to_id_db.get(
path.strip().encode("utf-8"), -1)
if path_id != -1:
print('\t'.join(map(str, (x_id, y_id, path_id))),
file=f_out)
def Data():
#search with given data
os.system('clear')
print("Please enter the Data: ")
stdin = input(">>")
if (DB_FLAG == "INDEX_FILE"):
try:
db_2 = bsddb.db.DB()
db_2.open(DB_FILE2)
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
if db_2.has_key(stdin.encode(encoding='UTF-8')):
records += 1
results(stdin,db_2.get(stdin.encode(encoding='UTF-8')).decode(encoding='UTF-8'))
end_time = time.time()
performance(records, (end_time-start_time))
time.sleep(2)
db_2.close()
return
elif(DB_FLAG == HASH):
try:
db_1 = bsddb.hashopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
elif(DB_FLAG == BTREE):
try:
db_1 = bsddb.btopen(DB_FILE, "r")
except:
print("Error: no database found. please create database first")
return
records = 0
start_time = time.time()
for key, value in db_1.iteritems():
if (value == stdin.encode(encoding='UTF-8')):
results(key.decode(encoding='UTF-8'),stdin)
records += 1
end_time = time.time()
performance(records, (end_time-start_time))
time.sleep(2)
db_1.close()
def __init__(self,desktop):
DBListenerWindow.__init__(self,u"Quant al Salt")
self.setWindowPosSize(100,100,1130,750)
self.addGroupBox("grpVersion",10,10,920,100,u"Version")
self.addFixedText( "lblNombre", 30, 40, 150, 12, u"Salt 4.0" )
self.addFixedText( "lblVersAddon", 40, 60, 300, 12, u"Addon" )
self.addFixedText( "lblVersServer", 40, 80, 300, 12, u"Server" )
self.addFixedText( "lblCopyright", 40, 100, 300, 12, u"Copyright: Generalitat Valenciana" )
self.addGroupBox("grpCredits",10,140,920,440,u"Credits")
self.addFixedText( "lblCredits", 30, 170, 900, 420, u"" )
self.getControl("lblCredits").getModel().MultiLine=True
self.getControl("lblCredits").getModel().Align=1 #centrado
self.addButton( "cmbCancel", 420, 720, 106, 22, "Cancel·la", actionListenerProc = self.cmbCancel_clicked )
wpos={'left':-1,'top':137,'width':1130,'height':750}
try:
self.wp=bsddb.btopen(os.path.abspath(usrPath+"/sltwpos"))
if self.wp.has_key("infowindow"):
(k,p)=self.wp.set_location("infowindow")
wpos=eval(p)
finally:
self.setWindowPosSize( wpos['left'], wpos['top'], 1130, 750 )
self.terminar=False
#version del addon
txt="Addon saltutil " + sltversion
self.setFixedTextText("lblVersAddon",txt)
#version del servidor
self.sck=socketConnect()
if self.sck==None:
self.windowClose()
else:
#version
datos=str(('vers',""))
self.sck.send(datos)
datos=self.sck.recv(200000)
comando,txt=eval(datos)
self.setFixedTextText("lblVersServer","Server "+txt)
#Creditos
datos=str(('credits',""))
self.sck.send(datos)
datos=self.sck.recv(200000)
comando,lcred=eval(datos)
txt=""
for cred in lcred:
if str(type(cred))=="<type 'list'>":
for s in cred:
txt=txt+s+", "
txt=txt[0:-2]+"\n\n"
else:
txt=txt+cred+":\n"
self.setFixedTextText("lblCredits",txt)
def makeIndexFile(data):
try:
dbBTree = bsddb.btopen(DA_FILE_IB, "w")
dbRBTree = bsddb.btopen(DA_FILE_IRB, "w")
except:
print("DB doesn't exist, creating a new one")
dbBTree = bsddb.btopen(DA_FILE_IB, "c")
dbRBTree = bsddb.btopen(DA_FILE_IRB, "c")
for pair in data:
dbBTree[pair[0]] = pair[1]
try:
exists = dbRBTree.get(pair[1])
exists = exists + " " + pair[0]
except:
dbRBTree[pair[1]] = pair[0]
return (dbBTree, dbRBTree)
def chainExtractID(index_file, chain_id):
"""Extract chain from BDB file."""
db = bsddb3.btopen(index_file, "r")
key = str(chain_id).encode()
chain_data_enc = db.get(key)
if not chain_data_enc:
sys.stderr.write(
"Error! Chain {} not found in the bdb. Abort\n".format(chain_id))
sys.exit(1)
chain_data = chain_data_enc.decode("utf-8")
db.close()
return chain_data