def main():
termdatabase = db.DB()
termdatabase.open("../phase2/te.idx", None, db.DB_BTREE, db.DB_CREATE,
db.DB_DUP)
datedatabase = db.DB()
datedatabase.open("../phase2/da.idx")
tweetdatabase = db.DB()
tweetdatabase.open("../phase2/tw.idx")
databases = {
"terms": termdatabase,
"tweets": tweetdatabase,
"dates": datedatabase
}
while 1:
query = input("Enter your query #>")
if query == 'exit':
break
if query == 'help':
print_help()
if query == "":
pass
else:
run_query(query.lower(), databases)
for key, value in databases.items():
value.close()
def setUp(self):
n = 100000
self.weibo_ids = self._load_items(n)
self.db_env = db.DBEnv()
self.db_env.set_tmp_dir(BDB_TMP_PATH)
self.db_env.set_lg_dir(BDB_LOG_PATH)
self.db_env.set_cachesize(0, 8 * (2 << 25), 1)
self.db_env.open(BDB_DATA_PATH, db.DB_INIT_CDB | db.DB_INIT_MPOOL)
weibo_hash_db = db.DB(self.db_env)
weibo_hash_db.open('weibo_hash', None, db.DB_HASH, db.DB_CREATE)
self.weibo_hash_db = weibo_hash_db
weibo_hash_db_rdonly = db.DB(self.db_env)
weibo_hash_db_rdonly.open('weibo_hash', None, db.DB_HASH, db.DB_RDONLY)
self.weibo_hash_db_rdonly = weibo_hash_db_rdonly
weibo_btree_db = db.DB(self.db_env)
weibo_btree_db.open('weibo_btree', None, db.DB_BTREE, db.DB_CREATE)
self.weibo_btree_db = weibo_btree_db
weibo_btree_db_rdonly = db.DB(self.db_env)
weibo_btree_db_rdonly.open('weibo_btree', None, db.DB_BTREE,
db.DB_RDONLY)
self.weibo_btree_db_rdonly = weibo_btree_db_rdonly
def open_db(self, direct):
if not os.path.exists(os.path.join(direct, 'jumpstart.db')):
wxMessageBox(
'This does not look like a JumpStart database directory',
style=wxOK | wxICON_ERROR)
return
self.env = db.DBEnv()
self.env.open(direct, db.DB_JOINENV)
self.dbFileChunks = db.DB(self.env)
self.dbFileChunks.open('Jumpstart.db', "File-Chunks", db.DB_UNKNOWN,
db.DB_RDONLY)
self.dbHashName = db.DB(self.env)
self.dbHashName.open('Jumpstart.db', "Hash-Name", db.DB_UNKNOWN,
db.DB_RDONLY)
curs = self.dbFileChunks.cursor()
rec = curs.first()
while rec is not None:
filehash = rec[0]
#print b2a_hex(filehash)
self.listFiles.Append(
[self.hash_to_name(filehash),
b2a_hex(filehash)])
rec = curs.next()
self.dbHashName.close()
self.dbFileChunks.close()
self.env.close()
self.panelSelections.Enable(1)
def process_one(q):
results = []
if q[0] == 'year':
database = db.DB()
database.open('ye.idx', None, db.DB_BTREE, db.DB_RDONLY)
curs = database.cursor()
if q[1] == ':':
results = equal_situation(q, curs)
elif q[1] == '>':
results = greater_situation(q, curs)
elif q[1] == '<':
results = smaller_situation(q, curs)
else:
print("Invalid input")
else:
database = db.DB()
database.open('te.idx', None, db.DB_BTREE, db.DB_RDONLY)
curs = database.cursor()
if len(q) > 1:
if q[1] == ":":
results = term_search(q, curs)
else:
results = terms_search(q, curs)
database.close()
return results
def create(dbname,
primary_namespace,
secondary_namespaces,
formatname="unknown"):
os.mkdir(dbname)
config_filename = os.path.join(dbname, "config.dat")
BaseDB.write_config(config_filename=config_filename,
index_type=INDEX_TYPE,
primary_namespace=primary_namespace,
secondary_namespaces=secondary_namespaces,
fileid_info={},
formatname=formatname)
dbenv = db.DBEnv(0)
envflags = db.DB_THREAD | db.DB_INIT_MPOOL
dbenv.open(dbname, envflags | db.DB_CREATE)
primary_table = db.DB(dbenv)
primary_table.open("key_%s" % (primary_namespace, ), None, db.DB_BTREE,
db.DB_CREATE, 0660)
secondary_tables = {}
for namespace in secondary_namespaces:
x = db.DB(dbenv)
x.open("id_%s" % (namespace, ), None, db.DB_BTREE, db.DB_CREATE, 0)
secondary_tables[namespace] = x
for x in secondary_tables.values():
x.close()
primary_table.close()
dbenv.close()
return open(dbname, "rw")
def close_fs_db(self):
self.dbh_blocks = bsdDB.DB() # blocks table
self.dbh_replicas.close()
self.dbh_tree.close()
self.dbh_id.close()
self.dbh_paths = bsdDB.DB() #path index
self.dbh_objects.close()
def test00_associateDBError(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test00_associateDBError..." % \
self.__class__.__name__
dupDB = db.DB(self.env)
dupDB.set_flags(db.DB_DUP)
dupDB.open(self.filename, "primary", db.DB_BTREE, db.DB_CREATE)
secDB = db.DB(self.env)
secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE)
# dupDB has been configured to allow duplicates, it can't
# associate with a secondary. BerkeleyDB will return an error.
try:
def f(a, b):
return a + b
dupDB.associate(secDB, f)
except db.DBError:
# good
secDB.close()
dupDB.close()
else:
secDB.close()
dupDB.close()
self.fail("DBError exception was expected")
def digest(): #opens the database
print("Opening idx files...\n")
database1 = db.DB() #handle for Berkeley DB database
database2 = db.DB()
database3 = db.DB()
database4 = db.DB()
DB_File = "ad.idx" #our file name
database1.open("ad.idx")
DB_File = "te.idx" #our file name
database2.open("te.idx")
'''
DB_File = "da.idx" #our file name
database3.open("pr.idx")
DB_File = "pr.idx" #our file name
database4.open("pr.idx")
'''
#database.open(DB_File ,None, db.DB_HASH, db.DB_CREATE) #another way to do it
cursor = database2.cursor()
#check idx database
iter = cursor.first()
while iter:
print(iter)
iter = cursor.next()
return cursor
def __init__(self, dbenv, name, flags, txn):
self.indexdb = db.DB(dbEnv=dbenv)
self.indexdb.open(name[0] + 'index.db', dbtype=db.DB_BTREE, flags=flags, txn=txn)
self.dagdb = db.DB(dbEnv=dbenv)
self.dagdb.open(name + '-dag.db', dbtype=db.DB_BTREE, flags=flags, txn=txn)
self.mergedb = db.DB(dbEnv=dbenv)
self.mergedb.open(name + '-merges.db', dbtype=db.DB_BTREE, flags=flags, txn=txn)
return
def test02_WithSource(self):
"""
A Recno file that is given a "backing source file" is essentially a
simple ASCII file. Normally each record is delimited by \n and so is
just a line in the file, but you can set a different record delimiter
if needed.
"""
homeDir = os.path.join(tempfile.gettempdir(), 'db_home')
source = os.path.join(homeDir, 'test_recno.txt')
if not os.path.isdir(homeDir):
os.mkdir(homeDir)
f = open(source, 'w') # create the file
f.close()
d = db.DB()
# This is the default value, just checking if both int
d.set_re_delim(0x0A)
d.set_re_delim('\n') # and char can be used...
d.set_re_source(source)
d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
data = "The quick brown fox jumped over the lazy dog".split()
for datum in data:
d.append(datum)
d.sync()
d.close()
# get the text from the backing source
text = open(source, 'r').read()
text = text.strip()
if verbose:
print text
print data
print text.split('\n')
assert text.split('\n') == data
# open as a DB again
d = db.DB()
d.set_re_source(source)
d.open(self.filename, db.DB_RECNO)
d[3] = 'reddish-brown'
d[8] = 'comatose'
d.sync()
d.close()
text = open(source, 'r').read()
text = text.strip()
if verbose:
print text
print text.split('\n')
assert text.split('\n') == \
"The quick reddish-brown fox jumped over the comatose dog".split()
def queryR(conditions):
inter = []
terms = []
for i in conditions:
print(i)
col = i['col']
op = i['operator']
con = i['condition']
if col == 'date':
matches = []
dtb = db.DB()
dtb.open('da.idx',None, db.DB_BTREE, db.DB_CREATE)
curs = dtb.cursor()
itr = curs.first()
while itr:
key = str(itr[0].decode("utf-8"))
data = str(itr[1].decode("utf-8"))
if ops[op](key,con):
matches.append(int(data))
itr = curs.next()
curs.close()
dtb.close()
inter.append(matches)
elif col in ["bcc", "cc", "from", "to"]:
matches = []
dtb = db.DB()
dtb.open('em.idx',None, db.DB_BTREE, db.DB_CREATE)
curs = dtb.cursor()
itr = curs.first()
while itr:
key = str(itr[0].decode("utf-8"))
data = str(itr[1].decode("utf-8"))
quer = str(col+'-'+con)
if ops[op](quer,key):
matches.append(int(data))
itr = curs.next()
curs.close()
dtb.close()
inter.append(matches)
elif col in ["term", "subj", "body"]:
terms.append(i)
if len(terms) > 0:
inter.append(dbFindFast(terms))
try:
setAll = set.intersection(*map(set,inter))
except:
setAll = None
if setAll == None:
return 'No matches'
return setAll
def __init__(self):
"""
Initialize the databases and setflags
"""
# Intitialize the db
self.reviews_db = db.DB()
self.scores_db = db.DB()
self.pterms_db = db.DB()
self.rterms_db = db.DB()
def main():
# Tweets
try:
tweets = open('tweets.txt', 'r')
except FileNotFoundError:
raise FileNotFoundError("tweets.txt doesn't exist")
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('tw.idx', None, db.DB_HASH, db.DB_CREATE)
cur = database.cursor()
for line in tweets:
tid, rec = line[0:9], line[10:]
cur.put(bytes(tid.strip('\n'), encoding="ascii"), rec.strip('\n'), db.DB_KEYFIRST)
database.close()
cur.close()
# Terms
try:
terms = open('terms.txt', 'r')
except FileNotFoundError:
raise FileNotFoundError("terms.txt doesn't exist")
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('te.idx', None, db.DB_BTREE, db.DB_CREATE)
cur = database.cursor()
for line in terms:
term, tid = line.split(":")
cur.put(bytes(term.strip('\n'), encoding="ascii"), tid.strip('\n'), db.DB_KEYFIRST)
database.close()
cur.close()
# Dates
try:
dates = open('dates.txt', 'r')
except FileNotFoundError:
raise FileNotFoundError("dates.txt doesn't exist")
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('da.idx', None, db.DB_BTREE, db.DB_CREATE)
cur = database.cursor()
for line in dates:
date, tid = line.split(":")
cur.put(bytes(date.strip('\n'), encoding="ascii"), tid.strip('\n'), db.DB_KEYFIRST)
database.close()
cur.close()
def openDatabases():
global ads, terms, pdates, prices
ads = db.DB()
ads.open("ad.idx")
terms = db.DB()
terms.open("te.idx")
pdates = db.DB()
pdates.open("da.idx")
prices = db.DB()
prices.open("pr.idx")
def create_indexes():
database1 = db.DB()
database1.set_flags(db.DB_DUP)
database1.open("re.idx", None, db.DB_HASH, db.DB_CREATE)
database1.close()
database2 = db.DB()
database2.set_flags(db.DB_DUP)
database2.open("te.idx", None, db.DB_BTREE, db.DB_CREATE)
database2.close()
database3 = db.DB()
database3.set_flags(db.DB_DUP)
database3.open("ye.idx", None, db.DB_BTREE, db.DB_CREATE)
database3.close()
def create_database():
global cursor1, cursor2, cursor3, cursor4, database1, database2, database3, database4
database1 = db.DB()
database2 = db.DB()
database3 = db.DB()
database4 = db.DB()
database1.open("re.idx", None, db.DB_HASH, db.DB_CREATE)
database2.open("te.idx", None, db.DB_BTREE, db.DB_CREATE)
database3.open("em.idx", None, db.DB_BTREE, db.DB_CREATE)
database4.open("da.idx", None, db.DB_BTREE, db.DB_CREATE)
cursor1 = database1.cursor()
cursor2 = database2.cursor()
cursor3 = database3.cursor()
cursor4 = database4.cursor()
def main():
os.system('sort -u years.txt -o years_output.txt')
os.system('sort -u terms.txt -o terms_output.txt')
os.system('sort -u recs.txt -o recs_output.txt')
year = open('years_output.txt', 'r')
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('ye.idx', None, db.DB_BTREE, db.DB_CREATE)
curs = database.cursor()
for line in year:
y=line[:4]
record=line[5:]
database.put(y.encode("utf-8"),record)
curs.close()
database.close()
#######################################################
terms = open('terms_output.txt', 'r')
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('te.idx', None, db.DB_BTREE, db.DB_CREATE)
cur = database.cursor()
for line in terms:
a=line.split(":")
term=a[0]
record=a[1]
database.put(term.encode("utf-8"), record)
database.close()
cur.close()
#############################################################
recs = open('recs_output.txt', 'r')
database = db.DB()
database.set_flags(db.DB_DUP)
database.open('re.idx', None, db.DB_HASH, db.DB_CREATE)
cur = database.cursor()
for line in recs:
a = line.split(":")
record=a[0]
b=a[1]
database .put(record.encode("utf-8"),b)
os.system('db_dump -p ye.idx')
os.system('db_dump -p te.idx')
os.system('db_dump -p re.idx')
database.close()
cur.close()
def set_current_log(self):
"""Return the log DB for the current write."""
cursor = self.logindex.cursor()
last = cursor.last()
cursor.close()
if not last:
name = LOG_PREFIX + '.1'
self.logindex.append(name)
self.logindex.sync()
self._current_idx = 1
log = db.DB(self.env)
log.open(name, None, db.DB_RECNO, db.DB_CREATE)
else:
idx, value = last
self._current_idx = idx
name = value.decode('utf-8')
log = db.DB(self.env)
try:
log.open(name, None, db.DB_RECNO)
except Exception as exc:
# The last db was deleted
name = LOG_PREFIX + '.' + str(idx + 1)
self._current_idx = idx + 1
log = db.DB(self.env)
log.open(name, None, db.DB_RECNO, db.DB_CREATE)
cursor = log.cursor()
last = cursor.last()
cursor.close()
if last:
eidx, _ = last
if eidx >= self.max_log_events:
log.close()
name = LOG_PREFIX + '.' + str(idx + 1)
self.logindex.append(name)
self.logindex.sync()
self._current_idx = idx + 1
log = db.DB(self.env)
log.open(name, None, db.DB_RECNO, db.DB_CREATE)
self._current_log = log
return self._current_log
def title_review_fetch(fp, data):
text = data[1].lower()
like = data[2]
idSet = set()
temp_db = db.DB()
temp_db.open(fp)
cursor = temp_db.cursor()
results = cursor.first()
found = False
text_key = text.encode("utf-8")
result = temp_db.get(text_key)
cursor = temp_db.cursor()
print(result, cursor.next())
"""
while results:
title = results[0].decode("utf-8")
_id = results[1]
if like:
if text == title[0:len(text)]:
idSet.add(_id)
else:
if text == title:
idSet.add(_id)
results = cursor.next()
"""
cursor.close()
temp_db.close()
return idSet
def test13_associate_in_transaction(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test13_associateAutoCommit..." % \
self.__class__.__name__
txn = self.env.txn_begin()
try:
self.createDB(txn=txn)
self.secDB = db.DB(self.env)
self.secDB.set_flags(db.DB_DUP)
self.secDB.set_get_returns_none(2)
self.secDB.open(self.filename,
"secondary",
db.DB_BTREE,
db.DB_CREATE | db.DB_THREAD,
txn=txn)
if db.version() >= (4, 1):
self.getDB().associate(self.secDB, self.getGenre, txn=txn)
else:
self.getDB().associate(self.secDB, self.getGenre)
self.addDataToDB(self.getDB(), txn=txn)
except:
txn.abort()
raise
self.txn_finish_test(self.secDB, txn=txn)
def test01_basic(self):
d = db.DB()
get_returns_none = d.set_get_returns_none(2)
d.set_get_returns_none(get_returns_none)
d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
for x in letters:
recno = d.append(x * 60)
assert type(recno) == type(0)
assert recno >= 1
if verbose:
print recno,
if verbose: print
stat = d.stat()
if verbose:
pprint(stat)
for recno in range(1, len(d) + 1):
data = d[recno]
if verbose:
print data
assert type(data) == type("")
assert data == d.get(recno)
try:
data = d[0] # This should raise a KeyError!?!?!
except db.DBInvalidArgError, val:
assert val[0] == db.EINVAL
if verbose: print val
def open_dbs(self):
self.env = db.DBEnv()
self.env.open(self.homeDir, db.DB_JOINENV)
self.dbFileChunks = db.DB(self.env)
self.dbFileChunks.open('Jumpstart.db', "File-Chunks", db.DB_UNKNOWN,
db.DB_RDONLY)
self.dbHashName = db.DB(self.env)
self.dbHashName.open('Jumpstart.db', "Hash-Name", db.DB_UNKNOWN,
db.DB_RDONLY)
self.dbFileUserChunk = db.DB(self.env)
self.dbFileUserChunk.open('Jumpstart.db', "FileUser-Chunk",
db.DB_UNKNOWN, db.DB_RDONLY)
self.dbFileUserChunk_Blocks = db.DB(self.env)
self.dbFileUserChunk_Blocks.open('Jumpstart.db',
"FileUserChunk-Blocks", db.DB_UNKNOWN,
db.DB_RDONLY)
def __init__(self, grampsdb, path):
"""
Class constructor
"""
super(DbUndoBSDDB, self).__init__(grampsdb)
self.undodb = db.DB()
self.path = path
def lesser(Items,index):
valid=False
if index=='NONE':
return
ItemList=[]
database = db.DB()
database.open(index)
cursor = database.cursor()
while not valid:
Items=int(Items)-1.0
var=bytes(str(Items), 'utf-8')
if var in database:
valid=True
if Items<1:
#NO RESULTS WERE FOUND
return []
results=cursor.get(var, db.DB_SET)
ItemList.append(results[1].decode("utf-8"))
for x in range(cursor.count()-1):
ItemList.append(cursor.next_dup()[1].decode("utf-8"))
results=cursor.get(var, db.DB_SET)
item=cursor.get(var, db.DB_SET)
while item != None:
ItemList.append(item[1].decode('utf-8'))
item = cursor.prev()
return ItemList
def query5(MAX):
MIN = 0
#MAX = 20
database = db.DB()
DB_File = "prices.idx"
database.open(DB_File, None, db.DB_BTREE)
curs = database.cursor()
iter = curs.first()
full_list = []
while (iter):
#print(int(iter[0].decode("utf-8"))>= MIN)
#print(int(iter[0].decode("utf-8"))< MAX)
if int(iter[0].decode("utf-8")) >= MIN and int(
iter[0].decode("utf-8")) < MAX:
ad_id = str(iter[1].decode("utf-8"))
ad_id2 = ad_id.split(",")
a = ad_id2[0]
full_list.append(a)
#print(full_list)
curs.next()
iter = curs.next()
curs.close()
database.close()
return full_list
def query4b(one_date):
#one_date = "2018/11/05"
#one_date = one_date.split("/")
database = db.DB()
DB_File = "pdates.idx"
database.open(DB_File, None, db.DB_BTREE)
curs = database.cursor()
iter = curs.first()
full_list = []
while (iter):
#print(int(iter[0].decode("utf-8"))>= MIN)
#print(int(iter[0].decode("utf-8"))< MAX)
index_date = str(iter[0].decode("utf-8"))
#index_dates = index_dates.split("/")
#dates <= input
#print(datetime(int(one_date[0]),int(one_date[1]),int(one_date[2])) <= datetime(int(one_date[0]),int(one_date[1]),int(one_date[2])))
newdate1 = time.strptime(one_date, "%Y/%m/%d") #date
newdate2 = time.strptime(index_date, "%Y/%m/%d") # ///
#print(newdate2 > newdate1)
if newdate2 < newdate1:
ad_id = str(iter[1].decode("utf-8"))
ad_id2 = ad_id.split(",")
a = ad_id2[0]
full_list.append(a)
iter = curs.next()
#print(full_list)
curs.close()
database.close()
return full_list
def emails_with_subject():
terms = db.DB()
terms.open(te_file, None, db.DB_BTREE, db.DB_CREATE)
curs = terms.cursor()
term = str(input('Enter a term to search for in subject field: '))
term = 'b-' + term
result = curs.set(term.encode('utf-8'))
matches = []
if result == None:
return
else:
matches.append(result[1])
result = curs.next_dup()
while result != None:
matches.append(result[1])
result = curs.next_dup()
print(matches)
def dates(query):
review_db = db.DB()
review_db.open("rw.idx")
cursor = review_db.cursor()
data = cursor.first()
date_asked = time.mktime(
datetime.datetime.strptime(query[2], "%Y/%m/%d").timetuple())
condition = query[1]
results = set()
while data:
_id = data[0]
#parsing the data in order to grab the date
review_split = str(data[1]).split('",')
review_split = review_split[2].split(',')
date = float(review_split[2]) if len(review_split) > 2 else False
if condition == ">" and date:
if date > date_asked:
results.add(_id)
if condition == "<" and date:
if date < date_asked:
results.add(_id)
data = cursor.next()
cursor.close()
review_db.close()
return results
def basic(fp, _id):
tmp_db = db.DB()
tmp_db.open(fp)
result = tmp_db.get(_id)
print("result", result)
tmp_db.close()
return result
class BaseThreadedTestCase(unittest.TestCase):
dbtype = db.DB_UNKNOWN # must be set in derived class
dbopenflags = 0
dbsetflags = 0
envflags = 0
def setUp(self):
if verbose:
dbutils._deadlock_VerboseFile = sys.stdout
homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
self.homeDir = homeDir
try:
os.mkdir(homeDir)
except OSError, e:
if e.errno <> errno.EEXIST: raise
self.env = db.DBEnv()
self.setEnvOpts()
self.env.open(homeDir, self.envflags | db.DB_CREATE)
self.filename = self.__class__.__name__ + '.db'
self.d = db.DB(self.env)
if self.dbsetflags:
self.d.set_flags(self.dbsetflags)
self.d.open(self.filename, self.dbtype,
self.dbopenflags | db.DB_CREATE)
