コード例 #1
0
ファイル: Query2.py プロジェクト: DT9/BerkleyDB
def hashTableQ2():
    answers = open('answers', 'a')
    iValue = input("Input Value: ")
    iValue = iValue.encode(encoding='UTF-8')
    start_time_hDB = time.time()
    HT_FILE = "/tmp/aredmond_db/hashtable"
    try:
        hDB = bsddb.hashopen(HT_FILE, "r")
    except:
        print("DB doesn't exist.")
        return 0
    hDB = bsddb.hashopen(HT_FILE, "w")
    hNumRecords = 0
    currentItem = hDB.first()
    for i in range(1, len(hDB)):       
        if (currentItem[1] == iValue):
            answers.write(currentItem[0].decode('UTF-8'))
            answers.write("\n")
            answers.write(currentItem[1].decode('UTF-8'))
            answers.write("\n")
            answers.write("\n")
            hNumRecords += 1
        currentItem = hDB.next()
    try:
        hDB.close()
        answers.close()
    except Exception as e:
        print (e)    
    end_time_hDB = time.time()
    print("Number of Records Retrieved: {}".format(hNumRecords))    
    print("--- {} microseconds ---".format((end_time_hDB - start_time_hDB) * 1000000))
コード例 #2
0
ファイル: python3.py プロジェクト: hpabst/291Project2
def main():
    try:
        db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "w")
        print("DB opend without error.")
        time.sleep(3)
    except:
        print("DB doesn't exist, creating a new one")
        db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "c")
        print("DB created.")
        time.sleep(1)
    random.seed(SEED)

    for index in range(DB_SIZE):
        krng = 64 + get_random()
        key = ""
        for i in range(krng):
            key += str(get_random_char())
        vrng = 64 + get_random()
        value = ""
        for i in range(vrng):
            value += str(get_random_char())
        print (key)
        print (value)
        print ("")
        key = key.encode(encoding='UTF-8')
        value = value.encode(encoding='UTF-8')
        db[key] = value
    try:
        db.close()
    except Exception as e:
        print (e)
コード例 #3
0
def openDB(string):
    if string == "btree":
        db = bsddb.btopen(DA_FILE_B, "r")
    elif string == "hash":
        db = bsddb.hashopen(DA_FILE_H, "r")
    elif string == "indexfile":
        db = (bsddb.hashopen(DA_FILE_IH, "r"), bsddb.hashopen(DA_FILE_IRH, "r"), bsddb.btopen(DA_FILE_IB, "r"))
    return db
コード例 #4
0
ファイル: scan_db.py プロジェクト: hazraa/samples
def process(dbo, user, dbfiles, dump=False, doip=False, dousage=False):

    count=0
    usrlist={}

    if type(dbfiles)==type(""):
        dbfiles=[dbfiles] 
    for dbf in dbfiles:
	if not user:
	    print "Scanning ",dbf

        try:
           db=bsddb3.hashopen(dbf, "r")
        except bsddb3._db.DBOldVersionError,msg:    # Old v1.85
           d=bsddb3.db.DB()
           d.upgrade(dbf)                     # upgrade on the fly
           db=bsddb3.hashopen(dbf, "r")

	fields=string.split(dbf,'.')
	dstr=str(fields[2])
	srv=str(fields[1])
        try:
            record=db.first()
	    while record:
	        count=count+1
	        (key, data) = record;
	        fields = string.split(data,chr(0))
	   
	        sid,usr=string.split(key,':',1)
		if dump==True:
		    if doip==True:
		        print "%d: %s:%s" % (count, key, fields[7])
		    else:
		        print "%d: %s" % (count, key)
	        if not usrlist.has_key(usr):
		    usrlist[usr]=[]
		# Session details for a user may be scattered between
		# Radius servers.
		nkey=dstr + ':' + srv + ':' + key
		#print nkey
	        usrlist[usr].append(nkey)	# Save all keys, incl any dups
		
		
		#if 0 and usr=="*****@*****.**":
		if user and usr==user:
		    data=db[key]
		    fields = string.split(data,chr(0))
		    if dousage==True:
		        print "%s %s %d %d" % (dbf, user, int(fields[9])/1000000, int(fields[10])/1000000)
	  	    else:
		        print fields
		#else:
		#    print usr,"=",fields

                record=db.next()

        except KeyError, msg:
	    pass
コード例 #5
0
ファイル: 291proj2.py プロジェクト: hpabst/291Project2
def populateDatabase():
    global typeOption
    global db
    global db2
    global dbPopFlag
    if(dbPopFlag == True):
        print("Error: database already populated, returning to menu.")
        return
    if(typeOption == "btree"):
        try:
            db = bsddb.btopen(DA_FILE, 'w')
        except:
            print("Error: Database does not exist, creating one.")
            db = bsddb.btopen(DA_FILE, 'c')
    elif(typeOption == "hash"):
        try:
            db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "w")           
        except:
            print("Error: Database does not exist, creating one.")
            db = bsddb.hashopen("/tmp/hpabst_db/hashdb.db", "c")
    elif(typeOption == "indexfile"):
        try:
            db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "w")
            db2 = bsddb.db.DB()
            db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
        except:
            print("Error: Database does not exist, creating one.")
            db = bsddb.btopen("/tmp/hpabst_db/indexdb1.db", "c")
            db2 = bsddb.db.DB()
            db2.open("/tmp/hpabst_db/indexdb2.db", None, bsddb.db.DB_BTREE, bsddb.db.DB_CREATE)
    random.seed(SEED)

    print("Populating the database...")
    for index in range(DB_SIZE):
        krng = 64 + get_random()
        key = ""
        while True:
            for i in range(krng):
                key += str(get_random_char())
            if(db.has_key(key.encode()) == False):
                break
            else:
                key = ""
        vrng = 64 + get_random()
        value = ""
        for i in range(vrng):
            value += str(get_random_char())
        key = key.encode(encoding="UTF-8")
        value = value.encode(encoding="UTF-8")
        db[key] = value
        if typeOption == "indexfile":
            db2[value] = key
        # Tracking Database Population Progress
        print('\b\b\b'+str(int((index/DB_SIZE)*100))+"%",end="")
    dbPopFlag = True
    print('\b\b\b\b', end="")
    print("100%\nDatabase population complete.")
コード例 #6
0
def createHashTable():
	## Create and populate hash Table
	print("Creating Hash Table")
	try:
		db = bsddb.hashopen(DA_FILE, "w")
	except:
		print("DB doesn't exist, creating a new one")
		db = bsddb.hashopen(DA_FILE, "c")
	populate(db)
	return db
コード例 #7
0
ファイル: project2.py プロジェクト: LingboTang/291-project
def create_and_populate(data_type):
    global db
    global inverted_db
    try:
        if data_type==1:
            db = bsddb.btopen(PATH, "w")
        elif data_type==2:
            db = bsddb.hashopen(PATH, "w")
        elif data_type==3:
            db = bsddb.btopen(PATH, "w")
            inverted_db = bsddb.btopen(INVERTED_PATH, "w")
            create_index_file()
    except:
        print("DB doesn't exist, creating a new one")
        if data_type==1:
            db = bsddb.btopen(PATH, "c")
        elif data_type==2:
            db = bsddb.hashopen(PATH, "c")
        elif data_type==3:
            db = bsddb.btopen(PATH, "c")
            inverted_db = bsddb.btopen(INVERTED_PATH, "c")            
    SEED = 10000000
    lib.set_seed(SEED)
    index=0
    while index<DB_SIZE:
        krng = 64 + lib.get_random() % 64
        key = ""
        for i in range(krng):
            key += str(chr(lib.get_random_char()))
        vrng = 64 + lib.get_random() % 64
        value = ""
        for i in range(vrng):
            value += str(chr(lib.get_random_char()))
        #print(key)
        #print(value)
        #print("")
        if data_type==3:
            ALPHA=key[0]
            #split_index=alpha.index(ALPHA)
        
        key = key.encode(encoding='UTF-8')
        value = value.encode(encoding='UTF-8')            
        if db.has_key(key):
            pass
        else:
            db[key] = value
            index+=1
            if data_type==3:
                if inverted_db.has_key(value):
                    new_key=inverted_db[value].decode(encoding='UTF-8')
                    new_key+=(","+key.decode(encoding='UTF-8'))
                    inverted_db[value]=new_key.encode(encoding='UTF-8')
                else:
                    inverted_db[value]=key
                ALL_DB[ALPHA][key]=value
コード例 #8
0
def makeHashTable(data):
    try:
        db = bsddb.hashopen(DA_FILE_H, "w")
    except:
        print("DB doesn't exist, creating a new one")
        db = bsddb.hashopen(DA_FILE_H, "c")

    for pair in data:
        db[pair[0]] = pair[1]    

    return db
コード例 #9
0
ファイル: CreateDatabase.py プロジェクト: DT9/BerkleyDB
def indexFileCD():
    data = open('data', 'a') 
    I1_FILE = "/tmp/aredmond_db/indexFile1"
    I2_FILE = "/tmp/aredmond_db/indexFile2"
    I3_FILE = "/tmp/aredmond_db/indexFile3"
    try:
        i1DB = bsddb.hashopen(I1_FILE, "w")
        i2DB = bsddb.hashopen(I2_FILE, "w")
        i3DB = bsddb.btopen(I3_FILE, "w")
    except:
        print("DB doesn't exist, creating a new one")
        i1DB = bsddb.hashopen(I1_File, "c")
        i2DB = bsddb.hashopen(I2_FILE, "c")
        i3DB = bsddb.btopen(I2_FILE, "c")
    random.seed(SEED)
    keySet = set()
    valueSet = set()
    for index in range(DB_SIZE):
        while(True):
            krng = 64 + get_random()
            key = "" 
            for i in range(krng):
                key += str(get_random_char())
            vrng = 64 + get_random()
            value = ""
            for i in range(vrng):
                value += str(get_random_char())
            print ("Key: " + key)
            data.write(key)
            data.write("\n")            
            print ("Value: " + value)
            data.write(value)
            data.write("\n")            
            print ("")
            data.write("\n")            
            key = key.encode(encoding='UTF-8')
            value = value.encode(encoding='UTF-8')
            if(not (key in keySet)):
                keySet.add(key)
                i1DB[key] = value
                i3DB[key] = value
                if(not (value in valueSet)):
                    valueSet.add(value)
                    i2DB[value] = key
                else:
                    i2DB[value] = i2DB[value] + ";;;".encode('UTF-8') + key
                break   
    try:
        i1DB.close()
        i2DB.close()
        data.close()
    except Exception as e:
        print (e)           
コード例 #10
0
def ret_DATA(filetype):
    if filetype == 'btree':
        db = bsddb.btopen(DB_FILE, 'r')
    elif filetype == 'hash':
        db = bsddb.hashopen(DB_FILE, 'r')
    elif filetype == 'indexfile':
        db = bsddb.btopen(DB_FILE, 'r')
        indexfile = bsddb.hashopen(SDB_FILE, 'r')
    else:
        print("Unknown type, function terminated\n")
        return

    # open answers for writing, appending to the end of the file
    answers = open('answers', 'a')

    result_lst = []
    data = input("Enter the data you want to search > ")
    data = data.encode(encoding = 'UTF-8')
    start_time = time.time()
    for key in db.keys():
        if db[key] == data:
            result_lst.append(key.decode(encoding = 'UTF-8'))
    end_time = time.time()
    
    elapse_time = (end_time - start_time) * 1000000

    print("Result:")

    data = data.decode(encoding = 'UTF-8')

    if result_lst:
        for key in result_lst:
            print('Key:', key)
            answers.write(key)
            answers.write('\n')
            print('Data:', data)
            answers.write(data)
            answers.write('\n')
            
            answers.write('\n')
    else:
        print("Data not found")
    print()
    print(len(result_lst), "record(s) received")
    print("Used", elapse_time, "micro seconds")
    print()
    answers.close()
    db.close()
    if filetype == 'indexfile':
        indexfile.close()
    return
コード例 #11
0
    def __init__(self, filename="background_model.counts.bdb"):
        import bsddb3

        self.all_counts = bsddb3.hashopen(filename, "c")
        # self.all_counts.open(filename, pytc.BDBOCREAT|pytc.BDBOWRITER|pytc.BDBOREADER)
        self.info = {"big_n": 0}
        self.counts = {"unigram": self.all_counts, "bigram": self.all_counts}
コード例 #12
0
def open_db3hash(*args):

    """Open a bsddb3 hash."""

    import bsddb3

    return bsddb3.hashopen(*args)
コード例 #13
0
ファイル: retrieve_1.py プロジェクト: megsum/c291_gp2
    def __init__(self, type_option, key):
        super(retrieve_1, self).__init__()
        start_time = time.time()
        self.value = None
        if type_option == "btree" or type_option == "indexfile":
            file = "/tmp/vanbelle_db/btree.db"
            db = bsddb.btopen(file, "r")
            try:
                self.value = db[key.encode(encoding="UTF-8")]
                try:
                    db.close()
                except Exception as e:
                    print(e)
                self.value = self.value.decode(encoding="UTF-8")
                print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
            except:
                print("invalid key")

        elif type_option == "hash":
            file = "/tmp/vanbelle_db/hash.db"
            db = bsddb.hashopen(file, "r")
            try:
                self.value = db[key.encode(encoding="UTF-8")]
                try:
                    db.close()
                except Exception as e:
                    print(e)
                self.value = self.value.decode(encoding="UTF-8")
                print("this function took %s microseconds to run" % ((time.time() - start_time) * 1000000))
            except:
                print("invalid key")
コード例 #14
0
ファイル: Query1.py プロジェクト: DT9/BerkleyDB
def indexFileQ1():
    answers = open('answers', 'a')    
    iKey = input("Input Key: ")
    iKey = iKey.encode(encoding='UTF-8')    
    start_time_iDB = time.time()
    I1_FILE = "/tmp/aredmond_db/indexFile1"
    try:
        iDB = bsddb.hashopen(I1_FILE, "r")
    except:
        print("DB doesn't exist.")
        return 0
    iNumRecords = 0
    if(iDB.has_key(iKey)):
        location = iDB.set_location(iKey)
        answers.write(location[0].decode('UTF-8'))
        answers.write("\n")
        answers.write(location[1].decode('UTF-8'))
        answers.write("\n")
        answers.write("\n")
        iNumRecords += 1
    try:
        iDB.close()
        answers.close()
    except Exception as e:
        print (e)        
    end_time_iDB = time.time()
    print("Number of Records Retrieved: {}".format(iNumRecords))
    print("--- {} microseconds ---".format((end_time_iDB - start_time_iDB) * 1000000))
コード例 #15
0
def subfunchash2():
    select = 'y'
    while select == 'y':
        os.system('cls' if os.name == 'nt' else 'clear')
        print('Retrieve records with a given key')
        Given_key = validity("Please select a number option from the options above: ", "Please select a valid option: ", 300, str)
        Retrieved_data = '' 
        try:
            db = bsddb.hashopen(DA_FILE,"r")
        except:
            print("Open method and file does not match to each other")
            break
        Given_key = Given_key.encode(encoding = 'UTF-8')
        try:
            db.has_key(Given_key)
            start_time = time.time()
            Retrieved_data = db[Given_key]
            end_time = time.time()
            Given_key = Given_key.decode(encoding = 'UTF-8')
            Retrieved_data = Retrieved_data.decode(encoding = 'UTF-8')
            time_used = end_time - start_time
            time_used *= 1000000
            db.close()
        except:
            print('Key is not found')
            break
        print('Key: %s\n'%Given_key+'Value: %s\n'%Retrieved_data)
        print("The program runs %.6f micro seconds"%time_used)
        select = validity("Do you want find another key? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
        #os.system('cls' if os.name == 'nt' else 'clear')
        #print()
コード例 #16
0
ファイル: db_explorer.py プロジェクト: exaile/exaile
def cvtdb(ctx, data, dbtype):
    '''
        Only used for testing purposes
    '''
    
    db = ctx.parent.params['db']
    newdb = db + '.new'
    
    if dbtype == 'gdbm':
        import gdbm
        new_d = gdbm.open(newdb, 'n')
    elif dbtype == 'dbm':
        import dbm
        new_d = dbm.open(newdb, 'n')
    elif dbtype == 'dbhash':
        import dbhash
        new_d = dbhash.open(newdb, 'n')
    elif dbtype == 'bsddb':
        new_d = bsddb.hashopen(newdb, 'n')
    elif dbtype == 'dumbdbm':
        import dumbdbm
        new_d = dumbdbm.open(newdb, 'n')
    else:
        raise click.ClickException("Invalid type %s" % dbtype)
    
    new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)
    
    for k, v in data.iteritems():
        new_data[k] = v
    
    new_data.sync()
    new_data.close()
コード例 #17
0
ファイル: dumpdb.py プロジェクト: euske/mcscanmap
def main(argv):
    import getopt
    def usage():
        print 'usage: %s [-k|-v] path ...' % argv[0]
        return 100
    try:
        (opts, args) = getopt.getopt(argv[1:], 'kv')
    except getopt.GetoptError:
        return usage()
    def dispitem(k,v):
        print '+%d,%d:%s->%s' % (len(k), len(v), k, v)
        return
    def dispkey(k,v):
        print '+%d:%s' % (len(k), k)
        return
    def dispvalue(k,v):
        print '+%d:%s' % (len(v), v)
        return
    func = dispitem
    for (k,v) in opts:
        if k == '-k':
            func = dispkey
        elif k == '-v':
            func = diskvalue
    if not args: return usage()
    for path in args:
        db = bsddb3.hashopen(path, 'r')
        for (k,v) in db.iteritems():
            func(k, v)
        db.close()
    return 0
コード例 #18
0
ファイル: part1getkeyrecord.py プロジェクト: 0x4849/cmput291
    def __init__(self):
        self.answers = open("part1keyanswers.txt", "w")

        self.db1Path = "part1BT"
        self.db2Path = "part1HT"
        self.db3Path = "indexfile"
        self.db3 = db.DB()

        self.db1MicroSeconds = []
        self.db1MilliSeconds = []

        self.db2MicroSeconds = []
        self.db2MilliSeconds = []

        self.db3MicroSeconds = []
        self.db3MilliSeconds = []

        self.db1 = bsddb3.btopen(self.db1Path, "c")
        self.db2 = bsddb3.hashopen(self.db2Path, "c")
        self.db3.open(self.db3Path, None, db.DB_HASH, db.DB_CREATE)
        # self.db1.open(self.db1Path, None, db.DB_BTREE, db.DB_CREATE)
        # self.db2.open(self.db2Path, None, db.DB_HASH, db.DB_CREATE)
        # self.db3.open(self.db3Path, None, db.DB_BTREE, db.DB_CREATE)

        self.mainMenu()
コード例 #19
0
ファイル: scanmap.py プロジェクト: euske/mcscanmap
 def __init__(self, dbpath, basehref, minlevel, maxlevel):
     print >>sys.stderr, 'opening: %r' % dbpath
     self.db = bsddb3.hashopen(dbpath)
     self.basehref = basehref
     self.minlevel = minlevel
     self.maxlevel = maxlevel
     self.session = requests.session()
     return
コード例 #20
0
ファイル: corpus.py プロジェクト: cypreess/corpora
 def __init__(self, path):
     self.corpus_path = path
     self.chunks = {}
     self.idx = rnopen(
         os.path.join(self.corpus_path, Corpus.IDX_FILE)
     )  # FIXME: issue #2 this should be implemented with bsddb3 Queue
     self.ridx = hashopen(os.path.join(self.corpus_path, Corpus.RIDX_FILE))
     self.properties = yaml.load(file(os.path.join(self.corpus_path, Corpus.CONFIG_FILE), "r"))
コード例 #21
0
ファイル: geocode.py プロジェクト: sirrice/locjoin
 def __init__(self, fname='/tmp/geocache.bdb'):
     try:
         self.cache = bsddb3.hashopen(fname)
     except:
         self.cache = {}
     try:
         self.ncalls = int(self.cache.get('__ncalls__', '0'))
     except:
         self.ncalls = 0
コード例 #22
0
def Data():
    #search with given data
    os.system('clear')    
    print("Please enter the Data: ")
    stdin = input(">>")
    
    if (DB_FLAG == "INDEX_FILE"):
        try:
            db_2 = bsddb.db.DB()
            db_2.open(DB_FILE2)
        except:
            print("Error: no database found. please create database first")
            return
        
        records = 0
        start_time = time.time()    

        if db_2.has_key(stdin.encode(encoding='UTF-8')):
            records += 1
            results(stdin,db_2.get(stdin.encode(encoding='UTF-8')).decode(encoding='UTF-8'))        
        
        end_time = time.time()
        performance(records, (end_time-start_time))
        
        time.sleep(2)
        db_2.close()

        return
    
    elif(DB_FLAG == HASH):
        try:
            db_1 = bsddb.hashopen(DB_FILE, "r")
        except:
            print("Error: no database found. please create database first")
            return
    elif(DB_FLAG == BTREE):
        try:
            db_1 = bsddb.btopen(DB_FILE, "r")
        except:
            print("Error: no database found. please create database first")
            return

    records = 0
    start_time = time.time()  
    
    for key, value in db_1.iteritems():
        if (value == stdin.encode(encoding='UTF-8')):
            results(key.decode(encoding='UTF-8'),stdin)
            records += 1
    end_time = time.time()
    performance(records, (end_time-start_time))
    time.sleep(2)
    
    db_1.close() 
コード例 #23
0
ファイル: util.py プロジェクト: elbaschid/postr
def get_buddyicon(flickr, data, size=48):
    """Lookup the buddyicon from the data in @data using @flickr and resize it
    to @size pixels."""
    from twisted.web.client import getPage
    import bsddb3

    global __buddy_cache
    if __buddy_cache is None:
        folder = os.path.join (get_cache_path(), "postr")
        if not os.path.exists(folder):
            os.makedirs(folder)
        path = os.path.join (folder, "buddyicons")
        try:
            __buddy_cache = bsddb3.hashopen(path, "c")
        except bsddb3.db.DBInvalidArgError:
            # The database needs upgrading, so delete it
            os.remove(path)
            __buddy_cache = bsddb3.hashopen(path, "c")

    def load_thumb(page, size):
        loader = gtk.gdk.PixbufLoader()
        loader.set_size (size, size)
        loader.write(page)
        loader.close()
        return loader.get_pixbuf()

    def got_data(page, url, size):
        __buddy_cache[url] = page
        return load_thumb(page, size)
    
    if int(data.get("iconfarm")) > 0:
        url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
    else:
        url = "http://www.flickr.com/images/buddyicon.jpg"

    if __buddy_cache.has_key(url):
        from twisted.internet import defer
        return defer.succeed(load_thumb(__buddy_cache[url], size))
    else:
        return getPage(url).addCallback(got_data, url, size)
コード例 #24
0
ファイル: retrieve_3.py プロジェクト: megsum/c291_gp2
    def __init__(self,type_option,low,high):
        super(retrieve_3,self).__init__()
        start_time = time.time()
        self.keys = []
        
        if type_option == 'btree':
            file = '/tmp/vanbelle_db/btree.db'
            db = bsddb.btopen(file,'r')
            for key, value in db.items():
                key = key.decode(encoding='UTF-8')
                value = value.decode(encoding='UTF-8')             
                if key >= low and key <= high:
                    self.keys.append((key,value))
            try:
                db.close()
            except Exception as e:
                print (e)        
            print('this function retrived %s records' %len(self.keys))
            print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
            
        elif type_option == 'hash':
            file = '/tmp/vanbelle_db/hash.db'
            db = bsddb.hashopen(file,'r')
            for key, value in db.items():
                key = key.decode(encoding='UTF-8')
                value = value.decode(encoding='UTF-8')             
                if key >= low and key <= high:
                    self.keys.append((key,value))
            try:
                db.close()
            except Exception as e:
                print (e)        
            print('this function retrived %s records' %len(self.keys))
            print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
            
        elif type_option == 'indexfile':
            file = '/tmp/vanbelle_db/index.db'
            dc = bsddb.btopen(file,'r')
            datas = dc.keys()
            for i in datas:
                i = i.decode(encoding='UTF-8')
                if i >= low and i <= high:
                    self.keys.append(dc[i.encode(encoding='UTF-8')])

            for i in range(len(self.keys)):
                self.keys[i] = self.keys[i].decode(encoding='UTF-8')
            try:
                dc.close()
            except Exception as e:
                print (e)              
            print('this function retrived %s records' %len(self.keys))
            print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
コード例 #25
0
ファイル: common.py プロジェクト: exaile/exaile
def open_shelf(path):
    '''
        Opens a python shelf file, used to store various types of metadata
    '''
    # As of Exaile 4, new DBs will only be created as Berkeley DB Hash databases
    # using either bsddb3 (external) or bsddb (stdlib but sometimes removed).
    # Existing DBs created with other backends will be migrated to Berkeley DB.
    # We do this because BDB is generally considered more performant,
    # and because gdbm currently doesn't work at all in MSYS2.

    # Some DBM modules don't use the path we give them, but rather they have
    # multiple filenames. If the specified path doesn't exist, double check
    # to see if whichdb returns a result before trying to open it with bsddb
    force_migrate = False
    if not os.path.exists(path):
        from whichdb import whichdb

        if whichdb(path) is not None:
            force_migrate = True

    if not force_migrate:
        try:
            db = bsddb.hashopen(path, 'c')
            return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
        except bsddb.db.DBInvalidArgError:
            logger.warning("%s was created with an old backend, migrating it", path)
        except Exception:
            raise

    # special case: zero-length file
    if not force_migrate and os.path.getsize(path) == 0:
        os.unlink(path)
    else:
        from xl.migrations.database.to_bsddb import migrate

        migrate(path)

    db = bsddb.hashopen(path, 'c')
    return shelve.BsdDbShelf(db, protocol=PICKLE_PROTOCOL)
コード例 #26
0
ファイル: retrieve_2.py プロジェクト: megsum/c291_gp2
 def __init__(self, type_option, data):
     super(retrieve_2,self).__init__()
     start_time = time.time()
     self.keys = []
     
     if type_option == 'btree':
         self.file = '/tmp/vanbelle_db/btree.db'
         db = bsddb.btopen(self.file,'r')
         
         for key, value in db.items():
             value = value.decode(encoding='UTF-8') 
             if data == value:           
                 key = key.decode(encoding='UTF-8')
                 self.keys.append(key)
         try:
             db.close()
         except Exception as e:
             print (e)        
         print('this function retrived %s records' %len(self.keys))
         print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
         
     elif type_option == 'hash':
         self.file = '/tmp/vanbelle_db/hash.db'
         db = bsddb.hashopen(self.file,'r') 
         for key, value in db.items():
             value = value.decode(encoding='UTF-8') 
             if data == value:           
                 key = key.decode(encoding='UTF-8')
                 self.keys.append(key)
         try:
             db.close()
         except Exception as e:
             print (e)        
         print('this function retrived %s records' %len(self.keys))
         print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
         
     elif type_option == 'indexfile':
         file = '/tmp/vanbelle_db/index.db'
         dc = bsddb.btopen(file,'r')
         try:
             self.keys.append(dc[data.encode(encoding='UTF-8')]) 
             for i in range(len(self.keys)):
                 self.keys[i] = self.keys[i].decode(encoding='UTF-8')
             try:
                 dc.close()
             except Exception as e:
                 print (e)
             print('this function retrived %s records' %len(self.keys))
             print('this function took %s microseconds to run' %((time.time() - start_time)*1000000))
         except:
             print('invalid data value')
コード例 #27
0
ファイル: Proj2.py プロジェクト: Gknoblau/291Project1
def create_db(db_type):
    # The flag for opening 'w' seems to creating a new database
    # even of there isn't one... which isn't what it's supposed to
    
    # https://docs.python.org/2/library/bsddb.html
    global db
    # Open DB for binary tree
    if(db_type == 'btree'):
        if(os.path.isfile(db_fileType)):
            print("Database already exists")
            db = bsddb.btopen(db_fileType, "w")
            return False
        else:
            db = bsddb.btopen(db_fileType, "w")
            return True
            
    #Open DB for hash
    elif(db_type == 'hash'):
        if(os.path.isfile(db_fileType)):
            print("Database already exists")
            db = bsddb.hashopen(db_fileType, "w") 
            return False 
        else:        
            db = bsddb.hashopen(db_fileType, "w")
            return True
        
    #Open Index file
    elif(db_type == 'index'):
        global db2
        if(os.path.isfile(db_fileType) and os.path.isfile(db_fileType_2)):
            print("Database already exists")
            db = bsddb.btopen(db_fileType, "w")
            db2 = bsddb.btopen(db_fileType_2, "w")
            return False
        else: 
            db = bsddb.btopen(db_fileType, "w")
            db2 = bsddb.btopen(db_fileType_2,  "w")
            return True
コード例 #28
0
ファイル: subfunchash.py プロジェクト: LingboTang/291-project
def subfunchash():
    select = 'y'
    while select == 'y':
        os.system('cls' if os.name == 'nt' else 'clear')
        os.system('mkdir /tmp/my_db')
        print('Create and Populate the database')
        #main code start from here
        time.time()
        try:
            db = bsddb.hashopen(DA_FILE, "w")
        except:
            print("DB doesn't exist, creating a new one")
            db = bsddb.hashopen(DA_FILE, "c")
        random.seed(SEED)        
        for index in range(DB_SIZE):
            krng = 64 + get_random()
            key = ""
            for i in range(krng):
                key += str(get_random_char())
            vrng = 64 + get_random()
            value = ""
            for i in range(vrng):
                value += str(get_random_char())
            print (key)
            print (value)
            print ("")
            key = key.encode(encoding='UTF-8')
            value = value.encode(encoding='UTF-8')
            db[key] = value
        try:
            db.close()
            print('Database created')
            print("The program runs %.6f seconds"%time.clock())
            select = validity("Do you want to repopulate the database? y/n: ", "Please enter a valid option: ", 1, str, ['y', 'n'], 'lower')
            os.system('cls' if os.name == 'nt' else 'clear')            
        except Exception as e:
            print (e)
コード例 #29
0
ファイル: CreateDatabase.py プロジェクト: DT9/BerkleyDB
def hashTableCD():
    data = open('data', 'a') 
    HT_FILE = "/tmp/aredmond_db/hashtable"
    try:
        hDB = bsddb.hashopen(HT_FILE, "w")
    except:
        print("DB doesn't exist, creating a new one")
        hDB = bsddb.hashopen(HT_FILE, "c")
    random.seed(SEED)
    keySet = set()
    for index in range(DB_SIZE):
        while(True):
            krng = 64 + get_random()
            key = "" 
            for i in range(krng):
                key += str(get_random_char())
            vrng = 64 + get_random()
            value = ""
            for i in range(vrng):
                value += str(get_random_char())
            data.write(key)
            data.write("\n")            
            data.write(value)
            data.write("\n")            
            data.write("\n")            
            key = key.encode(encoding='UTF-8')
            value = value.encode(encoding='UTF-8')
            if(not (key in keySet)):
                keySet.add(key)
                hDB[key] = value
                break
    try:
        hDB.close()
        data.close()
    except Exception as e:
        print (e)    
コード例 #30
0
def makeIndexFile(data):
    try:
        dbHash = bsddb.hashopen(DA_FILE_IH, "w")
        dbRHash = bsddb.hashopen(DA_FILE_IRH, "w")
        dbBTree = bsddb.btopen(DA_FILE_IB, "w")

    except:
        print("DB doesn't exist, creating a new one")
        dbHash = bsddb.hashopen(DA_FILE_IH, "c")
        dbRHashTree = bsddb.hashopen(DA_FILE_IRH, "c")
        dbBTree = bsddb.btopen(DA_FILE_IB, "c")

    for pair in data:
        dbHash[pair[0]] = pair[1]
        
        try:
            exists = dbRHash.get(pair[1])
            exists = exists + " " + pair[0]
        except:
            dbRHash[pair[1]] = pair[0]

        dbBTree[pair[0]] = pair[1]

    return (dbHash, dbRHash, dbBTree)
コード例 #31
0
def db_put(key, value):
    with open('mapping_db.lock', 'w') as lockfile:
        fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX)
        with closing(bsddb.hashopen('mapping_db', 'c')) as db:
            db[codecs.encode(key, 'ascii')] = codecs.encode(value, 'ascii')
コード例 #32
0
#!/usr/bin/env python
# encoding: utf-8
import json
import bsddb3 as bsddb
db = bsddb.hashopen('mouse.bdb')
for key in db:
    j = json.loads(db[key + ""])
    print j
コード例 #33
0
ファイル: Main.py プロジェクト: CMPUTproject291/project2
def Retrieverecordswithagivendata(mode):
    if mode == 1:
        DA_FILE = "/tmp/weijie2_db/bt_db"
        db = bsddb.btopen(DA_FILE, "r")
        invertDA_FILE = "/tmp/weijie2_db/invert_bt_db"
        invertdb = bsddb.btopen(invertDA_FILE, "r")
    elif mode == 2:
        DA_FILE = "/tmp/weijie2_db/hash_db"
        db = bsddb.hashopen(DA_FILE, "r")
        invertDA_FILE = "/tmp/weijie2_db/invert_hash_db"
        invertdb = bsddb.hashopen(invertDA_FILE, "r")
    elif mode == 3:
        DA_FILE = "/tmp/weijie2_db/indexfile_db"
        db = bsddb.btopen(DA_FILE, "r")
        invertDA_FILE = "/tmp/weijie2_db/invert_indexfile_db"
        invertdb = bsddb.btopen(invertDA_FILE, "r")
    while True:
        try:
            if mode == 1:
                value = input("Input a Valid value:").encode(encoding='UTF-8')
                keyList = []
                start = time.time()
                try:
                    for key in db.keys():
                        if db[key] == value:
                            keyList.append(key.decode(encoding='UTF-8'))
                except:
                    print("not found")
                    break
                end = time.time()
                print("Time:", 1000000 * (end - start), "micro seconds")
                print("Receive", len(keyList), "records")
                file = open("answers", "a")
                for i in keyList:
                    print("Key is:", i)
                    file.write(key.decode(encoding='UTF-8') + "\n")
                    file.write(value.decode(encoding='UTF-8') + "\n\n")
                file.close()
                db.close()
                break
            elif mode == 2:
                value = input("Input a Valid value:").encode(encoding='UTF-8')
                keyList = []
                start = time.time()
                try:
                    for key in db.keys():
                        if db[key] == value:
                            keyList.append(key.decode(encoding='UTF-8'))
                except:
                    print("not found")
                    break
                end = time.time()
                print("Time :", 1000000 * (end - start), "micro seconds")
                print("Receive", len(keyList), "records")
                file = open("answers", "a")

                for i in keyList:
                    print("Key is:", i)
                    file.write(key.decode(encoding='UTF-8') + "\n")
                    file.write(value.decode(encoding='UTF-8') + "\n\n")
                file.close()

                db.close()
                break
            elif mode == 3:
                value = input("Input a Valid value:").encode(encoding='UTF-8')
                keyList = []
                start = time.time()
                try:
                    for key in db.keys():
                        if db[key] == value:
                            keyList.append(key.decode(encoding='UTF-8'))
                except:
                    print("not found")
                    break
                end = time.time()
                print("Time:", 1000000 * (end - start), "micro seconds")
                print("Receive", len(keyList), "records")
                file = open("answers", "a")
                for i in keyList:
                    print("Key is:", i)
                    file.write(key.decode(encoding='UTF-8') + "\n")
                    file.write(value.decode(encoding='UTF-8') + "\n\n")
                file.close()
                db.close()
                break
        except Exception as e:
            print(e)
    return 0
コード例 #34
0
def updateMovie():
    while True:
        name = raw_input("Movie name to be updated: ")
        flag = True
        index = -1
        db = bsddb3.hashopen('movies.db', 'c')
        for key in db.keys():
            index = index + 1
            movie = pickle.loads(db[key])
            if movie.name == name:
                while True:
                    print "1.Name\n2.Directors\n3.Actors\n4.Genres\n5.Run time\n6.Production House\n7.Country\n8.Release Date:\n9.Languages:\n"
                    field = int(
                        raw_input(
                            "Enter field to be updated(enter choice no. ): "))
                    if field == 1:
                        movie.name = raw_input("Enter updated movie name: ")
                    elif field == 5:
                        movie.runTime = int(
                            raw_input("Enter updated run time: "))
                    elif field == 6:
                        movie.productionHouse = raw_input(
                            "Enter updated productionHouse: ")
                    elif field == 7:
                        movie.country = raw_input("Enter updated country: ")
                    elif field == 3:
                        del movie.actors[:]
                        act_list = raw_input(
                            "Enter updated actors list(comma separated list): "
                        )
                        while act_list == "":
                            act_list = raw_input(
                                "List cannot be empty.Please enter: ")
                        for actor in act_list.split(','):
                            movie.actors.append(actor)
                    elif field == 2:
                        del movie.directors[:]
                        dir_list = raw_input(
                            "Enter updated directors list(comma separated list): "
                        )
                        while dir_list == "":
                            dir_list = raw_input(
                                "List cannot be empty.Please enter: ")
                        for director in dir_list.split(','):
                            movie.directors.append(director)
                    elif field == 9:
                        del movie.languages[:]
                        lan_list = raw_input(
                            "Enter updated languages list(comma separated list): "
                        )
                        while lan_list == "":
                            lan_list = raw_input(
                                "List cannot be empty.Please enter: ")
                        for language in lan_list.split(','):
                            movie.languages.append(language)
                    elif field == 4:
                        genres = raw_input(
                            "Select genre(comma separated list): adventure/drama/sci_fi "
                        )
                        while genres == "":
                            genres = raw_input(
                                "List cannot be empty.Plese enter. ")
                        del movie.genres[:]
                        for genre in genres.split(','):
                            if genre.upper() == "ADVENTURE":
                                movie.genres.append(movie_pb2.ADVENTURE)
                            elif genre.upper() == "SCI_FI":
                                movie.genres.append(movie_pb2.SCI_FI)
                            elif genre.upper() == "DRAMA":
                                movie.genres.append(movie_pb2.DRAMA)
                    elif field == 8:
                        date = raw_input("Release date(DD/MM/YYYY): ")
                        list_temp = date.split("/")
                        movie.releaseDate.day = int(list_temp[0])
                        mon = int(list_temp[1])
                        if mon == 1:
                            movie.releaseDate.month = movie_pb2.JANUARY
                        elif mon == 2:
                            movie.releaseDate.month = movie_pb2.FEBRUARY
                        elif mon == 3:
                            movie.releaseDate.month = movie_pb2.MARCH
                        elif mon == 4:
                            movie.releaseDate.month = movie_pb2.APRIL
                        elif mon == 5:
                            movie.releaseDate.month = movie_pb2.MAY
                        elif mon == 6:
                            movie.releaseDate.month = movie_pb2.JUNE
                        elif mon == 7:
                            movie.releaseDate.month = movie_pb2.JULY
                        elif mon == 8:
                            movie.releaseDate.month = movie_pb2.AUGUST
                        elif mon == 9:
                            movie.releaseDate.month = movie_pb2.SEPTEMBER
                        elif mon == 10:
                            movie.releaseDate.month = movie_pb2.OCTOBER
                        elif mon == 11:
                            movie.releaseDate.month = movie_pb2.NOVEMBER
                        elif mon == 12:
                            movie.releaseDate.month = movie_pb2.DECEMBER
                        movie.releaseDate.year = int(list_temp[2])
                    db[key] = pickle.dumps(movie)
                    flag = False
                    if raw_input(
                            "Do you want to update any other fields(or leave blank to finish): "
                    ) == "":
                        break
                break
        if index == -1:
            print "No movies to show"
            break
        if flag:
            print "Movie not found.Continuing...."
        if raw_input(
                "Do you want to update another movie(or leave blank to finish): "
        ) == "":
            break
コード例 #35
0
ファイル: Main.py プロジェクト: CMPUTproject291/project2
def Receivetracekeyrange(mode):
    while True:
        try:
            while True:

                lower = input("Please enter your lower bound key:").encode(
                    encoding='UTF-8')
                upper = input("Please enter your upper bound key:").encode(
                    encoding='UTF-8')

                if lower > upper:
                    print("invalid input lower should before upper ")
                else:
                    break
            if mode == 1:
                try:
                    DA_FILE = "/tmp/weijie2_db/bt_db"
                    db = bsddb.btopen(DA_FILE, "r")
                except:
                    print("open error")
                    return 0
                file = open("answers", "a")
                start = time.time()
                count = 0
                for key in db.keys():
                    if key >= lower and key <= upper:
                        count += 1
                        file.write(key.decode(encoding='UTF-8') + "\n")
                        file.write(db[key].decode(encoding='UTF-8') + "\n\n")
                end = time.time()
                print("There are totall", count, "records in this range.")
                print("Time :", 1000000 * (end - start), "micro seconds")
                file.close()
            elif mode == 2:
                try:
                    DA_FILE = "/tmp/weijie2_db/hash_db"
                    db = bsddb.hashopen(DA_FILE, "r")
                except:
                    print("open error")
                    return 0
                file = open("answers", "a")
                start = time.time()
                count = 0
                for key in db.keys():
                    if key >= lower and key <= upper:
                        count += 1
                        file.write(key.decode(encoding='UTF-8') + "\n")
                        file.write(db[key].decode(encoding='UTF-8') + "\n\n")
                end = time.time()
                print("There are totall", count, "records in this range.")
                print("Time :", 1000000 * (end - start), "micro seconds")
                file.close()
            elif mode == 3:
                try:
                    DA_FILE = "/tmp/weijie2_db/indexfile_db"
                    db = bsddb.btopen(DA_FILE, "r")
                except:
                    print("open error")
                    return 0
                file = open("answers", "a")
                keys = db.keys()
                begin = binarysearch(keys, lower)
                finish = binarysearch(keys, upper)
                count = 0
                start = time.time()
                while begin <= finish:
                    key = keys[begin]
                    count += 1
                    file.write(key.decode(encoding='UTF-8') + "\n")
                    file.write(db[key].decode(encoding='UTF-8') + "\n\n")
                    begin += 1
                end = time.time()
                print("Time :", 1000000 * (end - start), "micro seconds")
                print("There are", count, "records in this range.")
            break

        except Exception as e:
            print(e)
コード例 #36
0
ファイル: Main.py プロジェクト: CMPUTproject291/project2
def Createandpopulateadatabase(mode):
    DB_SIZE = 100000
    #DB_SIZE = 1000

    SEED = 10000000

    if mode == 1:
        DA_FILE = "/tmp/weijie2_db/bt_db"
        invertDA_FILE = "/tmp/weijie2_db/invert_bt_db"
        try:
            db = bsddb.btopen(DA_FILE, "w")
            invertdb = bsddb.btopen(invertDA_FILE, "w")
        except:
            print("DB doesn't exist, creating a new one")
            db = bsddb.btopen(DA_FILE, "c")
            invertdb = bsddb.btopen(invertDA_FILE, "c")
        random.seed(SEED)
    elif mode == 2:
        DA_FILE = "/tmp/weijie2_db/hash_db"
        invertDA_FILE = "/tmp/weijie2_db/invert_hash_db"
        try:
            db = bsddb.hashopen(DA_FILE, "w")
            invertdb = bsddb.hashopen(invertDA_FILE, "w")
        except:
            print("DB doesn't exist, creating a new one")
            db = bsddb.hashopen(DA_FILE, "c")
            invertdb = bsddb.hashopen(invertDA_FILE, "c")
        random.seed(SEED)
    elif mode == 3:
        DA_FILE = "/tmp/weijie2_db/indexfile_db"
        invertDA_FILE = "/tmp/weijie2_db/invert_indexfile_db"
        try:
            db = bsddb.btopen(DA_FILE, "w")
            invertdb = bsddb.btopen(invertDA_FILE, "w")
        except:
            print("DB doesn't exist, creating a new one")
            db = bsddb.btopen(DA_FILE, "c")
            invertdb = bsddb.btopen(invertDA_FILE, "c")
        random.seed(SEED)

    index = 0
    while index < DB_SIZE:
        k = 64 + random.randint(0, 63)
        key = ""
        for i in range(k):
            key += str(chr(97 + random.randint(0, 25)))
        v = 64 + random.randint(0, 63)
        value = ""
        for j in range(v):
            value += str(chr(97 + random.randint(0, 25)))

        print("key:", key, "\nvalue:", value, "\n")
        key = key.encode(encoding='UTF-8')
        value = value.encode(encoding='UTF-8')
        db[key] = value
        invertdb[value] = key
        index += 1
    try:
        db.close()
        invertdb.close()
    except Exception as e:
        print(e)
コード例 #37
0
ファイル: Main.py プロジェクト: CMPUTproject291/project2
def Retrieverecordswithagivenkey(mode):
    if mode == 1:
        DA_FILE = "/tmp/weijie2_db/bt_db"
        db = bsddb.btopen(DA_FILE, "w")
    elif mode == 2:
        DA_FILE = "/tmp/weijie2_db/hash_db"
        db = bsddb.hashopen(DA_FILE, "w")

    elif mode == 3:
        DA_FILE = "/tmp/weijie2_db/indexfile_db"
        #db = bsddb.btopen(DA_FILE,"w")
    else:
        print("Mode error")
        return 1
    while True:

        try:
            if mode == 1:
                DA_FILE = "/tmp/weijie2_db/bt_db"
                db = bsddb.btopen(DA_FILE, "r")
                key = input("Input a Valid Key :").encode(encoding='UTF-8')
                start = time.time()
                try:
                    value = db[key]
                    print("found")
                except:
                    print("Key does not exist")
                end = time.time()
                decodevalue = value.decode(encoding='UTF-8')
                print("value is :", decodevalue)
                print("Time:", 1000000 * (end - start), "micro seconds")
                file = open("answers", "a")
                file.write(key.decode(encoding='UTF-8') + "\n")
                file.write(decodevalue + "\n\n")
                file.close()
                break

            elif mode == 2:
                DA_FILE = "/tmp/weijie2_db/hash_db"
                db = bsddb.hashopen(DA_FILE, "r")
                key = input("Input a Valid Key :").encode(encoding='UTF-8')
                start = time.time()
                try:
                    value = db[key]
                    print("found")
                except:
                    print("Key does not exist")
                end = time.time()
                decodevalue = value.decode(encoding='UTF-8')
                print("value is :", decodevalue)
                print("Time:", 1000000 * (end - start), "micro seconds")
                file = open("answers", "a")
                file.write(key.decode(encoding='UTF-8') + "\n")
                file.write(decodevalue + "\n\n")
                file.close()
                break
            elif mode == 3:
                DA_FILE = "/tmp/weijie2_db/indexfile_db"
                db = bsddb.btopen(DA_FILE, "r")
                key = input("Input a Valid Key :").encode(encoding='UTF-8')
                start = time.time()
                try:
                    value = db[key]
                    print("found")
                except:
                    print("Key does not exist")
                end = time.time()
                decodevalue = value.decode(encoding='UTF-8')
                print("value is :", decodevalue)
                print("Time:", 1000000 * (end - start), "micro seconds")
                file = open("answers", "a")
                file.write(key.decode(encoding='UTF-8') + "\n")
                file.write(decodevalue + "\n\n")
                file.close()
                break
            db.close()
        except Exception as e:
            print(e)
コード例 #38
0
def db_get(key):
    with closing(bsddb.hashopen('mapping_db', 'c')) as db:
        return codecs.decode(db.get(codecs.encode(key, 'ascii')), 'ascii')
コード例 #39
0
ファイル: __init__.py プロジェクト: tariqhamid/kopano-core
def dbopen(path):
    if sys.hexversion >= 0x03000000:
        return bsddb.hashopen(path, 'c')
    else:  # pragma: no cover
        return bsddb.hashopen(_encode(path), 'c')