def __init__(self, engine, **kw): super(LevelDBStore, self).__init__(engine, **kw) self._store = leveldb.LevelDB(self._engine)
import leveldb, json import csv user_db = leveldb.LevelDB('./user_db') cnt = 0 for k, v in user_db.RangeIter(): obj = json.loads(v.decode('utf8')) #print(cnt, k, obj['PlayerName']) cnt += 1 print cnt
import leveldb fname_db = 'test_db' # LevelDBオープン db = leveldb.LevelDB(fname_db) # valueが'Japan'のものを列挙 clue = 'Japan'.encode() result = [value[0].decode() for value in db.RangeIter() if value[1] == clue] # 件数表示 print('{}件'.format(len(result)))
def reopen(self): del self.db self.db = leveldb.LevelDB(self.dbfile)
def open(self): self._db = leveldb.LevelDB(self.db_path)
def __init__(self, config, shared): Processor.__init__(self) self.shared = shared self.config = config self.up_to_date = False self.watched_addresses = [] self.history_cache = {} self.chunk_cache = {} self.cache_lock = threading.Lock() self.headers_data = '' self.mempool_addresses = {} self.mempool_hist = {} self.mempool_hashes = [] self.mempool_lock = threading.Lock() self.address_queue = Queue() self.dbpath = config.get('leveldb', 'path') self.pruning_limit = config.getint('leveldb', 'pruning_limit') self.db_version = 1 # increase this when database needs to be updated self.dblock = threading.Lock() try: self.db = leveldb.LevelDB(self.dbpath) except: traceback.print_exc(file=sys.stdout) self.shared.stop() self.bitcoind_url = 'http://%s:%s@%s:%s/' % ( config.get('bitcoind', 'user'), config.get('bitcoind', 'password'), config.get('bitcoind', 'host'), config.get('bitcoind', 'port')) self.height = 0 self.is_test = False self.sent_height = 0 self.sent_header = None try: hist = self.deserialize(self.db.Get('height')) self.last_hash, self.height, db_version = hist[0] print_log("Database version", self.db_version) print_log("Blockchain height", self.height) except: traceback.print_exc(file=sys.stdout) print_log('initializing database') self.height = 0 self.last_hash = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' db_version = self.db_version # check version if self.db_version != db_version: print_log( "Your database '%s' is deprecated. Please create a new database" % self.dbpath) self.shared.stop() return # catch_up headers self.init_headers(self.height) threading.Timer(0, lambda: self.catch_up(sync=False)).start() while not shared.stopped() and not self.up_to_date: try: time.sleep(1) except: print "keyboard interrupt: stopping threads" shared.stop() sys.exit(0) print_log("blockchain is up to date.") threading.Timer(10, self.main_iteration).start()
def generate(dataset, dirs, threshold, size, unit_size, sample_rate, gpu, model, force, tt, net = '', js = None): if dataset not in support: raise ValueError('The Dataset {} is not supported right now '.format(dataset)) dictionary = dict() dictionary['train'] = dict() dictionary['train']['foreground'] = [] dictionary['train']['background'] = [] dictionary['train']['annotations'] = dict() dictionary['train']['response'] = dict() dictionary['train']['iou'] = dict() db = dict() db['train'] = leveldb.LevelDB(dataset + '_threshold_{}_train'.format(threshold)) if tt > 1: db['test'] = leveldb.LevelDB(dataset + '_threshold_{}_test'.format(threshold)) dictionary['test'] = dict() dictionary['test']['foreground'] = [] dictionary['test']['background'] = [] dictionary['test']['annotations'] = dict() dictionary['test']['response'] = dict() dictionary['test']['iou'] = dict() b = leveldb.WriteBatch() miss = [] fg_counter = 0 bg_counter = 0 if dataset == 'ActivityNet-1.3': j = json.load(open('ActivityNet/activity_net.v1-3.min.json')) for v in j['database'].keys(): # iterate over name of videos vpath = os.path.join(dirs['video'], v + '.mp4') # check if video is downloaded if not os.path.isfile(vpath): miss.append(v) continue fpath = os.path.join(dirs['feature'], '{}_US[{}]_SR[{}].h5'.format(v, unit_size, sample_rate)) # check if extracted feature is exist exist = os.path.isfile(fpath) if not exist and not force: # sampling unit level feature print('[!] Unit Level Feature [ {} ] is not exist'.format(fpath)) print('[*] Extracting ... ') net = unit.sampling(v + '.mp4', size, unit_size, sample_rate, net, gpu, model, dirs['video'], dirs['feature'], reuse = True) #raise ValueError('[!] Unit Level Feature [ {} ] is not exist'.format(fpath)) elif exist: print('Feature Path {} is exist'.format(fpath)) elif force: print('[!] Unit Level Feature [ {} ] is not exist, Ignoring...'.format(fpath)) if not force or (force and exist) : with h5py.File(fpath) as ff:#, cv2.VideoCapture(vpath) as capture: #capture = cv2.VideoCapture(vpath) #length = capture.get(cv2.CAP_PROP_FRAME_COUNT) #fps = capture.get(cv2.CAP_PROP_FPS) #capture.release() print('[*] Current Video => [ {} ]'.format(fpath)) fps = np.asarray(ff['fps']) units = list(ff.keys()) # name of units-level feature fduration = [] #annotations['annotations'][v] = [] for fs in range(len(j['database'][v]['annotations'])): # each video contains more than one foreground segment fduration.append((np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps).astype(int)) # frame duration [start frame, end frame] #annotations['annotations'][v].append(fduration[fs]) print(fduration[fs]) for u in units: #print('units : {}'.format(u)) if u == 'feature_size' or u == 'unit_size' or u == 'sample_rate' or u == 'nframes' or u == 'fps': continue fg = False #print(len(j['database'][v]['annotations'])) for fs in range(len(j['database'][v]['annotations'])): # each video contains more than one foreground segment #print('fs {}'.format(fs)) #print(np.asarray(j['database'][v]['annotations'][fs]['segment']) * 8) # convert start time and end time to start frame and end frame respectively """ foreground duration : time * fps => index of frame proposal duration : index of frame """ #fduration = (np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps).astype(np.int) # frame duration [start frame, end frame] pduration = np.asarray(u.split('_'), dtype = int) iou = unit.iou(pduration, fduration[fs], 'clip') if not iou: break """ if iou < 1 and iou > 0: print('Duration Time : {}'.format(j['database'][v]['annotations'][fs]['segment'])) print('Duration Time : {}'.format(np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps)) print('fduration : {}'.format(fduration[fs])) print('pduration : {}'.format(pduration)) print('iou : {}'.format(iou)) """ if iou > threshold: vu = '{}_{}'.format(v, u) if fg_counter >= tt: dictionary['test']['foreground'].append(vu) dictionary['test']['response'][vu] = '{}_{}'.format(fduration[fs][0], fduration[fs][1]) dictionary['test']['iou'][vu] = iou fg_counter = 0 else: dictionary['train']['foreground'].append(vu) dictionary['train']['response'][vu] = '{}_{}'.format(fduration[fs][0], fduration[fs][1]) dictionary['train']['iou'][vu] = iou fg_counter += 1 fg = True break if not fg: if bg_counter >= tt: dictionary['test']['background'].append('{}_{}'.format(v, u)) bg_counter = 0 else: dictionary['train']['background'].append('{}_{}'.format(v, u)) bg_counter += 1 #print(ujson.dumps(annotations)) db['train'].Put('annotations'.encode(), ujson.dumps(dictionary['train']).encode()) if tt > 1: db['test'].Put('annotations'.encode(), ujson.dumps(dictionary['test']).encode()) db['train'].Write(b, sync = True) print('[*] Cache is save to [ {} ]'.format(dataset + '_threshold_{}_train'.format(threshold))) if tt > 1: db['test'].Write(b, sync = True) print('[*] Cache is save to [ {} ]'.format(dataset + '_threshold_{}_test'.format(threshold))) print('[!] Missing {} Video Files'.format(len(miss)))
def __init__(self, dbfile): if dbfile not in databases: databases[dbfile] = leveldb.LevelDB(dbfile) self.db = databases[dbfile] self.uncommitted = {}
try: tld = tldextract.extract(urlparse(uri).netloc) except UnicodeError: return print tld.domain.encode("utf8", "ignore"), json.dumps(res) key = " ".join((tld, uri, args.crawl)) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument('db', help='leveldb root directory') args = parser.parse_args(sys.argv[1:]) db = leveldb.LevelDB(args.db) stats = defaultdict(lambda: defaultdict(int)) header = None full_domain = None valid_languages = [l.lower() for l in args.lang] for linenr, line in enumerate(sys.stdin): if line.startswith(magic_number): header = parse_line(line) full_domain = "%s.%s" % (header["domain"], header["tld"]) continue lang, percent, confidence = line.split()
import leveldb db = leveldb.LevelDB('/Learning/NLP100knock/section7/music_list.ldb') count = 0 for k, v in db.RangeIter(): if v == 'Japan'.encode('utf-8'): count += 1 print(count)
def __init__(self, dbfile): self.dbfile = dbfile if dbfile not in databases: databases[dbfile] = ( leveldb.LevelDB(dbfile), dict(), threading.Lock()) self.db, self.uncommitted, self.lock = databases[dbfile]
def main(leveldb_dir, limit): datum = caffe_pb2.Datum() db = leveldb.LevelDB(leveldb_dir) for i in range(0, limit): datum.ParseFromString(db.Get(str(i))) print datum.float_data, datum.label
import json import leveldb from settings import LEVELDB_PATH from pysyncobj import SyncObj, SyncObjConf, replicated ldb = leveldb.LevelDB(LEVELDB_PATH) class KVStorage(SyncObj): def __init__(self, selfAddress, partnerAddrs, dumpFile=None): self.ldb = ldb conf = SyncObjConf(fullDumpFile=dumpFile, ) super(KVStorage, self).__init__(selfAddress, partnerAddrs, conf) def get(self, key): try: if not isinstance(key, bytes): key = key.encode() value = self.ldb.Get(key) value = json.loads(value.decode()) except KeyError: value = "" return value # @replicated def set(self, key, value): if isinstance(key, bytes): key = key.decode() if isinstance(value, bytes): value = value.decode() value = json.dumps(value)
if not (options.host and options.dbfile): optparser.print_help() print "Starting leveldb-server %s" % options.host context = zmq.Context() frontend = context.socket(zmq.XREP) frontend.bind(options.host) backend = context.socket(zmq.XREQ) backend.bind('inproc://backend') poll = zmq.Poller() poll.register(frontend, zmq.POLLIN) poll.register(backend, zmq.POLLIN) db = leveldb.LevelDB(options.dbfile) workers = [] for i in xrange(3): worker = workerThread(context, db) worker.start() workers.append(worker) try: while True: sockets = dict(poll.poll()) if frontend in sockets: if sockets[frontend] == zmq.POLLIN: msg = frontend.recv_multipart() backend.send_multipart(msg)
import leveldb import sys from utils import to_hex if __name__ == '__main__': server_id = sys.argv[1] idx_ETH = sys.argv[2] idx_TOK = sys.argv[3] masked_amt_ETH = to_hex(sys.argv[4]) masked_amt_TOK = to_hex(sys.argv[5]) file = f"Scripts/hbswap/data/Pool-P{server_id}.data" pool_ETH, pool_TOK = 0, 0 with open(file, 'rb') as f: pool_ETH = f.read(16) pool_TOK = f.read(16) db = leveldb.LevelDB(f"Scripts/hbswap/db/server{server_id}") mask_share_ETH = bytes(db.Get(idx_ETH.encode())) mask_share_TOK = bytes(db.Get(idx_TOK.encode())) file = f"Persistence/Transactions-P{server_id}.data" with open(file, 'wb') as f: f.write(pool_ETH + pool_TOK + mask_share_ETH + mask_share_TOK + masked_amt_ETH + masked_amt_TOK)
def __init__(self, filename): self.db = leveldb.LevelDB(filename) self._last_key = None
アーティスト名(name)から活動場所(area)を検索するためのデータベースを構築せよ. ''' import sys import gzip import json import leveldb def message(text): sys.stderr.write(f"\33[92m{text}\33[0m\n") fname = 'artist.json.gz' db_name = 'name2area' db = leveldb.LevelDB(db_name) if sum(1 for _ in db.RangeIter(include_value=False)) == 921337: message("[*] skip") else: with gzip.open(fname, 'rt') as f: for json_data in map(json.loads, f): name = f"{json_data['name']}\t{json_data['id']}" area = json_data.get('area', '""') db.Put(name.encode(), area.encode()) fname_db_size = sum(1 for _ in db.RangeIter(include_value=False)) message(f'[+] {db_name} のサイズ -> {fname_db_size}') ''' * LevelDB - https://github.com/google/leveldb
def __init__(self, dbfile): self._nbop = 0 self._db = leveldb.LevelDB(dbfile) self._batch = leveldb.WriteBatch() return
def test_load_complex_pgn(self): # 131563 db = leveldb.LevelDB('book/polyglot_index.db') self.get_game(db, 131563) self.get_game(db, 523009)
import leveldb db = leveldb.LevelDB('./db') # single put db.Put('hello', 'hello world') print db.Get('hello') # multiple put/delete applied atomically, and committed to disk batch = leveldb.WriteBatch() batch.Put('hello', 'world') batch.Put('hello again', 'world') batch.Delete('hello') db.Write(batch, sync=True)
import gzip import json import leveldb if __name__ == '__main__': db = leveldb.LevelDB('./lvdb') with gzip.open(filename='artist.json.gz') as f: for line in f: data_json = json.loads(line.decode().rstrip()) key = data_json['name'] value = data_json.get('area') if value: db.Put(key.encode(), value.encode())
def __init__(self, dbpath): self._map_db = leveldb.LevelDB(dbpath)
def main(): db_id = leveldb.LevelDB('./db') db_seq = leveldb.LevelDB('./dc') db_qua = leveldb.LevelDB('./de') #lid = [] num = 0 fp = open("./test.fastq", "r") #fp = open("./all.pacbio.fastq", "r") while 1: line = fp.readline() if not line: break #print str(line) #if (str(line)[0] == "@"): num += 1 sid = 'ps' + repr(num).zfill(8) #lid.append(sid) db_id.Put(sid, line.strip('\n')) line = fp.readline() db_seq.Put(sid, line.strip('\n')) line = fp.readline() line = fp.readline() db_qua.Put(sid, line.strip('\n')) fp.close() adjust = "????????????????????????????????????????????????????????????????????????????????" c_pos = 0 c_chr = "" c_seq = "" c_qua = "" t_seq = "" t_qua = "" fp = open("./test.pass", "r") while 1: line = fp.readline() if not line: break elif line[0] == "#": pass else: t = line.split("\t") if t[0] == c_chr: if c_pos > int(t[1]) - 1: pass else: c_seq += t_seq[c_pos + 1:int(t[1])] c_qua += t_qua[c_pos + 1:int(t[1])] c_seq += t[4] c_qua += adjust[0:len(t[4])] c_pos = int(t[1]) - 1 + len(t[3]) - 1 else: if c_pos: c_seq += t_seq[(c_pos + 1):len(t_seq)] c_qua += t_qua[(c_pos + 1):len(t_qua)] #''' db_seq.Put(c_chr,c_seq) db_qua.Put(c_chr,c_qua) #''' ''' print $seq_hash{$c_chr}{"t"}; print "\n"; print "$c_seq\n"; print "\n\+\n"; print "$c_qua\n"; $ced_hash{$c_chr} += 1; ''' #last tail; c_pos = 0 # a new seq begin c_chr = t[0] #print c_chr c_seq = "" c_qua = "" #''' t_seq = db_seq.Get(c_chr) t_qua = db_qua.Get(c_chr) db_seq.Delete(c_chr) db_qua.Delete(c_chr) #''' c_seq += t_seq[c_pos:int(t[1])] c_qua += t_qua[c_pos:int(t[1])] c_seq += t[4]; # change c_qua += adjust[0:len(t[4])]; c_pos = int(t[1]) - 1 + len(t[3]) - 1; # 0-start; fp.close() c_seq += t_seq[(c_pos + 1):len(t_seq)] c_qua += t_qua[(c_pos + 1):len(t_qua)] db_seq.Put(c_chr,c_seq) db_qua.Put(c_chr,c_qua) for x in xrange(1,num+1): sid = 'ps' + repr(num).zfill(8) print db_id.Get(sid) print db_seq.Get(sid) print "+" print db_qua.Get(sid) import shutil shutil.rmtree('db') shutil.rmtree('dc') shutil.rmtree('de')
def __init__(self, dbpath): self._dir_db = leveldb.LevelDB(dbpath) self.l = Lock()
# -*- coding: utf-8 -*- import scrapy from scrapy.http import Request from datetime import datetime from chotot.items import generalItem import leveldb db = leveldb.LevelDB("db/dienlanh") def insert(item): db.Put(item['id'].encode('UTF-8'), item['tel'].encode('UTF-8')) def search(item): query = db.Get(item['id'].encode('UTF-8')) return query.decode() def validate_time(string): if string == "Tin ưu tiên" or string.find("trước") > -1: return True else: return False class DienlanhSpider(scrapy.Spider): name = 'dienlanh' start_urls = ['http://www.chotot.com/toan-quoc/mua-ban-tu-lanh-may-lanh-may-giat/'] custom_settings = {'FEED_URI': "output/chotot_dienlanh_%(time)s.csv", 'FEED_FORMAT': 'csv'} def parse(self, response): item_urls = response.xpath('//a[@class="adItem___2GCVQ"]/@href').extract() item_infos = response.xpath('//span[@class="item___eld8Q"]/text()').extract()
import leveldb import binascii import numpy as np import caffe import sys from caffe.proto import caffe_pb2 ## parse input argument dbName = sys.argv[1] # open leveldb files db = leveldb.LevelDB(dbName) # get db iterator it = db.RangeIter() f1 = open('./mdb_reader2.txt', 'w+') for key, value in it: # convert string to datum datum = caffe_pb2.Datum.FromString(db.Get(key)) # convert datum to numpy string f1.write('key:' + str(key) + '\n') arr = caffe.io.datum_to_array(datum) i = 0 # convert to svm format # print 1 key and 10 probability only, only for googlenet and vgg
import leveldb DB = './db' HELP = '''usage: leveldb_test.py [-h] [-i count] [-q times] desc.: [-h] --- get usage help [-i count] --- insert count of number into database [-q times] --- query the database the given times and get the QPS ''' MAX = 10000 # random max number TIME = 10000 # query times QUERY = 1 # the key to query, init here, and random in use place # connect the database db = leveldb.LevelDB(DB) def insert(): print 'insert count :%s' % MAX # insert data into database for i in range(MAX): key = str(i) value = str(random.randint(0, MAX)) #print key, value db.Put(key, value) def query(): print 'query times: %d' % TIME start_time = int(round(time.time() * 1000))
def main(argv): leveldb_name = sys.argv[1] db_save_name = sys.argv[4] print "%s" % sys.argv[1] print "%s" % sys.argv[2] print "%s" % sys.argv[3] print "%s" % sys.argv[4] print "%s" % sys.argv[5] temp = int(sys.argv[5]) # window_num = 1000; # window_num = 12736; window_num = int(sys.argv[2]) # window_num = 2845; start = time.time() if 'db' not in locals().keys(): db = leveldb.LevelDB(leveldb_name) datum = feat_helper_pb2.Datum() db_save = lmdb.open(db_save_name, map_size=int(1e12)) #batch = leveldb.WriteBatch() datum_save = feat_helper_pb2.Datum() #ft = np.zeros((window_num, int(sys.argv[3]))) for im_idx in range(window_num): key_str = '%08d' % (im_idx) datum.ParseFromString(db.Get(key_str)) ft0 = np.array(datum.float_data) #print ft0.shape ft1 = ft0 / temp m_ft1 = ft1.max() ft1 = ft1 - m_ft1 exp_ft1 = np.exp(ft1) exp_ft1_s = exp_ft1.sum() p_ft1 = exp_ft1 / exp_ft1_s del datum_save.float_data[:] datum_save.float_data.extend(p_ft1.tolist()) #batch.Put(key_str, datum_save.SerializeToString()) db_save.begin(write=True).put(key_str, datum_save.SerializeToString()) if (im_idx % 10000 == 0): print "%d images processed." % (im_idx) # if (im_idx % 500 == 0 ): # #print np.array(datum.float_data) # print "sample %d" %(im_idx) # print p_ft1.max() # #calculate p when t=1 # m_ft0 = ft0.max() # ft0 = ft0 - m_ft0 # exp_ft0 = np.exp(ft0) # exp_ft0_s = exp_ft0.sum() # p_ft0 = exp_ft0 / exp_ft0_s # print p_ft0.max() #write leveldb #db_save.Write(batch, sync = True) db_save.close() print 'temperature is %d' % (temp) print 'time 1: %f' % (time.time() - start) #sio.savemat(sys.argv[4], {'feats':ft},oned_as='row') print 'time 2: %f' % (time.time() - start) print 'done!'
""" 661. KVSの検索 60で構築したデータベースを用い,特定の(指定された)アーティストの活動場所を取得せよ. """ import gzip import json import leveldb begin = 0 end = 10 db = leveldb.LevelDB('./artist_db') name = input('Who ? :') for ii, data in enumerate(db.RangeIter()): # if ii >= begin and ii <= end: #print(list(data)) #print(data[0].decode("utf-8", "ignore"), data[1].decode("utf-8", "ignore")) A = name.strip() B = data[0].decode("utf-8", "ignore") if A == B: print('Exactly Match!') print('Musician:{}\tActivity Area :{}'.\ format(B,data[1].decode("utf-8", "ignore"))) """ else: if len(set(A) & set(B)) > 3: print('May be you want') print('Musician:{}\tActivity Area :{}'.\
import csv data = csv.DictReader(open('World-z10.tsv'), delimiter="\t") import fiona import json import leveldb db = leveldb.LevelDB("db/geonames") for row in data: geonameid = str(row['geonameid']) try: correct_name = db.Get(geonameid) if row['name'] != correct_name: print "{0} should be {1}".format(row['name'], correct_name) except Exception: pass