def __init__(self, engine, **kw):
super(LevelDBStore, self).__init__(engine, **kw)
self._store = leveldb.LevelDB(self._engine)
import leveldb, json
import csv
user_db = leveldb.LevelDB('./user_db')
cnt = 0
for k, v in user_db.RangeIter():
obj = json.loads(v.decode('utf8'))
#print(cnt, k, obj['PlayerName'])
cnt += 1
print cnt
import leveldb
fname_db = 'test_db'
# LevelDBオープン
db = leveldb.LevelDB(fname_db)
# valueが'Japan'のものを列挙
clue = 'Japan'.encode()
result = [value[0].decode() for value in db.RangeIter() if value[1] == clue]
# 件数表示
print('{}件'.format(len(result)))
def reopen(self):
del self.db
self.db = leveldb.LevelDB(self.dbfile)
def open(self):
self._db = leveldb.LevelDB(self.db_path)
def __init__(self, config, shared):
Processor.__init__(self)
self.shared = shared
self.config = config
self.up_to_date = False
self.watched_addresses = []
self.history_cache = {}
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = []
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
self.dbpath = config.get('leveldb', 'path')
self.pruning_limit = config.getint('leveldb', 'pruning_limit')
self.db_version = 1 # increase this when database needs to be updated
self.dblock = threading.Lock()
try:
self.db = leveldb.LevelDB(self.dbpath)
except:
traceback.print_exc(file=sys.stdout)
self.shared.stop()
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'user'), config.get('bitcoind', 'password'),
config.get('bitcoind', 'host'), config.get('bitcoind', 'port'))
self.height = 0
self.is_test = False
self.sent_height = 0
self.sent_header = None
try:
hist = self.deserialize(self.db.Get('height'))
self.last_hash, self.height, db_version = hist[0]
print_log("Database version", self.db_version)
print_log("Blockchain height", self.height)
except:
traceback.print_exc(file=sys.stdout)
print_log('initializing database')
self.height = 0
self.last_hash = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
db_version = self.db_version
# check version
if self.db_version != db_version:
print_log(
"Your database '%s' is deprecated. Please create a new database"
% self.dbpath)
self.shared.stop()
return
# catch_up headers
self.init_headers(self.height)
threading.Timer(0, lambda: self.catch_up(sync=False)).start()
while not shared.stopped() and not self.up_to_date:
try:
time.sleep(1)
except:
print "keyboard interrupt: stopping threads"
shared.stop()
sys.exit(0)
print_log("blockchain is up to date.")
threading.Timer(10, self.main_iteration).start()
def generate(dataset, dirs, threshold, size, unit_size, sample_rate, gpu, model, force, tt, net = '', js = None):
if dataset not in support:
raise ValueError('The Dataset {} is not supported right now '.format(dataset))
dictionary = dict()
dictionary['train'] = dict()
dictionary['train']['foreground'] = []
dictionary['train']['background'] = []
dictionary['train']['annotations'] = dict()
dictionary['train']['response'] = dict()
dictionary['train']['iou'] = dict()
db = dict()
db['train'] = leveldb.LevelDB(dataset + '_threshold_{}_train'.format(threshold))
if tt > 1:
db['test'] = leveldb.LevelDB(dataset + '_threshold_{}_test'.format(threshold))
dictionary['test'] = dict()
dictionary['test']['foreground'] = []
dictionary['test']['background'] = []
dictionary['test']['annotations'] = dict()
dictionary['test']['response'] = dict()
dictionary['test']['iou'] = dict()
b = leveldb.WriteBatch()
miss = []
fg_counter = 0
bg_counter = 0
if dataset == 'ActivityNet-1.3':
j = json.load(open('ActivityNet/activity_net.v1-3.min.json'))
for v in j['database'].keys(): # iterate over name of videos
vpath = os.path.join(dirs['video'], v + '.mp4') # check if video is downloaded
if not os.path.isfile(vpath):
miss.append(v)
continue
fpath = os.path.join(dirs['feature'], '{}_US[{}]_SR[{}].h5'.format(v, unit_size, sample_rate)) # check if extracted feature is exist
exist = os.path.isfile(fpath)
if not exist and not force:
# sampling unit level feature
print('[!] Unit Level Feature [ {} ] is not exist'.format(fpath))
print('[*] Extracting ... ')
net = unit.sampling(v + '.mp4', size, unit_size, sample_rate, net, gpu, model, dirs['video'], dirs['feature'], reuse = True)
#raise ValueError('[!] Unit Level Feature [ {} ] is not exist'.format(fpath))
elif exist:
print('Feature Path {} is exist'.format(fpath))
elif force:
print('[!] Unit Level Feature [ {} ] is not exist, Ignoring...'.format(fpath))
if not force or (force and exist) :
with h5py.File(fpath) as ff:#, cv2.VideoCapture(vpath) as capture:
#capture = cv2.VideoCapture(vpath)
#length = capture.get(cv2.CAP_PROP_FRAME_COUNT)
#fps = capture.get(cv2.CAP_PROP_FPS)
#capture.release()
print('[*] Current Video => [ {} ]'.format(fpath))
fps = np.asarray(ff['fps'])
units = list(ff.keys()) # name of units-level feature
fduration = []
#annotations['annotations'][v] = []
for fs in range(len(j['database'][v]['annotations'])): # each video contains more than one foreground segment
fduration.append((np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps).astype(int)) # frame duration [start frame, end frame]
#annotations['annotations'][v].append(fduration[fs])
print(fduration[fs])
for u in units:
#print('units : {}'.format(u))
if u == 'feature_size' or u == 'unit_size' or u == 'sample_rate' or u == 'nframes' or u == 'fps':
continue
fg = False
#print(len(j['database'][v]['annotations']))
for fs in range(len(j['database'][v]['annotations'])): # each video contains more than one foreground segment
#print('fs {}'.format(fs))
#print(np.asarray(j['database'][v]['annotations'][fs]['segment']) * 8)
# convert start time and end time to start frame and end frame respectively
"""
foreground duration : time * fps => index of frame
proposal duration : index of frame
"""
#fduration = (np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps).astype(np.int) # frame duration [start frame, end frame]
pduration = np.asarray(u.split('_'), dtype = int)
iou = unit.iou(pduration, fduration[fs], 'clip')
if not iou:
break
"""
if iou < 1 and iou > 0:
print('Duration Time : {}'.format(j['database'][v]['annotations'][fs]['segment']))
print('Duration Time : {}'.format(np.asarray(j['database'][v]['annotations'][fs]['segment']) * fps))
print('fduration : {}'.format(fduration[fs]))
print('pduration : {}'.format(pduration))
print('iou : {}'.format(iou))
"""
if iou > threshold:
vu = '{}_{}'.format(v, u)
if fg_counter >= tt:
dictionary['test']['foreground'].append(vu)
dictionary['test']['response'][vu] = '{}_{}'.format(fduration[fs][0], fduration[fs][1])
dictionary['test']['iou'][vu] = iou
fg_counter = 0
else:
dictionary['train']['foreground'].append(vu)
dictionary['train']['response'][vu] = '{}_{}'.format(fduration[fs][0], fduration[fs][1])
dictionary['train']['iou'][vu] = iou
fg_counter += 1
fg = True
break
if not fg:
if bg_counter >= tt:
dictionary['test']['background'].append('{}_{}'.format(v, u))
bg_counter = 0
else:
dictionary['train']['background'].append('{}_{}'.format(v, u))
bg_counter += 1
#print(ujson.dumps(annotations))
db['train'].Put('annotations'.encode(), ujson.dumps(dictionary['train']).encode())
if tt > 1:
db['test'].Put('annotations'.encode(), ujson.dumps(dictionary['test']).encode())
db['train'].Write(b, sync = True)
print('[*] Cache is save to [ {} ]'.format(dataset + '_threshold_{}_train'.format(threshold)))
if tt > 1:
db['test'].Write(b, sync = True)
print('[*] Cache is save to [ {} ]'.format(dataset + '_threshold_{}_test'.format(threshold)))
print('[!] Missing {} Video Files'.format(len(miss)))
def __init__(self, dbfile):
if dbfile not in databases:
databases[dbfile] = leveldb.LevelDB(dbfile)
self.db = databases[dbfile]
self.uncommitted = {}
try:
tld = tldextract.extract(urlparse(uri).netloc)
except UnicodeError:
return
print tld.domain.encode("utf8", "ignore"), json.dumps(res)
key = " ".join((tld, uri, args.crawl))
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('db', help='leveldb root directory')
args = parser.parse_args(sys.argv[1:])
db = leveldb.LevelDB(args.db)
stats = defaultdict(lambda: defaultdict(int))
header = None
full_domain = None
valid_languages = [l.lower() for l in args.lang]
for linenr, line in enumerate(sys.stdin):
if line.startswith(magic_number):
header = parse_line(line)
full_domain = "%s.%s" % (header["domain"], header["tld"])
continue
lang, percent, confidence = line.split()
import leveldb
db = leveldb.LevelDB('/Learning/NLP100knock/section7/music_list.ldb')
count = 0
for k, v in db.RangeIter():
if v == 'Japan'.encode('utf-8'):
count += 1
print(count)
def __init__(self, dbfile):
self.dbfile = dbfile
if dbfile not in databases:
databases[dbfile] = (
leveldb.LevelDB(dbfile), dict(), threading.Lock())
self.db, self.uncommitted, self.lock = databases[dbfile]
def main(leveldb_dir, limit):
datum = caffe_pb2.Datum()
db = leveldb.LevelDB(leveldb_dir)
for i in range(0, limit):
datum.ParseFromString(db.Get(str(i)))
print datum.float_data, datum.label
import json
import leveldb
from settings import LEVELDB_PATH
from pysyncobj import SyncObj, SyncObjConf, replicated
ldb = leveldb.LevelDB(LEVELDB_PATH)
class KVStorage(SyncObj):
def __init__(self, selfAddress, partnerAddrs, dumpFile=None):
self.ldb = ldb
conf = SyncObjConf(fullDumpFile=dumpFile, )
super(KVStorage, self).__init__(selfAddress, partnerAddrs, conf)
def get(self, key):
try:
if not isinstance(key, bytes):
key = key.encode()
value = self.ldb.Get(key)
value = json.loads(value.decode())
except KeyError:
value = ""
return value
# @replicated
def set(self, key, value):
if isinstance(key, bytes):
key = key.decode()
if isinstance(value, bytes):
value = value.decode()
value = json.dumps(value)
if not (options.host and options.dbfile):
optparser.print_help()
print "Starting leveldb-server %s" % options.host
context = zmq.Context()
frontend = context.socket(zmq.XREP)
frontend.bind(options.host)
backend = context.socket(zmq.XREQ)
backend.bind('inproc://backend')
poll = zmq.Poller()
poll.register(frontend, zmq.POLLIN)
poll.register(backend, zmq.POLLIN)
db = leveldb.LevelDB(options.dbfile)
workers = []
for i in xrange(3):
worker = workerThread(context, db)
worker.start()
workers.append(worker)
try:
while True:
sockets = dict(poll.poll())
if frontend in sockets:
if sockets[frontend] == zmq.POLLIN:
msg = frontend.recv_multipart()
backend.send_multipart(msg)
import leveldb
import sys
from utils import to_hex
if __name__ == '__main__':
server_id = sys.argv[1]
idx_ETH = sys.argv[2]
idx_TOK = sys.argv[3]
masked_amt_ETH = to_hex(sys.argv[4])
masked_amt_TOK = to_hex(sys.argv[5])
file = f"Scripts/hbswap/data/Pool-P{server_id}.data"
pool_ETH, pool_TOK = 0, 0
with open(file, 'rb') as f:
pool_ETH = f.read(16)
pool_TOK = f.read(16)
db = leveldb.LevelDB(f"Scripts/hbswap/db/server{server_id}")
mask_share_ETH = bytes(db.Get(idx_ETH.encode()))
mask_share_TOK = bytes(db.Get(idx_TOK.encode()))
file = f"Persistence/Transactions-P{server_id}.data"
with open(file, 'wb') as f:
f.write(pool_ETH + pool_TOK + mask_share_ETH + mask_share_TOK +
masked_amt_ETH + masked_amt_TOK)
def __init__(self, filename):
self.db = leveldb.LevelDB(filename)
self._last_key = None
アーティスト名（name）から活動場所（area）を検索するためのデータベースを構築せよ．
'''
import sys
import gzip
import json
import leveldb
def message(text):
sys.stderr.write(f"\33[92m{text}\33[0m\n")
fname = 'artist.json.gz'
db_name = 'name2area'
db = leveldb.LevelDB(db_name)
if sum(1 for _ in db.RangeIter(include_value=False)) == 921337:
message("[*] skip")
else:
with gzip.open(fname, 'rt') as f:
for json_data in map(json.loads, f):
name = f"{json_data['name']}\t{json_data['id']}"
area = json_data.get('area', '""')
db.Put(name.encode(), area.encode())
fname_db_size = sum(1 for _ in db.RangeIter(include_value=False))
message(f'[+] {db_name} のサイズ -> {fname_db_size}')
'''
* LevelDB
- https://github.com/google/leveldb
def __init__(self, dbfile):
self._nbop = 0
self._db = leveldb.LevelDB(dbfile)
self._batch = leveldb.WriteBatch()
return
def test_load_complex_pgn(self):
# 131563
db = leveldb.LevelDB('book/polyglot_index.db')
self.get_game(db, 131563)
self.get_game(db, 523009)
import leveldb
db = leveldb.LevelDB('./db')
# single put
db.Put('hello', 'hello world')
print db.Get('hello')
# multiple put/delete applied atomically, and committed to disk
batch = leveldb.WriteBatch()
batch.Put('hello', 'world')
batch.Put('hello again', 'world')
batch.Delete('hello')
db.Write(batch, sync=True)
import gzip
import json
import leveldb
if __name__ == '__main__':
db = leveldb.LevelDB('./lvdb')
with gzip.open(filename='artist.json.gz') as f:
for line in f:
data_json = json.loads(line.decode().rstrip())
key = data_json['name']
value = data_json.get('area')
if value:
db.Put(key.encode(), value.encode())
def __init__(self, dbpath):
self._map_db = leveldb.LevelDB(dbpath)
def main():
db_id = leveldb.LevelDB('./db')
db_seq = leveldb.LevelDB('./dc')
db_qua = leveldb.LevelDB('./de')
#lid = []
num = 0
fp = open("./test.fastq", "r")
#fp = open("./all.pacbio.fastq", "r")
while 1:
line = fp.readline()
if not line:
break
#print str(line)
#if (str(line)[0] == "@"):
num += 1
sid = 'ps' + repr(num).zfill(8)
#lid.append(sid)
db_id.Put(sid, line.strip('\n'))
line = fp.readline()
db_seq.Put(sid, line.strip('\n'))
line = fp.readline()
line = fp.readline()
db_qua.Put(sid, line.strip('\n'))
fp.close()
adjust = "????????????????????????????????????????????????????????????????????????????????"
c_pos = 0
c_chr = ""
c_seq = ""
c_qua = ""
t_seq = ""
t_qua = ""
fp = open("./test.pass", "r")
while 1:
line = fp.readline()
if not line:
break
elif line[0] == "#":
pass
else:
t = line.split("\t")
if t[0] == c_chr:
if c_pos > int(t[1]) - 1:
pass
else:
c_seq += t_seq[c_pos + 1:int(t[1])]
c_qua += t_qua[c_pos + 1:int(t[1])]
c_seq += t[4]
c_qua += adjust[0:len(t[4])]
c_pos = int(t[1]) - 1 + len(t[3]) - 1
else:
if c_pos:
c_seq += t_seq[(c_pos + 1):len(t_seq)]
c_qua += t_qua[(c_pos + 1):len(t_qua)]
#'''
db_seq.Put(c_chr,c_seq)
db_qua.Put(c_chr,c_qua)
#'''
'''
print $seq_hash{$c_chr}{"t"};
print "\n";
print "$c_seq\n";
print "\n\+\n";
print "$c_qua\n";
$ced_hash{$c_chr} += 1;
'''
#last tail;
c_pos = 0 # a new seq begin
c_chr = t[0]
#print c_chr
c_seq = ""
c_qua = ""
#'''
t_seq = db_seq.Get(c_chr)
t_qua = db_qua.Get(c_chr)
db_seq.Delete(c_chr)
db_qua.Delete(c_chr)
#'''
c_seq += t_seq[c_pos:int(t[1])]
c_qua += t_qua[c_pos:int(t[1])]
c_seq += t[4]; # change
c_qua += adjust[0:len(t[4])];
c_pos = int(t[1]) - 1 + len(t[3]) - 1; # 0-start;
fp.close()
c_seq += t_seq[(c_pos + 1):len(t_seq)]
c_qua += t_qua[(c_pos + 1):len(t_qua)]
db_seq.Put(c_chr,c_seq)
db_qua.Put(c_chr,c_qua)
for x in xrange(1,num+1):
sid = 'ps' + repr(num).zfill(8)
print db_id.Get(sid)
print db_seq.Get(sid)
print "+"
print db_qua.Get(sid)
import shutil
shutil.rmtree('db')
shutil.rmtree('dc')
shutil.rmtree('de')
def __init__(self, dbpath):
self._dir_db = leveldb.LevelDB(dbpath)
self.l = Lock()
# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from datetime import datetime
from chotot.items import generalItem
import leveldb
db = leveldb.LevelDB("db/dienlanh")
def insert(item):
db.Put(item['id'].encode('UTF-8'), item['tel'].encode('UTF-8'))
def search(item):
query = db.Get(item['id'].encode('UTF-8'))
return query.decode()
def validate_time(string):
if string == "Tin ưu tiên" or string.find("trước") > -1:
return True
else:
return False
class DienlanhSpider(scrapy.Spider):
name = 'dienlanh'
start_urls = ['http://www.chotot.com/toan-quoc/mua-ban-tu-lanh-may-lanh-may-giat/']
custom_settings = {'FEED_URI': "output/chotot_dienlanh_%(time)s.csv",
'FEED_FORMAT': 'csv'}
def parse(self, response):
item_urls = response.xpath('//a[@class="adItem___2GCVQ"]/@href').extract()
item_infos = response.xpath('//span[@class="item___eld8Q"]/text()').extract()
import leveldb
import binascii
import numpy as np
import caffe
import sys
from caffe.proto import caffe_pb2
## parse input argument
dbName = sys.argv[1]
# open leveldb files
db = leveldb.LevelDB(dbName)
# get db iterator
it = db.RangeIter()
f1 = open('./mdb_reader2.txt', 'w+')
for key, value in it:
# convert string to datum
datum = caffe_pb2.Datum.FromString(db.Get(key))
# convert datum to numpy string
f1.write('key:' + str(key) + '\n')
arr = caffe.io.datum_to_array(datum)
i = 0
# convert to svm format
# print 1 key and 10 probability only, only for googlenet and vgg
import leveldb
DB = './db'
HELP = '''usage: leveldb_test.py [-h] [-i count] [-q times]
desc.:
[-h] --- get usage help
[-i count] --- insert count of number into database
[-q times] --- query the database the given times and get the QPS
'''
MAX = 10000 # random max number
TIME = 10000 # query times
QUERY = 1 # the key to query, init here, and random in use place
# connect the database
db = leveldb.LevelDB(DB)
def insert():
print 'insert count :%s' % MAX
# insert data into database
for i in range(MAX):
key = str(i)
value = str(random.randint(0, MAX))
#print key, value
db.Put(key, value)
def query():
print 'query times: %d' % TIME
start_time = int(round(time.time() * 1000))
def main(argv):
leveldb_name = sys.argv[1]
db_save_name = sys.argv[4]
print "%s" % sys.argv[1]
print "%s" % sys.argv[2]
print "%s" % sys.argv[3]
print "%s" % sys.argv[4]
print "%s" % sys.argv[5]
temp = int(sys.argv[5])
# window_num = 1000;
# window_num = 12736;
window_num = int(sys.argv[2])
# window_num = 2845;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
db_save = lmdb.open(db_save_name, map_size=int(1e12))
#batch = leveldb.WriteBatch()
datum_save = feat_helper_pb2.Datum()
#ft = np.zeros((window_num, int(sys.argv[3])))
for im_idx in range(window_num):
key_str = '%08d' % (im_idx)
datum.ParseFromString(db.Get(key_str))
ft0 = np.array(datum.float_data)
#print ft0.shape
ft1 = ft0 / temp
m_ft1 = ft1.max()
ft1 = ft1 - m_ft1
exp_ft1 = np.exp(ft1)
exp_ft1_s = exp_ft1.sum()
p_ft1 = exp_ft1 / exp_ft1_s
del datum_save.float_data[:]
datum_save.float_data.extend(p_ft1.tolist())
#batch.Put(key_str, datum_save.SerializeToString())
db_save.begin(write=True).put(key_str, datum_save.SerializeToString())
if (im_idx % 10000 == 0):
print "%d images processed." % (im_idx)
# if (im_idx % 500 == 0 ):
# #print np.array(datum.float_data)
# print "sample %d" %(im_idx)
# print p_ft1.max()
# #calculate p when t=1
# m_ft0 = ft0.max()
# ft0 = ft0 - m_ft0
# exp_ft0 = np.exp(ft0)
# exp_ft0_s = exp_ft0.sum()
# p_ft0 = exp_ft0 / exp_ft0_s
# print p_ft0.max()
#write leveldb
#db_save.Write(batch, sync = True)
db_save.close()
print 'temperature is %d' % (temp)
print 'time 1: %f' % (time.time() - start)
#sio.savemat(sys.argv[4], {'feats':ft},oned_as='row')
print 'time 2: %f' % (time.time() - start)
print 'done!'
"""
661. KVSの検索
60で構築したデータベースを用い，特定の（指定された）アーティストの活動場所を取得せよ．
"""
import gzip
import json
import leveldb
begin = 0
end = 10
db = leveldb.LevelDB('./artist_db')
name = input('Who ? :')
for ii, data in enumerate(db.RangeIter()):
# if ii >= begin and ii <= end:
#print(list(data))
#print(data[0].decode("utf-8", "ignore"), data[1].decode("utf-8", "ignore"))
A = name.strip()
B = data[0].decode("utf-8", "ignore")
if A == B:
print('Exactly Match!')
print('Musician:{}\tActivity Area :{}'.\
format(B,data[1].decode("utf-8", "ignore")))
"""
else:
if len(set(A) & set(B)) > 3:
print('May be you want')
print('Musician:{}\tActivity Area :{}'.\
import csv
data = csv.DictReader(open('World-z10.tsv'), delimiter="\t")
import fiona
import json
import leveldb
db = leveldb.LevelDB("db/geonames")
for row in data:
geonameid = str(row['geonameid'])
try:
correct_name = db.Get(geonameid)
if row['name'] != correct_name:
print "{0} should be {1}".format(row['name'], correct_name)
except Exception:
pass
