def __init__(self, fp, dataDir, settings, chain): self.is_running = True self.fail_code = 0 self.fp = fp self.dataDir = dataDir self.settings = settings self.core_version = None # Set during start() self.daemon_running = False self.blockBuffer = 100 # Work n blocks from the tip to avoid forks, should be > COINBASE_MATURITY self.mode = settings.get('mode', 'master') self.binDir = os.path.expanduser(settings['ghostbindir']) self.ghostDataDir = os.path.expanduser(settings['ghostdatadir']) self.chain = chain self.debug = settings.get('debug', DEBUG) self.poolAddrHrp = 'gcs' if self.chain == 'mainnet' else 'tpcs' self.poolAddr = settings['pooladdress'] self.poolAddrReward = settings['rewardaddress'] self.poolHeight = settings.get('startheight', 0) self.maxOutputsPerTx = settings.get('maxoutputspertx', 48) # Default parameters self.poolFeePercent = 2 self.stakeBonusPercent = 5 self.payoutThreshold = int(0.5 * COIN) self.minBlocksBetweenPayments = 100 # Minimum number of blocks between payment runs self.minOutputValue = int( 0.1 * COIN ) # Ignore any outputs of lower value when accumulating rewards self.tx_fee_per_kb = None self.smsg_fee_rate_target = None self.dbPath = os.path.join(dataDir, 'stakepooldb') db = plyvel.DB(self.dbPath, create_if_missing=True) n = db.get(bytes([DBT_DATA]) + b'current_height') if n is None: logmt(self.fp, 'First run\n') db.put( bytes([DBT_DATA]) + b'db_version', struct.pack('>i', CURRENT_DB_VERSION)) else: self.poolHeight = struct.unpack('>i', n)[0] self.lastHeightParametersSet = -1 self.setParameters(self.poolHeight) self.zmqContext = zmq.Context() self.zmqSubscriber = self.zmqContext.socket(zmq.SUB) self.zmqSubscriber.connect(self.settings['zmqhost'] + ':' + str(self.settings['zmqport'])) self.zmqSubscriber.setsockopt_string(zmq.SUBSCRIBE, 'hashblock') self.debugDir = os.path.join(dataDir, 'poolDebug') if self.debug and not os.path.exists(self.debugDir): os.makedirs(self.debugDir) with open(os.path.join(self.debugDir, 'pool.csv'), 'a') as fp: fp.write( 'height,blockReward,blockOutput,poolReward,poolRewardTotal,poolCoinTotal,Disbursed,fees,totalFees\n' ) if self.mode == 'master': try: self.min_blocks_between_withdrawals = self.settings[ 'poolownerwithdrawal']['frequency'] assert (self.min_blocks_between_withdrawals > self.blockBuffer) self.owner_withdrawal_addr = self.settings[ 'poolownerwithdrawal']['address'] assert (self.settings['poolownerwithdrawal']['reserve'] >= 0.005) assert (self.settings['poolownerwithdrawal']['threshold'] >= 0.0) self.have_withdrawal_info = True except Exception: traceback.print_exc() self.have_withdrawal_info = False # If pool was synced in observer mode 'pool_fees_detected' may be higher than 'pool_fees' # 'pool_fees_detected' is tracked at chain tip - buffer, while 'pool_fees' is tracked as the pool makes transactions n = db.get(bytes([DBT_DATA]) + b'pool_fees_detected') pool_fees_detected = 0 if n is None else int.from_bytes(n, 'big') dbkey = bytes([DBT_DATA]) + b'pool_fees' n = db.get(dbkey) pool_fees = 0 if n is None else int.from_bytes(n, 'big') if pool_fees_detected > pool_fees: logmt( self.fp, 'Replacing pool_fees with pool_fees_detected: %s, %s' % (format8(pool_fees), format8(pool_fees_detected))) db.put(dbkey, pool_fees_detected.to_bytes(8, 'big')) else: self.have_withdrawal_info = False addr = db.get(bytes([DBT_DATA]) + b'pool_addr') if addr is not None: self.poolAddr = bech32Encode(self.poolAddrHrp, addr) else: db.put( bytes([DBT_DATA]) + b'pool_addr', bech32Decode(self.poolAddrHrp, self.poolAddr)) addr = db.get(bytes([DBT_DATA]) + b'reward_addr') if addr is not None: self.poolAddrReward = encodeAddress(addr) else: db.put( bytes([DBT_DATA]) + b'reward_addr', decodeAddress(self.poolAddrReward)) n = db.get(bytes([DBT_DATA]) + b'db_version') self.db_version = 0 if n is None else struct.unpack('>i', n)[0] db.close() # Wait for daemon to start authcookiepath = os.path.join( self.ghostDataDir, '' if self.chain == 'mainnet' else self.chain, '.cookie') for i in range(10): if not os.path.exists(authcookiepath): time.sleep(0.5) with open(authcookiepath) as fp: self.rpc_auth = fp.read() # Todo: Read rpc port from .conf file self.rpc_port = settings.get( 'rpcport', 51725 if self.chain == 'mainnet' else 51925)
#!/usr/bin/python3 import plyvel import json import sys from os import path data = json.load(sys.stdin) if len(sys.argv) < 3: sys.stderr.write("configure.py chrome-dir ext-id") sys.exit(1) chrome_dir = sys.argv[1] ext_id = sys.argv[2] db_path = path.join(chrome_dir, "Default/Local Extension Settings", ext_id) db = plyvel.DB(db_path, create_if_missing=True) for k, v in data.items(): db.put(bytes(k, encoding='utf-8'), bytes(v, encoding='utf-8')) db.close()
def num2Bytes(n): return n.to_bytes(8, "big") def addressFromHex(hex_str): return tronapi.common.account.Address().from_hex(hex_str) def addressFromBytes(addr): return tronapi.common.account.Address().from_hex(bytes.decode(b2hs(addr))) # 会遇到问题 UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb6 in position 3: invalid start byte # db = plyvel.DB("/data2/20210425/output-directory/database/trans") blockIndexDB = plyvel.DB( "/data2/20210425/output-directory/database/block-index") blockDB = plyvel.DB("/data2/20210425/output-directory/database/block") blockIndexIT = blockIndexDB.iterator() k, v = next(blockIndexIT) k = b2l(k) print(k) vs = binascii.hexlify(v) print(vs) blockIndexITrev = blockIndexDB.iterator(reverse=True) k, v = next(blockIndexITrev) k = b2l(k) print(k)
def post_init(self): import plyvel self._db = plyvel.DB(self.data_path, create_if_missing=True)
def open(self): #check if pickle exists self.db = plyvel.DB(self.name + '/leveldb/', create_if_missing=True)
def _init_db(db): lookup = plyvel.DB(os.path.abspath(db)) return lookup
def writelines(self, datas): self.stream.writelines(datas) self.stream.flush() def __getattr__(self, attr): return getattr(self.stream, attr) import sys sys.stdout = Unbuffered(sys.stdout) # ======== DO NOT TOUCH ================== level_dan = plyvel.DB("./danju-gzipJson.ldb", create_if_missing=True, compression=None) level_shuang = plyvel.DB("./shuangju-gzipJson.ldb", create_if_missing=True, compression=None) Data = glob.glob("./json-zhcn/poet.song.*.json") + glob.glob( "./json-zhcn/poet.tang.*.json") #======= 先制造集合,再转成正则 ========== Comma = ",,。、!!??" # 分隔符,逗号/顿号/句号/感叹号/问号 NotComma = "".join( list(set(zhon.hanzi.punctuation + string.punctuation) - set(Comma))) Comma = "[%s]" % (re.escape(Comma))
with levelDBMapLock: if index not in levelDBMap: db = plyvel.DB(index, compression=None) blockIndexes = [ DBBlockIndex(format_hash(k[1:]), v) for k, v in db.iterator() if k[0] == ord('b') ] db.close() blockIndexes.sort(key=lambda x: x.height) levelDBMap[index] = blockIndexes return levelDBMap[index] blockchain.__getBlockIndexes = loadIndex pldb = plyvel.DB('/blockchain/tx_to_block/', create_if_missing=False) pldbLock = threading.Lock() app.logger.debug("OK") class IllegalState(Exception): pass @app.before_request def before_request(): g.request_start_time = time.time() g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time)
def LevelDBAggregator(manager_params, status_queue, batch_size=100): """ Receives <key, value> pairs from other processes and writes them to the central database. Executes queries until being told to die (then it will finish work and shut down). This process should never be terminated un-gracefully. <manager_params> TaskManager configuration parameters <status_queue> is a queue connect to the TaskManager used for communication <batch_size> is the size of the write batch """ # sets up logging connection logger = loggingclient(*manager_params['logger_address']) # sets up the serversocket to start accepting connections sock = serversocket() status_queue.put(sock.sock.getsockname()) # let TM know location sock.start_accepting() # sets up DB connection db_path = os.path.join(manager_params['data_directory'], 'javascript.ldb') db = plyvel.DB(db_path, create_if_missing = True, lru_cache_size = 10**9, write_buffer_size = 128*10**4, compression = None) batch = db.write_batch() counter = 0 # number of executions made since last write commit_time = 0 # keep track of time since last write while True: # received KILL command from TaskManager if not status_queue.empty(): status_queue.get() sock.close() drain_queue(sock.queue, batch, db, counter, logger) break # no command for now -> sleep to avoid pegging CPU on blocking get if sock.queue.empty(): time.sleep(0.1) # commit every five seconds to avoid blocking the db for too long if counter > 0 and time.time() - commit_time > 5: batch.write() batch = db.write_batch() continue # process record script = sock.queue.get() process_script(script, batch, db, counter, logger) # batch commit if necessary if counter >= batch_size: counter = 0 commit_time = time.time() batch.write() batch = db.write_batch() # finishes work and gracefully stops batch.write() db.close()
help='how number of url this program will scrape') parser.add_argument('--mode', help='you can specify mode...') parser.add_argument('--refresh', help='create snapshot(true|false)') parser.add_argument('--file', help='input filespath') parser.add_argument('--active', help='spcific active thread number') args_obj = vars(parser.parse_args()) depth = (lambda x: int(x) if x else 10)(args_obj.get('depth')) mode = (lambda x: x if x else 'undefined')(args_obj.get('mode')) refresh = (lambda x: False if x == 'false' else True)(args_obj.get('refresh')) active = (lambda x: 15 if x == None else int(x))(args_obj.get('active')) filename = args_obj.get('file') if mode == 'scrape' or mode == 'level': db = plyvel.DB('pixiv_htmls', create_if_missing=True) """ SnapshotDealデーモンが出力するデータをもとにスクレイピングを行う NOTE: SQLの全アクセスは非常に動作コストが高く、推奨されない NOTE: Snapshotが何もない場合、initialize_parse_and_map_data_to_local_dbを呼び出して初期化を行う """ seed = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=60675452" def save_links(links): """ 一応 pickleで保存 """ open("links.pkl", "wb").write(pickle.dumps(links)) try: links = pickle.loads(open("links.pkl", "rb").read()) links.add(seed) except FileNotFoundError as e:
def __init__(self): self.tmpdir = tempfile.TemporaryDirectory() self.db = DB_ENGINE.DB(self.tmpdir.name, create_if_missing=True)
#!/usr/bin/env python3 from datetime import datetime import plyvel print('Recovery Start : ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) plyvel.DB('./.storage/db_icon_dex', create_if_missing=True, max_open_files=1024) print('Recovery End : ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
def main(): my_DataBese = plyvel.DB('test.ldb', create_if_missing=True) print my_DataBese.get(sys.argc[1])
import json import gzip import plyvel import pickle my_db = plyvel.DB('database', create_if_missing = True) with gzip.open('artist.json.gz', 'r', 'utf-8') as f: for line in f: obj = json.loads(line.decode('utf-8')) name = obj['name'] if 'area' in obj: area = obj['area'] else: area = '無い' my_db.put(pickle.dumps(name), pickle.dumps(area)) my_db.close()
def make_db(path: str, create_if_missing: bool = True) -> plyvel.DB: if not os.path.exists(path): os.makedirs(path) return plyvel.DB(path, create_if_missing=create_if_missing)
def initialize(): db = plyvel.DB('/home/jiapeng/leveldb', create_if_missing=True) return db
LOG = [] # The log as a list of dictionaries AUTHORS = [] # The list if author names DATES = [] # The first days the authors commit FONT = ImageFont.truetype('/usr/share/fonts/truetype/ubuntu-font-family/Ubuntu-M.ttf', 12) LONGEST_NAME = 0 LONGEST_DATE = 0 NUM_WEEKS = 0 FIRST_DATE = None LAST_DATE = None def find_week(date): return int((date - FIRST_DATE).days / 7) if __name__ == '__main__': commitdb = plyvel.DB('../data/commits/', create_if_missing=True) orgs = commitdb.prefixed_db(b'zalando') project = orgs.prefixed_db(b'nakadi') # Data preparation for date, commit_data in project: commit = json.loads(commit_data) # Convert the date to a datetime obj dt = parser.parse(date) if not FIRST_DATE: FIRST_DATE = dt LAST_DATE = dt # Check if we know the name # Check if this is the longest name if not commit['commit']['author']['name'] in AUTHORS:
from core import core, transmission from storage import config import plyvel import json import os db = plyvel.DB(os.getenv('DB_PATH_OVERRIDE', config.get('database', 'path')), create_if_missing=True) def get_head(): ''' Get the blockId of the newest block ''' return db.get('head'.encode('utf-8')).decode('utf-8') def _set_head(blockId): ''' Internal function: Set newest blockId ''' db.put('head'.encode('utf-8'), blockId.encode('utf-8')) def put_block(block): ''' Put a core/Transmission block into the chain ''' if not isinstance(block, transmission.Transmission): raise TypeError('Not a Core.Transmission block object') assert block.previous_hash == get_head() db.put(block.transmission_hash.encode('utf-8'), block.to_json().encode('utf-8')) _set_head(block.transmission_hash)
def init_leveldb(self): self.db_attack = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/train_data_attack/") self.db_noattack = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/train_data_noattack/") self.db_attack_test = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_attack/") self.db_noattack_test = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_noattack/") self.db_attack_poi = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/train_data_attack_poi/") self.db_noattack_poi = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/train_data_noattack_poi/" ) self.db_attack_test_poi = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_attack_poi/") self.db_noattack_test_poi = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_noattack_poi/") self.db_partial_attack = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_partial_attack/", ) self.db_partial_attack_google = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_partial_attack_google/", ) self.db_partial_attack_poi = plyvel.DB( "/home/bourne/Workstation/AntiGPS/results/test_data_partial_attack_poi/", create_if_missing=True, )
#!/usr/bin/python3 import plyvel import pickle import sys import time import block import os ts = time.time() db = plyvel.DB("/home/ben/mof-bc") blocks = 0 with db.iterator() as it: for k,v in it: if k == b'last': continue blocks += 1 block = pickle.loads(v) if block.merkle_tree.root != 'root': block.merkle_tree.print_tree_txs() else: print(block, '-> root') tf = time.time() print(blocks, 'blocks') print("Time taken", tf-ts, "seconds")
for x in args: while x: ret += bytes([(x & 0x7f) | (int(x > 0x07f) << 7)]) x >>= 7 return ret def unvarint(bs): ret = 0 for byte in reversed(bs): ret <<= 7 ret |= byte & (0x7f) return ret db = plyvel.DB('../server/run') # list(db.iterator(start=b'M'+varint(1,1), stop=b'M'+varint(1,2))) ssm = lambda x: SignedServerMessage.FromString(x) sm = lambda x: SignedServerMessage.ServerMessage.FromString(x) def SSM(*args): return [ ssm(v) for (k, v) in db.iterator(start=b'M' + varint(*args), stop=b'M' + varint(*(list(args[:-1]) + [args[-1] + 1]))) ]
if (dty < 3): num += 1 dtyc = np_utils.to_categorical(dty, 2) dtyc.shape = (1, 2) df = (dtx, dtyc, log) df1 = pickle.dumps(df, protocol=2) db.put(struct.pack('>l', num), df1) del (dtx) del (df) del (df1) return (num) if __name__ == '__main__': db = plyvel.DB('../eegdb5_10/', create_if_missing=True) a = glob.glob('../Raw/*.vhdr') j = 0 k = str(len(a)) num = 0 for i in a: j += 1 log = str(j) + "/" + k + " " + str(i) alldata, allid = readraweeg(i, (-.5, .5)) num = putdb(alldata, allid, db, j, log, num=num) indexdb = np.random.randint(num, size=num) np.savetxt("indexdb5_10.csv", indexdb) print(num) db.close()
red = redis.Redis(host="localhost", port=6379, db=0) red.flushall() blockchain = Blockchain("datas") # force the creation of the index if os.path.exists("super-big-index.pickle"): os.remove("super-big-index.pickle") print("Creating index") next( blockchain.get_ordered_blocks("datas/index", cache="super-big-index.pickle")) print("Index created") pldb = plyvel.DB('tx_to_block/', create_if_missing=False) class IllegalState(Exception): pass @app.before_request def before_request(): g.request_start_time = time.time() g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time) def get_block_transactions(block_height): blockchain = Blockchain("datas")
def create_benchmark_archive(blk_hash): blk = json.loads(subprocess.check_output([BITCOINZ_CLI, 'getblock', blk_hash])) print 'Height: %d' % blk['height'] print 'Transactions: %d' % len(blk['tx']) os.mkdir('benchmark') with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f: f.write(binascii.unhexlify(subprocess.check_output([BITCOINZ_CLI, 'getblock', blk_hash, 'false']).strip())) txs = [json.loads(subprocess.check_output([BITCOINZ_CLI, 'getrawtransaction', tx, '1']) ) for tx in blk['tx']] js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0]) if js_txs: print 'Block contains %d JoinSplit-containing transactions' % js_txs return inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin'] if x.has_key('txid')] print 'Total inputs: %d' % len(inputs) unique_inputs = {} for i in sorted(inputs): if unique_inputs.has_key(i[0]): unique_inputs[i[0]].append(i[1]) else: unique_inputs[i[0]] = [i[1]] print 'Unique input transactions: %d' % len(unique_inputs) db_path = 'benchmark/block-%d-inputs' % blk['height'] db = plyvel.DB(db_path, create_if_missing=True) wb = db.write_batch() bar = progressbar.ProgressBar(redirect_stdout=True) print 'Collecting input coins for block' for tx in bar(unique_inputs.keys()): rawtx = json.loads(subprocess.check_output([BITCOINZ_CLI, 'getrawtransaction', tx, '1'])) mask_size = 0 mask_code = 0 b = 0 while 2+b*8 < len(rawtx['vout']): zero = True i = 0 while i < 8 and 2+b*8+i < len(rawtx['vout']): if 2+b*8+i in unique_inputs[tx]: zero = False i += 1 if not zero: mask_size = b + 1 mask_code += 1 b += 1 coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0] first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx] second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx] code = 8*(mask_code - (0 if first or second else 1)) + \ (1 if coinbase else 0) + \ (2 if first else 0) + \ (4 if second else 0) coins = bytearray() # Serialized format: # - VARINT(nVersion) coins.extend(encode_varint(rawtx['version'])) # - VARINT(nCode) coins.extend(encode_varint(code)) # - unspentness bitvector, for vout[2] and further; least significant byte first for b in range(mask_size): avail = 0 i = 0 while i < 8 and 2+b*8+i < len(rawtx['vout']): if 2+b*8+i in unique_inputs[tx]: avail |= (1 << i) i += 1 coins.append(avail) # - the non-spent CTxOuts (via CTxOutCompressor) for i in range(len(rawtx['vout'])): if i in unique_inputs[tx]: coins.extend(encode_varint(compress_amount(int(rawtx['vout'][i]['valueZat'])))) coins.extend(compress_script( binascii.unhexlify(rawtx['vout'][i]['scriptPubKey']['hex']))) # - VARINT(nHeight) coins.extend(encode_varint(json.loads( subprocess.check_output([BITCOINZ_CLI, 'getblockheader', rawtx['blockhash']]) )['height'])) db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1]) db_val = bytes(coins) wb.put(db_key, db_val) wb.write() db.close() # Make reproducible archive os.remove('%s/LOG' % db_path) files = subprocess.check_output(['find', 'benchmark']).strip().split('\n') archive_name = 'block-%d.tar' % blk['height'] tar = tarfile.open(archive_name, 'w') for name in sorted(files): tar.add(name, recursive=False, filter=deterministic_filter) tar.close() subprocess.check_call(['xz', '-6', archive_name]) print 'Created archive %s.xz' % archive_name subprocess.call(['rm', '-r', 'benchmark'])
def extraction_core_txs_optimized_four(directory, args): logging.info("processing db dir: " + directory + "/txs") logging.info("algo: extract-core optimized 4") if (args.sorted): logging.info("sorting: True") else: logging.info("sorting: False") if not args.liteonmemory: logging.info("intensive on memory, but fastest.") else: logging.info("lite on memory, but slower.") number_of_addreses = 0 count = 0 show_interval = 1000 currentaddress = ["thisisatemplateusedforbootstrapping-", 0] currentaddresslen = len(currentaddress) list_of_addresses = "" #logging.info("--------------------------------" ) if not args.dry_run: output_file = codecs.open(args.output_file, 'w', 'utf8') for key, value in plyvel.DB(directory + "/txs", create_if_missing=False): #print key[0][0:2] if key[2] != "a": break if not currentaddress == key[5:currentaddresslen + 5]: currentaddress = key[5:6 + (key[5:].index("-"))] currentaddresslen = len(currentaddress) number_of_addreses += 1 if not args.liteonmemory: list_of_addresses = list_of_addresses + currentaddress else: if not args.dry_run: output_file.write(currentaddress) count += 1 if (count % show_interval) == 0: logging.info("processed txs: " + "\t" + str(count)) show_interval = show_interval * 2 if not args.dry_run: if not args.liteonmemory: output_file.write(list_of_addresses) output_file.close() if (args.sorted): logging.info('sorting addresses') subprocess.call([ "tr '-' '\n' < " + args.output_file + " | sort -o " + args.output_file ], shell=True) else: logging.info('processing file') subprocess.call([ "tr '-' '\n' < " + args.output_file + " > " + "temp_" + args.output_file ], shell=True) subprocess.call([ "mv -f" + " temp_" + args.output_file + " " + args.output_file ], shell=True) #subprocess.call(["awk -F'-' '{print $2}' " + args.output_file + " | sort -u -o " + args.output_file], shell=True) logging.info('output file: ' + args.output_file) logging.info('# of transactions: ' + str(count)) logging.info(' # of addresses: ' + str(number_of_addreses)) logging.info("total time: " + '\033[92m' + (datetime.datetime.now() - start_time).__str__().split('.')[0] + '\033[0m' + ' on ' + datetime.datetime.today().strftime('%b, %d, %Y')) logging.info('\033[92m' + 'done.' + '\033[0m') print("") print("") exit(0)
def __init__(self, db, records, **kwargs): self.db = plyvel.DB(db, **kwargs, create_if_missing=True)
def main(): my_DataBese = plyvel.DB('test.ldb', create_if_missing=True) print my_DataBese.get('黒木メイサ')
import plyvel import utils2 import types import re import time import timeout_decorator g = types.SimpleNamespace() g.shuangju = plyvel.DB("./shuangju-gzipJson.ldb", compression='snappy') g.danju = plyvel.DB("./danju-gzipJson.ldb", compression='snappy') def danshuangTransfer(startKeys, targetKeys, mode): # 全部key都应该是顺序无关的 if mode == "SHUANGSTART": startShuang = {i: list() for i in startKeys} startDan = {} elif mode == "DANSTART": startShuang = {} startDan = {i: list() for i in startKeys} else: raise Exception("Unknown MODE: <%s>" % mode) targetDan = {i: list() for i in targetKeys} targetShuang = {i: list() for i in targetKeys} print(targetDan, targetShuang, startDan, startShuang, sep="\n")
import plyvel import json db = plyvel.DB('./db_test/', create_if_missing=True) with open('artist.json') as f: for l in f: js = json.loads(l) key = js['name'] + '\t' + str(js['id']) value = js.get('area') if value == None: value = 'None' db.put(key.encode(), value.encode())
def processBlock(self, height): logmt(self.fp, 'processBlock height %d' % (height)) reward = callrpc(self.rpc_port, self.rpc_auth, 'getblockreward', [ height, ]) db = plyvel.DB(self.dbPath, create_if_missing=True) n = db.get(bytes([DBT_DATA]) + b'current_height') if n is not None: poolDBHeight = struct.unpack('>i', n)[0] if poolDBHeight >= height: logmt( self.fp, 'Block %d already processed, pooldb height %d' % (height, poolDBHeight)) self.poolHeight = poolDBHeight db.close() return self.setParameters(height) if 'coinstake' not in reward: # logm('No coinstake txn found in block ' + str(height)) db.put( bytes([DBT_DATA]) + b'current_height', struct.pack('>i', height)) db.close() self.poolHeight = height return batchBalances = dict() b = db.write_batch(transaction=True) b.put(bytes([DBT_DATA]) + b'current_height', struct.pack('>i', height)) self.findPayments(height, reward['coinstake'], db, b, batchBalances) for out in reward['outputs']: try: if self.poolAddrReward == out['script']['spendaddr']: if out['value'] != reward['blockreward']: logmt( self.fp, 'WARNING: Pool reward mismatch at height %d\n' % (height)) try: self.processPoolBlock(height, reward, db, b, batchBalances) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb) traceback.print_exception(exc_type, exc_value, exc_tb, file=self.fp) self.fp.flush() break except Exception: pass b.write() n = db.get(bytes([DBT_DATA]) + b'last_payment_run') lastPaymentRunHeight = 0 if n is None else struct.unpack('>i', n)[0] if lastPaymentRunHeight + self.minBlocksBetweenPayments <= height: with db.write_batch(transaction=True) as b: self.processPayments(height, db, b) if self.have_withdrawal_info: n = db.get(bytes([DBT_DATA]) + b'last_withdrawal_run') last_withdrawal_run = 0 if n is None else struct.unpack('>i', n)[0] if last_withdrawal_run + self.min_blocks_between_withdrawals <= height: with db.write_batch(transaction=True) as b: self.processPoolRewardWithdrawal(height, db, b) db.close() self.poolHeight = height