async def block_transaction_id_list(pointer, limit, page, order, app): pool = app["db_pool"] qt = time.time() async with pool.acquire() as conn: if isinstance(pointer, bytes): stmt = await conn.prepare( "SELECT height FROM blocks WHERE hash = $1 LIMIT 1;") pointer = await stmt.fetchval(pointer) if pointer is None: raise APIException(NOT_FOUND, "block not found", status=404) if app["block_transaction_id_list"].has_key(pointer): transactions = app["block_transaction_id_list"][pointer] else: async with pool.acquire() as conn: rows = await conn.fetch( "SELECT tx_id " "FROM transaction WHERE pointer >= $1 AND pointer < $2 " "ORDER BY pointer %s LIMIT $3 OFFSET $4;" % order, pointer << 39, (pointer + 1) << 39, limit, limit * (page - 1)) transactions = [rh2s(t["tx_id"]) for t in rows] app["block_transaction_id_list"][pointer] = transactions resp = {"data": transactions, "time": round(time.time() - qt, 4)} return resp
async def _get_missed(self, block_hash=False, block_time=None, block_height=None): if block_hash: t = time.time() block = self.block_preload.pop(block_hash) if not block: result = await self.rpc.getblock(block_hash, 0) try: if not block: block = decode_block_tx(result) self.log.info("block downloaded %s" % round(time.time() - t, 4)) for index, tx in enumerate(block): if rh2s(block[tx]["txId"]) in self.missed_tx_list: self.loop.create_task( self._new_transaction(block[tx], block_time, block_height, index)) return except Exception as err: self.log.error("_get_missed exception %s " % str(err)) self.log.error(str(traceback.format_exc())) self.await_tx_list = [] self.block_txs_request.cancel() if self.get_missed_tx_threads > self.get_missed_tx_threads_limit: return self.get_missed_tx_threads += 1 # start more threads if len(self.missed_tx_list) > 1: self.loop.create_task( self._get_missed(False, block_time, block_height)) while True: if not self.missed_tx_list: break try: batch = list() while self.missed_tx_list: batch.append( ["getrawtransaction", self.missed_tx_list.pop()]) if len(batch) >= self.batch_limit: break result = await self.rpc.batch(batch) for r in result: try: tx = Transaction(r["result"], format="raw") except: self.log.error("Transaction decode failed: %s" % r["result"]) raise Exception("Transaction decode failed") self.loop.create_task( self._new_transaction(tx, block_time, None, None)) except Exception as err: self.log.error("_get_missed exception %s " % str(err)) self.log.error(str(traceback.format_exc())) self.await_tx_list = [] self.block_txs_request.cancel() self.get_missed_tx_threads -= 1
async def blocks_last_n_hours(n, app): pool = app["db_pool"] q = time.time() async with pool.acquire() as conn: rows = await conn.fetch( "SELECT height," " hash," " header," " adjusted_timestamp " "FROM blocks " "WHERE adjusted_timestamp >= $1 " "ORDER BY height desc;", int(time.time()) - n * 60 * 60) if rows is None: raise APIException(NOT_FOUND, "blocks not found", status=404) r = [] for row in rows: block = dict() block["height"] = row["height"] block["hash"] = rh2s(row["hash"]) block["header"] = base64.b64encode(row["header"]).decode() block["adjustedTimestamp"] = row["adjusted_timestamp"] r.append(block) resp = {"data": r, "time": round(time.time() - q, 4)} return resp
async def wait_tx_then_add(self, raw_tx_hash, tx): tx_hash = rh2s(tx["hash"]) try: if not self.await_tx_future[raw_tx_hash].done(): await self.await_tx_future[raw_tx_hash] self.loop.create_task(self._new_transaction(tx)) except: self.tx_in_process.remove(tx_hash)
async def tx_hash_by_pointer(pointer, app): q = time.time() async with app["db_pool"].acquire() as conn: row = await conn.fetchrow( "SELECT tx_id " "FROM transaction " "WHERE pointer = $1 LIMIT 1;", pointer) if row is None: raise APIException(NOT_FOUND, "transaction not found", status=404) return {"data": rh2s(row["tx_id"]), "time": round(time.time() - q, 4)}
async def block_utxo(pointer, limit, page, order, app): pool = app["db_pool"] q = time.time() async with pool.acquire() as conn: if isinstance(pointer, bytes): stmt = await conn.prepare("SELECT height " "FROM blocks WHERE hash = $1 LIMIT 1;") pointer = await stmt.fetchval(pointer) if pointer is None: raise APIException(NOT_FOUND, "block not found", status=404) rows = await conn.fetch( "SELECT outpoint, pointer, address, amount " "FROM connector_utxo WHERE pointer >= $1 AND pointer < $2 " "ORDER BY pointer %s LIMIT $3 OFFSET $4;" % order, pointer << 39, (pointer + 1) << 39, limit + 1, limit * (page - 1)) utxo = list() for row in rows[:limit]: address = row["address"] address_type = SCRIPT_N_TYPES[address[0]] if address[0] in (0, 1, 5, 6): script_hash = True if address[0] in (1, 6) else False witness = 0 if address[0] in (1, 6) else None address = hash_to_address(address[1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness) script = address_to_script(address, hex=1) elif address[0] == 2: script = address[1:].hex() address = script_to_address(address[1:], testnet=app["testnet"]) else: script = address[1:].hex() address = None utxo.append({ "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]), "txIndex": (row["pointer"] >> 20) & 1048575, "amount": row["amount"], "address": address, "script": script, "type": address_type }) last_page = False if len(rows) > limit else True resp = { "data": utxo, "time": round(time.time() - q, 4), "lastPage": last_page } return resp
async def _new_transaction(self, tx, block_time=None, block_height=None, block_index=None): tx_hash = rh2s(tx["txId"]) ft = self.await_tx_future if tx_hash in self.await_tx_list else None if tx_hash in self.tx_in_process: return self.tx_in_process.add(tx_hash) # Check is transaction new tx_id = self.tx_cache.get(tx["txId"]) if tx_id is not None: self.tx_in_process.remove(tx_hash) return try: # call external handler if self.tx_handler: tx_id = await self.tx_handler(tx, ft, block_time, block_height, block_index) # insert new transaction to dublicate filter if not self.external_dublicate_filter: tx_id = await insert_new_tx(self, tx["txId"]) self.tx_cache.set(tx["txId"], tx_id) if tx_hash in self.await_tx_list: self.await_tx_list.remove(tx_hash) self.await_tx_id_list.append(tx_id) if not self.await_tx_future[unhexlify(tx_hash)[::-1]].done(): self.await_tx_future[unhexlify(tx_hash)[::-1]].set_result( True) if not self.await_tx_list: self.block_txs_request.set_result(True) except DependsTransaction as err: self.block_dependency_tx += 1 self.loop.create_task(self.wait_tx_then_add(err.raw_tx_hash, tx)) except Exception as err: if tx_hash in self.await_tx_list: self.await_tx_list = [] self.await_tx_id_list = [] self.block_txs_request.cancel() for i in self.await_tx_future: if not self.await_tx_future[i].done(): self.await_tx_future[i].cancel() self.log.debug("new transaction error %s " % err) self.log.debug(str(traceback.format_exc())) finally: self.tx_in_process.remove(tx_hash)
async def block_by_pointer(pointer, app): pool = app["db_pool"] q = time.time() async with pool.acquire() as conn: if pointer == 'last': stmt = await conn.prepare( "SELECT height," " hash," " header," " adjusted_timestamp " "FROM blocks ORDER BY height desc LIMIT 1;") row = await stmt.fetchrow() else: if type(pointer) == bytes: stmt = await conn.prepare( "SELECT height," " hash," " header," " adjusted_timestamp " "FROM blocks WHERE hash = $1 LIMIT 1;") row = await stmt.fetchrow(pointer) elif type(pointer) == int: stmt = await conn.prepare( "SELECT height," " hash," " header," " adjusted_timestamp " "FROM blocks WHERE height = $1 LIMIT 1;") row = await stmt.fetchrow(pointer) if row is None: raise APIException(NOT_FOUND, "block not found", status=404) block = dict() block["height"] = row["height"] block["hash"] = rh2s(row["hash"]) block["header"] = base64.b64encode(row["header"]).decode() block["adjustedTimestamp"] = row["adjusted_timestamp"] resp = {"data": block, "time": round(time.time() - q, 4)} return resp
async def tx_hash_by_pointers(pointers, app): q = time.time() async with app["db_pool"].acquire() as conn: rows = await conn.fetch("SELECT pointer, tx_id " "FROM transaction " "WHERE pointer = ANY($1);", pointers) d = dict() r = dict() for row in rows: d[row["pointer"]] = row["tx_id"] for pointer in pointers: try: key= "%s:%s" % (pointer >> 39, (pointer >> 20) & 1048575) r[key] = rh2s(d[pointer]) except: r[key] = None return {"data": r, "time": round(time.time() - q, 4)}
async def create_db_model(app, conn): # check db isolation level level = await conn.fetchval("SHOW TRANSACTION ISOLATION LEVEL;") if level != "repeatable read": raise Exception( "Postgres repeatable read isolation level required! current isolation level is %s" % level) await conn.execute( open("./db_model/sql/schema.sql", "r", encoding='utf-8').read().replace("\n", "")) # blocks data if app.blocks_data: await conn.execute( open("./db_model/sql/block.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute( "INSERT INTO service (name,value) VALUES ('blocks_data','1') ON CONFLICT(name) DO NOTHING;" ) else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('blocks_data', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT value FROM service WHERE name ='blocks_data' LIMIT 1;") app.log.info("Option blocks_data = %s" % m) if bool(int(m)) != app.blocks_data: app.log.critical( "blocks_data option not match db structure; you should drop db and recreate it" ) raise Exception("DB structure invalid") # transaction if app.transaction: await conn.execute( open("./db_model/sql/transaction.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute( "INSERT INTO service (name,value) VALUES ('transaction','1') ON CONFLICT(name) DO NOTHING;" ) else: await conn.execute( "INSERT INTO service (name,value) VALUES ('transaction','0') ON CONFLICT(name) DO NOTHING;" ) m = await conn.fetchval( "SELECT value FROM service WHERE name ='transaction' LIMIT 1;") app.log.info("Option transaction = %s" % m) if bool(int(m)) != app.transaction: app.log.critical( "transaction option not match db structure; you should drop db and recreate it" ) raise Exception("DB structure invalid") # mempool_analytica if app.mempool_analytica: if not app.transaction: app.log.critical( "transaction mempool_analytica required transaction option enabled" ) raise Exception("configuration invalid") await conn.execute( open("./db_model/sql/mempool_analytica.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute( "INSERT INTO service (name,value) VALUES ('mempool_analytica','1') " "ON CONFLICT(name) DO NOTHING;") else: await conn.execute( "INSERT INTO service (name,value) VALUES ('mempool_analytica','0') " "ON CONFLICT(name) DO NOTHING;") m = await conn.fetchval( "SELECT value FROM service WHERE name ='mempool_analytica' LIMIT 1;") app.log.info("Option mempool_analytica = %s" % m) if bool(int(m)) != app.mempool_analytica: app.log.critical( "mempool_analytica option not match db structure; you should drop db and recreate it" ) raise Exception("DB structure invalid") # merkle proof module if app.merkle_proof: await conn.execute( open("./db_model/sql/merkle_proof.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute(""" INSERT INTO service (name, value) VALUES ('merkle_proof', '1') ON CONFLICT(name) DO NOTHING; """) else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('merkle_proof', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT service.value FROM service WHERE service.name ='merkle_proof' LIMIT 1;" ) app.log.info("Option merkle_proof = %s" % m) if int(m) == 1 and not app.merkle_proof or app.merkle_proof and int( m) == 0: app.log.critical( "merkle_proof config option not match db structure; you should drop db and recreate it." ) raise Exception("DB structure invalid") if app.block_filters: await conn.execute( open("./db_model/sql/filters.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute(""" INSERT INTO service (name, value) VALUES ('block_filters', '1') ON CONFLICT(name) DO NOTHING; """) else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('block_filters', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT service.value FROM service WHERE service.name ='block_filters' LIMIT 1;" ) app.log.info("Option block_filters = %s" % m) if int(m) == 1 and not app.block_filters or app.block_filters and int( m) == 0: app.log.critical( "block_filters config option not match db structure; you should drop db and recreate it." ) raise Exception("DB structure invalid") # transaction history if app.transaction_history: if not app.transaction: app.log.critical( "transaction_history option required transaction option enabled" ) raise Exception("configuration invalid") await conn.execute( open("./db_model/sql/transaction_history.sql", "r", encoding='utf-8').read().replace("\n", "")) t = await conn.fetchval( """SELECT EXISTS ( SELECT FROM information_schema.tables WHERE table_name = 'transaction_map_1');""") if not t: for i in range(app.transaction_map_partitions): await conn.execute(""" CREATE TABLE transaction_map_%s PARTITION OF transaction_map FOR VALUES WITH (MODULUS %s, REMAINDER %s) WITH (fillfactor=100); """ % (i + 1, app.transaction_map_partitions, i)) await conn.execute(""" INSERT INTO service (name, value) VALUES ('transaction_history', '1') ON CONFLICT(name) DO NOTHING; """) app.transaction_map_start_block = await conn.fetchval( "SELECT pointer FROM stxo " "ORDER BY pointer DESC LIMIT 1;") if app.transaction_map_start_block is None: app.transaction_map_start_block = 0 else: app.transaction_map_start_block = app.transaction_map_start_block >> 39 else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('transaction_history', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT value FROM service WHERE name ='transaction_history' LIMIT 1;") app.log.info("Option transaction_history = %s" % m) if bool(int(m)) != app.transaction_history: app.log.critical("transaction_history option not match db structure; " "you should drop db and recreate it") raise Exception("DB structure invalid") # address_state module if app.address_state: if not app.transaction_history: app.log.critical( "address_state option required transaction_history option enabled" ) raise Exception("configuration invalid") await conn.execute( open("./db_model/sql/address_state.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute(""" INSERT INTO service (name, value) VALUES ('address_state', '1') ON CONFLICT(name) DO NOTHING; """) await conn.execute(""" INSERT INTO service (name, value) VALUES ('address_state_rollback', '0') ON CONFLICT(name) DO UPDATE SET value = '0'; """) # for i in range(50): # await conn.execute(""" # CREATE TABLE IF NOT EXISTS address_%s # PARTITION OF address # FOR VALUES WITH (MODULUS %s, REMAINDER %s) # WITH (fillfactor=90); # """ % (i + 1, 50, i)) else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('address_state', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT service.value FROM service WHERE service.name ='address_state' LIMIT 1;" ) app.log.info("Option address_state = %s" % m) # address_timeline module if app.address_timeline: if not app.transaction_history: app.log.critical( "address_timeline option required transaction_history option enabled" ) raise Exception("configuration invalid") if not app.address_state: app.log.critical( "address_timeline option required address_state option enabled" ) raise Exception("configuration invalid") await conn.execute( open("./db_model/sql/address_timeline.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute(""" INSERT INTO service (name, value) VALUES ('address_timeline', '1') ON CONFLICT(name) DO NOTHING; """) else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('address_timeline', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT service.value FROM service WHERE service.name ='address_timeline' LIMIT 1;" ) app.log.info("Option address_timeline = %s" % m) if int(m ) == 1 and not app.address_timeline or app.address_timeline and int( m) == 0: app.log.critical( "address_timeline config option not match db structure; you should drop db and recreate it." ) raise Exception("DB structure invalid") # blockchain_analytica module if app.blockchain_analytica: await conn.execute( open("./db_model/sql/blockchain_analytica.sql", "r", encoding='utf-8').read().replace("\n", "")) await conn.execute(""" INSERT INTO service (name, value) VALUES ('blockchain_analytica', '1') ON CONFLICT(name) DO NOTHING; """) if not app.mempool_analytica: app.log.critical( "mempool_analytica option required for blockchain_analytica option enabled" ) raise Exception("configuration invalid") # b = await conn.fetchval("SELECT blockchain FROM blocks_stat ORDER BY height DESC LIMIT 1;") # if b is not None: # app.blockchain_stat = json.loads(b) # else: # app.blockchain_stat = { # "outputs": {"count": {"total": 0}, # total outputs count # # What is the total quantity of # # coins in bitcoin blockchain? # # "amount": {"min": {"pointer": 0, # output with minimal amount # "value": 0}, # What is the minimum amount of a coins? # # "max": {"pointer": 0, # output with maximal amount # "value": 0}, # What is the maximal amount of a coins? # # "total": 0, # total amount of all outputs # # "map": {"count": dict(), # quantity distribution by amount # # How many coins exceed 1 BTC? # # "amount": dict()} # amounts distribution by amount # # What is the total amount of all coins # # exceeding 10 BTC? # }, # # "type": {"map": {"count": dict(), # quantity distribution by type # # How many P2SH coins? # # "amount": dict(), # amounts distribution by type # # What is the total amount of # # all P2PKH coins? # # "size": dict()}}, # sizes distribution by type # # What is the total size # # of all P2PKH coins? # # "age": {"map": {"count": dict(), # distribution of counts by age # # How many coins older then 1 year? # # "amount": dict(), # distribution of amount by age # # What amount of coins older then 1 month? # # "type": dict() # distribution of counts by type # # How many P2SH coins older then 1 year? # }} # # }, # # "inputs": {"count": {"total": 0}, # total inputs count # # What is the total quantity of # # spent coins in bitcoin blockchain? # # "amount": {"min": {"pointer": 0, # input with minimal amount # "value": 0}, # What is the smallest coin spent? # # "max": {"pointer": 0, # input with maximal amount # "value": 0}, # what is the greatest coin spent? # # "total": 0, # total amount of all inputs # # What is the total amount of # # all spent coins? # # "map": {"count": dict(), # quantity distribution by amount # # How many spent coins exceed 1 BTC? # # "amount": dict()} # amounts distribution by amount # # What is the total amount of # # all spent coins exceeding 10 BTC? # }, # "type": { # "map": {"count": dict(), # quantity distribution by type # # How many P2SH spent coins? # # "amount": dict(), # amounts distribution by type # # What is the total amount # # of all P2PKH spent? # # "size": dict() # sizes distribution by type # # What is the total # # size of all P2PKH spent? # # }}, # # # # P2SH redeem script statistics # "P2SH": { # "type": {"map": {"count": dict(), # "amount": dict(), # "size": dict()} # } # }, # # # P2WSH redeem script statistics # "P2WSH": { # "type": {"map": {"count": dict(), # "amount": dict(), # "size": dict()} # } # } # }, # # "transactions": {"count": {"total": 0}, # # "amount": {"min": {"pointer": 0, # "value": 0}, # # "max": {"pointer": 0, # "value": 0}, # # "map": {"count": dict(), # "amount": dict(), # "size": dict()}, # "total": 0}, # "size": {"min": {"pointer": 0, "value": 0}, # "max": {"pointer": 0, "value": 0}, # "total": {"size": 0, "bSize": 0, "vSize": 0}, # "map": {"count": dict(), "amount": dict()}}, # # "type": {"map": {"count": dict(), "size": dict(), # "amount": dict()}}, # # "fee": {"min": {"pointer": 0, "value": 0}, # "max": {"pointer": 0, "value": 0}, # "total": 0} # } # } else: await conn.execute(""" INSERT INTO service (name, value) VALUES ('blockchain_analytica', '0') ON CONFLICT(name) DO NOTHING; """) m = await conn.fetchval( "SELECT service.value FROM service WHERE service.name ='blockchain_analytica' LIMIT 1;" ) app.log.info("Option blockchain_analytica = %s" % m) if int( m ) == 1 and not app.blockchain_analytica or app.blockchain_analytica and int( m) == 0: app.log.critical( "blockchain_analytica config option not match db structure; you should drop db and recreate it." ) raise Exception("DB structure invalid") # transaction_fee_analytica app.log.info("Option transaction_fee_analytica = 0") # nodes_discovery app.log.info("Option nodes_discovery = 0") # market_data app.log.info("Option market_data = 0") # deterministic_wallet app.log.info("Option deterministic_wallet = 0") # payment_forwarding app.log.info("Option payment_forwarding = 0") # hot_wallet app.log.info("Option hot_wallet = 0") start_block = await conn.fetchval( "SELECT height FROM blocks ORDER BY height DESC LIMIT 1;") app.start_checkpoint = -1 if start_block is None else start_block if "flush_mempool_on_start" in app.config["OPTIONS"]: if app.config["OPTIONS"]["flush_mempool_on_start"] == "on": app.log.info("Option flush_mempool_on_start = 1") app.log.info("Flush mempool ...") if app.transaction: await conn.execute("TRUNCATE TABLE unconfirmed_transaction;") await conn.execute("TRUNCATE TABLE invalid_transaction;") if app.transaction_history: await conn.execute( "TRUNCATE TABLE unconfirmed_transaction_map;") await conn.execute("TRUNCATE TABLE invalid_transaction_map;") await conn.execute("TRUNCATE TABLE connector_unconfirmed_utxo;") await conn.execute("TRUNCATE TABLE connector_unconfirmed_stxo;") rows = await conn.fetch( "SELECT hash from blocks order by height desc limit 100;") for row in rows: app.chain_tail.append(rh2s(row["hash"])) await conn.execute( """ INSERT INTO service (name, value) VALUES ('block_filters_bootstrap', '0') ON CONFLICT(name) DO UPDATE SET value = '0'; """)
async def outpoints_info(outpoints, app): q = time.time() o = [i[2] for i in outpoints] t = [i[0] for i in outpoints] out_map_pointer = dict() async with app["db_pool"].acquire() as conn: utxo = await conn.fetch( "SELECT outpoint, pointer, address, amount " "FROM connector_utxo " "WHERE outpoint = ANY($1) LIMIT $2;", o, len(o)) uutxo = await conn.fetch( "SELECT outpoint, address, amount " "FROM connector_unconfirmed_utxo " "WHERE outpoint = ANY($1) LIMIT $2;", o, len(o)) ustxo = await conn.fetch( "SELECT outpoint, tx_id " "FROM connector_unconfirmed_stxo " "WHERE outpoint = ANY($1);", o) stxo = [] if len(utxo) + len(uutxo) < len(o): tx_pointers = await conn.fetch( "SELECT pointer, tx_id FROM transaction " "WHERE tx_id = ANY($1) LIMIT $2;", t, len(t)) print(tx_pointers) tx_map_pointer = dict() for row in tx_pointers: tx_map_pointer[row["tx_id"]] = row["pointer"] s = set() for i in outpoints: try: s.add(tx_map_pointer[i[0]] + (1 << 19) + i[1]) out_map_pointer[tx_map_pointer[i[0]] + (1 << 19) + i[1]] = i[2] except: pass stxo = await conn.fetch( "SELECT stxo.pointer, stxo.address, stxo.amount, transaction.tx_id " "FROM stxo " "JOIN transaction " "ON transaction.pointer = (stxo.s_pointer >> 18)<<18" "WHERE stxo.pointer = ANY($1) LIMIT $2", s, len(s)) print(stxo) outpoints_map = dict() for row in utxo: outpoints_map[row["outpoint"]] = [ row["pointer"] >> 39, row["address"], row["amount"], [] ] for row in uutxo: outpoints_map[row["outpoint"]] = [ None, row["address"], row["amount"], [] ] for row in ustxo: try: outpoints_map[row["outpoint"]][3].append(rh2s(row["tx_id"])) except: pass for row in stxo: outpoint = out_map_pointer[row["pointer"]] outpoints_map[outpoint] = [ row["pointer"] >> 39, row["address"], row["amount"], [rh2s(row["tx_id"])] ] result = [] for i in outpoints: outpoint = "%s:%s" % (rh2s(i[0]), i[1]) try: h, address, amount, s = outpoints_map[i[2]] r = dict() if address[0] in (0, 1, 5, 6): script_hash = True if address[0] in (1, 6) else False witness_version = None if address[0] < 5 else 0 try: r["address"] = hash_to_address( address[1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) r["script"] = address_to_script(r["address"], hex=1) except: raise elif address[0] == 2: r["address"] = script_to_address(address[1:], testnet=app["testnet"]) r["script"] = r["address"][1:].hex() else: r["script"] = r["address"][1:].hex() r["scriptOpcodes"] = decode_script(r["script"]) r["scriptAsm"] = decode_script(r["script"], 1) r["height"] = h r["spent"] = s r["type"] = SCRIPT_N_TYPES[address[0]] r["amount"] = amount result.append({outpoint: r}) except: result.append({outpoint: None}) return {"data": result, "time": round(time.time() - q, 4)}
async def processor(self): utxo_sequence = 0 stxo_sequence = 0 tx_sequence = 0 best_fee = 1 dbs = set() dbs_childs = set() outputs, inputs, transactions = self.refresh_stat() truncate_dbs_table = True best_fee_hourly = ListCache(60 * 60) best_fee_4h = ListCache(60 * 60 * 4) async with self.db_pool.acquire() as conn: rows = await conn.fetch( "SELECT minute, transactions->'feeRate'->'best' as best FROM mempool_analytica " "order by minute desc LIMIT 240;") c = 0 for row in rows: if row["best"] is not None: if c < 60: best_fee_hourly.set(float(row["best"])) best_fee_4h.set(float(row["best"])) c += 1 while True: try: if not self.bootstrap_completed: async with self.db_pool.acquire() as conn: v = await conn.fetchval( "SELECT value FROM service " "WHERE name = 'bootstrap_completed' LIMIT 1;") if v == '1': self.bootstrap_completed = True self.log.info("Mempool analytica task started") async with self.db_pool.acquire() as conn: self.last_day = await conn.fetchval( "SELECT max(day) FROM mempool_analytica WHERE day IS NOT NULL " ) self.last_hour = await conn.fetchval( "SELECT max(hour) FROM mempool_analytica WHERE day IS NOT NULL " ) if self.last_day is None: self.last_day = 0 if self.last_hour is None: self.last_hour = 0 else: await asyncio.sleep(60) q = time.time() await self.load_block_map() async with self.db_pool.acquire() as conn: async with conn.transaction(): last_hash = await conn.fetchval( "SELECT height FROM blocks order by height desc LIMIT 1;" ) stxo = await conn.fetch( "SELECT tx_id, out_tx_id, address, amount, pointer, sequence, outpoint, id " "FROM connector_unconfirmed_stxo " "WHERE id > $1;", stxo_sequence) utxo = await conn.fetch( "SELECT out_tx_id as tx_id, address, amount, id " "FROM connector_unconfirmed_utxo " "WHERE id > $1;", utxo_sequence) tx = await conn.fetch( "SELECT tx_id, size, b_size, rbf, fee, " "amount, segwit, timestamp, id FROM unconfirmed_transaction " "WHERE id > $1;", tx_sequence) row = await conn.fetchval( "select min(feerate) " "from (select feerate, sum((size + b_size * 4)/4) " "over (order by feerate desc) as block " "from unconfirmed_transaction) t where block <= 920000;" ) if row is not None: best_fee = row if last_hash != self.last_hash: self.last_hash = last_hash outputs, inputs, transactions = self.refresh_stat() utxo_sequence = 0 stxo_sequence = 0 tx_sequence = 0 dbs = set() dbs_childs = set() truncate_dbs_table = True if not tx: s_minute = int(time.time()) // 60 if s_minute % 60 == 0 and self.last_hour < s_minute // 60: s_hour = s_minute // 60 self.last_hour = s_hour if s_hour % 24 == 0 and self.last_day < s_hour // 24: s_day = s_hour // 24 self.last_day = s_day else: s_day = None else: s_hour = None s_day = None async with self.db_pool.acquire() as conn: async with conn.transaction(): await conn.execute( "INSERT INTO mempool_analytica " "(minute, hour, day, inputs, outputs, transactions)" " VALUES " "($1, $2, $3, $4, $5, $6) " "ON CONFLICT (minute) " "DO UPDATE SET " " inputs = $4," " outputs = $5, " " transactions = $6", s_minute, s_hour, s_day, json.dumps(inputs), json.dumps(outputs), json.dumps(transactions)) txsi = set() txso = set() dbs_outs = set() dbs_set = set() if tx: inputs["count"] += len(stxo) for row in stxo: if stxo_sequence < row["id"]: stxo_sequence = row["id"] if row["sequence"] > 0: dbs_outs.add(row["outpoint"]) dbs_set.add(row["tx_id"]) txsi.add(row["tx_id"]) inputs["amount"]["total"] += row["amount"] if inputs["amount"]["max"]["value"] is None or \ inputs["amount"]["max"]["value"] < row["amount"]: inputs["amount"]["max"]["value"] = row["amount"] inputs["amount"]["max"]["txId"] = rh2s( row["tx_id"]) if inputs["amount"]["min"]["value"] is None or \ inputs["amount"]["min"]["value"] > row["amount"]: inputs["amount"]["min"]["value"] = row["amount"] inputs["amount"]["min"]["txId"] = rh2s( row["tx_id"]) try: inputs["typeMap"][row["address"][0]]["count"] += 1 inputs["typeMap"][row["address"] [0]]["amount"] += row["amount"] except: inputs["typeMap"][row["address"][0]] = { "count": 1, "amount": row["amount"] } amount = row["amount"] key = None if amount == 0 else str( math.floor(math.log10(amount))) try: inputs["amountMap"][key]["count"] += 1 inputs["amountMap"][key]["amount"] += row["amount"] except: inputs["amountMap"][key] = { "count": 1, "amount": row["amount"] } try: key = time.time() - self.block_map_timestamp[ row["pointer"] >> 39] if key < 3600: key = "1h" elif key < 43200: key = "12h" elif key < 86400: key = "1d" elif key < 259200: key = "3d" elif key < 604800: key = "1w" elif key < 2592000: key = "1m" else: key = "%sy" % (int(key // 31536000) + 1) except: key = None try: inputs["ageMap"][key]["count"] += 1 inputs["ageMap"][key]["amount"] += row["amount"] except: inputs["ageMap"][key] = { "count": 1, "amount": row["amount"] } async with self.db_pool.acquire() as conn: dbs_rows = await conn.fetch( "SELECT tx_id, outpoint " "FROM connector_unconfirmed_stxo " "WHERE outpoint = ANY($1);", dbs_outs) out_map = set() for row in dbs_rows: if row["outpoint"] in out_map: if row["tx_id"] in dbs_set: dbs.add(row["tx_id"]) else: out_map.add(row["outpoint"]) l_dbs_size = 0 while True: for row in stxo: if row["out_tx_id"] in dbs or row[ "out_tx_id"] in dbs_childs: if row["tx_id"] not in dbs: dbs_childs.add(row["tx_id"]) if l_dbs_size != len(dbs_childs): l_dbs_size = len(dbs_childs) else: break outputs["count"] += len(utxo) for row in utxo: if utxo_sequence < row["id"]: utxo_sequence = row["id"] txso.add(row["tx_id"]) outputs["amount"]["total"] += row["amount"] if outputs["amount"]["max"]["value"] is None or \ outputs["amount"]["max"]["value"] < row["amount"]: outputs["amount"]["max"]["value"] = row["amount"] outputs["amount"]["max"]["txId"] = rh2s( row["tx_id"]) if outputs["amount"]["min"]["value"] is None or \ outputs["amount"]["min"]["value"] > row["amount"]: if row["amount"] > 0: outputs["amount"]["min"]["value"] = row[ "amount"] outputs["amount"]["min"]["txId"] = rh2s( row["tx_id"]) try: outputs["typeMap"][row["address"][0]]["count"] += 1 outputs["typeMap"][row["address"] [0]]["amount"] += row["amount"] except: outputs["typeMap"][row["address"][0]] = { "count": 1, "amount": row["amount"] } amount = row["amount"] key = None if amount == 0 else str( math.floor(math.log10(amount))) try: outputs["amountMap"][key]["count"] += 1 outputs["amountMap"][key]["amount"] += row[ "amount"] except: outputs["amountMap"][key] = { "count": 1, "amount": row["amount"] } transactions["doublespend"]["count"] = len(dbs) transactions["doublespendChilds"]["count"] = len( dbs_childs) transactions["count"] += len(tx) dbs_records = deque() for row in tx: v_size = math.ceil( (row["b_size"] * 3 + row["size"]) / 4) if tx_sequence < row["id"]: tx_sequence = row["id"] if row["tx_id"] in dbs: transactions["doublespend"]["amount"] += row[ "amount"] transactions["doublespend"]["size"] += row["size"] transactions["doublespend"]["vSize"] += v_size dbs_records.append( (row["tx_id"], row["timestamp"], 0)) if row["tx_id"] in dbs_childs: transactions["doublespendChilds"]["amount"] += row[ "amount"] transactions["doublespendChilds"]["size"] += row[ "size"] transactions["doublespendChilds"][ "vSize"] += v_size dbs_records.append( (row["tx_id"], row["timestamp"], 1)) if row["amount"] > 0: transactions["amount"]["total"] += row["amount"] if transactions["amount"]["max"]["value"] is None or \ transactions["amount"]["max"]["value"] < row["amount"]: transactions["amount"]["max"]["value"] = row[ "amount"] transactions["amount"]["max"]["txId"] = rh2s( row["tx_id"]) if transactions["amount"]["min"]["value"] is None or \ transactions["amount"]["min"]["value"] > row["amount"]: transactions["amount"]["min"]["value"] = row[ "amount"] transactions["amount"]["min"]["txId"] = rh2s( row["tx_id"]) if row["fee"] is not None: transactions["fee"]["total"] += row["fee"] if transactions["fee"]["max"]["value"] is None or \ transactions["fee"]["max"]["value"] < row["fee"]: transactions["fee"]["max"]["value"] = row[ "fee"] transactions["fee"]["max"]["txId"] = rh2s( row["tx_id"]) if transactions["fee"]["min"]["value"] is None or \ transactions["fee"]["min"]["value"] > row["fee"]: transactions["fee"]["min"]["value"] = row[ "fee"] transactions["fee"]["min"]["txId"] = rh2s( row["tx_id"]) fee_rate = math.ceil(row["fee"] / v_size) if transactions["feeRate"]["max"]["value"] is None or \ transactions["feeRate"]["max"]["value"] < fee_rate: transactions["feeRate"]["max"][ "value"] = fee_rate transactions["feeRate"]["max"]["txId"] = rh2s( row["tx_id"]) if transactions["feeRate"]["min"]["value"] is None or \ transactions["feeRate"]["min"]["value"] > fee_rate: transactions["feeRate"]["min"][ "value"] = fee_rate transactions["feeRate"]["min"]["txId"] = rh2s( row["tx_id"]) key = fee_rate if key > 10 and key < 20: key = math.floor(key / 2) * 2 elif key > 20 and key < 200: key = math.floor(key / 10) * 10 elif key > 200: key = math.floor(key / 25) * 25 try: transactions["feeRateMap"][key]["count"] += 1 transactions["feeRateMap"][key]["size"] += row[ "size"] transactions["feeRateMap"][key][ "vSize"] += v_size except: transactions["feeRateMap"][key] = { "count": 1, "size": row["size"], "vSize": v_size } if row["rbf"]: transactions["rbfCount"] += 1 if row["segwit"]: transactions["segwitCount"] += 1 if row["size"]: transactions["size"]["total"] += row["size"] transactions["vSize"]["total"] += v_size if transactions["size"]["max"]["value"] is None or \ transactions["size"]["max"]["value"] < row["size"]: transactions["size"]["max"]["value"] = row[ "size"] transactions["size"]["max"]["txId"] = rh2s( row["tx_id"]) if transactions["vSize"]["max"]["value"] is None or \ transactions["vSize"]["max"]["value"] < v_size: transactions["vSize"]["max"]["value"] = v_size transactions["vSize"]["max"]["txId"] = rh2s( row["tx_id"]) if transactions["size"]["min"]["value"] is None or \ transactions["size"]["min"]["value"] > row["size"]: transactions["size"]["min"]["value"] = row[ "size"] transactions["size"]["min"]["txId"] = rh2s( row["tx_id"]) if transactions["vSize"]["min"]["value"] is None or \ transactions["vSize"]["min"]["value"] > v_size: transactions["vSize"]["min"]["value"] = v_size transactions["vSize"]["min"]["txId"] = rh2s( row["tx_id"]) if transactions["vSize"]["total"] > 1000000 / 10: transactions["feeRate"]["best"] = round(best_fee, 2) else: transactions["feeRate"]["best"] = 1 async with self.db_pool.acquire() as conn: async with conn.transaction(): if truncate_dbs_table: await conn.execute( "truncate table mempool_dbs;") truncate_dbs_table = False await conn.copy_records_to_table( 'mempool_dbs', columns=["tx_id", "timestamp", "child"], records=dbs_records) s_minute = int(time.time()) // 60 if s_minute % 60 == 0 and self.last_hour < s_minute // 60: s_hour = s_minute // 60 self.last_hour = s_hour if s_hour % 24 == 0 and self.last_day < s_hour // 24: s_day = s_hour // 24 self.last_day = s_day else: s_day = None else: s_hour = None s_day = None if self.last_minute != s_minute or transactions[ "feeRate"]["bestHourly"] == 1: best_fee_hourly.set( transactions["feeRate"]["best"]) f = 0 for i in best_fee_hourly.items: f += i f4 = 0 for i in best_fee_4h.items: f4 += i if len(best_fee_hourly.items): transactions["feeRate"][ "bestHourly"] = round( f / len(best_fee_hourly.items), 2) else: transactions["feeRate"][ "bestHourly"] = transactions[ "feeRate"]["best"] if len(best_fee_4h.items): transactions["feeRate"]["best4h"] = round( f4 / len(best_fee_4h.items), 2) else: transactions["feeRate"][ "best4h"] = transactions["feeRate"][ "best"] await conn.execute( "INSERT INTO mempool_analytica " "(minute, hour, day, inputs, outputs, transactions)" " VALUES " "($1, $2, $3, $4, $5, $6) " "ON CONFLICT (minute) " "DO UPDATE SET " " inputs = $4," " outputs = $5, " " transactions = $6", s_minute, s_hour, s_day, json.dumps(inputs), json.dumps(outputs), json.dumps(transactions)) if s_hour is not None: self.log.warning( "Mempool analytica hourly point saved %s" % s_hour) self.log.info( "Mempool transactions %s; STXO : %s; UTXO %s; DBS %s; round time %s;" % (transactions["count"], inputs["count"], outputs["count"], transactions["doublespend"]["count"] + transactions["doublespendChilds"]["count"], q)) q = time.time() - q if q < 1: await asyncio.sleep(1 - q) if q > 10: self.log.warning("Mempool analytica is to slow %s" % q) if self.last_minute != s_minute or transactions["feeRate"][ "best4h"] == 1: self.last_minute = s_minute self.log.debug( "Mempool TX %s; STXO %s; UTXO %s; DBS %s; %s; %s; Best fee %s/%s/%s; Round time %s;" % (transactions["count"], inputs["count"], outputs["count"], transactions["doublespend"]["count"] + transactions["doublespendChilds"]["count"], format_bytes(transactions["size"]["total"]), format_vbytes(transactions["vSize"]["total"]), transactions["feeRate"]["best"], transactions["feeRate"]["bestHourly"], transactions["feeRate"]["best4h"], round(q, 4))) else: await asyncio.sleep(2) # assert len(tx) == len(txsi) # assert len(tx) == len(txso) # # async with self.db_pool.acquire() as conn: # v = await conn.fetch("SELECT invalid_transaction.tx_id FROM invalid_transaction " # " JOIN connector_unconfirmed_stxo ON connector_unconfirmed_stxo.tx_id = invalid_transaction.tx_id " # " ;") # k = [t["tx_id"] for t in v] # for t in v: # print(rh2s(t["tx_id"])) # v = await conn.fetch("SELECT outpoint, sequence FROM connector_unconfirmed_stxo WHERE tx_id = ANY($1);", k) # print("u", len(v)) # uu = set() # pp = set() # for r in v: # uu.add(r["outpoint"]) # pp.add((r["outpoint"], r["sequence"])) # v = await conn.fetch("SELECT outpoint, sequence FROM invalid_stxo WHERE tx_id = ANY($1);", k) # print("i", len(v)) # ii = set() # for r in v: # ii.add((r["outpoint"], r["sequence"])) # e = 0 # for i in ii: # if i[0] not in uu: # print("none", i[1]) # else: # e += 1 # print(">>", e) # # v = await conn.fetch("SELECT count(*) from connector_unconfirmed_utxo WHERE out_tx_id = ANY($1);", k) # print("connector_unconfirmed_utxo", v) # v = await conn.fetch("SELECT count(*) from unconfirmed_transaction WHERE tx_id = ANY($1);", k) # print("unconfirmed_transaction", v) # v = await conn.fetch("SELECT count(*) from unconfirmed_transaction_map WHERE tx_id = ANY($1);", k) # print("unconfirmed_transaction_map", v) # ff = 0 # for i in pp: # v = await conn.fetchval("SELECT count(*) from invalid_stxo WHERE outpoint = $1 and sequence = $2;", i[0], i[1]) # ff += v # print("ff", ff) # ll = list() # v = await conn.fetch("SELECT outpoint, sequence, out_tx_id, tx_id, input_index, address, amount, pointer from connector_unconfirmed_stxo WHERE tx_id = ANY($1);", k) # for i in v: # ll.append((i["outpoint"], # i["sequence"], # i["out_tx_id"], # i["tx_id"], # i["input_index"], # i["address"], # i["amount"], # i["pointer"], # )) # print("ll", len(ll)) # try: # # await conn.copy_records_to_table('invalid_stxo', # # columns=["outpoint", # # "sequence", # # "out_tx_id", # # "tx_id", # # "input_index", # # "address", # # "amount", # # "pointer",], # # records=ll) # # print("iok") # ###v = await conn.fetch("DELETE FROM connector_unconfirmed_stxo WHERE tx_id = ANY($1);", k) # except Exception as err: # print(err) # await asyncio.sleep(50000) # v = await conn.fetch("DELETE FROM unconfirmed_transaction_map WHERE tx_id = ANY($1);", k) # print(v) # # v = await conn.fetch("DELETE FROM connector_unconfirmed_stxo WHERE tx_id = ANY($1);", k) # # print(v) # v = await conn.fetch("SELECT tx_id FROM connector_unconfirmed_stxo WHERE tx_id = ANY($1);", k) # print(v) # v = await conn.fetch("SELECT out_tx_id FROM connector_unconfirmed_utxo WHERE out_tx_id = ANY($1);", k) # print(v) # v = await conn.fetch("DELETE FROM connector_unconfirmed_utxo WHERE out_tx_id = ANY($1);", k) # print(v) # v = await conn.fetch("SELECT out_tx_id FROM connector_unconfirmed_utxo WHERE out_tx_id = ANY($1);", k) # print(v) # if v == []: # await conn.fetch("DELETE FROM unconfirmed_transaction WHERE tx_id = ANY($1);", k) except asyncio.CancelledError: self.log.warning("Mempool analytica task canceled") break except Exception as err: self.log.error("Mempool analytica task error: %s" % err) print(traceback.format_exc()) await asyncio.sleep(10)
async def block_data_by_pointer(pointer, stat, app): pool = app["db_pool"] qt = time.time() async with pool.acquire() as conn: if pointer == 'last': stmt = await conn.prepare( "SELECT height," " hash," " miner," " timestamp_received," " data," " header," " adjusted_timestamp " "FROM blocks ORDER BY height desc LIMIT 1;") row = await stmt.fetchrow() else: if type(pointer) == bytes: stmt = await conn.prepare( "SELECT height," " hash," " miner," " timestamp_received," " data," " header," " adjusted_timestamp " "FROM blocks WHERE hash = $1 LIMIT 1;") row = await stmt.fetchrow(pointer) elif type(pointer) == int: stmt = await conn.prepare( "SELECT height," " hash," " miner," " timestamp_received," " data," " header," " adjusted_timestamp " "FROM blocks WHERE height = $1 LIMIT 1;") row = await stmt.fetchrow(pointer) if row is None: raise APIException(NOT_FOUND, "block not found", status=404) block = dict() block["height"] = row["height"] if block["height"] not in app["block_map_time"]: await block_map_update(app) if block["height"] not in app["block_map_time"]: raise Exception("internal error") block["hash"] = rh2s(row["hash"]) block["header"] = base64.b64encode(row["header"]).decode() d = json.loads(row["data"]) for k in d: block[k] = d[k] try: block["miner"] = json.loads(row["miner"]) except: block["miner"] = None print(block["height"] not in app["block_map_time"]) block["medianBlockTime"] = app["block_map_time"][block["height"]][2] block["blockTime"] = app["block_map_time"][block["height"]][1] block["receivedTimestamp"] = row["timestamp_received"] block["adjustedTimestamp"] = row["adjusted_timestamp"] block["bitsHex"] = block["bits"] block["bits"] = bytes_to_int(bytes_from_hex(block["bits"])) block["nonceHex"] = block["nonce"].to_bytes(4, byteorder="big").hex() block["versionHex"] = int_to_bytes(block["version"]).hex() block["difficulty"] = block["targetDifficulty"] q = int.from_bytes(s2rh(block["hash"]), byteorder="little") block["blockDifficulty"] = target_to_difficulty(q) del block["targetDifficulty"] next = await conn.fetchval( "SELECT " " hash " "FROM blocks WHERE height = $1;", block["height"] + 1) if next is not None: block["nextBlockHash"] = rh2s(next) else: block["nextBlockHash"] = None # get coinbase transaction cb = await conn.fetchval( "SELECT raw_transaction " "FROM transaction WHERE pointer = $1 LIMIT 1;", block["height"] << 39) tx = Transaction(cb, format="raw") block["estimatedBlockReward"] = 200 * 100000000 >> block[ "height"] // 52500 block["blockReward"] = tx["amount"] if tx["amount"] > block["estimatedBlockReward"]: block["blockReward"] = block["estimatedBlockReward"] block["blockFeeReward"] = tx["amount"] - block[ "estimatedBlockReward"] else: block["blockReward"] = tx["amount"] block["blockFeeReward"] = 0 block["confirmations"] = app["last_block"] - block["height"] + 1 block["transactionsCount"] = var_int_to_int(row["header"][80:]) block["coinbase"] = tx["vIn"][0]["scriptSig"].hex() if stat and app["blockchain_analytica"]: async with pool.acquire() as conn: stat = await conn.fetchval( "SELECT block FROM block_stat WHERE height = $1 LIMIT 1;", row["height"]) if stat is not None: block["statistics"] = json.loads(stat) else: block["statistics"] = None resp = {"data": block, "time": round(time.time() - qt, 4)} return resp
async def load_blocks(self, height, limit): start_height = height start_limit = limit self.destroyed_coins = MRU() self.coins = MRU() try: self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout) blocks, missed = dict(), deque() v, t, limit = height + limit, 0, 30 while height < v and height <= self.target_height: batch, h_list = list(), list() while len(batch) < limit and height < v and height <= self.target_height: batch.append(["getblockhash", height]) h_list.append(height) height += 1 result = await self.rpc.batch(batch) h, batch = list(), list() for lh, r in zip(h_list, result): if r["result"] is not None: batch.append(["getblock", r["result"], 0]) h.append(lh) result = await self.rpc.batch(batch) for x, y in zip(h, result): if y["result"] is not None: block = decode_block_tx(y["result"]) block["p2pkMapHash"] = [] if self.option_tx_map: block["txMap"], block["stxo"] = set(), deque() if self.option_block_filters: block["filter"] = set() if self.option_analytica: block["stat"] = {"inputs": {"count": 0, "amount": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "typeMap": {}}, "outputs": {"count": 0, "amount": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "typeMap": {}}, "transactions": {"count": 0, "amount": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "size": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "vSize": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "fee": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}, "total": 0}, "feeRate": {"max": {"value": None, "txId": None}, "min": {"value": None, "txId": None}}, "amountMap": {}, "feeRateMap": {}, "typeMap": {"segwit": {"count": 0, "amount": 0, "size": 0}, "rbf": {"count": 0, "amount": 0, "size": 0}}} } if self.option_merkle_proof: mt = merkle_tree(block["rawTx"][i]["txId"] for i in block["rawTx"]) coinbase = block["rawTx"][0]["vIn"][0]["scriptSig"] block["miner"] = None for tag in MINER_COINBASE_TAG: if coinbase.find(tag) != -1: block["miner"] = json.dumps(MINER_COINBASE_TAG[tag]) break else: try: address_hash = block["rawTx"][0]["vOut"][0]["addressHash"] script_hash = False if block["rawTx"][0]["vOut"][0]["nType"] == 1 else True a = hash_to_address(address_hash, script_hash=script_hash) if a in MINER_PAYOUT_TAG: block["miner"] = json.dumps(MINER_PAYOUT_TAG[a]) except: pass if self.utxo_data: # handle outputs for z in block["rawTx"]: if self.option_merkle_proof: block["rawTx"][z]["merkleProof"] = b''.join(merkle_proof(mt, z, return_hex=False)) tx_pointer = (x << 39)+(z << 20) for i in block["rawTx"][z]["vOut"]: out= block["rawTx"][z]["vOut"][i] o = b"".join((block["rawTx"][z]["txId"], int_to_bytes(i))) pointer = (x << 39)+(z << 20)+(1 << 19) + i out_type = out["nType"] try: if out_type == 2: block["p2pkMapHash"].append((out["addressHash"], out["scriptPubKey"])) raise Exception("P2PK") address = b"".join((bytes([out_type]), out["addressHash"])) except: address = b"".join((bytes([out_type]), out["scriptPubKey"])) if out_type in (0, 1, 2, 5, 6): if self.option_block_filters: e = b"".join((bytes([out_type]), z.to_bytes(4, byteorder="little"), out["addressHash"])) block["filter"].add(e) if self.option_tx_map: block["txMap"].add((address, tx_pointer)) out["_address"] = address self.coins[o] = (pointer, out["value"], address) if self.option_analytica: tx = block["rawTx"][z] out_stat = block["stat"]["outputs"] out_stat["count"] += 1 out_stat["amount"]["total"] += out["value"] if out_stat["amount"]["min"]["value"] is None or \ out_stat["amount"]["min"]["value"] > out["value"]: if out["value"] > 0: out_stat["amount"]["min"]["value"] = out["value"] out_stat["amount"]["min"]["txId"] = rh2s(tx["txId"]) out_stat["amount"]["max"]["vOut"] = i if out_stat["amount"]["max"]["value"] is None or \ out_stat["amount"]["max"]["value"] < out["value"]: out_stat["amount"]["max"]["value"] = out["value"] out_stat["amount"]["max"]["txId"] = rh2s(tx["txId"]) out_stat["amount"]["max"]["vOut"] = i key = None if out["value"] == 0 else str(math.floor(math.log10(out["value"]))) out_type = SCRIPT_N_TYPES[out_type] a = out["value"] try: out_stat["typeMap"][out_type]["count"] += 1 out_stat["typeMap"][out_type]["amount"] += a except: out_stat["typeMap"][out_type] = {"count": 1, "amount": a, "amountMap": {}} try: out_stat["typeMap"][out_type]["amountMap"][key]["count"] += 1 out_stat["typeMap"][out_type]["amountMap"][key]["amount"] += a except: out_stat["typeMap"][out_type]["amountMap"][key] = {"count": 1, "amount": a} if self.option_analytica: tx = block["rawTx"][z] tx["inputsAmount"] = 0 tx_stat = block["stat"]["transactions"] tx_stat["count"] += 1 for k in ("amount", "size", "vSize"): tx_stat[k]["total"] += tx[k] if tx_stat[k]["min"]["value"] is None or tx_stat[k]["min"]["value"] > tx[k]: tx_stat[k]["min"]["value"] = tx[k] tx_stat[k]["min"]["txId"] = rh2s(tx["txId"]) if tx_stat[k]["max"]["value"] is None or tx_stat[k]["max"]["value"] < tx[k]: tx_stat[k]["max"]["value"] = tx[k] tx_stat[k]["max"]["txId"] = rh2s(tx["txId"]) key = None if tx["amount"] == 0 else str(math.floor(math.log10(tx["amount"]))) try: tx_stat["amountMap"][key]["count"] += 1 tx_stat["amountMap"][key]["amount"] += tx["amount"] tx_stat["amountMap"][key]["size"] += tx["amount"] except: tx_stat["amountMap"][key] = {"count": 1, "amount": tx["amount"], "size": tx["amount"]} if tx["segwit"]: tx_stat["typeMap"]["segwit"]["count"] += 1 tx_stat["typeMap"]["segwit"]["amount"] += tx["amount"] tx_stat["typeMap"]["segwit"]["size"] += tx["size"] if tx["rbf"]: tx_stat["typeMap"]["rbf"]["count"] += 1 tx_stat["typeMap"]["rbf"]["amount"] += tx["amount"] tx_stat["typeMap"]["rbf"]["size"] += tx["size"] # handle inputs for z in block["rawTx"]: if not block["rawTx"][z]["coinbase"]: for i in block["rawTx"][z]["vIn"]: inp = block["rawTx"][z]["vIn"][i] outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) block["rawTx"][z]["vIn"][i]["_outpoint"] = outpoint tx_pointer = (x<<39)+(z<<20) try: r = self.coins.delete(outpoint) block["rawTx"][z]["vIn"][i]["_a_"] = r self.destroyed_coins[r[0]] = True out_type = r[2][0] if self.option_block_filters: if out_type in (0, 1, 5, 6): e = b"".join((bytes([out_type]), z.to_bytes(4, byteorder="little"), r[2][1:])) block["filter"].add(e) elif out_type == 2: a = parse_script(r[2][1:])["addressHash"] e = b"".join((bytes([out_type]), z.to_bytes(4, byteorder="little"), a[:20])) block["filter"].add(e) if self.option_tx_map: block["txMap"].add((r[2], tx_pointer)) block["stxo"].append((r[0], (x<<39)+(z<<20)+i, r[2], r[1])) t += 1 if self.option_analytica: a = r[1] in_type = SCRIPT_N_TYPES[r[2][0]] try: tx = block["rawTx"][z] input_stat = block["stat"]["inputs"] input_stat["count"] += 1 tx["inputsAmount"] += a input_stat["amount"]["total"] += a if input_stat["amount"]["min"]["value"] is None or \ input_stat["amount"]["min"]["value"] > a: input_stat["amount"]["min"]["value"] = a input_stat["amount"]["min"]["txId"] = rh2s(tx["txId"]) input_stat["amount"]["min"]["vIn"] = i if input_stat["amount"]["max"]["value"] is None or \ input_stat["amount"]["max"]["value"] < a: input_stat["amount"]["max"]["value"] = a input_stat["amount"]["max"]["txId"] = rh2s(tx["txId"]) input_stat["amount"]["max"]["vIn"] = i key = None if a == 0 else str(math.floor(math.log10(a))) try: input_stat["typeMap"][in_type]["count"] += 1 input_stat["typeMap"][in_type]["amount"] += a except: input_stat["typeMap"][in_type] = {"count": 1, "amount": a, "amountMap": {}} try: input_stat["typeMap"][in_type]["amountMap"][key]["count"] += 1 input_stat["typeMap"][in_type]["amountMap"][key]["amount"] += a except: input_stat["typeMap"][in_type]["amountMap"][key] = {"count": 1, "amount": a} except: print(traceback.format_exc()) except: if self.dsn: missed.append(outpoint) blocks[x] = block m, n = 0, 0 if self.utxo_data and missed and self.dsn: async with self.db.acquire() as conn: rows = await conn.fetch("SELECT outpoint, " " pointer," " address," " amount " "FROM connector_utxo " "WHERE outpoint = ANY($1);", missed) m += len(rows) p = dict() for row in rows: p[row["outpoint"]] = (row["pointer"], row["amount"], row["address"]) for h in blocks: for z in blocks[h]["rawTx"]: tx_pointer = (h<<39)+(z<<20) if not blocks[h]["rawTx"][z]["coinbase"]: for i in blocks[h]["rawTx"][z]["vIn"]: outpoint = blocks[h]["rawTx"][z]["vIn"][i]["_outpoint"] try: blocks[h]["rawTx"][z]["vIn"][i]["_l_"] = p[outpoint] try: out_type = p[outpoint][2][0] if self.option_block_filters: if out_type in (0, 1, 5, 6): e = b"".join((bytes([out_type]), z.to_bytes(4, byteorder="little"), p[outpoint][2][1:])) blocks[h]["filter"].add(e) elif out_type == 2: a = parse_script(p[outpoint][2][1:])["addressHash"] e = b"".join((bytes([out_type]), z.to_bytes(4, byteorder="little"), a[:20])) blocks[h]["filter"].add(e) if self.option_tx_map: blocks[h]["txMap"].add((p[outpoint][2], tx_pointer)) blocks[h]["stxo"].append((p[outpoint][0], (h<<39)+(z<<20)+i, p[outpoint][2], p[outpoint][1])) if self.option_analytica: a = p[outpoint][1] in_type = SCRIPT_N_TYPES[p[outpoint][2][0]] tx = blocks[h]["rawTx"][z] input_stat = blocks[h]["stat"]["inputs"] input_stat["count"] += 1 tx["inputsAmount"] += a input_stat["amount"]["total"] += a if input_stat["amount"]["min"]["value"] is None or \ input_stat["amount"]["min"]["value"] > a: input_stat["amount"]["min"]["value"] = a input_stat["amount"]["min"]["txId"] = rh2s(tx["txId"]) input_stat["amount"]["min"]["vIn"] = i if input_stat["amount"]["max"]["value"] is None or \ input_stat["amount"]["max"]["value"] < a: input_stat["amount"]["max"]["value"] = a input_stat["amount"]["max"]["txId"] = rh2s(tx["txId"]) input_stat["amount"]["max"]["vIn"] = i key = None if a == 0 else str(math.floor(math.log10(a))) try: input_stat["typeMap"][in_type]["count"] += 1 input_stat["typeMap"][in_type]["amount"] += a except: input_stat["typeMap"][in_type] = {"count": 1, "amount": a, "amountMap": {}} try: input_stat["typeMap"][in_type]["amountMap"][key]["count"] += 1 input_stat["typeMap"][in_type]["amountMap"][key]["amount"] += a except: input_stat["typeMap"][in_type]["amountMap"][key] = {"count": 1, "amount": a} t += 1 n += 1 except: print(traceback.format_exc()) except: pass if self.utxo_data and blocks: blocks[x]["checkpoint"] = x for x in blocks: if self.utxo_data: for y in blocks[x]["rawTx"]: for i in blocks[x]["rawTx"][y]["vOut"]: try: r = self.destroyed_coins.delete((x<<39)+(y<<20)+(1<<19)+i) blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r except: pass if self.option_block_filters: blocks[x]["filter"] = bytearray(b"".join(blocks[x]["filter"])) blocks[x] = pickle.dumps(blocks[x]) await self.pipe_sent_msg(b'result', pickle.dumps(blocks)) except concurrent.futures.CancelledError: pass except Exception as err: self.log.error("block loader restarting: %s" % err) print(traceback.format_exc()) await asyncio.sleep(1) self.loop.create_task(self.load_blocks(start_height, start_limit)) finally: try: await self.rpc.close() except: pass
async def _new_block(self, block): self.block_dependency_tx = 0 """ 0 Check if block already exist in db 1 Check parent block in db: If no parent get last block height from db if last block height >= recent block height this is orphan ignore it else: remove top block from db and ask block with hrecent block height -1 return 2 add all transactions from block to db ask full block from node parse txs and add to db in case not exist 3 call before add block handler^ if this handler rise exception block adding filed 4 add block to db and commit 5 after block add handelr 6 ask next block """ if not self.active or not self.active_block.done(): return if block is None: self.sync = False self.log.debug('Block synchronization completed') return self.active_block = asyncio.Future() binary_block_hash = unhexlify(block["hash"]) binary_previousblock_hash = \ unhexlify(block["previousblockhash"]) \ if "previousblockhash" in block else None block_height = int(block["height"]) next_block_height = block_height + 1 self.log.info("New block %s %s" % (block_height, block["hash"])) bt = q = tm() try: async with self._db_pool.acquire() as con: # blockchain position check block_exist = self.block_cache.get(binary_block_hash) if block_exist is not None: self.log.info("block already exist in db %s" % block["hash"]) return # Get parent from db if binary_previousblock_hash is not None: parent_height = self.block_cache.get( binary_previousblock_hash) else: parent_height = None # self.log.warning("parent height %s" % parent_height) if parent_height is None: # have no mount point in local chain # self.log.warning("last local height %s" % self.last_block_height) if self.last_block_height is not None: if self.last_block_height >= block_height: self.log.critical( "bitcoin node out of sync block %s" % block["hash"]) return if self.last_block_height + 1 == block_height: if self.orphan_handler: tq = tm() await self.orphan_handler( self.last_block_height, con) self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq))) tq = tm() await remove_orphan(self, con) self.log.info("remove orphan %s [%s]" % (self.last_block_height + 1, tm(tq))) next_block_height -= 2 if next_block_height > self.last_block_height: next_block_height = self.last_block_height + 1 if self.sync and next_block_height >= self.sync: if self.sync_requested: next_block_height = self.last_block_height + 1 else: self.sync = next_block_height return else: if self.start_block is not None and block[ "height"] != self.start_block: self.log.info("Start from block %s" % self.start_block) next_block_height = self.start_block return else: if self.last_block_height + 1 != block_height: if self.orphan_handler: tq = tm() await self.orphan_handler(self.last_block_height, con) self.log.info("orphan handler %s [%s]" % (self.last_block_height, tm(tq))) await remove_orphan(self, con) next_block_height -= 1 self.log.debug("requested %s" % next_block_height) return self.log.debug("blockchain position check [%s]" % tm(q)) # add all block transactions q = tm() binary_tx_hash_list = [unhexlify(t)[::-1] for t in block["tx"]] if block["height"] in (91842, 91880): # BIP30 Fix self.tx_cache.pop( s2rh( "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599" )) self.tx_cache.pop( s2rh( "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468" )) tx_id_list, missed = await get_tx_id_list( self, binary_tx_hash_list, con) if len(tx_id_list) + len(missed) != len(block["tx"]): raise Exception("tx count mismatch") self.await_tx_id_list = tx_id_list if self.before_block_handler: sn = await self.before_block_handler( block, missed, self.sync_tx_lock, self.node_last_block, con) if sn and missed: self.await_tx_id_list = self.await_tx_id_list + [ 0 for i in range(len(missed)) ] missed = [] cq = tm() missed = [rh2s(t) for t in missed] self.log.info("Transactions already exist: %s missed %s [%s]" % (len(tx_id_list), len(missed), tm(q))) if missed: self.log.debug("Request missed transactions") self.missed_tx_list = list(missed) self.await_tx_list = missed self.await_tx_future = dict() for i in missed: self.await_tx_future[unhexlify(i) [::-1]] = asyncio.Future() self.block_txs_request = asyncio.Future() if len(missed) == len(block["tx"]): self.loop.create_task( self._get_missed(block["hash"], block["time"], block["height"])) else: self.loop.create_task( self._get_missed(False, block["time"], block["height"])) try: await asyncio.wait_for(self.block_txs_request, timeout=self.block_timeout) except asyncio.CancelledError: # refresh rpc connection session await self.rpc.close() self.rpc = aiojsonrpc.rpc(self.rpc_url, self.loop, timeout=self.rpc_timeout) raise RuntimeError("block transaction request timeout") if len(block["tx"]) != len(self.await_tx_id_list): self.log.error("get block transactions failed") self.log.error(str(self.await_tx_id_list)) raise Exception("get block transactions failed") tx_count = len(self.await_tx_id_list) self.total_received_tx += tx_count self.total_received_tx_time += tm(q) rate = round(self.total_received_tx / self.total_received_tx_time) self.log.info( "Transactions received: %s [%s] rate tx/s ->> %s <<" % (tx_count, tm(cq), rate)) async with con.transaction(): if self.block_received_handler: await self.block_received_handler(block, con) # insert new block await insert_new_block(self, binary_block_hash, block["height"], binary_previousblock_hash, block["time"], con) if not self.external_dublicate_filter: self.loop.create_task( update_block_height(self, block["height"], list(self.await_tx_id_list))) if self.sync == block["height"]: self.sync += 1 next_block_height = self.sync # after block added handler if self.block_handler: await self.block_handler(block, con) self.last_block_height = block["height"] self.block_cache.set(binary_block_hash, block["height"]) except Exception as err: if self.await_tx_list: self.await_tx_list = [] self.log.error(str(traceback.format_exc())) self.log.error("new block error %s" % str(err)) next_block_height = None finally: self.active_block.set_result(True) self.log.debug("block processing completed") if next_block_height is not None: self.sync_requested = True self.loop.create_task( self.get_block_by_height(next_block_height)) self.log.info( "%s block [%s tx/ %s size] (dp %s) processing time %s cache [%s/%s]" % (block["height"], len(block["tx"]), block["size"] / 1000000, self.block_dependency_tx, tm(bt), len(self.block_hashes_preload._store), len(self.block_preload._store)))
async def block_transactions(pointer, option_raw_tx, limit, page, order, mode, app): pool = app["db_pool"] qt = time.time() async with pool.acquire() as conn: if isinstance(pointer, bytes): stmt = await conn.prepare( "SELECT height, hash, header, adjusted_timestamp " "FROM blocks WHERE hash = $1 LIMIT 1;") block_row = await stmt.fetchrow(pointer) if block_row is None: raise APIException(NOT_FOUND, "block not found", status=404) pointer = block_row["height"] block_height = block_row["height"] else: block_height = pointer stmt = await conn.prepare( "SELECT height, hash, header, adjusted_timestamp " "FROM blocks WHERE height = $1 LIMIT 1;") block_row = await stmt.fetchrow(pointer) if block_row is None: raise APIException(NOT_FOUND, "block not found", status=404) count = var_int_to_int(block_row["header"][80:]) pages = math.ceil(count / limit) if app["merkle_proof"]: rows = await conn.fetch( "SELECT tx_id, raw_transaction, timestamp, pointer, merkle_proof " "FROM transaction WHERE pointer >= $1 AND pointer < $2 " "ORDER BY pointer %s LIMIT $3 OFFSET $4;" % order, pointer << 39, (pointer + 1) << 39, limit, limit * (page - 1)) else: rows = await conn.fetch( "SELECT tx_id, raw_transaction, timestamp, pointer " "FROM transaction WHERE pointer >= $1 AND pointer < $2 " "ORDER BY pointer %s LIMIT $3 OFFSET $4;" % order, pointer << 39, (pointer + 1) << 39, limit, limit * (page - 1)) block_time = unpack("<L", block_row["header"][68:68 + 4])[0] transactions = list() for row in rows: tx = Transaction(row["raw_transaction"], format="decoded", testnet=app["testnet"], keep_raw_tx=option_raw_tx) tx["blockIndex"] = (row["pointer"] >> 20) & 524287 tx["blockTime"] = block_time tx["timestamp"] = row["timestamp"] tx["confirmations"] = app["last_block"] - block_height + 1 if app["merkle_proof"]: tx["merkleProof"] = base64.b64encode( row["merkle_proof"]).decode() del tx["blockHash"] del tx["blockTime"] del tx["format"] del tx["testnet"] del tx["time"] del tx["fee"] if not option_raw_tx: del tx["rawTx"] if app["transaction_history"]: tx["inputsAmount"] = 0 if mode == "brief": tx["outputAddresses"] = 0 tx["inputAddresses"] = 0 for z in tx["vOut"]: if "address" in tx["vOut"][z]: tx["outputAddresses"] += 1 transactions.append(tx) app["block_transactions"][block_row["hash"]] = transactions if app["transaction_history"]: # get information about spent input coins rows = await conn.fetch( "SELECT pointer," " s_pointer," " address, " " amount " "FROM stxo " "WHERE stxo.s_pointer >= $1 and stxo.s_pointer < $2 order by stxo.s_pointer asc;", (block_height << 39) + ((limit * (page - 1)) << 20), ((block_height) << 39) + ((limit * (page)) << 20)) for r in rows: s = r["s_pointer"] i = (s - ((s >> 19) << 19)) m = ((s - ((s >> 39) << 39)) >> 20) - limit * (page - 1) transactions[m]["inputsAmount"] += r["amount"] if mode == "verbose": transactions[m]["vIn"][i]["type"] = SCRIPT_N_TYPES[ r["address"][0]] transactions[m]["vIn"][i]["amount"] = r["amount"] transactions[m]["vIn"][i][ "blockHeight"] = r["pointer"] >> 39 transactions[m]["vIn"][i]["confirmations"] = app[ "last_block"] - (r["pointer"] >> 39) + 1 if r["address"][0] in (0, 1, 5, 6): script_hash = True if r["address"][0] in (1, 6) else False witness_version = None if r["address"][0] < 5 else 0 transactions[m]["vIn"][i]["address"] = hash_to_address( r["address"][1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) transactions[m]["vIn"][i][ "scriptPubKey"] = address_to_script( transactions[m]["vIn"][i]["address"], hex=1) elif r["address"][0] == 2: transactions[m]["vIn"][i][ "address"] = script_to_address( r["address"][1:], testnet=app["testnet"]) transactions[m]["vIn"][i]["scriptPubKey"] = r[ "address"][1:].hex() else: transactions[m]["vIn"][i]["scriptPubKey"] = r[ "address"][1:].hex() transactions[m]["vIn"][i][ "scriptPubKeyOpcodes"] = decode_script( transactions[m]["vIn"][i]["scriptPubKey"]) transactions[m]["vIn"][i][ "scriptPubKeyAsm"] = decode_script( transactions[m]["vIn"][i]["scriptPubKey"], 1) else: if r["address"][0] in (0, 1, 2, 5, 6): if mode == "brief": transactions[m]["inputAddresses"] += 1 for m in range(len(transactions)): if transactions[m]["coinbase"]: transactions[m]["fee"] = 0 else: transactions[m]["fee"] = transactions[m][ "inputsAmount"] - transactions[m]["amount"] transactions[m]["outputsAmount"] = transactions[m]["amount"] if mode != "verbose": transactions[m]["inputs"] = len(transactions[m]["vIn"]) transactions[m]["outputs"] = len(transactions[m]["vOut"]) del transactions[m]["vIn"] del transactions[m]["vOut"] # get information about spent output coins if mode == "verbose": # get information about spent output coins rows = await conn.fetch( "SELECT outpoint," " input_index," " tx_id " "FROM connector_unconfirmed_stxo " "WHERE out_tx_id = ANY($1);", [s2rh(t["txId"]) for t in transactions]) out_map = dict() for v in rows: i = bytes_to_int(v["outpoint"][32:]) try: out_map[(rh2s(v["outpoint"][:32]), i)].append({ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }) except: out_map[(rh2s(v["outpoint"][:32]), i)] = [{ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }] rows = await conn.fetch( "SELECT stxo.pointer," " stxo.s_pointer," " transaction.tx_id " "FROM stxo " "JOIN transaction " "ON transaction.pointer = (stxo.s_pointer >> 18)<<18 " "WHERE stxo.pointer >= $1 and " "stxo.pointer < $2 order by stxo.pointer ;", (block_height << 39) + ((limit * (page - 1)) << 20), ((block_height) << 39) + ((limit * page) << 20)) p_out_map = dict() for v in rows: p_out_map[v["pointer"]] = [{ "txId": rh2s(v["tx_id"]), "vIn": v["s_pointer"] & 0b111111111111111111 }] for t in range(len(transactions)): o_pointer = (block_height << 39) + ( transactions[t]["blockIndex"] << 20) + (1 << 19) for i in transactions[t]["vOut"]: try: transactions[t]["vOut"][i]["spent"] = p_out_map[ o_pointer + i] except: try: transactions[t]["vOut"][i]["spent"] = out_map[( transactions[t]["txId"], int(i))] except: transactions[t]["vOut"][i]["spent"] = [] resp = { "data": { "list": transactions, "page": page, "pages": pages, "total": count, "limit": limit }, "time": round(time.time() - qt, 4) } return resp
async def tx_by_pointer_opt_tx(pointer, option_raw_tx, app): q = time.time() block_height = None block_index = None block_hash = None mempool_rank = None invalid_tx = False invalidation_timestamp = None mempool_conflict = False last_dep = dict() conflict_outpoints = deque() print('tx_by_pointer_opt_tx') async with app["db_pool"].acquire() as conn: if isinstance(pointer, bytes): row = await conn.fetchrow( "SELECT tx_id " "FROM mempool_dbs " "WHERE tx_id = $1 LIMIT 1;", pointer) if row: mempool_conflict = True row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp " "FROM unconfirmed_transaction " "WHERE tx_id = $1 LIMIT 1;", pointer) if row is None: if app["merkle_proof"]: row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp, pointer, " "merkle_proof " "FROM transaction " "WHERE tx_id = $1 LIMIT 1;", pointer) else: row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp, pointer " "FROM transaction " "WHERE tx_id = $1 LIMIT 1;", pointer) if row is None: row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp, invalidation_timestamp " "FROM invalid_transaction " "WHERE tx_id = $1 LIMIT 1;", pointer) if row is None: raise APIException(NOT_FOUND, "transaction not found", status=404) invalid_tx = True invalidation_timestamp = row["invalidation_timestamp"] if not invalid_tx: block_height = row["pointer"] >> 39 block_index = (row["pointer"] >> 20) & 524287 block_row = await conn.fetchrow( "SELECT hash, header, adjusted_timestamp " "FROM blocks " "WHERE height = $1 LIMIT 1;", block_height) adjusted_timestamp = block_row["adjusted_timestamp"] block_time = unpack("<L", block_row["header"][68:68 + 4])[0] block_hash = block_row["hash"] else: mempool_rank = await conn.fetchval( """SELECT ranks.rank FROM (SELECT tx_id, rank() OVER(ORDER BY feerate DESC) as rank FROM unconfirmed_transaction) ranks WHERE tx_id = $1 LIMIT 1""", pointer) else: if app["merkle_proof"]: row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp, pointer, " "merkle_proof " "FROM transaction " "WHERE pointer = $1 LIMIT 1;", pointer) else: row = await conn.fetchrow( "SELECT tx_id, raw_transaction, timestamp, pointer " "FROM transaction " "WHERE pointer = $1 LIMIT 1;", pointer) if row is None: raise APIException(NOT_FOUND, "transaction not found", status=404) block_height = row["pointer"] >> 39 block_index = (row["pointer"] >> 20) & 524287 block_row = await conn.fetchrow( "SELECT hash, header, adjusted_timestamp " "FROM blocks " "WHERE height = $1 LIMIT 1;", block_height) adjusted_timestamp = block_row["adjusted_timestamp"] block_time = unpack("<L", block_row["header"][68:68 + 4])[0] block_hash = block_row["hash"] tx = Transaction(row["raw_transaction"], format="decoded", testnet=app["testnet"], keep_raw_tx=option_raw_tx) if app["transaction_history"]: for i in tx["vOut"]: tx["vOut"][i]['spent'] = [] # get information about spent input coins if block_height is not None: s_pointers = [(block_height << 39) + (block_index << 20) + i for i in tx["vIn"]] rows = await conn.fetch( "SELECT pointer," " s_pointer," " address, " " amount " "FROM stxo " "WHERE stxo.s_pointer = ANY($1);", s_pointers) tx["inputsAmount"] = 0 if not tx["coinbase"]: assert len(rows) == len(s_pointers) for r in rows: s = r["s_pointer"] i = (s - ((s >> 19) << 19)) tx["vIn"][i]["type"] = SCRIPT_N_TYPES[r["address"][0]] tx["vIn"][i]["amount"] = r["amount"] tx["inputsAmount"] += r["amount"] tx["vIn"][i]["blockHeight"] = r["pointer"] >> 39 tx["vIn"][i]["confirmations"] = app["last_block"] - ( r["pointer"] >> 39) + 1 if r["address"][0] in (0, 1, 5, 6): script_hash = True if r["address"][0] in (1, 6) else False witness_version = None if r["address"][0] < 5 else 0 tx["vIn"][i]["address"] = hash_to_address( r["address"][1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) tx["vIn"][i]["scriptPubKey"] = address_to_script( tx["vIn"][i]["address"], hex=1) elif r["address"][0] == 2: tx["vIn"][i]["address"] = script_to_address( r["address"][1:], testnet=app["testnet"]) tx["vIn"][i]["scriptPubKey"] = r["address"][1:].hex() else: tx["vIn"][i]["scriptPubKey"] = r["address"][1:].hex() tx["vIn"][i]["scriptPubKeyOpcodes"] = decode_script( tx["vIn"][i]["scriptPubKey"]) tx["vIn"][i]["scriptPubKeyAsm"] = decode_script( tx["vIn"][i]["scriptPubKey"], 1) else: if invalid_tx: rows = await conn.fetch( "SELECT outpoint," " input_index," " out_tx_id, " " tx_id," " invalid_stxo.address, " " invalid_stxo.amount " " " "FROM invalid_stxo " "WHERE tx_id = $1;", s2rh(tx["txId"])) else: rows = await conn.fetch( "SELECT outpoint," " input_index," " out_tx_id, " " tx_id," " connector_unconfirmed_stxo.address, " " connector_unconfirmed_stxo.amount " " " "FROM connector_unconfirmed_stxo " "WHERE tx_id = $1;", s2rh(tx["txId"])) c_rows = await conn.fetch( """ SELECT pointer, tx_id FROM transaction WHERE tx_id = ANY($1); """, [s2rh(tx["vIn"][q]["txId"]) for q in tx["vIn"]]) tx_id_map_pointer = dict() for r in c_rows: tx_id_map_pointer[rh2s(r["tx_id"])] = r["pointer"] tx["inputsAmount"] = 0 if not tx["coinbase"]: assert len(rows) == len(tx["vIn"]) for r in rows: i = r["input_index"] tx["vIn"][i]["type"] = SCRIPT_N_TYPES[r["address"][0]] tx["vIn"][i]["amount"] = r["amount"] tx["inputsAmount"] += r["amount"] try: p = tx_id_map_pointer[tx["vIn"][i]["txId"]] tx["vIn"][i]["blockHeight"] = p >> 39 tx["vIn"][i]["confirmations"] = app["last_block"] - ( p >> 39) + 1 except: tx["vIn"][i]["blockHeight"] = None tx["vIn"][i]["confirmations"] = None if invalid_tx or mempool_conflict: op = b"%s%s" % (s2rh(tx["vIn"][i]["txId"]), int_to_bytes(tx["vIn"][i]["vOut"])) conflict_outpoints.append(op) try: last_dep[op[:32]].add(tx["vIn"][i]["vOut"]) except: last_dep[op[:32]] = {tx["vIn"][i]["vOut"]} if r["address"][0] in (0, 1, 5, 6): script_hash = True if r["address"][0] in (1, 6) else False witness_version = None if r["address"][0] < 5 else 0 try: tx["vIn"][i]["address"] = hash_to_address( r["address"][1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) tx["vIn"][i]["scriptPubKey"] = address_to_script( tx["vIn"][i]["address"], hex=1) except: print("???", r["address"].hex()) raise elif r["address"][0] == 2: tx["vIn"][i]["address"] = script_to_address( r["address"][1:], testnet=app["testnet"]) tx["vIn"][i]["scriptPubKey"] = r["address"][1:].hex() else: tx["vIn"][i]["scriptPubKey"] = r["address"][1:].hex() tx["vIn"][i]["scriptPubKeyOpcodes"] = decode_script( tx["vIn"][i]["scriptPubKey"]) tx["vIn"][i]["scriptPubKeyAsm"] = decode_script( tx["vIn"][i]["scriptPubKey"], 1) # get information about spent output coins if invalid_tx: rows = await conn.fetch( "SELECT outpoint," " input_index," " tx_id " "FROM invalid_stxo " "WHERE out_tx_id = $1;", s2rh(tx["txId"])) else: rows = await conn.fetch( "SELECT outpoint," " input_index," " tx_id " "FROM connector_unconfirmed_stxo " "WHERE out_tx_id = $1;", s2rh(tx["txId"])) for r in rows: i = bytes_to_int(r["outpoint"][32:]) tx["vOut"][i]['spent'].append({ "txId": rh2s(r["tx_id"]), "vIn": r["input_index"] }) if block_height is not None: pointers = [ (block_height << 39) + (block_index << 20) + (1 << 19) + i for i in tx["vOut"] ] rows = await conn.fetch( "SELECT stxo.pointer," " stxo.s_pointer," " transaction.tx_id " "FROM stxo " "JOIN transaction " "ON transaction.pointer = (stxo.s_pointer >> 18)<<18 " "WHERE stxo.pointer = ANY($1);", pointers) for r in rows: i = r["pointer"] & 0b111111111111111111 tx["vOut"][i]['spent'].append({ "txId": rh2s(r["tx_id"]), "vIn": r["s_pointer"] & 0b111111111111111111 }) if not tx["coinbase"]: tx["fee"] = tx["inputsAmount"] - tx["amount"] else: tx["fee"] = 0 tx["outputsAmount"] = tx["amount"] tx["blockHeight"] = block_height tx["blockIndex"] = block_index if block_hash is not None: tx["blockHash"] = rh2s(block_hash) tx["adjustedTimestamp"] = adjusted_timestamp tx["blockTime"] = block_time else: tx["blockHash"] = None tx["adjustedTimestamp"] = None tx["blockTime"] = None tx["time"] = row["timestamp"] if block_height: tx["confirmations"] = app["last_block"] - block_height + 1 if app["merkle_proof"]: tx["merkleProof"] = base64.b64encode(row["merkle_proof"]).decode() del tx["format"] del tx["testnet"] if not app["transaction_history"]: del tx["fee"] if not option_raw_tx: del tx["rawTx"] else: tx["rawTx"] = base64.b64encode(bytes_from_hex(tx["rawTx"])).decode() if mempool_rank is not None: tx["mempoolRank"] = mempool_rank tx["valid"] = not invalid_tx tx["feeRate"] = round(tx["fee"] / tx["vSize"], 2) if invalid_tx or mempool_conflict: m_conflict = [] i_conflict_chain = [] dep_chain = [[tx["txId"]]] conflict_outpoints_chain = [conflict_outpoints] # find out mainnet competitors and invalid tx chains while last_dep: async with app["db_pool"].acquire() as conn: rows = await conn.fetch( "SELECT tx_id, pointer FROM transaction where tx_id = ANY($1)", last_dep.keys()) if len(rows) >= len(last_dep): break for row in rows: try: del last_dep[row["tx_id"]] except: pass dep_chain.append([rh2s(k) for k in last_dep.keys()]) # go deeper async with app["db_pool"].acquire() as conn: if invalid_tx: rows = await conn.fetch( "SELECT out_tx_id," " outpoint " "from invalid_stxo " "WHERE tx_id = ANY($1)", last_dep.keys()) else: rows = await conn.fetch( "SELECT out_tx_id," " outpoint " "from connector_unconfirmed_stxo " "WHERE tx_id = ANY($1)", last_dep.keys()) last_dep = dict() conflict_outpoints = deque() for row in rows: v_out = bytes_to_int(row["outpoint"][32:]) try: last_dep[row["out_tx_id"]].add(v_out) except: last_dep[row["out_tx_id"]] = {v_out} conflict_outpoints.append(row["outpoint"]) conflict_outpoints_chain.append(conflict_outpoints) o_pointers = set() pointer_map = dict() for row in rows: pointer = row["pointer"] t = row["tx_id"] for v_out in last_dep[t]: pointer_map[pointer + (1 << 19) + v_out] = [rh2s(t), v_out] o_pointers.add(pointer + (1 << 19) + v_out) if mempool_conflict: m_conflict.append({ "outpoint": { "txId": rh2s(t), "vOut": v_out, "block": pointer >> 39 } }) if invalid_tx: async with app["db_pool"].acquire() as conn: rows = await conn.fetch( "SELECT transaction.tx_id, stxo.s_pointer, stxo.pointer " "FROM stxo " "JOIN transaction on transaction.pointer = (stxo.s_pointer >> 18)<<18 " "WHERE stxo.pointer = ANY($1);", o_pointers) for row in rows: m_conflict.append({ "doublespend": { "txId": pointer_map[row["pointer"]][0], "vOut": pointer_map[row["pointer"]][1], "block": row["pointer"] >> 39 }, "competitor": { "txId": rh2s(row["tx_id"]), "vIn": row["s_pointer"] & 524287, "block": row["s_pointer"] >> 39 } }) chain = dep_chain atx = set() atx.add(s2rh(tx["txId"])) # check invalid transactions competitors for conflict_outpoints in conflict_outpoints_chain: [atx.add(t[:32]) for t in conflict_outpoints] if invalid_tx: async with app["db_pool"].acquire() as conn: irows = await conn.fetch( "SELECT invalid_stxo.tx_id," " invalid_stxo.out_tx_id, " " invalid_stxo.input_index, " " invalid_stxo.outpoint, " " invalid_transaction.size, " " invalid_transaction.b_size, " " invalid_transaction.feerate, " " invalid_transaction.fee," " invalid_transaction.rbf, " " invalid_transaction.segwit " "from invalid_stxo " "JOIN invalid_transaction " "ON invalid_transaction.tx_id = invalid_stxo.tx_id " "WHERE outpoint = ANY($1)", conflict_outpoints) else: async with app["db_pool"].acquire() as conn: irows = await conn.fetch( "SELECT connector_unconfirmed_stxo.tx_id," " connector_unconfirmed_stxo.out_tx_id, " " connector_unconfirmed_stxo.input_index, " " connector_unconfirmed_stxo.outpoint, " " unconfirmed_transaction.size, " " unconfirmed_transaction.b_size, " " unconfirmed_transaction.feerate, " " unconfirmed_transaction.fee," " unconfirmed_transaction.rbf, " " unconfirmed_transaction.segwit " "from connector_unconfirmed_stxo " "JOIN unconfirmed_transaction " "ON unconfirmed_transaction.tx_id = connector_unconfirmed_stxo.tx_id " "WHERE outpoint = ANY($1)", conflict_outpoints) i_conflict = [] for row in irows: h = rh2s(row["tx_id"]) if row["tx_id"] not in atx: i_conflict.append({ "doublespend": { "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]) }, "competitor": { "txId": h, "vIn": row["input_index"], "size": row["size"], "bSize": row["b_size"], "feeRate": round(row["feerate"], 2), "fee": row["fee"], "rbf": bool(row["rbf"]), "segwit": bool(row["segwit"]) } }) i_conflict_chain.append(i_conflict) if invalid_tx: del tx["blockHash"] del tx["confirmations"] del tx["blockTime"] del tx["blockIndex"] del tx["blockHeight"] del tx["adjustedTimestamp"] if "flag" in tx: del tx["flag"] if invalid_tx: tx["conflict"] = { "blockchain": m_conflict, "invalidTxCompetitors": i_conflict_chain, "invalidTxChain": chain, "invalidationTimestamp": invalidation_timestamp } else: tx["conflict"] = { "blockchain": m_conflict, "txCompetitors": i_conflict_chain, "unconfirmedChain": chain } return {"data": tx, "time": round(time.time() - q, 4)}
async def address_unconfirmed_utxo(address, type, order, limit, page, app): q = time.time() utxo = [] if address[0] == 0 and type is None: a = [address] async with app["db_pool"].acquire() as conn: script = await conn.fetchval( "SELECT script from connector_unconfirmed_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is None: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: a.append(b"\x02" + script) rows = await conn.fetch( "SELECT outpoint, amount, address " "FROM connector_unconfirmed_utxo " "WHERE address = ANY($1) " "order by amount %s LIMIT $2 OFFSET $3;" % order, a, limit if limit else "ALL", limit * (page - 1)) for row in rows: utxo.append({ "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]), "amount": row["amount"], "type": SCRIPT_N_TYPES[row["address"][0]] }) else: async with app["db_pool"].acquire() as conn: if address[0] == 0: if type == 2: script = await conn.fetchval( "SELECT script from connector_unconfirmed_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is None: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: address = b"\x02" + script else: return { "data": utxo, "time": round(time.time() - q, 4) } rows = await conn.fetch( "SELECT outpoint, amount, address " "FROM connector_unconfirmed_utxo " "WHERE address = $1 " "order by amount %s LIMIT $2 OFFSET $3;" % order, address, limit if limit else "ALL", limit * (page - 1)) for row in rows: utxo.append({ "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]), "amount": row["amount"] }) return {"data": utxo, "time": round(time.time() - q, 4)}
async def address_transactions(address, type, limit, page, order, mode, timeline, app): q = time.time() qt = time.time() pages = 0 if address[0] == 0 and type is None: a = [address] async with app["db_pool"].acquire() as conn: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: a.append(b"\x02" + script) else: async with app["db_pool"].acquire() as conn: if address[0] == 0: if type == 2: script = await conn.fetchval( "SELECT script from connector_unconfirmed_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is None: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: address = b"\x02" + script else: return { "data": { "confirmed": 0, "unconfirmed": 0 }, "time": round(time.time() - q, 4) } a = [address] print("1", time.time() - qt) qt = time.time() pages = 0 for z in a: s = await address_state_extended(z, app) pages += s["data"]["receivedTxCount"] + s["data"]["sentTxCount"] pages = math.ceil(pages / limit) async with app["db_pool"].acquire() as conn: # get total transactions count to determine pages count print("4", time.time() - qt) qt = time.time() tx_id_list = await conn.fetch( "SELECT pointer FROM transaction_map " "WHERE address = ANY($1) order by pointer %s " "LIMIT $2 OFFSET $3;" % order, a, limit, limit * (page - 1)) l = [t["pointer"] for t in tx_id_list] print("4.1", time.time() - qt) qt = time.time() rows = await conn.fetch( "SELECT transaction.pointer," " transaction.raw_transaction," " transaction.tx_id, " " transaction.timestamp " "FROM transaction " "WHERE pointer = ANY($1) " "order by pointer %s " "LIMIT $2 ;" % order, l, limit) print("3", time.time() - qt) qt = time.time() target_scripts = [] tx_list = [] s_pointers = [] tx_id_set = set() o_pointers = [] for row in rows: tx_id_set.add(row["tx_id"]) tx = Transaction(row["raw_transaction"], testnet=app["testnet"]) tx["blockHeight"] = row["pointer"] >> 39 tx["blockIndex"] = (row["pointer"] >> 20) & 524287 tx["timestamp"] = row["timestamp"] tx["confirmations"] = app["last_block"] - tx["blockHeight"] + 1 try: tx["blockTime"] = app["block_map_time"][tx["blockHeight"]][1] except: pass tx_pointer = (tx["blockHeight"] << 39) + (tx["blockIndex"] << 20) for i in tx["vIn"]: s_pointers.append(row["pointer"] + i) for i in tx["vOut"]: o_pointers.append(tx_pointer + (1 << 19) + i) tx_list.append(tx) ts = dict() for d in a: if d[0] in (0, 1, 5, 6): ts[hash_to_script(d[1:], d[0], hex=True)] = { "r": 0, "s": 0, "i": 0, "o": 0 } else: ts[d[1:].hex()] = {"r": 0, "s": 0, "i": 0, "o": 0} target_scripts.append(ts) async with app["db_pool"].acquire() as conn: stxo = await conn.fetch( "SELECT s_pointer, pointer, amount, address FROM stxo " "WHERE stxo.s_pointer = ANY($1);", s_pointers) stxo_map = {} for row in stxo: stxo_map[row["s_pointer"]] = (row["address"], row["amount"], row["pointer"]) for i in range(len(tx_list)): tx_list[i]["inputsAmount"] = 0 tx_list[i]["inputAddressCount"] = 0 tx_list[i]["outAddressCount"] = 0 tx_list[i]["inputsCount"] = len(tx_list[i]["vIn"]) tx_list[i]["outsCount"] = len(tx_list[i]["vOut"]) tx_pointer = (tx_list[i]["blockHeight"] << 39) + ( tx_list[i]["blockIndex"] << 20) if not tx_list[i]["coinbase"]: for k in tx_list[i]["vIn"]: d = stxo_map[tx_pointer + k] tx_list[i]["vIn"][k]["type"] = SCRIPT_N_TYPES[d[0][0]] tx_list[i]["vIn"][k]["amount"] = d[1] tx_list[i]["inputsAmount"] += d[1] pointer = d[2] tx_list[i]["vIn"][k]["blockHeight"] = pointer >> 39 tx_list[i]["vIn"][k]["confirmations"] = app["last_block"] - ( pointer >> 39) + 1 if d[0][0] in (0, 1, 2, 5, 6): script_hash = True if d[0][0] in (1, 6) else False witness_version = None if d[0][0] < 5 else 0 try: if d[0][0] == 2: ad = b"\x02" + parse_script( d[0][1:])["addressHash"] else: ad = d[0] tx_list[i]["vIn"][k]["address"] = hash_to_address( ad[1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) tx_list[i]["vIn"][k][ "scriptPubKey"] = address_to_script( tx_list[i]["vIn"][k]["address"], hex=1) except: print(tx_list[i]["txId"]) print("??", d[0].hex()) raise tx_list[i]["inputAddressCount"] += 1 else: tx_list[i]["vIn"][k]["scriptPubKey"] = d[0][1:].hex() tx_list[i]["vIn"][k]["scriptPubKeyOpcodes"] = decode_script( tx_list[i]["vIn"][k]["scriptPubKey"]) tx_list[i]["vIn"][k]["scriptPubKeyAsm"] = decode_script( tx_list[i]["vIn"][k]["scriptPubKey"], 1) for ti in target_scripts[i]: if ti == tx_list[i]["vIn"][k]["scriptPubKey"]: target_scripts[i][ti]["s"] += tx_list[i]["vIn"][k][ "amount"] target_scripts[i][ti]["i"] += 1 if not tx_list[i]["coinbase"]: tx_list[i][ "fee"] = tx_list[i]["inputsAmount"] - tx_list[i]["amount"] else: tx_list[i]["fee"] = 0 tx_list[i]["outputsAmount"] = tx_list[i]["amount"] print("2", time.time() - qt) qt = time.time() # get information about spent output coins async with app["db_pool"].acquire() as conn: rows = await conn.fetch( "SELECT outpoint," " input_index," " tx_id " "FROM connector_unconfirmed_stxo " "WHERE out_tx_id = ANY($1);", tx_id_set) out_map = dict() for v in rows: i = bytes_to_int(v["outpoint"][32:]) try: out_map[(rh2s(v["outpoint"][:32]), i)].append({ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }) except: out_map[(rh2s(v["outpoint"][:32]), i)] = [{ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }] async with app["db_pool"].acquire() as conn: rows = await conn.fetch( "SELECT stxo.pointer," " stxo.s_pointer," " transaction.tx_id, " " transaction.timestamp " "FROM stxo " "JOIN transaction " "ON transaction.pointer = (stxo.s_pointer >> 18)<<18 " "WHERE stxo.pointer = ANY($1);", o_pointers) p_out_map = dict() for v in rows: p_out_map[v["pointer"]] = [{ "txId": rh2s(v["tx_id"]), "vIn": v["s_pointer"] & 0b111111111111111111 }] print("1", time.time() - qt) qt = time.time() for t in range(len(tx_list)): if tx_list[t]["blockHeight"] is not None: o_pointer = (tx_list[t]["blockHeight"] << 39) + ( tx_list[t]["blockIndex"] << 20) + (1 << 19) for i in tx_list[t]["vOut"]: try: tx_list[t]["vOut"][i]["spent"] = p_out_map[o_pointer + i] except: try: tx_list[t]["vOut"][i]["spent"] = out_map[( tx_list[t]["txId"], int(i))] except: tx_list[t]["vOut"][i]["spent"] = [] for ti in target_scripts[t]: if ti == tx_list[t]["vOut"][i]["scriptPubKey"]: target_scripts[t][ti]["r"] += tx_list[t]["vOut"][i][ "value"] target_scripts[t][ti]["o"] += 1 if "address" in tx_list[t]["vOut"][i]: tx_list[t]["outAddressCount"] += 1 address_amount = 0 address_received = 0 address_outputs = 0 address_sent = 0 address_inputs = 0 for ti in target_scripts[t]: address_amount += target_scripts[t][ti]["r"] - target_scripts[ t][ti]["s"] address_received += target_scripts[t][ti]["r"] address_outputs += target_scripts[t][ti]["o"] address_sent += target_scripts[t][ti]["s"] address_inputs += target_scripts[t][ti]["i"] tx_list[t]["amount"] = address_amount tx_list[t]["addressReceived"] = address_received tx_list[t]["addressOuts"] = address_outputs tx_list[t]["addressSent"] = address_sent tx_list[t]["addressInputs"] = address_inputs if mode != "verbose": del tx_list[t]["vIn"] del tx_list[t]["vOut"] del tx_list[t]["format"] del tx_list[t]["testnet"] del tx_list[t]["rawTx"] del tx_list[t]["hash"] del tx_list[t]["blockHash"] del tx_list[t]["time"] try: del tx_list[t]["flag"] except: pass if timeline and tx_list: tx_pointer = (tx_list[-1]["blockHeight"] << 39) + ( tx_list[-1]["blockIndex"] << 20) print(tx_list[-1]["blockHeight"]) r = await address_state_extended_in_pointer(a, tx_pointer, app) print(r) state = { "receivedAmount": r['data']["receivedAmount"], "receivedTxCount": r['data']["receivedTxCount"], "sentAmount": r['data']["sentAmount"], "sentTxCount": r['data']["sentTxCount"], "receivedOutsCount": r['data']["receivedOutsCount"], "spentOutsCount": r['data']["spentOutsCount"] } k = len(tx_list) l = len(tx_list) while k: k -= 1 if order == "desc": i = k else: i = l - (k + 1) tx = tx_list[i] state["receivedAmount"] += tx["addressReceived"] state["receivedOutsCount"] += tx["addressOuts"] state["sentAmount"] += tx["addressSent"] state["spentOutsCount"] += tx["addressInputs"] if tx["addressReceived"] >= tx["addressSent"]: state["receivedTxCount"] += 1 else: state["sentTxCount"] += 1 tx_list[i]["timelineState"] = { "receivedAmount": state["receivedAmount"], "receivedTxCount": state["receivedTxCount"], "sentAmount": state["sentAmount"], "sentTxCount": state["sentTxCount"], "receivedOutsCount": state["receivedOutsCount"], "spentOutsCount": state["spentOutsCount"] } return { "data": { "page": page, "limit": limit, "pages": pages, "list": tx_list }, "time": round(time.time() - q, 4) }
async def data_last_n_blocks(n, app): pool = app["db_pool"] qt = time.time() async with pool.acquire() as conn: rows = await conn.fetch( "SELECT height," " hash," " miner," " timestamp_received," " data," " header," " adjusted_timestamp " "FROM blocks ORDER BY height desc LIMIT $1;", n) nx_map = dict() cb_pointers = [] for row in rows: cb_pointers.append(row["height"] << 39) nx_map[row["height"] - 1] = rh2s(row["hash"]) cb_pointers = [row["height"] << 39 for row in rows] cb_rows = await conn.fetch( "SELECT pointer, raw_transaction " "FROM transaction WHERE pointer = ANY($1);", cb_pointers) cb_map = dict() for cb in cb_rows: cb_map[cb["pointer"] >> 39] = Transaction(cb["raw_transaction"], format="raw") if rows is None: raise APIException(NOT_FOUND, "blocks not found", status=404) r = [] for row in rows: block = dict() block["height"] = row["height"] if block["height"] not in app["block_map_time"]: print(1) await block_map_update(app) if block["height"] not in app["block_map_time"]: print(block["height"], app["last_block"]) raise Exception("internal error") block["hash"] = rh2s(row["hash"]) block["header"] = base64.b64encode(row["header"]).decode() d = json.loads(row["data"]) for k in d: block[k] = d[k] if row["miner"]: block["miner"] = json.loads(row["miner"]) else: block["miner"] = None block["medianBlockTime"] = app["block_map_time"][block["height"]][2] block["blockTime"] = app["block_map_time"][block["height"]][1] block["receivedTimestamp"] = row["timestamp_received"] block["adjustedTimestamp"] = row["adjusted_timestamp"] block["bitsHex"] = block["bits"] block["bits"] = bytes_to_int(bytes_from_hex(block["bits"])) block["nonceHex"] = block["nonce"].to_bytes(4, byteorder="big").hex() block["versionHex"] = int_to_bytes(block["version"]).hex() block["difficulty"] = block["targetDifficulty"] q = int.from_bytes(s2rh(block["hash"]), byteorder="little") block["blockDifficulty"] = target_to_difficulty(q) del block["targetDifficulty"] try: block["nextBlockHash"] = nx_map[block["height"]] except: block["nextBlockHash"] = None # get coinbase transaction tx = cb_map[block["height"]] block["estimatedBlockReward"] = 200 * 100000000 >> block[ "height"] // 52500 block["blockReward"] = tx["amount"] if tx["amount"] > block["estimatedBlockReward"]: block["blockReward"] = block["estimatedBlockReward"] block["blockFeeReward"] = tx["amount"] - block[ "estimatedBlockReward"] else: block["blockReward"] = tx["amount"] block["blockFeeReward"] = 0 block["confirmations"] = app["last_block"] - block["height"] + 1 block["transactionsCount"] = var_int_to_int(row["header"][80:]) block["coinbase"] = tx["vIn"][0]["scriptSig"].hex() r.append(block) resp = {"data": r, "time": round(time.time() - qt, 4)} return resp
async def tx_by_pointers_opt_tx(pointers, hashes, option_raw_tx, app): q = time.time() async with app["db_pool"].acquire() as conn: h_rows, uh_rows, p_row = [], [], [] if hashes: if app["merkle_proof"]: h_rows = await conn.fetch("SELECT tx_id, raw_transaction, timestamp, pointer, merkle_proof," " blocks.hash, header, adjusted_timestamp " "FROM transaction " "JOIN blocks ON " "blocks.height = pointer >> 39 " "WHERE tx_id = ANY($1);", hashes) else: h_rows = await conn.fetch("SELECT tx_id, raw_transaction, timestamp, pointer, " " blocks.hash, header, adjusted_timestamp " "FROM transaction " "JOIN blocks ON " "blocks.height = pointer >> 39 " "WHERE tx_id = ANY($1);", hashes) if len(h_rows) < len(hashes): uh_rows = await conn.fetch("SELECT tx_id, raw_transaction, timestamp " "FROM unconfirmed_transaction " "WHERE tx_id = ANY($1);", hashes) if pointers: if app["merkle_proof"]: p_row = await conn.fetch("SELECT tx_id, raw_transaction, timestamp, pointer, merkle_proof, " " blocks.hash, header, adjusted_timestamp " "FROM transaction " " JOIN blocks ON " "blocks.height = pointer >> 39 " "WHERE pointer = ANY($1);", pointers) else: p_row = await conn.fetch("SELECT tx_id, raw_transaction, timestamp, pointer, merkle_proof, " " blocks.hash, header, adjusted_timestamp " "FROM transaction " "JOIN blocks ON " "blocks.height = pointer >> 39 " "WHERE pointer = ANY($1);", pointers) r = dict() txs = dict() s_pointers = [] o_pointers = [] tx_id_list = [] for row in h_rows: block_height = row["pointer"] >> 39 block_index = (row["pointer"] >> 20) & 524287 block_time = unpack("<L", row["header"][68: 68 + 4])[0] tx = Transaction(row["raw_transaction"],format="decoded",testnet=app["testnet"],keep_raw_tx=option_raw_tx) tx["blockHeight"] = block_height tx["blockIndex"] = block_index tx["blockHash"] = rh2s(row["hash"]) tx["adjustedTimestamp"] = row["adjusted_timestamp"] tx["blockTime"] = block_time tx["timestamp"] = row["timestamp"] tx["confirmations"] = app["last_block"] - block_height + 1 if app["merkle_proof"]: tx["merkleProof"] = base64.b64encode(row["merkle_proof"]).decode() del tx["format"] del tx["testnet"] del tx["fee"] del tx["rawTx"] r[tx["txId"]] = tx if app["transaction_history"]: [s_pointers.append((block_height << 39) + (block_index << 20) + i) for i in tx["vIn"]] [o_pointers.append((block_height << 39) + (block_index << 20) + (1 << 19) + i) for i in tx["vIn"]] tx_id_list.append(s2rh(tx["txId"])) us_pointers = [] us_pointers_inputs = [] for row in uh_rows: block_height = None block_index = None block_time = None tx = Transaction(row["raw_transaction"],format="decoded",testnet=app["testnet"],keep_raw_tx=option_raw_tx) tx["blockHeight"] = block_height tx["blockIndex"] = block_index tx["blockHash"] = None tx["adjustedTimestamp"] = None tx["blockTime"] = block_time tx["time"] = row["timestamp"] tx["confirmations"] = 0 del tx["format"] del tx["testnet"] del tx["fee"] if not option_raw_tx: del tx["rawTx"] r[tx["txId"]] = tx if app["transaction_history"]: us_pointers.append(s2rh(tx["txId"])) tx_id_list.append(s2rh(tx["txId"])) [us_pointers_inputs.append(s2rh(tx["vIn"][v]["txId"])) for v in tx["vIn"]] for row in p_row: block_height = row["pointer"] >> 39 block_index = (row["pointer"] >> 20) & 524287 block_time = unpack("<L", row["header"][68: 68 + 4])[0] tx = Transaction(row["raw_transaction"],format="decoded",testnet=app["testnet"],keep_raw_tx=option_raw_tx) tx["blockHeight"] = block_height tx["blockIndex"] = block_index tx["blockHash"] = rh2s(row["hash"]) tx["adjustedTimestamp"] = row["adjusted_timestamp"] tx["blockTime"] = block_time tx["timestamp"] = row["timestamp"] tx["confirmations"] = app["last_block"] - block_height + 1 if app["merkle_proof"]: tx["merkleProof"] = base64.b64encode(row["merkle_proof"]).decode() del tx["format"] del tx["testnet"] del tx["fee"] if not option_raw_tx: del tx["rawTx"] pointer = row["pointer"] r["%s:%s" % (pointer >> 39, (pointer >> 20) & 524287)] = tx if app["transaction_history"]: [s_pointers.append((block_height << 39) + (block_index << 20) + i) for i in tx["vIn"]] tx_id_list.append(s2rh(tx["txId"])) [o_pointers.append((block_height << 39) + (block_index << 20) + (1 << 19) + i) for i in tx["vIn"]] if app["transaction_history"]: # get information about spent input coins if s_pointers: rows = await conn.fetch("SELECT pointer," " s_pointer," " address, " " amount " "FROM stxo " "WHERE stxo.s_pointer = ANY($1);", s_pointers) s_pointers_map = dict() for v in rows: s_pointers_map[v["s_pointer"]] = v for t in r: if r[t]["blockHeight"] is None: continue s_pointer = (r[t]["blockHeight"] << 39) + (r[t]["blockIndex"] << 20) r[t]["inputsAmount"] = 0 for i in r[t]["vIn"]: if r[t]["coinbase"]: continue d = s_pointers_map[s_pointer + i] r[t]["vIn"][i]["type"] = SCRIPT_N_TYPES[d["address"][0]] r[t]["vIn"][i]["amount"] = d["amount"] r[t]["inputsAmount"] += d["amount"] r[t]["vIn"][i]["blockHeight"] = d["pointer"] >> 39 r[t]["vIn"][i]["confirmations"] = app["last_block"] - (d["pointer"] >> 39) + 1 if d["address"][0] in (0,1,2,5,6): script_hash = True if d["address"][0] in (1, 6) else False witness_version = None if d["address"][0] < 5 else 0 r[t]["vIn"][i]["address"] = hash_to_address(d["address"][1:], testnet=app["testnet"], script_hash = script_hash, witness_version = witness_version) r[t]["vIn"][i]["scriptPubKey"] = address_to_script(r[t]["vIn"][i]["address"], hex=1) elif r["address"][0] == 2: r[t]["vIn"][i]["address"] = script_to_address(d["address"][1:], testnet=app["testnet"]) r[t]["vIn"][i]["scriptPubKey"] = d["address"][1:].hex() else: r[t]["vIn"][i]["scriptPubKey"] = d["address"][1:].hex() r[t]["vIn"][i]["scriptPubKeyOpcodes"] = decode_script(r[t]["vIn"][i]["scriptPubKey"]) r[t]["vIn"][i]["scriptPubKeyAsm"] = decode_script(r[t]["vIn"][i]["scriptPubKey"], 1) if us_pointers: rows = await conn.fetch("SELECT outpoint," " input_index," " out_tx_id, " " tx_id," " address, " " amount " " " "FROM connector_unconfirmed_stxo " "WHERE tx_id = ANY($1);", us_pointers) c_rows = await conn.fetch(""" SELECT pointer, tx_id FROM transaction WHERE tx_id = ANY($1); """, us_pointers_inputs) tx_id_map_pointer = dict() for v in c_rows: tx_id_map_pointer[rh2s(v["tx_id"])] = v["pointer"] us_pointers = dict() for v in rows: us_pointers[(rh2s(v["tx_id"]), v["input_index"])] = v for t in r: if r[t]["blockHeight"] is not None: continue r[t]["inputsAmount"] = 0 for i in r[t]["vIn"]: d = us_pointers[(r[t]["txId"], i)] r[t]["vIn"][i]["type"] = SCRIPT_N_TYPES[d["address"][0]] r[t]["vIn"][i]["amount"] = d["amount"] r[t]["inputsAmount"] += d["amount"] try: pointer = tx_id_map_pointer[r[t]["vIn"][i]["txId"]] r[t]["vIn"][i]["blockHeight"] = pointer >> 39 r[t]["vIn"][i]["confirmations"] = app["last_block"] - (pointer >> 39) + 1 except: r[t]["vIn"][i]["blockHeight"] = None r[t]["vIn"][i]["confirmations"] = None if d["address"][0] in (0,1,2,5,6): script_hash = True if d["address"][0] in (1, 6) else False witness_version = None if d["address"][0] < 5 else 0 try: if d["address"][0] == 2: ad = b"\x02" + parse_script(d["address"][1:])["addressHash"] else: ad = d["address"] r[t]["vIn"][i]["address"] = hash_to_address(ad[1:], testnet=app["testnet"], script_hash = script_hash, witness_version = witness_version) r[t]["vIn"][i]["scriptPubKey"] = address_to_script(r[t]["vIn"][i]["address"], hex=1) except: print(r[t]["txId"]) print("??", d["address"].hex()) raise elif r["address"][0] == 2: r[t]["vIn"][i]["address"] = script_to_address(d["address"][1:], testnet=app["testnet"]) r[t]["vIn"][i]["scriptPubKey"] = d["address"][1:].hex() else: r[t]["vIn"][i]["scriptPubKey"] = d["address"][1:].hex() r[t]["vIn"][i]["scriptPubKeyOpcodes"] = decode_script(r[t]["vIn"][i]["scriptPubKey"]) r[t]["vIn"][i]["scriptPubKeyAsm"] = decode_script(r[t]["vIn"][i]["scriptPubKey"], 1) # get information about spent output coins rows = await conn.fetch("SELECT outpoint," " input_index," " tx_id " "FROM connector_unconfirmed_stxo " "WHERE out_tx_id = ANY($1);", tx_id_list) out_map = dict() for v in rows: i = bytes_to_int(v["outpoint"][32:]) try: out_map[(rh2s(v["outpoint"][:32]), i)].append({"txId": rh2s(v["tx_id"]), "vIn": v["input_index"]}) except: out_map[(rh2s(v["outpoint"][:32]), i)] = [{"txId": rh2s(v["tx_id"]), "vIn": v["input_index"]}] rows = await conn.fetch("SELECT stxo.pointer," " stxo.s_pointer," " transaction.tx_id " "FROM stxo " "JOIN transaction " "ON transaction.pointer = (stxo.s_pointer >> 18)<<18 " "WHERE stxo.pointer = ANY($1);", o_pointers) p_out_map = dict() for v in rows: p_out_map[v["pointer"]] = [{"txId": rh2s(v["tx_id"]), "vIn": v["s_pointer"] & 0b111111111111111111}] for t in r: if r[t]["blockHeight"] is not None: o_pointer = (r[t]["blockHeight"] << 39) + (r[t]["blockIndex"] << 20) + (1 << 19) for i in r[t]["vOut"]: try: r[t]["vOut"][i]["spent"] = p_out_map[o_pointer + i] except: try: r[t]["vOut"][i]["spent"] = out_map[(r[t]["txId"], int(i))] except: r[t]["vOut"][i]["spent"] = [] else: for i in r[t]["vOut"]: try: r[t]["vOut"][i]["spent"] = out_map[(r[t]["txId"], int(i))] except: r[t]["vOut"][i]["spent"] = [] for pointer in pointers: key = "%s:%s" % (pointer >> 39, (pointer >> 20) & 524287) try: txs[key] = r[key] except: txs[key] = None for h in hashes: h = rh2s(h) try: txs[h] = r[h] except: txs[h] = None return {"data": txs, "time": round(time.time() - q, 4)}
async def address_unconfirmed_transactions(address, type, limit, page, order, mode, app): q = time.time() if address[0] == 0 and type is None: a = [address] async with app["db_pool"].acquire() as conn: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is None: script = await conn.fetchval( "SELECT script from connector_unconfirmed_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: a.append(b"\x02" + script) else: async with app["db_pool"].acquire() as conn: if address[0] == 0: if type == 2: script = await conn.fetchval( "SELECT script from connector_unconfirmed_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is None: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: address = b"\x02" + script else: return { "data": { "confirmed": 0, "unconfirmed": 0 }, "time": round(time.time() - q, 4) } a = [address] async with app["db_pool"].acquire() as conn: count = await conn.fetchval( "SELECT count(tx_id) FROM unconfirmed_transaction_map " "WHERE address = ANY($1);", a) pages = math.ceil(count / limit) rows = await conn.fetch( "SELECT " " unconfirmed_transaction.raw_transaction," " unconfirmed_transaction.tx_id, " " unconfirmed_transaction.timestamp " "FROM unconfirmed_transaction_map " "JOIN unconfirmed_transaction " "on unconfirmed_transaction.tx_id = unconfirmed_transaction_map.tx_id " "WHERE unconfirmed_transaction_map.address = ANY($1) " "order by unconfirmed_transaction.timestamp %s " "LIMIT $2 OFFSET $3;" % order, a, limit, limit * (page - 1)) t = set() mempool_rank_map = dict() for row in rows: t.add(row["tx_id"]) if t: ranks = await conn.fetch( """SELECT ranks.rank, ranks.tx_id FROM (SELECT tx_id, rank() OVER(ORDER BY feerate DESC) as rank FROM unconfirmed_transaction) ranks WHERE tx_id = ANY($1) LIMIT $2""", t, len(t)) for r in ranks: mempool_rank_map[r["tx_id"]] = r["rank"] target_scripts = [] tx_list = [] tx_id_set = set() for row in rows: tx_id_set.add(row["tx_id"]) tx = Transaction(row["raw_transaction"], testnet=app["testnet"]) tx["timestamp"] = row["timestamp"] tx_list.append(tx) try: tx["mempoolRank"] = mempool_rank_map[row["tx_id"]] except: pass ts = dict() for d in a: if d[0] in (0, 1, 5, 6): ts[hash_to_script(d[1:], d[0], hex=True)] = 0 else: ts[d[1:].hex()] = 0 target_scripts.append(ts) async with app["db_pool"].acquire() as conn: stxo = await conn.fetch( "SELECT connector_unconfirmed_stxo.input_index," " connector_unconfirmed_stxo.tx_id," " connector_unconfirmed_stxo.amount," " connector_unconfirmed_stxo.address," " transaction.pointer " "FROM connector_unconfirmed_stxo " "LEFT OUTER JOIN transaction " "ON connector_unconfirmed_stxo.out_tx_id = transaction.tx_id " "WHERE connector_unconfirmed_stxo.tx_id = ANY($1);", tx_id_set) stxo_map = {} for row in stxo: stxo_map[(rh2s(row["tx_id"]), row["input_index"])] = (row["address"], row["amount"], row["pointer"]) for i in range(len(tx_list)): tx_list[i]["inputsAmount"] = 0 tx_list[i]["inputAddressCount"] = 0 tx_list[i]["outAddressCount"] = 0 tx_list[i]["inputsCount"] = len(tx_list[i]["vIn"]) tx_list[i]["outsCount"] = len(tx_list[i]["vOut"]) if not tx_list[i]["coinbase"]: for k in tx_list[i]["vIn"]: d = stxo_map[(tx_list[i]["txId"], k)] tx_list[i]["vIn"][k]["type"] = SCRIPT_N_TYPES[d[0][0]] tx_list[i]["vIn"][k]["amount"] = d[1] tx_list[i]["inputsAmount"] += d[1] pointer = d[2] if pointer is not None: tx_list[i]["vIn"][k]["blockHeight"] = pointer >> 39 tx_list[i]["vIn"][k]["confirmations"] = app[ "last_block"] - (pointer >> 39) + 1 else: tx_list[i]["vIn"][k]["blockHeight"] = None tx_list[i]["vIn"][k]["confirmations"] = None if d[0][0] in (0, 1, 2, 5, 6): script_hash = True if d[0][0] in (1, 6) else False witness_version = None if d[0][0] < 5 else 0 try: if d[0][0] == 2: ad = b"\x02" + parse_script( d[0][1:])["addressHash"] else: ad = d[0] tx_list[i]["vIn"][k]["address"] = hash_to_address( ad[1:], testnet=app["testnet"], script_hash=script_hash, witness_version=witness_version) tx_list[i]["vIn"][k][ "scriptPubKey"] = address_to_script( tx_list[i]["vIn"][k]["address"], hex=1) except: print(tx_list[i]["txId"]) print("??", d[0].hex()) raise tx_list[i]["inputAddressCount"] += 1 else: tx_list[i]["vIn"][k]["scriptPubKey"] = d[0][1:].hex() tx_list[i]["vIn"][k]["scriptPubKeyOpcodes"] = decode_script( tx_list[i]["vIn"][k]["scriptPubKey"]) tx_list[i]["vIn"][k]["scriptPubKeyAsm"] = decode_script( tx_list[i]["vIn"][k]["scriptPubKey"], 1) for ti in target_scripts[i]: if ti == tx_list[i]["vIn"][k]["scriptPubKey"]: target_scripts[i][ti] -= tx_list[i]["vIn"][k]["amount"] if tx_list[i]["vIn"][k]["sequence"] < 0xfffffffe: tx_list[i]["vIn"][k]["rbf"] = True if not tx_list[i]["coinbase"]: tx_list[i][ "fee"] = tx_list[i]["inputsAmount"] - tx_list[i]["amount"] else: tx_list[i]["fee"] = 0 tx_list[i]["outputsAmount"] = tx_list[i]["amount"] # get information about spent output coins async with app["db_pool"].acquire() as conn: rows = await conn.fetch( "SELECT outpoint," " input_index," " tx_id " "FROM connector_unconfirmed_stxo " "WHERE out_tx_id = ANY($1);", tx_id_set) out_map = dict() for v in rows: i = bytes_to_int(v["outpoint"][32:]) try: out_map[(rh2s(v["outpoint"][:32]), i)].append({ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }) except: out_map[(rh2s(v["outpoint"][:32]), i)] = [{ "txId": rh2s(v["tx_id"]), "vIn": v["input_index"] }] # todo get information about double spent coins # async with app["db_pool"].acquire() as conn: # rows = await conn.fetch("SELECT outpoint," # " input_index," # " tx_id " # "FROM connector_unconfirmed_stxo " # "WHERE out_tx_id = ANY($1);", tx_id_set) for t in range(len(tx_list)): for i in tx_list[t]["vOut"]: try: tx_list[t]["vOut"][i]["spent"] = out_map[(tx_list[t]["txId"], int(i))] except: tx_list[t]["vOut"][i]["spent"] = [] for ti in target_scripts[t]: if ti == tx_list[t]["vOut"][i]["scriptPubKey"]: target_scripts[t][ti] += tx_list[t]["vOut"][i]["value"] if "address" in tx_list[t]["vOut"][i]: tx_list[t]["outAddressCount"] += 1 address_amount = 0 for ti in target_scripts[t]: address_amount += target_scripts[t][ti] tx_list[t]["amount"] = address_amount if mode != "verbose": del tx_list[t]["vIn"] del tx_list[t]["vOut"] del tx_list[t]["format"] del tx_list[t]["testnet"] del tx_list[t]["rawTx"] del tx_list[t]["hash"] del tx_list[t]["blockHash"] del tx_list[t]["time"] del tx_list[t]["confirmations"] del tx_list[t]["blockIndex"] try: del tx_list[t]["flag"] except: pass return { "data": { "page": page, "limit": limit, "pages": pages, "list": tx_list }, "time": round(time.time() - q, 4) }
async def address_confirmed_utxo(address, type, from_block, order, order_by, limit, page, app): q = time.time() utxo = [] if from_block: from_block = " AND pointer >= " + str(from_block << 39) else: from_block = "" if address[0] == 0 and type is None: a = [address] async with app["db_pool"].acquire() as conn: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: a.append(b"\x02" + script) rows = await conn.fetch( "SELECT outpoint, amount, pointer, address " "FROM connector_utxo " "WHERE address = ANY($1) %s " "order by %s %s LIMIT $2 OFFSET $3;" % (from_block, order_by, order), a, limit if limit else "ALL", limit * (page - 1)) outpoints = [row["outpoint"] for row in rows] urows = await conn.fetch( "SELECT outpoint FROM connector_unconfirmed_stxo " "WHERE outpoint = ANY($1);", outpoints) outpoints = set([row["outpoint"] for row in urows]) for row in rows: if row["outpoint"] not in outpoints: utxo.append({ "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]), "block": row["pointer"] >> 39, "txIndex": (row["pointer"] - ((row["pointer"] >> 39) << 39)) >> 20, "amount": row["amount"], "type": SCRIPT_N_TYPES[row["address"][0]] }) else: async with app["db_pool"].acquire() as conn: if address[0] == 0: if type == 2: script = await conn.fetchval( "SELECT script from connector_p2pk_map " "WHERE address = $1 LIMIT 1;", address[1:]) if script is not None: address = b"\x02" + script else: return { "data": utxo, "time": round(time.time() - q, 4) } rows = await conn.fetch( "SELECT outpoint, amount, pointer " "FROM connector_utxo " "WHERE address = $1 %s " "order by %s %s LIMIT $2 OFFSET $3;" % (from_block, order_by, order), address, limit, limit * (page - 1)) outpoints = [row["outpoint"] for row in rows] urows = await conn.fetch( "SELECT outpoint FROM connector_unconfirmed_stxo " "WHERE outpoint = ANY($1);", outpoints) outpoints = set([row["outpoint"] for row in urows]) for row in rows: if row["outpoint"] not in outpoints: utxo.append({ "txId": rh2s(row["outpoint"][:32]), "vOut": bytes_to_int(row["outpoint"][32:]), "block": row["pointer"] >> 39, "txIndex": (row["pointer"] - ((row["pointer"] >> 39) << 39)) >> 20, "amount": row["amount"] }) return {"data": utxo, "time": round(time.time() - q, 4)}