def _check_uncle(self, block): """ 检查当前区块的叔块,看是否需要回滚 """ # 检查当前传入的区块高度和最新的区块高度是否一致,相差大于100说明当前是处理比较老的区块了则不用检查叔块 current_height = hex_to_int(block["number"]) newest_height = self.rpc.eth_block_number() if newest_height - current_height > 100: return False # 没有叔块也不用检查叔块 uncles = block["uncles"] if not uncles: return False G_LOGGER.info(f"区块{current_height}发现叔块:{','.join(uncles)}") # 获取叔块对应最小区块高度的hash uncle_height = [] for uncle in uncles: uncle_block = self.rpc.eth_get_block_by_hash(uncle, tx_detail=False) if uncle_block: uncle_height.append(hex_to_int(uncle_block["number"])) if uncle_height: min_height = min(uncle_height) # 查询缓存的区块中是否有uncle hash push_block_hash = self.db.redis.get_cache_block(min_height) # 已经推送的该高度对应的区块hash如果不在uncles中,说明之前推送的区块就是正确的区块,不用继续处理 if push_block_hash not in uncles: return False raise ForkError(min_height, f"{min_height}高度同步过程中出现临时分叉")
def parse_block(self, block_num): """ 解析区块的交易详情 :return: 返回待推送信息列表 """ while True: try: if isinstance(block_num, int): block_num = hex(block_num) if isinstance(block_num, str) and not block_num.startswith('0x'): block_num = hex(int(block_num)) block = self.rpc.get_block(block_num) txs = block.get("transactions", []) push_list = [] for tx in txs: tx_receipt = self.rpc.get_tx_receipt(tx.get("hash")) mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["Txid"] = tx.get("hash") mq_tx["Type"] = "TOMO" mq_tx["From"] = tx.get("from") mq_tx["To"] = tx.get("to") mq_tx["Amount"] = int(tx.get("value"), 16) mq_tx["Fee"] = int(tx["gasPrice"], 16) * int( tx_receipt["gasUsed"], 16) status = int(tx_receipt["status"], 16) mq_tx["Valid"] = True if status else False mq_tx["status"] = "true" if status else "false" mq_tx["Time"] = int(block.get("timestamp"), 16) mq_tx["BlockNumber"] = int(tx.get("blockNumber"), 16) push_list.append(mq_tx) return push_list except Exception as e: G_LOGGER.info(f"出现异常,尝试重新获取。异常原因:{str(e)}") time.sleep(1)
def parse_txs(self, txs, height=None): """ 解析区块内交易 """ push_list = [] push_cache = [] if self.rollback and height: cache_tag = self.push_cache.get(str(height)) push_cache = self.push_cache.pop(str(height)) if cache_tag else [] G_LOGGER.info('Start process block:{}, rollback_status:{}, rollback_count: {}, cache_data_len:{}, block txcounts:{}'.format(height, self.rollback, self.rollback_count, len(push_cache), len(txs))) coinbase_txs = self.parse_coinbase_tx(txs.pop(0), push_cache=push_cache) total_fees = 0 process_tx_counts = 0 for tx_ids in seperate_big_list(txs, chunk=self.request_chunk_size): tx_details = self.rpc.get_transactions(tx_ids) # G_LOGGER.info('Block:{}, current process tx counts:{}'.format(height, len(tx_details))) if not tx_details: continue process_tx_counts += len(tx_details) txs_list, fees = self.parse_normal_tx(tx_details, push_cache=push_cache) total_fees += fees push_list.extend(txs_list) # 给coinbase交易添加fee字段值 for tx in coinbase_txs: tx['Fee'] = str(total_fees) push_list.append(tx) G_LOGGER.info('Block:{}, process txcounts:{}, push txcounts: {}'.format(height, process_tx_counts + 1, len(push_list))) return push_list
def mempool_process(cls): """ 未确认交易推送进程 """ # 只有比特系币种才有内存池 coin_name = G_CFG.coin.coin_dict.get("name") btc_serial = [ "btc", "bch", "bsv", "bcx", "bcd", "btg", "dash", "doge", "god", "ipc", "ltc", "qtum", "sbtc", "zec", 'usdt' ] if coin_name not in btc_serial: return None G_LOGGER.info(f"进程{os.getpid()}启动Mempool的推送任务") cls.coin_push().loop_mempool()
def kafka_push(self, data): try: # 兼容处理eth_multi和其他币种两套数据模板 if "Type" in data: # 把空字符串、None值转化为0,避免eval处理出错 if not data["Amount"]: data["Amount"] = 0 if not data["Fee"]: data["Fee"] = 0 if not data["BlockNumber"]: data["BlockNumber"] = 0 if not data["Time"]: data["Time"] = 0 # eval可以把小数型字符串和16进制字符串转换为小数和整数类型 if isinstance(data["Amount"], str): data["Amount"] = eval(data["Amount"]) if isinstance(data["Fee"], str): data["Fee"] = eval(data["Fee"]) if isinstance(data["BlockNumber"], str): data["BlockNumber"] = eval(data["BlockNumber"]) if isinstance(data["Time"], str): data["Time"] = eval(data["Time"]) # 根据每个币种精度,转化为整型数值 if data["Type"] in ["EOS"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4) if data["Type"] in ["IOST"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8) # 去除小数位无效的0 data["Amount"] = int(data["Amount"]) data["Fee"] = int(data["Fee"]) data["BlockNumber"] = int(data["BlockNumber"]) data["Time"] = int(data["Time"]) * 1000 else: data["time"] = eval(data["time"]) * 1000 data["value"] = eval(data["value"]) data["Fee"] = eval(data["Fee"]) partition, offset = self.db.kafka.send(data) G_LOGGER.info("Process:{} kafka push success, partition={}, offset={}, push_data={}".format(os.getpid(), partition, offset, data)) except Exception as e: G_LOGGER.error("Process:{} kafka push failed, push_data={}, error={}".format(os.getpid(), data, str(e)))
def push_sync(self, height, num, rollback_count=0): block_num = height try: for block_num in range(height, height + num): self.block_num = block_num # 获取待推送数据 push_data = self.coin.push_list(block_num, rollback_count=rollback_count) G_LOGGER.info("当前推送区块高度为:{}, push_counts={}".format(self.block_num, len(push_data))) for data in push_data: self.rocket_push(data) time.sleep(0.5) except ForkError as e: raise ForkError(e.height, e.msg) except Exception as e: # error_info = traceback.format_exc() error_info = str(e) raise SyncError(block_num, f"{block_num}{str(error_info)}")
def rocket_push(self, data): try: # 兼容处理eth_multi和其他币种两套数据模板 if "Type" in data: # 把空字符串、None值转化为0,避免eval处理出错 if not data["Amount"]: data["Amount"] = 0 if not data["Fee"]: data["Fee"] = 0 if not data["BlockNumber"]: data["BlockNumber"] = 0 if not data["Time"]: data["Time"] = 0 # eval可以把小数型字符串和16进制字符串转换为小数和整数类型 if isinstance(data["Amount"], str): data["Amount"] = eval(data["Amount"]) if isinstance(data["Fee"], str): data["Fee"] = eval(data["Fee"]) if isinstance(data["BlockNumber"], str): data["BlockNumber"] = eval(data["BlockNumber"]) if isinstance(data["Time"], str): data["Time"] = eval(data["Time"]) # 根据每个币种精度,转化为整型数值 if data["Type"] in ["EOS"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4) if data["Type"] in ["IOST"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8) # 去除小数位无效的0 data["Amount"] = int(data["Amount"]) data["Fee"] = int(data["Fee"]) data["BlockNumber"] = int(data["BlockNumber"]) data["Time"] = int(data["Time"]) * 1000 else: data["time"] = eval(data["time"]) * 1000 data["value"] = eval(data["value"]) data["Fee"] = eval(data["Fee"]) msg_id = self.db.rocket.push_data(data) G_LOGGER.info("success, msg_id={}, data={}".format(msg_id, data)) except Exception as e: G_LOGGER.error("failed, data={}, error={}".format(data, str(e)))
def loop_fetch_block(self): """ 从redis取出待处理的块进行解析 """ while True: block_num = self.db.redis.get_pending_block() if len(block_num) > 0: block_num = block_num[0].decode("utf-8") diff_num = 1 G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块") try: self.push_sync(int(block_num), diff_num) except Exception as e: self.db.redis.save_pending_block(block_num) # err_info = traceback.format_exc() err_info = str(e) G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块异常, 详情::{err_info}") time.sleep(0.1)
def push_mempool_info(self): """ 推送mempool中的未确认交易信息 """ redis_mempool_list = self.db.redis.get_mempool_info() G_LOGGER.info('Mempool中未确认交易数量:{}'.format(len(redis_mempool_list))) try: old_tx_ids, new_tx_ids, push_data = self.coin.push_mempool_list(redis_mempool_list) for data in push_data: self.kafka_push(data) update_redis_mempool_list = [tx_id for tx_id in redis_mempool_list if tx_id not in old_tx_ids] update_redis_mempool_list.extend(new_tx_ids) self.db.redis.update_mempool_info(update_redis_mempool_list) # 避免频繁请求节点,每隔6秒检测mempool time.sleep(6) except Exception: msg = traceback.format_exc() G_LOGGER.info("推送MemPool信息出错, 错误: {}.".format(msg))
def _wrap(*args, **kwargs): def _loop(): func(*args, **kwargs) mode = G_CFG.coin.coin_dict["mode"] while True: if mode == "prod": try: _loop() except Exception: error_info = traceback.format_exc() G_LOGGER.error(f"出现异常, 请处理. {error_info}") except KeyboardInterrupt: G_LOGGER.info(f"{func.__name__}手动取消任务, 退出循环执行任务") elif mode == "dev": _loop() # 每秒循环一次 time.sleep(5)
def parse_block(self, block_num): """ 解析区块的交易详情 :return: 返回待推送信息列表 """ while True: try: block = self.rpc.get_block(block_num) txs = block.get("tx", []) push_list = [] for tx in txs: G_LOGGER.info("tx={}".format(tx)) mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["Txid"] = tx.get("txHash") mq_tx["Type"] = "BNB" mq_tx["From"] = tx.get("fromAddr") mq_tx["To"] = tx.get("toAddr") mq_tx["Amount"] = int( float(tx.get("value") if tx.get("value") else 0) * 100000000) mq_tx["Fee"] = int( float(tx.get("txFee") if tx.get("txFee") else 0) * 100000000) mq_tx["Valid"] = True mq_tx["Time"] = utcstr_to_timestamp( tx.get("timeStamp")) if tx.get("timeStamp") else 0 mq_tx["BlockNumber"] = tx.get("blockHeight") mq_tx["TxIndex"] = 0 mq_tx["Memo"] = tx.get("memo") if tx.get("memo") else "" G_LOGGER.info("mq_tx={}".format(mq_tx)) push_list.append(mq_tx) return push_list except Exception as e: G_LOGGER.info(f"出现异常,尝试重新获取。异常原因:{str(e)}") time.sleep(1)
def parse_block(self, block_num): """ 解析区块的交易详情 :return: 返回待推送信息列表 """ block = None while True: try: block = self.rpc.get_block(block_num) ledger = block.get("ledger", []) if block else [] txs = ledger.get("transactions", []) if ledger else [] push_list = [] for tx in txs: if tx["TransactionType"] == "Payment" and isinstance( tx['Amount'], str): # 交易类型为Payment且状态为成功 mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["Txid"] = tx.get("hash", "") mq_tx["Type"] = "XRP" mq_tx["From"] = tx.get("Account", "") mq_tx["To"] = tx.get("Destination", "") mq_tx["Fee"] = tx.get("Fee", "") mq_tx["Memo"] = tx.get("DestinationTag", "") meta_data = tx.get("metaData", {}) status = meta_data.get("TransactionResult") mq_tx["Valid"] = True if ( status and status == "tesSUCCESS") else False mq_tx["status"] = "true" if ( status and status == "tesSUCCESS") else "false" mq_tx["Amount"] = meta_data.get( "delivered_amount", "0") mq_tx["Time"] = ledger.get("close_time", 0) + self.time_diff mq_tx["BlockNumber"] = ledger.get("ledger_index", 0) push_list.append(mq_tx) return push_list except Exception as e: G_LOGGER.info( f"区块{block_num}出现异常,区块内容:{block},尝试重新获取。异常原因:{str(e)}") time.sleep(1)
def parse_block(self, block_height): """ 解析区块 """ if not isinstance(block_height, int): block_height = int(block_height) block_hash = self.rpc.get_block_hash(block_height) if not block_hash: return [] self.current_height = block_height block = self.rpc.get_block(block_hash) txs = block['tx'] if block else [] if not txs: return [] push_list = self.parse_txs(txs, height=block_height) len_of_cacahe = len(self.push_cache) G_LOGGER.info('{} push_cache_length: {}, rollback_count: {}'.format(self.coin_type, len_of_cacahe, self.rollback_count)) # 非回溯类的才会缓存 if not self.rollback: if len_of_cacahe <= (int(self.rollback_count) + 1): self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list]) else: G_LOGGER.info('{} push_cache_length: {} out of limit, reset push_cache_dict'.format(self.coin_type, len_of_cacahe)) self.push_cache = {} self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list]) G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys()))) return push_list
def parse_block(self, block_num): """ 解析区块的交易列表 :return: 返回待推送信息列表 """ block = self.rpc.get_block(block_num) txs = block.get("tx", []) if block else [] push_list = [] # 回溯缓存推送 push_cache = [] if self.rollback and block_num: cache_tag = self.push_cache.get(str(block_num)) push_cache = self.push_cache.pop(str(block_num)) if cache_tag else [] for tx in txs: try: sub_push = self.parse_tx(tx, push_cache=push_cache) # 每一个tx都加上区块时间和区块高度 for push in sub_push: push["BlockNumber"] = block["index"] push["Time"] = block["time"] push_list.extend(sub_push) except Exception as e: G_LOGGER.error(f"解析交易出现异常,详情:{e}") len_of_cacahe = len(self.push_cache) G_LOGGER.info('NEO push_cache_length: {}, rollback_count: {}'.format(len_of_cacahe, self.rollback_count)) # 非回溯类的才会缓存 if not self.rollback: if len_of_cacahe <= (int(self.rollback_count) + 1): self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list]) else: G_LOGGER.info('NEO push_cache_length: {} out of limit, reset push_cache_dict'.format(len_of_cacahe)) self.push_cache = {} self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list]) G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys()))) return push_list
def parse_tx(self, tx_ids): """ 解析交易详情 :param tx_ids: 交易ids :return: """ push_list = [] for tx_id in tx_ids: # 根据tx_id查询交易详情 tx = self.rpc.get_transaction(tx_id) # 获取operations下的第一个type字段(交易类型) tx_type = tx[1]["trx"]["operations"][0]["type"] asset_id = tx[1]["trx"]["alp_inport_asset"]["asset_id"] if tx_type == "transaction_op_type": # 合约交易 tx = self.rpc.get_pretty_contract_transaction(tx_id) elif asset_id == 0: """ 已知tx_type withdraw_op_type: ACT Transfer (1f279cf35e8ec5cc785642bf06c57b1ba33a645b) deposit_op_type: ACT Transfer define_slate_op_type: ACT Transfer (09749dbf3aba3a54a5a5ac6f1723869becaf2b32) withdraw_pay_op_type: Agent Gets Paid (d81726e747a4e7c14ef12839cfb0dbd68117757f) register_account_op_type: Account Registration (f2de809451b084e8d80758efa7949d7ec8e80036) """ # ACT交易 tx = self.rpc.get_pretty_transaction(tx_id) else: G_LOGGER.info( f"unkown asset_id:{str(id)}, tx_type:{tx_type}, hash:{tx_id}" ) continue tx["tx_type"] = tx_type mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["Type"] = "ACT" mq_tx["BlockNumber"] = tx["block_num"] mq_tx["Time"] = date_to_timestamp(tx["timestamp"]) mq_tx["TxIndex"] = 0 # 只要转账的合约交易 if tx["tx_type"] == "transaction_op_type": reserved = tx["reserved"] if reserved and tx["reserved"][0] == "transfer_to": # 区块链浏览器普遍使用该交易ID mq_tx["Txid"] = tx["orig_trx_id"] # 发起转账的地址 mq_tx["From"] = tx['to_contract_ledger_entry'][ 'from_account'] # 转账金额 mq_tx["Amount"] = tx["to_contract_ledger_entry"]["amount"][ "amount"] # 合约地址 mq_tx["Contract"] = tx["to_contract_ledger_entry"][ "to_account"] # 合约执行函数 if reserved[1].find("|") > -1: # 合约转账金额 mq_tx["Amount"] = reserved[1].split("|")[1] # 实际合约转账地址 mq_tx["To"] = reserved[1].split("|")[0] else: mq_tx["To"] = reserved[1] else: # 交易id mq_tx["Txid"] = tx["trx_id"] # 发起转账的地址 mq_tx["From"] = tx['ledger_entries'][0]['from_account'] # 接收转账的地址 mq_tx["To"] = tx["ledger_entries"][0]["to_account"] # 转账金额 mq_tx["Amount"] = tx["ledger_entries"][0]["amount"]["amount"] mq_tx["status"] = "true" push_list.append(mq_tx) return push_list
parser.add_argument("coin", help="指定要开启推送的货币名称") parser.add_argument("rpc", help="节点rpc请求地址") parser.add_argument("-m", "--mode", help="设置运行模式,dev开发环境,prod生产环境") parser.add_argument("-p", "--process", type=int, help="设置进程数量,默认1") args = parser.parse_args() # 根据传入的参数更新全局配置 G_CFG.coin.coin_dict["name"] = args.coin process = args.process if args.process else 0 G_CFG.coin.coin_dict["process"] = process G_CFG.rpc.rpc_dict["url"] = args.rpc if args.mode: G_CFG.coin.coin_dict["mode"] = args.mode G_CFG.log.log_dict["filename"] = f"log/{args.coin}.log" mode = G_CFG.coin.coin_dict["mode"] if mode not in ["prod", "dev"]: G_LOGGER.info("未知的运行模式") exit() if args.process and args.process > 1: events = [ Event.push_process, Event.mempool_process, Event.fetch_process ] else: events = [Event.push_process, Event.mempool_process] process_len = len(events) + process # 开启多进程 pool = Pool(processes=process_len) for e in events: result = pool.apply_async(e) if args.process and args.process > 1 and e == Event.fetch_process: for i in range(args.process - 1): result = pool.apply_async(e)
def parse_block(self, block_height): """ 解析区块的交易详情 :return: 返回待推送信息列表 """ accounts = self.get_xmr_accounts() block_detail = self.rpc.get_block(block_height) if not block_detail: return [] tx_ids = block_detail.get("tx_hashes", []) if not tx_ids: return [] tx_details = self.rpc.get_transactions(tx_ids) push_list = [] for tx_detail in tx_details: tx_as_json = tx_detail.get("as_json") block_timestamp = tx_detail.get("block_timestamp") tx_global_indexes = tx_detail['output_indices'] txid = tx_detail.get("tx_hash") if not txid: continue tx_json = json.loads(tx_as_json) extra_list = tx_json['extra'] extra = ''.join('{:02x}'.format(x) for x in extra_list) extra_mem = ctypes.c_char_p() extra_mem.value = extra.encode() tx_pub_key_mem = ctypes.create_string_buffer(100) tx_pub_key_result = self.monero_lib.monero_from_extra_get_tx_pub_key( extra_mem, tx_pub_key_mem) if not tx_pub_key_result: continue # G_LOGGER.info(f"height:{block_height} txid:{txid}") tx_vin = tx_json.get("vin", []) v_in_keys = [i['key'] for i in tx_vin] key_images = [i.get("k_image") for i in v_in_keys] tx_vout = tx_json.get("vout", []) rct_signatures = tx_json.get("rct_signatures") if not rct_signatures: continue ecdhInfos = rct_signatures.get("ecdhInfo") if not ecdhInfos: continue outPks = rct_signatures.get("outPk") if not outPks: continue tx_fee = rct_signatures.get("txnFee", '0') mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["Txid"] = txid mq_tx["Type"] = "XMR" mq_tx["From"] = "" mq_tx["To"] = "" mq_tx["Amount"] = "" mq_tx["Memo"] = "" mq_tx["Fee"] = str(tx_fee) mq_tx["Valid"] = True mq_tx["status"] = "true" mq_tx["Time"] = block_timestamp mq_tx["BlockNumber"] = block_height mq_tx["VoutsIndex"] = 0 v_in_address = [] for key in key_images: if not key: continue redis_key = key[:6] + key[18:24] + key[-6:] addr_info = self.redis.get_and_delete(redis_key) if not addr_info: continue addr_info = json.loads(addr_info.decode()) G_LOGGER.info( f"height:{block_height}, txid:{txid}, key_image: {key}, addr_info: {addr_info}" ) v_in_address.append(addr_info) v_out_infos = [] for v_out_index, vout in enumerate(tx_vout): rct = outPks[v_out_index] global_index = tx_global_indexes[v_out_index] target = vout.get("target") if not target: continue v_out_address = target.get("key") if not v_out_address: continue random_utxo = { "global_index": str(global_index), "public_key": v_out_address, "rct": rct } self.redis.push_utxo_cache(self.random_utxo_redis_key, random_utxo) for account in accounts: address = account.get("addr") private_view_key = account.get("pr_vk") public_spend_key = account.get("pb_sk") private_view_key_mem = ctypes.c_char_p() private_view_key_mem.value = private_view_key.encode() public_spend_key_mem = ctypes.c_char_p() public_spend_key_mem.value = public_spend_key.encode() derivation_mem = ctypes.create_string_buffer(256) one_time_address_mem = ctypes.create_string_buffer(128) amount_echo_init = ecdhInfos[v_out_index] if not amount_echo_init: continue amount_echo = amount_echo_init.get("amount") if not amount_echo: continue if address and str(address).startswith("8"): additional_tx_pub_key_mem = ctypes.create_string_buffer( 100) additional_tx_pub_key_result = self.monero_lib.monero_from_extra_get_additional_tx_pub_key( extra_mem, 0, v_out_index, additional_tx_pub_key_mem) if not additional_tx_pub_key_result: der_result = self.monero_lib.monero_generate_derivation_byte32( tx_pub_key_mem, private_view_key_mem, derivation_mem) else: der_result = self.monero_lib.monero_generate_derivation_byte32( additional_tx_pub_key_mem, private_view_key_mem, derivation_mem) else: der_result = self.monero_lib.monero_generate_derivation_byte32( tx_pub_key_mem, private_view_key_mem, derivation_mem) if not der_result: continue address_result = self.monero_lib.monero_generate_one_time_public_key( derivation_mem, public_spend_key_mem, v_out_index, one_time_address_mem) if not address_result: continue one_time_addr = (one_time_address_mem.value).decode() # 判断是否命中 if v_out_address.strip() != one_time_addr.strip(): # G_LOGGER.info(f"Addr:{address}, height:{block_height}, txid:{txid}, out:{v_out_index} No") continue G_LOGGER.info( f"Addr:{address}, height:{block_height}, txid:{txid}, out:{v_out_index} Yes" ) amount_echo_mem = ctypes.c_char_p() amount_echo_mem.value = amount_echo.encode() real_amount_mem = ctypes.create_string_buffer(128) # payment_id_mem = ctypes.create_string_buffer(32) amount_result = self.monero_lib.monero_decrypt_amount( derivation_mem, v_out_index, amount_echo_mem, 0, real_amount_mem) amount = '0' if amount_result: amount = (real_amount_mem.value).decode() amount = int("0x" + amount, 0) mq_tx["Amount"] = amount # 暂时不推送memo信息,子地址不需要memo # pyamentid_result = self.monero_lib.monero_from_extra_get_payment_id(extra_mem, derivation_mem, payment_id_mem) # payment_id = '0' # if pyamentid_result: # payment_id = (payment_id_mem.value).decode() # tmp_tx["Memo"] = payment_id tmp_tx = mq_tx.copy() tmp_tx["Memo"] = "" tmp_tx["VoutsIndex"] = v_out_index tmp_tx["To"] = address v_out_infos.append(tmp_tx) break # if (not v_in_address) or (not v_out_infos): # continue v_in_length = len(v_in_address) v_out_length = len(v_out_infos) G_LOGGER.info( f"height:{block_height}, txid:{txid}, v_in_length:{v_in_length}, v_out_length:{v_out_length}" ) G_LOGGER.info(f"v_in_address: {v_in_address}") G_LOGGER.info(f"v_out_infos: {v_out_infos}") v_out_sort_infos = sorted(v_out_infos, key=lambda x: x["VoutsIndex"]) max_len = max(v_out_length, v_in_length) for num in range(max_len): v_in_idx = num if num < v_in_length else -1 v_out_idx = num if num < v_out_length else -1 tx_tp = v_out_sort_infos[ v_out_idx] if v_out_length - 1 >= v_out_idx else v_out_sort_infos[ -1] tmp = tx_tp.copy() tmp['From'] = v_in_address[v_in_idx] if v_in_address else '' push_list.append(tmp) return push_list
def parse_block(self, block_num): """ 解析区块的交易详情 :return: 返回待推送信息列表 """ t1 = time.time() push_list = [] node, block = self.http.get_block(block_num) t2 = time.time() if isinstance(block, str): # 出现大块,比如110588067,api.eossweden.org超级节点偶发性返回部分数据,不能被json反序列化,需要多次重试才能获得结果,造成消息积压 # eos.greymass.com超级节点可以正确拿到数据,但是访问频率受限,所以异常以后要重试获取数据 G_LOGGER.info( "eos_pass, block_num={}, node={}, lenght={}, head={}, tail={}". format(block_num, node, len(block), block[:38], block[-30:])) # self.redis.client.rpush("eos_pass_block", block_num) # return push_list timestamp = str(date_to_timestamp(block['timestamp'].split(".")[0])) txs = block.get("transactions", []) for tx in txs: tx_status = tx.get("status") if (not tx_status) or (tx_status not in ("executed")): continue trx = tx.get("trx", "") if not isinstance(trx, dict): continue tx_id = trx['id'] transaction = trx["transaction"] acts = transaction.get('actions', []) act_index = 0 get_amount_and_symbol = self.get_amount_and_symbol for act in acts: account, name, Contract = act.get('account'), act['name'], "" # if name in self.SUPPORTED and self.action_account.get(name) == account: if name in self.SUPPORTED: data = act['data'] if not isinstance(data, dict): continue mq_tx = G_PUSH_TEMPLATE.copy() if name == "delegatebw" and self.action_account.get( name) != account: from_addr = account to_addr = data.get("to") amount, symbol = get_amount_and_symbol( "other_delegatebw", data) elif name == "undelegatebw" and self.action_account.get( name) != account: from_addr = data.get("from") to_addr = account amount, symbol = get_amount_and_symbol( "other_undelegatebw", data) else: from_addr = data.get(self.action_convert[name]['from']) to_addr = data.get(self.action_convert[name]['to']) if not from_addr or not to_addr: continue amount, symbol = get_amount_and_symbol(name, data) # 无金额或者金额小于0.0001, 略过 if (not amount) or (float(amount) <= 0.0001): continue if account and str( account) == "eosio.token" and symbol in ["EOS"]: Contract = "eosio.token" else: Contract = str(account) + "|" + str(symbol).upper() memo_info = data.get("memo", "") mq_tx["BlockNumber"] = block_num mq_tx["Time"] = timestamp mq_tx["Txid"] = tx_id mq_tx["Type"] = "EOS" mq_tx['From'] = from_addr mq_tx['To'] = to_addr mq_tx['Amount'] = amount mq_tx["Action"] = name mq_tx['Contract'] = Contract mq_tx["VoutsIndex"] = act_index mq_tx["Memo"] = memo_info mq_tx["Valid"] = True mq_tx["status"] = "true" push_list.append(mq_tx) act_index += 1 t3 = time.time() http_time = int((t2 - t1) * 1000) / 1000 parse_time = int((t3 - t2) * 1000) / 1000 G_LOGGER.info( "eos_parse_block, block_num={}, length={}, node={}, 网络耗时={}, 处理耗时={}" .format(block_num, len(push_list), node, http_time, parse_time)) return push_list
def fetch_process(cls): process = G_CFG.coin.coin_dict.get("process") if int(process) <= 1: return None G_LOGGER.info(f"进程{os.getpid()}已开启高并发推送模式") cls.coin_push().loop_fetch_block()
def parse_block(self, block_num): """ 解析区块的交易列表(支持多发送) :return: 返回待推送信息列表 """ while True: try: block = self.rpc.eth_get_block_by_number(block_num) if block is None: raise Exception(f"获取到最新高度{block_num}区块详情为None,触发异常,尝试重新获取") G_LOGGER.info( f"当前推送的区块:高度{int(block.get('number', '0x0'), 16)},哈希{block.get('hash', '')}" ) self._check_uncle(block) # 遍历交易 push_list = [] if block.get("transactions") is None: time.sleep(3) continue tx_hashes = [tx["hash"] for tx in block["transactions"]] receipts = self.rpc.eth_get_transaction_receipt( tx_hashes) if tx_hashes else [] assert len(receipts) == len(block["transactions"]) for i, tx in enumerate(block["transactions"]): _input = tx.get('input', '') _to = tx.get('to', '') _from = tx.get('from', '') multi_send_token = self.is_multi_send_token(_input) mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["BlockNumber"] = hex_to_int(tx["blockNumber"]) mq_tx["Txid"] = tx["hash"] mq_tx["Type"] = self.coin_type mq_tx["From"] = tx["from"] mq_tx["Time"] = block["timestamp"] is_token_transfer = self._is_token(_input) # 交易状态 mq_tx["Valid"] = self.get_status(receipts[i], is_token_transfer) mq_tx["status"] = self.get_tx_status( receipts[i], is_token_transfer) # 手续费 gas_price = tx.get("gasPrice", "0x0") gas_used = receipts[i].get("gasUsed", "0x0") if receipts[i] else "0x0" mq_tx["Fee"] = hex(int(gas_price, 16) * int(gas_used, 16)) if is_token_transfer: if _from != _to: tx['contract'] = _to _to = self._get_token_to_address(_input) _value = self._get_token_to_value(_input) tx["to"] = _to if _to != "0x" else f"0x{'0'*40}" tx["value"] = _value if _value != "0x" else "0x0" elif multi_send_token: continue _len = self.multi_abi_len _del_0x_input = del_0x(_input) if multi_send_token == 'create': # 多发送合约创建的交易,保存到数据库存起来 data = dict() data['contract'] = tx['contract'] = tx['creates'] data['blockNumber'] = hex_to_int(tx["blockNumber"]) data['timestamp'] = block["timestamp"] data['hash'] = tx["hash"] # 保存合约 self.db.mysql.contract.insert_contract(data) G_LOGGER.info(f"发现新的多发送合约{data['contract']}并保存") continue else: token = '' address_list = amount_list = [] # 不是我们自己的多发送合约则不处理 contract = self.db.mysql.contract.get_by_contract( tx["to"]) if not contract: continue if multi_send_token == 'multisend': _multisend_abi = del_0x( self.multi_abi_deal['multisend']) result = self.parse_multi_data( _del_0x_input[len(_multisend_abi):]) address_count = hex_to_int(result[2]) address_list = result[3:3 + address_count] amount_count = hex_to_int( result[3 + address_count]) amount_list = result[4 + address_count:4 + address_count + amount_count] elif multi_send_token == 'multisendToken': _multisend_token_abi = del_0x( self.multi_abi_deal['multisendToken']) token = add_0x( self.parse_multi_data_pos( _del_0x_input, _multisend_token_abi, 0)) result = self.parse_multi_data( _del_0x_input[len(_multisend_token_abi):]) address_count = hex_to_int(result[3]) address_list = result[4:4 + address_count] amount_count = hex_to_int( result[4 + address_count]) amount_list = result[5 + address_count:5 + address_count + amount_count] for k, v in enumerate(address_list): mq_tx = mq_tx.copy() mq_tx["To"] = v mq_tx["Amount"] = int_to_hex( hex_to_int(amount_list[k])) mq_tx["Contract"] = token push_list.append(mq_tx) return push_list if tx["to"]: mq_tx["To"] = tx["to"] mq_tx["Amount"] = tx["value"] if 'contract' in tx.keys(): mq_tx["Contract"] = tx["contract"] push_list.append(mq_tx) return push_list except ForkError as ex: raise ForkError(ex.height, ex.msg) except Exception as ex: traceback.print_exc() G_LOGGER.info(f"获取块出现异常,尝试重新获取。异常原因:{str(ex)}") time.sleep(10)
def _send_data(self, method, url, _params=None, _data=None, _json=None, _cookies=None): data = _params or _data or _json G_LOGGER.debug('接口: {}, 方法: {}, 请求数据: {}'.format(url, method, data)) t1 = time.time() retry_time = 0.001 while True: try: with self.session.request(method, url, params=_params, data=_data, json=_json, auth=self.auth, headers=self.headers, timeout=self.timeout, cookies=_cookies) as rsp: G_LOGGER.debug('返回数据: {}'.format(rsp.text)) if 300 > rsp.status_code >= 200: result = rsp.text if self.is_json: try: t2 = time.time() net_time = int((t2 - t1) * 1000) / 1000 res = json.loads(rsp.text) t3 = time.time() json_time = int((t3 - t2) * 1000) / 1000 all_time = int((t3 - t1) * 1000) / 1000 # G_LOGGER.info("json success, {}, {}, length={}, net_time={}, json_time={}, all_time={}, ".format(url, data, len(rsp.text), net_time, json_time, all_time)) return res except Exception: # G_LOGGER.info("json fail, {}, {}, length={}, net_time={}".format(url, data, len(rsp.text), net_time)) pass return result elif rsp.status_code == 404: G_LOGGER.warn("[404] URL不存在, 请检查URL".format(self.auth)) raise UrlError("[404] URL不存在, 请检查URL") elif rsp.status_code == 401: G_LOGGER.warn("[401] auth验证错误. auth: {}".format(self.auth)) raise RequestError(rsp.status_code, "[401] auth验证错误. auth: {}".format(self.auth)) elif rsp.status_code == 403: G_LOGGER.warn("[403] 没有权限操作此URL. url: {}".format(url)) raise RequestError(rsp.status_code, "[403] 没有权限操作此URL. url: {}".format(url)) elif 500 > rsp.status_code >= 400: G_LOGGER.warn("[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text)) raise RequestError(rsp.status_code, "[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text), rsp) elif 599 > rsp.status_code >= 500: G_LOGGER.warn("[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text)) raise RequestError(rsp.status_code, "[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text), rsp) except (UrlError, RequestError): raise except (request_error.Timeout, request_error.ConnectionError) as e: if retry_time < 10: retry_time = min(max(retry_time, retry_time * 2), 10) G_LOGGER.info("{}超时或被拒绝, retry_time={}, {} {}, 错误:{}".format(url, retry_time, method, data, str(e))) continue except request_error.InvalidURL: G_LOGGER.error('请求URL无效, 请检查: {}'.format(url)) raise UrlError('URL无效.') except Exception as e: G_LOGGER.error('请求出现不可预知异常, 请处理. \r\n' '请求url: {}\r\n' '请求方法: {}\r\n' '请求参数: params: {}, data: {}, json: {}\r\n' '验证用户: {}\r\n' '错误详情: {}'.format(url, method, _params, _data, _json, self.auth, traceback.format_exc())) raise RequestError(0, "请求出现不可预知异常, 请处理. 详情简要: {}".format(e))
G_CFG.mq.mq_dict[ "routing_key"] = args.routing_key if args.routing_key else args.coin if args.exchange_name: G_CFG.mq.mq_dict["exchange_name"] = args.exchange_name if args.vhost: G_CFG.mq.mq_dict["vhost"] = args.vhost if args.mq_host: G_CFG.mq.mq_dict["host"] = args.mq_host if args.mq_user: G_CFG.mq.mq_dict["username"] = args.mq_user if args.mq_password: G_CFG.mq.mq_dict["password"] = args.mq_password if args.mode: G_CFG.coin.coin_dict["mode"] = args.mode G_CFG.mysql.mysql_dict["db"] = args.coin G_CFG.log.log_dict["filename"] = f"log/{args.coin}.log" G_CFG.message.message_dict["monitor_path"] = f"log/{args.coin}.txt" mode = G_CFG.coin.coin_dict["mode"] if mode not in ["prod", "dev"]: G_LOGGER.info("未知的运行模式") exit() # 手动推送区块数据 coin_push = Event.coin_push() coin_name = args.coin block_num = args.block_num count = args.count print('手动推送区块数据,coin: {}, start_block: {}, end_blcok: {}'.format( coin_name, block_num, block_num + count - 1)) coin_push.push_sync(block_num, count)
def parse_block(self, block_num): """ 解析区块的交易列表 """ try: block = self.rpc.eth_get_block_by_number(block_num) if block is None: raise Exception(f"获取区块高度{block_num}区块详情为None,触发异常,尝试重新获取") G_LOGGER.info( f"当前推送的区块:高度{int(block.get('number', '0x0'), 16)},哈希{block.get('hash', '')}" ) self._check_uncle(block) # 遍历交易 push_list = [] if block.get("transactions") is None: raise Exception( f"获取区块高度{block_num}区块transactions为None,触发异常,尝试重新获取") tx_hashes = [tx["hash"] for tx in block["transactions"]] receipts = self.rpc.eth_get_transaction_receipt( tx_hashes) if tx_hashes else [] assert len(receipts) == len(block["transactions"]) for i, tx in enumerate(block["transactions"]): _input = tx.get('input', '') _to = tx.get('to', '') _from = tx.get('from', '') is_token_transfer = self._is_token(_input) if is_token_transfer: if _from != _to: tx['contract'] = _to _to = self._get_token_to_address(_input) _value = self._get_token_to_value(_input) tx["to"] = _to if _to != "0x" else f"0x{'0'*40}" tx["value"] = _value if _value != "0x" else "0x0" mq_tx = G_PUSH_TEMPLATE.copy() mq_tx["BlockNumber"] = int(tx["blockNumber"], 16) mq_tx["Txid"] = tx["hash"] mq_tx["Type"] = self.coin_type mq_tx["From"] = tx["from"] # 排除to为空的 if not tx["to"]: continue mq_tx["To"] = tx["to"] mq_tx["Time"] = block["timestamp"] mq_tx["Amount"] = tx["value"] # 手续费 gas_price = tx.get("gasPrice", "0x0") gas_used = receipts[i].get("gasUsed", "0x0") if receipts[i] else "0x0" mq_tx["Fee"] = hex(int(gas_price, 16) * int(gas_used, 16)) # 交易状态 mq_tx["Valid"] = self.get_status(receipts[i], is_token_transfer) mq_tx["status"] = self.get_tx_status(receipts[i], is_token_transfer) if 'contract' in tx.keys(): mq_tx["Contract"] = tx["contract"] push_list.append(mq_tx) # 缓存已推送的区块高度和hash self.db.redis.save_cache_block(hex_to_int(block["number"]), block["hash"]) return push_list except ForkError as ex: raise ForkError(ex.height, ex.msg) except Exception as e: traceback.print_exc() G_LOGGER.info(f"获取块出现异常,尝试重新获取。异常原因:{str(e)}")
def push_process(cls): """ 交易推送进程 """ G_LOGGER.info(f"进程{os.getpid()}启动TX的推送任务") cls.coin_push().loop_push()
def push_main(self, save_redis=False): """ 推送处理 """ if not self.coin_name: raise Exception("unknow coin name") basic_info = self.db.get_basic(self.coin_name) if not basic_info: G_LOGGER.info("币种{}没有相关配置,请检查数据库配置".format(self.coin_name)) return True save_num = current_height = basic_info["current_height"] if save_redis: newest_block_height = self.coin.newest_height() diff_num = newest_block_height - current_height else: newest_block_height = self.coin.newest_height() diff_num = newest_block_height - current_height if diff_num <= 0: G_LOGGER.info("最新高度{},已同步高度{},无需推送".format(newest_block_height, current_height)) return True try: if save_redis: step = 100 # 最大处理100个块存入redis diff_num = step if diff_num > step else diff_num G_LOGGER.info("当前最新高度为{}, 数据库中的高度{}, 本次需要放入redis的块数为{}".format(newest_block_height, current_height, diff_num)) # 待推送的区块高度存入到redis队列中 for height in range(current_height + 1, current_height + diff_num + 1): self.block_num = height G_LOGGER.info("redis_save_pending_block: {}".format(self.block_num)) self.db.redis.save_pending_block(self.block_num) else: G_LOGGER.info("最新高度{},已同步高度{},本次需要同步块数{}".format(newest_block_height, current_height, diff_num)) self.push_sync(current_height + 1, diff_num, rollback_count=self.rollback_count) except ForkError as e: G_LOGGER.error("同步过程出现临时分叉, 即将回滚至高度{}重新推送".format(e.height)) save_num = e.height except SyncError as e: err_info = traceback.format_exc() G_LOGGER.error("{}块同步过程出现异常, 请处理此块. 详情: {}".format(e.height, str(err_info))) except Exception as e: G_LOGGER.error("同步过程出现异常, 请处理此块. 详情{}".format(e)) except KeyboardInterrupt: G_LOGGER.info("同步过程手动取消任务, 同步至高度: {}. 已保存".format(self.block_num)) save_num = self.block_num else: save_num = self.block_num finally: basic_info['current_height'] = save_num basic_info['newest_height'] = newest_block_height self.db.update_basic(basic_info) G_LOGGER.info("push success ,已同步保存至区块高度: {}".format(save_num)) return True