def _check_uncle(self, block):
"""
检查当前区块的叔块，看是否需要回滚
"""
# 检查当前传入的区块高度和最新的区块高度是否一致，相差大于100说明当前是处理比较老的区块了则不用检查叔块
current_height = hex_to_int(block["number"])
newest_height = self.rpc.eth_block_number()
if newest_height - current_height > 100:
return False
# 没有叔块也不用检查叔块
uncles = block["uncles"]
if not uncles:
return False
G_LOGGER.info(f"区块{current_height}发现叔块:{','.join(uncles)}")
# 获取叔块对应最小区块高度的hash
uncle_height = []
for uncle in uncles:
uncle_block = self.rpc.eth_get_block_by_hash(uncle,
tx_detail=False)
if uncle_block:
uncle_height.append(hex_to_int(uncle_block["number"]))
if uncle_height:
min_height = min(uncle_height)
# 查询缓存的区块中是否有uncle hash
push_block_hash = self.db.redis.get_cache_block(min_height)
# 已经推送的该高度对应的区块hash如果不在uncles中，说明之前推送的区块就是正确的区块，不用继续处理
if push_block_hash not in uncles:
return False
raise ForkError(min_height, f"{min_height}高度同步过程中出现临时分叉")
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
while True:
try:
if isinstance(block_num, int):
block_num = hex(block_num)
if isinstance(block_num,
str) and not block_num.startswith('0x'):
block_num = hex(int(block_num))
block = self.rpc.get_block(block_num)
txs = block.get("transactions", [])
push_list = []
for tx in txs:
tx_receipt = self.rpc.get_tx_receipt(tx.get("hash"))
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("hash")
mq_tx["Type"] = "TOMO"
mq_tx["From"] = tx.get("from")
mq_tx["To"] = tx.get("to")
mq_tx["Amount"] = int(tx.get("value"), 16)
mq_tx["Fee"] = int(tx["gasPrice"], 16) * int(
tx_receipt["gasUsed"], 16)
status = int(tx_receipt["status"], 16)
mq_tx["Valid"] = True if status else False
mq_tx["status"] = "true" if status else "false"
mq_tx["Time"] = int(block.get("timestamp"), 16)
mq_tx["BlockNumber"] = int(tx.get("blockNumber"), 16)
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(f"出现异常，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def parse_txs(self, txs, height=None):
"""
解析区块内交易
"""
push_list = []
push_cache = []
if self.rollback and height:
cache_tag = self.push_cache.get(str(height))
push_cache = self.push_cache.pop(str(height)) if cache_tag else []
G_LOGGER.info('Start process block:{}, rollback_status:{}, rollback_count: {}, cache_data_len:{}, block txcounts:{}'.format(height, self.rollback, self.rollback_count, len(push_cache), len(txs)))
coinbase_txs = self.parse_coinbase_tx(txs.pop(0), push_cache=push_cache)
total_fees = 0
process_tx_counts = 0
for tx_ids in seperate_big_list(txs, chunk=self.request_chunk_size):
tx_details = self.rpc.get_transactions(tx_ids)
# G_LOGGER.info('Block:{}, current process tx counts:{}'.format(height, len(tx_details)))
if not tx_details:
continue
process_tx_counts += len(tx_details)
txs_list, fees = self.parse_normal_tx(tx_details, push_cache=push_cache)
total_fees += fees
push_list.extend(txs_list)
# 给coinbase交易添加fee字段值
for tx in coinbase_txs:
tx['Fee'] = str(total_fees)
push_list.append(tx)
G_LOGGER.info('Block:{}, process txcounts:{}, push txcounts: {}'.format(height, process_tx_counts + 1, len(push_list)))
return push_list
def mempool_process(cls):
"""
未确认交易推送进程
"""
# 只有比特系币种才有内存池
coin_name = G_CFG.coin.coin_dict.get("name")
btc_serial = [
"btc", "bch", "bsv", "bcx", "bcd", "btg", "dash", "doge", "god",
"ipc", "ltc", "qtum", "sbtc", "zec", 'usdt'
]
if coin_name not in btc_serial:
return None
G_LOGGER.info(f"进程{os.getpid()}启动Mempool的推送任务")
cls.coin_push().loop_mempool()
def kafka_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
partition, offset = self.db.kafka.send(data)
G_LOGGER.info("Process:{} kafka push success, partition={}, offset={}, push_data={}".format(os.getpid(), partition, offset, data))
except Exception as e:
G_LOGGER.error("Process:{} kafka push failed, push_data={}, error={}".format(os.getpid(), data, str(e)))
def push_sync(self, height, num, rollback_count=0):
block_num = height
try:
for block_num in range(height, height + num):
self.block_num = block_num
# 获取待推送数据
push_data = self.coin.push_list(block_num, rollback_count=rollback_count)
G_LOGGER.info("当前推送区块高度为:{}, push_counts={}".format(self.block_num, len(push_data)))
for data in push_data:
self.rocket_push(data)
time.sleep(0.5)
except ForkError as e:
raise ForkError(e.height, e.msg)
except Exception as e:
# error_info = traceback.format_exc()
error_info = str(e)
raise SyncError(block_num, f"{block_num}{str(error_info)}")
def rocket_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
msg_id = self.db.rocket.push_data(data)
G_LOGGER.info("success, msg_id={}, data={}".format(msg_id, data))
except Exception as e:
G_LOGGER.error("failed, data={}, error={}".format(data, str(e)))
def loop_fetch_block(self):
"""
从redis取出待处理的块进行解析
"""
while True:
block_num = self.db.redis.get_pending_block()
if len(block_num) > 0:
block_num = block_num[0].decode("utf-8")
diff_num = 1
G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块")
try:
self.push_sync(int(block_num), diff_num)
except Exception as e:
self.db.redis.save_pending_block(block_num)
# err_info = traceback.format_exc()
err_info = str(e)
G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块异常, 详情::{err_info}")
time.sleep(0.1)
def push_mempool_info(self):
"""
推送mempool中的未确认交易信息
"""
redis_mempool_list = self.db.redis.get_mempool_info()
G_LOGGER.info('Mempool中未确认交易数量:{}'.format(len(redis_mempool_list)))
try:
old_tx_ids, new_tx_ids, push_data = self.coin.push_mempool_list(redis_mempool_list)
for data in push_data:
self.kafka_push(data)
update_redis_mempool_list = [tx_id for tx_id in redis_mempool_list if tx_id not in old_tx_ids]
update_redis_mempool_list.extend(new_tx_ids)
self.db.redis.update_mempool_info(update_redis_mempool_list)
# 避免频繁请求节点,每隔6秒检测mempool
time.sleep(6)
except Exception:
msg = traceback.format_exc()
G_LOGGER.info("推送MemPool信息出错, 错误: {}.".format(msg))
def _wrap(*args, **kwargs):
def _loop():
func(*args, **kwargs)
mode = G_CFG.coin.coin_dict["mode"]
while True:
if mode == "prod":
try:
_loop()
except Exception:
error_info = traceback.format_exc()
G_LOGGER.error(f"出现异常, 请处理. {error_info}")
except KeyboardInterrupt:
G_LOGGER.info(f"{func.__name__}手动取消任务, 退出循环执行任务")
elif mode == "dev":
_loop()
# 每秒循环一次
time.sleep(5)
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
while True:
try:
block = self.rpc.get_block(block_num)
txs = block.get("tx", [])
push_list = []
for tx in txs:
G_LOGGER.info("tx={}".format(tx))
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("txHash")
mq_tx["Type"] = "BNB"
mq_tx["From"] = tx.get("fromAddr")
mq_tx["To"] = tx.get("toAddr")
mq_tx["Amount"] = int(
float(tx.get("value") if tx.get("value") else 0) *
100000000)
mq_tx["Fee"] = int(
float(tx.get("txFee") if tx.get("txFee") else 0) *
100000000)
mq_tx["Valid"] = True
mq_tx["Time"] = utcstr_to_timestamp(
tx.get("timeStamp")) if tx.get("timeStamp") else 0
mq_tx["BlockNumber"] = tx.get("blockHeight")
mq_tx["TxIndex"] = 0
mq_tx["Memo"] = tx.get("memo") if tx.get("memo") else ""
G_LOGGER.info("mq_tx={}".format(mq_tx))
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(f"出现异常，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
block = None
while True:
try:
block = self.rpc.get_block(block_num)
ledger = block.get("ledger", []) if block else []
txs = ledger.get("transactions", []) if ledger else []
push_list = []
for tx in txs:
if tx["TransactionType"] == "Payment" and isinstance(
tx['Amount'], str):
# 交易类型为Payment且状态为成功
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("hash", "")
mq_tx["Type"] = "XRP"
mq_tx["From"] = tx.get("Account", "")
mq_tx["To"] = tx.get("Destination", "")
mq_tx["Fee"] = tx.get("Fee", "")
mq_tx["Memo"] = tx.get("DestinationTag", "")
meta_data = tx.get("metaData", {})
status = meta_data.get("TransactionResult")
mq_tx["Valid"] = True if (
status and status == "tesSUCCESS") else False
mq_tx["status"] = "true" if (
status and status == "tesSUCCESS") else "false"
mq_tx["Amount"] = meta_data.get(
"delivered_amount", "0")
mq_tx["Time"] = ledger.get("close_time",
0) + self.time_diff
mq_tx["BlockNumber"] = ledger.get("ledger_index", 0)
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(
f"区块{block_num}出现异常，区块内容:{block}，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def parse_block(self, block_height):
"""
解析区块
"""
if not isinstance(block_height, int):
block_height = int(block_height)
block_hash = self.rpc.get_block_hash(block_height)
if not block_hash:
return []
self.current_height = block_height
block = self.rpc.get_block(block_hash)
txs = block['tx'] if block else []
if not txs:
return []
push_list = self.parse_txs(txs, height=block_height)
len_of_cacahe = len(self.push_cache)
G_LOGGER.info('{} push_cache_length: {}, rollback_count: {}'.format(self.coin_type, len_of_cacahe, self.rollback_count))
# 非回溯类的才会缓存
if not self.rollback:
if len_of_cacahe <= (int(self.rollback_count) + 1):
self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list])
else:
G_LOGGER.info('{} push_cache_length: {} out of limit, reset push_cache_dict'.format(self.coin_type, len_of_cacahe))
self.push_cache = {}
self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list])
G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys())))
return push_list
def parse_block(self, block_num):
"""
解析区块的交易列表
:return: 返回待推送信息列表
"""
block = self.rpc.get_block(block_num)
txs = block.get("tx", []) if block else []
push_list = []
# 回溯缓存推送
push_cache = []
if self.rollback and block_num:
cache_tag = self.push_cache.get(str(block_num))
push_cache = self.push_cache.pop(str(block_num)) if cache_tag else []
for tx in txs:
try:
sub_push = self.parse_tx(tx, push_cache=push_cache)
# 每一个tx都加上区块时间和区块高度
for push in sub_push:
push["BlockNumber"] = block["index"]
push["Time"] = block["time"]
push_list.extend(sub_push)
except Exception as e:
G_LOGGER.error(f"解析交易出现异常，详情：{e}")
len_of_cacahe = len(self.push_cache)
G_LOGGER.info('NEO push_cache_length: {}, rollback_count: {}'.format(len_of_cacahe, self.rollback_count))
# 非回溯类的才会缓存
if not self.rollback:
if len_of_cacahe <= (int(self.rollback_count) + 1):
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
else:
G_LOGGER.info('NEO push_cache_length: {} out of limit, reset push_cache_dict'.format(len_of_cacahe))
self.push_cache = {}
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys())))
return push_list
def parse_tx(self, tx_ids):
"""
解析交易详情
:param tx_ids: 交易ids
:return:
"""
push_list = []
for tx_id in tx_ids:
# 根据tx_id查询交易详情
tx = self.rpc.get_transaction(tx_id)
# 获取operations下的第一个type字段（交易类型）
tx_type = tx[1]["trx"]["operations"][0]["type"]
asset_id = tx[1]["trx"]["alp_inport_asset"]["asset_id"]
if tx_type == "transaction_op_type":
# 合约交易
tx = self.rpc.get_pretty_contract_transaction(tx_id)
elif asset_id == 0:
"""
已知tx_type
withdraw_op_type: ACT Transfer (1f279cf35e8ec5cc785642bf06c57b1ba33a645b)
deposit_op_type: ACT Transfer
define_slate_op_type: ACT Transfer (09749dbf3aba3a54a5a5ac6f1723869becaf2b32)
withdraw_pay_op_type: Agent Gets Paid (d81726e747a4e7c14ef12839cfb0dbd68117757f)
register_account_op_type: Account Registration (f2de809451b084e8d80758efa7949d7ec8e80036)
"""
# ACT交易
tx = self.rpc.get_pretty_transaction(tx_id)
else:
G_LOGGER.info(
f"unkown asset_id:{str(id)}, tx_type:{tx_type}, hash:{tx_id}"
)
continue
tx["tx_type"] = tx_type
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Type"] = "ACT"
mq_tx["BlockNumber"] = tx["block_num"]
mq_tx["Time"] = date_to_timestamp(tx["timestamp"])
mq_tx["TxIndex"] = 0
# 只要转账的合约交易
if tx["tx_type"] == "transaction_op_type":
reserved = tx["reserved"]
if reserved and tx["reserved"][0] == "transfer_to":
# 区块链浏览器普遍使用该交易ID
mq_tx["Txid"] = tx["orig_trx_id"]
# 发起转账的地址
mq_tx["From"] = tx['to_contract_ledger_entry'][
'from_account']
# 转账金额
mq_tx["Amount"] = tx["to_contract_ledger_entry"]["amount"][
"amount"]
# 合约地址
mq_tx["Contract"] = tx["to_contract_ledger_entry"][
"to_account"]
# 合约执行函数
if reserved[1].find("|") > -1:
# 合约转账金额
mq_tx["Amount"] = reserved[1].split("|")[1]
# 实际合约转账地址
mq_tx["To"] = reserved[1].split("|")[0]
else:
mq_tx["To"] = reserved[1]
else:
# 交易id
mq_tx["Txid"] = tx["trx_id"]
# 发起转账的地址
mq_tx["From"] = tx['ledger_entries'][0]['from_account']
# 接收转账的地址
mq_tx["To"] = tx["ledger_entries"][0]["to_account"]
# 转账金额
mq_tx["Amount"] = tx["ledger_entries"][0]["amount"]["amount"]
mq_tx["status"] = "true"
push_list.append(mq_tx)
return push_list
parser.add_argument("coin", help="指定要开启推送的货币名称")
parser.add_argument("rpc", help="节点rpc请求地址")
parser.add_argument("-m", "--mode", help="设置运行模式，dev开发环境,prod生产环境")
parser.add_argument("-p", "--process", type=int, help="设置进程数量,默认1")
args = parser.parse_args()
# 根据传入的参数更新全局配置
G_CFG.coin.coin_dict["name"] = args.coin
process = args.process if args.process else 0
G_CFG.coin.coin_dict["process"] = process
G_CFG.rpc.rpc_dict["url"] = args.rpc
if args.mode:
G_CFG.coin.coin_dict["mode"] = args.mode
G_CFG.log.log_dict["filename"] = f"log/{args.coin}.log"
mode = G_CFG.coin.coin_dict["mode"]
if mode not in ["prod", "dev"]:
G_LOGGER.info("未知的运行模式")
exit()
if args.process and args.process > 1:
events = [
Event.push_process, Event.mempool_process, Event.fetch_process
]
else:
events = [Event.push_process, Event.mempool_process]
process_len = len(events) + process
# 开启多进程
pool = Pool(processes=process_len)
for e in events:
result = pool.apply_async(e)
if args.process and args.process > 1 and e == Event.fetch_process:
for i in range(args.process - 1):
result = pool.apply_async(e)
def parse_block(self, block_height):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
accounts = self.get_xmr_accounts()
block_detail = self.rpc.get_block(block_height)
if not block_detail:
return []
tx_ids = block_detail.get("tx_hashes", [])
if not tx_ids:
return []
tx_details = self.rpc.get_transactions(tx_ids)
push_list = []
for tx_detail in tx_details:
tx_as_json = tx_detail.get("as_json")
block_timestamp = tx_detail.get("block_timestamp")
tx_global_indexes = tx_detail['output_indices']
txid = tx_detail.get("tx_hash")
if not txid:
continue
tx_json = json.loads(tx_as_json)
extra_list = tx_json['extra']
extra = ''.join('{:02x}'.format(x) for x in extra_list)
extra_mem = ctypes.c_char_p()
extra_mem.value = extra.encode()
tx_pub_key_mem = ctypes.create_string_buffer(100)
tx_pub_key_result = self.monero_lib.monero_from_extra_get_tx_pub_key(
extra_mem, tx_pub_key_mem)
if not tx_pub_key_result:
continue
# G_LOGGER.info(f"height:{block_height} txid:{txid}")
tx_vin = tx_json.get("vin", [])
v_in_keys = [i['key'] for i in tx_vin]
key_images = [i.get("k_image") for i in v_in_keys]
tx_vout = tx_json.get("vout", [])
rct_signatures = tx_json.get("rct_signatures")
if not rct_signatures:
continue
ecdhInfos = rct_signatures.get("ecdhInfo")
if not ecdhInfos:
continue
outPks = rct_signatures.get("outPk")
if not outPks:
continue
tx_fee = rct_signatures.get("txnFee", '0')
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = txid
mq_tx["Type"] = "XMR"
mq_tx["From"] = ""
mq_tx["To"] = ""
mq_tx["Amount"] = ""
mq_tx["Memo"] = ""
mq_tx["Fee"] = str(tx_fee)
mq_tx["Valid"] = True
mq_tx["status"] = "true"
mq_tx["Time"] = block_timestamp
mq_tx["BlockNumber"] = block_height
mq_tx["VoutsIndex"] = 0
v_in_address = []
for key in key_images:
if not key:
continue
redis_key = key[:6] + key[18:24] + key[-6:]
addr_info = self.redis.get_and_delete(redis_key)
if not addr_info:
continue
addr_info = json.loads(addr_info.decode())
G_LOGGER.info(
f"height:{block_height}, txid:{txid}, key_image: {key}, addr_info: {addr_info}"
)
v_in_address.append(addr_info)
v_out_infos = []
for v_out_index, vout in enumerate(tx_vout):
rct = outPks[v_out_index]
global_index = tx_global_indexes[v_out_index]
target = vout.get("target")
if not target:
continue
v_out_address = target.get("key")
if not v_out_address:
continue
random_utxo = {
"global_index": str(global_index),
"public_key": v_out_address,
"rct": rct
}
self.redis.push_utxo_cache(self.random_utxo_redis_key,
random_utxo)
for account in accounts:
address = account.get("addr")
private_view_key = account.get("pr_vk")
public_spend_key = account.get("pb_sk")
private_view_key_mem = ctypes.c_char_p()
private_view_key_mem.value = private_view_key.encode()
public_spend_key_mem = ctypes.c_char_p()
public_spend_key_mem.value = public_spend_key.encode()
derivation_mem = ctypes.create_string_buffer(256)
one_time_address_mem = ctypes.create_string_buffer(128)
amount_echo_init = ecdhInfos[v_out_index]
if not amount_echo_init:
continue
amount_echo = amount_echo_init.get("amount")
if not amount_echo:
continue
if address and str(address).startswith("8"):
additional_tx_pub_key_mem = ctypes.create_string_buffer(
100)
additional_tx_pub_key_result = self.monero_lib.monero_from_extra_get_additional_tx_pub_key(
extra_mem, 0, v_out_index,
additional_tx_pub_key_mem)
if not additional_tx_pub_key_result:
der_result = self.monero_lib.monero_generate_derivation_byte32(
tx_pub_key_mem, private_view_key_mem,
derivation_mem)
else:
der_result = self.monero_lib.monero_generate_derivation_byte32(
additional_tx_pub_key_mem,
private_view_key_mem, derivation_mem)
else:
der_result = self.monero_lib.monero_generate_derivation_byte32(
tx_pub_key_mem, private_view_key_mem,
derivation_mem)
if not der_result:
continue
address_result = self.monero_lib.monero_generate_one_time_public_key(
derivation_mem, public_spend_key_mem, v_out_index,
one_time_address_mem)
if not address_result:
continue
one_time_addr = (one_time_address_mem.value).decode()
# 判断是否命中
if v_out_address.strip() != one_time_addr.strip():
# G_LOGGER.info(f"Addr:{address}, height:{block_height}, txid:{txid}, out:{v_out_index} No")
continue
G_LOGGER.info(
f"Addr:{address}, height:{block_height}, txid:{txid}, out:{v_out_index} Yes"
)
amount_echo_mem = ctypes.c_char_p()
amount_echo_mem.value = amount_echo.encode()
real_amount_mem = ctypes.create_string_buffer(128)
# payment_id_mem = ctypes.create_string_buffer(32)
amount_result = self.monero_lib.monero_decrypt_amount(
derivation_mem, v_out_index, amount_echo_mem, 0,
real_amount_mem)
amount = '0'
if amount_result:
amount = (real_amount_mem.value).decode()
amount = int("0x" + amount, 0)
mq_tx["Amount"] = amount
# 暂时不推送memo信息,子地址不需要memo
# pyamentid_result = self.monero_lib.monero_from_extra_get_payment_id(extra_mem, derivation_mem, payment_id_mem)
# payment_id = '0'
# if pyamentid_result:
# payment_id = (payment_id_mem.value).decode()
# tmp_tx["Memo"] = payment_id
tmp_tx = mq_tx.copy()
tmp_tx["Memo"] = ""
tmp_tx["VoutsIndex"] = v_out_index
tmp_tx["To"] = address
v_out_infos.append(tmp_tx)
break
# if (not v_in_address) or (not v_out_infos):
# continue
v_in_length = len(v_in_address)
v_out_length = len(v_out_infos)
G_LOGGER.info(
f"height:{block_height}, txid:{txid}, v_in_length:{v_in_length}, v_out_length:{v_out_length}"
)
G_LOGGER.info(f"v_in_address: {v_in_address}")
G_LOGGER.info(f"v_out_infos: {v_out_infos}")
v_out_sort_infos = sorted(v_out_infos,
key=lambda x: x["VoutsIndex"])
max_len = max(v_out_length, v_in_length)
for num in range(max_len):
v_in_idx = num if num < v_in_length else -1
v_out_idx = num if num < v_out_length else -1
tx_tp = v_out_sort_infos[
v_out_idx] if v_out_length - 1 >= v_out_idx else v_out_sort_infos[
-1]
tmp = tx_tp.copy()
tmp['From'] = v_in_address[v_in_idx] if v_in_address else ''
push_list.append(tmp)
return push_list
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
t1 = time.time()
push_list = []
node, block = self.http.get_block(block_num)
t2 = time.time()
if isinstance(block, str):
# 出现大块，比如110588067，api.eossweden.org超级节点偶发性返回部分数据，不能被json反序列化，需要多次重试才能获得结果，造成消息积压
# eos.greymass.com超级节点可以正确拿到数据，但是访问频率受限，所以异常以后要重试获取数据
G_LOGGER.info(
"eos_pass, block_num={}, node={}, lenght={}, head={}, tail={}".
format(block_num, node, len(block), block[:38], block[-30:]))
# self.redis.client.rpush("eos_pass_block", block_num)
# return push_list
timestamp = str(date_to_timestamp(block['timestamp'].split(".")[0]))
txs = block.get("transactions", [])
for tx in txs:
tx_status = tx.get("status")
if (not tx_status) or (tx_status not in ("executed")):
continue
trx = tx.get("trx", "")
if not isinstance(trx, dict):
continue
tx_id = trx['id']
transaction = trx["transaction"]
acts = transaction.get('actions', [])
act_index = 0
get_amount_and_symbol = self.get_amount_and_symbol
for act in acts:
account, name, Contract = act.get('account'), act['name'], ""
# if name in self.SUPPORTED and self.action_account.get(name) == account:
if name in self.SUPPORTED:
data = act['data']
if not isinstance(data, dict):
continue
mq_tx = G_PUSH_TEMPLATE.copy()
if name == "delegatebw" and self.action_account.get(
name) != account:
from_addr = account
to_addr = data.get("to")
amount, symbol = get_amount_and_symbol(
"other_delegatebw", data)
elif name == "undelegatebw" and self.action_account.get(
name) != account:
from_addr = data.get("from")
to_addr = account
amount, symbol = get_amount_and_symbol(
"other_undelegatebw", data)
else:
from_addr = data.get(self.action_convert[name]['from'])
to_addr = data.get(self.action_convert[name]['to'])
if not from_addr or not to_addr:
continue
amount, symbol = get_amount_and_symbol(name, data)
# 无金额或者金额小于0.0001, 略过
if (not amount) or (float(amount) <= 0.0001):
continue
if account and str(
account) == "eosio.token" and symbol in ["EOS"]:
Contract = "eosio.token"
else:
Contract = str(account) + "|" + str(symbol).upper()
memo_info = data.get("memo", "")
mq_tx["BlockNumber"] = block_num
mq_tx["Time"] = timestamp
mq_tx["Txid"] = tx_id
mq_tx["Type"] = "EOS"
mq_tx['From'] = from_addr
mq_tx['To'] = to_addr
mq_tx['Amount'] = amount
mq_tx["Action"] = name
mq_tx['Contract'] = Contract
mq_tx["VoutsIndex"] = act_index
mq_tx["Memo"] = memo_info
mq_tx["Valid"] = True
mq_tx["status"] = "true"
push_list.append(mq_tx)
act_index += 1
t3 = time.time()
http_time = int((t2 - t1) * 1000) / 1000
parse_time = int((t3 - t2) * 1000) / 1000
G_LOGGER.info(
"eos_parse_block, block_num={}, length={}, node={}, 网络耗时={}, 处理耗时={}"
.format(block_num, len(push_list), node, http_time, parse_time))
return push_list
def fetch_process(cls):
process = G_CFG.coin.coin_dict.get("process")
if int(process) <= 1:
return None
G_LOGGER.info(f"进程{os.getpid()}已开启高并发推送模式")
cls.coin_push().loop_fetch_block()
def parse_block(self, block_num):
"""
解析区块的交易列表(支持多发送)
:return: 返回待推送信息列表
"""
while True:
try:
block = self.rpc.eth_get_block_by_number(block_num)
if block is None:
raise Exception(f"获取到最新高度{block_num}区块详情为None，触发异常，尝试重新获取")
G_LOGGER.info(
f"当前推送的区块：高度{int(block.get('number', '0x0'), 16)}，哈希{block.get('hash', '')}"
)
self._check_uncle(block)
# 遍历交易
push_list = []
if block.get("transactions") is None:
time.sleep(3)
continue
tx_hashes = [tx["hash"] for tx in block["transactions"]]
receipts = self.rpc.eth_get_transaction_receipt(
tx_hashes) if tx_hashes else []
assert len(receipts) == len(block["transactions"])
for i, tx in enumerate(block["transactions"]):
_input = tx.get('input', '')
_to = tx.get('to', '')
_from = tx.get('from', '')
multi_send_token = self.is_multi_send_token(_input)
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["BlockNumber"] = hex_to_int(tx["blockNumber"])
mq_tx["Txid"] = tx["hash"]
mq_tx["Type"] = self.coin_type
mq_tx["From"] = tx["from"]
mq_tx["Time"] = block["timestamp"]
is_token_transfer = self._is_token(_input)
# 交易状态
mq_tx["Valid"] = self.get_status(receipts[i],
is_token_transfer)
mq_tx["status"] = self.get_tx_status(
receipts[i], is_token_transfer)
# 手续费
gas_price = tx.get("gasPrice", "0x0")
gas_used = receipts[i].get("gasUsed",
"0x0") if receipts[i] else "0x0"
mq_tx["Fee"] = hex(int(gas_price, 16) * int(gas_used, 16))
if is_token_transfer:
if _from != _to:
tx['contract'] = _to
_to = self._get_token_to_address(_input)
_value = self._get_token_to_value(_input)
tx["to"] = _to if _to != "0x" else f"0x{'0'*40}"
tx["value"] = _value if _value != "0x" else "0x0"
elif multi_send_token:
continue
_len = self.multi_abi_len
_del_0x_input = del_0x(_input)
if multi_send_token == 'create':
# 多发送合约创建的交易，保存到数据库存起来
data = dict()
data['contract'] = tx['contract'] = tx['creates']
data['blockNumber'] = hex_to_int(tx["blockNumber"])
data['timestamp'] = block["timestamp"]
data['hash'] = tx["hash"]
# 保存合约
self.db.mysql.contract.insert_contract(data)
G_LOGGER.info(f"发现新的多发送合约{data['contract']}并保存")
continue
else:
token = ''
address_list = amount_list = []
# 不是我们自己的多发送合约则不处理
contract = self.db.mysql.contract.get_by_contract(
tx["to"])
if not contract:
continue
if multi_send_token == 'multisend':
_multisend_abi = del_0x(
self.multi_abi_deal['multisend'])
result = self.parse_multi_data(
_del_0x_input[len(_multisend_abi):])
address_count = hex_to_int(result[2])
address_list = result[3:3 + address_count]
amount_count = hex_to_int(
result[3 + address_count])
amount_list = result[4 + address_count:4 +
address_count +
amount_count]
elif multi_send_token == 'multisendToken':
_multisend_token_abi = del_0x(
self.multi_abi_deal['multisendToken'])
token = add_0x(
self.parse_multi_data_pos(
_del_0x_input, _multisend_token_abi,
0))
result = self.parse_multi_data(
_del_0x_input[len(_multisend_token_abi):])
address_count = hex_to_int(result[3])
address_list = result[4:4 + address_count]
amount_count = hex_to_int(
result[4 + address_count])
amount_list = result[5 + address_count:5 +
address_count +
amount_count]
for k, v in enumerate(address_list):
mq_tx = mq_tx.copy()
mq_tx["To"] = v
mq_tx["Amount"] = int_to_hex(
hex_to_int(amount_list[k]))
mq_tx["Contract"] = token
push_list.append(mq_tx)
return push_list
if tx["to"]:
mq_tx["To"] = tx["to"]
mq_tx["Amount"] = tx["value"]
if 'contract' in tx.keys():
mq_tx["Contract"] = tx["contract"]
push_list.append(mq_tx)
return push_list
except ForkError as ex:
raise ForkError(ex.height, ex.msg)
except Exception as ex:
traceback.print_exc()
G_LOGGER.info(f"获取块出现异常，尝试重新获取。异常原因：{str(ex)}")
time.sleep(10)
def _send_data(self, method, url, _params=None, _data=None, _json=None, _cookies=None):
data = _params or _data or _json
G_LOGGER.debug('接口: {}, 方法: {}, 请求数据: {}'.format(url, method, data))
t1 = time.time()
retry_time = 0.001
while True:
try:
with self.session.request(method, url, params=_params, data=_data, json=_json, auth=self.auth,
headers=self.headers, timeout=self.timeout, cookies=_cookies) as rsp:
G_LOGGER.debug('返回数据: {}'.format(rsp.text))
if 300 > rsp.status_code >= 200:
result = rsp.text
if self.is_json:
try:
t2 = time.time()
net_time = int((t2 - t1) * 1000) / 1000
res = json.loads(rsp.text)
t3 = time.time()
json_time = int((t3 - t2) * 1000) / 1000
all_time = int((t3 - t1) * 1000) / 1000
# G_LOGGER.info("json success, {}, {}, length={}, net_time={}, json_time={}, all_time={}, ".format(url, data, len(rsp.text), net_time, json_time, all_time))
return res
except Exception:
# G_LOGGER.info("json fail, {}, {}, length={}, net_time={}".format(url, data, len(rsp.text), net_time))
pass
return result
elif rsp.status_code == 404:
G_LOGGER.warn("[404] URL不存在, 请检查URL".format(self.auth))
raise UrlError("[404] URL不存在, 请检查URL")
elif rsp.status_code == 401:
G_LOGGER.warn("[401] auth验证错误. auth: {}".format(self.auth))
raise RequestError(rsp.status_code, "[401] auth验证错误. auth: {}".format(self.auth))
elif rsp.status_code == 403:
G_LOGGER.warn("[403] 没有权限操作此URL. url: {}".format(url))
raise RequestError(rsp.status_code, "[403] 没有权限操作此URL. url: {}".format(url))
elif 500 > rsp.status_code >= 400:
G_LOGGER.warn("[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text))
raise RequestError(rsp.status_code,
"[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text),
rsp)
elif 599 > rsp.status_code >= 500:
G_LOGGER.warn("[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text))
raise RequestError(rsp.status_code,
"[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text),
rsp)
except (UrlError, RequestError):
raise
except (request_error.Timeout, request_error.ConnectionError) as e:
if retry_time < 10:
retry_time = min(max(retry_time, retry_time * 2), 10)
G_LOGGER.info("{}超时或被拒绝, retry_time={}, {} {}, 错误:{}".format(url, retry_time, method, data, str(e)))
continue
except request_error.InvalidURL:
G_LOGGER.error('请求URL无效, 请检查: {}'.format(url))
raise UrlError('URL无效.')
except Exception as e:
G_LOGGER.error('请求出现不可预知异常, 请处理. \r\n'
'请求url: {}\r\n'
'请求方法: {}\r\n'
'请求参数: params: {}, data: {}, json: {}\r\n'
'验证用户: {}\r\n'
'错误详情: {}'.format(url, method, _params, _data, _json, self.auth,
traceback.format_exc()))
raise RequestError(0, "请求出现不可预知异常, 请处理. 详情简要: {}".format(e))
G_CFG.mq.mq_dict[
"routing_key"] = args.routing_key if args.routing_key else args.coin
if args.exchange_name:
G_CFG.mq.mq_dict["exchange_name"] = args.exchange_name
if args.vhost:
G_CFG.mq.mq_dict["vhost"] = args.vhost
if args.mq_host:
G_CFG.mq.mq_dict["host"] = args.mq_host
if args.mq_user:
G_CFG.mq.mq_dict["username"] = args.mq_user
if args.mq_password:
G_CFG.mq.mq_dict["password"] = args.mq_password
if args.mode:
G_CFG.coin.coin_dict["mode"] = args.mode
G_CFG.mysql.mysql_dict["db"] = args.coin
G_CFG.log.log_dict["filename"] = f"log/{args.coin}.log"
G_CFG.message.message_dict["monitor_path"] = f"log/{args.coin}.txt"
mode = G_CFG.coin.coin_dict["mode"]
if mode not in ["prod", "dev"]:
G_LOGGER.info("未知的运行模式")
exit()
# 手动推送区块数据
coin_push = Event.coin_push()
coin_name = args.coin
block_num = args.block_num
count = args.count
print('手动推送区块数据，coin: {}, start_block: {}, end_blcok: {}'.format(
coin_name, block_num, block_num + count - 1))
coin_push.push_sync(block_num, count)
def parse_block(self, block_num):
"""
解析区块的交易列表
"""
try:
block = self.rpc.eth_get_block_by_number(block_num)
if block is None:
raise Exception(f"获取区块高度{block_num}区块详情为None，触发异常，尝试重新获取")
G_LOGGER.info(
f"当前推送的区块：高度{int(block.get('number', '0x0'), 16)}，哈希{block.get('hash', '')}"
)
self._check_uncle(block)
# 遍历交易
push_list = []
if block.get("transactions") is None:
raise Exception(
f"获取区块高度{block_num}区块transactions为None，触发异常，尝试重新获取")
tx_hashes = [tx["hash"] for tx in block["transactions"]]
receipts = self.rpc.eth_get_transaction_receipt(
tx_hashes) if tx_hashes else []
assert len(receipts) == len(block["transactions"])
for i, tx in enumerate(block["transactions"]):
_input = tx.get('input', '')
_to = tx.get('to', '')
_from = tx.get('from', '')
is_token_transfer = self._is_token(_input)
if is_token_transfer:
if _from != _to:
tx['contract'] = _to
_to = self._get_token_to_address(_input)
_value = self._get_token_to_value(_input)
tx["to"] = _to if _to != "0x" else f"0x{'0'*40}"
tx["value"] = _value if _value != "0x" else "0x0"
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["BlockNumber"] = int(tx["blockNumber"], 16)
mq_tx["Txid"] = tx["hash"]
mq_tx["Type"] = self.coin_type
mq_tx["From"] = tx["from"]
# 排除to为空的
if not tx["to"]:
continue
mq_tx["To"] = tx["to"]
mq_tx["Time"] = block["timestamp"]
mq_tx["Amount"] = tx["value"]
# 手续费
gas_price = tx.get("gasPrice", "0x0")
gas_used = receipts[i].get("gasUsed",
"0x0") if receipts[i] else "0x0"
mq_tx["Fee"] = hex(int(gas_price, 16) * int(gas_used, 16))
# 交易状态
mq_tx["Valid"] = self.get_status(receipts[i],
is_token_transfer)
mq_tx["status"] = self.get_tx_status(receipts[i],
is_token_transfer)
if 'contract' in tx.keys():
mq_tx["Contract"] = tx["contract"]
push_list.append(mq_tx)
# 缓存已推送的区块高度和hash
self.db.redis.save_cache_block(hex_to_int(block["number"]),
block["hash"])
return push_list
except ForkError as ex:
raise ForkError(ex.height, ex.msg)
except Exception as e:
traceback.print_exc()
G_LOGGER.info(f"获取块出现异常，尝试重新获取。异常原因：{str(e)}")
def push_process(cls):
"""
交易推送进程
"""
G_LOGGER.info(f"进程{os.getpid()}启动TX的推送任务")
cls.coin_push().loop_push()
def push_main(self, save_redis=False):
"""
推送处理
"""
if not self.coin_name:
raise Exception("unknow coin name")
basic_info = self.db.get_basic(self.coin_name)
if not basic_info:
G_LOGGER.info("币种{}没有相关配置,请检查数据库配置".format(self.coin_name))
return True
save_num = current_height = basic_info["current_height"]
if save_redis:
newest_block_height = self.coin.newest_height()
diff_num = newest_block_height - current_height
else:
newest_block_height = self.coin.newest_height()
diff_num = newest_block_height - current_height
if diff_num <= 0:
G_LOGGER.info("最新高度{},已同步高度{},无需推送".format(newest_block_height, current_height))
return True
try:
if save_redis:
step = 100 # 最大处理100个块存入redis
diff_num = step if diff_num > step else diff_num
G_LOGGER.info("当前最新高度为{}, 数据库中的高度{}, 本次需要放入redis的块数为{}".format(newest_block_height, current_height, diff_num))
# 待推送的区块高度存入到redis队列中
for height in range(current_height + 1, current_height + diff_num + 1):
self.block_num = height
G_LOGGER.info("redis_save_pending_block: {}".format(self.block_num))
self.db.redis.save_pending_block(self.block_num)
else:
G_LOGGER.info("最新高度{},已同步高度{},本次需要同步块数{}".format(newest_block_height, current_height, diff_num))
self.push_sync(current_height + 1, diff_num, rollback_count=self.rollback_count)
except ForkError as e:
G_LOGGER.error("同步过程出现临时分叉, 即将回滚至高度{}重新推送".format(e.height))
save_num = e.height
except SyncError as e:
err_info = traceback.format_exc()
G_LOGGER.error("{}块同步过程出现异常, 请处理此块. 详情: {}".format(e.height, str(err_info)))
except Exception as e:
G_LOGGER.error("同步过程出现异常, 请处理此块. 详情{}".format(e))
except KeyboardInterrupt:
G_LOGGER.info("同步过程手动取消任务, 同步至高度: {}. 已保存".format(self.block_num))
save_num = self.block_num
else:
save_num = self.block_num
finally:
basic_info['current_height'] = save_num
basic_info['newest_height'] = newest_block_height
self.db.update_basic(basic_info)
G_LOGGER.info("push success ,已同步保存至区块高度: {}".format(save_num))
return True