def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
while True:
try:
if isinstance(block_num, int):
block_num = hex(block_num)
if isinstance(block_num,
str) and not block_num.startswith('0x'):
block_num = hex(int(block_num))
block = self.rpc.get_block(block_num)
txs = block.get("transactions", [])
push_list = []
for tx in txs:
tx_receipt = self.rpc.get_tx_receipt(tx.get("hash"))
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("hash")
mq_tx["Type"] = "TOMO"
mq_tx["From"] = tx.get("from")
mq_tx["To"] = tx.get("to")
mq_tx["Amount"] = int(tx.get("value"), 16)
mq_tx["Fee"] = int(tx["gasPrice"], 16) * int(
tx_receipt["gasUsed"], 16)
status = int(tx_receipt["status"], 16)
mq_tx["Valid"] = True if status else False
mq_tx["status"] = "true" if status else "false"
mq_tx["Time"] = int(block.get("timestamp"), 16)
mq_tx["BlockNumber"] = int(tx.get("blockNumber"), 16)
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(f"出现异常，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def set(self, tx_hash, value):
"""设置数据"""
key = self.build_key(tx_hash)
value = json.dumps(value)
G_LOGGER.debug("设置key: {} 值: {}".format(key, value))
return self.client.set(key, value)
def send_mail(self,
receiver,
title,
msg,
mail_type="plain",
file_paths=[],
file_names=[],
image_paths=None):
"""发送邮件"""
assert receiver and isinstance(receiver, list)
sender = self.__mail_user
mail_type = mail_type.lower()
if mail_type in ["plain", "html"]:
message = MIMEText(msg, mail_type, "utf-8")
elif mail_type in ["file", "image"]:
message = MIMEMultipart()
else:
return False
try:
message["From"] = Header(self.__mail_user, "utf-8")
message["To"] = Header(",".join(receiver), "utf-8")
message["Subject"] = Header(title, "utf-8")
if mail_type in ["file", "image"]:
# 邮件正文内容
if image_paths is not None:
message.attach(MIMEText(msg, "html", "utf-8"))
# 添加图片
if image_paths is not None:
for index, image_path in enumerate(image_paths, 1):
# 指定图片为当前目录
fp = open(image_path, "rb")
msg_image = MIMEImage(fp.read())
fp.close()
# 定义图片 ID，在 HTML 文本中引用
msg_image.add_header("Content-ID",
"<image" + str(index) + ">")
message.attach(msg_image)
else:
message.attach(MIMEText(msg, "plain", "utf-8"))
# 构造附件，传送filePath制定文件
for filePath, fileName in zip(file_paths, file_names):
att = MIMEText(
open(filePath, "rb").read(), "base64", "utf-8")
att["Content-Type"] = "application/octet-stream"
# 邮件中显示文件名
att["Content-Disposition"] = "attachment; filename=" " + fileName + " ""
message.attach(att)
except Exception as e:
G_LOGGER.error(f"构造邮件发生错误,详情:{str(e)}")
return False
try:
smtp = smtplib.SMTP_SSL(self.__mail_host, self.__mail_port)
smtp.login(self.__mail_user, self.__mail_pass)
smtp.sendmail(sender, receiver, message.as_string())
smtp.quit()
except Exception as e:
G_LOGGER.error(f"发送邮件发生错误,详情:{str(e)}")
return False
return True
def parse_txs(self, txs, height=None):
"""
解析区块内交易
"""
push_list = []
push_cache = []
if self.rollback and height:
cache_tag = self.push_cache.get(str(height))
push_cache = self.push_cache.pop(str(height)) if cache_tag else []
G_LOGGER.info('Start process block:{}, rollback_status:{}, rollback_count: {}, cache_data_len:{}, block txcounts:{}'.format(height, self.rollback, self.rollback_count, len(push_cache), len(txs)))
coinbase_txs = self.parse_coinbase_tx(txs.pop(0), push_cache=push_cache)
total_fees = 0
process_tx_counts = 0
for tx_ids in seperate_big_list(txs, chunk=self.request_chunk_size):
tx_details = self.rpc.get_transactions(tx_ids)
# G_LOGGER.info('Block:{}, current process tx counts:{}'.format(height, len(tx_details)))
if not tx_details:
continue
process_tx_counts += len(tx_details)
txs_list, fees = self.parse_normal_tx(tx_details, push_cache=push_cache)
total_fees += fees
push_list.extend(txs_list)
# 给coinbase交易添加fee字段值
for tx in coinbase_txs:
tx['Fee'] = str(total_fees)
push_list.append(tx)
G_LOGGER.info('Block:{}, process txcounts:{}, push txcounts: {}'.format(height, process_tx_counts + 1, len(push_list)))
return push_list
def _check_uncle(self, block):
"""
检查当前区块的叔块，看是否需要回滚
"""
# 检查当前传入的区块高度和最新的区块高度是否一致，相差大于100说明当前是处理比较老的区块了则不用检查叔块
current_height = hex_to_int(block["number"])
newest_height = self.rpc.eth_block_number()
if newest_height - current_height > 100:
return False
# 没有叔块也不用检查叔块
uncles = block["uncles"]
if not uncles:
return False
G_LOGGER.info(f"区块{current_height}发现叔块:{','.join(uncles)}")
# 获取叔块对应最小区块高度的hash
uncle_height = []
for uncle in uncles:
uncle_block = self.rpc.eth_get_block_by_hash(uncle,
tx_detail=False)
if uncle_block:
uncle_height.append(hex_to_int(uncle_block["number"]))
if uncle_height:
min_height = min(uncle_height)
# 查询缓存的区块中是否有uncle hash
push_block_hash = self.db.redis.get_cache_block(min_height)
# 已经推送的该高度对应的区块hash如果不在uncles中，说明之前推送的区块就是正确的区块，不用继续处理
if push_block_hash not in uncles:
return False
raise ForkError(min_height, f"{min_height}高度同步过程中出现临时分叉")
def coin_push(cls):
plugin_manager = DirectoryPluginManager()
coin_name = G_CFG.coin.coin_dict.get("name")
plugin_manager.load_plugins(coin_name)
coin = plugin_manager.get_plugins(coin_name)
if coin:
return CoinPush(coin)
else:
G_LOGGER.error(f"未安装{coin_name}的交易数据解析插件")
exit()
def processor(data):
"""JSONRPC协议中, result与error互斥, 两者不可能同时拥有值."""
results = []
for d in data:
if not isinstance(d, dict):
continue
results.append(d.get('result'))
if ignore_err:
return results
if not all(results):
G_LOGGER.debug('many post中返回数据错误: {}'.format(data))
raise JsonRpcError(-1, "many post中返回数据中错误.")
def mempool_process(cls):
"""
未确认交易推送进程
"""
# 只有比特系币种才有内存池
coin_name = G_CFG.coin.coin_dict.get("name")
btc_serial = [
"btc", "bch", "bsv", "bcx", "bcd", "btg", "dash", "doge", "god",
"ipc", "ltc", "qtum", "sbtc", "zec", 'usdt'
]
if coin_name not in btc_serial:
return None
G_LOGGER.info(f"进程{os.getpid()}启动Mempool的推送任务")
cls.coin_push().loop_mempool()
def _read_dir(self, plugin_list, directory):
"""
递归遍历插件目录
:return:
"""
try:
for f in os.listdir(directory):
sub_path = os.path.join(directory, f)
if os.path.isdir(sub_path):
self._read_dir(plugin_list, sub_path)
else:
if f.endswith(".py") and f != "__init__.py":
plugin_list.append((f[:-3], sub_path))
except OSError:
G_LOGGER.error("Failed to access: %s" % directory)
return plugin_list
def kafka_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
partition, offset = self.db.kafka.send(data)
G_LOGGER.info("Process:{} kafka push success, partition={}, offset={}, push_data={}".format(os.getpid(), partition, offset, data))
except Exception as e:
G_LOGGER.error("Process:{} kafka push failed, push_data={}, error={}".format(os.getpid(), data, str(e)))
def push_sync(self, height, num, rollback_count=0):
block_num = height
try:
for block_num in range(height, height + num):
self.block_num = block_num
# 获取待推送数据
push_data = self.coin.push_list(block_num, rollback_count=rollback_count)
G_LOGGER.info("当前推送区块高度为:{}, push_counts={}".format(self.block_num, len(push_data)))
for data in push_data:
self.rocket_push(data)
time.sleep(0.5)
except ForkError as e:
raise ForkError(e.height, e.msg)
except Exception as e:
# error_info = traceback.format_exc()
error_info = str(e)
raise SyncError(block_num, f"{block_num}{str(error_info)}")
def gen_address_p2pk(self, script_bytes):
"""
根据p2pkh的签名信息解析出pubkey,pubkey--->address
param: magicbyte 版本前缀
"""
magicbyte, magicbyte_length = self.get_address_magicbyte(address_type='P2PKH')
curosr = 0
while curosr < len(script_bytes):
n, length = self.get_curosr_vrant(script_bytes, curosr)
curosr += n
content = script_bytes[curosr: curosr + length]
curosr += length
address = self.pubkey_to_address(content, magicbyte=magicbyte, magicbyte_length=magicbyte_length)
G_LOGGER.debug("{} ---> P2PKH Address: {}".format(self.coin_type, address))
return address
def gen_address_p2sh(self, script_bytes):
"""
p2sh根据验证信息解析出reeddemscript, reeddemscript--->address
param: magicbyte 版本前缀
"""
magicbyte, magicbyte_length = self.get_address_magicbyte(address_type='P2SH')
curosr = 1
while curosr < len(script_bytes):
n, length = self.get_curosr_vrant(script_bytes, curosr)
curosr += n
content = script_bytes[curosr: curosr + length]
curosr += length
address = self.p2sh_scriptaddr(content.hex(), magicbyte=magicbyte, magicbyte_length=magicbyte_length)
G_LOGGER.debug("{} ---> P2PKH Address: {}".format(self.coin_type, address))
return address
def rocket_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
msg_id = self.db.rocket.push_data(data)
G_LOGGER.info("success, msg_id={}, data={}".format(msg_id, data))
except Exception as e:
G_LOGGER.error("failed, data={}, error={}".format(data, str(e)))
def _wrap(*args, **kwargs):
def _loop():
func(*args, **kwargs)
mode = G_CFG.coin.coin_dict["mode"]
while True:
if mode == "prod":
try:
_loop()
except Exception:
error_info = traceback.format_exc()
G_LOGGER.error(f"出现异常, 请处理. {error_info}")
except KeyboardInterrupt:
G_LOGGER.info(f"{func.__name__}手动取消任务, 退出循环执行任务")
elif mode == "dev":
_loop()
# 每秒循环一次
time.sleep(5)
def push_mempool_info(self):
"""
推送mempool中的未确认交易信息
"""
redis_mempool_list = self.db.redis.get_mempool_info()
G_LOGGER.info('Mempool中未确认交易数量:{}'.format(len(redis_mempool_list)))
try:
old_tx_ids, new_tx_ids, push_data = self.coin.push_mempool_list(redis_mempool_list)
for data in push_data:
self.kafka_push(data)
update_redis_mempool_list = [tx_id for tx_id in redis_mempool_list if tx_id not in old_tx_ids]
update_redis_mempool_list.extend(new_tx_ids)
self.db.redis.update_mempool_info(update_redis_mempool_list)
# 避免频繁请求节点,每隔6秒检测mempool
time.sleep(6)
except Exception:
msg = traceback.format_exc()
G_LOGGER.info("推送MemPool信息出错, 错误: {}.".format(msg))
def loop_fetch_block(self):
"""
从redis取出待处理的块进行解析
"""
while True:
block_num = self.db.redis.get_pending_block()
if len(block_num) > 0:
block_num = block_num[0].decode("utf-8")
diff_num = 1
G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块")
try:
self.push_sync(int(block_num), diff_num)
except Exception as e:
self.db.redis.save_pending_block(block_num)
# err_info = traceback.format_exc()
err_info = str(e)
G_LOGGER.info(f"进程{os.getpid()}正在处理{block_num}区块异常, 详情::{err_info}")
time.sleep(0.1)
def get_address_from_scriptsig(self, script_sig):
"""
从解锁脚本中解析出地址,尚不支持隔离见证类型的解锁脚本(签名)
"""
address = None
try:
script_sig_byte = bytes.fromhex(script_sig)
if not script_sig_byte:
return None
n = script_sig_byte[0]
if n == 0:
address = self.gen_address_p2sh(script_sig_byte)
else:
address = self.gen_address_p2pk(script_sig_byte)
except Exception:
err_info = traceback.format_exc()
G_LOGGER.error("{} GET_ADDRESS_FROM_SCRIPTSIG ERROR：{}".format(self.coin_type, err_info))
return address
def __init__(self, cfg):
super().__init__()
self.client = RedisPool(cfg).get_client()
self.coin = cfg.coin.coin_dict["name"]
# 推送成功的近1000条数据
self.last_push_cache = {
"cache_name": f"{self.coin}_last_push_cache",
"cache_length": 1000
}
# 推送失败的数据
self.fail_push_log = f"{self.coin}_fail_push_log"
# basic
self.basic_key = f"{self.coin}_basic"
# mempool
self.mempool_key = f"{self.coin}_mempool"
# pending block
self.pending_block_key = f"{self.coin}_pending_block"
G_LOGGER.debug("与redis建立连接成功.")
G_LOGGER.debug("币种: {} redis连接信息: {}".format(self.coin,
cfg.redis.redis_dict))
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
block = None
while True:
try:
block = self.rpc.get_block(block_num)
ledger = block.get("ledger", []) if block else []
txs = ledger.get("transactions", []) if ledger else []
push_list = []
for tx in txs:
if tx["TransactionType"] == "Payment" and isinstance(
tx['Amount'], str):
# 交易类型为Payment且状态为成功
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("hash", "")
mq_tx["Type"] = "XRP"
mq_tx["From"] = tx.get("Account", "")
mq_tx["To"] = tx.get("Destination", "")
mq_tx["Fee"] = tx.get("Fee", "")
mq_tx["Memo"] = tx.get("DestinationTag", "")
meta_data = tx.get("metaData", {})
status = meta_data.get("TransactionResult")
mq_tx["Valid"] = True if (
status and status == "tesSUCCESS") else False
mq_tx["status"] = "true" if (
status and status == "tesSUCCESS") else "false"
mq_tx["Amount"] = meta_data.get(
"delivered_amount", "0")
mq_tx["Time"] = ledger.get("close_time",
0) + self.time_diff
mq_tx["BlockNumber"] = ledger.get("ledger_index", 0)
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(
f"区块{block_num}出现异常，区块内容:{block}，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def parse_block(self, block_height):
"""
解析区块
"""
if not isinstance(block_height, int):
block_height = int(block_height)
block_hash = self.rpc.get_block_hash(block_height)
if not block_hash:
return []
self.current_height = block_height
block = self.rpc.get_block(block_hash)
txs = block['tx'] if block else []
if not txs:
return []
push_list = self.parse_txs(txs, height=block_height)
len_of_cacahe = len(self.push_cache)
G_LOGGER.info('{} push_cache_length: {}, rollback_count: {}'.format(self.coin_type, len_of_cacahe, self.rollback_count))
# 非回溯类的才会缓存
if not self.rollback:
if len_of_cacahe <= (int(self.rollback_count) + 1):
self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list])
else:
G_LOGGER.info('{} push_cache_length: {} out of limit, reset push_cache_dict'.format(self.coin_type, len_of_cacahe))
self.push_cache = {}
self.push_cache[str(block_height)] = set([tx['Txid'] for tx in push_list])
G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys())))
return push_list
def parse_block(self, block_num):
"""
解析区块的交易详情
:return: 返回待推送信息列表
"""
while True:
try:
block = self.rpc.get_block(block_num)
txs = block.get("tx", [])
push_list = []
for tx in txs:
G_LOGGER.info("tx={}".format(tx))
mq_tx = G_PUSH_TEMPLATE.copy()
mq_tx["Txid"] = tx.get("txHash")
mq_tx["Type"] = "BNB"
mq_tx["From"] = tx.get("fromAddr")
mq_tx["To"] = tx.get("toAddr")
mq_tx["Amount"] = int(
float(tx.get("value") if tx.get("value") else 0) *
100000000)
mq_tx["Fee"] = int(
float(tx.get("txFee") if tx.get("txFee") else 0) *
100000000)
mq_tx["Valid"] = True
mq_tx["Time"] = utcstr_to_timestamp(
tx.get("timeStamp")) if tx.get("timeStamp") else 0
mq_tx["BlockNumber"] = tx.get("blockHeight")
mq_tx["TxIndex"] = 0
mq_tx["Memo"] = tx.get("memo") if tx.get("memo") else ""
G_LOGGER.info("mq_tx={}".format(mq_tx))
push_list.append(mq_tx)
return push_list
except Exception as e:
G_LOGGER.info(f"出现异常，尝试重新获取。异常原因：{str(e)}")
time.sleep(1)
def parse_block(self, block_num):
"""
解析区块的交易列表
:return: 返回待推送信息列表
"""
block = self.rpc.get_block(block_num)
txs = block.get("tx", []) if block else []
push_list = []
# 回溯缓存推送
push_cache = []
if self.rollback and block_num:
cache_tag = self.push_cache.get(str(block_num))
push_cache = self.push_cache.pop(str(block_num)) if cache_tag else []
for tx in txs:
try:
sub_push = self.parse_tx(tx, push_cache=push_cache)
# 每一个tx都加上区块时间和区块高度
for push in sub_push:
push["BlockNumber"] = block["index"]
push["Time"] = block["time"]
push_list.extend(sub_push)
except Exception as e:
G_LOGGER.error(f"解析交易出现异常，详情：{e}")
len_of_cacahe = len(self.push_cache)
G_LOGGER.info('NEO push_cache_length: {}, rollback_count: {}'.format(len_of_cacahe, self.rollback_count))
# 非回溯类的才会缓存
if not self.rollback:
if len_of_cacahe <= (int(self.rollback_count) + 1):
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
else:
G_LOGGER.info('NEO push_cache_length: {} out of limit, reset push_cache_dict'.format(len_of_cacahe))
self.push_cache = {}
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys())))
return push_list
def scard(self, address):
key = self.build_key(address)
count = self.client.scard(key)
G_LOGGER.debug('查看用户元素数量 -- 用户: {} 数量: {}'.format(key, count))
return count
def smembers(self, address):
key = self.build_key(address)
members = self.client.smembers(key)
G_LOGGER.debug('查看用户所有元素 -- 用户: {} 添加值: {}'.format(key, members))
return members
def sadd(self, address, *values):
key = self.build_key(address)
G_LOGGER.debug('key: {} 添加值: {}'.format(key, values))
self.client.sadd(key, *values)
def get_and_delete(self, redis_key):
key = self.build_key(redis_key)
result = self.client.get(key)
G_LOGGER.debug("获取key: {} 值为: {}".format(key, result))
self.client.delete(key)
return result
def get(self, tx_hash):
key = self.build_key(tx_hash)
result = self.client.get(key)
G_LOGGER.debug("获取key: {} 值为: {}".format(key, result))
return result
def srem(self, address, *values):
# 删除一个地址的中tx_hash
key = self.build_key(address)
rm = self.client.srem(key, *values)
G_LOGGER.debug('删除用户元素 -- 用户: {} 元素: {}'.format(key, values))
return rm
def sismember(self, address, value):
key = self.build_key(address)
is_mem = self.client.sismember(key, value)
G_LOGGER.debug('是否为用户元素 -- 用户: {} 元素: {} 结果: {}'.format(
key, value, is_mem))
return is_mem