def send_mail(self,
receiver,
title,
msg,
mail_type="plain",
file_paths=[],
file_names=[],
image_paths=None):
"""发送邮件"""
assert receiver and isinstance(receiver, list)
sender = self.__mail_user
mail_type = mail_type.lower()
if mail_type in ["plain", "html"]:
message = MIMEText(msg, mail_type, "utf-8")
elif mail_type in ["file", "image"]:
message = MIMEMultipart()
else:
return False
try:
message["From"] = Header(self.__mail_user, "utf-8")
message["To"] = Header(",".join(receiver), "utf-8")
message["Subject"] = Header(title, "utf-8")
if mail_type in ["file", "image"]:
# 邮件正文内容
if image_paths is not None:
message.attach(MIMEText(msg, "html", "utf-8"))
# 添加图片
if image_paths is not None:
for index, image_path in enumerate(image_paths, 1):
# 指定图片为当前目录
fp = open(image_path, "rb")
msg_image = MIMEImage(fp.read())
fp.close()
# 定义图片 ID，在 HTML 文本中引用
msg_image.add_header("Content-ID",
"<image" + str(index) + ">")
message.attach(msg_image)
else:
message.attach(MIMEText(msg, "plain", "utf-8"))
# 构造附件，传送filePath制定文件
for filePath, fileName in zip(file_paths, file_names):
att = MIMEText(
open(filePath, "rb").read(), "base64", "utf-8")
att["Content-Type"] = "application/octet-stream"
# 邮件中显示文件名
att["Content-Disposition"] = "attachment; filename=" " + fileName + " ""
message.attach(att)
except Exception as e:
G_LOGGER.error(f"构造邮件发生错误,详情:{str(e)}")
return False
try:
smtp = smtplib.SMTP_SSL(self.__mail_host, self.__mail_port)
smtp.login(self.__mail_user, self.__mail_pass)
smtp.sendmail(sender, receiver, message.as_string())
smtp.quit()
except Exception as e:
G_LOGGER.error(f"发送邮件发生错误,详情:{str(e)}")
return False
return True
def coin_push(cls):
plugin_manager = DirectoryPluginManager()
coin_name = G_CFG.coin.coin_dict.get("name")
plugin_manager.load_plugins(coin_name)
coin = plugin_manager.get_plugins(coin_name)
if coin:
return CoinPush(coin)
else:
G_LOGGER.error(f"未安装{coin_name}的交易数据解析插件")
exit()
def _read_dir(self, plugin_list, directory):
"""
递归遍历插件目录
:return:
"""
try:
for f in os.listdir(directory):
sub_path = os.path.join(directory, f)
if os.path.isdir(sub_path):
self._read_dir(plugin_list, sub_path)
else:
if f.endswith(".py") and f != "__init__.py":
plugin_list.append((f[:-3], sub_path))
except OSError:
G_LOGGER.error("Failed to access: %s" % directory)
return plugin_list
def kafka_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
partition, offset = self.db.kafka.send(data)
G_LOGGER.info("Process:{} kafka push success, partition={}, offset={}, push_data={}".format(os.getpid(), partition, offset, data))
except Exception as e:
G_LOGGER.error("Process:{} kafka push failed, push_data={}, error={}".format(os.getpid(), data, str(e)))
def rocket_push(self, data):
try:
# 兼容处理eth_multi和其他币种两套数据模板
if "Type" in data:
# 把空字符串、None值转化为0，避免eval处理出错
if not data["Amount"]:
data["Amount"] = 0
if not data["Fee"]:
data["Fee"] = 0
if not data["BlockNumber"]:
data["BlockNumber"] = 0
if not data["Time"]:
data["Time"] = 0
# eval可以把小数型字符串和16进制字符串转换为小数和整数类型
if isinstance(data["Amount"], str):
data["Amount"] = eval(data["Amount"])
if isinstance(data["Fee"], str):
data["Fee"] = eval(data["Fee"])
if isinstance(data["BlockNumber"], str):
data["BlockNumber"] = eval(data["BlockNumber"])
if isinstance(data["Time"], str):
data["Time"] = eval(data["Time"])
# 根据每个币种精度，转化为整型数值
if data["Type"] in ["EOS"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4)
if data["Type"] in ["IOST"]:
data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8)
data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8)
# 去除小数位无效的0
data["Amount"] = int(data["Amount"])
data["Fee"] = int(data["Fee"])
data["BlockNumber"] = int(data["BlockNumber"])
data["Time"] = int(data["Time"]) * 1000
else:
data["time"] = eval(data["time"]) * 1000
data["value"] = eval(data["value"])
data["Fee"] = eval(data["Fee"])
msg_id = self.db.rocket.push_data(data)
G_LOGGER.info("success, msg_id={}, data={}".format(msg_id, data))
except Exception as e:
G_LOGGER.error("failed, data={}, error={}".format(data, str(e)))
def _wrap(*args, **kwargs):
def _loop():
func(*args, **kwargs)
mode = G_CFG.coin.coin_dict["mode"]
while True:
if mode == "prod":
try:
_loop()
except Exception:
error_info = traceback.format_exc()
G_LOGGER.error(f"出现异常, 请处理. {error_info}")
except KeyboardInterrupt:
G_LOGGER.info(f"{func.__name__}手动取消任务, 退出循环执行任务")
elif mode == "dev":
_loop()
# 每秒循环一次
time.sleep(5)
def get_address_from_scriptsig(self, script_sig):
"""
从解锁脚本中解析出地址,尚不支持隔离见证类型的解锁脚本(签名)
"""
address = None
try:
script_sig_byte = bytes.fromhex(script_sig)
if not script_sig_byte:
return None
n = script_sig_byte[0]
if n == 0:
address = self.gen_address_p2sh(script_sig_byte)
else:
address = self.gen_address_p2pk(script_sig_byte)
except Exception:
err_info = traceback.format_exc()
G_LOGGER.error("{} GET_ADDRESS_FROM_SCRIPTSIG ERROR：{}".format(self.coin_type, err_info))
return address
def parse_block(self, block_num):
"""
解析区块的交易列表
:return: 返回待推送信息列表
"""
block = self.rpc.get_block(block_num)
txs = block.get("tx", []) if block else []
push_list = []
# 回溯缓存推送
push_cache = []
if self.rollback and block_num:
cache_tag = self.push_cache.get(str(block_num))
push_cache = self.push_cache.pop(str(block_num)) if cache_tag else []
for tx in txs:
try:
sub_push = self.parse_tx(tx, push_cache=push_cache)
# 每一个tx都加上区块时间和区块高度
for push in sub_push:
push["BlockNumber"] = block["index"]
push["Time"] = block["time"]
push_list.extend(sub_push)
except Exception as e:
G_LOGGER.error(f"解析交易出现异常，详情：{e}")
len_of_cacahe = len(self.push_cache)
G_LOGGER.info('NEO push_cache_length: {}, rollback_count: {}'.format(len_of_cacahe, self.rollback_count))
# 非回溯类的才会缓存
if not self.rollback:
if len_of_cacahe <= (int(self.rollback_count) + 1):
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
else:
G_LOGGER.info('NEO push_cache_length: {} out of limit, reset push_cache_dict'.format(len_of_cacahe))
self.push_cache = {}
self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list])
G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys())))
return push_list
def push_main(self, save_redis=False):
"""
推送处理
"""
if not self.coin_name:
raise Exception("unknow coin name")
basic_info = self.db.get_basic(self.coin_name)
if not basic_info:
G_LOGGER.info("币种{}没有相关配置,请检查数据库配置".format(self.coin_name))
return True
save_num = current_height = basic_info["current_height"]
if save_redis:
newest_block_height = self.coin.newest_height()
diff_num = newest_block_height - current_height
else:
newest_block_height = self.coin.newest_height()
diff_num = newest_block_height - current_height
if diff_num <= 0:
G_LOGGER.info("最新高度{},已同步高度{},无需推送".format(newest_block_height, current_height))
return True
try:
if save_redis:
step = 100 # 最大处理100个块存入redis
diff_num = step if diff_num > step else diff_num
G_LOGGER.info("当前最新高度为{}, 数据库中的高度{}, 本次需要放入redis的块数为{}".format(newest_block_height, current_height, diff_num))
# 待推送的区块高度存入到redis队列中
for height in range(current_height + 1, current_height + diff_num + 1):
self.block_num = height
G_LOGGER.info("redis_save_pending_block: {}".format(self.block_num))
self.db.redis.save_pending_block(self.block_num)
else:
G_LOGGER.info("最新高度{},已同步高度{},本次需要同步块数{}".format(newest_block_height, current_height, diff_num))
self.push_sync(current_height + 1, diff_num, rollback_count=self.rollback_count)
except ForkError as e:
G_LOGGER.error("同步过程出现临时分叉, 即将回滚至高度{}重新推送".format(e.height))
save_num = e.height
except SyncError as e:
err_info = traceback.format_exc()
G_LOGGER.error("{}块同步过程出现异常, 请处理此块. 详情: {}".format(e.height, str(err_info)))
except Exception as e:
G_LOGGER.error("同步过程出现异常, 请处理此块. 详情{}".format(e))
except KeyboardInterrupt:
G_LOGGER.info("同步过程手动取消任务, 同步至高度: {}. 已保存".format(self.block_num))
save_num = self.block_num
else:
save_num = self.block_num
finally:
basic_info['current_height'] = save_num
basic_info['newest_height'] = newest_block_height
self.db.update_basic(basic_info)
G_LOGGER.info("push success ,已同步保存至区块高度: {}".format(save_num))
return True
def _send_data(self, method, url, _params=None, _data=None, _json=None, _cookies=None):
data = _params or _data or _json
G_LOGGER.debug('接口: {}, 方法: {}, 请求数据: {}'.format(url, method, data))
t1 = time.time()
retry_time = 0.001
while True:
try:
with self.session.request(method, url, params=_params, data=_data, json=_json, auth=self.auth,
headers=self.headers, timeout=self.timeout, cookies=_cookies) as rsp:
G_LOGGER.debug('返回数据: {}'.format(rsp.text))
if 300 > rsp.status_code >= 200:
result = rsp.text
if self.is_json:
try:
t2 = time.time()
net_time = int((t2 - t1) * 1000) / 1000
res = json.loads(rsp.text)
t3 = time.time()
json_time = int((t3 - t2) * 1000) / 1000
all_time = int((t3 - t1) * 1000) / 1000
# G_LOGGER.info("json success, {}, {}, length={}, net_time={}, json_time={}, all_time={}, ".format(url, data, len(rsp.text), net_time, json_time, all_time))
return res
except Exception:
# G_LOGGER.info("json fail, {}, {}, length={}, net_time={}".format(url, data, len(rsp.text), net_time))
pass
return result
elif rsp.status_code == 404:
G_LOGGER.warn("[404] URL不存在, 请检查URL".format(self.auth))
raise UrlError("[404] URL不存在, 请检查URL")
elif rsp.status_code == 401:
G_LOGGER.warn("[401] auth验证错误. auth: {}".format(self.auth))
raise RequestError(rsp.status_code, "[401] auth验证错误. auth: {}".format(self.auth))
elif rsp.status_code == 403:
G_LOGGER.warn("[403] 没有权限操作此URL. url: {}".format(url))
raise RequestError(rsp.status_code, "[403] 没有权限操作此URL. url: {}".format(url))
elif 500 > rsp.status_code >= 400:
G_LOGGER.warn("[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text))
raise RequestError(rsp.status_code,
"[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text),
rsp)
elif 599 > rsp.status_code >= 500:
G_LOGGER.warn("[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text))
raise RequestError(rsp.status_code,
"[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text),
rsp)
except (UrlError, RequestError):
raise
except (request_error.Timeout, request_error.ConnectionError) as e:
if retry_time < 10:
retry_time = min(max(retry_time, retry_time * 2), 10)
G_LOGGER.info("{}超时或被拒绝, retry_time={}, {} {}, 错误:{}".format(url, retry_time, method, data, str(e)))
continue
except request_error.InvalidURL:
G_LOGGER.error('请求URL无效, 请检查: {}'.format(url))
raise UrlError('URL无效.')
except Exception as e:
G_LOGGER.error('请求出现不可预知异常, 请处理. \r\n'
'请求url: {}\r\n'
'请求方法: {}\r\n'
'请求参数: params: {}, data: {}, json: {}\r\n'
'验证用户: {}\r\n'
'错误详情: {}'.format(url, method, _params, _data, _json, self.auth,
traceback.format_exc()))
raise RequestError(0, "请求出现不可预知异常, 请处理. 详情简要: {}".format(e))