def send_mail(self, receiver, title, msg, mail_type="plain", file_paths=[], file_names=[], image_paths=None): """发送邮件""" assert receiver and isinstance(receiver, list) sender = self.__mail_user mail_type = mail_type.lower() if mail_type in ["plain", "html"]: message = MIMEText(msg, mail_type, "utf-8") elif mail_type in ["file", "image"]: message = MIMEMultipart() else: return False try: message["From"] = Header(self.__mail_user, "utf-8") message["To"] = Header(",".join(receiver), "utf-8") message["Subject"] = Header(title, "utf-8") if mail_type in ["file", "image"]: # 邮件正文内容 if image_paths is not None: message.attach(MIMEText(msg, "html", "utf-8")) # 添加图片 if image_paths is not None: for index, image_path in enumerate(image_paths, 1): # 指定图片为当前目录 fp = open(image_path, "rb") msg_image = MIMEImage(fp.read()) fp.close() # 定义图片 ID,在 HTML 文本中引用 msg_image.add_header("Content-ID", "<image" + str(index) + ">") message.attach(msg_image) else: message.attach(MIMEText(msg, "plain", "utf-8")) # 构造附件,传送filePath制定文件 for filePath, fileName in zip(file_paths, file_names): att = MIMEText( open(filePath, "rb").read(), "base64", "utf-8") att["Content-Type"] = "application/octet-stream" # 邮件中显示文件名 att["Content-Disposition"] = "attachment; filename=" " + fileName + " "" message.attach(att) except Exception as e: G_LOGGER.error(f"构造邮件发生错误,详情:{str(e)}") return False try: smtp = smtplib.SMTP_SSL(self.__mail_host, self.__mail_port) smtp.login(self.__mail_user, self.__mail_pass) smtp.sendmail(sender, receiver, message.as_string()) smtp.quit() except Exception as e: G_LOGGER.error(f"发送邮件发生错误,详情:{str(e)}") return False return True
def coin_push(cls): plugin_manager = DirectoryPluginManager() coin_name = G_CFG.coin.coin_dict.get("name") plugin_manager.load_plugins(coin_name) coin = plugin_manager.get_plugins(coin_name) if coin: return CoinPush(coin) else: G_LOGGER.error(f"未安装{coin_name}的交易数据解析插件") exit()
def _read_dir(self, plugin_list, directory): """ 递归遍历插件目录 :return: """ try: for f in os.listdir(directory): sub_path = os.path.join(directory, f) if os.path.isdir(sub_path): self._read_dir(plugin_list, sub_path) else: if f.endswith(".py") and f != "__init__.py": plugin_list.append((f[:-3], sub_path)) except OSError: G_LOGGER.error("Failed to access: %s" % directory) return plugin_list
def kafka_push(self, data): try: # 兼容处理eth_multi和其他币种两套数据模板 if "Type" in data: # 把空字符串、None值转化为0,避免eval处理出错 if not data["Amount"]: data["Amount"] = 0 if not data["Fee"]: data["Fee"] = 0 if not data["BlockNumber"]: data["BlockNumber"] = 0 if not data["Time"]: data["Time"] = 0 # eval可以把小数型字符串和16进制字符串转换为小数和整数类型 if isinstance(data["Amount"], str): data["Amount"] = eval(data["Amount"]) if isinstance(data["Fee"], str): data["Fee"] = eval(data["Fee"]) if isinstance(data["BlockNumber"], str): data["BlockNumber"] = eval(data["BlockNumber"]) if isinstance(data["Time"], str): data["Time"] = eval(data["Time"]) # 根据每个币种精度,转化为整型数值 if data["Type"] in ["EOS"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4) if data["Type"] in ["IOST"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8) # 去除小数位无效的0 data["Amount"] = int(data["Amount"]) data["Fee"] = int(data["Fee"]) data["BlockNumber"] = int(data["BlockNumber"]) data["Time"] = int(data["Time"]) * 1000 else: data["time"] = eval(data["time"]) * 1000 data["value"] = eval(data["value"]) data["Fee"] = eval(data["Fee"]) partition, offset = self.db.kafka.send(data) G_LOGGER.info("Process:{} kafka push success, partition={}, offset={}, push_data={}".format(os.getpid(), partition, offset, data)) except Exception as e: G_LOGGER.error("Process:{} kafka push failed, push_data={}, error={}".format(os.getpid(), data, str(e)))
def rocket_push(self, data): try: # 兼容处理eth_multi和其他币种两套数据模板 if "Type" in data: # 把空字符串、None值转化为0,避免eval处理出错 if not data["Amount"]: data["Amount"] = 0 if not data["Fee"]: data["Fee"] = 0 if not data["BlockNumber"]: data["BlockNumber"] = 0 if not data["Time"]: data["Time"] = 0 # eval可以把小数型字符串和16进制字符串转换为小数和整数类型 if isinstance(data["Amount"], str): data["Amount"] = eval(data["Amount"]) if isinstance(data["Fee"], str): data["Fee"] = eval(data["Fee"]) if isinstance(data["BlockNumber"], str): data["BlockNumber"] = eval(data["BlockNumber"]) if isinstance(data["Time"], str): data["Time"] = eval(data["Time"]) # 根据每个币种精度,转化为整型数值 if data["Type"] in ["EOS"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 4) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 4) if data["Type"] in ["IOST"]: data["Amount"] = Decimal(str(data["Amount"])) * pow(10, 8) data["Fee"] = Decimal(str(data["Fee"])) * pow(10, 8) # 去除小数位无效的0 data["Amount"] = int(data["Amount"]) data["Fee"] = int(data["Fee"]) data["BlockNumber"] = int(data["BlockNumber"]) data["Time"] = int(data["Time"]) * 1000 else: data["time"] = eval(data["time"]) * 1000 data["value"] = eval(data["value"]) data["Fee"] = eval(data["Fee"]) msg_id = self.db.rocket.push_data(data) G_LOGGER.info("success, msg_id={}, data={}".format(msg_id, data)) except Exception as e: G_LOGGER.error("failed, data={}, error={}".format(data, str(e)))
def _wrap(*args, **kwargs): def _loop(): func(*args, **kwargs) mode = G_CFG.coin.coin_dict["mode"] while True: if mode == "prod": try: _loop() except Exception: error_info = traceback.format_exc() G_LOGGER.error(f"出现异常, 请处理. {error_info}") except KeyboardInterrupt: G_LOGGER.info(f"{func.__name__}手动取消任务, 退出循环执行任务") elif mode == "dev": _loop() # 每秒循环一次 time.sleep(5)
def get_address_from_scriptsig(self, script_sig): """ 从解锁脚本中解析出地址,尚不支持隔离见证类型的解锁脚本(签名) """ address = None try: script_sig_byte = bytes.fromhex(script_sig) if not script_sig_byte: return None n = script_sig_byte[0] if n == 0: address = self.gen_address_p2sh(script_sig_byte) else: address = self.gen_address_p2pk(script_sig_byte) except Exception: err_info = traceback.format_exc() G_LOGGER.error("{} GET_ADDRESS_FROM_SCRIPTSIG ERROR:{}".format(self.coin_type, err_info)) return address
def parse_block(self, block_num): """ 解析区块的交易列表 :return: 返回待推送信息列表 """ block = self.rpc.get_block(block_num) txs = block.get("tx", []) if block else [] push_list = [] # 回溯缓存推送 push_cache = [] if self.rollback and block_num: cache_tag = self.push_cache.get(str(block_num)) push_cache = self.push_cache.pop(str(block_num)) if cache_tag else [] for tx in txs: try: sub_push = self.parse_tx(tx, push_cache=push_cache) # 每一个tx都加上区块时间和区块高度 for push in sub_push: push["BlockNumber"] = block["index"] push["Time"] = block["time"] push_list.extend(sub_push) except Exception as e: G_LOGGER.error(f"解析交易出现异常,详情:{e}") len_of_cacahe = len(self.push_cache) G_LOGGER.info('NEO push_cache_length: {}, rollback_count: {}'.format(len_of_cacahe, self.rollback_count)) # 非回溯类的才会缓存 if not self.rollback: if len_of_cacahe <= (int(self.rollback_count) + 1): self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list]) else: G_LOGGER.info('NEO push_cache_length: {} out of limit, reset push_cache_dict'.format(len_of_cacahe)) self.push_cache = {} self.push_cache[str(block_num)] = set([tx['Txid'] for tx in push_list]) G_LOGGER.info("current push_cache_keys: {}".format(list(self.push_cache.keys()))) return push_list
def push_main(self, save_redis=False): """ 推送处理 """ if not self.coin_name: raise Exception("unknow coin name") basic_info = self.db.get_basic(self.coin_name) if not basic_info: G_LOGGER.info("币种{}没有相关配置,请检查数据库配置".format(self.coin_name)) return True save_num = current_height = basic_info["current_height"] if save_redis: newest_block_height = self.coin.newest_height() diff_num = newest_block_height - current_height else: newest_block_height = self.coin.newest_height() diff_num = newest_block_height - current_height if diff_num <= 0: G_LOGGER.info("最新高度{},已同步高度{},无需推送".format(newest_block_height, current_height)) return True try: if save_redis: step = 100 # 最大处理100个块存入redis diff_num = step if diff_num > step else diff_num G_LOGGER.info("当前最新高度为{}, 数据库中的高度{}, 本次需要放入redis的块数为{}".format(newest_block_height, current_height, diff_num)) # 待推送的区块高度存入到redis队列中 for height in range(current_height + 1, current_height + diff_num + 1): self.block_num = height G_LOGGER.info("redis_save_pending_block: {}".format(self.block_num)) self.db.redis.save_pending_block(self.block_num) else: G_LOGGER.info("最新高度{},已同步高度{},本次需要同步块数{}".format(newest_block_height, current_height, diff_num)) self.push_sync(current_height + 1, diff_num, rollback_count=self.rollback_count) except ForkError as e: G_LOGGER.error("同步过程出现临时分叉, 即将回滚至高度{}重新推送".format(e.height)) save_num = e.height except SyncError as e: err_info = traceback.format_exc() G_LOGGER.error("{}块同步过程出现异常, 请处理此块. 详情: {}".format(e.height, str(err_info))) except Exception as e: G_LOGGER.error("同步过程出现异常, 请处理此块. 详情{}".format(e)) except KeyboardInterrupt: G_LOGGER.info("同步过程手动取消任务, 同步至高度: {}. 已保存".format(self.block_num)) save_num = self.block_num else: save_num = self.block_num finally: basic_info['current_height'] = save_num basic_info['newest_height'] = newest_block_height self.db.update_basic(basic_info) G_LOGGER.info("push success ,已同步保存至区块高度: {}".format(save_num)) return True
def _send_data(self, method, url, _params=None, _data=None, _json=None, _cookies=None): data = _params or _data or _json G_LOGGER.debug('接口: {}, 方法: {}, 请求数据: {}'.format(url, method, data)) t1 = time.time() retry_time = 0.001 while True: try: with self.session.request(method, url, params=_params, data=_data, json=_json, auth=self.auth, headers=self.headers, timeout=self.timeout, cookies=_cookies) as rsp: G_LOGGER.debug('返回数据: {}'.format(rsp.text)) if 300 > rsp.status_code >= 200: result = rsp.text if self.is_json: try: t2 = time.time() net_time = int((t2 - t1) * 1000) / 1000 res = json.loads(rsp.text) t3 = time.time() json_time = int((t3 - t2) * 1000) / 1000 all_time = int((t3 - t1) * 1000) / 1000 # G_LOGGER.info("json success, {}, {}, length={}, net_time={}, json_time={}, all_time={}, ".format(url, data, len(rsp.text), net_time, json_time, all_time)) return res except Exception: # G_LOGGER.info("json fail, {}, {}, length={}, net_time={}".format(url, data, len(rsp.text), net_time)) pass return result elif rsp.status_code == 404: G_LOGGER.warn("[404] URL不存在, 请检查URL".format(self.auth)) raise UrlError("[404] URL不存在, 请检查URL") elif rsp.status_code == 401: G_LOGGER.warn("[401] auth验证错误. auth: {}".format(self.auth)) raise RequestError(rsp.status_code, "[401] auth验证错误. auth: {}".format(self.auth)) elif rsp.status_code == 403: G_LOGGER.warn("[403] 没有权限操作此URL. url: {}".format(url)) raise RequestError(rsp.status_code, "[403] 没有权限操作此URL. url: {}".format(url)) elif 500 > rsp.status_code >= 400: G_LOGGER.warn("[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text)) raise RequestError(rsp.status_code, "[{}] 客户端请求错误. 错误详情: {}".format(rsp.status_code, rsp.text), rsp) elif 599 > rsp.status_code >= 500: G_LOGGER.warn("[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text)) raise RequestError(rsp.status_code, "[{}] 服务器响应错误. 错误文本: {}".format(rsp.status_code, rsp.text), rsp) except (UrlError, RequestError): raise except (request_error.Timeout, request_error.ConnectionError) as e: if retry_time < 10: retry_time = min(max(retry_time, retry_time * 2), 10) G_LOGGER.info("{}超时或被拒绝, retry_time={}, {} {}, 错误:{}".format(url, retry_time, method, data, str(e))) continue except request_error.InvalidURL: G_LOGGER.error('请求URL无效, 请检查: {}'.format(url)) raise UrlError('URL无效.') except Exception as e: G_LOGGER.error('请求出现不可预知异常, 请处理. \r\n' '请求url: {}\r\n' '请求方法: {}\r\n' '请求参数: params: {}, data: {}, json: {}\r\n' '验证用户: {}\r\n' '错误详情: {}'.format(url, method, _params, _data, _json, self.auth, traceback.format_exc())) raise RequestError(0, "请求出现不可预知异常, 请处理. 详情简要: {}".format(e))