def redis_instance(self): """ 启动redis消费 :return: """ try: """获取订阅信息""" kafka_topic = "get_%s_queue" % self.task_choice_pattern.replace( ".", "_") """读取当前redis内数据长度""" queue_length = BusinessUtil.get_redis_llen_by_key(kafka_topic) """读取最大量""" consume_max_redis_read_length = BusiBaseConfService( ).search_key_return_value( (busi_config.CONSUME_MAX_REDIS_READ_LENGTH)) self.logger.info("单次最大处理量为:%s" % consume_max_redis_read_length) """读取redis列表""" result = BusinessUtil.get_redis_lpop_by_key_and_length( kafka_topic, queue_length, int(consume_max_redis_read_length)) """判空""" if not result: self.logger.info("【消费】未读取到有效数据,流程结束") return """处理消息""" self.processing_task(result) """处理结束""" self.logger.info("【消费】处理结束") except Exception as e: self.logger.error("【消费】启动消费异常,信息为:%s" % traceback.format_exc())
def search(self, task_code): """ 查询有效模板数据 :param task_code: :return: """ try: template_code = self.search_template_code(task_code) """未查到模板编码""" if not template_code: return False """查询redis""" template_redis_key = "%s_%s" % (template_code, busi_config.TEMPLATE_EXT_REDIS_KEY) result = BusinessUtil.get_redis_by_key(template_redis_key) if result: return result _db = MySqlHelper() result = _db.fetch_all(template_sql.search, template_code) """如果未查到值""" if not result: return False """redis入库""" BusinessUtil.set_redis_by_key(template_redis_key, result, busi_config.REDIS_CACHE_TIME) return result except Exception as e: self.logger.error("【查询有效服务器】查询异常信息为:%s" % traceback.format_exc()) return False
def parseEML(self, item): """ 解析eml后缀文件 获取mobile、batchCode、subBatchCode、templateCode :param item: :return: """ result = { "mobile": "", "batchCode": "", "subBatchCode": "", "templateCode": "", "email": "" } eml_path = item.get("resultAttach", "") """文件是否存在 或后缀名不正确""" if not eml_path or eml_path.find("eml") < 0: return result """解析邮件内容""" parse_eml_result = BusinessUtil.parse_eml(eml_path) """获取发送账号""" result.update({"email": parse_eml_result.get("To")}) """获取文件内容""" parse_content = parse_eml_result.get("content") """解析数据""" parse_text_result = BusinessUtil.parse_eml_text_param(parse_content) """更新""" result.update(parse_text_result) return result
def search_key(self, param): """ 查询业务配置 通过key值 :return: """ try: if not param: self.logger.info("【查询业务配置】请求参数为为空,流程结束") return False """查询redis""" result = BusinessUtil.get_redis_by_key( busi_config.BUSI_BASE_CONF_REDIS_KEY) if not result: """查询数据库""" _db = MySqlHelper() result = _db.fetch_all(busi_base_conf_sql.search, None) else: """转化数据""" result = json.loads(result) """存储到redis""" BusinessUtil.set_redis_by_key(busi_config.BUSI_BASE_CONF_REDIS_KEY, result, busi_config.REDIS_CACHE_TIME) """遍历结果""" busi_base_list = [] for item in result: if item.get("busiKey") != param: continue busi_base_list.append(item) break if not busi_base_list: return False return busi_base_list[0] except Exception as e: self.logger.error("【查询业务配置】查询异常信息为:%s" % traceback.format_exc()) return False
def search_lock_domain_name(self): """ 查询域名状态 :return: """ try: """查询redis""" result = BusinessUtil.get_redis_by_key( busi_config.LOCK_ACCOUNT_REDIS_KEY) if result: return result """查询数据库""" _db = MySqlHelper() result = _db.fetch_one( send_email_account_sql.search_lock_domain_name) if not result: return False """获取域名""" domain_name = result.get("domainName", "") """如果不为空 则添加到redis""" if domain_name: BusinessUtil.set_redis_by_key( busi_config.LOCK_ACCOUNT_REDIS_KEY, domain_name, busi_config.REDIS_CACHE_TIME) return domain_name except Exception as e: self.logger.error("【查询发送邮箱账号】查询异常信息为:%s" % traceback.format_exc()) return False
def get_send_email_content(result): """ 获取邮件内容 :param result: :return: """ """获取文件路径""" templatePath = result.get("templatePath") """读取文件内容""" file_content = BusinessUtil.read_file(templatePath) if not file_content: raise Exception("读取文件内容异常") """替换内容参数""" randomNum = ''.join( random.sample([ 'z', 'y', 'x', 'w', 'v', 'u', 't', 's', 'r', 'q', 'p', 'o', 'n', 'm', 'l', 'k', 'j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a' ], 10)) result["randomNum"] = randomNum """邮箱加密""" email = result.get("email") secret_email = BusinessUtil.encrypt_aes("adsawdws", email) result["secret_email"] = secret_email file_content = replace_file_content(file_content, result) return file_content
def instance(self): """ :return: """ self.logger.info( "---------------------------------浮屠长生 开始提取数据---------------------------------" ) batch_code = None try: """生成主批次号""" batch_code = BusinessUtil.get_uniqu_time() self.logger.info("【提取数据】生成当前主批次号为:%s" % batch_code) """提取数据""" result = self.get_primamary_list(batch_code) """获取待发送邮箱,并集""" marketing_email = BusiBaseConfService().search_key_return_value( (busi_config.MARKETING_EMAIL_KEY), "139.com") result = BusinessUtil.convert_data(result, marketing_email, False) """数据处理""" self.observer(result) except Exception as e: self.logger.error("【数据提取】数据提取异常,回滚批次为:%s,异常信息为:%s" % (batch_code, traceback.format_exc())) self.roll_back(batch_code) self.logger.info( "---------------------------------浮屠长生 提取数据结束---------------------------------" )
def parseHtml(self, item): """ 解析退信内容 获取mobile、batchCode、subBatchCode、templateCode :param item: :return: """ result = { "sendResult": item.get("sendSubject", ""), "mobile": "", "batchCode": "", "subBatchCode": "", "templateCode": "", "email": "" } html_path = item.get("resultHtml", "") """文件是否存在 或后缀名不正确""" if not html_path or html_path.find("html") < 0: return result """读取内容""" file_info = BusinessUtil.read_file(html_path) """判断内容是否为空""" if not file_info: return result """解析数据""" parse_text_result = BusinessUtil.parse_eml_text_param(file_info) """更新""" result.update(parse_text_result) return result
def update(self, param=None): """ 更新数据 :param param: :return: """ _db = None try: BusinessUtil.delete_redis_by_key( busi_config.SERVER_CONFIG_REDIS_KEY) if param is None: self.logger.info("【更新服务器】请求参数为空,流程结束") return False _db = MySqlHelper() _db.begin() count = _db.update(server_conf_sql.update, param) _db.end() return count except Exception as e: if _db: _db.isEnd = 2 _db.end() self.logger.error("【更新服务器】更新异常,事务回滚,异常信息为:%s" % traceback.format_exc()) return False
def get_send_msg(smtp_server, result): """ 处理发送消息 :param smtp_server: :param result: :return: """ """修改标题""" templateFrom = BusinessUtil.replace_email_from(result.get("templateFrom")) fr = "%s<%s>" % (templateFrom, result.get("userName")) to = result.get("templateTo") if result.get("templateTo") else result.get( "email") email_subject = result.get("templateSubject") templateContent = result.get("templateContent") """获取邮件内容""" fileContent = get_send_email_content(result) #msg = smtp_server.html_attach_msg(fileContent) msg = smtp_server.text_html_attach_msg(templateContent, fileContent) """获取头部信息""" head_msg = smtp_server.get_header(fr, to, email_subject) """更新内容信息""" msg['From'] = head_msg.get('From') msg['To'] = head_msg.get('To') msg['Subject'] = head_msg.get('Subject') msg['Date'] = head_msg.get('Date') msg['Message-ID'] = head_msg.get('Message-ID') return msg
def verify_email(self, need_verify_email, verify_email, time_out=10): """ 验证邮箱 :param need_verify_email: 需要验证的邮箱(不确定是否真实存在) :param verify_email: 验证邮箱(真实存在的) :param time_out: :return: """ if not need_verify_email or not verify_email: self.logger.info("请求参数为空,请确认") return False """邮箱格式校验""" format_email = BusinessUtil.search_send_email(need_verify_email) if not format_email: self.logger.info("邮箱格式错误,请确认") return False """分割邮箱""" name, host = need_verify_email.split('@') """判断邮箱校验类型""" if host in HTTP_CHECK_HOST_163: result = self.verify_email_163(need_verify_email) else: result = self.check_by_smtp(need_verify_email, verify_email, time_out) return result
def processing_task(self, key, queue_length): """ 按任务优先级处理数据 :param key: 任务队列key值 :param queue_length: 当前队列长度 :return: """ """读取最大量""" max_redis_read_length = BusiBaseConfService().search_key_return_value( (busi_config.MAX_REDIS_READ_LENGTH)) self.logger.info("单次最大处理量为:%s" % max_redis_read_length) """读取redis列表""" result = BusinessUtil.get_redis_lpop_by_key_and_length( key, queue_length, int(max_redis_read_length)) """判空""" if not result: return """最大线程数""" thread_max_workers = BusiBaseConfService().search_key_return_value( (busi_config.THREAD_MAX_WORKERS)) """按发送邮箱名称分组""" task_dict = {} with ThreadPoolExecutor( max_workers=int(thread_max_workers)) as executor: future_list = [] """循环""" for item in result: if item: future = executor.submit(self.task_thread_submit, item, key, task_dict) future_list.append(future) """等待子线程结束""" wait(future_list, timeout=10) """处理队列数据""" self.deal_with_result_data(task_dict)
def update(self): """ 限速检查 :return: """ """读取账号""" self.logger.info("【发送邮件】1、限速检查") result = self.subject.email_item user_name = result.get("userName") """设置令牌桶""" self.set_token_bucket(user_name) """计算令牌""" BusinessUtil.lock_item(user_name, user_name, consume_token, 300000, result) consume_result = result.get("is_smtp") if not consume_result: raise BusinessExcetion("T01", "触发限速规则")
def __init__(self): self.logger = UserLogger.getlog() """当前服务器IP地址""" self.public_ip = BusinessUtil.get_public_ip() """设置全局变量""" global_argument.set_value(self.public_ip) self.task_code = None self.relId = None
def consume_token(*keysValue): """ 消费队列 :param keysValue: :return: """ item = keysValue[0] result = BusinessUtil.get_redis_by_key(item.get("userName")) if type(result) == str: result = json.loads(result) """读取令牌""" token_result = BusinessUtil.consume_token(result) if not token_result: item["is_smtp"] = False else: """设置新值""" item["is_smtp"] = BusinessUtil.set_redis_reset_time_ex(item.get("userName"), token_result)
def __init__(self): self.logger = UserLogger.getlog() """当前服务器IP地址""" self.public_ip = BusinessUtil.get_public_ip() """设置全局变量""" global_argument.set_value(self.public_ip) """选择消费机制""" self.task_choice_pattern = self.public_ip
def search_key(self, param): """ 查询黑名单 通过content :return: """ try: if not param: return False """从redis读取字段""" result = BusinessUtil.get_redis_by_key(param) if result: return result """从数据库内查询""" _db = MySqlHelper() result = _db.fetch_all(black_roll_call_sql.search_key) """未查到数据""" if not result: return False black_dict = {} """循环遍历数据""" for item in result: if not item: continue """生成分组""" item_key = BusinessUtil.get_verify_key(item.get("content")) if not item_key: continue if item_key in black_dict: black_dict[item_key].append(item.get("content")) else: black_dict.setdefault(item_key, []) black_dict[item_key].append(item.get("content")) """存入redis""" result_str = None for key in list(black_dict.keys()): item = black_dict[key] item_list = list(set(item)) item_str = ",".join(item_list) if key == param: result_str = item_str BusinessUtil.set_redis_reset_time_ex(key, item_str) return result_str except Exception as e: self.logger.error("【查询黑名单】查询异常信息为:%s" % traceback.format_exc()) return False
def __init__(self): """ 初始化 """ self.logger = UserLogger.getlog() """当前服务器IP地址""" self.public_ip = BusinessUtil.get_public_ip() """设置全局变量""" global_argument.set_value(self.public_ip)
def get_task_list(self): """ 获取任务列表 :return: """ pattern = "%s_%s_pagoda_[0-9]*" % (self.task_code, self.relId) result = BusinessUtil.get_redis_name_by_keys(pattern) self.logger.info("【数据提取】读取到的任务列表为:%s" % result) return result
def parseText(self, item): """ 解析退信内容 获取sendResult :param item: :return: """ result = {"sendResult": ""} text_path = item.get("resultText", "") """文件是否存在""" if not text_path: return result """读取文件内容""" text_content = BusinessUtil.read_file(text_path) """解析""" sendResult = BusinessUtil.search_send_result(text_content) """结果不为空""" if sendResult: result["sendResult"] = sendResult[0].strip() return result
def get_task_list(self): """ 获取数据列表 (例:201905241427196394251011058_1_pagoda_1,201905241427196394251011058_2_pagoda_2) :return: """ def sort_key(key): taskCode, server_index, name, index = key.split("_") return index current_pattern = "%s_%s_pagoda_[0-9]*" % (self.task_code, self.relId) other_pattern = "%s_[^%s]*_pagoda_[0-9]*" % (self.task_code, self.relId) current_list = BusinessUtil.get_redis_name_by_keys(current_pattern) other_list = BusinessUtil.get_redis_name_by_keys(other_pattern) """排序""" current_list.sort(key=sort_key) other_list.sort(key=sort_key) """数组合并""" result = current_list + other_list return result
def search(self, param=None): """ 查询有效服务器数据 优先查询redis内数据,未查得后查询数据库数据并更新到redis内 :param param: :return: result """ try: result = BusinessUtil.get_redis_by_key( busi_config.SERVER_CONFIG_REDIS_KEY) if result: return json.loads(result) _db = MySqlHelper() result = _db.fetch_all(server_conf_sql.search, param) if not result: self.logger.info("【查询有效服务器】响应结果为空,结束流程") return False BusinessUtil.set_redis_by_key(busi_config.SERVER_CONFIG_REDIS_KEY, result, busi_config.REDIS_CACHE_TIME) return result except Exception as e: self.logger.error("【查询有效服务器】查询异常信息为:%s" % traceback.format_exc()) return False
def search_valid_account(self, server_ip): """ 查询发送有效账号(含剩余数量) 缓存5分钟 可能存在账号多发情况 :param server_ip: :return: """ try: if not server_ip: self.logger.info("【查询发送邮箱账号】传入参数有误,请确认") return False """选取发送账号""" valid_account_key = "%s_%s" % (server_ip.replace( ".", "_"), busi_config.VALID_ACCOUNT_REDIS_KEY) """查询redis""" result = BusinessUtil.get_redis_by_key(valid_account_key) if not result: """查询数据库""" _db = MySqlHelper() """设置当前日期""" timeStr = str( time.strftime('%Y-%m-%d', time.localtime(time.time()))) start = "%s 00:00:00" % timeStr end = "%s 23:59:59" % timeStr result = _db.fetch_all( send_email_account_sql.search_valid_account, (server_ip, start, end)) else: """转化数据""" result = json.loads(result) """判断非空""" if not result: return False BusinessUtil.set_redis_by_key(valid_account_key, result, busi_config.REDIS_CACHE_TIME) return result except Exception as e: self.logger.error("【查询发送邮箱账号】查询异常信息为:%s" % traceback.format_exc()) return False
def process_message(self, account, msg, resultText): """ 解析邮件正文 :param account: :param msg: :param resultText: :return: """ try: if msg.is_multipart(): for part in msg.get_payload(): self.process_message(account, part, resultText) else: content_type = msg.get_content_type() if content_type == 'text/plain': content = msg.get_payload(decode=True) # 要检测文本编码: charset = msg.get_content_charset(msg) if charset: content = content.decode(charset) file_path = BusinessUtil.get_uniq_pop_file_name(account, content, 'txt') BusinessUtil.save_file(file_path, content, 'w+') resultText["resultText"] = file_path elif content_type == 'text/html': content = msg.get_payload(decode=True) # 要检测文本编码: charset = msg.get_content_charset(msg) if charset: content = content.decode(charset) file_path = BusinessUtil.get_uniq_pop_file_name(account, content, 'html') BusinessUtil.save_file(file_path, content, 'w+') resultText["resultHtml"] = file_path else: # 不是文本,作为附件处理: file_name = self.get_file_name(msg) # 获取后缀 suffix = file_name.split(".")[1] content = msg.get_payload(decode=True) file_path = BusinessUtil.get_uniq_pop_file_name(account, content, suffix, 'attach') BusinessUtil.save_file(file_path, content, 'wb') resultText["resultAttach"] = file_path except Exception as e: self.logger.info("解析邮件失败:%s" % (traceback.format_exc()))
def search_template_code(self, task_code): try: template_redis_key = "%s_%s" % (task_code, busi_config.TEMPLATE_REDIS_KEY) result = BusinessUtil.get_redis_by_key(template_redis_key) if not result: _db = MySqlHelper() result = _db.fetch_all(template_sql.search_template_code, task_code) if not result: return False """redis入库""" BusinessUtil.set_redis_by_key(template_redis_key, result, busi_config.REDIS_CACHE_TIME) else: if type(result) == str: result = json.loads(result) """筛选templateCode""" item = random.choice(result) return item.get("templateCode") except Exception as e: self.logger.error("【查询有效服务器】查询异常信息为:%s" % traceback.format_exc()) return False
def verify_email_is_in_black_roll_call(self, email): """ 校验是否在黑名单内 :param email: :return: """ """获取redis key值""" redis_key = BusinessUtil.get_verify_key(email) """如果未获取到值""" if not redis_key: return False """通过key值查询redis""" redis_verify_email_value = BlackRollCallService().search_key(redis_key) """未找到值""" if not redis_verify_email_value: return False """从黑名单内查到字段""" if redis_verify_email_value.find(email) > -1: return True return False
def instance(self): """ 调度处理任务队列数据 :return: """ try: """获取休眠时间""" product_sleep_time = BusiBaseConfService().search_key_return_value( (busi_config.PRODUCT_SLEEP_TIME)) sleep_time = product_sleep_time.split("|") time_out = random.randint(int(sleep_time[0]), int(sleep_time[1])) """判断是否存在发送中的任务 不存在 则直接返回""" if not self.search_task_code_by_server_ip(): self.logger.info("【服务进程】不存在发送中的任务,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """读取任务队列""" task_name_list = self.get_task_list() self.logger.info("【服务进程】任务列表为:%s" % task_name_list) if not task_name_list: self.logger.info("【服务进程】任务列表为空,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """判断账号""" if not self.check_valid_account(): self.logger.info("【服务进程】无有效账号,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """按任务优先级读取""" for task in task_name_list: queue_length = BusinessUtil.get_redis_llen_by_key(task) """队列为空""" if not queue_length or queue_length == 0: continue """出队列""" self.processing_task(task, queue_length) break except Exception as e: self.logger.error("【服务进程】服务进程异常,异常信息为:%s" % traceback.format_exc())
def insert_update_more(self, result): """ 添加或更新数据 :param result: :return: """ _db = None try: if not result: self.logger.info("【添加/更新营销数据】请求参数为为空,流程结束") """查询数据""" _db = MySqlHelper() for param in result: param["subBatchCode"] = BusinessUtil.get_uniqu_time() count = _db.count(marketing_data_sql.count, (param.get("batchCode"), param.get("mobile"), param.get("email"))) if count > 0: result = _db.update( marketing_data_sql.update_insert, (param.get("subBatchCode"), int(param.get("status", "0")), param.get("batchCode"), param.get("mobile"), param.get("email"))) else: result = _db.insert_one( marketing_data_sql.insert, (param.get("originalBatchCode"), param.get("batchCode"), param.get("subBatchCode"), param.get("mobile"), param.get("email"), int(param.get("status", "0")))) _db.end() return result except Exception as e: if _db: _db.isEnd = 2 _db.end() self.logger.error("【添加/更新营销数据】保存异常信息为:%s" % traceback.format_exc()) return False
def set_token_bucket(self, key): """ 设置令牌桶 (后续考虑到将信息添加到数据库) :return: """ BusinessUtil.set_token_bucket(key)