def redis_instance(self):
"""
启动redis消费
:return:
"""
try:
"""获取订阅信息"""
kafka_topic = "get_%s_queue" % self.task_choice_pattern.replace(
".", "_")
"""读取当前redis内数据长度"""
queue_length = BusinessUtil.get_redis_llen_by_key(kafka_topic)
"""读取最大量"""
consume_max_redis_read_length = BusiBaseConfService(
).search_key_return_value(
(busi_config.CONSUME_MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % consume_max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
kafka_topic, queue_length, int(consume_max_redis_read_length))
"""判空"""
if not result:
self.logger.info("【消费】未读取到有效数据，流程结束")
return
"""处理消息"""
self.processing_task(result)
"""处理结束"""
self.logger.info("【消费】处理结束")
except Exception as e:
self.logger.error("【消费】启动消费异常，信息为：%s" % traceback.format_exc())
def search(self, task_code):
"""
查询有效模板数据
:param task_code:
:return:
"""
try:
template_code = self.search_template_code(task_code)
"""未查到模板编码"""
if not template_code:
return False
"""查询redis"""
template_redis_key = "%s_%s" % (template_code,
busi_config.TEMPLATE_EXT_REDIS_KEY)
result = BusinessUtil.get_redis_by_key(template_redis_key)
if result:
return result
_db = MySqlHelper()
result = _db.fetch_all(template_sql.search, template_code)
"""如果未查到值"""
if not result:
return False
"""redis入库"""
BusinessUtil.set_redis_by_key(template_redis_key, result,
busi_config.REDIS_CACHE_TIME)
return result
except Exception as e:
self.logger.error("【查询有效服务器】查询异常信息为：%s" % traceback.format_exc())
return False
def parseEML(self, item):
"""
解析eml后缀文件 获取mobile、batchCode、subBatchCode、templateCode
:param item:
:return:
"""
result = {
"mobile": "",
"batchCode": "",
"subBatchCode": "",
"templateCode": "",
"email": ""
}
eml_path = item.get("resultAttach", "")
"""文件是否存在 或后缀名不正确"""
if not eml_path or eml_path.find("eml") < 0:
return result
"""解析邮件内容"""
parse_eml_result = BusinessUtil.parse_eml(eml_path)
"""获取发送账号"""
result.update({"email": parse_eml_result.get("To")})
"""获取文件内容"""
parse_content = parse_eml_result.get("content")
"""解析数据"""
parse_text_result = BusinessUtil.parse_eml_text_param(parse_content)
"""更新"""
result.update(parse_text_result)
return result
def search_key(self, param):
"""
查询业务配置 通过key值
:return:
"""
try:
if not param:
self.logger.info("【查询业务配置】请求参数为为空，流程结束")
return False
"""查询redis"""
result = BusinessUtil.get_redis_by_key(
busi_config.BUSI_BASE_CONF_REDIS_KEY)
if not result:
"""查询数据库"""
_db = MySqlHelper()
result = _db.fetch_all(busi_base_conf_sql.search, None)
else:
"""转化数据"""
result = json.loads(result)
"""存储到redis"""
BusinessUtil.set_redis_by_key(busi_config.BUSI_BASE_CONF_REDIS_KEY,
result, busi_config.REDIS_CACHE_TIME)
"""遍历结果"""
busi_base_list = []
for item in result:
if item.get("busiKey") != param:
continue
busi_base_list.append(item)
break
if not busi_base_list:
return False
return busi_base_list[0]
except Exception as e:
self.logger.error("【查询业务配置】查询异常信息为：%s" % traceback.format_exc())
return False
def search_lock_domain_name(self):
"""
查询域名状态
:return:
"""
try:
"""查询redis"""
result = BusinessUtil.get_redis_by_key(
busi_config.LOCK_ACCOUNT_REDIS_KEY)
if result:
return result
"""查询数据库"""
_db = MySqlHelper()
result = _db.fetch_one(
send_email_account_sql.search_lock_domain_name)
if not result:
return False
"""获取域名"""
domain_name = result.get("domainName", "")
"""如果不为空 则添加到redis"""
if domain_name:
BusinessUtil.set_redis_by_key(
busi_config.LOCK_ACCOUNT_REDIS_KEY, domain_name,
busi_config.REDIS_CACHE_TIME)
return domain_name
except Exception as e:
self.logger.error("【查询发送邮箱账号】查询异常信息为：%s" % traceback.format_exc())
return False
def get_send_email_content(result):
"""
获取邮件内容
:param result:
:return:
"""
"""获取文件路径"""
templatePath = result.get("templatePath")
"""读取文件内容"""
file_content = BusinessUtil.read_file(templatePath)
if not file_content:
raise Exception("读取文件内容异常")
"""替换内容参数"""
randomNum = ''.join(
random.sample([
'z', 'y', 'x', 'w', 'v', 'u', 't', 's', 'r', 'q', 'p', 'o', 'n',
'm', 'l', 'k', 'j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'
], 10))
result["randomNum"] = randomNum
"""邮箱加密"""
email = result.get("email")
secret_email = BusinessUtil.encrypt_aes("adsawdws", email)
result["secret_email"] = secret_email
file_content = replace_file_content(file_content, result)
return file_content
def instance(self):
"""
:return:
"""
self.logger.info(
"---------------------------------浮屠长生 开始提取数据---------------------------------"
)
batch_code = None
try:
"""生成主批次号"""
batch_code = BusinessUtil.get_uniqu_time()
self.logger.info("【提取数据】生成当前主批次号为：%s" % batch_code)
"""提取数据"""
result = self.get_primamary_list(batch_code)
"""获取待发送邮箱，并集"""
marketing_email = BusiBaseConfService().search_key_return_value(
(busi_config.MARKETING_EMAIL_KEY), "139.com")
result = BusinessUtil.convert_data(result, marketing_email, False)
"""数据处理"""
self.observer(result)
except Exception as e:
self.logger.error("【数据提取】数据提取异常，回滚批次为：%s，异常信息为：%s" %
(batch_code, traceback.format_exc()))
self.roll_back(batch_code)
self.logger.info(
"---------------------------------浮屠长生 提取数据结束---------------------------------"
)
def parseHtml(self, item):
"""
解析退信内容 获取mobile、batchCode、subBatchCode、templateCode
:param item:
:return:
"""
result = {
"sendResult": item.get("sendSubject", ""),
"mobile": "",
"batchCode": "",
"subBatchCode": "",
"templateCode": "",
"email": ""
}
html_path = item.get("resultHtml", "")
"""文件是否存在 或后缀名不正确"""
if not html_path or html_path.find("html") < 0:
return result
"""读取内容"""
file_info = BusinessUtil.read_file(html_path)
"""判断内容是否为空"""
if not file_info:
return result
"""解析数据"""
parse_text_result = BusinessUtil.parse_eml_text_param(file_info)
"""更新"""
result.update(parse_text_result)
return result
def update(self, param=None):
"""
更新数据
:param param:
:return:
"""
_db = None
try:
BusinessUtil.delete_redis_by_key(
busi_config.SERVER_CONFIG_REDIS_KEY)
if param is None:
self.logger.info("【更新服务器】请求参数为空，流程结束")
return False
_db = MySqlHelper()
_db.begin()
count = _db.update(server_conf_sql.update, param)
_db.end()
return count
except Exception as e:
if _db:
_db.isEnd = 2
_db.end()
self.logger.error("【更新服务器】更新异常，事务回滚，异常信息为：%s" %
traceback.format_exc())
return False
def get_send_msg(smtp_server, result):
"""
处理发送消息
:param smtp_server:
:param result:
:return:
"""
"""修改标题"""
templateFrom = BusinessUtil.replace_email_from(result.get("templateFrom"))
fr = "%s<%s>" % (templateFrom, result.get("userName"))
to = result.get("templateTo") if result.get("templateTo") else result.get(
"email")
email_subject = result.get("templateSubject")
templateContent = result.get("templateContent")
"""获取邮件内容"""
fileContent = get_send_email_content(result)
#msg = smtp_server.html_attach_msg(fileContent)
msg = smtp_server.text_html_attach_msg(templateContent, fileContent)
"""获取头部信息"""
head_msg = smtp_server.get_header(fr, to, email_subject)
"""更新内容信息"""
msg['From'] = head_msg.get('From')
msg['To'] = head_msg.get('To')
msg['Subject'] = head_msg.get('Subject')
msg['Date'] = head_msg.get('Date')
msg['Message-ID'] = head_msg.get('Message-ID')
return msg
def verify_email(self, need_verify_email, verify_email, time_out=10):
"""
验证邮箱
:param need_verify_email: 需要验证的邮箱(不确定是否真实存在)
:param verify_email: 验证邮箱(真实存在的)
:param time_out:
:return:
"""
if not need_verify_email or not verify_email:
self.logger.info("请求参数为空，请确认")
return False
"""邮箱格式校验"""
format_email = BusinessUtil.search_send_email(need_verify_email)
if not format_email:
self.logger.info("邮箱格式错误，请确认")
return False
"""分割邮箱"""
name, host = need_verify_email.split('@')
"""判断邮箱校验类型"""
if host in HTTP_CHECK_HOST_163:
result = self.verify_email_163(need_verify_email)
else:
result = self.check_by_smtp(need_verify_email, verify_email,
time_out)
return result
def processing_task(self, key, queue_length):
"""
按任务优先级处理数据
:param key: 任务队列key值
:param queue_length: 当前队列长度
:return:
"""
"""读取最大量"""
max_redis_read_length = BusiBaseConfService().search_key_return_value(
(busi_config.MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
key, queue_length, int(max_redis_read_length))
"""判空"""
if not result:
return
"""最大线程数"""
thread_max_workers = BusiBaseConfService().search_key_return_value(
(busi_config.THREAD_MAX_WORKERS))
"""按发送邮箱名称分组"""
task_dict = {}
with ThreadPoolExecutor(
max_workers=int(thread_max_workers)) as executor:
future_list = []
"""循环"""
for item in result:
if item:
future = executor.submit(self.task_thread_submit, item,
key, task_dict)
future_list.append(future)
"""等待子线程结束"""
wait(future_list, timeout=10)
"""处理队列数据"""
self.deal_with_result_data(task_dict)
def update(self):
"""
限速检查
:return:
"""
"""读取账号"""
self.logger.info("【发送邮件】1、限速检查")
result = self.subject.email_item
user_name = result.get("userName")
"""设置令牌桶"""
self.set_token_bucket(user_name)
"""计算令牌"""
BusinessUtil.lock_item(user_name, user_name, consume_token, 300000, result)
consume_result = result.get("is_smtp")
if not consume_result:
raise BusinessExcetion("T01", "触发限速规则")
def __init__(self):
self.logger = UserLogger.getlog()
"""当前服务器IP地址"""
self.public_ip = BusinessUtil.get_public_ip()
"""设置全局变量"""
global_argument.set_value(self.public_ip)
self.task_code = None
self.relId = None
def consume_token(*keysValue):
"""
消费队列
:param keysValue:
:return:
"""
item = keysValue[0]
result = BusinessUtil.get_redis_by_key(item.get("userName"))
if type(result) == str:
result = json.loads(result)
"""读取令牌"""
token_result = BusinessUtil.consume_token(result)
if not token_result:
item["is_smtp"] = False
else:
"""设置新值"""
item["is_smtp"] = BusinessUtil.set_redis_reset_time_ex(item.get("userName"), token_result)
def __init__(self):
self.logger = UserLogger.getlog()
"""当前服务器IP地址"""
self.public_ip = BusinessUtil.get_public_ip()
"""设置全局变量"""
global_argument.set_value(self.public_ip)
"""选择消费机制"""
self.task_choice_pattern = self.public_ip
def search_key(self, param):
"""
查询黑名单 通过content
:return:
"""
try:
if not param:
return False
"""从redis读取字段"""
result = BusinessUtil.get_redis_by_key(param)
if result:
return result
"""从数据库内查询"""
_db = MySqlHelper()
result = _db.fetch_all(black_roll_call_sql.search_key)
"""未查到数据"""
if not result:
return False
black_dict = {}
"""循环遍历数据"""
for item in result:
if not item:
continue
"""生成分组"""
item_key = BusinessUtil.get_verify_key(item.get("content"))
if not item_key:
continue
if item_key in black_dict:
black_dict[item_key].append(item.get("content"))
else:
black_dict.setdefault(item_key, [])
black_dict[item_key].append(item.get("content"))
"""存入redis"""
result_str = None
for key in list(black_dict.keys()):
item = black_dict[key]
item_list = list(set(item))
item_str = ",".join(item_list)
if key == param:
result_str = item_str
BusinessUtil.set_redis_reset_time_ex(key, item_str)
return result_str
except Exception as e:
self.logger.error("【查询黑名单】查询异常信息为：%s" % traceback.format_exc())
return False
def __init__(self):
"""
初始化
"""
self.logger = UserLogger.getlog()
"""当前服务器IP地址"""
self.public_ip = BusinessUtil.get_public_ip()
"""设置全局变量"""
global_argument.set_value(self.public_ip)
def get_task_list(self):
"""
获取任务列表
:return:
"""
pattern = "%s_%s_pagoda_[0-9]*" % (self.task_code, self.relId)
result = BusinessUtil.get_redis_name_by_keys(pattern)
self.logger.info("【数据提取】读取到的任务列表为：%s" % result)
return result
def parseText(self, item):
"""
解析退信内容 获取sendResult
:param item:
:return:
"""
result = {"sendResult": ""}
text_path = item.get("resultText", "")
"""文件是否存在"""
if not text_path:
return result
"""读取文件内容"""
text_content = BusinessUtil.read_file(text_path)
"""解析"""
sendResult = BusinessUtil.search_send_result(text_content)
"""结果不为空"""
if sendResult:
result["sendResult"] = sendResult[0].strip()
return result
def get_task_list(self):
"""
获取数据列表 (例：201905241427196394251011058_1_pagoda_1,201905241427196394251011058_2_pagoda_2)
:return:
"""
def sort_key(key):
taskCode, server_index, name, index = key.split("_")
return index
current_pattern = "%s_%s_pagoda_[0-9]*" % (self.task_code, self.relId)
other_pattern = "%s_[^%s]*_pagoda_[0-9]*" % (self.task_code,
self.relId)
current_list = BusinessUtil.get_redis_name_by_keys(current_pattern)
other_list = BusinessUtil.get_redis_name_by_keys(other_pattern)
"""排序"""
current_list.sort(key=sort_key)
other_list.sort(key=sort_key)
"""数组合并"""
result = current_list + other_list
return result
def search(self, param=None):
"""
查询有效服务器数据 优先查询redis内数据，未查得后查询数据库数据并更新到redis内
:param param:
:return: result
"""
try:
result = BusinessUtil.get_redis_by_key(
busi_config.SERVER_CONFIG_REDIS_KEY)
if result:
return json.loads(result)
_db = MySqlHelper()
result = _db.fetch_all(server_conf_sql.search, param)
if not result:
self.logger.info("【查询有效服务器】响应结果为空，结束流程")
return False
BusinessUtil.set_redis_by_key(busi_config.SERVER_CONFIG_REDIS_KEY,
result, busi_config.REDIS_CACHE_TIME)
return result
except Exception as e:
self.logger.error("【查询有效服务器】查询异常信息为：%s" % traceback.format_exc())
return False
def search_valid_account(self, server_ip):
"""
查询发送有效账号(含剩余数量)
缓存5分钟 可能存在账号多发情况
:param server_ip:
:return:
"""
try:
if not server_ip:
self.logger.info("【查询发送邮箱账号】传入参数有误，请确认")
return False
"""选取发送账号"""
valid_account_key = "%s_%s" % (server_ip.replace(
".", "_"), busi_config.VALID_ACCOUNT_REDIS_KEY)
"""查询redis"""
result = BusinessUtil.get_redis_by_key(valid_account_key)
if not result:
"""查询数据库"""
_db = MySqlHelper()
"""设置当前日期"""
timeStr = str(
time.strftime('%Y-%m-%d', time.localtime(time.time())))
start = "%s 00:00:00" % timeStr
end = "%s 23:59:59" % timeStr
result = _db.fetch_all(
send_email_account_sql.search_valid_account,
(server_ip, start, end))
else:
"""转化数据"""
result = json.loads(result)
"""判断非空"""
if not result:
return False
BusinessUtil.set_redis_by_key(valid_account_key, result,
busi_config.REDIS_CACHE_TIME)
return result
except Exception as e:
self.logger.error("【查询发送邮箱账号】查询异常信息为：%s" % traceback.format_exc())
return False
def process_message(self, account, msg, resultText):
"""
解析邮件正文
:param account:
:param msg:
:param resultText:
:return:
"""
try:
if msg.is_multipart():
for part in msg.get_payload():
self.process_message(account, part, resultText)
else:
content_type = msg.get_content_type()
if content_type == 'text/plain':
content = msg.get_payload(decode=True)
# 要检测文本编码:
charset = msg.get_content_charset(msg)
if charset:
content = content.decode(charset)
file_path = BusinessUtil.get_uniq_pop_file_name(account, content, 'txt')
BusinessUtil.save_file(file_path, content, 'w+')
resultText["resultText"] = file_path
elif content_type == 'text/html':
content = msg.get_payload(decode=True)
# 要检测文本编码:
charset = msg.get_content_charset(msg)
if charset:
content = content.decode(charset)
file_path = BusinessUtil.get_uniq_pop_file_name(account, content, 'html')
BusinessUtil.save_file(file_path, content, 'w+')
resultText["resultHtml"] = file_path
else:
# 不是文本,作为附件处理:
file_name = self.get_file_name(msg)
# 获取后缀
suffix = file_name.split(".")[1]
content = msg.get_payload(decode=True)
file_path = BusinessUtil.get_uniq_pop_file_name(account, content, suffix, 'attach')
BusinessUtil.save_file(file_path, content, 'wb')
resultText["resultAttach"] = file_path
except Exception as e:
self.logger.info("解析邮件失败：%s" % (traceback.format_exc()))
def search_template_code(self, task_code):
try:
template_redis_key = "%s_%s" % (task_code,
busi_config.TEMPLATE_REDIS_KEY)
result = BusinessUtil.get_redis_by_key(template_redis_key)
if not result:
_db = MySqlHelper()
result = _db.fetch_all(template_sql.search_template_code,
task_code)
if not result:
return False
"""redis入库"""
BusinessUtil.set_redis_by_key(template_redis_key, result,
busi_config.REDIS_CACHE_TIME)
else:
if type(result) == str:
result = json.loads(result)
"""筛选templateCode"""
item = random.choice(result)
return item.get("templateCode")
except Exception as e:
self.logger.error("【查询有效服务器】查询异常信息为：%s" % traceback.format_exc())
return False
def verify_email_is_in_black_roll_call(self, email): """ 校验是否在黑名单内 :param email: :return: """ """获取redis key值""" redis_key = BusinessUtil.get_verify_key(email) """如果未获取到值""" if not redis_key: return False """通过key值查询redis""" redis_verify_email_value = BlackRollCallService().search_key(redis_key) """未找到值""" if not redis_verify_email_value: return False """从黑名单内查到字段""" if redis_verify_email_value.find(email) > -1: return True return False
def instance(self):
"""
调度处理任务队列数据
:return:
"""
try:
"""获取休眠时间"""
product_sleep_time = BusiBaseConfService().search_key_return_value(
(busi_config.PRODUCT_SLEEP_TIME))
sleep_time = product_sleep_time.split("|")
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
"""判断是否存在发送中的任务 不存在 则直接返回"""
if not self.search_task_code_by_server_ip():
self.logger.info("【服务进程】不存在发送中的任务，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""读取任务队列"""
task_name_list = self.get_task_list()
self.logger.info("【服务进程】任务列表为：%s" % task_name_list)
if not task_name_list:
self.logger.info("【服务进程】任务列表为空，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""判断账号"""
if not self.check_valid_account():
self.logger.info("【服务进程】无有效账号，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""按任务优先级读取"""
for task in task_name_list:
queue_length = BusinessUtil.get_redis_llen_by_key(task)
"""队列为空"""
if not queue_length or queue_length == 0:
continue
"""出队列"""
self.processing_task(task, queue_length)
break
except Exception as e:
self.logger.error("【服务进程】服务进程异常，异常信息为：%s" % traceback.format_exc())
def insert_update_more(self, result):
"""
添加或更新数据
:param result:
:return:
"""
_db = None
try:
if not result:
self.logger.info("【添加/更新营销数据】请求参数为为空，流程结束")
"""查询数据"""
_db = MySqlHelper()
for param in result:
param["subBatchCode"] = BusinessUtil.get_uniqu_time()
count = _db.count(marketing_data_sql.count,
(param.get("batchCode"), param.get("mobile"),
param.get("email")))
if count > 0:
result = _db.update(
marketing_data_sql.update_insert,
(param.get("subBatchCode"),
int(param.get("status", "0")), param.get("batchCode"),
param.get("mobile"), param.get("email")))
else:
result = _db.insert_one(
marketing_data_sql.insert,
(param.get("originalBatchCode"),
param.get("batchCode"), param.get("subBatchCode"),
param.get("mobile"), param.get("email"),
int(param.get("status", "0"))))
_db.end()
return result
except Exception as e:
if _db:
_db.isEnd = 2
_db.end()
self.logger.error("【添加/更新营销数据】保存异常信息为：%s" % traceback.format_exc())
return False
def set_token_bucket(self, key): """ 设置令牌桶 （后续考虑到将信息添加到数据库） :return: """ BusinessUtil.set_token_bucket(key)
