def deal_with_result_data(self, task_dict):
"""
按服务器IP注册队列
:param task_dict:
:return:
"""
"""获取休眠时间"""
account_sleep_time = BusiBaseConfService().search_key_return_value(
(busi_config.ACCOUNT_SLEEP_TIME))
sleep_time = account_sleep_time.split("|")
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
while task_dict:
"""按服务器生成队列"""
server_dict = {}
queue_length = 0
for key in list(task_dict.keys()):
"""判断是否为空"""
if not task_dict[key]:
del task_dict[key]
continue
"""取值"""
item = task_dict[key].pop()
queue_length += 1
server_ip_name = "get_%s_queue" % item.get("serverIp").replace(
".", "_")
if server_ip_name in task_dict:
server_dict[server_ip_name].append(item)
else:
server_dict.setdefault(server_ip_name, [])
server_dict[server_ip_name].append(item)
"""判空"""
if task_dict:
self.redis_queue_manager_helper(server_dict)
self.logger.info("账号休眠，休眠时长：%s秒" % time_out)
time.sleep(time_out)
def update(self):
"""
筛选发送服务器
:return:
"""
self.logger.info("【数据提取】3、筛选服务器")
result = self.subject.email_item
"""查询服务器数据"""
server_list = ServerConfService().search()
"""查询筛选规则"""
resend_rules = BusiBaseConfService().search_key_return_value(
(busi_config.RESEND_RULES), "1|1|1")
"""分割规则"""
resend_rules_list = resend_rules.split("|")
"""判断是否有服务器IP"""
server_ip = result.get("serverIp", "")
if server_ip:
"""服务器IP存在，筛选当前服务器信息"""
server_item = self.select_server_ip(server_list, server_ip)
else:
"""待过滤服务器ip"""
filter_ip = result.get("reSendServerIp", "")
"""筛选服务器"""
if not filter_ip:
filter_ip = ""
server_item = self.filter_server_ip(server_list,
resend_rules_list[0],
filter_ip)
if not server_item:
raise BusinessExcetion("T04", "未查到有效的服务器地址")
self.logger.info("【任务队列】筛选服务器IP：%s" % server_item.get("serverIp"))
result.update(server_item)
"""更新/添加使用量"""
def update(self):
"""
选择发送内容
:return:
"""
self.logger.info("【任务队列】2、筛选发送内容")
result = self.subject.email_item
"""当前task_code"""
task_code = result.get("task_code")
"""查询发送内容"""
template_list = TemplateService().search(task_code)
"""查询筛选规则"""
resend_rules = BusiBaseConfService().search_key_return_value(
(busi_config.RESEND_RULES), "1|1|1")
"""分割规则"""
resend_rules_list = resend_rules.split("|")
"""判断是否有发送内容"""
template_ext_code = result.get("templateExtCode", "")
if template_ext_code:
"""发送内容存在，筛选当前发送内容"""
send_email_content_item = self.select_send_email_content(
template_list, template_ext_code)
else:
"""待过滤发送内容"""
filter_template_ext_code = result.get("reSendTemplateExtCode", "")
if not filter_template_ext_code:
filter_template_ext_code = ""
"""筛选发送内容"""
send_email_content_item = self.filter_send_template_ext_code(
template_list, resend_rules_list[2], filter_template_ext_code)
if not send_email_content_item:
raise BusinessExcetion("T02", "未查到有效邮件内容，进行回滚")
self.logger.info("【任务队列】筛选发送内容：%s" %
send_email_content_item.get("templateExtCode"))
result.update(send_email_content_item)
def get_primamary_list(self, batchCode):
"""
获取营销数据
:return:
"""
if not self.task_code:
self.logger.info("【提取数据】不存在发送中的任务，流程结束")
return False
"""拉取单批次发送量"""
batchMaxLength = BusiBaseConfService().search_key_return_value((busi_config.BATCH_MAX_LENGTH), "3")
primaryDataService = PrimaryDataService()
"""判断类型"""
if str(self.task_type) == "1":
"""更新数据"""
primaryDataService.update((batchCode, self.task_code, int(batchMaxLength)))
"""查询数据"""
result = primaryDataService.search((batchCode))
return result
"""获取允许重发的ip"""
allow_resend_server_ip = BusiBaseConfService().search_key_return_value((busi_config.ALLOW_RESEND_SERVER_IP))
"""判断是否为主程序"""
if allow_resend_server_ip != self.public_ip:
return False
"""文件中读取"""
result = self.search_result_by_file(batchCode, int(batchMaxLength))
return result
def processing_task(self, key, queue_length):
"""
按任务优先级处理数据
:param key: 任务队列key值
:param queue_length: 当前队列长度
:return:
"""
"""读取最大量"""
max_redis_read_length = BusiBaseConfService().search_key_return_value(
(busi_config.MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
key, queue_length, int(max_redis_read_length))
"""判空"""
if not result:
return
"""最大线程数"""
thread_max_workers = BusiBaseConfService().search_key_return_value(
(busi_config.THREAD_MAX_WORKERS))
"""按发送邮箱名称分组"""
task_dict = {}
with ThreadPoolExecutor(
max_workers=int(thread_max_workers)) as executor:
future_list = []
"""循环"""
for item in result:
if item:
future = executor.submit(self.task_thread_submit, item,
key, task_dict)
future_list.append(future)
"""等待子线程结束"""
wait(future_list, timeout=10)
"""处理队列数据"""
self.deal_with_result_data(task_dict)
def get_send_min_max_time(self):
"""
获取发送时间段
:return:
"""
send_min_max_time = BusiBaseConfService().search_key_return_value(
(busi_config.SEND_MIN_MAX_TIME), "9|20")
"""数据分割"""
min_time, max_time = send_min_max_time.split("|")
return [min_time, max_time]
def get_batch_sleep_time(self):
"""
获取休眠时间
:return:
"""
batch_sleep_time = BusiBaseConfService().search_key_return_value(
(busi_config.BUSI_BATCH_SLEEP_TIME), "5|8")
min_time, max_time = batch_sleep_time.split("|")
"""休眠时长"""
time_out = random.randint(int(min_time), int(max_time))
return time_out
def redis_instance(self):
"""
启动redis消费
:return:
"""
try:
"""获取订阅信息"""
kafka_topic = "get_%s_queue" % self.task_choice_pattern.replace(
".", "_")
"""读取当前redis内数据长度"""
queue_length = BusinessUtil.get_redis_llen_by_key(kafka_topic)
"""读取最大量"""
consume_max_redis_read_length = BusiBaseConfService(
).search_key_return_value(
(busi_config.CONSUME_MAX_REDIS_READ_LENGTH))
self.logger.info("单次最大处理量为：%s" % consume_max_redis_read_length)
"""读取redis列表"""
result = BusinessUtil.get_redis_lpop_by_key_and_length(
kafka_topic, queue_length, int(consume_max_redis_read_length))
"""判空"""
if not result:
self.logger.info("【消费】未读取到有效数据，流程结束")
return
"""处理消息"""
self.processing_task(result)
"""处理结束"""
self.logger.info("【消费】处理结束")
except Exception as e:
self.logger.error("【消费】启动消费异常，信息为：%s" % traceback.format_exc())
def instance(self):
"""
:return:
"""
self.logger.info(
"---------------------------------浮屠长生 开始提取数据---------------------------------"
)
batch_code = None
try:
"""生成主批次号"""
batch_code = BusinessUtil.get_uniqu_time()
self.logger.info("【提取数据】生成当前主批次号为：%s" % batch_code)
"""提取数据"""
result = self.get_primamary_list(batch_code)
"""获取待发送邮箱，并集"""
marketing_email = BusiBaseConfService().search_key_return_value(
(busi_config.MARKETING_EMAIL_KEY), "139.com")
result = BusinessUtil.convert_data(result, marketing_email, False)
"""数据处理"""
self.observer(result)
except Exception as e:
self.logger.error("【数据提取】数据提取异常，回滚批次为：%s，异常信息为：%s" %
(batch_code, traceback.format_exc()))
self.roll_back(batch_code)
self.logger.info(
"---------------------------------浮屠长生 提取数据结束---------------------------------"
)
def instance(self):
"""
收取邮件
:return:
"""
self.logger.info(
"=============================浮屠长生 开始收取邮件============================="
)
"""获取发送账号"""
result = self.get_send_email_account_list()
if not result:
"""未查到发送账号"""
self.logger.info(
"=============================浮屠长生 收取邮件结束============================="
)
return
"""账号间隔休眠时长"""
collect_account_sleep_time = BusiBaseConfService(
).search_key_return_value((busi_config.COLLECT_ACCOUNT_SLEEP_TIME),
"0")
self.logger.info("账号间隔休眠时长[%s]秒" % collect_account_sleep_time)
for item in result:
try:
self.observer(item)
time.sleep(float(collect_account_sleep_time))
except Exception as e:
self.logger.error("【收取邮件】收取邮件异常，异常信息为：%s" %
traceback.format_exc())
self.logger.error(
"=============================浮屠长生 收取邮件结束============================="
)
def task_collect_time(self):
"""
收取邮件
:return:
"""
while True:
"""开始收取邮件"""
self.instance()
"""休眠"""
collect_sleep_time = BusiBaseConfService().search_key_return_value((busi_config.COLLECT_SLEEP_TIME), "15")
self.logger.info("收取邮件完成，进行休眠，休眠时长[%s]分钟" % collect_sleep_time)
time.sleep(60 * int(collect_sleep_time))
def update(self):
result = self.subject.email_list
self.logger.info("【数据提取】1、队列黑名单检查")
"""是否校验黑名单"""
verify_black_roll_call = BusiBaseConfService().search_key_return_value((busi_config.VERIFY_BLACK_ROLL_CALL), "0")
"""黑名单"""
if verify_black_roll_call == "0":
return
for item in result:
email = item.get("email")
"""黑名单校验 True 在黑名单内 Flase 不再黑名单内"""
if self.verify_email_is_in_black_roll_call(email):
item["status"] = "4"
def instance(self):
"""
调度处理任务队列数据
:return:
"""
try:
"""获取休眠时间"""
product_sleep_time = BusiBaseConfService().search_key_return_value(
(busi_config.PRODUCT_SLEEP_TIME))
sleep_time = product_sleep_time.split("|")
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
"""判断是否存在发送中的任务 不存在 则直接返回"""
if not self.search_task_code_by_server_ip():
self.logger.info("【服务进程】不存在发送中的任务，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""读取任务队列"""
task_name_list = self.get_task_list()
self.logger.info("【服务进程】任务列表为：%s" % task_name_list)
if not task_name_list:
self.logger.info("【服务进程】任务列表为空，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""判断账号"""
if not self.check_valid_account():
self.logger.info("【服务进程】无有效账号，日常生成休眠，休眠时长：%s分" % time_out)
time.sleep(time_out * 60)
return
"""按任务优先级读取"""
for task in task_name_list:
queue_length = BusinessUtil.get_redis_llen_by_key(task)
"""队列为空"""
if not queue_length or queue_length == 0:
continue
"""出队列"""
self.processing_task(task, queue_length)
break
except Exception as e:
self.logger.error("【服务进程】服务进程异常，异常信息为：%s" % traceback.format_exc())
def resend_instance(self):
"""
邮件重发
:return:
"""
self.logger.info("---------------------------------浮屠长生 开始提取重发数据---------------------------------")
try:
"""查询配置文件 最大重发量"""
reSendMaxLength = BusiBaseConfService().search_key_return_value((busi_config.RE_SEND_MAX_LENGTH), "5")
result = MarketingDataService().resend_search((self.task_code, reSendMaxLength))
"""数据处理"""
self.observer(result)
except Exception as e:
self.logger.error("【数据提取】提取重发数据异常，异常信息为：%s" % traceback.format_exc())
self.logger.info("---------------------------------浮屠长生 提取重发数据结束---------------------------------")
def update(self):
"""
筛选发送账号
:return:
"""
self.logger.info("【任务队列】1、筛选发送账号")
result = self.subject.email_item
"""当前IP"""
public_ip = result.get("public_ip")
account_list = SendEmailAccountService().search_valid_account(public_ip)
if not account_list:
self.logger.info("【发送邮件】开始变更账号信息")
account_reopen_time = BusiBaseConfService().search_key_return_value((busi_config.ACCOUNT_REOPEN_TIME))
SendEmailAccountService().search_and_update_account(public_ip, int(account_reopen_time))
raise BusinessExcetion("T02", "未查询到有效账号")
"""查询筛选规则"""
resend_rules = BusiBaseConfService().search_key_return_value((busi_config.RESEND_RULES),"1|1|1")
"""分割规则"""
resend_rules_list = resend_rules.split("|")
"""判断是否有发送账号"""
send_account = result.get("sendAccount", "")
if send_account:
"""发送账号存在，筛选当前账号信息"""
send_account_item = self.select_send_account(account_list, send_account)
else:
"""待过滤发送账号"""
filter_send_account = result.get("reSendAccount", "")
if not filter_send_account:
filter_send_account = ""
"""筛选发送账号"""
send_account_item = self.filter_send_account(account_list, resend_rules_list[1], filter_send_account)
if not send_account_item:
raise BusinessExcetion("T02", "未查到有效账号，进行回滚")
self.logger.info("【任务队列】筛选发送账号：%s" % send_account_item.get("userName"))
result.update(send_account_item)
self.set_used_of_count_record(send_account_item.get("userName"), 2)
def check_valid_account(self):
"""
校验有效邮箱
:return:
"""
account_list = SendEmailAccountService().search_valid_account(
self.public_ip)
if account_list:
return True
"""获取账号重开间隔时间"""
account_reopen_time = BusiBaseConfService().search_key_return_value(
(busi_config.ACCOUNT_REOPEN_TIME))
"""查询触发频率过快的账号剩余量， 如还有剩余，尝试重新激活"""
return SendEmailAccountService().search_lock_account(
int(account_reopen_time))
def processing_task(self, message):
"""
处理消费队列
:param message:
:return:
"""
"""最大线程数"""
thread_max_workers = BusiBaseConfService().search_key_return_value(
(busi_config.CONSUME_THREAD_MAX_WORKERS))
"""线程数"""
with ThreadPoolExecutor(
max_workers=int(thread_max_workers)) as executor:
future_list = []
"""读取任务队列数据"""
for msg in message:
item = json.loads(msg)
future = executor.submit(self.task_thread_submit, item)
future_list.append(future)
future_list.append(future)
"""等待子线程结束"""
wait(future_list, timeout=10)
def manager_processing_task(self, task_queue, result_queue):
"""
处理任务队列
:param task_queue:
:param result_queue:
:return:
"""
"""最大线程数"""
thread_max_workers = BusiBaseConfService().search_key_return_value(
(busi_config.THREAD_MAX_WORKERS))
"""线程数"""
with ThreadPoolExecutor(
max_workers=int(thread_max_workers)) as executor:
future_list = []
"""读取任务队列数据"""
while not task_queue.empty():
item = task_queue.get(True, timeout=5)
self.logger.error("【任务进程】读取队列项数据为：%s" % item)
future = executor.submit(self.task_thread_submit, item,
result_queue)
future_list.append(future)
"""等待子线程结束"""
wait(future_list, timeout=10)
def task_extract_time(self):
"""
定时任务
:return:
"""
send_count = 0
send_report = 0
while True:
self.logger.info("=============================浮屠长生 开始第[%s]定时任务==================================" % send_count)
"""获取发送时间段"""
send_min_max_time = BusiBaseConfService().search_key_return_value((busi_config.SEND_MIN_MAX_TIME), "9|20")
"""数据分割"""
send_time = send_min_max_time.split("|")
"""获取休眠时间"""
batch_sleep_time = BusiBaseConfService().search_key_return_value((busi_config.BUSI_BATCH_SLEEP_TIME), "5|8")
sleep_time = batch_sleep_time.split("|")
"""获取允许重发的ip"""
allow_resend_server_ip = BusiBaseConfService().search_key_return_value((busi_config.ALLOW_RESEND_SERVER_IP))
"""休眠时长"""
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
"""获取当前时间"""
hour = int(time.strftime('%H', time.localtime(time.time())))
if hour > int(send_time[1]):
"""
收取邮件
"""
Collect().instance_server_ip()
self.logger.info("当前时间未在发送时间段内容，进行休眠，休眠时长[%s]分钟" % time_out)
time.sleep(60 * time_out)
elif hour < int(send_time[0]):
"""判断是否在发送时间段内，如果不存在的话跳过 0~9"""
"""若标志位为0"""
if send_report == 0 and allow_resend_server_ip == self.public_ip and self.get_task_list():
send_report = 1
"""更新账号"""
SendEmailAccountService().update_all_status_valid()
send_count = 0
self.logger.info("当前时间未在发送时间段内容，进行休眠，休眠时长[%s]分钟" % time_out)
time.sleep(60 * time_out)
elif not self.search_task_code_by_server_ip():
self.logger.info("【提取数据】不存在发送中的任务，流程结束")
time.sleep(60 * time_out)
elif send_count != 0 and not self.get_task_list():
"""如果不是当天第一次发送，且缓存记录为空"""
"""发送次数"""
send_count += 1
send_report = 0
"""
收取邮件
"""
Collect().instance_server_ip()
"""
数据二次营销 大于5次就不再营销
"""
if allow_resend_server_ip == self.public_ip:
self.resend_instance()
self.instance()
"""休眠"""
self.logger.info("当前第[%s]次发送，提取完成，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60 * time_out)
elif self.get_task_list():
"""如果缓存记录不为空，则休眠"""
send_report = 0
self.logger.info("当前第[%s]次发送，缓存信息不为空，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60 * time_out)
else:
"""如果缓存记录为空，进行消费生产"""
send_count += 1
send_report = 0
"""生产"""
self.instance()
self.logger.info("当前第[%s]次发送，生成缓存信息，进行休眠，休眠时长[%s]分钟" % (send_count, time_out))
time.sleep(60*time_out)
self.logger.info("=============================浮屠长生 结束本次定时任务==================================")
def get_consume_sleep_time(self):
consume_sleep_time = BusiBaseConfService().search_key_return_value(
(busi_config.CONSUME_SLEEP_TIME))
sleep_time = consume_sleep_time.split("|")
time_out = random.randint(int(sleep_time[0]), int(sleep_time[1]))
return time_out