def deal_with_result_data(self, task_dict): """ 按服务器IP注册队列 :param task_dict: :return: """ """获取休眠时间""" account_sleep_time = BusiBaseConfService().search_key_return_value( (busi_config.ACCOUNT_SLEEP_TIME)) sleep_time = account_sleep_time.split("|") time_out = random.randint(int(sleep_time[0]), int(sleep_time[1])) while task_dict: """按服务器生成队列""" server_dict = {} queue_length = 0 for key in list(task_dict.keys()): """判断是否为空""" if not task_dict[key]: del task_dict[key] continue """取值""" item = task_dict[key].pop() queue_length += 1 server_ip_name = "get_%s_queue" % item.get("serverIp").replace( ".", "_") if server_ip_name in task_dict: server_dict[server_ip_name].append(item) else: server_dict.setdefault(server_ip_name, []) server_dict[server_ip_name].append(item) """判空""" if task_dict: self.redis_queue_manager_helper(server_dict) self.logger.info("账号休眠,休眠时长:%s秒" % time_out) time.sleep(time_out)
def update(self): """ 筛选发送服务器 :return: """ self.logger.info("【数据提取】3、筛选服务器") result = self.subject.email_item """查询服务器数据""" server_list = ServerConfService().search() """查询筛选规则""" resend_rules = BusiBaseConfService().search_key_return_value( (busi_config.RESEND_RULES), "1|1|1") """分割规则""" resend_rules_list = resend_rules.split("|") """判断是否有服务器IP""" server_ip = result.get("serverIp", "") if server_ip: """服务器IP存在,筛选当前服务器信息""" server_item = self.select_server_ip(server_list, server_ip) else: """待过滤服务器ip""" filter_ip = result.get("reSendServerIp", "") """筛选服务器""" if not filter_ip: filter_ip = "" server_item = self.filter_server_ip(server_list, resend_rules_list[0], filter_ip) if not server_item: raise BusinessExcetion("T04", "未查到有效的服务器地址") self.logger.info("【任务队列】筛选服务器IP:%s" % server_item.get("serverIp")) result.update(server_item) """更新/添加使用量"""
def update(self): """ 选择发送内容 :return: """ self.logger.info("【任务队列】2、筛选发送内容") result = self.subject.email_item """当前task_code""" task_code = result.get("task_code") """查询发送内容""" template_list = TemplateService().search(task_code) """查询筛选规则""" resend_rules = BusiBaseConfService().search_key_return_value( (busi_config.RESEND_RULES), "1|1|1") """分割规则""" resend_rules_list = resend_rules.split("|") """判断是否有发送内容""" template_ext_code = result.get("templateExtCode", "") if template_ext_code: """发送内容存在,筛选当前发送内容""" send_email_content_item = self.select_send_email_content( template_list, template_ext_code) else: """待过滤发送内容""" filter_template_ext_code = result.get("reSendTemplateExtCode", "") if not filter_template_ext_code: filter_template_ext_code = "" """筛选发送内容""" send_email_content_item = self.filter_send_template_ext_code( template_list, resend_rules_list[2], filter_template_ext_code) if not send_email_content_item: raise BusinessExcetion("T02", "未查到有效邮件内容,进行回滚") self.logger.info("【任务队列】筛选发送内容:%s" % send_email_content_item.get("templateExtCode")) result.update(send_email_content_item)
def get_primamary_list(self, batchCode): """ 获取营销数据 :return: """ if not self.task_code: self.logger.info("【提取数据】不存在发送中的任务,流程结束") return False """拉取单批次发送量""" batchMaxLength = BusiBaseConfService().search_key_return_value((busi_config.BATCH_MAX_LENGTH), "3") primaryDataService = PrimaryDataService() """判断类型""" if str(self.task_type) == "1": """更新数据""" primaryDataService.update((batchCode, self.task_code, int(batchMaxLength))) """查询数据""" result = primaryDataService.search((batchCode)) return result """获取允许重发的ip""" allow_resend_server_ip = BusiBaseConfService().search_key_return_value((busi_config.ALLOW_RESEND_SERVER_IP)) """判断是否为主程序""" if allow_resend_server_ip != self.public_ip: return False """文件中读取""" result = self.search_result_by_file(batchCode, int(batchMaxLength)) return result
def processing_task(self, key, queue_length): """ 按任务优先级处理数据 :param key: 任务队列key值 :param queue_length: 当前队列长度 :return: """ """读取最大量""" max_redis_read_length = BusiBaseConfService().search_key_return_value( (busi_config.MAX_REDIS_READ_LENGTH)) self.logger.info("单次最大处理量为:%s" % max_redis_read_length) """读取redis列表""" result = BusinessUtil.get_redis_lpop_by_key_and_length( key, queue_length, int(max_redis_read_length)) """判空""" if not result: return """最大线程数""" thread_max_workers = BusiBaseConfService().search_key_return_value( (busi_config.THREAD_MAX_WORKERS)) """按发送邮箱名称分组""" task_dict = {} with ThreadPoolExecutor( max_workers=int(thread_max_workers)) as executor: future_list = [] """循环""" for item in result: if item: future = executor.submit(self.task_thread_submit, item, key, task_dict) future_list.append(future) """等待子线程结束""" wait(future_list, timeout=10) """处理队列数据""" self.deal_with_result_data(task_dict)
def get_send_min_max_time(self): """ 获取发送时间段 :return: """ send_min_max_time = BusiBaseConfService().search_key_return_value( (busi_config.SEND_MIN_MAX_TIME), "9|20") """数据分割""" min_time, max_time = send_min_max_time.split("|") return [min_time, max_time]
def get_batch_sleep_time(self): """ 获取休眠时间 :return: """ batch_sleep_time = BusiBaseConfService().search_key_return_value( (busi_config.BUSI_BATCH_SLEEP_TIME), "5|8") min_time, max_time = batch_sleep_time.split("|") """休眠时长""" time_out = random.randint(int(min_time), int(max_time)) return time_out
def redis_instance(self): """ 启动redis消费 :return: """ try: """获取订阅信息""" kafka_topic = "get_%s_queue" % self.task_choice_pattern.replace( ".", "_") """读取当前redis内数据长度""" queue_length = BusinessUtil.get_redis_llen_by_key(kafka_topic) """读取最大量""" consume_max_redis_read_length = BusiBaseConfService( ).search_key_return_value( (busi_config.CONSUME_MAX_REDIS_READ_LENGTH)) self.logger.info("单次最大处理量为:%s" % consume_max_redis_read_length) """读取redis列表""" result = BusinessUtil.get_redis_lpop_by_key_and_length( kafka_topic, queue_length, int(consume_max_redis_read_length)) """判空""" if not result: self.logger.info("【消费】未读取到有效数据,流程结束") return """处理消息""" self.processing_task(result) """处理结束""" self.logger.info("【消费】处理结束") except Exception as e: self.logger.error("【消费】启动消费异常,信息为:%s" % traceback.format_exc())
def instance(self): """ :return: """ self.logger.info( "---------------------------------浮屠长生 开始提取数据---------------------------------" ) batch_code = None try: """生成主批次号""" batch_code = BusinessUtil.get_uniqu_time() self.logger.info("【提取数据】生成当前主批次号为:%s" % batch_code) """提取数据""" result = self.get_primamary_list(batch_code) """获取待发送邮箱,并集""" marketing_email = BusiBaseConfService().search_key_return_value( (busi_config.MARKETING_EMAIL_KEY), "139.com") result = BusinessUtil.convert_data(result, marketing_email, False) """数据处理""" self.observer(result) except Exception as e: self.logger.error("【数据提取】数据提取异常,回滚批次为:%s,异常信息为:%s" % (batch_code, traceback.format_exc())) self.roll_back(batch_code) self.logger.info( "---------------------------------浮屠长生 提取数据结束---------------------------------" )
def instance(self): """ 收取邮件 :return: """ self.logger.info( "=============================浮屠长生 开始收取邮件=============================" ) """获取发送账号""" result = self.get_send_email_account_list() if not result: """未查到发送账号""" self.logger.info( "=============================浮屠长生 收取邮件结束=============================" ) return """账号间隔休眠时长""" collect_account_sleep_time = BusiBaseConfService( ).search_key_return_value((busi_config.COLLECT_ACCOUNT_SLEEP_TIME), "0") self.logger.info("账号间隔休眠时长[%s]秒" % collect_account_sleep_time) for item in result: try: self.observer(item) time.sleep(float(collect_account_sleep_time)) except Exception as e: self.logger.error("【收取邮件】收取邮件异常,异常信息为:%s" % traceback.format_exc()) self.logger.error( "=============================浮屠长生 收取邮件结束=============================" )
def task_collect_time(self): """ 收取邮件 :return: """ while True: """开始收取邮件""" self.instance() """休眠""" collect_sleep_time = BusiBaseConfService().search_key_return_value((busi_config.COLLECT_SLEEP_TIME), "15") self.logger.info("收取邮件完成,进行休眠,休眠时长[%s]分钟" % collect_sleep_time) time.sleep(60 * int(collect_sleep_time))
def update(self): result = self.subject.email_list self.logger.info("【数据提取】1、队列黑名单检查") """是否校验黑名单""" verify_black_roll_call = BusiBaseConfService().search_key_return_value((busi_config.VERIFY_BLACK_ROLL_CALL), "0") """黑名单""" if verify_black_roll_call == "0": return for item in result: email = item.get("email") """黑名单校验 True 在黑名单内 Flase 不再黑名单内""" if self.verify_email_is_in_black_roll_call(email): item["status"] = "4"
def instance(self): """ 调度处理任务队列数据 :return: """ try: """获取休眠时间""" product_sleep_time = BusiBaseConfService().search_key_return_value( (busi_config.PRODUCT_SLEEP_TIME)) sleep_time = product_sleep_time.split("|") time_out = random.randint(int(sleep_time[0]), int(sleep_time[1])) """判断是否存在发送中的任务 不存在 则直接返回""" if not self.search_task_code_by_server_ip(): self.logger.info("【服务进程】不存在发送中的任务,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """读取任务队列""" task_name_list = self.get_task_list() self.logger.info("【服务进程】任务列表为:%s" % task_name_list) if not task_name_list: self.logger.info("【服务进程】任务列表为空,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """判断账号""" if not self.check_valid_account(): self.logger.info("【服务进程】无有效账号,日常生成休眠,休眠时长:%s分" % time_out) time.sleep(time_out * 60) return """按任务优先级读取""" for task in task_name_list: queue_length = BusinessUtil.get_redis_llen_by_key(task) """队列为空""" if not queue_length or queue_length == 0: continue """出队列""" self.processing_task(task, queue_length) break except Exception as e: self.logger.error("【服务进程】服务进程异常,异常信息为:%s" % traceback.format_exc())
def resend_instance(self): """ 邮件重发 :return: """ self.logger.info("---------------------------------浮屠长生 开始提取重发数据---------------------------------") try: """查询配置文件 最大重发量""" reSendMaxLength = BusiBaseConfService().search_key_return_value((busi_config.RE_SEND_MAX_LENGTH), "5") result = MarketingDataService().resend_search((self.task_code, reSendMaxLength)) """数据处理""" self.observer(result) except Exception as e: self.logger.error("【数据提取】提取重发数据异常,异常信息为:%s" % traceback.format_exc()) self.logger.info("---------------------------------浮屠长生 提取重发数据结束---------------------------------")
def update(self): """ 筛选发送账号 :return: """ self.logger.info("【任务队列】1、筛选发送账号") result = self.subject.email_item """当前IP""" public_ip = result.get("public_ip") account_list = SendEmailAccountService().search_valid_account(public_ip) if not account_list: self.logger.info("【发送邮件】开始变更账号信息") account_reopen_time = BusiBaseConfService().search_key_return_value((busi_config.ACCOUNT_REOPEN_TIME)) SendEmailAccountService().search_and_update_account(public_ip, int(account_reopen_time)) raise BusinessExcetion("T02", "未查询到有效账号") """查询筛选规则""" resend_rules = BusiBaseConfService().search_key_return_value((busi_config.RESEND_RULES),"1|1|1") """分割规则""" resend_rules_list = resend_rules.split("|") """判断是否有发送账号""" send_account = result.get("sendAccount", "") if send_account: """发送账号存在,筛选当前账号信息""" send_account_item = self.select_send_account(account_list, send_account) else: """待过滤发送账号""" filter_send_account = result.get("reSendAccount", "") if not filter_send_account: filter_send_account = "" """筛选发送账号""" send_account_item = self.filter_send_account(account_list, resend_rules_list[1], filter_send_account) if not send_account_item: raise BusinessExcetion("T02", "未查到有效账号,进行回滚") self.logger.info("【任务队列】筛选发送账号:%s" % send_account_item.get("userName")) result.update(send_account_item) self.set_used_of_count_record(send_account_item.get("userName"), 2)
def check_valid_account(self): """ 校验有效邮箱 :return: """ account_list = SendEmailAccountService().search_valid_account( self.public_ip) if account_list: return True """获取账号重开间隔时间""" account_reopen_time = BusiBaseConfService().search_key_return_value( (busi_config.ACCOUNT_REOPEN_TIME)) """查询触发频率过快的账号剩余量, 如还有剩余,尝试重新激活""" return SendEmailAccountService().search_lock_account( int(account_reopen_time))
def processing_task(self, message): """ 处理消费队列 :param message: :return: """ """最大线程数""" thread_max_workers = BusiBaseConfService().search_key_return_value( (busi_config.CONSUME_THREAD_MAX_WORKERS)) """线程数""" with ThreadPoolExecutor( max_workers=int(thread_max_workers)) as executor: future_list = [] """读取任务队列数据""" for msg in message: item = json.loads(msg) future = executor.submit(self.task_thread_submit, item) future_list.append(future) future_list.append(future) """等待子线程结束""" wait(future_list, timeout=10)
def manager_processing_task(self, task_queue, result_queue): """ 处理任务队列 :param task_queue: :param result_queue: :return: """ """最大线程数""" thread_max_workers = BusiBaseConfService().search_key_return_value( (busi_config.THREAD_MAX_WORKERS)) """线程数""" with ThreadPoolExecutor( max_workers=int(thread_max_workers)) as executor: future_list = [] """读取任务队列数据""" while not task_queue.empty(): item = task_queue.get(True, timeout=5) self.logger.error("【任务进程】读取队列项数据为:%s" % item) future = executor.submit(self.task_thread_submit, item, result_queue) future_list.append(future) """等待子线程结束""" wait(future_list, timeout=10)
def task_extract_time(self): """ 定时任务 :return: """ send_count = 0 send_report = 0 while True: self.logger.info("=============================浮屠长生 开始第[%s]定时任务==================================" % send_count) """获取发送时间段""" send_min_max_time = BusiBaseConfService().search_key_return_value((busi_config.SEND_MIN_MAX_TIME), "9|20") """数据分割""" send_time = send_min_max_time.split("|") """获取休眠时间""" batch_sleep_time = BusiBaseConfService().search_key_return_value((busi_config.BUSI_BATCH_SLEEP_TIME), "5|8") sleep_time = batch_sleep_time.split("|") """获取允许重发的ip""" allow_resend_server_ip = BusiBaseConfService().search_key_return_value((busi_config.ALLOW_RESEND_SERVER_IP)) """休眠时长""" time_out = random.randint(int(sleep_time[0]), int(sleep_time[1])) """获取当前时间""" hour = int(time.strftime('%H', time.localtime(time.time()))) if hour > int(send_time[1]): """ 收取邮件 """ Collect().instance_server_ip() self.logger.info("当前时间未在发送时间段内容,进行休眠,休眠时长[%s]分钟" % time_out) time.sleep(60 * time_out) elif hour < int(send_time[0]): """判断是否在发送时间段内,如果不存在的话跳过 0~9""" """若标志位为0""" if send_report == 0 and allow_resend_server_ip == self.public_ip and self.get_task_list(): send_report = 1 """更新账号""" SendEmailAccountService().update_all_status_valid() send_count = 0 self.logger.info("当前时间未在发送时间段内容,进行休眠,休眠时长[%s]分钟" % time_out) time.sleep(60 * time_out) elif not self.search_task_code_by_server_ip(): self.logger.info("【提取数据】不存在发送中的任务,流程结束") time.sleep(60 * time_out) elif send_count != 0 and not self.get_task_list(): """如果不是当天第一次发送,且缓存记录为空""" """发送次数""" send_count += 1 send_report = 0 """ 收取邮件 """ Collect().instance_server_ip() """ 数据二次营销 大于5次就不再营销 """ if allow_resend_server_ip == self.public_ip: self.resend_instance() self.instance() """休眠""" self.logger.info("当前第[%s]次发送,提取完成,进行休眠,休眠时长[%s]分钟" % (send_count, time_out)) time.sleep(60 * time_out) elif self.get_task_list(): """如果缓存记录不为空,则休眠""" send_report = 0 self.logger.info("当前第[%s]次发送,缓存信息不为空,进行休眠,休眠时长[%s]分钟" % (send_count, time_out)) time.sleep(60 * time_out) else: """如果缓存记录为空,进行消费生产""" send_count += 1 send_report = 0 """生产""" self.instance() self.logger.info("当前第[%s]次发送,生成缓存信息,进行休眠,休眠时长[%s]分钟" % (send_count, time_out)) time.sleep(60*time_out) self.logger.info("=============================浮屠长生 结束本次定时任务==================================")
def get_consume_sleep_time(self): consume_sleep_time = BusiBaseConfService().search_key_return_value( (busi_config.CONSUME_SLEEP_TIME)) sleep_time = consume_sleep_time.split("|") time_out = random.randint(int(sleep_time[0]), int(sleep_time[1])) return time_out