def save(d): date, id = d['mail_ident'].split(',')[:2] while True: try: with atomic(): mail = get_mail_model(date).objects.get(id=id) if 'bounce_result' in d: mail.bounce_time = d['bounce_time'] mail.bounce_result = 'success' if d[ 'bounce_result'] else 'fail' mail.bounce_message = d['bounce_message'] if mail.state == 'bounce': mail.state = 'fail_finished' if mail.return_code else 'reject' else: log.error( u'save: unexpected state, mail_ident={}, state={}'. format(d['mail_ident'], mail.state)) mail.save(update_fields=[ 'bounce_time', 'bounce_result', 'bounce_message', 'state' ]) return except (DatabaseError, InterfaceError) as e: # 如果报数据库异常,关闭连接,重新处理任务 log.warning(u'DatabaseError: exception', exc_info=1) connection.close() except BaseException as e: log.warning(u'save: exception', exc_info=1) gevent.sleep(10)
def load_mail(d): date, id = d['mail_ident'].split(',')[:2] for _ in xrange(5): try: with atomic(): mail = get_mail_model(date).objects.get(id=id) settings = Settings.objects.get() try: mail.customer except ObjectDoesNotExist: return 'finish', None d['sender'] = mail.mail_from d['receiver'] = mail.mail_to if mail.dispatch_data is None: return init_dispatch_data(d, mail, settings) else: return update_dispatch_data(d, mail, settings) except (DatabaseError, InterfaceError) as e: # 如果报数据库异常,关闭连接,重新处理任务 log.warning(u'load_mail: database exception', exc_info=1) connection.close() except BaseException as e: log.warning(u'load_mail: exception, mail_ident={}'.format( d['mail_ident']), exc_info=1) gevent.sleep(10) else: return 'retry', None
def __init__(self, task): # 邮件基本信息 self.task = task self.task_date, self.task_id = task.split(',')[:2] model = get_mail_model(self.task_date) self.mail_obj = model.objects.get(pk=self.task_id) self.uid = self.mail_obj.customer.id self.size = self.mail_obj.size self.mail_from = self.mail_obj.mail_from.lower() self.mail_to = self.mail_obj.mail_to.lower() self.mail_path = self.mail_obj.get_mail_path() self.subject = self.mail_obj.subject self.same_mails = model.objects.filter(mail_id=self.task_id, state__in=['check', 'review']) self.mails_count = self.same_mails.count() self.same_keys = [m.get_mail_filename() for m in self.same_mails] self.key = self.mail_obj.get_mail_filename() self.parse_obj = ParseEmail(self.mail_obj.get_mail_content()) self.parse_data = self.parse_obj.parseMailTemplate() # self.content = strip_tags(self.parse_obj.get_content(self.parse_data)) self.content = self.parse_obj.get_content(self.parse_data) self.mail_to_length = model.objects.filter( mail_id=self.task_id).count() + 1 self.check_result = '' self.attachments = self.parse_data.get('attachments', []) self.customer_setting, _ = CustomerSetting.objects.get_or_create( customer_id=self.uid)
def _do_bulk_customer(date): print date bulk_max = setting.bulk_customer # check_list = ['bulk_email', 'error_format', 'recipient_blacklist', 'spam'] mail_model = get_mail_model(date.strftime('%Y%m%d')) # customers = mail_model.objects.filter(check_result__in=check_list).values('customer').annotate( # count=Count("customer")).filter(count__gt=bulk_max).order_by('-count') customers = mail_model.objects.filter( state='reject').values('customer').annotate( count=Count("customer")).filter( count__gt=bulk_max).order_by('-count') print customers for c in customers: customer = c['customer'] # sender_dict = mail_model.objects.filter(check_result__in=check_list, customer=customer).values( # 'mail_from').annotate(count=Count('mail_from')).order_by('-count') sender_dict = mail_model.objects.filter( state='reject', customer=customer).values('mail_from').annotate( count=Count('mail_from')).order_by('-count') print sender_dict bulk_obj, _ = BulkCustomer.objects.get_or_create(customer_id=customer, date=date) bulk_obj.spam_count = c['count'] bulk_obj.sender_count = len(sender_dict) bulk_obj.sender = json.dumps(list(sender_dict)) bulk_obj.recent_count = BulkCustomer.objects.filter( customer=customer, date__lt=date, date__gt=(date - datetime.timedelta(days=3))).count() bulk_obj.save()
def bounce(j): d = log_model.load(json.loads(j)) r = d['result'][-1] date, id = d['mail_ident'].split(',')[:2] while True: try: mail = get_mail_model(date).objects.get(id=id) try: enable = CustomerSetting.objects.filter(customer=mail.customer, bounce=True).first() except ObjectDoesNotExist: enable = False if enable: bounce_setting = BounceSettings.objects.get() sender = bounce_setting.mailbox receiver = d['sender'] html_text = safe_format(bounce_setting.template_cn, receiver=d['receiver'], subject=mail.subject, date=r['deliver_time'], reason=mail.return_message_display()) origin_message = mail.get_mail_content() # origin_message = lib.del_attach_from_msg.del_attach_from_msg(d['mail_ident']) message = bounce_message(sender, receiver, html_text, origin_message) break except BaseException as e: log.warning(u'bounce: exception', exc_info=1) gevent.sleep(60) if enable: code, msg = send(bounce_setting.server, bounce_setting.port, sender, bounce_setting.password, receiver, message) msg = decode_msg(msg) d['bounce_time'] = datetime.datetime.now() d['bounce_result'] = code == 250 d['bounce_message'] = msg log.info( u'bounce: bounced message, mail_ident={}, code={}, msg={}, reason={}' .format(d['mail_ident'], code, msg, mail.return_message_display())) else: log.info(u'bounce: skip bounced message, mail_ident={}'.format( d['mail_ident'])) del d['result'] (redis.pipeline().lpush('control_bounced', json.dumps(bounce_model.dump(d))).lrem( 'control_bounce_waiting', 0, j).execute())
def get_mail_count(get_mail_model, state): date_end = datetime.datetime.today() date_start = date_end - datetime.timedelta(days=3) date = date_start count = 0 while date <= date_end: count += get_mail_model( date.strftime('%Y%m%d')).objects.filter(state=state).count() date = date + datetime.timedelta(days=1) return count
def get_review_count(get_mail_model): date_end = datetime.datetime.today() date_start = date_end - datetime.timedelta(days=3) date = date_start count = 0 while date <= date_end: count += get_mail_model(date.strftime('%Y%m%d')).objects.exclude( check_result='cyber_spam').filter(state='review').count() date = date + datetime.timedelta(days=1) return count
def relay_limit_notice(): """ 定时一次性发送 中继用户超过限制通知, 当用户超过设置值时,发送通知 :return: """ global notice_setting, bounce_setting notice_type = "relay" while True: try: cur = datetime.datetime.now() if cur.hour != 1 or cur.minute != 25: gevent.sleep(30) continue outinfo('relay limit notice') all_content = "" content_format = notice_setting.relay_content manager = notice_setting.manager flag = Notification.objects.filter( manager=manager, type=notice_type, created__contains=datetime.date.today()) pre = cur + datetime.timedelta(-1) if not flag: mail_model = get_mail_model(pre.strftime('%Y%m%d')) customers = mail_model.objects.exclude( customer__company__icontains=u'临时信任').values( 'customer').annotate( count=Count("customer")).order_by('-count') for c in customers: customer = Customer.objects.get(pk=c['customer']) company_id = customer.id company = customer.company relay_limit = customer.relay_limit sender_count = mail_model.objects.exclude( mail_from='<>', mail_from__isnull=False).filter( customer=customer).distinct('mail_from').count() if relay_limit != -1 and sender_count > relay_limit: content = Common.safe_format(content_format, company=company, company_id=company_id, setting=relay_limit, count=sender_count) all_content += content if all_content: subject = pre.strftime('%Y-%m-%d ') + u"中继用户超过限制通知" notice(subject=subject, content=all_content, type=notice_type, customer=None, manager=manager) except: outerror(traceback.format_exc()) finally: gevent.sleep(60)
def save_check_statistic(date, get_mail_model, statistics, check_list): model = get_mail_model(date.strftime('%Y%m%d')) all_dict = dict( model.objects.values_list("check_result").annotate(Count("id"))) pass_dict = dict( model.objects.filter(review_result__in=['pass', 'pass_undo']). values_list("check_result").annotate(Count("id"))) obj, _ = statistics.objects.get_or_create(date=date) for c in check_list: obj.__setattr__('{}_all'.format(c), all_dict.get(c, 0)) obj.__setattr__('{}_pass'.format(c), pass_dict.get(c, 0)) obj.save()
def review_result(): remote_ip = flask.request.remote_addr if remote_ip != config['review_help_server_ip']: log.warning(u'review_result: forbidden, ip={}'.format(remote_ip)) flask.abort(403) d = json.loads(flask.request.data) if d['origin'] == 'relay': mails = (get_mail_model(d['date']).objects.filter(state='review') .filter(Q(id=d['mail_id']) | Q(mail_id=d['mail_id']))) if len(mails) == 0: log.info(u'review_result: skip, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) elif d['review_result'] == 'pass': keys = [m.get_mail_filename() for m in mails] mails.update(state='dispatch', review_result=d['review_result'], review_time=datetime.datetime.now()) map(lambda key: redis.lpush('relay_dispatch', key), set(keys)) log.info(u'review_result: pass, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) elif d['review_result'] == 'reject': mails.update(state='reject', review_result=d['review_result'], review_time=datetime.datetime.now()) log.info(u'review_result: reject, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) else: assert 0 elif d['origin'] == 'collect': mails = (c_get_mail_model(d['date']).objects.filter(state='review') .filter(Q(id=d['mail_id']) | Q(mail_id=d['mail_id']))) if len(mails) == 0: log.info(u'review_result: skip, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) elif d['review_result'] == 'pass': keys = [m.get_mail_filename() for m in mails] mails.update(state='send', review_result=d['review_result'], review_time=datetime.datetime.now()) map(lambda key: redis.lpush('collect_send', key), set(keys)) log.info(u'review_result: pass, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) elif d['review_result'] == 'reject': mails.update(state='reject', review_result=d['review_result'], review_time=datetime.datetime.now()) log.info(u'review_result: reject, ip={}, key={},{},{}' .format(remote_ip, d['origin'], d['date'], d['mail_id'])) else: assert 0 else: assert 0 return ''
def save_review_statistic(date, get_mail_model, statistics, check_list, type='all'): model = get_mail_model(date.strftime('%Y%m%d')) mails = model.objects.filter(check_result__in=check_list) if type == 'all': _save_review_statistic(model, date, mails, statistics, type) else: reviewers = set( mails.filter(reviewer__isnull=False).values_list('reviewer', flat=True)) for r in reviewers: _ = mails.filter(reviewer_id=r) _save_review_statistic(model, date, _, statistics, type, r)
def load_mail(key, origin, date, mail_id): while True: try: if origin == 'relay': return get_mail_model(date).objects.get(id=mail_id) elif origin == 'collect': return c_get_mail_model(date).objects.get(id=mail_id) else: assert 0 except (DatabaseError, InterfaceError) as e: # 如果报数据库异常,关闭连接,重新处理任务 log.warning(u'load_mail: database exception, key={}'.format(key), exc_info=1) connection.close() except BaseException as e: log.warning(u'load_mail: exception, key={}'.format(key), exc_info=1) gevent.sleep(10)
def _do_save_review_resutl(mail, result): """ ———自动审核——— 关键词/动态SPAM/高危邮件的审核策略调整: 1、在X时间内,如果同一个发件人和同一个收件人连续X次通过/拒绝,则记住对应关系,下次直接放行/拒绝。 2、这个规则不适合多个收件人 3、规则有效期为X天 :param mail: :param resutl: :return: """ check_list = [ 'subject_blacklist', 'keyword_blacklist', 'active_spam', 'high_risk' ] check_result = mail.check_result if check_result in check_list and mail.mail_id == 0 and not get_mail_model( mail.get_date()).objects.filter(mail_id=mail.id): _save_review_result(mail.mail_from, mail.mail_to, result, check_result)
def determine_and_save(j, d): date, id = d['mail_ident'].split(',')[:2] while True: try: with atomic(): mail = get_mail_model(date).objects.get(id=id) action = determine(j, d, mail) save(d, mail) if action is not None: action(get_redis_connection(mail.server_id)) return True except (DatabaseError, InterfaceError) as e: # 如果报数据库异常,关闭连接,重新处理任务 log.warning(u'determine_and_save: database exception', exc_info=1) connection.close() except BaseException as e: log.warning(u'determine_and_save: exception, mail_ident={}'.format(d['mail_ident']), exc_info=1) gevent.sleep(1) return False
def sender_credit(): while True: try: j = redis.brpoplpush(REDIS_KEY, REDIS_KEY_temp) mail_id, reason = j.split('----') date, id = mail_id.split('_') m = get_mail_model(date).objects.get(id=id) if m.mail_id != 0: redis.lrem(REDIS_KEY_temp, j, 0) continue sender = m.mail_from.lower().split('=')[-1] expect_value = getattr(setting, reason, 0) if expect_value < 0: key = 'sender_credit-{}'.format(date) value_limit = int(setting.reduce_limit) else: key = 'sender_credit+{}'.format(date) value_limit = int(setting.increase_limit) try: value_count = int(redis.hget(key, sender)) except: value_count = 0 value = 0 if value_count >= value_limit else expect_value redis.hincrby(key, sender, abs(expect_value)) SenderCreditLog.objects.create(sender=sender, expect_value=expect_value, value=value, reason=reason, mail_id=mail_id) if value != 0: obj, _ = SenderCredit.objects.get_or_create(sender=sender) obj.credit += value obj.save() outinfo('{}: expect_value({}), value({})'.format(j, expect_value, value)) redis.lrem(REDIS_KEY_temp, j, 0) except (DatabaseError, InterfaceError), e: #如果报数据库异常,关闭连接,重新处理任务 outerror('DatabaseError: {}'.format(j)) outerror(traceback.format_exc()) connection.close() gevent.sleep(10) return except BaseException as e: outerror('exception: {}'.format(j)) outerror(traceback.format_exc())
def _do_save_keyword_review_status(date, get_mail_model, check_result, relay=True): model = get_mail_model(date.strftime('%Y%m%d')) mails = model.objects.filter(check_result=check_result) subject_dict = {} subject_pass_dict = {} for m in mails: k = m.check_message.split('----')[0] if k in subject_dict: subject_dict[k] += 1 else: subject_dict[k] = 1 if m.state != 'reject': if k in subject_pass_dict: subject_pass_dict[k] += 1 else: subject_pass_dict[k] = 1 if check_result == 'subject_blacklist': blacklist_model = SubjectKeywordBlacklist else: blacklist_model = KeywordBlacklist for k, v in subject_dict.iteritems(): objs = blacklist_model.objects.filter(keyword=k) if objs: obj = objs[0] if relay: obj.relay_all += v else: obj.collect_all += v if k in subject_pass_dict: if relay: obj.relay_pass += subject_pass_dict[k] else: obj.collect_pass += subject_pass_dict[k] obj.save()
def save(d): date, id = d['mail_ident'].split(',')[:2] while True: try: with atomic(): mail = get_mail_model(date).objects.get(id=id) mail.dispatch_data = json.dumps(d['dispatch_data']) mail.deliver_ip = d['deliver_ip'] if mail.state == 'dispatch': if 'state' in d: mail.state = d['state'] else: mail.state = 'send' elif mail.state == 'retry': if 'state' in d: mail.state = d['state'] else: mail.state = 'retry' else: log.error( u'save: unexpected state, mail_ident={}, state={}'. format(d['mail_ident'], mail.state)) if d['dispatch_data'][-1]['attachment_removed']: mail.is_del_attach = True mail.save(update_fields=[ 'dispatch_data', 'deliver_ip', 'state', 'is_del_attach' ]) return except (DatabaseError, InterfaceError) as e: # 如果报数据库异常,关闭连接,重新处理任务 log.warning(u'save: database exception', exc_info=1) connection.close() except BaseException as e: log.warning(u'save: exception, mail_ident={}'.format( d['mail_ident']), exc_info=1) gevent.sleep(10)
def push_data(): from_date = '20160801' to_date = '20160809' while from_date != to_date: outinfo('push data: {}'.format(from_date)) mails = get_mail_model(from_date).objects.filter(mail_id=0) for m in mails: if m.state == 'reject' and m.check_result == 'spam': reason = 'check_dspam' elif m.state == 'reject' and m.check_result == 'k_auto_reject': reason = 'check_auto_reject' elif m.state == 'fail_finished' and m.error_type == 2: reason = 'send_not_exist' elif m.state == 'fail_finished' and m.error_type == 5: reason = 'send_spam' elif m.reviewer and m.review_result.find('reject') != -1: reason = 'review_reject' elif m.reviewer and m.review_result.find('pass') != -1: reason = 'review_pass' else: continue d = '{}----{}'.format(m.date_id(), reason) redis.lpush(REDIS_KEY, d) from_date = (datetime.datetime.strptime(from_date, "%Y%m%d") + datetime.timedelta(days=1)).strftime("%Y%m%d")
def del_attach_from_msg(date_id, f='relay'): date, id = date_id.split(',') mail_obj = get_mail_model(date).objects.get( id=id) if 'relay' == f else get_mail_model2(date).objects.get(id=id) email_content = mail_obj.get_mail_content() message = email.message_from_string(email_content) s = Settings.objects.all() if s: days = s[0].back_days else: days = 10 message = del_attach_ins(message) p = ParseEmail(email_content) data = p.parseMailTemplate() expire_day = (mail_obj.created + datetime.timedelta(days=days)).strftime('%Y-%m-%d %H:%M') context = {'m': data, 'mail_obj': mail_obj, 'expire_day': expire_day} # attachment_django_template = get_django_template() attachment_django_template = get_render_attchtemplate() t = template.Template(attachment_django_template) django_html = t.render(Context(context)) l = [] for part in message.walk(): if not part.is_multipart(): content_type = part.get_content_type() filename = part.get_filename() if content_type == 'text/html' and filename is None: l.append(part) if django_html and len(l) > 0: part = l[0] payload = part.get_payload(decode=True) charset = part.get_content_charset() html = decode_str(payload, charset) # server = '{}'.format( getattr(settings, 'WEB_USER_SHENZHEN') ) # key = base64.b64encode('MAILDOWNLOAD' + '_' + date_id.replace(',', '_')) # word = base64.b64encode(date_id.replace(',', '_')) # link = 'http://{}/mail/dowload_mail_real_attachment?key={}&id={}&cid=\g<cid>'.format(server, key, word) # html = re.sub('"cid:(?P<cid>.*?)"', link, html) # html2 = html_add_footer(html, django_html) html2 = heml_add_header(html, django_html) payload2 = base64.b64encode(html2.encode('utf-8')) part.set_payload(payload2) del part['content-type'] del part['content-transfer-encoding'] part['content-type'] = 'text/html; charset="utf-8"' part['content-transfer-encoding'] = 'base64' elif django_html: if not message.is_multipart(): payload = message.get_payload(decode=True) charset = part.get_content_charset() html = decode_str(payload, charset) # html2 = html_add_footer(html, django_html) html2 = heml_add_header(html, django_html) payload2 = base64.b64encode(html2.encode('utf-8')) message.set_payload(payload2) del message['content-type'] del message['content-transfer-encoding'] message['content-type'] = 'text/html; charset="utf-8"' message['content-transfer-encoding'] = 'base64' else: msg = MIMEText(django_html, 'html', 'utf-8') message.attach(msg) msg = message.as_string() with open('{}_del_attach'.format(mail_obj.get_mail_path()), 'w') as f: f.write(msg) return msg
def statistics(p_date): mails = get_mail_model(p_date.replace( '-', '')).objects.filter(return_code__isnull=False) # type==all rs = get_data(mails) obj, bool = Statistics.objects.get_or_create(date=p_date, type='all') outinfo('all: {}'.format(rs)) save_statistics(obj, rs) # type==ip ips = mails.values_list('deliver_ip', flat=True).distinct() for ip in ips: rs = get_data(mails.filter(deliver_ip=ip)) if rs.get('count', 0): try: obj, bool = Statistics.objects.get_or_create(date=p_date, type='ip', ip=ip) save_statistics(obj, rs) outinfo('ip({}): {}'.format(ip, rs)) except: outerror(traceback.format_exc()) #type==ip_pool pools = IpPool.objects.all() for p in pools: ips = list(p.clusterip_set.values_list('ip', flat=True)) rs = get_data(mails.filter(deliver_ip__in=ips)) if rs.get('count', 0): try: obj, bool = Statistics.objects.get_or_create(date=p_date, type='ip_pool', ip_pool=p) save_statistics(obj, rs) outinfo('ip_pool({}): {}'.format(p, rs)) except: outerror(traceback.format_exc()) #type==cluster cls = Cluster.objects.all() for c in cls: ips = list(c.cluster.values_list('ip', flat=True)) rs = get_data(mails.filter(deliver_ip__in=ips)) if rs.get('count', 0): try: obj, bool = Statistics.objects.get_or_create(date=p_date, type='cluster', cluster=c) save_statistics(obj, rs) outinfo('cluster({}): {}'.format(c, rs)) except: outerror(traceback.format_exc()) #type==customer # customers = Customer.objects.all() customers = mails.values_list('customer_id', flat=True).distinct() for c in customers: rs = get_data(mails.filter(customer_id=c)) if rs.get('count', 0): try: obj, bool = Statistics.objects.get_or_create(date=p_date, type='customer', customer_id=c) save_statistics(obj, rs) outinfo('customer({}): {}'.format(c, rs)) except: outerror(traceback.format_exc())
def run(self): if not os.path.exists(self.mail_path): return date = time.strftime('%Y%m%d') #创建django的mail模型 mail_model = get_mail_model(date) if date not in _dates: install_model(mail_model) #创建的目录 用户组也有权限修改 Common.make_dir([os.path.join(data_path, date)], permit=0775) _dates.append(date) uid, mail_id, client_ip = self.task.split(',')[:3] with open(self.mail_path) as f: email_str = f.read() with open(self.from_path) as f: mail_from = f.read() mail_from = mail_from.split('=')[-1] with open(self.rcpt_path) as f: rcpt_list = f.readlines() # subject = get_mail_subject(mail_data=email_str) p = ParseEmail(email_str) if int(uid) in replace_sender_customers: email_str = p.replace_sender(mail_from) subject = p.get_attr('subject') try: sender_name = re.search('(.*?)\(.*?\)', p.get_attr('from')).group(1) except: outerror('get sender_name error: {}'.format(self.task)) outerror(traceback.format_exc()) sender_name = '' data = p.parseMailTemplate() attaches = data.get('attachments', []) try: attach_name = u'----'.join( [a.get('decode_name', '') for a in attaches]) except: outerror('get attach_name error: {}'.format(self.task)) outerror(traceback.format_exc()) attach_name = '' #被推送到检测队列中的邮件 #当有多个收件人时, 只推送mail_id=0的一个对象到检测队列 #mail_id=0一定不是无效地址中的邮件, 如果都在无效地址中, check_mail=None, 则无邮件推送到检测队列 check_mail = None outinfo(self.task) outinfo('rcpt_list:{}'.format(rcpt_list)) #mail id列表 mail_ids = [] is_white = is_contains(mail_from, invalidsender_whitelist) for rcpt in rcpt_list: if rcpt == '': continue # remove \n rcpt = rcpt.replace('\n', '') # 去掉字符串中的引号 #rcpt = rcpt.replace('"', '').replace("'", "") rcpt = addr_get(rcpt) try: mail_obj = mail_model(customer_id=uid, mail_from=mail_from.lower(), mail_to=rcpt.lower(), client_ip=client_ip, subject=subject, size=len(email_str), sender_name=sender_name, server_id=server_id, attach_name=attach_name) mail_obj.save() except (DatabaseError, InterfaceError), e: outerror(traceback.format_exc()) continue with open(mail_obj.get_mail_path(), 'w') as f: f.write(email_str) filename = mail_obj.get_mail_filename() if not check_email_format(rcpt, suffix_list) or is_same_domain( rcpt, mail_from): outinfo('{}: error format'.format(filename)) self._do_invalid_mail(mail_obj, 'error_format') elif not is_white and check_invalid_mail(rcpt): outinfo('{}: invalid mail'.format(filename)) self._do_invalid_mail(mail_obj, 'invalid_mail') elif is_contains(rcpt, recipient_blacklist): outinfo('{}: recipient blacklist'.format(filename)) self._do_invalid_mail(mail_obj, 'recipient_blacklist') else: outinfo('{}: pass'.format(filename)) if not check_mail: check_mail = mail_obj mail_ids.append(mail_obj.id)
def get_mail_count(mail_obj, date): model = get_mail_model(date.replace('-', '')) mail_id = mail_obj.mail_id if mail_obj.mail_id else mail_obj.id return model.objects.filter(mail_id=mail_id).count() + 1
def bulk_notice(): """ 群发邮件通知 用户群发邮件的时候通知客户和对应的技术支持 :return: """ global notice_setting, bounce_setting subject = u'违规邮件通知' while True: try: mail_model = get_mail_model(time.strftime('%Y%m%d')) if notice_setting: outinfo('bulk notice') bulk_count = notice_setting.bulk_count bulk_interval = notice_setting.bulk_interval content_format = notice_setting.bulk_content customers = mail_model.objects.filter( state='reject').values('customer').annotate( count=Count("customer")).filter( count__gt=bulk_count).order_by('-count') for c in customers: customer = Customer.objects.get(pk=c['customer']) need_time = datetime.datetime.now() - datetime.timedelta( minutes=bulk_interval) # 判断是否在发送间隔时间内, 如果在间隔时间内,则不发送通知 if Notification.objects.filter(customer=customer, created__gt=need_time, type='bulk'): gevent.sleep(300) continue """ # 当天群发超过300 发通知一次 发第二次时检测 是否超过300*2 第三次是否超过300*3 依次类推 # 当天已发送群发通知的数量 times = Notification.objects.filter(customer=customer, created__contains=datetime.date.today(), type='bulk').count() if not times: times = 1 if bulk_count and count > bulk_count * times: """ notices = Notification.objects.filter( customer=customer, created__contains=datetime.date.today(), type='bulk').order_by('-created') if notices: count = mail_model.objects.filter( state='reject', customer=customer, created__gt=notices[0].created).count() else: count = c['count'] if count > bulk_count: mails = mail_model.objects.exclude( mail_from='<>').filter(state='reject', customer=customer) if notices: mails = mails.filter( created__gt=notices[0].created) sender_dict = mails.values('mail_from').annotate( count=Count('mail_from')).filter( count__gt=10).order_by('-count') if sender_dict: account_list = [] for s in sender_dict: mail_from = re.sub('prvs=(.*?)=', '', s['mail_from']) if mail_from not in account_list: account_list.append(mail_from) if len(account_list) >= 7: break content = Common.safe_format( content_format, count=str(count), account=','.join(account_list)) notice(subject, content, 'bulk', customer) gevent.sleep(300) except (DatabaseError, InterfaceError), e: # 如果报数据库异常,关闭连接,重新处理任务 outerror('DatabaseError') outerror(traceback.format_exc()) connection.close() gevent.sleep(2) except: