def ip_notice():
"""
IP不通邮件通知 当发送机连续10分钟接收邮件都失败时，则通知相应管理员该发送机IP不通
:return:
"""
global notice_setting, bounce_setting
subject = u'发送机IP不通通知'
type = 'ip'
redis = get_redis()
while True:
try:
if notice_setting:
outinfo('ip notice')
ip_interval = notice_setting.ip_interval
content_format = notice_setting.ip_content
manager = notice_setting.manager
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=ip_interval)
_, ip = redis.brpop('control_dispatch_exception')
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(manager=manager,
created__gt=need_time,
type=type,
content__icontains=ip):
gevent.sleep(300)
continue
content = Common.safe_format(content_format, ip=ip)
notice(subject, content, type, manager=manager)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def save_keyword_review_status():
"""
统计某个关键字 检测次数和 通过次数
:return:
"""
outinfo('save keword review status start')
for d in range(2, 0, -1):
date = datetime.date.today() - datetime.timedelta(days=d)
_do_save_keyword_review_status(date,
get_mail_model,
'subject_blacklist',
relay=True)
_do_save_keyword_review_status(date,
get_mail_model,
'keyword_blacklist',
relay=True)
_do_save_keyword_review_status(date,
get_col_mail_model,
'subject_blacklist',
relay=False)
_do_save_keyword_review_status(date,
get_col_mail_model,
'keyword_blacklist',
relay=False)
_do_save_keyword_review_status_auto_reject(date)
outinfo('save keword review status end')
def _do_check_format(self):
if self.mail_from != '<>' and not check_email_format(
self.mail_from, is_check_suffix=False):
self.mail_obj.check_result = 'error_format'
outinfo(u'[ErrorFormat]:{}: {}'.format(self.task, self.mail_from))
return True
return False
def _do_attach_check(self):
"""
小危附件：自动删除 非中文 邮件中 xxx 附件类型 且 小于XXX KB的邮件，直接删除，不审核，不学习。过滤顺序在 发件人白名单检测 之后。
:return:
"""
attachments = self.attachments
if setting.collect_attachment_min_size and len(attachments) == 1:
try:
attachment = attachments[0]
filename = attachment['decode_name']
if filename.split(
'.'
)[-1] in attach_type_blacklist and attachment[
'size'] <= setting.collect_attachment_min_size and re.match(
RES_CYBER_REJECT,
filename) and (not self.subject or re.match(
RES_CYBER_REJECT, self.subject)):
self.mail_obj.check_result = 'auto_reject_attach'
self.mail_obj.check_message = filename
outinfo(u'[SmallRiskAttach(min_attachment)]: {}'.format(
self.task))
return True
except:
outerror(u'attach_type_blacklist error:{}'.format(self.task))
outerror(traceback.format_exc())
return False
def _do_check_recipient_whitelist(self, mail_obj):
"""
中继收件人域名白名单，凡是收件人域名在白名单中，只做格式检测和DSPAM过滤，然后就直接发送。
:return:
"""
# 有多个收件人的不进行处理
# if self.mails_count:
# return False
# 发件人在发件人白名单中不进行处理
if self.mail_obj.check_result == 'sender_whitelist':
return False
try:
if (mail_obj.mail_to.lower() in recipient_whitelist or mail_obj.mail_to.split('@')[-1].lower() in recipient_whitelist_domain) \
and self.check_result not in ['spam', 'innocent', 'error_format'] and is_credit_gt_default(mail_obj.mail_from):
#if mail_obj.mail_to.split('@')[-1] in recipient_whitelist and self.check_result not in ['spam', 'innocent',
# 'error_format']:
outinfo(u'[RecipientWhitelist]:{}'.format(
mail_obj.get_mail_filename()))
# mail_obj.check_result = self.check_result
mail_obj.check_result = 'recipient_whitelist'
mail_obj.check_message = self.mail_obj.check_message
mail_obj.review_result = 'pass'
mail_obj.state = 'dispatch'
mail_obj.save(update_fields=[
'check_result', 'check_message', 'review_result', 'state',
'dspam_sig'
])
mail_obj.save_mail_for_pop()
redis.lpush('relay_dispatch', mail_obj.get_mail_filename())
return True
except BaseException as e:
outerror('recipient whitelist error:{}'.format(
mail_obj.get_mail_filename()))
outerror(traceback.format_exc())
return False
def stat_sender_grey_list_1():
outinfo('stat_sender_grey_list_1: start')
# config
diff_sender_name_count_min = check_setting.hrisk_diff_sender_count
expire = check_setting.hrisk_diff_sender_time * 60
today = datetime.date.today()
yesterday = today - datetime.timedelta(days=1)
t1 = today.strftime('mail_%Y%m%d')
t2 = yesterday.strftime('mail_%Y%m%d')
table = '((select * from {}) union all (select * from {})) as u'.format(
t1, t2)
with connection.cursor() as curs:
curs.execute(sender_name_count_sql.format(table),
[diff_sender_name_count_min])
grey_list = [mail_from for mail_from, in curs]
now = time.time()
old_dict = redis.hgetall(KEY_1)
new_dict = {}
for sender, add_time in old_dict.iteritems():
if float(add_time) + expire >= now:
new_dict[sender] = add_time
for sender in grey_list:
if sender not in new_dict:
new_dict[sender] = now
outinfo('stat_sender_grey_list_1: update redis, size={}'.format(
len(new_dict)))
redis.delete(KEY_1_TEMP)
redis.hmset(KEY_1_TEMP, new_dict)
redis.rename(KEY_1_TEMP, KEY_1)
def can_decode(s):
try:
s.decode('utf-8')
return True
except UnicodeDecodeError:
outinfo('can not decode {!r}'.format(s))
return False
def _ctasd_spam(self):
if setting.ctasd_max_size and self.size > setting.ctasd_max_size:
return False
try:
with gevent.Timeout(60):
r = ctasd.check_out_mail_data(self.mail_obj.get_mail_content(),
self.mail_from.encode('utf-8'),
self.mail_to_length)
res_spam = r['X-CTCH-Spam'].lower()
res_virus = r['X-CTCH-VOD'].lower()
if res_spam in ctasd_spam_status:
self.mail_obj.check_result = 'cyber_spam'
self.mail_obj.check_message = 'X-CTCH-Spam: {}'.format(
r['X-CTCH-Spam'])
outinfo(u'[Ctasd:Spam] {}: {}'.format(self.task, res_spam))
return True
elif res_virus in ctasd_virus_status:
self.mail_obj.check_result = 'cyber_spam'
self.mail_obj.check_message = 'X-CTCH-VOD: {}'.format(
res_virus)
outinfo(u'[Ctasd:Virus] {}: {}'.format(
self.task, res_virus))
return True
except gevent.Timeout:
outerror('ctasd check time out :{}'.format(self.task))
outerror(traceback.format_exc())
except BaseException as e:
outerror('ctasd check error :{}'.format(self.task))
outerror(traceback.format_exc())
return False
def review_notice():
"""
审核邮件通知 需审核的邮件超过XXX封（中继+网关-cyber）发送通知给默认审核人员
:return:
"""
global notice_setting, bounce_setting
subject = u'审核邮件通知'
type = 'review'
while True:
try:
if notice_setting:
outinfo('review notice')
review_count = notice_setting.review_count
review_interval = notice_setting.review_interval
content_format = notice_setting.review_content
reviewer = notice_setting.reviewer
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=review_interval)
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(manager=reviewer,
created__gt=need_time,
type=type):
gevent.sleep(300)
continue
count = get_mail_count(get_mail_model, 'review')
# count = get_review_count(get_mail_model) + get_review_count(get_mail_model2)
if count >= review_count:
content = Common.safe_format(content_format,
count=str(count))
notice(subject, content, type, manager=reviewer)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def review_statistic():
"""
邮件审核人员管理：
1、统计审核的邮件总数，通过/拒绝的邮件数
2、统计邮件平均审核时间，就是待审核邮件的平均等待时间。
3、统计邮件的审核正确率，xx.xx%显示格式（被其他人员纠正的邮件，就是误判的邮件）
:return:
"""
outinfo('review statistic start')
for d in range(2, 0, -1):
date = datetime.date.today() - datetime.timedelta(days=d)
print date
check_list = [
'active_spam', 'high_risk', 'keyword_blacklist',
'sender_blacklist', 'subject_blacklist', 'cyber_spam',
'spamassassin', 'recipient_blacklist'
]
save_review_statistic(date, get_mail_model, ReviewStatistics,
check_list, 'all')
save_review_statistic(date, get_mail_model, ReviewStatistics,
check_list, 'reviewer')
check_list = [
'high_risk', 'keyword_blacklist', 'sender_blacklist',
'subject_blacklist', 'cyber_spam', 'spamassassin', 'spf',
'force_check'
]
save_review_statistic(date, get_col_mail_model, ColReviewStatistics,
check_list, 'all')
save_review_statistic(date, get_col_mail_model, ColReviewStatistics,
check_list, 'reviewer')
outinfo('review statistic end')
def dspam_pass(key, get_model):
rkey = '{}_pass'.format(key)
tmp_rkey = '{}_pass_temp'.format(key)
while True:
try:
j = redis.brpoplpush(rkey, tmp_rkey)
date, id = j.split(',')[:2]
mail = get_model(date).objects.get(id=id)
if mail.mail_id == 0:
dspamc(file_path=mail.get_mail_path(),
report='innocent',
sig=mail.dspam_sig)
mail.dspam_study = 2
mail.save(update_fields=['dspam_study'])
redis.lrem(tmp_rkey, 0, j)
# 将mail_id=0的邮件 保存到POP目录
if mail.mail_id == 0:
mail.save_mail_for_pop()
outinfo('{}: {}'.format(rkey, j))
if key.find('collect') == -1:
_do_save_review_resutl(mail, 'pass')
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
def _do_auto_review(self, step, direct_reject=False):
# 判断是否直接拒绝
if direct_reject:
outinfo(u'{}: {} direct_reject'.format(self.task_info, step))
self.mail_obj.check_result = 'auto_reject'
self.mail_obj.check_message += '({})'.format(step)
return
def dspam_reject_from_path(dspam_path, get_model):
while True:
mails = os.listdir(dspam_path)
if not mails:
gevent.sleep(1)
continue
for mail in mails:
try:
mail_path = os.path.join(dspam_path, mail)
if not os.path.exists(mail_path):
continue
try:
date, id = mail.split(',')[:2]
sig = get_model(date).objects.get(id=id).dspam_sig
except Exception, e:
sig = ''
dspamc(file_path=mail_path, report='spam', sig=sig)
outinfo('{}: imap reject'.format(mail))
try:
os.remove(mail_path)
except:
outerror('remove error: {}'.format(mail))
outerror(traceback.format_exc())
except BaseException as e:
outerror('exception: {}'.format(mail))
outerror(traceback.format_exc())
def _do_high_sender(self):
if redis.hexists(RELAY_SENDER_GREY_LIST, self.mail_from):
self.mail_obj.check_result = 'high_sender'
outinfo(u'[HighSender] {}'.format(self.task))
return True
return False
def init_resource_routine():
while True:
try:
outinfo('init resouce routine')
init_resource()
except BaseException as e:
outerror('init_resource_routine exception')
gevent.sleep(600)
def _esets_old(self):
res = esets(self.mail_path)
if res['action'] not in ('accepted', 'exception'):
self.mail_obj.check_result = 'esets'
self.mail_obj.check_message = res['message']
outinfo(u'[Esets] {}'.format(self.task_info))
return True
return False
def _do_check_sender_whitelist(self):
sender = self.mail_from
if sender in sender_whitelist:
self.mail_obj.check_result = 'sender_whitelist'
self.mail_obj.check_message = u'发件人白名单'
outinfo(u'[SenderWhitelist] {}'.format(self.task))
return True
return False
def clear_redis_routine():
while True:
clear_date = (datetime.datetime.now() - datetime.timedelta(days=7)).strftime('%Y%m%d')
outinfo('clear redis: {}'.format(clear_date))
redis.delete('sender_credit-{}'.format(clear_date))
redis.delete('sender_credit+{}'.format(clear_date))
redis.delete('csender_credit-{}'.format(clear_date))
redis.delete('csender_credit+{}'.format(clear_date))
gevent.sleep(3600)
def _do_active_spam(self):
if redis.hexists(SPAM_SENDER, self.mail_from):
self.mail_obj.check_result = 'active_spam'
if not self.mail_obj.check_message:
self.mail_obj.check_message = ''
outinfo(u'[ActiveSpam] {}: {}'.format(self.task, self.mail_from))
self._do_auto_review('active_spam')
return True
return False
def _do_check_keyword(self, check_str, keyword_key, direct_reject=False):
search_s, s = blacklist[keyword_key].search(check_str, is_dr=direct_reject)
if search_s:
outinfo(u'[{}]:{}: {}'.format(keyword_key, self.task_info, search_s))
self.mail_obj.check_result = keyword_key if keyword_key != 'attach_blacklist' else 'high_risk'
self.mail_obj.check_message = u'{}----{}'.format(search_s, s)
self._do_auto_review(keyword_key, direct_reject)
return True
return False
def _savi(self):
for attach in self.attachments:
res = savscan_attach(attach)
if res:
message = ','.join(res)
self.mail_obj.check_message = message
self.mail_obj.check_result = 'savi'
outinfo(u'[Savi] {}: {}'.format(self.task_info, message))
return True
return False
def set_bulk_customer():
"""
判断群发客户
:return:
"""
outinfo('set bulk customer start')
for d in range(0, -1, -1):
date = datetime.date.today() - datetime.timedelta(days=d)
_do_bulk_customer(date)
outinfo('set bulk customer end')
def relay_limit_notice():
"""
定时一次性发送
中继用户超过限制通知, 当用户超过设置值时，发送通知
:return:
"""
global notice_setting, bounce_setting
notice_type = "relay"
while True:
try:
cur = datetime.datetime.now()
if cur.hour != 1 or cur.minute != 25:
gevent.sleep(30)
continue
outinfo('relay limit notice')
all_content = ""
content_format = notice_setting.relay_content
manager = notice_setting.manager
flag = Notification.objects.filter(
manager=manager,
type=notice_type,
created__contains=datetime.date.today())
pre = cur + datetime.timedelta(-1)
if not flag:
mail_model = get_mail_model(pre.strftime('%Y%m%d'))
customers = mail_model.objects.exclude(
customer__company__icontains=u'临时信任').values(
'customer').annotate(
count=Count("customer")).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
company_id = customer.id
company = customer.company
relay_limit = customer.relay_limit
sender_count = mail_model.objects.exclude(
mail_from='<>', mail_from__isnull=False).filter(
customer=customer).distinct('mail_from').count()
if relay_limit != -1 and sender_count > relay_limit:
content = Common.safe_format(content_format,
company=company,
company_id=company_id,
setting=relay_limit,
count=sender_count)
all_content += content
if all_content:
subject = pre.strftime('%Y-%m-%d ') + u"中继用户超过限制通知"
notice(subject=subject,
content=all_content,
type=notice_type,
customer=None,
manager=manager)
except:
outerror(traceback.format_exc())
finally:
gevent.sleep(60)
def collect_limit_notice():
"""
定时一次性发送前一天的统计数据,hour_to设置定时发送时间，
网关用户超过限制通知, 当用户超过设置值时，发送通知
:return:
"""
global notice_setting, bounce_setting
hour_to = 1
notice_type = "collect"
while True:
try:
cur = datetime.datetime.now()
if cur.hour == hour_to and notice_setting:
outinfo('collect limit notice')
all_content = ""
content_format = notice_setting.collect_content
manager = notice_setting.manager
flag = Notification.objects.filter(
manager=manager,
type=notice_type,
created__contains=datetime.date.today())
pre = cur + datetime.timedelta(-1)
if not flag:
mail_model = get_mail_model2(pre.strftime('%Y%m%d'))
customers = mail_model.objects.exclude(
customer__company__icontains=u'临时信任').values(
'customer').annotate(
count=Count("customer")).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
company_id = customer.id
company = customer.company
collect_limit = customer.collect_limit
rcp_count = mail_model.objects.exclude(
mail_to='<>', mail_to__isnull=False).filter(
customer=customer).distinct('mail_to').count()
if collect_limit != -1 and rcp_count > collect_limit:
content = Common.safe_format(content_format,
company=company,
company_id=company_id,
setting=collect_limit,
count=rcp_count)
all_content += content
if all_content:
subject = pre.strftime('%Y-%m-%d ') + u"网关用户超过限制通知"
notice(subject=subject,
content=all_content,
type=notice_type,
customer=None,
manager=manager)
except:
outerror(traceback.format_exc())
finally:
gevent.sleep(900)
def _do_check_spf(self):
if self.mail_obj.check_result == 'spf':
return True
domain = self.mail_from.split('@')[-1].lower()
if self.customer_setting.check_spf or check_list(domain, spf_checklist):
if not self._do_spf_ip_whitelist() and not check_spf(self.mail_from, self.real_client_ip):
self.mail_obj.check_result = 'spf'
outinfo(u'[spf]:{}: {} {}'.format(self.task_info, self.mail_from, self.real_client_ip))
return True
return False
def init_resource_routine():
while redis.rpoplpush(MAIL_INCHECK_ERROR_QUEUE, MAIL_INCHECK_QUEUE) is not None:
pass
while True:
if signal_stop: break
try:
outinfo(u'init resouce routine')
init_resource()
except BaseException as e:
outerror(u'init_resource_routine exception')
gevent.sleep(180)
def clear_invalid_email():
"""
清除过期的无效地址
:return:
"""
outinfo('clear invalid start')
expire_days = setting.invalid_mail_expire_days
active_time = datetime.datetime.now() - datetime.timedelta(
days=expire_days)
InvalidMail.objects.filter(created__lte=active_time).delete()
outinfo('clear invalid end')
def _do_check_for_same_mail(self):
mail_obj = self.mail_obj
for m in self.same_mails:
if not self._do_check_recipient_whitelist(m):
m.check_result = mail_obj.check_result
m.check_message = mail_obj.check_message
m.state = mail_obj.state
m.review_result = mail_obj.review_result
m.save(update_fields=['check_result', 'check_message', 'review_result', 'state', 'dspam_sig'])
if mail_obj.check_result in check_result_pass:
redis.lpush(MAIL_SEND_QUEUE, m.get_mail_filename())
outinfo(u'check same mail:{}'.format(m.get_mail_filename()))
def _do_check_format(self):
if not self.subject and not self.content and not self.attachments:
self.mail_obj.check_result = 'error_format'
outinfo(u'[ErrorFormat]:{}: no content'.format(self.task_info))
return True
if not self.customer_setting.check_format:
return False
if not check_email_format(self.mail_from, is_check_suffix=False):
self.mail_obj.check_result = 'error_format'
outinfo(u'[ErrorFormat]:{}: {}'.format(self.task_info, self.mail_from))
return True
return False
def _check_spf(self):
if self._do_check_spf():
#强制SPF检查域名库 中的免审直接拒绝
domain = self.mail_from.split('@')[-1].lower()
for k, v in spf_checklist.iteritems():
#if domain.endswith(k) and v:
if domain == k and v:
self.mail_obj.check_result = 'auto_reject'
self.mail_obj.check_message = 'spf'
outinfo(u'[spf]:direct reject: {} {}'.format(self.task_info, self.mail_from))
return True
return True
return False
