def ip_notice(): """ IP不通邮件通知 当发送机连续10分钟接收邮件都失败时,则通知相应管理员该发送机IP不通 :return: """ global notice_setting, bounce_setting subject = u'发送机IP不通通知' type = 'ip' redis = get_redis() while True: try: if notice_setting: outinfo('ip notice') ip_interval = notice_setting.ip_interval content_format = notice_setting.ip_content manager = notice_setting.manager need_time = datetime.datetime.now() - datetime.timedelta( minutes=ip_interval) _, ip = redis.brpop('control_dispatch_exception') # 判断是否在发送间隔时间内, 如果在间隔时间内,则不发送通知 if Notification.objects.filter(manager=manager, created__gt=need_time, type=type, content__icontains=ip): gevent.sleep(300) continue content = Common.safe_format(content_format, ip=ip) notice(subject, content, type, manager=manager) gevent.sleep(300) except: outerror(traceback.format_exc()) gevent.sleep(300)
def save_keyword_review_status(): """ 统计某个关键字 检测次数和 通过次数 :return: """ outinfo('save keword review status start') for d in range(2, 0, -1): date = datetime.date.today() - datetime.timedelta(days=d) _do_save_keyword_review_status(date, get_mail_model, 'subject_blacklist', relay=True) _do_save_keyword_review_status(date, get_mail_model, 'keyword_blacklist', relay=True) _do_save_keyword_review_status(date, get_col_mail_model, 'subject_blacklist', relay=False) _do_save_keyword_review_status(date, get_col_mail_model, 'keyword_blacklist', relay=False) _do_save_keyword_review_status_auto_reject(date) outinfo('save keword review status end')
def _do_check_format(self): if self.mail_from != '<>' and not check_email_format( self.mail_from, is_check_suffix=False): self.mail_obj.check_result = 'error_format' outinfo(u'[ErrorFormat]:{}: {}'.format(self.task, self.mail_from)) return True return False
def _do_attach_check(self): """ 小危附件:自动删除 非中文 邮件中 xxx 附件类型 且 小于XXX KB的邮件,直接删除,不审核,不学习。过滤顺序在 发件人白名单检测 之后。 :return: """ attachments = self.attachments if setting.collect_attachment_min_size and len(attachments) == 1: try: attachment = attachments[0] filename = attachment['decode_name'] if filename.split( '.' )[-1] in attach_type_blacklist and attachment[ 'size'] <= setting.collect_attachment_min_size and re.match( RES_CYBER_REJECT, filename) and (not self.subject or re.match( RES_CYBER_REJECT, self.subject)): self.mail_obj.check_result = 'auto_reject_attach' self.mail_obj.check_message = filename outinfo(u'[SmallRiskAttach(min_attachment)]: {}'.format( self.task)) return True except: outerror(u'attach_type_blacklist error:{}'.format(self.task)) outerror(traceback.format_exc()) return False
def _do_check_recipient_whitelist(self, mail_obj): """ 中继收件人域名白名单,凡是收件人域名在白名单中,只做格式检测和DSPAM过滤,然后就直接发送。 :return: """ # 有多个收件人的不进行处理 # if self.mails_count: # return False # 发件人在发件人白名单中不进行处理 if self.mail_obj.check_result == 'sender_whitelist': return False try: if (mail_obj.mail_to.lower() in recipient_whitelist or mail_obj.mail_to.split('@')[-1].lower() in recipient_whitelist_domain) \ and self.check_result not in ['spam', 'innocent', 'error_format'] and is_credit_gt_default(mail_obj.mail_from): #if mail_obj.mail_to.split('@')[-1] in recipient_whitelist and self.check_result not in ['spam', 'innocent', # 'error_format']: outinfo(u'[RecipientWhitelist]:{}'.format( mail_obj.get_mail_filename())) # mail_obj.check_result = self.check_result mail_obj.check_result = 'recipient_whitelist' mail_obj.check_message = self.mail_obj.check_message mail_obj.review_result = 'pass' mail_obj.state = 'dispatch' mail_obj.save(update_fields=[ 'check_result', 'check_message', 'review_result', 'state', 'dspam_sig' ]) mail_obj.save_mail_for_pop() redis.lpush('relay_dispatch', mail_obj.get_mail_filename()) return True except BaseException as e: outerror('recipient whitelist error:{}'.format( mail_obj.get_mail_filename())) outerror(traceback.format_exc()) return False
def stat_sender_grey_list_1(): outinfo('stat_sender_grey_list_1: start') # config diff_sender_name_count_min = check_setting.hrisk_diff_sender_count expire = check_setting.hrisk_diff_sender_time * 60 today = datetime.date.today() yesterday = today - datetime.timedelta(days=1) t1 = today.strftime('mail_%Y%m%d') t2 = yesterday.strftime('mail_%Y%m%d') table = '((select * from {}) union all (select * from {})) as u'.format( t1, t2) with connection.cursor() as curs: curs.execute(sender_name_count_sql.format(table), [diff_sender_name_count_min]) grey_list = [mail_from for mail_from, in curs] now = time.time() old_dict = redis.hgetall(KEY_1) new_dict = {} for sender, add_time in old_dict.iteritems(): if float(add_time) + expire >= now: new_dict[sender] = add_time for sender in grey_list: if sender not in new_dict: new_dict[sender] = now outinfo('stat_sender_grey_list_1: update redis, size={}'.format( len(new_dict))) redis.delete(KEY_1_TEMP) redis.hmset(KEY_1_TEMP, new_dict) redis.rename(KEY_1_TEMP, KEY_1)
def can_decode(s): try: s.decode('utf-8') return True except UnicodeDecodeError: outinfo('can not decode {!r}'.format(s)) return False
def _ctasd_spam(self): if setting.ctasd_max_size and self.size > setting.ctasd_max_size: return False try: with gevent.Timeout(60): r = ctasd.check_out_mail_data(self.mail_obj.get_mail_content(), self.mail_from.encode('utf-8'), self.mail_to_length) res_spam = r['X-CTCH-Spam'].lower() res_virus = r['X-CTCH-VOD'].lower() if res_spam in ctasd_spam_status: self.mail_obj.check_result = 'cyber_spam' self.mail_obj.check_message = 'X-CTCH-Spam: {}'.format( r['X-CTCH-Spam']) outinfo(u'[Ctasd:Spam] {}: {}'.format(self.task, res_spam)) return True elif res_virus in ctasd_virus_status: self.mail_obj.check_result = 'cyber_spam' self.mail_obj.check_message = 'X-CTCH-VOD: {}'.format( res_virus) outinfo(u'[Ctasd:Virus] {}: {}'.format( self.task, res_virus)) return True except gevent.Timeout: outerror('ctasd check time out :{}'.format(self.task)) outerror(traceback.format_exc()) except BaseException as e: outerror('ctasd check error :{}'.format(self.task)) outerror(traceback.format_exc()) return False
def review_notice(): """ 审核邮件通知 需审核的邮件超过XXX封(中继+网关-cyber)发送通知给默认审核人员 :return: """ global notice_setting, bounce_setting subject = u'审核邮件通知' type = 'review' while True: try: if notice_setting: outinfo('review notice') review_count = notice_setting.review_count review_interval = notice_setting.review_interval content_format = notice_setting.review_content reviewer = notice_setting.reviewer need_time = datetime.datetime.now() - datetime.timedelta( minutes=review_interval) # 判断是否在发送间隔时间内, 如果在间隔时间内,则不发送通知 if Notification.objects.filter(manager=reviewer, created__gt=need_time, type=type): gevent.sleep(300) continue count = get_mail_count(get_mail_model, 'review') # count = get_review_count(get_mail_model) + get_review_count(get_mail_model2) if count >= review_count: content = Common.safe_format(content_format, count=str(count)) notice(subject, content, type, manager=reviewer) gevent.sleep(300) except: outerror(traceback.format_exc()) gevent.sleep(300)
def review_statistic(): """ 邮件审核人员管理: 1、统计审核的邮件总数,通过/拒绝的邮件数 2、统计邮件平均审核时间,就是待审核邮件的平均等待时间。 3、统计邮件的审核正确率,xx.xx%显示格式(被其他人员纠正的邮件,就是误判的邮件) :return: """ outinfo('review statistic start') for d in range(2, 0, -1): date = datetime.date.today() - datetime.timedelta(days=d) print date check_list = [ 'active_spam', 'high_risk', 'keyword_blacklist', 'sender_blacklist', 'subject_blacklist', 'cyber_spam', 'spamassassin', 'recipient_blacklist' ] save_review_statistic(date, get_mail_model, ReviewStatistics, check_list, 'all') save_review_statistic(date, get_mail_model, ReviewStatistics, check_list, 'reviewer') check_list = [ 'high_risk', 'keyword_blacklist', 'sender_blacklist', 'subject_blacklist', 'cyber_spam', 'spamassassin', 'spf', 'force_check' ] save_review_statistic(date, get_col_mail_model, ColReviewStatistics, check_list, 'all') save_review_statistic(date, get_col_mail_model, ColReviewStatistics, check_list, 'reviewer') outinfo('review statistic end')
def dspam_pass(key, get_model): rkey = '{}_pass'.format(key) tmp_rkey = '{}_pass_temp'.format(key) while True: try: j = redis.brpoplpush(rkey, tmp_rkey) date, id = j.split(',')[:2] mail = get_model(date).objects.get(id=id) if mail.mail_id == 0: dspamc(file_path=mail.get_mail_path(), report='innocent', sig=mail.dspam_sig) mail.dspam_study = 2 mail.save(update_fields=['dspam_study']) redis.lrem(tmp_rkey, 0, j) # 将mail_id=0的邮件 保存到POP目录 if mail.mail_id == 0: mail.save_mail_for_pop() outinfo('{}: {}'.format(rkey, j)) if key.find('collect') == -1: _do_save_review_resutl(mail, 'pass') except BaseException as e: outerror('exception: {}'.format(j)) outerror(traceback.format_exc())
def _do_auto_review(self, step, direct_reject=False): # 判断是否直接拒绝 if direct_reject: outinfo(u'{}: {} direct_reject'.format(self.task_info, step)) self.mail_obj.check_result = 'auto_reject' self.mail_obj.check_message += '({})'.format(step) return
def dspam_reject_from_path(dspam_path, get_model): while True: mails = os.listdir(dspam_path) if not mails: gevent.sleep(1) continue for mail in mails: try: mail_path = os.path.join(dspam_path, mail) if not os.path.exists(mail_path): continue try: date, id = mail.split(',')[:2] sig = get_model(date).objects.get(id=id).dspam_sig except Exception, e: sig = '' dspamc(file_path=mail_path, report='spam', sig=sig) outinfo('{}: imap reject'.format(mail)) try: os.remove(mail_path) except: outerror('remove error: {}'.format(mail)) outerror(traceback.format_exc()) except BaseException as e: outerror('exception: {}'.format(mail)) outerror(traceback.format_exc())
def _do_high_sender(self): if redis.hexists(RELAY_SENDER_GREY_LIST, self.mail_from): self.mail_obj.check_result = 'high_sender' outinfo(u'[HighSender] {}'.format(self.task)) return True return False
def init_resource_routine(): while True: try: outinfo('init resouce routine') init_resource() except BaseException as e: outerror('init_resource_routine exception') gevent.sleep(600)
def _esets_old(self): res = esets(self.mail_path) if res['action'] not in ('accepted', 'exception'): self.mail_obj.check_result = 'esets' self.mail_obj.check_message = res['message'] outinfo(u'[Esets] {}'.format(self.task_info)) return True return False
def _do_check_sender_whitelist(self): sender = self.mail_from if sender in sender_whitelist: self.mail_obj.check_result = 'sender_whitelist' self.mail_obj.check_message = u'发件人白名单' outinfo(u'[SenderWhitelist] {}'.format(self.task)) return True return False
def clear_redis_routine(): while True: clear_date = (datetime.datetime.now() - datetime.timedelta(days=7)).strftime('%Y%m%d') outinfo('clear redis: {}'.format(clear_date)) redis.delete('sender_credit-{}'.format(clear_date)) redis.delete('sender_credit+{}'.format(clear_date)) redis.delete('csender_credit-{}'.format(clear_date)) redis.delete('csender_credit+{}'.format(clear_date)) gevent.sleep(3600)
def _do_active_spam(self): if redis.hexists(SPAM_SENDER, self.mail_from): self.mail_obj.check_result = 'active_spam' if not self.mail_obj.check_message: self.mail_obj.check_message = '' outinfo(u'[ActiveSpam] {}: {}'.format(self.task, self.mail_from)) self._do_auto_review('active_spam') return True return False
def _do_check_keyword(self, check_str, keyword_key, direct_reject=False): search_s, s = blacklist[keyword_key].search(check_str, is_dr=direct_reject) if search_s: outinfo(u'[{}]:{}: {}'.format(keyword_key, self.task_info, search_s)) self.mail_obj.check_result = keyword_key if keyword_key != 'attach_blacklist' else 'high_risk' self.mail_obj.check_message = u'{}----{}'.format(search_s, s) self._do_auto_review(keyword_key, direct_reject) return True return False
def _savi(self): for attach in self.attachments: res = savscan_attach(attach) if res: message = ','.join(res) self.mail_obj.check_message = message self.mail_obj.check_result = 'savi' outinfo(u'[Savi] {}: {}'.format(self.task_info, message)) return True return False
def set_bulk_customer(): """ 判断群发客户 :return: """ outinfo('set bulk customer start') for d in range(0, -1, -1): date = datetime.date.today() - datetime.timedelta(days=d) _do_bulk_customer(date) outinfo('set bulk customer end')
def relay_limit_notice(): """ 定时一次性发送 中继用户超过限制通知, 当用户超过设置值时,发送通知 :return: """ global notice_setting, bounce_setting notice_type = "relay" while True: try: cur = datetime.datetime.now() if cur.hour != 1 or cur.minute != 25: gevent.sleep(30) continue outinfo('relay limit notice') all_content = "" content_format = notice_setting.relay_content manager = notice_setting.manager flag = Notification.objects.filter( manager=manager, type=notice_type, created__contains=datetime.date.today()) pre = cur + datetime.timedelta(-1) if not flag: mail_model = get_mail_model(pre.strftime('%Y%m%d')) customers = mail_model.objects.exclude( customer__company__icontains=u'临时信任').values( 'customer').annotate( count=Count("customer")).order_by('-count') for c in customers: customer = Customer.objects.get(pk=c['customer']) company_id = customer.id company = customer.company relay_limit = customer.relay_limit sender_count = mail_model.objects.exclude( mail_from='<>', mail_from__isnull=False).filter( customer=customer).distinct('mail_from').count() if relay_limit != -1 and sender_count > relay_limit: content = Common.safe_format(content_format, company=company, company_id=company_id, setting=relay_limit, count=sender_count) all_content += content if all_content: subject = pre.strftime('%Y-%m-%d ') + u"中继用户超过限制通知" notice(subject=subject, content=all_content, type=notice_type, customer=None, manager=manager) except: outerror(traceback.format_exc()) finally: gevent.sleep(60)
def collect_limit_notice(): """ 定时一次性发送前一天的统计数据,hour_to设置定时发送时间, 网关用户超过限制通知, 当用户超过设置值时,发送通知 :return: """ global notice_setting, bounce_setting hour_to = 1 notice_type = "collect" while True: try: cur = datetime.datetime.now() if cur.hour == hour_to and notice_setting: outinfo('collect limit notice') all_content = "" content_format = notice_setting.collect_content manager = notice_setting.manager flag = Notification.objects.filter( manager=manager, type=notice_type, created__contains=datetime.date.today()) pre = cur + datetime.timedelta(-1) if not flag: mail_model = get_mail_model2(pre.strftime('%Y%m%d')) customers = mail_model.objects.exclude( customer__company__icontains=u'临时信任').values( 'customer').annotate( count=Count("customer")).order_by('-count') for c in customers: customer = Customer.objects.get(pk=c['customer']) company_id = customer.id company = customer.company collect_limit = customer.collect_limit rcp_count = mail_model.objects.exclude( mail_to='<>', mail_to__isnull=False).filter( customer=customer).distinct('mail_to').count() if collect_limit != -1 and rcp_count > collect_limit: content = Common.safe_format(content_format, company=company, company_id=company_id, setting=collect_limit, count=rcp_count) all_content += content if all_content: subject = pre.strftime('%Y-%m-%d ') + u"网关用户超过限制通知" notice(subject=subject, content=all_content, type=notice_type, customer=None, manager=manager) except: outerror(traceback.format_exc()) finally: gevent.sleep(900)
def _do_check_spf(self): if self.mail_obj.check_result == 'spf': return True domain = self.mail_from.split('@')[-1].lower() if self.customer_setting.check_spf or check_list(domain, spf_checklist): if not self._do_spf_ip_whitelist() and not check_spf(self.mail_from, self.real_client_ip): self.mail_obj.check_result = 'spf' outinfo(u'[spf]:{}: {} {}'.format(self.task_info, self.mail_from, self.real_client_ip)) return True return False
def init_resource_routine(): while redis.rpoplpush(MAIL_INCHECK_ERROR_QUEUE, MAIL_INCHECK_QUEUE) is not None: pass while True: if signal_stop: break try: outinfo(u'init resouce routine') init_resource() except BaseException as e: outerror(u'init_resource_routine exception') gevent.sleep(180)
def clear_invalid_email(): """ 清除过期的无效地址 :return: """ outinfo('clear invalid start') expire_days = setting.invalid_mail_expire_days active_time = datetime.datetime.now() - datetime.timedelta( days=expire_days) InvalidMail.objects.filter(created__lte=active_time).delete() outinfo('clear invalid end')
def _do_check_for_same_mail(self): mail_obj = self.mail_obj for m in self.same_mails: if not self._do_check_recipient_whitelist(m): m.check_result = mail_obj.check_result m.check_message = mail_obj.check_message m.state = mail_obj.state m.review_result = mail_obj.review_result m.save(update_fields=['check_result', 'check_message', 'review_result', 'state', 'dspam_sig']) if mail_obj.check_result in check_result_pass: redis.lpush(MAIL_SEND_QUEUE, m.get_mail_filename()) outinfo(u'check same mail:{}'.format(m.get_mail_filename()))
def _do_check_format(self): if not self.subject and not self.content and not self.attachments: self.mail_obj.check_result = 'error_format' outinfo(u'[ErrorFormat]:{}: no content'.format(self.task_info)) return True if not self.customer_setting.check_format: return False if not check_email_format(self.mail_from, is_check_suffix=False): self.mail_obj.check_result = 'error_format' outinfo(u'[ErrorFormat]:{}: {}'.format(self.task_info, self.mail_from)) return True return False
def _check_spf(self): if self._do_check_spf(): #强制SPF检查域名库 中的免审直接拒绝 domain = self.mail_from.split('@')[-1].lower() for k, v in spf_checklist.iteritems(): #if domain.endswith(k) and v: if domain == k and v: self.mail_obj.check_result = 'auto_reject' self.mail_obj.check_message = 'spf' outinfo(u'[spf]:direct reject: {} {}'.format(self.task_info, self.mail_from)) return True return True return False