def handle_error(self):
try:
error_file = os.path.join(error_path, self.task)
os.rename(self.mail_path, error_file)
except:
outerror('handle error:{}'.format(self.task))
outerror(traceback.format_exc())
def ip_notice():
"""
IP不通邮件通知 当发送机连续10分钟接收邮件都失败时，则通知相应管理员该发送机IP不通
:return:
"""
global notice_setting, bounce_setting
subject = u'发送机IP不通通知'
type = 'ip'
redis = get_redis()
while True:
try:
if notice_setting:
outinfo('ip notice')
ip_interval = notice_setting.ip_interval
content_format = notice_setting.ip_content
manager = notice_setting.manager
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=ip_interval)
_, ip = redis.brpop('control_dispatch_exception')
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(manager=manager,
created__gt=need_time,
type=type,
content__icontains=ip):
gevent.sleep(300)
continue
content = Common.safe_format(content_format, ip=ip)
notice(subject, content, type, manager=manager)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def dspam_pass(key, get_model):
rkey = '{}_pass'.format(key)
tmp_rkey = '{}_pass_temp'.format(key)
while True:
try:
j = redis.brpoplpush(rkey, tmp_rkey)
date, id = j.split(',')[:2]
mail = get_model(date).objects.get(id=id)
if mail.mail_id == 0:
dspamc(file_path=mail.get_mail_path(),
report='innocent',
sig=mail.dspam_sig)
mail.dspam_study = 2
mail.save(update_fields=['dspam_study'])
redis.lrem(tmp_rkey, 0, j)
# 将mail_id=0的邮件 保存到POP目录
if mail.mail_id == 0:
mail.save_mail_for_pop()
outinfo('{}: {}'.format(rkey, j))
if key.find('collect') == -1:
_do_save_review_resutl(mail, 'pass')
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
def review_notice():
"""
审核邮件通知 需审核的邮件超过XXX封（中继+网关-cyber）发送通知给默认审核人员
:return:
"""
global notice_setting, bounce_setting
subject = u'审核邮件通知'
type = 'review'
while True:
try:
if notice_setting:
outinfo('review notice')
review_count = notice_setting.review_count
review_interval = notice_setting.review_interval
content_format = notice_setting.review_content
reviewer = notice_setting.reviewer
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=review_interval)
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(manager=reviewer,
created__gt=need_time,
type=type):
gevent.sleep(300)
continue
count = get_mail_count(get_mail_model, 'review')
# count = get_review_count(get_mail_model) + get_review_count(get_mail_model2)
if count >= review_count:
content = Common.safe_format(content_format,
count=str(count))
notice(subject, content, type, manager=reviewer)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def _do_attach_check(self):
"""
小危附件：自动删除 非中文 邮件中 xxx 附件类型 且 小于XXX KB的邮件，直接删除，不审核，不学习。过滤顺序在 发件人白名单检测 之后。
:return:
"""
attachments = self.attachments
if setting.collect_attachment_min_size and len(attachments) == 1:
try:
attachment = attachments[0]
filename = attachment['decode_name']
if filename.split(
'.'
)[-1] in attach_type_blacklist and attachment[
'size'] <= setting.collect_attachment_min_size and re.match(
RES_CYBER_REJECT,
filename) and (not self.subject or re.match(
RES_CYBER_REJECT, self.subject)):
self.mail_obj.check_result = 'auto_reject_attach'
self.mail_obj.check_message = filename
outinfo(u'[SmallRiskAttach(min_attachment)]: {}'.format(
self.task))
return True
except:
outerror(u'attach_type_blacklist error:{}'.format(self.task))
outerror(traceback.format_exc())
return False
def _do_check_recipient_whitelist(self, mail_obj):
"""
中继收件人域名白名单，凡是收件人域名在白名单中，只做格式检测和DSPAM过滤，然后就直接发送。
:return:
"""
# 有多个收件人的不进行处理
# if self.mails_count:
# return False
# 发件人在发件人白名单中不进行处理
if self.mail_obj.check_result == 'sender_whitelist':
return False
try:
if (mail_obj.mail_to.lower() in recipient_whitelist or mail_obj.mail_to.split('@')[-1].lower() in recipient_whitelist_domain) \
and self.check_result not in ['spam', 'innocent', 'error_format'] and is_credit_gt_default(mail_obj.mail_from):
#if mail_obj.mail_to.split('@')[-1] in recipient_whitelist and self.check_result not in ['spam', 'innocent',
# 'error_format']:
outinfo(u'[RecipientWhitelist]:{}'.format(
mail_obj.get_mail_filename()))
# mail_obj.check_result = self.check_result
mail_obj.check_result = 'recipient_whitelist'
mail_obj.check_message = self.mail_obj.check_message
mail_obj.review_result = 'pass'
mail_obj.state = 'dispatch'
mail_obj.save(update_fields=[
'check_result', 'check_message', 'review_result', 'state',
'dspam_sig'
])
mail_obj.save_mail_for_pop()
redis.lpush('relay_dispatch', mail_obj.get_mail_filename())
return True
except BaseException as e:
outerror('recipient whitelist error:{}'.format(
mail_obj.get_mail_filename()))
outerror(traceback.format_exc())
return False
def finish(self):
if not self.mail_obj.check_result:
self.mail_obj.check_result = 'innocent'
check_result = self.mail_obj.check_result
try:
# 清空邮件
# 直接拒绝
if check_result in check_result_reject:
self.mail_obj.review_result = 'reject'
self.mail_obj.state = 'reject'
elif check_result in check_result_pass:
self.mail_obj.review_result = 'pass'
self.mail_obj.state = 'send'
# 需要审核
else:
self.mail_obj.state = 'review'
redis.lpush('collect_sync', self.key)
self._do_check_for_same_mail()
if not self._do_check_recipient_whitelist(self.mail_obj):
self.mail_obj.save(update_fields=['check_result', 'check_message', 'review_result', 'state', 'dspam_sig'])
if check_result in check_result_pass:
self.mail_obj.save_mail_for_pop()
redis.lpush(MAIL_SEND_QUEUE, self.key)
if check_result in check_auto_study:
redis.lpush('dspam_collect_reject', self.key)
if review_help_mode == 'client' and self.mail_obj.state == 'review':
redis.lpush('control_review_help', 'collect,' + self.task)
except BaseException as e:
outerror(u'finish error:{}'.format(self.task_info))
outerror(traceback.format_exc())
def init_resource_routine():
while True:
try:
outinfo('init resouce routine')
init_resource()
except BaseException as e:
outerror('init_resource_routine exception')
gevent.sleep(600)
def worker(task):
try:
p = Processor(task)
p.run()
except ObjectDoesNotExist, e:
outerror('ObjectDoesNotExist: {}'.format(task))
outerror(traceback.format_exc())
# redis.rpush(MAIL_INCHECK_QUEUE, task)
return
def _do_virus(self):
# 进行病毒邮件检测
try:
pyclamd.init_unix_socket(clamav_sock)
res = pyclamd.scan_file(self.mail_path)
except Exception, err:
outerror(u'virus check error :{}'.format(self.task_info))
outerror(traceback.format_exc())
return False
def worker(task):
p = Processor(task)
try:
with gevent.Timeout(60):
p.run()
except gevent.Timeout, e:
p.handle_error()
outerror('Process timeout:{}'.format(task))
outerror(traceback.format_exc())
def relay_limit_notice():
"""
定时一次性发送
中继用户超过限制通知, 当用户超过设置值时，发送通知
:return:
"""
global notice_setting, bounce_setting
notice_type = "relay"
while True:
try:
cur = datetime.datetime.now()
if cur.hour != 1 or cur.minute != 25:
gevent.sleep(30)
continue
outinfo('relay limit notice')
all_content = ""
content_format = notice_setting.relay_content
manager = notice_setting.manager
flag = Notification.objects.filter(
manager=manager,
type=notice_type,
created__contains=datetime.date.today())
pre = cur + datetime.timedelta(-1)
if not flag:
mail_model = get_mail_model(pre.strftime('%Y%m%d'))
customers = mail_model.objects.exclude(
customer__company__icontains=u'临时信任').values(
'customer').annotate(
count=Count("customer")).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
company_id = customer.id
company = customer.company
relay_limit = customer.relay_limit
sender_count = mail_model.objects.exclude(
mail_from='<>', mail_from__isnull=False).filter(
customer=customer).distinct('mail_from').count()
if relay_limit != -1 and sender_count > relay_limit:
content = Common.safe_format(content_format,
company=company,
company_id=company_id,
setting=relay_limit,
count=sender_count)
all_content += content
if all_content:
subject = pre.strftime('%Y-%m-%d ') + u"中继用户超过限制通知"
notice(subject=subject,
content=all_content,
type=notice_type,
customer=None,
manager=manager)
except:
outerror(traceback.format_exc())
finally:
gevent.sleep(60)
def init_resource_routine():
while redis.rpoplpush(MAIL_INCHECK_ERROR_QUEUE, MAIL_INCHECK_QUEUE) is not None:
pass
while True:
if signal_stop: break
try:
outinfo(u'init resouce routine')
init_resource()
except BaseException as e:
outerror(u'init_resource_routine exception')
gevent.sleep(180)
def collect_limit_notice():
"""
定时一次性发送前一天的统计数据,hour_to设置定时发送时间，
网关用户超过限制通知, 当用户超过设置值时，发送通知
:return:
"""
global notice_setting, bounce_setting
hour_to = 1
notice_type = "collect"
while True:
try:
cur = datetime.datetime.now()
if cur.hour == hour_to and notice_setting:
outinfo('collect limit notice')
all_content = ""
content_format = notice_setting.collect_content
manager = notice_setting.manager
flag = Notification.objects.filter(
manager=manager,
type=notice_type,
created__contains=datetime.date.today())
pre = cur + datetime.timedelta(-1)
if not flag:
mail_model = get_mail_model2(pre.strftime('%Y%m%d'))
customers = mail_model.objects.exclude(
customer__company__icontains=u'临时信任').values(
'customer').annotate(
count=Count("customer")).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
company_id = customer.id
company = customer.company
collect_limit = customer.collect_limit
rcp_count = mail_model.objects.exclude(
mail_to='<>', mail_to__isnull=False).filter(
customer=customer).distinct('mail_to').count()
if collect_limit != -1 and rcp_count > collect_limit:
content = Common.safe_format(content_format,
company=company,
company_id=company_id,
setting=collect_limit,
count=rcp_count)
all_content += content
if all_content:
subject = pre.strftime('%Y-%m-%d ') + u"网关用户超过限制通知"
notice(subject=subject,
content=all_content,
type=notice_type,
customer=None,
manager=manager)
except:
outerror(traceback.format_exc())
finally:
gevent.sleep(900)
def worker(task):
try:
p = Processor(task)
p.run()
except (DatabaseError, InterfaceError), e:
#如果报数据库异常，关闭连接，重新处理任务
outerror('DatabaseError: {}'.format(task))
outerror(traceback.format_exc())
connection.close()
#redis.rpush(MAIL_INCHECK_QUEUE, task)
gevent.sleep(10)
return
def _ctasd_spam(self):
# check size
if not self.customer_setting.check_ctasd:
return False
if setting.ctasd_max_size and self.size > setting.ctasd_max_size:
return False
try:
with gevent.Timeout(60):
r = ctasd.check_in_mail_data(self.mail_content,
self.client_ip.encode('utf-8'),
self.mail_from.encode('utf-8'))
res_spam = r['X-CTCH-Spam'].lower()
res_virus = r['X-CTCH-VOD'].lower()
if res_spam in ctasd_spam_status:
if re.match(RES_CYBER_REJECT, self.subject):
self.mail_obj.check_result = 'auto_reject_cyber'
outinfo(u'[Ctasd:Spam] {}: {}: auto_reject_cyber'.format(self.task_info, res_spam))
else:
self.mail_obj.check_result = 'cyber_spam'
outinfo(u'[Ctasd:Spam] {}: {}'.format(self.task_info, res_spam))
self.mail_obj.check_message = 'X-CTCH-Spam: {}'.format(r['X-CTCH-Spam'])
return True
elif res_virus in ctasd_virus_status:
self.mail_obj.check_result = 'cyber_spam'
self.mail_obj.check_message = 'X-CTCH-VOD: {}'.format(res_virus)
outinfo(u'[Ctasd:Virus] {}: {}'.format(self.task_info, res_virus))
return True
except gevent.Timeout:
outerror(u'ctasd check time out :{}'.format(self.task_info))
outerror(traceback.format_exc())
except BaseException as e:
outerror(u'ctasd check error :{}'.format(self.task_info))
outerror(traceback.format_exc())
return False
def _ctasd_spam(self):
if setting.ctasd_max_size and self.size > setting.ctasd_max_size:
return False
try:
with gevent.Timeout(60):
r = ctasd.check_out_mail_data(self.mail_obj.get_mail_content(),
self.mail_from.encode('utf-8'),
self.mail_to_length)
res_spam = r['X-CTCH-Spam'].lower()
res_virus = r['X-CTCH-VOD'].lower()
if res_spam in ctasd_spam_status:
self.mail_obj.check_result = 'cyber_spam'
self.mail_obj.check_message = 'X-CTCH-Spam: {}'.format(
r['X-CTCH-Spam'])
outinfo(u'[Ctasd:Spam] {}: {}'.format(self.task, res_spam))
return True
elif res_virus in ctasd_virus_status:
self.mail_obj.check_result = 'cyber_spam'
self.mail_obj.check_message = 'X-CTCH-VOD: {}'.format(
res_virus)
outinfo(u'[Ctasd:Virus] {}: {}'.format(
self.task, res_virus))
return True
except gevent.Timeout:
outerror('ctasd check time out :{}'.format(self.task))
outerror(traceback.format_exc())
except BaseException as e:
outerror('ctasd check error :{}'.format(self.task))
outerror(traceback.format_exc())
return False
def dspam_reject_from_path(dspam_path, get_model):
while True:
mails = os.listdir(dspam_path)
if not mails:
gevent.sleep(1)
continue
for mail in mails:
try:
mail_path = os.path.join(dspam_path, mail)
if not os.path.exists(mail_path):
continue
try:
date, id = mail.split(',')[:2]
sig = get_model(date).objects.get(id=id).dspam_sig
except Exception, e:
sig = ''
dspamc(file_path=mail_path, report='spam', sig=sig)
outinfo('{}: imap reject'.format(mail))
try:
os.remove(mail_path)
except:
outerror('remove error: {}'.format(mail))
outerror(traceback.format_exc())
except BaseException as e:
outerror('exception: {}'.format(mail))
outerror(traceback.format_exc())
def init_resource_routine():
global bulk_count
bulk_count = redis.get(bulk_count_key)
if not bulk_count:
bulk_count = -1
redis.set(bulk_count_key, -1)
bulk_count = int(bulk_count)
while True:
try:
outinfo('init resouce routine')
init_resource()
except BaseException as e:
outerror('init_resource_routine exception')
gevent.sleep(60)
def _do_recipient_checklist(self):
"""
网关收件人强制检测名单
:param key:
:return:
"""
try:
res, msg = recipient_checklist.search(self.mail_to)
if res:
outinfo(u'[force_check]{}: {}'.format(self.task_info, msg))
self.mail_obj.check_result = 'force_check'
self.mail_obj.check_message = msg
return True
except BaseException, e:
outerror(u'force check error:{}'.format(self.task_info))
outerror(traceback.format_exc())
def collect_dielver_exception_notice():
"""
网关客户服务器DOWN机要短信/邮件提醒管理员和客户管理员
IP不通邮件通知 当发送机连续10分钟接收邮件都失败时，则通知相应管理员该发送机IP不通
:return:
"""
global notice_setting
subject = u'网关客户服务器DOWN机通知'
type = 'c_down'
redis = get_redis()
while True:
try:
if notice_setting:
outinfo('collect deliver exception notice')
interval = notice_setting.c_deliver_exception_interval
content_format = notice_setting.c_deliver_exception_content
manager = notice_setting.manager
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=interval)
_, res = redis.brpop('collect_deliver_exception')
customer_id, domain = res.split(',')
customer = Customer.objects.get(id=customer_id)
ip = ','.join(
list(
ColCustomerDomain.objects.filter(
customer=customer, domain=domain,
disabled=False).values_list('forward_address',
flat=True)))
if customer.gateway_status == 'disabled' or not ip or get_mail_model2(
datetime.date.today().strftime('%Y%m%d')
).objects.filter(state__in=['retry', 'send'],
customer=customer).count() < 10:
continue
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(customer=customer,
created__gt=need_time,
type=type):
gevent.sleep(300)
continue
content = Common.safe_format(content_format, **locals())
notice(subject, content, type, customer=customer)
notice(subject, content, type, manager=customer.tech)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def _esets(self):
try:
if setting.esets_max_size and self.size > setting.esets_max_size:
return False
res = self.parse_obj.get_attr('X-EsetResult', '').lower()
res_anti = self.parse_obj.get_attr('X-ESET-Antispam', '').lower()
if res_anti == 'spam':
if res == 'infected':
self.mail_obj.check_result = 'esets_av'
else:
self.mail_obj.check_result = 'esets'
outinfo(u'[Esets] {}'.format(self.task_info))
return True
except BaseException as e:
outerror(u'esets error :{}'.format(self.task_info))
outerror(traceback.format_exc())
return False
def _do_domain_blacklist(self):
domain = self.mail_from.split('@')[-1]
for d in domain_blacklist:
try:
r = re.search(d, domain, flags=re.IGNORECASE | re.UNICODE)
if r:
outinfo(u'[DomainBlacklist]:{}: {}'.format(
self.task, domain))
self.mail_obj.check_result = 'domain_blacklist'
self.mail_obj.check_message = u'{}----{}'.format(
d, r.group(0))
return True
except:
outerror('domain_blacklist error:{}'.format(d))
outerror(traceback.format_exc())
continue
return False
def finish(self):
check_result = self.mail_obj.check_result
self.check_result = check_result
try:
# 直接拒绝
if check_result in check_result_reject:
self.mail_obj.review_result = 'reject'
self.mail_obj.state = 'reject'
# 直接通过
elif check_result in check_result_pass:
self.mail_obj.review_result = 'pass'
self.mail_obj.state = 'dispatch'
# 需要审核
else:
self.mail_obj.state = 'review'
redis.lpush('relay_sync', self.key)
#发件人信誉度处理
if check_result in ['k_auto_reject', 'spam']:
reason = 'check_dspam' if check_result == 'spam' else 'check_auto_reject'
redis.lpush('sender_credit',
'{}----{}'.format(self.mail_obj.date_id(), reason))
self._do_check_for_same_mail()
if not self._do_check_recipient_whitelist(self.mail_obj):
# 清空邮件
if check_result in check_result_clear_mail:
#群发样本邮件不清除
if not (check_result
in ['bulk_email', 'bulk_email_subject']
and self._save_bulk_email()):
self.mail_obj.clear_mail()
self.mail_obj.save(update_fields=[
'check_result', 'check_message', 'review_result', 'state',
'dspam_sig', 'bulk_sample'
])
if check_result in check_result_pass:
self.mail_obj.save_mail_for_pop()
redis.lpush('relay_dispatch', self.key)
if review_help_mode == 'client' and self.mail_obj.state == 'review':
redis.lpush('control_review_help', 'relay,' + self.task)
except BaseException as e:
outerror('finish error:{}'.format(self.task))
outerror(traceback.format_exc())
def jam_notice():
"""
实时监控服务器处理状态，当中继检测数＋中继传输数+中继重试数＋网关检测数超过阀值时，发送通知
:return:
"""
global notice_setting, bounce_setting
subject = u'服务器拥堵通知'
redis = get_redis()
while True:
try:
if notice_setting:
outinfo('jam notice')
jam_interval = notice_setting.jam_interval
content_format = notice_setting.jam_content
count_max = notice_setting.jam_count
manager = notice_setting.manager
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=jam_interval)
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(manager=manager,
created__gt=need_time,
type='jam'):
gevent.sleep(300)
continue
relay_check = get_mail_count(get_mail_model, 'check')
relay_dispatch = get_mail_count(get_mail_model, 'dispatch')
relay_retry = get_mail_count(get_mail_model, 'retry')
collect_check = get_mail_count(get_mail_model2, 'check')
collect_send = get_mail_count(get_mail_model2, 'send')
relay_error = Common.get_file_count(
'/home/umail/data/mails-error/')
collect_error = Common.get_file_count(
'/home/umail/data/collect-mails-error/')
count = relay_check + relay_dispatch + collect_check + collect_send + relay_retry + relay_error + collect_error
if count >= count_max:
content = Common.safe_format(content_format, **locals())
notice(subject, content, 'jam', manager=manager)
gevent.sleep(300)
except:
outerror(traceback.format_exc())
gevent.sleep(300)
def _do_check_recipient_whitelist(self, mail_obj):
"""
网关收件人白名单, 如果收件人在白名单中，网关对该发件人不做任何过滤
:return:
"""
try:
if mail_obj.mail_to in recipient_whitelist:
outinfo(u'[RecipientWhitelist]:{}'.format(mail_obj.get_mail_filename()))
mail_obj.check_result = 'recipient_whitelist'
mail_obj.check_message = self.mail_obj.check_message
mail_obj.review_result = 'pass'
mail_obj.state = 'send'
mail_obj.save(update_fields=['check_result', 'check_message', 'review_result', 'state'])
redis.lpush(MAIL_SEND_QUEUE, mail_obj.get_mail_filename())
return True
except BaseException as e:
outerror('recipient whitelist error:{}'.format(mail_obj.get_mail_filename()))
outerror(traceback.format_exc())
return False
def sync_mail(key, get_model):
rkey = '{}_sync'.format(key)
tmp_rkey = '{}_sync_temp'.format(key)
server = settings.SYNC_SERVER.upper()
is_allow_sync = settings.IS_ALLOW_SYNC
while True:
try:
j = redis.brpoplpush(rkey, tmp_rkey)
date, id = j.split(',')[:2]
if is_allow_sync:
mail = get_model(date).objects.get(id=id)
scp(mail.get_mail_path(),
getattr(settings, 'HOST_{}'.format(server)),
getattr(settings, 'PORT_{}'.format(server)),
flat=False)
outinfo('{}: {}'.format(rkey, j))
redis.lrem(tmp_rkey, j, 0)
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
def dspam_reject(key, get_model):
rkey = '{}_reject'.format(key)
tmp_rkey = '{}_reject_temp'.format(key)
while True:
try:
j = redis.brpoplpush(rkey, tmp_rkey)
date, id = j.split(',')[:2]
mail = get_model(date).objects.get(id=id)
if mail.check_result not in ['spam', 'virus'
] and mail.mail_id == 0:
dspamc(file_path=mail.get_mail_path(),
report='spam',
sig=mail.dspam_sig)
mail.dspam_study = 1
mail.customer_report = 2
mail.save(update_fields=['dspam_study', 'customer_report'])
redis.lrem(tmp_rkey, 0, j)
outinfo('{}: {}'.format(rkey, j))
if key.find('collect') == -1:
_do_save_review_resutl(mail, 'reject')
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
def _do_sender_checklist(self, key):
"""
发件人白名单 黑名单检测
:param key: sender_whitelist or c_sender_blacklist
:return:
"""
try:
res, msg = sender_checklist[key].search(self.uid, self.mail_from)
if res:
# 发件人白名单必须检测spf
if key == 'sender_whitelist':
if not self._do_spf_ip_whitelist() and not check_spf(self.mail_from, self.real_client_ip):
self.mail_obj.check_result = 'spf'
outinfo(u'[spf]:{}: {} {}'.format(self.task_info, self.mail_from, self.real_client_ip))
return False
outinfo(u'[{}]{}: {}'.format(key, self.task_info, msg))
self.mail_obj.check_result = key
self.mail_obj.check_message = msg
return True
except BaseException, e:
outerror(u'{} error:{}'.format(key, self.task_info))
outerror(traceback.format_exc())
def csender_credit():
while True:
try:
j = redis.brpoplpush(CREDIS_KEY, CREDIS_KEY_temp)
mail_id, reason = j.split('----')
date, id = mail_id.split('_')
m = get_cmail_model(date).objects.get(id=id)
if m.mail_id != 0:
redis.lrem(CREDIS_KEY_temp, j, 0)
continue
sender = m.mail_from.lower().split('=')[-1]
expect_value = getattr(csetting, reason, 0)
if expect_value < 0:
key = 'csender_credit-{}'.format(date)
value_limit = int(csetting.reduce_limit)
else:
key = 'csender_credit+{}'.format(date)
value_limit = int(csetting.increase_limit)
try:
value_count = int(redis.hget(key, sender))
except:
value_count = 0
value = 0 if value_count >= value_limit else expect_value
redis.hincrby(key, sender, abs(expect_value))
CSenderCreditLog.objects.create(sender=sender, expect_value=expect_value, value=value, reason=reason, mail_id=mail_id)
if value != 0:
obj, _ = CSenderCredit.objects.get_or_create(sender=sender)
obj.credit += value
obj.save()
if obj.credit > csetting.whitelist_credit:
SenderWhitelist.objects.get_or_create(sender=sender, is_global=True)
outinfo('collect:{}: expect_value({}), value({})'.format(j, expect_value, value))
redis.lrem(CREDIS_KEY_temp, j, 0)
except (DatabaseError, InterfaceError), e:
#如果报数据库异常，关闭连接，重新处理任务
outerror('DatabaseError: {}'.format(j))
outerror(traceback.format_exc())
connection.close()
gevent.sleep(10)
return
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
