def save(d):
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
if 'bounce_result' in d:
mail.bounce_time = d['bounce_time']
mail.bounce_result = 'success' if d[
'bounce_result'] else 'fail'
mail.bounce_message = d['bounce_message']
if mail.state == 'bounce':
mail.state = 'fail_finished' if mail.return_code else 'reject'
else:
log.error(
u'save: unexpected state, mail_ident={}, state={}'.
format(d['mail_ident'], mail.state))
mail.save(update_fields=[
'bounce_time', 'bounce_result', 'bounce_message', 'state'
])
return
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'DatabaseError: exception', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'save: exception', exc_info=1)
gevent.sleep(10)
def load_mail(d):
date, id = d['mail_ident'].split(',')[:2]
for _ in xrange(5):
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
settings = Settings.objects.get()
try:
mail.customer
except ObjectDoesNotExist:
return 'finish', None
d['sender'] = mail.mail_from
d['receiver'] = mail.mail_to
if mail.dispatch_data is None:
return init_dispatch_data(d, mail, settings)
else:
return update_dispatch_data(d, mail, settings)
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'load_mail: database exception', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'load_mail: exception, mail_ident={}'.format(
d['mail_ident']),
exc_info=1)
gevent.sleep(10)
else:
return 'retry', None
def __init__(self, task):
# 邮件基本信息
self.task = task
self.task_date, self.task_id = task.split(',')[:2]
model = get_mail_model(self.task_date)
self.mail_obj = model.objects.get(pk=self.task_id)
self.uid = self.mail_obj.customer.id
self.size = self.mail_obj.size
self.mail_from = self.mail_obj.mail_from.lower()
self.mail_to = self.mail_obj.mail_to.lower()
self.mail_path = self.mail_obj.get_mail_path()
self.subject = self.mail_obj.subject
self.same_mails = model.objects.filter(mail_id=self.task_id,
state__in=['check', 'review'])
self.mails_count = self.same_mails.count()
self.same_keys = [m.get_mail_filename() for m in self.same_mails]
self.key = self.mail_obj.get_mail_filename()
self.parse_obj = ParseEmail(self.mail_obj.get_mail_content())
self.parse_data = self.parse_obj.parseMailTemplate()
# self.content = strip_tags(self.parse_obj.get_content(self.parse_data))
self.content = self.parse_obj.get_content(self.parse_data)
self.mail_to_length = model.objects.filter(
mail_id=self.task_id).count() + 1
self.check_result = ''
self.attachments = self.parse_data.get('attachments', [])
self.customer_setting, _ = CustomerSetting.objects.get_or_create(
customer_id=self.uid)
def _do_bulk_customer(date):
print date
bulk_max = setting.bulk_customer
# check_list = ['bulk_email', 'error_format', 'recipient_blacklist', 'spam']
mail_model = get_mail_model(date.strftime('%Y%m%d'))
# customers = mail_model.objects.filter(check_result__in=check_list).values('customer').annotate(
# count=Count("customer")).filter(count__gt=bulk_max).order_by('-count')
customers = mail_model.objects.filter(
state='reject').values('customer').annotate(
count=Count("customer")).filter(
count__gt=bulk_max).order_by('-count')
print customers
for c in customers:
customer = c['customer']
# sender_dict = mail_model.objects.filter(check_result__in=check_list, customer=customer).values(
# 'mail_from').annotate(count=Count('mail_from')).order_by('-count')
sender_dict = mail_model.objects.filter(
state='reject', customer=customer).values('mail_from').annotate(
count=Count('mail_from')).order_by('-count')
print sender_dict
bulk_obj, _ = BulkCustomer.objects.get_or_create(customer_id=customer,
date=date)
bulk_obj.spam_count = c['count']
bulk_obj.sender_count = len(sender_dict)
bulk_obj.sender = json.dumps(list(sender_dict))
bulk_obj.recent_count = BulkCustomer.objects.filter(
customer=customer,
date__lt=date,
date__gt=(date - datetime.timedelta(days=3))).count()
bulk_obj.save()
def bounce(j):
d = log_model.load(json.loads(j))
r = d['result'][-1]
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
mail = get_mail_model(date).objects.get(id=id)
try:
enable = CustomerSetting.objects.filter(customer=mail.customer,
bounce=True).first()
except ObjectDoesNotExist:
enable = False
if enable:
bounce_setting = BounceSettings.objects.get()
sender = bounce_setting.mailbox
receiver = d['sender']
html_text = safe_format(bounce_setting.template_cn,
receiver=d['receiver'],
subject=mail.subject,
date=r['deliver_time'],
reason=mail.return_message_display())
origin_message = mail.get_mail_content()
# origin_message = lib.del_attach_from_msg.del_attach_from_msg(d['mail_ident'])
message = bounce_message(sender, receiver, html_text,
origin_message)
break
except BaseException as e:
log.warning(u'bounce: exception', exc_info=1)
gevent.sleep(60)
if enable:
code, msg = send(bounce_setting.server, bounce_setting.port, sender,
bounce_setting.password, receiver, message)
msg = decode_msg(msg)
d['bounce_time'] = datetime.datetime.now()
d['bounce_result'] = code == 250
d['bounce_message'] = msg
log.info(
u'bounce: bounced message, mail_ident={}, code={}, msg={}, reason={}'
.format(d['mail_ident'], code, msg, mail.return_message_display()))
else:
log.info(u'bounce: skip bounced message, mail_ident={}'.format(
d['mail_ident']))
del d['result']
(redis.pipeline().lpush('control_bounced',
json.dumps(bounce_model.dump(d))).lrem(
'control_bounce_waiting', 0, j).execute())
def get_mail_count(get_mail_model, state):
date_end = datetime.datetime.today()
date_start = date_end - datetime.timedelta(days=3)
date = date_start
count = 0
while date <= date_end:
count += get_mail_model(
date.strftime('%Y%m%d')).objects.filter(state=state).count()
date = date + datetime.timedelta(days=1)
return count
def get_review_count(get_mail_model):
date_end = datetime.datetime.today()
date_start = date_end - datetime.timedelta(days=3)
date = date_start
count = 0
while date <= date_end:
count += get_mail_model(date.strftime('%Y%m%d')).objects.exclude(
check_result='cyber_spam').filter(state='review').count()
date = date + datetime.timedelta(days=1)
return count
def relay_limit_notice():
"""
定时一次性发送
中继用户超过限制通知, 当用户超过设置值时，发送通知
:return:
"""
global notice_setting, bounce_setting
notice_type = "relay"
while True:
try:
cur = datetime.datetime.now()
if cur.hour != 1 or cur.minute != 25:
gevent.sleep(30)
continue
outinfo('relay limit notice')
all_content = ""
content_format = notice_setting.relay_content
manager = notice_setting.manager
flag = Notification.objects.filter(
manager=manager,
type=notice_type,
created__contains=datetime.date.today())
pre = cur + datetime.timedelta(-1)
if not flag:
mail_model = get_mail_model(pre.strftime('%Y%m%d'))
customers = mail_model.objects.exclude(
customer__company__icontains=u'临时信任').values(
'customer').annotate(
count=Count("customer")).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
company_id = customer.id
company = customer.company
relay_limit = customer.relay_limit
sender_count = mail_model.objects.exclude(
mail_from='<>', mail_from__isnull=False).filter(
customer=customer).distinct('mail_from').count()
if relay_limit != -1 and sender_count > relay_limit:
content = Common.safe_format(content_format,
company=company,
company_id=company_id,
setting=relay_limit,
count=sender_count)
all_content += content
if all_content:
subject = pre.strftime('%Y-%m-%d ') + u"中继用户超过限制通知"
notice(subject=subject,
content=all_content,
type=notice_type,
customer=None,
manager=manager)
except:
outerror(traceback.format_exc())
finally:
gevent.sleep(60)
def save_check_statistic(date, get_mail_model, statistics, check_list):
model = get_mail_model(date.strftime('%Y%m%d'))
all_dict = dict(
model.objects.values_list("check_result").annotate(Count("id")))
pass_dict = dict(
model.objects.filter(review_result__in=['pass', 'pass_undo']).
values_list("check_result").annotate(Count("id")))
obj, _ = statistics.objects.get_or_create(date=date)
for c in check_list:
obj.__setattr__('{}_all'.format(c), all_dict.get(c, 0))
obj.__setattr__('{}_pass'.format(c), pass_dict.get(c, 0))
obj.save()
def review_result():
remote_ip = flask.request.remote_addr
if remote_ip != config['review_help_server_ip']:
log.warning(u'review_result: forbidden, ip={}'.format(remote_ip))
flask.abort(403)
d = json.loads(flask.request.data)
if d['origin'] == 'relay':
mails = (get_mail_model(d['date']).objects.filter(state='review')
.filter(Q(id=d['mail_id']) | Q(mail_id=d['mail_id'])))
if len(mails) == 0:
log.info(u'review_result: skip, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
elif d['review_result'] == 'pass':
keys = [m.get_mail_filename() for m in mails]
mails.update(state='dispatch', review_result=d['review_result'], review_time=datetime.datetime.now())
map(lambda key: redis.lpush('relay_dispatch', key), set(keys))
log.info(u'review_result: pass, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
elif d['review_result'] == 'reject':
mails.update(state='reject', review_result=d['review_result'], review_time=datetime.datetime.now())
log.info(u'review_result: reject, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
else:
assert 0
elif d['origin'] == 'collect':
mails = (c_get_mail_model(d['date']).objects.filter(state='review')
.filter(Q(id=d['mail_id']) | Q(mail_id=d['mail_id'])))
if len(mails) == 0:
log.info(u'review_result: skip, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
elif d['review_result'] == 'pass':
keys = [m.get_mail_filename() for m in mails]
mails.update(state='send', review_result=d['review_result'], review_time=datetime.datetime.now())
map(lambda key: redis.lpush('collect_send', key), set(keys))
log.info(u'review_result: pass, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
elif d['review_result'] == 'reject':
mails.update(state='reject', review_result=d['review_result'], review_time=datetime.datetime.now())
log.info(u'review_result: reject, ip={}, key={},{},{}'
.format(remote_ip, d['origin'], d['date'], d['mail_id']))
else:
assert 0
else:
assert 0
return ''
def save_review_statistic(date,
get_mail_model,
statistics,
check_list,
type='all'):
model = get_mail_model(date.strftime('%Y%m%d'))
mails = model.objects.filter(check_result__in=check_list)
if type == 'all':
_save_review_statistic(model, date, mails, statistics, type)
else:
reviewers = set(
mails.filter(reviewer__isnull=False).values_list('reviewer',
flat=True))
for r in reviewers:
_ = mails.filter(reviewer_id=r)
_save_review_statistic(model, date, _, statistics, type, r)
def load_mail(key, origin, date, mail_id):
while True:
try:
if origin == 'relay':
return get_mail_model(date).objects.get(id=mail_id)
elif origin == 'collect':
return c_get_mail_model(date).objects.get(id=mail_id)
else:
assert 0
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'load_mail: database exception, key={}'.format(key),
exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'load_mail: exception, key={}'.format(key),
exc_info=1)
gevent.sleep(10)
def _do_save_review_resutl(mail, result):
"""
———自动审核———
关键词/动态SPAM/高危邮件的审核策略调整：
1、在X时间内，如果同一个发件人和同一个收件人连续X次通过/拒绝，则记住对应关系，下次直接放行/拒绝。
2、这个规则不适合多个收件人
3、规则有效期为X天
:param mail:
:param resutl:
:return:
"""
check_list = [
'subject_blacklist', 'keyword_blacklist', 'active_spam', 'high_risk'
]
check_result = mail.check_result
if check_result in check_list and mail.mail_id == 0 and not get_mail_model(
mail.get_date()).objects.filter(mail_id=mail.id):
_save_review_result(mail.mail_from, mail.mail_to, result, check_result)
def determine_and_save(j, d):
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
action = determine(j, d, mail)
save(d, mail)
if action is not None:
action(get_redis_connection(mail.server_id))
return True
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'determine_and_save: database exception', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'determine_and_save: exception, mail_ident={}'.format(d['mail_ident']), exc_info=1)
gevent.sleep(1)
return False
def sender_credit():
while True:
try:
j = redis.brpoplpush(REDIS_KEY, REDIS_KEY_temp)
mail_id, reason = j.split('----')
date, id = mail_id.split('_')
m = get_mail_model(date).objects.get(id=id)
if m.mail_id != 0:
redis.lrem(REDIS_KEY_temp, j, 0)
continue
sender = m.mail_from.lower().split('=')[-1]
expect_value = getattr(setting, reason, 0)
if expect_value < 0:
key = 'sender_credit-{}'.format(date)
value_limit = int(setting.reduce_limit)
else:
key = 'sender_credit+{}'.format(date)
value_limit = int(setting.increase_limit)
try:
value_count = int(redis.hget(key, sender))
except:
value_count = 0
value = 0 if value_count >= value_limit else expect_value
redis.hincrby(key, sender, abs(expect_value))
SenderCreditLog.objects.create(sender=sender, expect_value=expect_value, value=value, reason=reason, mail_id=mail_id)
if value != 0:
obj, _ = SenderCredit.objects.get_or_create(sender=sender)
obj.credit += value
obj.save()
outinfo('{}: expect_value({}), value({})'.format(j, expect_value, value))
redis.lrem(REDIS_KEY_temp, j, 0)
except (DatabaseError, InterfaceError), e:
#如果报数据库异常，关闭连接，重新处理任务
outerror('DatabaseError: {}'.format(j))
outerror(traceback.format_exc())
connection.close()
gevent.sleep(10)
return
except BaseException as e:
outerror('exception: {}'.format(j))
outerror(traceback.format_exc())
def _do_save_keyword_review_status(date,
get_mail_model,
check_result,
relay=True):
model = get_mail_model(date.strftime('%Y%m%d'))
mails = model.objects.filter(check_result=check_result)
subject_dict = {}
subject_pass_dict = {}
for m in mails:
k = m.check_message.split('----')[0]
if k in subject_dict:
subject_dict[k] += 1
else:
subject_dict[k] = 1
if m.state != 'reject':
if k in subject_pass_dict:
subject_pass_dict[k] += 1
else:
subject_pass_dict[k] = 1
if check_result == 'subject_blacklist':
blacklist_model = SubjectKeywordBlacklist
else:
blacklist_model = KeywordBlacklist
for k, v in subject_dict.iteritems():
objs = blacklist_model.objects.filter(keyword=k)
if objs:
obj = objs[0]
if relay:
obj.relay_all += v
else:
obj.collect_all += v
if k in subject_pass_dict:
if relay:
obj.relay_pass += subject_pass_dict[k]
else:
obj.collect_pass += subject_pass_dict[k]
obj.save()
def save(d):
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
mail.dispatch_data = json.dumps(d['dispatch_data'])
mail.deliver_ip = d['deliver_ip']
if mail.state == 'dispatch':
if 'state' in d:
mail.state = d['state']
else:
mail.state = 'send'
elif mail.state == 'retry':
if 'state' in d:
mail.state = d['state']
else:
mail.state = 'retry'
else:
log.error(
u'save: unexpected state, mail_ident={}, state={}'.
format(d['mail_ident'], mail.state))
if d['dispatch_data'][-1]['attachment_removed']:
mail.is_del_attach = True
mail.save(update_fields=[
'dispatch_data', 'deliver_ip', 'state', 'is_del_attach'
])
return
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'save: database exception', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'save: exception, mail_ident={}'.format(
d['mail_ident']),
exc_info=1)
gevent.sleep(10)
def push_data():
from_date = '20160801'
to_date = '20160809'
while from_date != to_date:
outinfo('push data: {}'.format(from_date))
mails = get_mail_model(from_date).objects.filter(mail_id=0)
for m in mails:
if m.state == 'reject' and m.check_result == 'spam':
reason = 'check_dspam'
elif m.state == 'reject' and m.check_result == 'k_auto_reject':
reason = 'check_auto_reject'
elif m.state == 'fail_finished' and m.error_type == 2:
reason = 'send_not_exist'
elif m.state == 'fail_finished' and m.error_type == 5:
reason = 'send_spam'
elif m.reviewer and m.review_result.find('reject') != -1:
reason = 'review_reject'
elif m.reviewer and m.review_result.find('pass') != -1:
reason = 'review_pass'
else:
continue
d = '{}----{}'.format(m.date_id(), reason)
redis.lpush(REDIS_KEY, d)
from_date = (datetime.datetime.strptime(from_date, "%Y%m%d") + datetime.timedelta(days=1)).strftime("%Y%m%d")
def del_attach_from_msg(date_id, f='relay'):
date, id = date_id.split(',')
mail_obj = get_mail_model(date).objects.get(
id=id) if 'relay' == f else get_mail_model2(date).objects.get(id=id)
email_content = mail_obj.get_mail_content()
message = email.message_from_string(email_content)
s = Settings.objects.all()
if s:
days = s[0].back_days
else:
days = 10
message = del_attach_ins(message)
p = ParseEmail(email_content)
data = p.parseMailTemplate()
expire_day = (mail_obj.created +
datetime.timedelta(days=days)).strftime('%Y-%m-%d %H:%M')
context = {'m': data, 'mail_obj': mail_obj, 'expire_day': expire_day}
# attachment_django_template = get_django_template()
attachment_django_template = get_render_attchtemplate()
t = template.Template(attachment_django_template)
django_html = t.render(Context(context))
l = []
for part in message.walk():
if not part.is_multipart():
content_type = part.get_content_type()
filename = part.get_filename()
if content_type == 'text/html' and filename is None:
l.append(part)
if django_html and len(l) > 0:
part = l[0]
payload = part.get_payload(decode=True)
charset = part.get_content_charset()
html = decode_str(payload, charset)
# server = '{}'.format( getattr(settings, 'WEB_USER_SHENZHEN') )
# key = base64.b64encode('MAILDOWNLOAD' + '_' + date_id.replace(',', '_'))
# word = base64.b64encode(date_id.replace(',', '_'))
# link = 'http://{}/mail/dowload_mail_real_attachment?key={}&id={}&cid=\g<cid>'.format(server, key, word)
# html = re.sub('"cid:(?P<cid>.*?)"', link, html)
# html2 = html_add_footer(html, django_html)
html2 = heml_add_header(html, django_html)
payload2 = base64.b64encode(html2.encode('utf-8'))
part.set_payload(payload2)
del part['content-type']
del part['content-transfer-encoding']
part['content-type'] = 'text/html; charset="utf-8"'
part['content-transfer-encoding'] = 'base64'
elif django_html:
if not message.is_multipart():
payload = message.get_payload(decode=True)
charset = part.get_content_charset()
html = decode_str(payload, charset)
# html2 = html_add_footer(html, django_html)
html2 = heml_add_header(html, django_html)
payload2 = base64.b64encode(html2.encode('utf-8'))
message.set_payload(payload2)
del message['content-type']
del message['content-transfer-encoding']
message['content-type'] = 'text/html; charset="utf-8"'
message['content-transfer-encoding'] = 'base64'
else:
msg = MIMEText(django_html, 'html', 'utf-8')
message.attach(msg)
msg = message.as_string()
with open('{}_del_attach'.format(mail_obj.get_mail_path()), 'w') as f:
f.write(msg)
return msg
def statistics(p_date):
mails = get_mail_model(p_date.replace(
'-', '')).objects.filter(return_code__isnull=False)
# type==all
rs = get_data(mails)
obj, bool = Statistics.objects.get_or_create(date=p_date, type='all')
outinfo('all: {}'.format(rs))
save_statistics(obj, rs)
# type==ip
ips = mails.values_list('deliver_ip', flat=True).distinct()
for ip in ips:
rs = get_data(mails.filter(deliver_ip=ip))
if rs.get('count', 0):
try:
obj, bool = Statistics.objects.get_or_create(date=p_date,
type='ip',
ip=ip)
save_statistics(obj, rs)
outinfo('ip({}): {}'.format(ip, rs))
except:
outerror(traceback.format_exc())
#type==ip_pool
pools = IpPool.objects.all()
for p in pools:
ips = list(p.clusterip_set.values_list('ip', flat=True))
rs = get_data(mails.filter(deliver_ip__in=ips))
if rs.get('count', 0):
try:
obj, bool = Statistics.objects.get_or_create(date=p_date,
type='ip_pool',
ip_pool=p)
save_statistics(obj, rs)
outinfo('ip_pool({}): {}'.format(p, rs))
except:
outerror(traceback.format_exc())
#type==cluster
cls = Cluster.objects.all()
for c in cls:
ips = list(c.cluster.values_list('ip', flat=True))
rs = get_data(mails.filter(deliver_ip__in=ips))
if rs.get('count', 0):
try:
obj, bool = Statistics.objects.get_or_create(date=p_date,
type='cluster',
cluster=c)
save_statistics(obj, rs)
outinfo('cluster({}): {}'.format(c, rs))
except:
outerror(traceback.format_exc())
#type==customer
# customers = Customer.objects.all()
customers = mails.values_list('customer_id', flat=True).distinct()
for c in customers:
rs = get_data(mails.filter(customer_id=c))
if rs.get('count', 0):
try:
obj, bool = Statistics.objects.get_or_create(date=p_date,
type='customer',
customer_id=c)
save_statistics(obj, rs)
outinfo('customer({}): {}'.format(c, rs))
except:
outerror(traceback.format_exc())
def run(self):
if not os.path.exists(self.mail_path):
return
date = time.strftime('%Y%m%d')
#创建django的mail模型
mail_model = get_mail_model(date)
if date not in _dates:
install_model(mail_model)
#创建的目录 用户组也有权限修改
Common.make_dir([os.path.join(data_path, date)], permit=0775)
_dates.append(date)
uid, mail_id, client_ip = self.task.split(',')[:3]
with open(self.mail_path) as f:
email_str = f.read()
with open(self.from_path) as f:
mail_from = f.read()
mail_from = mail_from.split('=')[-1]
with open(self.rcpt_path) as f:
rcpt_list = f.readlines()
# subject = get_mail_subject(mail_data=email_str)
p = ParseEmail(email_str)
if int(uid) in replace_sender_customers:
email_str = p.replace_sender(mail_from)
subject = p.get_attr('subject')
try:
sender_name = re.search('(.*?)\(.*?\)',
p.get_attr('from')).group(1)
except:
outerror('get sender_name error: {}'.format(self.task))
outerror(traceback.format_exc())
sender_name = ''
data = p.parseMailTemplate()
attaches = data.get('attachments', [])
try:
attach_name = u'----'.join(
[a.get('decode_name', '') for a in attaches])
except:
outerror('get attach_name error: {}'.format(self.task))
outerror(traceback.format_exc())
attach_name = ''
#被推送到检测队列中的邮件
#当有多个收件人时, 只推送mail_id=0的一个对象到检测队列
#mail_id=0一定不是无效地址中的邮件, 如果都在无效地址中, check_mail=None, 则无邮件推送到检测队列
check_mail = None
outinfo(self.task)
outinfo('rcpt_list:{}'.format(rcpt_list))
#mail id列表
mail_ids = []
is_white = is_contains(mail_from, invalidsender_whitelist)
for rcpt in rcpt_list:
if rcpt == '': continue
# remove \n
rcpt = rcpt.replace('\n', '')
# 去掉字符串中的引号
#rcpt = rcpt.replace('"', '').replace("'", "")
rcpt = addr_get(rcpt)
try:
mail_obj = mail_model(customer_id=uid,
mail_from=mail_from.lower(),
mail_to=rcpt.lower(),
client_ip=client_ip,
subject=subject,
size=len(email_str),
sender_name=sender_name,
server_id=server_id,
attach_name=attach_name)
mail_obj.save()
except (DatabaseError, InterfaceError), e:
outerror(traceback.format_exc())
continue
with open(mail_obj.get_mail_path(), 'w') as f:
f.write(email_str)
filename = mail_obj.get_mail_filename()
if not check_email_format(rcpt, suffix_list) or is_same_domain(
rcpt, mail_from):
outinfo('{}: error format'.format(filename))
self._do_invalid_mail(mail_obj, 'error_format')
elif not is_white and check_invalid_mail(rcpt):
outinfo('{}: invalid mail'.format(filename))
self._do_invalid_mail(mail_obj, 'invalid_mail')
elif is_contains(rcpt, recipient_blacklist):
outinfo('{}: recipient blacklist'.format(filename))
self._do_invalid_mail(mail_obj, 'recipient_blacklist')
else:
outinfo('{}: pass'.format(filename))
if not check_mail:
check_mail = mail_obj
mail_ids.append(mail_obj.id)
def get_mail_count(mail_obj, date):
model = get_mail_model(date.replace('-', ''))
mail_id = mail_obj.mail_id if mail_obj.mail_id else mail_obj.id
return model.objects.filter(mail_id=mail_id).count() + 1
def bulk_notice():
"""
群发邮件通知 用户群发邮件的时候通知客户和对应的技术支持
:return:
"""
global notice_setting, bounce_setting
subject = u'违规邮件通知'
while True:
try:
mail_model = get_mail_model(time.strftime('%Y%m%d'))
if notice_setting:
outinfo('bulk notice')
bulk_count = notice_setting.bulk_count
bulk_interval = notice_setting.bulk_interval
content_format = notice_setting.bulk_content
customers = mail_model.objects.filter(
state='reject').values('customer').annotate(
count=Count("customer")).filter(
count__gt=bulk_count).order_by('-count')
for c in customers:
customer = Customer.objects.get(pk=c['customer'])
need_time = datetime.datetime.now() - datetime.timedelta(
minutes=bulk_interval)
# 判断是否在发送间隔时间内, 如果在间隔时间内，则不发送通知
if Notification.objects.filter(customer=customer,
created__gt=need_time,
type='bulk'):
gevent.sleep(300)
continue
"""
# 当天群发超过300 发通知一次 发第二次时检测 是否超过300*2 第三次是否超过300*3 依次类推
# 当天已发送群发通知的数量
times = Notification.objects.filter(customer=customer, created__contains=datetime.date.today(),
type='bulk').count()
if not times:
times = 1
if bulk_count and count > bulk_count * times:
"""
notices = Notification.objects.filter(
customer=customer,
created__contains=datetime.date.today(),
type='bulk').order_by('-created')
if notices:
count = mail_model.objects.filter(
state='reject',
customer=customer,
created__gt=notices[0].created).count()
else:
count = c['count']
if count > bulk_count:
mails = mail_model.objects.exclude(
mail_from='<>').filter(state='reject',
customer=customer)
if notices:
mails = mails.filter(
created__gt=notices[0].created)
sender_dict = mails.values('mail_from').annotate(
count=Count('mail_from')).filter(
count__gt=10).order_by('-count')
if sender_dict:
account_list = []
for s in sender_dict:
mail_from = re.sub('prvs=(.*?)=', '',
s['mail_from'])
if mail_from not in account_list:
account_list.append(mail_from)
if len(account_list) >= 7:
break
content = Common.safe_format(
content_format,
count=str(count),
account=','.join(account_list))
notice(subject, content, 'bulk', customer)
gevent.sleep(300)
except (DatabaseError, InterfaceError), e:
# 如果报数据库异常，关闭连接，重新处理任务
outerror('DatabaseError')
outerror(traceback.format_exc())
connection.close()
gevent.sleep(2)
except:
