def _do_dspam(ids, result):
"""
:param ids: 学习的ID列表
:param result: 结果
:return:
"""
keys = {}
# redis = get_redis_connection()
for k, v in deal_with_ids(ids).iteritems():
mail_model = get_mail_model(k)
servers = mail_model.objects.filter(id__in=v).values_list(
'server_id', flat=True).distinct()
for s in servers:
mails = mail_model.objects.filter(id__in=v, server_id=s)
keys.setdefault(s, []).extend(
map(lambda mail: mail.get_mail_filename(), mails))
# keys.extend(map(lambda mail: mail.get_mail_filename(), mails))
for k, v in keys.iteritems():
try:
redis = get_redis_connection2(k)
except:
redis = get_redis_connection2('default')
map(lambda key: redis.lpush('dspam_collect_{}'.format(result), key),
set(v))
def save(d):
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
if 'bounce_result' in d:
mail.bounce_time = d['bounce_time']
mail.bounce_result = 'success' if d[
'bounce_result'] else 'fail'
mail.bounce_message = d['bounce_message']
if mail.state == 'bounce':
mail.state = 'fail_finished' if mail.return_code else 'reject'
else:
log.error(
u'save: unexpected state, mail_ident={}, state={}'.
format(d['mail_ident'], mail.state))
mail.save(update_fields=[
'bounce_time', 'bounce_result', 'bounce_message', 'state'
])
return
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'DatabaseError: exception', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'save: exception', exc_info=1)
gevent.sleep(10)
def _resend(ids):
"""
将邮件推送到队列重新投递
:param ids:
:return:
"""
keys = {}
# redis = get_redis_connection()
for k, v in deal_with_ids(ids).iteritems():
mail_model = get_mail_model(k)
servers = mail_model.objects.filter(id__in=v).values_list(
'server_id', flat=True).distinct()
for s in servers:
mails = mail_model.objects.filter(
id__in=v,
server_id=s,
state__in=['reject', 'finished', 'fail_finished'])
keys.setdefault(s, []).extend(
map(lambda mail: mail.get_mail_filename(), mails))
mails.update(state='retry')
# keys.extend(map(lambda mail: mail.get_mail_filename(), mails))
for k, v in keys.iteritems():
try:
redis = get_redis_connection2(k)
except:
redis = get_redis_connection2('default')
map(lambda key: redis.lpush('collect_send', key), set(v))
def get_state_data(date_str):
key = '_cache:collect_mail_status:{}'.format(date_str)
if datetime.date.today().strftime('%Y-%m-%d') != date_str:
rs = cache.get(key)
if rs:
return rs
rs = {}
model = get_mail_model(date_str.replace('-', ''))
try:
rs['count'] = model.objects.count()
rs['count1'] = model.objects.filter(mail_id=0).count()
except:
return rs
rs['state'] = dict(
model.objects.values_list("state").annotate(Count("id")))
rs['check'] = dict(
model.objects.values_list("check_result").annotate(Count("id")))
rs['review'] = dict(
model.objects.values_list("review_result").annotate(Count("id")))
rs['send'] = {
'success':
model.objects.filter(return_code=250).count(),
'fail':
model.objects.filter(Q(state='finished') | Q(state='bounce')).exclude(
return_code=250).count()
}
rs['state']['review'] = model.objects.filter(state='review').exclude(
check_result='spam').count()
cache.set(key, rs, 0)
return rs
def manager_main():
"""
跟客户管理员发送报告
:return:
"""
try:
t3 = time.time()
mail_model = get_mail_model(get_mail_date())
customer_list = mail_model.objects.exclude(customer__gateway_status='disabled') \
.filter(state='reject', mail_to__isnull=False, review_result='reject',
customer__customersetting__m_spamrpt=True, customer__customersetting__is_spamrpt_sendtime=False) \
.distinct('customer_id') \
.values_list('customer_id', flat=True)
pool = gevent.pool.Pool(_MAXTHREAD)
for customer_id in customer_list:
pool.spawn(work_send, customer_id, '', 'manager')
log.info('waiting stop...')
pool.join()
t4 = time.time()
log.info('m_spam_rpt send total time={}'.format(t4 - t3))
return
except (DatabaseError, InterfaceError) as e:
log.error(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.error(u'spam_rpt: exception', exc_info=1)
gevent.sleep(10)
def _get_active_receiver_list(data, date):
mail_model = get_mail_model(date)
customer_id = data.get('customer_id', '')
if customer_id:
mails = mail_model.objects.filter(customer_id=customer_id)
else:
mails = mail_model.objects.all()
mails = mails.values("mail_to").annotate(
total_all=Count("id"),
size=Sum("size"),
total=Sum(
Case(When(mail_id=0, then=Value(1)),
default=Value(0),
output_field=IntegerField())),
total_success=Sum(
Case(When(state='finished', then=Value(1)),
default=Value(0),
output_field=IntegerField())),
size_success=Sum(
Case(When(state='finished', then='size'),
default=Value(0),
output_field=IntegerField())),
total_reject=Sum(
Case(When(Q(state='reject') | Q(state='c_reject'), then=Value(1)),
default=Value(0),
output_field=IntegerField())),
size_reject=Sum(
Case(When(Q(state='reject') | Q(state='c_reject'), then='size'),
default=Value(0),
output_field=IntegerField())),
total_fail=Sum(
Case(When(Q(state='fail_finished') | Q(state='bounce'),
then=Value(1)),
default=Value(0),
output_field=IntegerField())),
size_fail=Sum(
Case(When(Q(state='fail_finished') | Q(state='bounce'),
then='size'),
default=Value(0),
output_field=IntegerField())),
total_out=Sum(
Case(When(Q(state='finished') | Q(state='fail_finished')
| Q(state='bounce'),
then=Value(1)),
default=Value(0),
output_field=IntegerField())),
size_out=Sum(
Case(When(Q(state='finished') | Q(state='fail_finished')
| Q(state='bounce'),
then='size'),
default=Value(0),
output_field=IntegerField())),
)
try:
mails.exists()
except:
mails = []
return mails
def __init__(self, task):
# 邮件基本信息
self.task = task
self.task_date, self.task_id = task.split(',')[:2]
self.model = get_mail_model(self.task_date)
self.mail_obj = self.model.objects.get(pk=self.task_id)
self.uid = self.mail_obj.customer.id
def bounce(j):
d = json.loads(j)
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
mail = get_mail_model(date).objects.get(id=id)
try:
enable = CustomerSetting.objects.filter(customer=mail.customer,
c_bounce=True).first()
except ObjectDoesNotExist:
enable = False
if enable:
bounce_setting = BounceSettings.objects.get()
sender = bounce_setting.mailbox
receiver = d['sender']
html_text = safe_format(bounce_setting.template_cn,
receiver=d['receiver'],
subject=mail.subject,
date=d.get('deliver_time'),
reason=mail.return_message_display())
# origin_message = mail.get_mail_content()
try:
origin_message = lib.del_attach_from_msg.del_attach_from_msg(
d['mail_ident'], f='collect')
except IOError:
origin_message = ''
message = bounce_message(sender, receiver, html_text,
origin_message)
break
except BaseException as e:
log.warning(u'bounce: exception', exc_info=1)
gevent.sleep(60)
if enable:
code, msg = send(bounce_setting.server, bounce_setting.port, sender,
bounce_setting.password, receiver, message)
msg = decode_msg(msg)
d['bounce_time'] = datetime.datetime.now()
d['bounce_result'] = code == 250
d['bounce_message'] = msg
log.info(
u'bounce: bounced message, mail_ident={}, code={}, msg={}, reason={}'
.format(d['mail_ident'], code, msg, mail.return_message_display()))
else:
log.info(u'bounce: skip bounced message, mail_ident={}'.format(
d['mail_ident']))
(redis.pipeline().lpush('collect_bounced',
json.dumps(d, cls=lib.common.ComplexEncoder)).lrem(
'collect_bounce_waiting', 0, j).execute())
def state_logs(request, id):
date, mail_id = id.split('_')
logs = MailStateLog.objects.filter(date=date, mail_id=mail_id)
m = get_mail_model(date).objects.get(id=mail_id)
return render_to_response("collect_mail/state_logs.html", {
'logs': logs,
'm': m,
},
context_instance=RequestContext(request))
def get_mails_from_sendtime(customer_id,
sendtime,
mail_to=None,
check_result=None):
"""
获取sendtime往前24个小时内的邮件数据
:param customer_id:
:param sendtime:
:param mail_to:
:param check_result:
:return:
"""
mail_model = get_mail_model(get_mail_date())
mail_model_today = get_mail_model(time.strftime('%Y%m%d'))
start_time = datetime.combine((datetime.today() + timedelta(-1)), sendtime)
end_time = datetime.combine(datetime.today(), sendtime)
first_mails = mail_model.objects.filter(customer_id=customer_id,
state='reject',
mail_to__isnull=False,
review_result='reject',
created__gte=start_time)
last_mails = mail_model_today.objects.filter(customer_id=customer_id,
state='reject',
mail_to__isnull=False,
review_result='reject',
created__lt=end_time)
if check_result == 'virus':
first_mails = first_mails.filter(check_result='virus')
last_mails = last_mails.filter(check_result='virus')
elif check_result == 'not_virus':
first_mails = first_mails.exclude(check_result='virus')
last_mails = last_mails.exclude(check_result='virus')
if mail_to:
first_mails = first_mails.filter(mail_to=mail_to)
last_mails = last_mails.filter(mail_to=mail_to)
else:
first_mails = first_mails.order_by('mail_to')
last_mails = last_mails.order_by('mail_to')
mails = list(first_mails)
mails.extend(list(last_mails))
return mails
def mail_review_undo(request):
ids = request.GET.get('id', '')
date, id = ids.split('_')
mail_obj = get_mail_model(date).objects.get(id=id)
# if not mail_obj.get_mail_content():
# msg = u'操作失败：邮件内容不存在'
# else:
undo_review_result = 'pass' if mail_obj.review_result == 'reject' else 'reject'
_do_review([ids], undo_review_result, request, flag=False)
msg = u'误判操作成功：{}'.format(undo_review_result)
return HttpResponse(json.dumps({'msg': msg}),
content_type="application/json")
def resent(request):
ids = request.GET.get('id', '')
date, id = ids.split('_')
msg = u'发送失败，只有发送完成/失败的邮件可以再次发送'
m = get_mail_model(date).objects.get(id=id)
if m.state in ['finished', 'fail_finished']:
redis = get_redis_connection2(m.server_id)
redis.lpush('collect_send', m.get_mail_filename())
m.state = 'retry'
m.save()
msg = u'已再次发送邮件'
return HttpResponse(json.dumps({'msg': msg}),
content_type="application/json")
def get_mails_from_interval(customer_id,
interval,
mail_to=None,
check_result=None):
"""
获取当前时间往前interval个小时内的邮件数据
:param customer_id:
:param interval: 间隔时间
:param mail_to:
:param check_result:
:return:
"""
end_time = datetime.now().strftime("%Y%m%d%H")
end_time = datetime.strptime('{}0000'.format(end_time), "%Y%m%d%H%M%S")
start_time = end_time - timedelta(hours=interval)
mail_model_start = get_mail_model(start_time.strftime('%Y%m%d'))
mail_model_end = get_mail_model(end_time.strftime('%Y%m%d'))
if start_time.day == end_time.day:
mails = mail_model_start.objects.filter(customer_id=customer_id,
state='reject',
mail_to__isnull=False,
review_result='reject',
created__gte=start_time,
created__lt=end_time)
mails = _get_mails(mails, check_result, mail_to)
else:
first_mails = mail_model_start.objects.filter(customer_id=customer_id,
state='reject',
mail_to__isnull=False,
review_result='reject',
created__gte=start_time)
last_mails = mail_model_end.objects.filter(customer_id=customer_id,
state='reject',
mail_to__isnull=False,
review_result='reject',
created__lt=end_time)
mails = _get_mails(first_mails, check_result, mail_to)
mails.extend(_get_mails(last_mails, check_result, mail_to))
return mails
def get_send_info(obj, type):
date = datetime.date.today()
msg = u"""
<span>总:<a href='/collect_mail/mail_list?{type}={value}'>{total}</a>
拒:<a href='/collect_mail/mail_list?{type}={value}&review=reject'>{total_reject}</a>
<span>发:<a href='/collect_mail/mail_list?show=sendlog&{type}={value}'>{count}</a>
成:<a href='/collect_mail/mail_list?show=sendlog&send=success&{type}={value}'>{success}</a>
成功率:<code>{rate}</code>%
</span></br>
<a href='/collect_mail/statistics?{type}={value}&type={type}'><code>查看历史详情</code></a>
"""
class DefaultS:
count = 0
success = 0
rate = 0
total = 0
total_reject = 0
s = DefaultS()
objs = []
model = get_mail_model(date.strftime('%Y%m%d'))
if type == 'customer':
objs = Statistics.objects.filter(date=date, customer=obj)
if objs:
s = objs[0]
if type == 'customer':
try:
s.total = model.objects.filter(customer=obj).count()
except:
s.total = 0
try:
s.total_reject = model.objects.filter(
customer=obj, review_result='reject').count()
except:
s.total_reject = 0
rs = {
'total': s.total,
'total_reject': s.total_reject,
'type': type,
'value': obj.id if type != 'ip' else obj.ip,
'count': s.count,
'success': s.success,
'rate': '%.2f' % s.rate,
}
return msg.format(**rs)
def send_mail_file(request):
if not validate_key(request):
raise Http404
date_id = request.GET.get('id', '')
date, id = date_id.split('_')
if id and date:
mail_obj = get_mail_model(date).objects.get(id=id)
filename = mail_obj.get_mail_path()
if not os.path.exists(filename):
filename = mail_obj.get_mail_path_old()
if os.path.exists(filename):
wrapper = FileWrapper(file(filename))
response = HttpResponse(wrapper, content_type='text/plain')
response['Content-Length'] = os.path.getsize(filename)
return response
return HttpResponse('')
def mail_recheck(request):
date = request.GET.get('date', '')
if date:
mails = get_mail_model(date).objects.filter(check_result='error',
state='review',
mail_id=0)
# mails = get_mail_model(date).objects.filter(state='check', mail_id=0)
for mail in mails:
mail.state = 'check'
mail.save()
redis = get_redis_connection2(mail.server_id)
try:
redis.lpush('collect_incheck', mail.get_mail_filename())
except:
pass
messages.add_message(request, messages.SUCCESS, u'出错邮件已重新加入检测')
return HttpResponseRedirect(reverse('collect_mail_summary'))
def save(d):
date, id = d['mail_ident'].split(',')[:2]
while True:
try:
with atomic():
mail = get_mail_model(date).objects.get(id=id)
if mail.state in ('send', 'retry'):
mail.state = d['state']
else:
log.error(u'save: unexpected state, mail_ident={}, state={}'
.format(d['mail_ident'], mail.state))
if len(d['result']) == 0:
mail.save(update_fields=['state'])
else:
r = d['result'][-1]
mail.deliver_ip = r['deliver_ip']
mail.deliver_time = r['deliver_time']
mail.return_code = r['return_code']
mail.return_message = r['return_message']
mail.error_type = r['error_type']
mail.save(update_fields=['state', 'deliver_ip', 'deliver_time', 'return_code', 'return_message',
'error_type'])
for r in d['result']:
dl = DeliverLog(
date=date,
mail_id=id,
deliver_ip=r['deliver_ip'],
deliver_time=r['deliver_time'],
receive_ip=r['receive_ip'],
return_code=r['return_code'],
return_message=r['return_message'],
)
dl.save()
return
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'save: exception', exc_info=1)
gevent.sleep(10)
def op_keywordlist(request):
operate = request.GET.get('operate', 'modify')
msg = u'错误: 没有该关键字信息'
try:
# date = request.GET.get('date')
date, id = request.GET.get('id').split('_')
mail_obj = get_mail_model(date.replace('-', '')).objects.get(pk=id)
keyword = mail_obj.check_message.split('----', 1)[0]
if mail_obj.check_result == 'subject_blacklist' or 'subject_blacklist' in mail_obj.check_message:
obj = RSubjectKeywordBlacklist.objects.get(keyword=keyword)
url = reverse('subject_keyword_blacklist_modify', args=(obj.id, ))
elif mail_obj.check_result == 'keyword_blacklist' or 'keyword_blacklist' in mail_obj.check_message:
obj = RKeywordBlacklist.objects.get(keyword=keyword)
url = reverse('keyword_blacklist_modify', args=(obj.id, ))
elif mail_obj.check_result == 'sender_blacklist' or 'sender_blacklist' in mail_obj.check_message:
obj = RSenderBlacklist.objects.get(keyword=keyword)
url = reverse('sender_blacklist_modify', args=(obj.id, ))
elif mail_obj.check_result == 'sender_whitelist':
keyword = re.findall('\((.*?)\)',
mail_obj.check_message)[0].split('----', 1)[0]
obj = SenderWhitelist.objects.get(sender=keyword)
url = reverse('sender_whitelist_modify', args=(obj.id, ))
elif mail_obj.check_result in [
'attach_blacklist', 'high_risk'
] or 'attach_blacklist' in mail_obj.check_message:
obj = AttachmentBlacklist.objects.get(keyword=keyword)
url = reverse('attachment_blacklist_modify', args=(obj.id, ))
if operate == 'modify':
return HttpResponseRedirect(url)
elif operate == 'delete':
obj.delete()
msg = u'删除成功'
return HttpResponse(json.dumps({'msg': msg}),
content_type="application/json")
except Exception, e:
if operate == 'modify':
return HttpResponse(msg)
elif operate == 'delete':
return HttpResponse(json.dumps({'msg': msg}),
content_type="application/json")
def load_mail(mail_ident):
date, id = mail_ident.split(',')[:2]
while True:
try:
mail = get_mail_model(date).objects.get(id=id)
d = {
'customer_id': mail.customer_id,
'domain': address_domain(mail.mail_to),
'mail_ident': mail_ident,
'sender': mail.mail_from,
'receiver': mail.mail_to,
'result': []
}
return d, mail
except (DatabaseError, InterfaceError) as e:
# 如果报数据库异常，关闭连接，重新处理任务
log.warning(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.warning(u'load_mail: exception', exc_info=1)
gevent.sleep(10)
def get_search_data(data, date=''):
if not date:
date = data.get('date', '')
customers = data.get('customers', '')
mail_from = data.get('mail_from', '')
mail_to = data.get('mail_to', '')
subject = data.get('subject', '')
state = data.get('state', '')
error_type = data.get('error_type', '')
dspam_study = data.get('dspam_study', '')
check = data.get('check', '')
check_message = data.get('check_message', '')
not_check = data.get('not_check', '')
review = data.get('review', '')
send = data.get('send', '')
mail_id = data.get('mail_id', '')
customer = data.get('customer', '')
customer_report = data.get('customer_report', '')
search = data.get('search[value]', '')
order_column = data.get('order[0][column]', '')
order_dir = data.get('order[0][dir]', '')
show = data.get('show', '')
reviewer_id = data.get('reviewer_id', '')
client_ip = data.get('client_ip', '')
server_id = data.get('server_id', '')
filter_word = data.get('filter_word', '')
reviewer = data.get('reviewer', '')
all_check = data.get('all_check', '')
check_list = data.getlist('check', [])
time_start = data.get('time_start', '')
time_end = data.get('time_end', '')
check_list = data.getlist('check', '')
if not date:
date = time.strftime('%Y-%m-%d')
return_code = data.get('return_code', '')
if not date:
date = time.strftime('%Y%m%d')
else:
date = date.replace('-', '')
mail_model = get_mail_model(date)
mails = mail_model.objects.all()
if customers:
mails = mails.filter(
Q(customer__company__icontains=customers)
| Q(customer__username__icontains=customers))
if mail_from:
mails = mails.filter(mail_from__icontains=mail_from)
if mail_to:
mails = mails.filter(mail_to__icontains=mail_to)
if customer:
mails = mails.filter(customer_id=customer)
if error_type:
mails = mails.filter(error_type=error_type)
if dspam_study:
mails = mails.filter(dspam_study=dspam_study)
if isinstance(check_list, list) and len(check_list) > 1:
mails = mails.filter(check_result__in=check_list)
elif check:
if check == 'subject_and_keyword':
mails = mails.filter(check_result__in=[
'keyword_blacklist', 'subject_blacklist', 'sender_blacklist'
])
else:
mails = mails.filter(check_result=check)
if not_check:
mails = mails.exclude(check_result=not_check)
if review:
mails = mails.filter(review_result=review)
if show == 'sendlog':
mails = mails.exclude(return_code=None)
elif show == 'reject':
mails = mails.filter(state='reject').exclude(check_result__in=[
'auto_reject', 'k_auto_reject', 'auto_reject_attach',
'error_format'
])
elif show == 'review':
state = 'review'
if state:
mails = mails.filter(state=state)
if check_message:
mails = mails.filter(check_message__icontains=check_message)
if send == 'success':
return_code = '250'
elif send == 'fail':
return_code = '0'
if mail_id:
if mail_id == '0':
# all_check 搜索出mail_id=0被"收件人白名单"通过, 但其他收件人需要审核的邮件
if len(all_check) > 1:
_mail_ids = mails.exclude(mail_id=0).values_list(
'mail_id', flat=True).distinct()
mail_ids = mail_model.objects.filter(id__in=_mail_ids).exclude(
state='review').values_list('id', flat=True)
mails = mails.filter(Q(mail_id=0) | Q(mail_id__in=mail_ids))
else:
mails = mails.filter(mail_id=0)
else:
mails = mails.filter(Q(mail_id=mail_id) | Q(id=mail_id))
if subject:
mails = mails.filter(subject__icontains=subject)
if client_ip:
mails = mails.filter(client_ip=client_ip)
if server_id:
mails = mails.filter(server_id=server_id)
if reviewer_id:
mails = mails.filter(reviewer_id=reviewer_id)
if search:
mails = mails.filter(
Q(mail_from__icontains=search) | Q(mail_to__icontains=search)
| Q(subject__icontains=search))
if return_code:
if return_code == '0':
mails = mails.exclude(return_code=250)
else:
mails = mails.filter(return_code=return_code)
if customer_report:
mails = mails.filter(customer_report=customer_report)
if filter_word:
mails = mails.filter(check_message__icontains=filter_word)
if reviewer:
mails = mails.filter(reviewer__username=reviewer)
if time_start:
time_start = datetime.datetime.strptime(
'{} {}:00'.format(date, time_start), '%Y%m%d %H:%M:%S')
mails = mails.filter(created__gte=time_start)
if time_end:
time_end = datetime.datetime.strptime(
'{} {}:00'.format(date, time_end), '%Y%m%d %H:%M:%S')
mails = mails.filter(created__lte=time_end)
try:
mails.exists()
except:
mails = []
if show == 'review':
colums = [
'id', 'customer', 'mail_from', 'subject', 'size', 'created',
'check_message', 'check_result'
]
elif show == 'sendlog':
colums = [
'id', 'customer', 'mail_from', 'subject', 'size', 'mail_id',
'created', 'mail_to', 'return_code', 'deliver_time',
'return_message'
]
elif show == 'reject':
colums = [
'id', 'sender_name', 'subject', 'attach_name', 'attach_name',
'check_result'
]
else:
colums = [
'id', 'customer', 'mail_from', 'subject', 'size', 'mail_id',
'created', 'mail_to', 'state', 'check_result', 'review_result',
'return_code', 'deliver_time', 'return_message'
]
if isinstance(mails, query.QuerySet
) and order_column and int(order_column) < len(colums):
if order_dir == 'desc':
mails = mails.order_by('-%s' % colums[int(order_column)])
else:
mails = mails.order_by('%s' % colums[int(order_column)])
return mails
def get_cmail_count(mail_obj, date):
model = get_mail_model(date.replace('-', ''))
mail_id = mail_obj.mail_id if mail_obj.mail_id else mail_obj.id
return model.objects.filter(mail_id=mail_id).count() + 1
def get_mail_message(customer_id,
mail_to,
sender_from,
type='customer',
interval=0):
mail_model = get_mail_model(get_mail_date())
if type == 'customer':
if interval:
virus_mails = get_mails_from_interval(customer_id,
interval,
mail_to=mail_to,
check_result='virus')
other_mails = get_mails_from_interval(customer_id,
interval,
mail_to=mail_to,
check_result='not_virus')
else:
virus_mails = mail_model.objects.filter(customer__id=customer_id,
mail_to=mail_to,
state='reject',
check_result='virus',
review_result='reject')
other_mails = mail_model.objects.filter(
customer__id=customer_id,
mail_to=mail_to,
state='reject',
review_result='reject').exclude(check_result='virus')
t = template.Template(html_template)
mail_subject = u'[垃圾邮件隔离报告] {} — {}'.format(
mail_to,
datetime.today().strftime('%Y-%m-%d'))
else:
if interval:
virus_mails = get_mails_from_interval(customer_id,
interval,
mail_to='',
check_result='virus')
other_mails = get_mails_from_interval(customer_id,
interval,
mail_to='',
check_result='not_virus')
else:
virus_mails = mail_model.objects.filter(
customer__id=customer_id,
state='reject',
check_result='virus',
review_result='reject').order_by('mail_to')
other_mails = mail_model.objects.filter(
customer__id=customer_id,
state='reject',
review_result='reject').exclude(
check_result='virus').order_by('mail_to')
t = template.Template(m_html_template)
mail_subject = u'[垃圾邮件隔离报告] {}'.format(
datetime.today().strftime('%Y-%m-%d'))
if not virus_mails and not other_mails:
return None
context = {
'virus_mails': virus_mails,
'other_mails': other_mails,
}
html_content = t.render(Context(context))
mail_message = MIMEText(html_content, _subtype='html', _charset='utf-8')
mail_message['Subject'] = Header(mail_subject, 'utf-8')
mail_message['From'] = Header(sender_from, 'utf-8')
mail_message['To'] = Header(mail_to, 'utf-8')
return mail_message
def mail_read(request):
date_id = request.GET.get('id', '')
date, id = date_id.split('_')
cid = request.GET.get('cid', '')
aid = request.GET.get('aid', '')
download = request.GET.get('download', '')
view_body = request.GET.get('view_body', '')
view_source = request.GET.get('view_source', '')
export = request.GET.get('export', '')
if id and date:
mail_obj = get_mail_model(date).objects.get(id=id)
content = mail_obj.get_mail_content()
parse_obj = ParseEmail(content)
m = parse_obj.parseMailTemplate()
if cid or aid:
attachments = m['attachments']
if aid:
attach = attachments[int(aid)]
response = HttpResponse(attach.get('data', ''),
content_type=attach.get(
'content_type', '').split(';')[0])
else:
for one in attachments:
if one.get('content_id') == cid:
attach = one
response = HttpResponse(attach.get('data', ''),
content_type=attach.get(
'content_type',
'').split(';')[0])
break
if download:
response[
'Content-Disposition'] = 'attachment; filename="{}"'.format(
attach.get('decode_name',
'').replace('\n', ' ').encode('utf-8'))
return response
if view_body:
text = m.get('html_text', '')
charset = m.get('html_charset', '')
if not text:
text = m.get('plain_text', '')
charset = m.get('plain_charset', '')
link = '{}?id={}&cid=\g<cid>'.format(reverse('collect_mail_read'),
date_id)
text = re.sub('"cid:(?P<cid>.*?)"', link, text)
if mail_obj.check_result in [
'keyword_blacklist', 'custom_blacklist'
] and text and charset:
s = mail_obj.check_message.split('----', 1)[-1]
re_s = u'<span style="background-color: yellow"><b style="color:#A94442;">{}</b></span>'.format(
s).encode(charset)
s = s.encode(charset)
text = text.replace(s, re_s)
return HttpResponse(text, charset=charset)
if view_source:
return render_to_response(
"collect_mail/txt.html", {
'content': content.decode('gbk', 'ignore'),
},
context_instance=RequestContext(request))
if export:
response = HttpResponse(content, content_type='text/html')
response['Content-Disposition'] = 'attachment; filename="eml.eml"'
return response
subject = mail_obj.subject
if mail_obj.check_result in ['subject_blacklist', 'custom_blacklist']:
s = mail_obj.check_message.split('----', 1)[-1]
subject = high_light(subject, s)
hl_mail_from = mail_obj.mail_from
if mail_obj.check_result == 'sender_blacklist':
s = mail_obj.check_message.split('----', 1)[-1]
hl_mail_from = high_light(mail_obj.mail_from, s)
m['from'] = high_light(m['from'], s)
return render_to_response("collect_mail/mail_read.html", {
'm': m,
'id': id,
'date': date,
'mail_obj': mail_obj,
'subject': subject,
'hl_mail_from': hl_mail_from,
},
context_instance=RequestContext(request))
return HttpResponse(u'no email')
def run(self):
if not os.path.exists(self.mail_path):
return
date = time.strftime('%Y%m%d')
#创建django的mail模型
mail_model = get_mail_model(date)
if date not in _dates:
install_model(mail_model)
#创建的目录 用户组也有权限修改
common.make_dir([os.path.join(data_path, 'c_{}'.format(date))],
permit=0775)
_dates.append(date)
uid, mail_id, client_ip = self.task.split(',')[:3]
with open(self.mail_path) as f:
email_str = f.read()
with open(self.rcpt_path) as f:
rcpt_list = f.readlines()
with open(self.from_path) as f:
mail_from = f.read()
rcpt_list = [rcpt for rcpt in rcpt_list if can_decode(rcpt)]
# subject = get_mail_subject(mail_data=email_str)
p = ParseEmail(email_str)
subject = p.get_attr('subject')
try:
sender_name = re.search('(.*?)\(.*?\)',
p.get_attr('from')).group(1)
except:
outerror('get sender_name error: {}'.format(self.task))
outerror(traceback.format_exc())
sender_name = ''
data = p.parseMailTemplate()
attaches = data.get('attachments', [])
try:
attach_name = u'----'.join(
[a.get('decode_name', '') for a in attaches])
except:
outerror('get attach_name error: {}'.format(self.task))
outerror(traceback.format_exc())
attach_name = ''
#被推送到检测队列中的邮件
#当有多个收件人时, 只推送mail_id=0的一个对象到检测队列
check_mail = None
outinfo(self.task)
outinfo('rcpt_list:{}'.format(rcpt_list))
#mail id列表
mail_ids = []
for rcpt in rcpt_list:
if rcpt == '': continue
# remove \n
rcpt = rcpt.replace('\n', '')
# 去掉字符串中的引号
rcpt = rcpt.replace('"', '').replace("'", "")
rcpt = addr_get(rcpt)
try:
mail_obj = mail_model(customer_id=uid,
mail_from=mail_from.lower(),
mail_to=rcpt.lower(),
client_ip=client_ip,
subject=subject,
size=len(email_str),
sender_name=sender_name,
server_id=server_id,
attach_name=attach_name)
mail_obj.save()
except (DatabaseError, InterfaceError), e:
outerror(traceback.format_exc())
continue
with open(mail_obj.get_mail_path(), 'w') as f:
f.write(email_str)
filename = mail_obj.get_mail_filename()
if not check_email_format(rcpt,
is_check_suffix=False) or is_same_domain(
rcpt, mail_from):
outinfo('{}: error format'.format(filename))
self._do_invalid_mail(mail_obj, 'error_format')
else:
outinfo('{}: pass'.format(filename))
if not check_mail:
check_mail = mail_obj
mail_ids.append(mail_obj.id)
def mail_modify_spam(request):
date_id = request.GET.get('id', False)
mail_date, mail_id = date_id.split('_')
action = request.GET.get('action', False)
auth_key = request.GET.get('auth_key', False)
decoded = base64.b64decode(auth_key)
private_tr, action2, mail_date2, mail_id2 = decoded.split('_')
if mail_id and mail_date and action:
if action == action2 and mail_date == mail_date2 and mail_id == mail_id2:
try:
mail_obj = get_mail_model(mail_date).objects.get(id=mail_id)
except Exception as e:
return HttpResponse(u'页面不存在 (404)')
if action == 'delete':
file_path = mail_obj.get_mail_path()
if mail_obj.review_result == 'reject':
if os.path.exists(file_path):
mail_obj.clear_mail()
return HttpResponse(u'删除邮件成功')
else:
return HttpResponse(u'删除邮件成功')
else:
return HttpResponse(u'邮件正在处理中，不能进行删除操作')
# mail_obj.clear_mail()
# return HttpResponse(u'删除邮件成功')
if action == 'whitelist':
add_sender_whitelist(date_id, is_global=False, is_domain=False)
if mail_obj.review_result == 'reject':
undo_review_result = 'pass'
_do_review([date_id],
undo_review_result,
request,
flag=False)
return HttpResponse(u'添加白名单成功')
else:
return HttpResponse(u'添加白名单成功')
# undo_review_result = 'pass' if mail_obj.review_result == 'reject' else 'reject'
# _do_review([date_id], undo_review_result, request, flag=False)
# return HttpResponse(u'添加白名单成功')
if action == 'deliver':
if mail_obj.review_result == 'reject':
undo_review_result = 'pass'
_do_review([date_id],
undo_review_result,
request,
flag=False)
return HttpResponse(u'邮件放行成功')
else:
return HttpResponse(u'邮件放行成功')
# undo_review_result = 'pass' if mail_obj.review_result == 'reject' else 'reject'
# _do_review([date_id], undo_review_result, request, flag=False)
# return HttpResponse(u'邮件放行成功')
else:
return HttpResponse(u'页面不存在 (404)')
else:
return HttpResponse(u'页面不存在 (404)')
def _do_review(ids, result, request, flag=True):
"""
:param ids: 审核的ID列表
:param result: 审核结果
:param flag: 是否正常流程操作， 默认：True, 如果是误判操作，则为False
:return:
"""
keys = {}
# redis = get_redis_connection()
reviewer = request.user
review_time = datetime.datetime.now()
msg = u'通过' if result == 'pass' else u'拒绝'
for k, v in deal_with_ids(ids).iteritems():
if flag:
mails = get_mail_model(k).objects.filter(
state='review').filter(Q(id__in=v) | Q(mail_id__in=v))
else:
mails = get_mail_model(k).objects.filter(id__in=v)
servers = mails.values_list('server_id', flat=True).distinct()
for s in servers:
keys.setdefault(s, []).extend(
map(lambda mail: mail.get_mail_filename(),
mails.filter(server_id=s)))
# keys.extend(map(lambda mail: mail.get_mail_filename(), mails))
review_result = result if flag else '{}_undo'.format(result)
if result == 'pass':
if flag:
mails.update(review_result=review_result,
state='send',
reviewer=reviewer,
review_time=review_time)
else:
mails.update(review_result=review_result, state='send')
elif result == 'reject':
if flag:
mails.update(review_result=review_result,
state='reject',
reviewer=reviewer,
review_time=review_time)
else:
mails.update(review_result=review_result, state='reject')
count = 0
for k, v in keys.iteritems():
count += len(v)
try:
redis = get_redis_connection2(k)
except:
redis = get_redis_connection2('default')
for m in set(v):
if result == 'pass':
redis.lpush('collect_send', m)
reason = 'review_pass'
else:
reason = 'review_reject'
redis.lpush('dspam_collect_{}'.format(result), m)
redis.lpush('csender_credit',
'{}----{}'.format('_'.join(m.split(',')), reason))
"""
if result == 'pass':
map(lambda key: redis.lpush('collect_send', key), set(v))
map(lambda key: redis.lpush('dspam_collect_{}'.format(result), key), set(v))
"""
return msg, count
