def get_date(value):
try:
date = value.split(';')[-1].strip().split(' +0800')[0]
tuple_time = time.strptime(date, '%a, %d %b %Y %H:%M:%S')
value = time.mktime(tuple_time)
except Exception:
traceback.print_exc()
error_msg = "run error\t\n error_msg: \n%s" % traceback.format_exc()
log_mail(error_msg)
print('Send an email with an error message')
return value
def push_data(data_list):
# print(data_list)
dict_date = {
data_list[x].get('received_time'): data_list[x]
for x in range(len(data_list))
}
# print(dict_date)
list_date = [x for x in dict_date.keys()]
# print(list_date)
list_date.sort()
# print(list_date)
data_list = [dict_date.get(x) for x in list_date]
step = 10
data_sort = [data_list[i:i + step] for i in range(0, len(data_list), step)]
for data_s in data_sort:
num = len(data_s)
data = json.dumps({'data': data_s})
# 测试库
# url = settings.PUSH_TEST_URL
# 正式库
url = settings.PUSH_URL
for i in range(5):
error_data = None
try:
res = requests.post(url, data, timeout=settings.PUSH_TIMEOUT)
if json.loads(res.text).get('status') != 0:
error_data = 'return status error:%s' % res.text
print(error_data)
else:
if json.loads(res.text).get('err_num') > 0:
error_msg = "Upload data error\t\n error_msg: \n%s\t\n error_data: \n%s" % (
res.text, data)
log_mail(error_msg)
print('Send an email with an error message')
else:
# print(data)
print("pushData: %s" % num)
time.sleep(1)
break
except Exception:
# traceback.print_exc()
error_data = traceback.format_exc()
print('push data error')
finally:
if i == 4:
error_msg = "Retry 5 upload errors\t\n error_msg: \n%s\t\n error_data: \n%s" % (
error_data, data)
log_mail(error_msg)
print(
'Too many failures, give up uploading, send error mail'
)
time.sleep(5)
def forward_parse(msg_dict):
# print(msg_dict.get('From'))
"""
:param msg_dict: 传入分析的数据字典
"""
for key in carrier_parse:
if msg_dict.get('From').find(key) < 0:
continue
try:
# print(key)
data = carrier_parse.get(key).data_parse(msg_dict)
except Exception:
# 出现异常，保存异常信息并发送邮件
msg_dict.pop('text')
error_msg = "carrier: \n%s\t\n\n" \
"error_msg: \n%s\t\n\n" \
"error_data: \n%s" % (key, traceback.format_exc(), msg_dict)
log_mail(error_msg)
print('Send an email with an error message')
return
if data:
# 数据出现错误时发送日志邮件
if data.get('error_data'):
msg_dict.pop('text')
error_msg = "carrier: \n%s\t\n\n" \
"error_msg: \n%s\t\n\n" \
"error_data: \n%s" % (key, data.get('error_data'), msg_dict)
log_mail(error_msg)
print('Send an email with an error message')
return
# 增加邮件时间
data['received_time'] = msg_dict.get('Received')
data_list.append(data)
# coding: utf-8
import traceback
import time
from main.main import main
from mail.log_mail import log_mail
if __name__ == '__main__':
while True:
try:
main()
except:
traceback.print_exc()
error_msg = "run error\t\n error_msg: \n%s" % traceback.format_exc(
)
log_mail(error_msg)
print('Send an email with an error message')
time.sleep(20 * 60)
def mail_handle():
global num_163, num_qq
# 遍历邮箱服务商
for mailbox_server in mail_config:
print('crawling %s mailbox' % mailbox_server)
# 获取账号，密码，端口, 网络连接类型
email, password, pop3_server, transport_protocols = (
x for x in mail_config.get(mailbox_server))
# 登录邮件服务器,防止出错，再次请求
server = None
for num_error in range(10):
try:
server = login_mail(email, password, pop3_server,
transport_protocols)
break
except Exception as e:
# if num_error == 9:
# raise e
pass
# 显示邮件数量和占用空间
print('Messages:%s.\nSize:%s' % server.stat())
# # 显示邮件编号
# resp, mails, octets = server.list()
# 获取最新的邮件，索引是从最早的邮件开始
max_num = server.stat()[0]
# 获取任务数量，同时替换最大任务数据
if mailbox_server == '163':
task_num = max_num - num_163 + 50
num_163 = max_num
else:
task_num = max_num - num_qq + 50
num_qq = max_num
# max_num = 59319
# task_num = 1000
for i in range(task_num):
index = max_num - i
# index=60458
msg, error_data = None, ''
for x in range(5): # 为防止程序卡死，最多可重试5次
try:
msg = get_msg(server, index)
break
except:
print('获取邮件出错，等待1秒重新请求。')
error_data = traceback.format_exc()
server = login_mail(email, password, pop3_server,
transport_protocols)
if x == 4:
error_msg = "邮箱：%s\t\n\r出错邮件：%s\t\n\r错误信息：%s" % (
email, index, error_data)
log_mail(error_msg)
print('get mail error,send error msg')
time.sleep(1)
if not msg:
# 发现QQ邮箱的退信会出现这种问题
print("邮件内容出错，邮件编号：%s" % index)
continue
# 展示邮件信息
msg_dict = print_info(msg)
yield msg_dict
server.quit()