def res_email():
msg_style = '''<html>
<head>
<meta charset="utf-8">
</head>
<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; }
.table td{border:solid 1px gray; padding:6px;}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>
<body class="body">
<h2>web api 在线监控报表, 监控每5分钟跑一次</h2>'''
msg = sql_related()
msg = msg_style + msg + "<br><br>2开头是正常的<br><br>"
msg += '</body>' + '</html>'
msg = msg.replace('<table>',
'<table class="table">').replace('<th>',
"<th class='table'>")
common.sendEmail(title='web api 在线监控报表', message=msg)
def web_test(name, url, op, data=None):
if data is None:
data = dict()
s = requests.Session()
start = time.clock()
start2 = timeit.default_timer()
start3 = time.time()
if op == 'get':
r = s.get(url, params=data, timeout=15)
elif op == 'post':
r = s.post(url, data=data, timeout=15)
else:
logging.error('invalid http method')
logging.info('test url is {}'.format(r.url))
end = time.clock()
stop3 = time.time()
stop2 = timeit.default_timer()
logging.info('timeit {}'.format(stop2 - start2))
logging.info('time.time {}'.format(stop3 - start3))
resp = r.status_code
logging.info('response time time clock is {}'.format(end - start))
resp_time = stop2 - start2
if resp < 299:
# do redis
logging.info('url {} works normal, status_code is {}'.format(
url, resp))
else:
logging.error('http code error code is {}'.format(resp))
common.sendEmail(
title='{} auto test get err code {}'.format(name, resp),
message='RT, Web api test error, url is {}, code {}, time is {}'.
format(url, resp, resp_time))
return resp, resp_time
def res_email():
msg_style = '''<html>
<head>
<meta charset="utf-8">
</head>
<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; }
.table td{border:solid 1px gray; padding:6px;}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>
<body class="body">
<h2>talent 当天推送简历id列表，来源为2 的是本来就是talent表的数据，
来源为3 的表示从搜索库过来的， 目前现在测试服测试推送效果，可以了部署到正式服上去<br>
16:30 推送talent表来源的， 10:35和13:35推送搜索库来源的，搜索库来源的都是当天入库（更新的）</h2>'''
msg = res()
msg2 = res2()
msg = msg_style + msg + "<br><br>今天自动拒绝的简历id列表，拒绝理由随便写的<br><br>" + msg2
msg += '</body>' + '</html>'
msg = msg.replace('<table>', '<table class="table">').replace('<th>', "<th class='table'>")
common.sendEmail(title='talent表自动推送简历列表(测试服)', message=msg)
def web_test(name, url, op, data=None):
if data is None:
data = dict()
s = requests.Session()
start = time.clock()
start2 = timeit.default_timer()
start3 = time.time()
if op == 'get':
r = s.get(url, params=data, timeout=15)
elif op == 'post':
r = s.post(url, data=data, timeout=15)
else:
logging.error('invalid http method')
logging.info('test url is {}'.format(r.url))
end = time.clock()
stop3 = time.time()
stop2 = timeit.default_timer()
logging.info('timeit {}'.format(stop2 - start2))
logging.info('time.time {}'.format(stop3 - start3))
resp = r.status_code
logging.info('response time time clock is {}'.format(end-start))
resp_time = stop2 - start2
if resp < 299:
# do redis
logging.info('url {} works normal, status_code is {}'.format(url, resp))
else:
logging.error('http code error code is {}'.format(resp))
common.sendEmail(title='{} auto test get err code {}'.format(name, resp),
message='RT, Web api test error, url is {}, code {}, time is {}'.format(url, resp, resp_time))
return resp, resp_time
def res_email():
msg_style = '''<html>
<head>
<meta charset="utf-8">
</head>
<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; }
.table td{border:solid 1px gray; padding:6px;}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>
<body class="body">
<h2>web api 在线监控报表, 监控每5分钟跑一次</h2>'''
msg = sql_related()
msg = msg_style + msg + "<br><br>2开头是正常的<br><br>"
msg += '</body>' + '</html>'
msg = msg.replace('<table>', '<table class="table">').replace('<th>', "<th class='table'>")
common.sendEmail(title='web api 在线监控报表', message=msg)
def main():
'''统计前一天的重复简历数，并发送通知邮件。'''
# 先从日志文件过滤需要的信息
for k in log_file_dict:
log_filter(k)
# 生成邮件正文
msg = get_email_msg()
# 发送邮件
title = '简历抓取-重复简历统计'
common.sendEmail(title=title, message=msg)
def main():
'''统计前一天的重复简历数，并发送通知邮件。'''
# 先从日志文件过滤需要的信息
for k in log_file_dict:
log_filter(k)
# 生成邮件正文
msg = get_email_msg()
# 发送邮件
title = '简历抓取-重复简历统计'
common.sendEmail(title=title, message=msg)
def task2():
while True:
try:
logging.info("monitor running")
exts = get_ext()
for i in exts:
ext = i[0]
results = get_data(ext)
if results is not None and len(results) > 5:
content = "recharge error："
logging.info("a lot of error,so send mail")
for r in results:
content += r[1]+'\n'
common.sendEmail(content)
time.sleep(2*60)
except Exception as e:
logging.error('monitor err msg is {}'.format(e), exc_info=True)
def task2():
while True:
try:
logging.info("monitor running")
exts = get_ext()
for i in exts:
ext = i[0]
results = get_data(ext)
if results is not None and len(results) > 5:
content = "recharge error："
logging.info("a lot of error,so send mail")
for r in results:
content += r[1] + '\n'
common.sendEmail(content)
time.sleep(2 * 60)
except Exception as e:
logging.error('monitor err msg is {}'.format(e), exc_info=True)
def main():
sr_list = source_list()
html = ''
for i in sr_list:
# html_i = '<br>' + i + '</br>'
table_list = table_cr(i)
# big_dict = big_cr(table_list)
html_i = '<br>' + i + '</br>' + no_list2(table_list)
html += html_i
html = html.replace('<table>', '<table class="table">').replace(
'<td>',
'<td style="text-align:right">').replace('<th>', "<th class='table'>")
msg_style = """<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; padding:6px}
.table td{border:solid 1px gray; padding:6px}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>"""
msg_head = """<html><head><meta charset="utf-8"></head>""" + msg_style + "<body>"
msg = msg_head # + u"""<h2>资讯来源报表，只展示4天内的情况，表格中的时间是文章发布时间</h2>"""
data_pass, data_to = to_pass2()
count_cnb = cnb_count()
# data_pass, data_to = 0, 0
msg += u"<br>昨天抓取的文章总量： " + str(data_pass) + u"； 今天抓取的文章总量： " + str(data_to) + ' 这两个是抓取时间，遍历20W的发布时间太慢了<br>' + \
u" 资讯类数据总量： " + str(total_00(1)) + '<br>' + u" 博客类数据总量： " + str(total_00(2)) + '<br>' +\
u" 微信总量： " + str(total_00(3)) + '<br>' + \
u"<br> cnblog总数（单独抓历史文章）： " + str(count_cnb) + u" （暂时分4类，分类可以加多，news_info表 type 字段）<br>"
stop_l = stop_list()
msg = msg + stop_l + html + "</body></html>"
msg = msg.encode('utf8')
common.sendEmail(title='资讯抓取报表', message=msg)
def main():
'''统计简历分类消息队列中的消息数'''
msg = '<p>当前时间：</p>'
# 当前时间
now = datetime.datetime.now()
msg += '<p>%s</p>' % now
# 获取当前队列中的消息数
redis_instant = redis.StrictRedis(host=host, port=port, db=db)
if redis_instant.ping():
msg_count = redis_instant.llen(queue_name)
tmp_msg = '<p>消息队列长度：%s</p>' % msg_count
else:
'<p>redis未启动</p>'
msg += tmp_msg
print msg
# 发送邮件
title = '简历分类-消息队列长度监测'
common.sendEmail(title=title, message=msg)
def res_email():
msg_style = '''<html>
<head>
<meta charset="utf-8">
</head>
<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; }
.table td{border:solid 1px gray; padding:6px;}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>
<body class="body">
<h2>talent 当天推送简历id列表，来源为2 的是本来就是talent表的数据，
来源为3 的表示从搜索库过来的， 目前现在测试服测试推送效果，可以了部署到正式服上去<br>
16:30 推送talent表来源的， 10:35和13:35推送搜索库来源的，搜索库来源的都是当天入库（更新的）</h2>'''
msg = res()
msg2 = res2()
msg = msg_style + msg + "<br><br>今天自动拒绝的简历id列表，拒绝理由随便写的<br><br>" + msg2
msg += '</body>' + '</html>'
msg = msg.replace('<table>',
'<table class="table">').replace('<th>',
"<th class='table'>")
common.sendEmail(title='talent表自动推送简历列表(测试服)', message=msg)
msg_style = """<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; padding:6px}
.table td{border:solid 1px gray; padding:6px}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>"""
msg_head = """<html><head><meta charset="utf-8"></head>""" + msg_style + "<body>"
msg = msg_head + u"""<h2>资讯来源报表，只展示4天内的情况，表格中的时间是文章发布时间</h2>"""
msg += u"<br>昨天发布的文章总量： " + str(len(data_pass)) + u"； 今天发布的文章总量： " + str(len(data_to)) + '<br>' + \
u"资讯类数据总量： " + str(total_00(1)) + u" 博客类数据总量： " + str(total_00(2)) + \
u"微信总量： " + str(total_00(3)) + \
u"<br>cnblog总数（单独抓历史文章）： " + str(count_cnb) + u" （暂时分3类，分类可以加多，news_info表 type 字段）<br>"
msg = msg + html + html_no + "</body></html>"
msg = msg.encode('utf8')
common.sendEmail(title='资讯抓取报表', message=msg)
def momo(multi, sl):
""" 倍数，睡眠时间， 100倍的检测睡眠 3分钟，2倍的睡眠一小时 """
while True:
try:
logging.info("monitor {} multi sleep {} running".format(multi, sl))
ld = runing()
lll = compare(ld, multi)
content = ''
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if len(lll) > 0:
x = PrettyTable(["fun_name", "今天", "昨天", '前天', "平均值", "次数"])
x.align["fun_name"] = "l"
x.border = True
content += now + u'\n <span style="color:red"> {} </span> 倍于昨天与前天的平均值 \n'.format(
multi)
for i in lll:
# content += ' {} this hour today num is {}, yesterday is {}, the day before yesterday is {}'\
# .format(i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0))\
# + '\n'
if multi >= 100: # 大于100 的才采取这策略
if i not in func_dict:
x.add_row([
i, ld[i].get(0), ld[i].get(1,
0), ld[i].get(2, 0),
(int(ld[i].get(1, 0)) + int(ld[i].get(2, 0))) /
2, 1
])
func_dict[i] = {}
func_dict[i]['time'] = datetime.datetime.now(
) # 更新时间
func_dict[i]['num'] = 1
logger.info('add key in func_dict')
else:
now = datetime.datetime.now()
if datetime.timedelta(hours=1) >= now - func_dict[
i]['time'] > datetime.timedelta(
hours=0.5): # 把key 删了，邮件发出来
x.add_row([
i, ld[i].get(0), ld[i].get(1, 0),
ld[i].get(2, 0),
(int(ld[i].get(1, 0)) +
int(ld[i].get(2, 0))) / 2,
func_dict[i]['num'] + 1
])
func_dict.pop(i)
logger.info('send email and remove key')
elif now - func_dict[i][
'time'] > datetime.timedelta(
hours=1): # 这个不发邮件，直接移除
func_dict.pop(i)
logger.info(
'remove key because it appear before 1 hour'
)
else: # 更新时间， 更新数字，这个就不发邮件了
# func_dict[i]['time'] = datetime.datetime.now() # 更新时间
func_dict[i]['num'] += 1
logger.info('add num')
else:
x.add_row([
i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0),
(int(ld[i].get(1, 0)) + int(ld[i].get(2, 0))) / 2,
1
])
# print content
print content.encode('utf8')
print x.get_string().encode('utf8')
# logging.info('content is ' + content)
if x.get_html_string().count('</tr>') >= 2: # 有才发出邮件出来
html = x.get_html_string(sortby=u'今天')
msg = html_cr(content.encode('utf8'), html)
msg += u"""<br>两个进程<br> 3分钟检测一次，超过前天跟昨天同一小时平均值100倍而且大于120，就会发告警邮件。<br>
一小时检测一次，超过前天跟昨天一小时平均值的2倍，就会发告警邮件， <br>
次数是半小时内出现的次数（只针对100倍的检测），半小时内出现计数，半小时后再出现将次数发出来"""
common.sendEmail(title='fun_action warning',
message=msg,
des='test')
logger.info('sending email')
else:
logger.info('donot send email')
logging.info('multi {} sleep {} min'.format(multi, sl))
time.sleep(sl * 60)
except Exception as e:
logging.error('monitor err msg is {}'.format(e), exc_info=True)
print 'asdfkjasljkflasjdf'
time.sleep(60 * 60)
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; padding:6px}
.table td{border:solid 1px gray; padding:6px}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>"""
msg_head = """<html><head><meta charset="utf-8"></head>""" + msg_style + "<body>"
msg = msg_head + """<h2>简历下载账号信息</h2>"""
msg2 = grap_info()
msg = msg + msg2 + "</body></html>"
msg = msg.replace('<table>', '<table class="table">').replace(
'<td>',
'<td style="text-align:right">').replace('<th>', "<th class='table'>")
# print msg
return msg
if __name__ == '__main__':
data = grap_info()
msg = eformat(data)
common.sendEmail('main', '简历渠道账号信息', msg, msg_type=1, des='op')
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; padding:6px}
.table td{border:solid 1px gray; padding:6px}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>"""
msg_head = """<html><head><meta charset="utf-8"></head>""" + msg_style + "<body>"
msg = msg_head + """<h2>简历下载账号信息</h2>"""
msg2 = grap_info()
msg = msg + msg2 + "</body></html>"
msg = msg.replace('<table>', '<table class="table">').replace('<td>', '<td style="text-align:right">').replace('<th>', "<th class='table'>")
# print msg
return msg
if __name__ == '__main__':
data =grap_info()
msg = eformat(data)
common.sendEmail('main', '简历渠道账号信息', msg, msg_type=1, des= 'op')
def momo(multi, sl):
""" 倍数，睡眠时间， 100倍的检测睡眠 3分钟，2倍的睡眠一小时 """
while True:
try:
logging.info("monitor {} multi sleep {} running".format(multi, sl))
ld = runing()
lll = compare(ld, multi)
content = ''
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if len(lll) > 0:
x = PrettyTable(["fun_name", "今天", "昨天", '前天', "平均值", "次数"])
x.align["fun_name"] = "l"
x.border = True
content += now + u'\n <span style="color:red"> {} </span> 倍于昨天与前天的平均值 \n'.format(multi)
for i in lll:
# content += ' {} this hour today num is {}, yesterday is {}, the day before yesterday is {}'\
# .format(i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0))\
# + '\n'
if multi >= 100: # 大于100 的才采取这策略
if i not in func_dict:
x.add_row([i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0), (int(ld[i].get(1, 0)) +
int(ld[i].get(2, 0))) / 2, 1])
func_dict[i] = {}
func_dict[i]['time'] = datetime.datetime.now() # 更新时间
func_dict[i]['num'] = 1
logger.info('add key in func_dict')
else:
now = datetime.datetime.now()
if datetime.timedelta(hours=1) >= now - func_dict[i]['time'] > datetime.timedelta(hours=0.5): # 把key 删了，邮件发出来
x.add_row([i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0), (int(ld[i].get(1, 0)) +
int(ld[i].get(2, 0))) / 2, func_dict[i]['num'] + 1])
func_dict.pop(i)
logger.info('send email and remove key')
elif now - func_dict[i]['time'] > datetime.timedelta(hours=1): # 这个不发邮件，直接移除
func_dict.pop(i)
logger.info('remove key because it appear before 1 hour')
else: # 更新时间， 更新数字，这个就不发邮件了
# func_dict[i]['time'] = datetime.datetime.now() # 更新时间
func_dict[i]['num'] += 1
logger.info('add num')
else:
x.add_row([i, ld[i].get(0), ld[i].get(1, 0), ld[i].get(2, 0), (int(ld[i].get(1, 0)) +
int(ld[i].get(2, 0)))/2, 1])
# print content
print content.encode('utf8')
print x.get_string().encode('utf8')
# logging.info('content is ' + content)
if x.get_html_string().count('</tr>') >= 2: # 有才发出邮件出来
html = x.get_html_string(sortby=u'今天')
msg = html_cr(content.encode('utf8'), html)
msg += u"""<br>两个进程<br> 3分钟检测一次，超过前天跟昨天同一小时平均值100倍而且大于120，就会发告警邮件。<br>
一小时检测一次，超过前天跟昨天一小时平均值的2倍，就会发告警邮件， <br>
次数是半小时内出现的次数（只针对100倍的检测），半小时内出现计数，半小时后再出现将次数发出来"""
common.sendEmail(title='fun_action warning', message=msg, des='test')
logger.info('sending email')
else:
logger.info('donot send email')
logging.info('multi {} sleep {} min'.format(multi, sl))
time.sleep(sl*60)
except Exception as e:
logging.error('monitor err msg is {}'.format(e), exc_info=True)
print 'asdfkjasljkflasjdf'
time.sleep(60*60)
config = Config(env)
ip = config.getValue("db", "ip")
username = config.getValue("db", "username")
databaseName = config.getValue("db", "database")
password = config.getValue("db", "password")
conn = initDB(ip, databaseName, username, password)
ISOTIMEFORMAT = '%Y-%m-%d'
#todayTimeStr = datetime.date.today().strftime(ISOTIMEFORMAT)
#yesterdayTimeStr = (datetime.date.today()+datetime.timedelta(-1)).strftime(ISOTIMEFORMAT)
#tomorrorTimeStr = (datetime.date.today()+datetime.timedelta(1)).strftime(ISOTIMEFORMAT)
yesterdayTimeStr = "2015-11-27"
tomorrorTimeStr = "2015-11-29"
todayTimeStr = "2015-11-28"
todayTableName = "account"+todayTimeStr
yesterdayTableName = "account" + yesterdayTimeStr
todayAccountDataIter = conn.iter("SELECT * FROM `"+todayTableName+"`" )
yesterdayAccountDataIter = conn.iter("SELECT * FROM `" + yesterdayTableName+"`")
recordsIter = conn.iter("SELECT * FROM trade_record where update_time<\'" + tomorrorTimeStr +"\' AND update_time > \'" + yesterdayTimeStr+"\'")
YAD = dict()
TAD = dict()
for data in yesterdayAccountDataIter:
YAD[data['user_id']] = data
for data in todayAccountDataIter:
TAD[data['user_id']] = data
records = getRecord(YAD, TAD, recordsIter)
checkData(TAD, YAD, records)
#checkRecord(TAD, YAD, records)
checkCompanyData(YAD, TAD, records["1"])
print(EMAIL_MESSAGE)
sendEmail(['*****@*****.**', '*****@*****.**'])
msg_style = """<style type="text/css">
.body{
font-family: Monaco, Menlo, Consolas, "Courier New", "Lucida Sans Unicode", "Lucida Sans", "Lucida Console", monospace;
font-size: 14px;
line-height: 20px;
}
.table{ border-collapse:collapse; border:solid 1px gray; padding:6px}
.table td{border:solid 1px gray; padding:6px}
.color-ok {color: green;}
.color-warning {color: coral;}
.color-error {color: red;}
.bg-ok {background-color: lavender;}
.bg-warning {background-color: yellow;}
.bg-error {background-color: deeppink;}
</style>"""
msg_head = """<html><head><meta charset="utf-8"></head>""" + msg_style + "<body>"
msg = msg_head + u"""<h2>资讯来源报表，只展示4天内的情况，表格中的时间是文章发布时间</h2>"""
msg += u"<br>昨天发布的文章总量： " + str(len(data_pass)) + u"； 今天发布的文章总量： " + str(len(data_to)) + '<br>' + \
u"资讯类数据总量： " + str(total_00(1)) + u" 博客类数据总量： " + str(total_00(2)) + \
u"微信总量： " + str(total_00(3)) + \
u"<br>cnblog总数（单独抓历史文章）： " + str(count_cnb) + u" （暂时分3类，分类可以加多，news_info表 type 字段）<br>"
msg = msg + html + html_no + "</body></html>"
msg = msg.encode('utf8')
common.sendEmail(title='资讯抓取报表', message=msg)
def call(base_path, domain):
# vlink账号
appid = config.vlink_appid
token = config.vlink_token
today = date.today()
# 2. 获取当前月的第一天
first = today.replace(day=1)
# 3. 减一天，得到上个月的最后一天
last_month = first - timedelta(days=1)
# 表名(动态变化的)
table_name = "record_callin_" + last_month.strftime("%Y%m")
try:
conn = MySQLdb.Connect(host=config.db_host, user=config.db_user, passwd=config.db_pwd, db=config.db_name,
compress=1,
cursorclass=MySQLdb.cursors.DictCursor, charset='utf8', connect_timeout=5)
except:
# print 'connect db is error!!'
common.debug("connect db is error!!")
common.sendEmail()
sys.exit(0)
if (common.table_exists(conn, table_name) != 1):
common.log(conn, 'month', table_name+"表不存在")
sys.exit(0)
common.log(conn, 'month', "按月下载音频程序启动...当前表 "+table_name)
query = "select id,voice_url,source_url,operator,create_time from %s where voice_url is null " % table_name
cursor = conn.cursor()
cursor.execute(query)
data = cursor.fetchall()
num = len(data)
# 没有数据时直接关闭链接 退出
if num is 0:
common.log(conn, 'month', "按月下载音频程序结束：无下载的数据")
cursor.close()
conn.close()
sys.exit(0)
total = 0
for row in data:
url = row['source_url']
if row['operator'] =='vlink':
url = config.vlink_domain+url
if common.check_file_ext(url) is False:
continue
operator = row['operator'] # 运营商
if operator == "vlink":
# 获取动态签名
timestamp = str(int(time.time()))
sign = common.getSign(timestamp)
# url = config.vlink_domain+url
prefix = "?appId=%s×tamp=%s&sign=%s" %(appid, timestamp, sign)
url = url+prefix
print(url)
# 文件存储目录名(e.g. 20191212),以通话记录时间他创建目录
time_arr = time.strptime(str(row['create_time']), "%Y-%m-%d %H:%M:%S")
file_dir = time.strftime("%Y%m%d", time_arr)
file_url = common.download_file(url, base_path, file_dir)
try:
if file_url:
total = total + 1
file_url = domain + '/' + file_url if domain else file_url
query = "update %s set voice_url='%s' where id=%d" % (table_name, file_url, row['id'])
else:
query = "update %s set voice_url=null where id=%d" % (table_name, row['id'])
time.sleep(0.1)
cursor.execute(query)
conn.commit()
except Exception as e:
conn.rollback()
print('rollback')
common.log(conn, 'month', "按月下载音频程序结束：需下载数据"+str(num)+"条，处理"+str(total)+"条数据,问题条数：("+str(num-total)+")")
cursor.close()
conn.close()
