def event_handler(self, evt, msg, **kwargs):
if "START" == evt:
util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
return
if "DONE" == evt:
util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
return
def event_handler(self, evt, msg, **kwargs):
if "START" == evt:
util.send_email(["<*****@*****.**>"], "2cimport 任务", msg)
return
if "DONE" == evt:
util.send_email(["<*****@*****.**>"], "2cimport 任务", msg)
return
def event_handler(self, evt, msg, **kwargs):
if "START" == evt:
util.send_email(["<*****@*****.**>"], "{}_removePastPage 任务".format(self.channel), msg)
return
if "DONE" == evt:
util.send_email(["<*****@*****.**>"], "{}_removePastPage 任务".format(self.channel), msg)
return
def event_handler(self, evt, msg, **kwargs):
if evt == 'DONE':
util.send_email(["<*****@*****.**>"], "{} 样本对比".format(self.channel), msg)
# 最终结果存放文件
filedest = 'app/share/%s_result.txt' % self.channel
if os.path.exists(filedest):
now = util.get_date_with_day_duration()
history_fn = os.path.join(os.path.dirname(filedest), '%s_%s.txt' % (self.channel, '%d%02d%02d%02d%02d' % (now.year, now.month, now.day, now.hour, now.minute)))
# 将结果移动到历史文件
os.system('mv %s %s' % (filedest, history_fn))
# 将最后计算结果放到share 目录下， 提供下载
os.system('mv %s %s' % (self.result_file, filedest))
def clean(channel):
result_dir = "result/%s_result" % channel
if not os.path.exists(result_dir):
os.makedirs(result_dir)
now = time.localtime()
now_str = "%04d%02d%02d" % (now.tm_year, now.tm_mon, now.tm_mday)
fail_ids_file = "%s/%s_fail_ids_%s" % (result_dir, channel,now_str)
cmd = "cat result/%s_fail_ids* | sort | uniq > %s; rm result/%s_fail_ids*" % (channel, fail_ids_file, channel)
print cmd
os.system(cmd)
fail_cnt = util.statistic_line(fail_ids_file)
success_ids_file = "%s/%s_statistics_%s" % (result_dir, channel, now_str)
cmd = "cat result/%s_statistics* | sort | uniq > %s; rm result/%s_statistics*" % (channel, success_ids_file, channel)
print cmd
os.system(cmd)
success_cnt = util.statistic_line(success_ids_file)
dupli_ids_file = "%s/%s_duplicate_ids_%s" % (result_dir, channel, now_str)
cmd = "cat result/%s_duplicate_ids* | sort | uniq > %s; rm result/%s_duplicate_ids*" % (channel, dupli_ids_file, channel)
os.system(cmd)
dupli_cnt = util.statistic_line(dupli_ids_file)
notify_title = "%s etl" % channel
notify_msg = """
prog: %s
result：
success_cnt: %d,
fail_cnt: %d,
dupli_cnt: %d
""" % (' '.join(sys.argv), success_cnt, fail_cnt, dupli_cnt)
send_email(notify_emails, notify_title, notify_msg)
def event_handler(self, evt, msg, **kwargs):
if evt == 'DONE':
util.send_email(["<*****@*****.**>"], "{} 样本对比".format(self.channel), msg + '\n duplicate cvs: %d' % self._duplication_count +
'\n parsed count: %d' % self.parsed_cnt)