Ejemplo n.º 1
0
    def event_handler(self, evt, msg, **kwargs):
        if "START" == evt:
            util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
            return

        if "DONE" == evt:
            util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
            return
Ejemplo n.º 2
0
    def event_handler(self, evt, msg, **kwargs):
        if "START" == evt:
            util.send_email(["<*****@*****.**>"], "2cimport 任务", msg)
            return

        if "DONE" == evt:
            util.send_email(["<*****@*****.**>"], "2cimport 任务", msg)
            return
Ejemplo n.º 3
0
    def event_handler(self, evt, msg, **kwargs):
        if "START" == evt:
            util.send_email(["<*****@*****.**>"], "{}_removePastPage 任务".format(self.channel), msg)
            return

        if "DONE" == evt:
            util.send_email(["<*****@*****.**>"], "{}_removePastPage 任务".format(self.channel), msg)
            return
Ejemplo n.º 4
0
    def event_handler(self, evt, msg, **kwargs):
        if evt == 'DONE':
            util.send_email(["<*****@*****.**>"], "{} 样本对比".format(self.channel), msg)

            # 最终结果存放文件
            filedest = 'app/share/%s_result.txt' % self.channel

            if os.path.exists(filedest):
                now = util.get_date_with_day_duration()
                history_fn = os.path.join(os.path.dirname(filedest), '%s_%s.txt' % (self.channel, '%d%02d%02d%02d%02d' % (now.year, now.month, now.day, now.hour, now.minute)))

                # 将结果移动到历史文件
                os.system('mv %s %s' % (filedest, history_fn))

            # 将最后计算结果放到share 目录下, 提供下载
            os.system('mv %s %s' % (self.result_file, filedest))
Ejemplo n.º 5
0
def clean(channel):
    result_dir = "result/%s_result" % channel
    if not os.path.exists(result_dir):
        os.makedirs(result_dir)
    now = time.localtime()
    now_str = "%04d%02d%02d" % (now.tm_year, now.tm_mon, now.tm_mday)
    fail_ids_file = "%s/%s_fail_ids_%s" % (result_dir, channel,now_str)
    cmd = "cat result/%s_fail_ids* | sort | uniq > %s; rm result/%s_fail_ids*" % (channel,  fail_ids_file, channel)
    print cmd
    os.system(cmd)
    fail_cnt = util.statistic_line(fail_ids_file)

    success_ids_file = "%s/%s_statistics_%s" % (result_dir, channel, now_str)
    cmd = "cat result/%s_statistics* | sort | uniq > %s; rm result/%s_statistics*" % (channel, success_ids_file, channel)
    print cmd
    os.system(cmd)
    success_cnt = util.statistic_line(success_ids_file)

    dupli_ids_file = "%s/%s_duplicate_ids_%s" % (result_dir, channel, now_str)
    cmd = "cat result/%s_duplicate_ids* | sort | uniq > %s; rm result/%s_duplicate_ids*" % (channel, dupli_ids_file, channel)
    os.system(cmd)

    dupli_cnt = util.statistic_line(dupli_ids_file)

    notify_title = "%s etl" % channel
    notify_msg = """

    prog: %s

    result:
            success_cnt: %d,
            fail_cnt: %d,
            dupli_cnt: %d

    """ % (' '.join(sys.argv), success_cnt, fail_cnt, dupli_cnt)

    send_email(notify_emails, notify_title, notify_msg)
Ejemplo n.º 6
0
 def event_handler(self, evt, msg, **kwargs):
     if evt == 'DONE':
         util.send_email(["<*****@*****.**>"], "{} 样本对比".format(self.channel), msg + '\n duplicate cvs: %d' % self._duplication_count +
                         '\n parsed count: %d' % self.parsed_cnt)