Ejemplo n.º 1
0
def calc():
    start_day, end_day = calc_eage_days()
    print START_DESC.format(start_day, end_day, str(program_start_time)[:19])
    start_ts = int(time.mktime(
            time.strptime(start_day, "%Y-%m-%d")) * 1000)
    end_ts = int(time.mktime(
            time.strptime(end_day, "%Y-%m-%d")) * 1000 -1)
    ts_calc_list = slice_ts(start_ts, end_ts, grabber_num)
    ts_list = copy.deepcopy(ts_calc_list)

    # get some former time and newer time logs
    reserve = 6 # reserve hours former and newer
    first_ts = ts_list[0][0]
    end_ts = ts_list[-1][1]
    first_ts = first_ts - (60 * 60 * reserve) * 1000
    end_ts = end_ts + (60 * 60 * reserve) * 1000
    ts_list[0][0] = first_ts
    ts_list[-1][1] = end_ts
        
    gmgr = GrabberManager(msgh)
    for i in range(grabber_num):
        name = 'ActiveUserGrabber' + str(i)
        config = {'start':ts_calc_list[i][0], 'end':ts_calc_list[i][1],
                  'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
        gmgr.set_worker(name, ActiveUserGrabber, config)

    ts_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
    for i in range(nginx_grabber_num):
        name = 'ActiveUserNginxGrabber' + str(i)
        config = {'start':ts_list[i][0], 'end':ts_list[i][1],
                  'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
        gmgr.set_worker(name, ActiveUserNginxGrabber, config)
        
    config = {'start':start_day, 'end':end_day}
    cmgr = ActiveUserCalcMgr(msgh, config)
    for i in range(calc_num):
        name = 'ActiveUserCalc' + str(i)
        cmgr.set_worker(name, ActiveUserCalc, config)

    gmgr.start()
    cmgr.start()

    gmgr.join()
    cmgr.join()
    
    program_finish_time = datetime.datetime.now()
    
    print FINISH_DESC.format(start_day, end_day, str(program_start_time)[:19],
                             str(program_finish_time)[:19],
                             str(program_finish_time - program_start_time)[:19])
Ejemplo n.º 2
0
def calc():
    start_day, end_day = calc_eage_days()
    print START_DESC.format(start_day, end_day, str(program_start_time)[:19])
    start_ts = int(time.mktime(time.strptime(start_day, "%Y-%m-%d")) * 1000)
    end_ts = int(time.mktime(time.strptime(end_day, "%Y-%m-%d")) * 1000 - 1)
    ts_calc_list = slice_ts(start_ts, end_ts, grabber_num)
    ts_list = copy.deepcopy(ts_calc_list)

    # get some former time and newer time logs
    reserve = 6  # reserve hours former and newer
    first_ts = ts_list[0][0]
    print first_ts
    end_ts = ts_list[-1][1]
    first_ts = first_ts - (60 * 60 * reserve) * 1000
    print first_ts
    print "====", datetime.datetime.fromtimestamp(first_ts / 1000)
    end_ts = end_ts + (60 * 60 * reserve) * 1000
    ts_list[0][0] = first_ts
    ts_list[-1][1] = end_ts

    gmgr = GrabberManager(msgh)
    for i in range(grabber_num):
        name = "PlayUserGrabber" + str(i)
        config = {
            "start": ts_calc_list[i][0],
            "end": ts_calc_list[i][1],
            "log_start_ts": ts_list[i][0],
            "log_end_ts": ts_list[i][1],
        }
        gmgr.set_worker(name, PlayUserGrabber, config)

    config = {"start": start_day, "end": end_day}
    cmgr = PlayUserCalcMgr(msgh, config)
    for i in range(calc_num):
        name = "PlayUserCalc" + str(i)
        cmgr.set_worker(name, PlayUserCalc, config)

    gmgr.start()
    cmgr.start()

    gmgr.join()
    cmgr.join()

    program_finish_time = datetime.datetime.now()

    print FINISH_DESC.format(
        start_day,
        end_day,
        str(program_start_time)[:19],
        str(program_finish_time)[:19],
        str(program_finish_time - program_start_time)[:19],
    )
Ejemplo n.º 3
0
def calc():
    start_day, end_day = calc_eage_days()
    print START_DESC.format(start_day, end_day,
                            str(program_start_time)[:19], sys.argv[3])
    start_ts = int(time.mktime(
            time.strptime(start_day, "%Y-%m-%d")) * 1000)
    end_ts = int(time.mktime(
            time.strptime(end_day, "%Y-%m-%d")) * 1000 -1)
    ts_calc_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
    ts_list = copy.deepcopy(ts_calc_list)

    # get some former time and newer time logs
    first_ts = ts_list[0][0]
    end_ts = ts_list[-1][1]
    first_ts = first_ts - (60 * 60 * 12) * 1000
    end_ts = end_ts + (60 * 60 * 3) * 1000
    ts_list[0][0] = first_ts
    ts_list[-1][1] = end_ts
        
    gmgr = GrabberManager(msgh)
    ts_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
    for i in range(nginx_grabber_num):
        name = 'AccessFreqNginxGrabber' + str(i)
        config = {'start':ts_calc_list[i][0], 'end':ts_calc_list[i][1],
                  'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
        gmgr.set_worker(name, AccessFreqNginxGrabber, config)

    config = {'start':start_day, 'end':end_day,
              'start_analysis': '2014-04-01', 'end_analysis': '2014-12-31'}
    cmgr = AccessFreqCalcMgr(msgh, config)
    for i in range(calc_num):
        name = 'AccessFreqCalc' + str(i)
        cmgr.set_worker(name, AccessFreqCalc, config)

    gmgr.start()
    cmgr.start()

    gmgr.join()
    cmgr.join()
    
    program_finish_time = datetime.datetime.now()
    
    print FINISH_DESC.format(start_day, end_day, str(program_start_time)[:19],
                             str(program_finish_time)[:19],
                             str(program_finish_time - program_start_time)[:19],
                             sys.argv[3])