def calc():
start_day, end_day = calc_eage_days()
print START_DESC.format(start_day, end_day,
str(program_start_time)[:19], sys.argv[3])
start_ts = int(time.mktime(time.strptime(start_day, "%Y-%m-%d")) * 1000)
end_ts = int(time.mktime(time.strptime(end_day, "%Y-%m-%d")) * 1000 - 1)
ts_calc_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
ts_list = copy.deepcopy(ts_calc_list)
# get some former time and newer time logs
first_ts = ts_list[0][0]
end_ts = ts_list[-1][1]
first_ts = first_ts - (60 * 60 * 12) * 1000
end_ts = end_ts + (60 * 60 * 3) * 1000
ts_list[0][0] = first_ts
ts_list[-1][1] = end_ts
gmgr = GrabberManager(msgh)
ts_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
for i in range(nginx_grabber_num):
name = 'AccessFreqNginxGrabber' + str(i)
config = {
'start': ts_calc_list[i][0],
'end': ts_calc_list[i][1],
'log_start_ts': ts_list[i][0],
'log_end_ts': ts_list[i][1]
}
gmgr.set_worker(name, AccessFreqNginxGrabber, config)
config = {
'start': start_day,
'end': end_day,
'start_analysis': '2014-04-01',
'end_analysis': '2014-12-31'
}
cmgr = AccessFreqCalcMgr(msgh, config)
for i in range(calc_num):
name = 'AccessFreqCalc' + str(i)
cmgr.set_worker(name, AccessFreqCalc, config)
gmgr.start()
cmgr.start()
gmgr.join()
cmgr.join()
program_finish_time = datetime.datetime.now()
print FINISH_DESC.format(
start_day, end_day,
str(program_start_time)[:19],
str(program_finish_time)[:19],
str(program_finish_time - program_start_time)[:19], sys.argv[3])
def calc():
start_day, end_day = calc_eage_days()
print START_DESC.format(start_day, end_day, str(program_start_time)[:19])
start_ts = int(time.mktime(time.strptime(start_day, "%Y-%m-%d")) * 1000)
end_ts = int(time.mktime(time.strptime(end_day, "%Y-%m-%d")) * 1000 - 1)
ts_calc_list = slice_ts(start_ts, end_ts, grabber_num)
ts_list = copy.deepcopy(ts_calc_list)
# get some former time and newer time logs
reserve = 6 # reserve hours former and newer
first_ts = ts_list[0][0]
print first_ts
end_ts = ts_list[-1][1]
first_ts = first_ts - (60 * 60 * reserve) * 1000
print first_ts
print "====", datetime.datetime.fromtimestamp(first_ts / 1000)
end_ts = end_ts + (60 * 60 * reserve) * 1000
ts_list[0][0] = first_ts
ts_list[-1][1] = end_ts
gmgr = GrabberManager(msgh)
for i in range(grabber_num):
name = "PlayUserGrabber" + str(i)
config = {
"start": ts_calc_list[i][0],
"end": ts_calc_list[i][1],
"log_start_ts": ts_list[i][0],
"log_end_ts": ts_list[i][1],
}
gmgr.set_worker(name, PlayUserGrabber, config)
config = {"start": start_day, "end": end_day}
cmgr = PlayUserCalcMgr(msgh, config)
for i in range(calc_num):
name = "PlayUserCalc" + str(i)
cmgr.set_worker(name, PlayUserCalc, config)
gmgr.start()
cmgr.start()
gmgr.join()
cmgr.join()
program_finish_time = datetime.datetime.now()
print FINISH_DESC.format(
start_day,
end_day,
str(program_start_time)[:19],
str(program_finish_time)[:19],
str(program_finish_time - program_start_time)[:19],
)
def calc():
start_day, end_day = calc_eage_days()
print START_DESC.format(start_day, end_day, str(program_start_time)[:19])
start_ts = int(time.mktime(
time.strptime(start_day, "%Y-%m-%d")) * 1000)
end_ts = int(time.mktime(
time.strptime(end_day, "%Y-%m-%d")) * 1000 -1)
ts_calc_list = slice_ts(start_ts, end_ts, grabber_num)
ts_list = copy.deepcopy(ts_calc_list)
# get some former time and newer time logs
reserve = 6 # reserve hours former and newer
first_ts = ts_list[0][0]
end_ts = ts_list[-1][1]
first_ts = first_ts - (60 * 60 * reserve) * 1000
end_ts = end_ts + (60 * 60 * reserve) * 1000
ts_list[0][0] = first_ts
ts_list[-1][1] = end_ts
gmgr = GrabberManager(msgh)
for i in range(grabber_num):
name = 'ActiveUserGrabber' + str(i)
config = {'start':ts_calc_list[i][0], 'end':ts_calc_list[i][1],
'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
gmgr.set_worker(name, ActiveUserGrabber, config)
ts_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
for i in range(nginx_grabber_num):
name = 'ActiveUserNginxGrabber' + str(i)
config = {'start':ts_list[i][0], 'end':ts_list[i][1],
'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
gmgr.set_worker(name, ActiveUserNginxGrabber, config)
config = {'start':start_day, 'end':end_day}
cmgr = ActiveUserCalcMgr(msgh, config)
for i in range(calc_num):
name = 'ActiveUserCalc' + str(i)
cmgr.set_worker(name, ActiveUserCalc, config)
gmgr.start()
cmgr.start()
gmgr.join()
cmgr.join()
program_finish_time = datetime.datetime.now()
print FINISH_DESC.format(start_day, end_day, str(program_start_time)[:19],
str(program_finish_time)[:19],
str(program_finish_time - program_start_time)[:19])
def calc():
start_day, end_day = calc_eage_days()
print START_DESC.format(start_day, end_day,
str(program_start_time)[:19], sys.argv[3])
start_ts = int(time.mktime(
time.strptime(start_day, "%Y-%m-%d")) * 1000)
end_ts = int(time.mktime(
time.strptime(end_day, "%Y-%m-%d")) * 1000 -1)
ts_calc_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
ts_list = copy.deepcopy(ts_calc_list)
# get some former time and newer time logs
first_ts = ts_list[0][0]
end_ts = ts_list[-1][1]
first_ts = first_ts - (60 * 60 * 12) * 1000
end_ts = end_ts + (60 * 60 * 3) * 1000
ts_list[0][0] = first_ts
ts_list[-1][1] = end_ts
gmgr = GrabberManager(msgh)
ts_list = slice_ts(start_ts, end_ts, nginx_grabber_num)
for i in range(nginx_grabber_num):
name = 'AccessFreqNginxGrabber' + str(i)
config = {'start':ts_calc_list[i][0], 'end':ts_calc_list[i][1],
'log_start_ts': ts_list[i][0], 'log_end_ts': ts_list[i][1]}
gmgr.set_worker(name, AccessFreqNginxGrabber, config)
config = {'start':start_day, 'end':end_day,
'start_analysis': '2014-04-01', 'end_analysis': '2014-12-31'}
cmgr = AccessFreqCalcMgr(msgh, config)
for i in range(calc_num):
name = 'AccessFreqCalc' + str(i)
cmgr.set_worker(name, AccessFreqCalc, config)
gmgr.start()
cmgr.start()
gmgr.join()
cmgr.join()
program_finish_time = datetime.datetime.now()
print FINISH_DESC.format(start_day, end_day, str(program_start_time)[:19],
str(program_finish_time)[:19],
str(program_finish_time - program_start_time)[:19],
sys.argv[3])