def do_many_days_s3_APPS(argv):
if len(argv) < 3:
print "usage: start_day end_day"
print "usage: 2011-10-7 2011-10-13"
sys.exit()
QUERY_FILES = [
'euospcomp07.osp_query.log',
'euospcomp08.osp_query.log',
'euospsch01.osp_query.log',
'euospsch03.osp_query.log',
'euospsch01.2.osp_query.log',
'euospsch03.2.osp_query.log',
]
CLICK_FILES = [
'euospsch03.osp_click.log',
'euospcomp08.osp_click.log'
]
#QUERY_FILES = QUERY_FILES[:2]
daylist = handolUtil.get_day_list(argv[1],argv[2])
analyzer = LogAnalyzer("APPS", "%s-%s" %(daylist[0], daylist[-1]))
stopwatch = handolUtil.StopWatch()
for daystr in daylist:
for q in QUERY_FILES:
s3_file = "DEVELOPING/app/7nmc1m75ij/apps-log/query_log/%s/%s.%s" % (daystr, q, daystr)
analyzer.querylog_from_s3("sch-emr", s3_file)
print "Loading & ETL: %f sec" % (stopwatch.laptime())
analyzer.get_stats()
print "Calc Stats: %f sec" % (stopwatch.laptime())
analyzer.write_info()
def do_many_days_s3_LHRHVH(argv):
if len(argv) < 3:
print "usage: start_day end_day"
print "usage: 2011-10-7 2011-10-13"
sys.exit()
daylist = handolUtil.get_day_list(argv[1],argv[2])
analyzer = LogAnalyzer("LHRHVH", "%s-%s" %(daylist[0], daylist[-1]))
stopwatch = handolUtil.StopWatch()
for daystr in daylist:
s3_file = "DEVELOPING/app/7nmc1m75ij/hubs1-log/query_log/osp_query.log.%s" % (daystr)
analyzer.querylog_from_s3("sch-emr", s3_file)
print "Loading & ETL: %f sec" % (stopwatch.laptime())
s3_file = "DEVELOPING/app/7nmc1m75ij/hubs2-log/query_log/osp_query.log.%s" % (daystr)
analyzer.querylog_from_s3("sch-emr", s3_file)
print "Loading & ETL: %f sec" % (stopwatch.laptime())
analyzer.get_stats()
print "Calc Stats: %f sec" % (stopwatch.laptime())
analyzer.write_info()
analyzer.click_log_mem(mem)
sys.stdout.flush()
analyzer.get_rate()
analyzer.print_info(outf)
#analyzer.save_to_mongo()
#analyzer.save_keywords()
######
if __name__ == "__main__":
# print make_q_names('20130110')
# print make_c_names('20130110')
if len(sys.argv) < 3:
print "usage: start_day end_day"
print "usage: 2011-10-7 2011-10-13"
sys.exit()
daylist = handolUtil.get_day_list(sys.argv[1], sys.argv[2])
for daystr in daylist:
print datetime.datetime.now()
outf = "qc_log.%s.csv" % (daystr)
print "### PROCESSING - %s" % (daystr)
try:
analyze_oneday_log_s3(daystr, outf)
except:
print "FAILED - %s" % (daystr)
if daystr == daylist[-1]: raise
for data in db.keywords.find():
kor_sum = get_sum_days(days, data, "450")
uk_sum = get_sum_days(days, data, "234")
if kor_sum[0] > 0:
kor_list.append([data["_id"]] + kor_sum)
if uk_sum[0] > 0:
uk_list.append([data["_id"]] + uk_sum)
write_report(kor_list, "KOR.keywords.csv")
write_report(uk_list, "UK.keywords.csv")
def write_report(rep_list, outf):
res = sorted(rep_list, key=lambda x: x[1], reverse=True)
outf = codecs.open(outf, "wb", encoding='utf-8')
n = 0
for v in res:
line = "%d , %d , %d , " % (v[1], v[2], v[3])
outf.write(line)
outf.write(v[0])
outf.write("\n")
n += 1
#if n > 150: break
outf.close()
if __name__ == "__main__":
daylist = handolUtil.get_day_list('2013-01-01', '2013-02-01')
doit_per_MCC(daylist)
analyzer.print_info(outf)
#analyzer.save_to_mongo()
#analyzer.save_keywords()
######
if __name__=="__main__":
# print make_q_names('20130110')
# print make_c_names('20130110')
if len(sys.argv) < 3:
print "usage: start_day end_day"
print "usage: 2011-10-7 2011-10-13"
sys.exit()
daylist = handolUtil.get_day_list(sys.argv[1],sys.argv[2])
for daystr in daylist:
print datetime.datetime.now()
outf = "qc_log.%s.csv" % (daystr)
print "### PROCESSING - %s" % (daystr)
try:
analyze_oneday_log_s3(daystr, outf )
except:
print "FAILED - %s" % (daystr)
if daystr==daylist[-1]: raise
kor_sum = get_sum_days(days, data, "450")
uk_sum = get_sum_days(days, data, "234")
if kor_sum[0] > 0:
kor_list.append([data["_id"]] + kor_sum)
if uk_sum[0] > 0:
uk_list.append([data["_id"]] + uk_sum)
write_report(kor_list, "KOR.keywords.csv")
write_report(uk_list, "UK.keywords.csv")
def write_report(rep_list, outf):
res = sorted(rep_list, key=lambda x:x[1], reverse=True)
outf = codecs.open(outf, "wb", encoding='utf-8')
n = 0
for v in res:
line = "%d , %d , %d , " % (v[1], v[2], v[3])
outf.write(line)
outf.write(v[0])
outf.write("\n")
n += 1
#if n > 150: break
outf.close()
if __name__=="__main__":
daylist = handolUtil.get_day_list('2013-01-01', '2013-02-01')
doit_per_MCC(daylist)