def get_date(self, delta=0):
date = self.get_argument('date','')
if date:
tody = datetime.datetime.strptime(date,'%Y-%m-%d')
else:
tody = time_start(datetime.datetime.now()-datetime.timedelta(days=delta),'day')
return tody
def get_sum_stat(self):
tody = time_start(datetime.datetime.now(),'day')
#start6 = datetime.datetime.strptime('2012-06-01','%Y-%m-%d')
#end6 = datetime.datetime.strptime('2012-07-01','%Y-%m-%d')
q1 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-06-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-07-01','%Y-%m-%d'),charge_type='')
q2 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-07-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-08-01','%Y-%m-%d'),charge_type='')
q3 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-08-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-09-01','%Y-%m-%d'),charge_type='')
q4 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-09-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-10-01','%Y-%m-%d'),charge_type='')
q5 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-10-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-11-01','%Y-%m-%d'),charge_type='')
q6 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-11-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-12-01','%Y-%m-%d'),charge_type='')
q7 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-12-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-01-01','%Y-%m-%d'),charge_type='')
q8 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-01-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-02-01','%Y-%m-%d'),charge_type='')
q9 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-02-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-03-01','%Y-%m-%d'),charge_type='')
q10 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-03-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-04-01','%Y-%m-%d'),charge_type='')
q11 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-04-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-05-01','%Y-%m-%d'),charge_type='')
q12 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-05-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-06-01','%Y-%m-%d'),charge_type='')
q13 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-06-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-07-01','%Y-%m-%d'),charge_type='')
q14 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-07-01','%Y-%m-%d'),end=tody,charge_type='')
data = [q1,q2,q5,q6,q7,q8,q9,q10,q11,q12,q13,q14]
#data = [q1,q2]
for q in data:
print len(q)
book_id_list = self.get_all_book_id_in_ebk5()
#book_id_list = book_id_list[0:10]
res = self.merge_multi_month_data_by_bookid(book_id_list,data)
return res
def get_date(self, delta=0):
date = self.get_argument('date', '')
if date:
tody = datetime.datetime.strptime(date, '%Y-%m-%d')
else:
tody = time_start(
datetime.datetime.now() - datetime.timedelta(days=delta),
'day')
return tody
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now, 'week')
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def save_data_in_mysql(self,res):
tody = time_start(datetime.datetime.now(),'day')
for book in res:
try:
s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0]
s.consume_money = book['fee']
s.book_pv = book['pv']
s.book_uv = book['uv']
s.update(unikey='book_id')
print "importing %s" % s.book_id
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def normlize_time(self, mode, start):
assert mode in ('hour', 'day', 'week', 'month')
start = time_start(start, mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year + 1, 1, 1)
else:
end = datetime.datetime(start.year, start.month + 1, 1)
return start, end
def save_all_data_in_tmp(self,res):
import_num = 0
tody = time_start(datetime.datetime.now(),'day')
for book in res:
try:
s = Temp.new()
s.book_id = book['book_id']
s.pv = book['pv']
s.uv = book['uv']
s.fee = book['fee']
s.pay_down = book['pay_down']
s.free_down = book['free_down']
s.save()
import_num += 0
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
def inc_update_data_in_mysql(self,res):
import_num = 0
tody = time_start(datetime.datetime.now(),'day')
for book in res:
try:
s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0]
s.consume_money = float(s.consume_money) + float(book['fee'])
s.book_pv += book['pv']
s.book_uv += book['uv']
s.fee_download_num += book['pay_down']
s.free_download_num += book['free_down']
s.update(unikey='book_id')
import_num += 1
print "updating %s" % s.book_id
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
def save_all_data_in_ebk5(self):
import_num = 0
tody = time_start(datetime.datetime.now(),'day')
books = Temp.mgr().get_all_data()
print books
for book in books:
try:
s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0]
s.consume_money = float(book['fee'])
s.book_pv = book['pv']
s.book_uv = book['uv']
s.fee_download_num = book['pay_down']
s.free_download_num = book['free_down']
s.update(unikey='book_id')
import_num += 1
print "updating %s" % s.book_id
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
start = datetime.datetime.strptime(a,'%Y-%m-%d:%H')
if o == '--end':
end = datetime.datetime.strptime(a,'%Y-%m-%d:%H')
if o == '--mode':
mode = a
assert jobtype in ('download_v6','service_v6')
assert mode in ('hour','day')
lock_file_path = '/tmp/split_%s_%s_%s_%s.lock'%(jobtype,start,end,mode)
fd = os.open(lock_file_path, os.O_CREAT|os.O_RDWR, 0660)
try:
# source /etc/profile to include hadoop home
SplitLog.lock(fd)
spliter = SplitLog()
if mode == 'day':
if not start:
now = time_start(datetime.datetime.now(),'day')
start = now - datetime.timedelta(days=1)
end = start + datetime.timedelta(days=1)
if start and end:
while start < end:
spliter.process(jobtype,start)
start += datetime.timedelta(hours=1)
else:
spliter.process(jobtype)
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
finally:
os.close(fd)
os.remove(lock_file_path)
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
'''
entry to analyze and import stat into mysql
'''
n0 = datetime.datetime.now()
# clear cache
self.down.reset_cache()
self.service.reset_cache()
# extra sql for partner_ids
extra = None
if partner_ids:
extra = 'partner_id in (%s)' % partner_ids
self.down.set_extra(extra)
self.service.set_extra(extra)
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import the partner and product to scope
if not (stats or partner_ids):
self.import_partner_productname(mode,start)
# loop all the valid scopes
q = Scope.mgr().Q().extra("status in ('pas','nice')")
extra and q.extra(extra)
for scope in q:
mask = scope.mask.split(',')
modes = scope.modes.split(',')
if mode not in modes:
print 'ignoring ',mode,scope.id,scope.modes
continue
print 'processing',scope.id,mask,modes
# patch basic stat
if 'basic' in mask and stats and 'patch_basic' in stats:
self.patch_basic(mode,start,**scope)
if 'book' in mask and stats and 'patch_book' in stats:
self.patch_book(mode,start,**scope)
# basic stat
if 'basic' in mask and (not stats or 'basic' in stats):
self.import_basic_stat(mode,start,**scope)
# visit stat of page types
if 'visit' in mask and (not stats or 'visit' in stats):
self.import_visit_stat(mode,start,**scope)
# topN of the page types
if 'topn' in mask and (not stats or 'topn' in stats):
self.import_topN(mode,start,**scope)
# book stat
if 'book' in mask and (not stats or 'book' in stats):
self.import_book_stat(mode,start,num=0,**scope)
self.import_bookrefer_stat(mode,start,['search_main','search_button','hot_search_word'],**scope)
# machine stat
if 'product' in mask and (not stats or 'product' in stats):
self.import_product_stat(mode,start,num=9,**scope)
# update new_user_run stat
if not stats or 'update' in stats:
self.update_new_user_run_stat(start,mode)
self.update_zero_new_user_run_stat(start,mode,SHOULD_UPDATE_ZERO_NEW_USER_RUN_PARTNER_LIST)
# logstatV2 -> logstatv3
if not stats or 'basicv3' in stats:
self.import_factory_database(mode,start)
# merge basic to factory sum
if not stats or 'factory' in stats:
self.import_factory_stat(mode,start)
# merge tag according to book stat
if not stats or 'tag' in stats:
self.import_toptag_bybook(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def get_date(self, delta=0):
tody = time_start(datetime.datetime.now() - datetime.timedelta(days=delta),
'day')
return tody
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
'''
entry to analyze and import stat into mysql
'''
n0 = datetime.datetime.now()
# clear cache
self.down.reset_cache()
self.service.reset_cache()
# extra sql for partner_ids
extra = None
if partner_ids:
extra = 'partner_id in (%s)' % partner_ids
self.down.set_extra(extra)
self.service.set_extra(extra)
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now, 'week')
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import the partner and product to scope
if not (stats or partner_ids):
self.import_partner_productname(mode, start)
# loop all the valid scopes
q = Scope.mgr().Q().extra("status in ('pas','nice')")
extra and q.extra(extra)
for scope in q:
mask = scope.mask.split(',')
modes = scope.modes.split(',')
if mode not in modes:
print 'ignoring ', mode, scope.id, scope.modes
continue
print 'processing', scope.id, mask, modes
# patch basic stat
if 'basic' in mask and stats and 'patch_basic' in stats:
self.patch_basic(mode, start, **scope)
if 'book' in mask and stats and 'patch_book' in stats:
self.patch_book(mode, start, **scope)
# basic stat
if 'basic' in mask and (not stats or 'basic' in stats):
self.import_basic_stat(mode, start, **scope)
# visit stat of page types
if 'visit' in mask and (not stats or 'visit' in stats):
self.import_visit_stat(mode, start, **scope)
# topN of the page types
if 'topn' in mask and (not stats or 'topn' in stats):
self.import_topN(mode, start, **scope)
# book stat
if 'book' in mask and (not stats or 'book' in stats):
self.import_book_stat(mode, start, num=0, **scope)
self.import_bookrefer_stat(
mode, start,
['search_main', 'search_button', 'hot_search_word'],
**scope)
# machine stat
if 'product' in mask and (not stats or 'product' in stats):
self.import_product_stat(mode, start, num=9, **scope)
# update new_user_run stat
if not stats or 'update' in stats:
self.update_new_user_run_stat(start, mode)
self.update_zero_new_user_run_stat(
start, mode, SHOULD_UPDATE_ZERO_NEW_USER_RUN_PARTNER_LIST)
# logstatV2 -> logstatv3
if not stats or 'basicv3' in stats:
self.import_factory_database(mode, start)
# merge basic to factory sum
if not stats or 'factory' in stats:
self.import_factory_stat(mode, start)
# merge tag according to book stat
if not stats or 'tag' in stats:
self.import_toptag_bybook(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def get_yest_book_data(self,delta=1):
tody = time_start(datetime.datetime.now(),'day')
yest = tody - datetime.timedelta(days=delta)
q = BookStat.mgr().Q(time=yest).filter(scope_id=1,mode='day',time=yest)
return q
def get_date(self, delta=0):
tody = time_start(datetime.datetime.now()-datetime.timedelta(days=delta),'day')
return tody
