def get_date(self, delta=0): date = self.get_argument('date','') if date: tody = datetime.datetime.strptime(date,'%Y-%m-%d') else: tody = time_start(datetime.datetime.now()-datetime.timedelta(days=delta),'day') return tody
def get_sum_stat(self): tody = time_start(datetime.datetime.now(),'day') #start6 = datetime.datetime.strptime('2012-06-01','%Y-%m-%d') #end6 = datetime.datetime.strptime('2012-07-01','%Y-%m-%d') q1 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-06-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-07-01','%Y-%m-%d'),charge_type='') q2 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-07-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-08-01','%Y-%m-%d'),charge_type='') q3 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-08-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-09-01','%Y-%m-%d'),charge_type='') q4 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-09-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-10-01','%Y-%m-%d'),charge_type='') q5 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-10-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-11-01','%Y-%m-%d'),charge_type='') q6 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-11-01','%Y-%m-%d'),end=datetime.datetime.strptime('2012-12-01','%Y-%m-%d'),charge_type='') q7 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2012-12-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-01-01','%Y-%m-%d'),charge_type='') q8 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-01-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-02-01','%Y-%m-%d'),charge_type='') q9 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-02-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-03-01','%Y-%m-%d'),charge_type='') q10 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-03-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-04-01','%Y-%m-%d'),charge_type='') q11 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-04-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-05-01','%Y-%m-%d'),charge_type='') q12 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-05-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-06-01','%Y-%m-%d'),charge_type='') q13 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-06-01','%Y-%m-%d'),end=datetime.datetime.strptime('2013-07-01','%Y-%m-%d'),charge_type='') q14 = BookStat.mgr().get_data_by_multi_days(scope_id=1,mode='day',start=datetime.datetime.strptime('2013-07-01','%Y-%m-%d'),end=tody,charge_type='') data = [q1,q2,q5,q6,q7,q8,q9,q10,q11,q12,q13,q14] #data = [q1,q2] for q in data: print len(q) book_id_list = self.get_all_book_id_in_ebk5() #book_id_list = book_id_list[0:10] res = self.merge_multi_month_data_by_bookid(book_id_list,data) return res
def get_date(self, delta=0): date = self.get_argument('date', '') if date: tody = datetime.datetime.strptime(date, '%Y-%m-%d') else: tody = time_start( datetime.datetime.now() - datetime.timedelta(days=delta), 'day') return tody
def start_importing(self, mode, now=None, stats=None, partner_ids=None): n0 = datetime.datetime.now() # normalize time if not now: now = datetime.datetime.now() - datetime.timedelta(days=1) if mode == 'day': start = time_start(now,'day') elif mode == 'week' and now.weekday() == 0: start = time_start(now,'week') elif mode == 'month' and now.day == 1: start = time_start(now,'month') else: print '%s %s not supported' % (mode,now) return # import data if not stats or '2db' in stats: self.import_data(mode,start) print mode,start,'time:',datetime.datetime.now()-n0 return True
def start_importing(self, mode, now=None, stats=None, partner_ids=None): n0 = datetime.datetime.now() # normalize time if not now: now = datetime.datetime.now() - datetime.timedelta(days=1) if mode == 'day': start = time_start(now, 'day') elif mode == 'week' and now.weekday() == 0: start = time_start(now, 'week') elif mode == 'month' and now.day == 1: start = time_start(now, 'month') else: print '%s %s not supported' % (mode, now) return # import data if not stats or '2db' in stats: self.import_data(mode, start) print mode, start, 'time:', datetime.datetime.now() - n0 return True
def save_data_in_mysql(self,res): tody = time_start(datetime.datetime.now(),'day') for book in res: try: s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0] s.consume_money = book['fee'] s.book_pv = book['pv'] s.book_uv = book['uv'] s.update(unikey='book_id') print "importing %s" % s.book_id except Exception,e: logging.error('%s\n',str(e),exc_info=True)
def normlize_time(self, mode, start): assert mode in ('hour','day','week','month') start = time_start(start,mode) if mode == 'hour': end = start + datetime.timedelta(hours=1) elif mode == 'day': end = start + datetime.timedelta(days=1) elif mode == 'week': end = start + datetime.timedelta(days=7) elif mode == 'month': if start.month == 12: end = datetime.datetime(start.year+1,1,1) else: end = datetime.datetime(start.year,start.month+1,1) return start,end
def normlize_time(self, mode, start): assert mode in ('hour', 'day', 'week', 'month') start = time_start(start, mode) if mode == 'hour': end = start + datetime.timedelta(hours=1) elif mode == 'day': end = start + datetime.timedelta(days=1) elif mode == 'week': end = start + datetime.timedelta(days=7) elif mode == 'month': if start.month == 12: end = datetime.datetime(start.year + 1, 1, 1) else: end = datetime.datetime(start.year, start.month + 1, 1) return start, end
def save_all_data_in_tmp(self,res): import_num = 0 tody = time_start(datetime.datetime.now(),'day') for book in res: try: s = Temp.new() s.book_id = book['book_id'] s.pv = book['pv'] s.uv = book['uv'] s.fee = book['fee'] s.pay_down = book['pay_down'] s.free_down = book['free_down'] s.save() import_num += 0 except Exception,e: logging.error('%s\n',str(e),exc_info=True)
def inc_update_data_in_mysql(self,res): import_num = 0 tody = time_start(datetime.datetime.now(),'day') for book in res: try: s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0] s.consume_money = float(s.consume_money) + float(book['fee']) s.book_pv += book['pv'] s.book_uv += book['uv'] s.fee_download_num += book['pay_down'] s.free_download_num += book['free_down'] s.update(unikey='book_id') import_num += 1 print "updating %s" % s.book_id except Exception,e: logging.error('%s\n',str(e),exc_info=True)
def save_all_data_in_ebk5(self): import_num = 0 tody = time_start(datetime.datetime.now(),'day') books = Temp.mgr().get_all_data() print books for book in books: try: s = Ebk5BookExpand.mgr(ismaster=1).Q(time=tody).filter(book_id=book['book_id'])[0] s.consume_money = float(book['fee']) s.book_pv = book['pv'] s.book_uv = book['uv'] s.fee_download_num = book['pay_down'] s.free_download_num = book['free_down'] s.update(unikey='book_id') import_num += 1 print "updating %s" % s.book_id except Exception,e: logging.error('%s\n',str(e),exc_info=True)
start = datetime.datetime.strptime(a,'%Y-%m-%d:%H') if o == '--end': end = datetime.datetime.strptime(a,'%Y-%m-%d:%H') if o == '--mode': mode = a assert jobtype in ('download_v6','service_v6') assert mode in ('hour','day') lock_file_path = '/tmp/split_%s_%s_%s_%s.lock'%(jobtype,start,end,mode) fd = os.open(lock_file_path, os.O_CREAT|os.O_RDWR, 0660) try: # source /etc/profile to include hadoop home SplitLog.lock(fd) spliter = SplitLog() if mode == 'day': if not start: now = time_start(datetime.datetime.now(),'day') start = now - datetime.timedelta(days=1) end = start + datetime.timedelta(days=1) if start and end: while start < end: spliter.process(jobtype,start) start += datetime.timedelta(hours=1) else: spliter.process(jobtype) except Exception,e: logging.error('%s\n',str(e),exc_info=True) finally: os.close(fd) os.remove(lock_file_path)
def start_importing(self, mode, now=None, stats=None, partner_ids=None): ''' entry to analyze and import stat into mysql ''' n0 = datetime.datetime.now() # clear cache self.down.reset_cache() self.service.reset_cache() # extra sql for partner_ids extra = None if partner_ids: extra = 'partner_id in (%s)' % partner_ids self.down.set_extra(extra) self.service.set_extra(extra) # normalize time if not now: now = datetime.datetime.now() - datetime.timedelta(days=1) if mode == 'day': start = time_start(now,'day') elif mode == 'week' and now.weekday() == 0: start = time_start(now,'week') elif mode == 'month' and now.day == 1: start = time_start(now,'month') else: print '%s %s not supported' % (mode,now) return # import the partner and product to scope if not (stats or partner_ids): self.import_partner_productname(mode,start) # loop all the valid scopes q = Scope.mgr().Q().extra("status in ('pas','nice')") extra and q.extra(extra) for scope in q: mask = scope.mask.split(',') modes = scope.modes.split(',') if mode not in modes: print 'ignoring ',mode,scope.id,scope.modes continue print 'processing',scope.id,mask,modes # patch basic stat if 'basic' in mask and stats and 'patch_basic' in stats: self.patch_basic(mode,start,**scope) if 'book' in mask and stats and 'patch_book' in stats: self.patch_book(mode,start,**scope) # basic stat if 'basic' in mask and (not stats or 'basic' in stats): self.import_basic_stat(mode,start,**scope) # visit stat of page types if 'visit' in mask and (not stats or 'visit' in stats): self.import_visit_stat(mode,start,**scope) # topN of the page types if 'topn' in mask and (not stats or 'topn' in stats): self.import_topN(mode,start,**scope) # book stat if 'book' in mask and (not stats or 'book' in stats): self.import_book_stat(mode,start,num=0,**scope) self.import_bookrefer_stat(mode,start,['search_main','search_button','hot_search_word'],**scope) # machine stat if 'product' in mask and (not stats or 'product' in stats): self.import_product_stat(mode,start,num=9,**scope) # update new_user_run stat if not stats or 'update' in stats: self.update_new_user_run_stat(start,mode) self.update_zero_new_user_run_stat(start,mode,SHOULD_UPDATE_ZERO_NEW_USER_RUN_PARTNER_LIST) # logstatV2 -> logstatv3 if not stats or 'basicv3' in stats: self.import_factory_database(mode,start) # merge basic to factory sum if not stats or 'factory' in stats: self.import_factory_stat(mode,start) # merge tag according to book stat if not stats or 'tag' in stats: self.import_toptag_bybook(mode,start) print mode,start,'time:',datetime.datetime.now()-n0 return True
def get_date(self, delta=0): tody = time_start(datetime.datetime.now() - datetime.timedelta(days=delta), 'day') return tody
def start_importing(self, mode, now=None, stats=None, partner_ids=None): ''' entry to analyze and import stat into mysql ''' n0 = datetime.datetime.now() # clear cache self.down.reset_cache() self.service.reset_cache() # extra sql for partner_ids extra = None if partner_ids: extra = 'partner_id in (%s)' % partner_ids self.down.set_extra(extra) self.service.set_extra(extra) # normalize time if not now: now = datetime.datetime.now() - datetime.timedelta(days=1) if mode == 'day': start = time_start(now, 'day') elif mode == 'week' and now.weekday() == 0: start = time_start(now, 'week') elif mode == 'month' and now.day == 1: start = time_start(now, 'month') else: print '%s %s not supported' % (mode, now) return # import the partner and product to scope if not (stats or partner_ids): self.import_partner_productname(mode, start) # loop all the valid scopes q = Scope.mgr().Q().extra("status in ('pas','nice')") extra and q.extra(extra) for scope in q: mask = scope.mask.split(',') modes = scope.modes.split(',') if mode not in modes: print 'ignoring ', mode, scope.id, scope.modes continue print 'processing', scope.id, mask, modes # patch basic stat if 'basic' in mask and stats and 'patch_basic' in stats: self.patch_basic(mode, start, **scope) if 'book' in mask and stats and 'patch_book' in stats: self.patch_book(mode, start, **scope) # basic stat if 'basic' in mask and (not stats or 'basic' in stats): self.import_basic_stat(mode, start, **scope) # visit stat of page types if 'visit' in mask and (not stats or 'visit' in stats): self.import_visit_stat(mode, start, **scope) # topN of the page types if 'topn' in mask and (not stats or 'topn' in stats): self.import_topN(mode, start, **scope) # book stat if 'book' in mask and (not stats or 'book' in stats): self.import_book_stat(mode, start, num=0, **scope) self.import_bookrefer_stat( mode, start, ['search_main', 'search_button', 'hot_search_word'], **scope) # machine stat if 'product' in mask and (not stats or 'product' in stats): self.import_product_stat(mode, start, num=9, **scope) # update new_user_run stat if not stats or 'update' in stats: self.update_new_user_run_stat(start, mode) self.update_zero_new_user_run_stat( start, mode, SHOULD_UPDATE_ZERO_NEW_USER_RUN_PARTNER_LIST) # logstatV2 -> logstatv3 if not stats or 'basicv3' in stats: self.import_factory_database(mode, start) # merge basic to factory sum if not stats or 'factory' in stats: self.import_factory_stat(mode, start) # merge tag according to book stat if not stats or 'tag' in stats: self.import_toptag_bybook(mode, start) print mode, start, 'time:', datetime.datetime.now() - n0 return True
def get_yest_book_data(self,delta=1): tody = time_start(datetime.datetime.now(),'day') yest = tody - datetime.timedelta(days=delta) q = BookStat.mgr().Q(time=yest).filter(scope_id=1,mode='day',time=yest) return q
def get_date(self, delta=0): tody = time_start(datetime.datetime.now()-datetime.timedelta(days=delta),'day') return tody