def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.down = NewDownloadQuery()
self.service = NewServiceQuery()
self.org_down = DownloadQuery(conf['host'], conf['port'])
self.org_service = ServiceQuery(conf['host'], conf['port'])
self.factory = FactoryQuery(conf['host'], conf['port'])
self.wap = WapQuery(conf['host'], conf['port'])
self.is_factory_sum_stat_done = False
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
self.ip_finder = IPProvince()
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'], conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour', 'day', 'week', 'month')
start = time_start(start, mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year + 1, 1, 1)
else:
end = datetime.datetime(start.year, start.month + 1, 1)
return start, end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now, 'week')
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start, end = self.normlize_time(mode, time)
tmp_user_task = self.org_service.get_all_user_task_stat(start)
import_num = 0
for data in tmp_user_task:
try:
data = data.strip()
i = data.split('\t')
stat = TmpUserTask.new()
stat.time = i[0]
stat.username = i[1]
stat.action_type = i[2]
stat.task_id = i[3]
stat.task_name = i[4]
stat.ds = i[5]
stat.amount = i[6]
stat.save()
import_num += 1
except Exception, e:
print i
logging.error('%s\n', str(e), exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
self.ip_finder = IPProvince()
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
output_file = open('ip_province.txt', 'w')
output = ""
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
try:
import_num = 0
ips = self.org_service.get_new_ips(start,end)
#ips = self.org_service.get_all_ips(start,end)
#ips = ['www.baidu.com','www.alipay.com']
print "finding %s ips..." % str(len(ips))
for ip in ips:
ip = str(ip.strip())
province = self.ip_finder.find_ip(ip)
tmp = ip + '\t' + province + '\n'
output += tmp
output = output.encode('utf-8')
output_file.write(output)
output_file.close()
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
user_task = self.org_service.get_user_task_stat(start)
import_num = 0
for data in user_task:
try:
data = data.strip()
i = data.split('\t')
stat = UserTask.new()
stat.time = start.strftime('%Y-%m-%d')
stat.task_type = i[1]
stat.task_id = i[2]
stat.task_name = i[3]
stat.uv = i[4]
stat.save()
import_num += 1
except Exception,e:
print i
logging.error('%s\n',str(e),exc_info=True)
return import_num
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week':
#elif mode == 'week' and now.weekday() == 6:
start = time_start(now,'week')
print start
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or 'retention' in stats:
self.import_new_user_run_week_retention_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_new_user_run_week_retention_data(self, mode, time):
'''
start: start time
'''
if mode != 'week':
return
#time = time - datetime.timedelta(days=7)
lstart,lend = self.normlize_time(mode,time)
llstart = lstart - datetime.timedelta(days=7)
llend = lend - datetime.timedelta(days=7)
#print lstart,lend
#print llstart,llend
yest = datetime.datetime.now() - datetime.timedelta(days=1)
import_num = 0
stat1 = self.org_service.get_last_last_week_new_user_run_stat(llstart,llend,yest)
print stat1
stat2 = self.org_service.get_last_week_retention_stat(yest, llstart, llend, lstart, lend)
print stat2
if stat1 and stat2:
new_user_run = stat1[0]
retention = stat2[0]
#cur=datetime.datetime.now()
try:
stat = RetentionStat.new()
stat.start_time = lstart
stat.week_num = time.strftime("%W")
stat.new_user_run = new_user_run
stat.retention = retention
print stat
stat.save()
import_num += 1
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or 'fujin' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
try:
import_num = 0
fujin1 = self.org_service.get_fujin_pvuv_stat(start,end)[0].split('\t')
print fujin1
fujin2 = self.org_service.get_fujin_fee_stat(start,end)[0].split('\t')
print fujin2
stat = FuJin.new()
stat.time = start.strftime('%Y-%m-%d')
stat.pv = fujin1[0]
stat.uv = fujin1[1]
stat.user_pay = fujin2[0]
stat.fee = float(fujin2[1])
stat.save()
import_num += 1
except Exception,e:
print fujin1,fujin2
logging.error('%s\n',str(e),exc_info=True)
return import_num
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'], conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour', 'day', 'week', 'month')
start = time_start(start, mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year + 1, 1, 1)
else:
end = datetime.datetime(start.year, start.month + 1, 1)
return start, end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now, 'week')
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import data
if not stats or 'wechat' in stats:
self.import_wechat_data(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def import_wechat_data(self, mode, time):
'''
start: start time
return: one day's wechat stat
'''
if mode != 'day':
return
print time
start, end = self.normlize_time(mode, time)
import_num = 0
wechat = self.org_service.get_wechat_stat(start, end)
try:
for line in wechat:
line = line.strip()
data = line.split('\t')
stat = WechatStat.new()
stat.datetime = start.strftime('%Y-%m-%d')
stat.bookid = data[0]
stat.pv = data[1]
stat.uv = data[2]
stat.save()
import_num += 1
except Exception, e:
logging.error('%s\n', str(e), exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
pvuv = self.org_service.get_bookworm_pvuv_stat(start)[0]
right_pvuv = self.org_service.get_bookworm_right_pvuv_stat(start)[0]
wrong_pvuv = self.org_service.get_bookworm_wrong_pvuv_stat(start)[0]
pay_pvuv = self.org_service.get_bookworm_pay_pvuv_stat(start)[0]
recharge_pvuv = self.org_service.get_bookworm_recharge_pvuv_stat(start)[0]
amount_stat = self.org_service.get_bookworm_amount_stat(start)[0]
count1 = self.org_service.get_bookworm_pay_count_1(start)[0]
count2 = self.org_service.get_bookworm_pay_count_2(start)[0]
count3 = self.org_service.get_bookworm_pay_count_3(start)[0]
import_num = 0
try:
pvuv = pvuv.strip()
right_pvuv = right_pvuv.strip()
wrong_pvuv = wrong_pvuv.strip()
pay_pvuv = pay_pvuv.strip()
recharge_pvuv = recharge_pvuv.strip()
amount_stat = amount_stat.strip()
pvuv_tmp = pvuv.split('\t')
wrong_pvuv_tmp = right_pvuv.split('\t')
right_pvuv_tmp = wrong_pvuv.split('\t')
pay_pvuv_tmp = pay_pvuv.split('\t')
recharge_pvuv_tmp = recharge_pvuv.split('\t')
amount_pvuv_tmp = amount_stat.split('\t')
stat = BookWorm.new()
stat.time = start.strftime('%Y-%m-%d')
stat.pv = int(pvuv_tmp[0])
stat.uv = int(pvuv_tmp[1])
stat.right_user = right_pvuv_tmp[0]
stat.right_num = right_pvuv_tmp[1]
stat.wrong_user = wrong_pvuv_tmp[0]
stat.wrong_num = wrong_pvuv_tmp[1]
stat.pay_user = pay_pvuv_tmp[1]
stat.pay_times = pay_pvuv_tmp[0]
stat.recharge_page_pv = recharge_pvuv_tmp[0]
stat.recharge_page_uv = recharge_pvuv_tmp[1]
stat.recharge_user = amount_pvuv_tmp[0]
stat.amount = amount_pvuv_tmp[1]
stat.pay_num1 = count1
stat.pay_num2 = count2
stat.pay_num3 = count3
stat.save()
import_num += 1
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
return import_num
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import reserve_book
if not stats or 'arpu' in stats:
self.import_one_week_fee_and_new_user_run(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_one_week_fee_and_new_user_run(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
import_num = 0
one_week_fee = self.org_service.get_arpu_one_week_fee(start,end)
new_user_visit = self.org_service.get_arpu_one_day_new_user_run(start,end)
thirty_days_fee = self.org_service.get_arpu_30_days_fee(start,end)
ninety_days_fee = self.org_service.get_arpu_90_days_fee(start,end)
try:
stat = Arpu7DaysArpuStat().new()
stat.time = start - datetime.timedelta(days=6)
stat.one_week_fee = one_week_fee
stat.new_user_visit = new_user_visit
stat.save()
stat1 = Arpu30DaysArpuFeeStat.new()
stat1.time = start - datetime.timedelta(days=29)
stat1.thirty_days_fee = thirty_days_fee
stat1.save()
stat2 = Arpu90DaysArpuFeeStat.new()
stat2.time = start - datetime.timedelta(days=89)
stat2.ninety_days_fee = ninety_days_fee
stat2.save()
import_num += 1
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
return import_num
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'], conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour', 'day', 'week', 'month')
start = time_start(start, mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year + 1, 1, 1)
else:
end = datetime.datetime(start.year, start.month + 1, 1)
return start, end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now, 'week')
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import arpu
if not stats or 'arpu' in stats:
self.import_one_week_fee_and_new_user_run(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def import_one_week_fee_and_new_user_run(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start, end = self.normlize_time(mode, time)
import_num = 0
one_week_fee = self.org_service.get_arpu_one_week_fee(start, end)
new_user_visit = self.org_service.get_arpu_one_day_new_user_run(
start, end)
thirty_days_fee = self.org_service.get_arpu_30_days_fee(start, end)
ninety_days_fee = self.org_service.get_arpu_90_days_fee(start, end)
try:
stat = Arpu7DaysArpuStat().new()
stat.time = start - datetime.timedelta(days=6)
stat.one_week_fee = one_week_fee
stat.new_user_visit = new_user_visit
stat.save()
stat1 = Arpu30DaysArpuFeeStat.new()
stat1.time = start - datetime.timedelta(days=29)
stat1.thirty_days_fee = thirty_days_fee
stat1.save()
stat2 = Arpu90DaysArpuFeeStat.new()
stat2.time = start - datetime.timedelta(days=89)
stat2.ninety_days_fee = ninety_days_fee
stat2.save()
import_num += 1
except Exception, e:
logging.error('%s\n', str(e), exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or 'userbandv6' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
import_num = 0
userbandv6 = self.org_service.get_userbandv6_stat(start,end)
for line in userbandv6:
try:
line = line.strip()
data = line.split('\t')
stat = UserBandV6.new()
stat.datetime = data[0].strip("[").strip("]")
stat.username = data[1]
stat.type = data[2]
stat.uid = data[3]
stat.plan_id = data[4]
stat.inner_version = data[5]
if stat.inner_version in INNER_VERSION.keys():
stat.inner_version = INNER_VERSION[stat.inner_version]
stat.partner = data[6]
stat.productname = data[7]
stat.amount = data[8]
stat.ds = data[9]
stat.save()
import_num += 1
except Exception,e:
print data
logging.error('%s\n',str(e),exc_info=True)
return import_num
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'], conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour', 'day', 'week', 'month')
start = time_start(start, mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year + 1, 1, 1)
else:
end = datetime.datetime(start.year, start.month + 1, 1)
return start, end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now, 'day')
elif mode == 'week':
#elif mode == 'week' and now.weekday() == 6:
start = time_start(now, 'week')
print start
elif mode == 'month' and now.day == 1:
start = time_start(now, 'month')
else:
print '%s %s not supported' % (mode, now)
return
# import data
if not stats or 'retention' in stats:
self.import_new_user_run_week_retention_data(mode, start)
print mode, start, 'time:', datetime.datetime.now() - n0
return True
def import_new_user_run_week_retention_data(self, mode, time):
'''
start: start time
'''
if mode != 'week':
return
#time = time - datetime.timedelta(days=7)
lstart, lend = self.normlize_time(mode, time)
llstart = lstart - datetime.timedelta(days=7)
llend = lend - datetime.timedelta(days=7)
#print lstart,lend
#print llstart,llend
yest = datetime.datetime.now() - datetime.timedelta(days=1)
import_num = 0
stat1 = self.org_service.get_last_last_week_new_user_run_stat(
llstart, llend, yest)
print stat1
stat2 = self.org_service.get_last_week_retention_stat(
yest, llstart, llend, lstart, lend)
print stat2
if stat1 and stat2:
new_user_run = stat1[0]
retention = stat2[0]
#cur=datetime.datetime.now()
try:
stat = RetentionStat.new()
stat.start_time = lstart
stat.week_num = time.strftime("%W")
stat.new_user_run = new_user_run
stat.retention = retention
print stat
stat.save()
import_num += 1
except Exception, e:
logging.error('%s\n', str(e), exc_info=True)
return import_num
class Hive2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def handle_big_data(self, data):
data = str(data)
if 'E' in data:
tmp = data.strip().split('E')
num = float(tmp[0])
e = int(tmp[1])
return 10**e * num
else:
return
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or '2db' in stats:
self.import_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_data(self, mode, time):
'''
mode: hour,day,week,month
start: start time
return: one day's user run's a week fee
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
import_num = 0
#balance = self.org_service.get_balance_stat(start)
#for data in balance:
# try:
# data = data.strip()
# i = data.split('\t')
# stat = PaymentBalanceStat.new()
# stat.time = start.strftime('%Y-%m-%d')
# stat.recharge_balance = self.handle_big_data(i[0])
# stat.gift_balance = self.handle_big_data(i[1])
# stat.save()
# import_num += 1
# except Exception,e:
# print i
# logging.error('%s\n',str(e),exc_info=True)
recharge_amount_order = self.org_service.get_recharge_amount_order(start)
print recharge_amount_order
for data in recharge_amount_order:
try:
data = data.strip()
i = data.split('\t')
stat = PaymentRechargingAmountOrderStat.new()
stat.time = start.strftime('%Y-%m-%d')
stat.uv = i[0]
stat.pv = i[1]
stat.origin = i[2]
stat.partner_id = i[3]
stat.innerver = i[4]
stat.amount = i[5]
stat.recharging_type = i[6]
stat.save()
import_num += 1
except Exception,e:
print i
logging.error('%s\n',str(e),exc_info=True)
recharge_amount_finish = self.org_service.get_recharge_amount_finish(start)
for data in recharge_amount_finish:
try:
data = data.strip()
i = data.split('\t')
stat = PaymentRechargingAmountFinishStat.new()
stat.time = start.strftime('%Y-%m-%d')
stat.uv = i[0]
stat.pv = i[1]
stat.origin = i[2]
stat.partner_id = i[3]
stat.innerver = i[4]
stat.amount = i[5]
stat.recharging_type = i[6]
stat.save()
import_num += 1
except Exception,e:
print i
logging.error('%s\n',str(e),exc_info=True)
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'], conf['port'])
class EValue2db(object):
'''
query from hive and import the result to mysql
'''
def __init__(self, conf=None):
if not conf:
conf = HiveConf
self.org_service = ServiceQuery(conf['host'],conf['port'])
def normlize_time(self, mode, start):
assert mode in ('hour','day','week','month')
start = time_start(start,mode)
if mode == 'hour':
end = start + datetime.timedelta(hours=1)
elif mode == 'day':
end = start + datetime.timedelta(days=1)
elif mode == 'week':
end = start + datetime.timedelta(days=7)
elif mode == 'month':
if start.month == 12:
end = datetime.datetime(start.year+1,1,1)
else:
end = datetime.datetime(start.year,start.month+1,1)
return start,end
def start_importing(self, mode, now=None, stats=None, partner_ids=None):
n0 = datetime.datetime.now()
# normalize time
if not now:
now = datetime.datetime.now() - datetime.timedelta(days=1)
if mode == 'day':
start = time_start(now,'day')
elif mode == 'week' and now.weekday() == 0:
start = time_start(now,'week')
elif mode == 'month' and now.day == 1:
start = time_start(now,'month')
else:
print '%s %s not supported' % (mode,now)
return
# import data
if not stats or 'wechat' in stats:
self.import_wechat_data(mode,start)
print mode,start,'time:',datetime.datetime.now()-n0
return True
def import_wechat_data(self, mode, time):
'''
start: start time
return: one day's wechat stat
'''
if mode != 'day':
return
print time
start,end = self.normlize_time(mode,time)
import_num = 0
wechat = self.org_service.get_wechat_stat(start,end)
try:
for line in wechat:
line = line.strip()
data = line.split('\t')
stat = WechatStat.new()
stat.datetime = start.strftime('%Y-%m-%d')
stat.bookid = data[0]
stat.pv = data[1]
stat.uv = data[2]
stat.save()
import_num += 1
except Exception,e:
logging.error('%s\n',str(e),exc_info=True)
return import_num
