def check_stat_plan_log_file_reading_completion(stat_plan):
"""
self.log_table_name='raw_data_monitor'
self.log_oem_name=self.script_file_name
self.log_category=helper_ip.get_current_server_ip()+'_'+self.start_time_str.replace(' ','_').replace(':','_').replace('-','_')+'_'+self.uuid
helper_mysql.put_raw_data(oem_name=self.log_oem_name, \
category=self.log_category, \
key='original_file_size', \
sub_key=log_file_name, \
value=helper_file.get_file_size(log_file_name), \
table_name=self.log_table_name)
"""
# 1.check total log file number
current_date=helper_regex.extract(stat_plan.log_category,r'_(\d{4}_\d{2}_\d{2})_').replace('_','-')
previous_date=helper_regex.date_add(current_date,-1)
previoud_date_category_like=helper_regex.extract(stat_plan.log_category.replace(current_date,previous_date),r'([\d\.]+_\d{4}_\d{2}_\d{2})')
sql=r'''
select
(select count(distinct sub_key)
from raw_data_monitor
where oem_name='%s'
and category='%s')
-
(select count(distinct sub_key)
from raw_data_monitor
where oem_name='%s'
and category=(
select count(distinct sub_key)
from raw_data_monitor
where oem_name='%s'
and category like '%s%%'
))
''' % (stat_plan.log_oem_name,stat_plan.log_category,stat_plan.log_oem_name,stat_plan.log_oem_name,previoud_date_category_like)
print sql
distance=helper_mysql.get_one_value_string(sql)
print distance
return distance
where `oem_name`="Vodafone"
and `category`="moagent"
and `key`="from_app_request_by_monet_id_user_agent_first_text_value"
and `sub_key`='%s'
''')
result_sql_signature=form.getvalue("result_sql_signature", "")
collection=helper_mysql.get_raw_collection_by_id(collection_id)
#collection=list(collection)[0:10]
content=['<table style=\\"margin:0; padding:0; border:0; text-align:left;\\">']
for i in collection:
tmp_sql=result_sql % (i,)
result=helper_mysql.get_one_value_string(tmp_sql)
content.append('<tr><td>'+str(i)+'</td><td><div style=\\"text-align:left;\\">'+str(result)+'</div></td></tr>')
content.append('</table>')
content=''.join(content)
content='document.write("<div style=\\"float:left;\\">'+content+'</div>");'
#print "Content-type: application/x-unknown"
#print "Content-Disposition: attachment; filename=data.txt"
print "Content-type: application/javascript"
print "Content-Length: "+str(len(content))
print ''
print content
def calculate_ndays_unique(key_space,db_name,date_units):
# calculate n days' unique
min_date=helper_mysql.get_one_value_string(r'''
select
min(`date`)
from `%s`
where `oem_name`='%s'
and `category`='%s'
and `key`='%s_collection_id'
and `sub_key`='%s'
and `date`>='2011-12-16'
''' % (db_name,key_space['oem_name'],key_space['category'],key_space['key'],key_space['sub_key']))
max_date=helper_mysql.get_one_value_string(r'''
select
max(`date`)
from `%s`
where `oem_name`='%s'
and `category`='%s'
and `key`='%s_collection_id'
and `sub_key`='%s'
and `date`>='2011-12-16'
''' % (db_name,key_space['oem_name'],key_space['category'],key_space['key'],key_space['sub_key']))
if not min_date or not max_date:
print 'date error.'
return
date_temp=min_date
#print date_temp
#exit()
while True:
if date_temp>=max_date:
break
for date_unit in date_units:
unique,total,average=helper_math.calculate_count_distinct(date_unit=date_unit,oem_name=key_space['oem_name'],category=key_space['category'],key=key_space['key'],sub_key=key_space['sub_key'],date=date_temp,table_name=db_name,allow_collection_empty=True)
print 'distinct collection calc '+date_temp+': date_unit '+str(date_unit)+' unique '+str(unique)+' total '+str(total)+' average '+str(average)
#exit()
key_prefix=helper_regex.regex_replace('_unique$','',key_space['key'])
if unique>0:
suffix=str(date_unit)
if isinstance(date_unit, (int, long)):
suffix+='_days'
helper_mysql.put_raw_data(oem_name=key_space['oem_name'],category=key_space['category'],key=key_prefix+'_'+suffix+'_unique',sub_key=key_space['sub_key'],value=unique,date=date_temp,table_name=db_name)
helper_mysql.put_raw_data(oem_name=key_space['oem_name'],category=key_space['category'],key=key_prefix+'_'+suffix+'_unique_base',sub_key=key_space['sub_key'],value=total,date=date_temp,table_name=db_name)
helper_mysql.put_raw_data(oem_name=key_space['oem_name'],category=key_space['category'],key=key_prefix+'_'+suffix+'_unique_average',sub_key=key_space['sub_key'],value=average,date=date_temp,table_name=db_name)
date_temp=helper_regex.date_add(date_temp,1)
and `key` = 'app_page_daily_visitor_unique' /*'app_page_daily_visitor_unique'*/
/*and `key` not like '%%_collection_id'
and `key` not like '%%_by_url_%%'*/
/*and `sub_key` = ''*/
and `date`>='%s' and `date`<='%s'
'''
#get reference value base
reference_value_base_sql=sql_reference_columns_tpl % (sql_base_date_range_start,sql_base_date_range_end)
#print reference_value_base_sql
reference_value_base=helper_mysql.get_one_value_string(reference_value_base_sql)
print 'reference_value_base: '+reference_value_base
#loop taget dates
for offset in range(0,target_date_length):
current_date=helper_regex.date_add(target_date_start,offset)
print 'current_date: '+current_date
#get reference value of target date
reference_value_target=helper_mysql.get_one_value_string(
sql_reference_columns_tpl
% (current_date,current_date))
