def setattr(self, attr, v):
if attr == 'created_at':
str_original = None
if isinstance(v, int): #Epoch Time as int or datetime.datetime
str_original = util.time2str(v)
elif isinstance(v, datetime):
v = util.time2epoch(v)
str_original = util.time2str(v)
else: # str
str_original = v
v = util.time2epoch(v) #创建日期，转换为基于东八区的Epoch整数格式
#print '%25s\t:=\t%s' % ('c_at_or',v)
self.__setattr__('c_at_or', str_original)
#print '%25s\t:=\t%s' % (attr,v)
self.__setattr__(attr, v)
def setattr(self,attr,v):
if attr == 'created_at':
str_original = None
if isinstance(v,int): #Epoch Time as int or datetime.datetime
str_original = util.time2str(v)
elif isinstance(v,datetime):
v = util.time2epoch(v)
str_original = util.time2str(v)
else: # str
str_original = v
v = util.time2epoch(v) #创建日期，转换为基于东八区的Epoch整数格式
#print '%25s\t:=\t%s' % ('c_at_or',v)
self.__setattr__('c_at_or',str_original)
#print '%25s\t:=\t%s' % (attr,v)
self.__setattr__(attr,v)
def setattr(self, attr, v):
if attr in ['province', 'city', 'verified_type'] and not isinstance(v, str):
# 认证类别、省、市ID用用str存储
v = str(v)
elif attr == 'idstr' and v is None:
v = str(id)
elif attr == 'created_at':
str_original = None
if isinstance(v, int): # Epoch Time as int or datetime.datetime
str_original = util.time2str(v)
elif isinstance(v, datetime):
v = util.time2epoch(v)
str_original = util.time2str(v)
else: # str
str_original = v
v = util.time2epoch(v) # 创建日期，转换为基于东八区的Epoch整数格式
# print '%25s\t:=\t%s' % ('created_at_or',v)
self.__setattr__('created_at_or',str_original)
# print '%25s\t:=\t%s' % (attr,v)
self.__setattr__(attr,v)
def log(addr, _type, _dir, name="", data={}, err="", pg="", tag=""):
"""记录消息日志"""
try:
today = util.time2str(fmt="%Y-%m-%d")
# 日期发生变化
if MsgLog.log_date != today:
# 关闭
if MsgLog.log_file:
MsgLog.log_file.close()
MsgLog.log_file = None
# 打开
MsgLog.log_file = open("log/msg.%s.log" % today, "a")
MsgLog.log_date = today
MsgLog.log_file.write(
"%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" %
(util.time2str(fmt="%H:%M:%S"), pg, tag, addr, name, _dir, _type,
data if data else "", err if err else ""))
except:
logging.fatal(
"act.log(addr=%r, _type=%r, pg=%r, uid=%r, data=%r, err=%r)\nException: \n%s",
addr, _type, pg, name, data, err, format_exc())
def setattr(self, attr, v):
if attr in ['province', 'city', 'verified_type'
] and not isinstance(v, str):
#认证类别、省、市ID用用str存储
v = str(v)
elif attr == 'idstr' and v is None:
v = str(id)
elif attr == 'created_at':
str_original = None
if isinstance(v, int): #Epoch Time as int or datetime.datetime
str_original = util.time2str(v)
elif isinstance(v, datetime):
v = util.time2epoch(v)
str_original = util.time2str(v)
else: # str
str_original = v
v = util.time2epoch(v) #创建日期，转换为基于东八区的Epoch整数格式
#print '%25s\t:=\t%s' % ('created_at_or',v)
self.__setattr__('created_at_or', str_original)
#print '%25s\t:=\t%s' % (attr,v)
self.__setattr__(attr, v)
def log(addr, _type, conn_id, name, data={}, err="", pg=""):
"""记录消息日志"""
try:
today = util.time2str(fmt="%Y-%m-%d")
# 日期发生变化
if ConnLog.log_date != today:
# 关闭
if ConnLog.log_file:
ConnLog.log_file.close()
ConnLog.log_file = None
# 打开
ConnLog.log_file = open("log/connect.%s.log" % today, "a")
ConnLog.log_date = today
ConnLog.log_file.write(
"%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" %
(util.time2str(fmt="%H:%M:%S"), _type, conn_id, pg, addr, name,
data if data else "", err if err else ""))
except:
logging.fatal(
"Conn.log(addr=%r, _type=%r, pg=%r, name=%r, conn_id=%r, data=%r, err=%r)\nException: \n%s",
addr, _type, pg, name, conn_id, data, err, format_exc())
def query_topdiffspender(config, start, end, param):
today_start = calendar.timegm(start.timetuple())
ystd_start = calendar.timegm((start - timedelta(days=1)).timetuple())
lastwk_start = calendar.timegm((start - timedelta(days=7)).timetuple())
db_end = util.query_end_time(param['table'],
config,
timezone=param['timezone'])
end = min(end, db_end)
today_end = calendar.timegm(end.timetuple()) / 900 * 900
ystd_end = calendar.timegm((end - timedelta(days=1)).timetuple())
lastwk_end = calendar.timegm((end - timedelta(days=7)).timetuple())
param['today_start'] = util.time2str(today_start, DATE_FORMAT)
param['ystd_start'] = util.time2str(ystd_start, DATE_FORMAT)
param['lastwk_start'] = util.time2str(lastwk_start, DATE_FORMAT)
param['today_end'] = util.time2str(today_end, DATE_FORMAT)
param['ystd_end'] = util.time2str(ystd_end, DATE_FORMAT)
param['lastwk_end'] = util.time2str(lastwk_end, DATE_FORMAT)
fields = 'time,id,today_spend,ystd_spend,lastwk_spend,today_click,ystd_click,lastwk_click,today_imp,ystd_imp,lastwk_imp,today_serve,ystd_serve,lastwk_serve,dod_delta,wow_delta,tot_3day_spend,adv'.split(
',')
tabid = param.get('tabid')
if tabid == 'native_topdiffsection':
command = SECTION_DIFF_TEMPLATE.format(**param)
else:
param['today_query'] = iterate_day(today_start, today_end, param,
TODAY_TEMPLATE)
param['ystd_query'] = iterate_day(ystd_start, ystd_end, param,
YEST_TEMPLATE)
param['lastwk_query'] = iterate_day(lastwk_start, lastwk_end, param,
LASTWK_TEMPLATE)
command = CMPGN_ADV_DIFF_TEMPLATE.format(**param)
return util.fetch_sql_data(config, fields, command,
-util.total_seconds(timedelta(minutes=5)))
def fetch_data0(self, appconfig, query, args, start, end, offset, data,
_type):
query = copy.deepcopy(query)
dateformat = util.get(query, 'dateformat', util.DATE_ISO)
query['dateformat'] = util.convert_date_format(dateformat)
# timezone_switch_aware_offset is used to make time align in graph
adjusted_offset = util.timezone_switch_aware_offset(
start, offset, query['timezone'])
groupby = query.get('groupby')
fields = query['fields'].split(',')
if groupby:
query['fields'] = ",".join(
map(
lambda f: 'sum(%s) as %s' % (f, f)
if (f not in groupby and not re.match(
" as ", f, re.IGNORECASE)) else f, fields))
query['groupby'] = "group by " + groupby
else:
query['groupby'] = ""
query['orderby'] = util.get(query, 'orderby', '',
lambda x: 'order by ' + x)
query['sort'] = util.get(query, 'sort', '')
query['limit'] = util.get(query, 'limit', '', lambda x: 'limit ' + x)
# support parameters in conditions
query['conditions'] = query['conditions'].format(**query)
query['partition'] = util.get(query, 'partition', 'ds')
# support parameters to join 2 tables
join = util.get(query, 'join', None)
join_to = None
if join:
join_to = util.get(join, 'join_to', None)
if join_to:
query['table1'] = util.get(query, 'table', None)
query['groupby_t1'] = "group by " + util.get(
join, 'groupby', '')
query['conditions_t1'] = self.get_additional_conditions(
args, join)
query['raw_fields_t1'] = ",".join(
map(
lambda f: 'sum(%s) as %s' % (f, f)
if (f not in query['groupby_t1'] and not re.match(
" as ", f, re.IGNORECASE)) else f,
join['raw_fields'].split(',')))
query['join_field_t1'] = util.get(join, 'on', None)
query['table2'] = util.get(join_to, 'table', None)
query['groupby_t2'] = "group by " + util.get(
join_to, 'groupby', '')
query['conditions_t2'] = self.get_additional_conditions(
args, join_to)
query['raw_fields_t2'] = ",".join(
map(
lambda f: 'sum(%s) as %s' % (f, f)
if (f not in query['groupby_t2'] and not re.match(
" as ", f, re.IGNORECASE)) else f,
join_to['raw_fields'].split(',')))
query['join_field_t2'] = util.get(join_to, 'on', None)
offset_w_timezone = adjusted_offset + util.getOffsetSeconds(
start, query['timezone'])
if util.get(query, 'source', None) == 'hive':
query['start'] = util.time_to_utc(start - offset, dateformat,
query['timezone'])
query['end'] = util.time_to_utc(end - offset, dateformat,
query['timezone'])
sql = HIVE_TEMPLATE.format(**query)
res = util.fetch_hive_data(appconfig, fields, sql,
offset_w_timezone)
else:
#query['start'] & query['end'] need to be in timezone-free format,
#because SQL_TEMPLATE will convert it to timezone-specific time
query['start'] = util.time2str(start - offset, dateformat)
query['end'] = util.time2str(end - offset, dateformat)
if join and join_to:
sql = SQL_JOIN_TEMPLATE.format(**query)
else:
sql = SQL_TEMPLATE.format(**query)
res = util.fetch_sql_data(appconfig, fields, sql,
offset_w_timezone)
data[_type] = res
