def fun_map_json(json_string, time_key, time_interval_minute_list, time_key_prefix):
import json
reload(sys)
sys.setdefaultencoding("utf-8")
from datetime import datetime
obj = json.loads(json_string)
time = obj.get(time_key, '')
# print('= = ' * 10, '[myapp test_utils.fun_map_row] found time = ', time, ', obj = ', obj)
cst_time = date_string_utc2cst(time)
obj[time_key] = cst_time
if cst_time == '':
res = None
print('= = ' * 10, '[myapp test_utils.fun_map_row] found ivalid cst_time')
else:
labels = obj['labels']
minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute
for time_interval_minute in time_interval_minute_list:
minute_start = (minute / time_interval_minute) * time_interval_minute
minute_end = minute_start + time_interval_minute - 1
time_start_key = time_key_prefix + str(time_interval_minute) + '_start'
time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
labels[time_start_key] = time_start
labels[time_end_key] = time_end
obj['labels'] = labels
res = Row(**obj)
print('= = ' * 10, '[myapp test_utils.fun_map_row] res.time = ', res.time, ', labels = ', res.labels)
return res
def fun_time_in_rdd_map(row,
time_key,
time_interval_minute_list,
time_key_prefix,
cache_conf=None):
"""
输入: Row[**obj]
输出: Row[**obj]
"""
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from datetime import datetime
# obj = json.loads(line)
obj = row.asDict()
time = obj.get(time_key, '')
cst_time = date_string_utc2cst(time)
if cst_time == '':
ret = None
else:
obj[time_key] = cst_time
minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute
for time_interval_minute in time_interval_minute_list:
minute_start = (minute /
time_interval_minute) * time_interval_minute
minute_end = minute_start + time_interval_minute - 1
time_start_key = time_key_prefix + str(
time_interval_minute) + '_start'
time_end_key = time_key_prefix + str(
time_interval_minute) + '_end'
time_start = cst_time[0:14] + str(minute_start).zfill(
2) + ':00'
time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
obj[time_start_key] = time_start
obj[time_end_key] = time_end
# ret = json.dumps(obj)
ret = Row(**obj)
# print('= = ' * 15, '[myapp EnhanceTimeProcessor.process.fun_time_in_rdd_map] ret = ')
# print(ret)
return ret
def fun_map_json(json_string, time_key, time_interval_minute_list,
time_key_prefix):
import json
reload(sys)
sys.setdefaultencoding("utf-8")
from datetime import datetime
obj = json.loads(json_string)
time = obj.get(time_key, '')
# print('= = ' * 10, '[myapp test_utils.fun_map_row] found time = ', time, ', obj = ', obj)
cst_time = date_string_utc2cst(time)
obj[time_key] = cst_time
if cst_time == '':
res = None
print('= = ' * 10,
'[myapp test_utils.fun_map_row] found ivalid cst_time')
else:
labels = obj['labels']
minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute
for time_interval_minute in time_interval_minute_list:
minute_start = (minute /
time_interval_minute) * time_interval_minute
minute_end = minute_start + time_interval_minute - 1
time_start_key = time_key_prefix + str(
time_interval_minute) + '_start'
time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
labels[time_start_key] = time_start
labels[time_end_key] = time_end
obj['labels'] = labels
res = Row(**obj)
print('= = ' * 10, '[myapp test_utils.fun_map_row] res.time = ',
res.time, ', labels = ', res.labels)
return res
def fun_time_in_rdd_map(row, time_key, time_interval_minute_list, time_key_prefix, cache_conf=None):
"""
输入: Row[**obj]
输出: Row[**obj]
"""
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from datetime import datetime
# obj = json.loads(line)
obj = row.asDict()
time = obj.get(time_key, '')
cst_time = date_string_utc2cst(time)
if cst_time == '':
ret = None
else:
obj[time_key] = cst_time
minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute
for time_interval_minute in time_interval_minute_list:
minute_start = (minute / time_interval_minute) * time_interval_minute
minute_end = minute_start + time_interval_minute - 1
time_start_key = time_key_prefix + str(time_interval_minute) + '_start'
time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
obj[time_start_key] = time_start
obj[time_end_key] = time_end
# ret = json.dumps(obj)
ret = Row(**obj)
# print('= = ' * 15, '[myapp EnhanceTimeProcessor.process.fun_time_in_rdd_map] ret = ')
# print(ret)
return ret