def CheckLiveTime(log_time, live_info=[]):
"""
:summery 校验日志时间是否有效的活动时间期间,并返回活动id和活动摄像机id
:param log_time: 需要校验的日志时间时间戳
:param live_info: 日志对应活动的活动信息，list数组
:return: errnum,str,str 0-有效 －1 时间格式错误 －2 时间不在有效期内
"""
for live_info_one in live_info:
try:
startTime = time.strptime(live_info_one[3], "%Y-%m-%d %H:%M:%S")
endTime = time.strptime(live_info_one[4], "%Y-%m-%d %H:%M:%S")
startTime = int(time.mktime(startTime))
endTime = int(time.mktime(endTime))
if startTime <= int(log_time) <= endTime:
return 0, live_info_one[0], live_info_one[1]
else:
continue
except (KeyError, ValueError) as e:
pydotalog.error("check time failed: %s", str(e))
return -1, 0, 0
pydotalog.error("%s is not a valid time !", str(log_time))
return -2, 0, 0
def CheckLiveTime(log_time, live_info=[]):
"""
:summery 校验日志时间是否有效的活动时间期间,并返回活动id和活动摄像机id
:param log_time: 需要校验的日志时间时间戳
:param live_info: 日志对应活动的活动信息，list数组
:return: errnum,str,str 0-有效 －1 时间格式错误 －2 时间不在有效期内
"""
for live_info_one in live_info:
try:
startTime = time.strptime(live_info_one[3], "%Y-%m-%d %H:%M:%S")
endTime = time.strptime(live_info_one[4], "%Y-%m-%d %H:%M:%S")
startTime = int(time.mktime(startTime))
endTime = int(time.mktime(endTime))
if startTime <= int(log_time) <= endTime:
return 0, live_info_one[0], live_info_one[1]
else:
continue
except (KeyError, ValueError) as e:
pydotalog.error("check time failed: %s", str(e))
return -1, 0, 0
pydotalog.error("%s is not a valid time !", str(log_time))
return -2, 0, 0
def __check_time(log_time):
"""
:summary: 校验log_time是否合理
:param log_time: 201510101000
:return: boolean
"""
if len(log_time) != 12:
return False
try:
return 201000000000 < int(log_time) < 205000000000
except ValueError:
pydotalog.error("log_time[%s] is overtime", str(log_time))
return False
def __check_time(log_time):
"""
:summary: 校验log_time是否合理
:param log_time: 201510101000
:return: boolean
"""
if len(log_time) != 12:
return False
try:
return 201000000000 < int(log_time) < 205000000000
except ValueError:
pydotalog.error("log_time[%s] is overtime", str(log_time))
return False
def __genOutputFileName(log_time, topic, start_time, data_type):
"""
根据日志时间，topic，后缀，数据类型拼接写入文件
:param log_time: 需要写入的日志时间,精确到小时
:type log_time: basestring 201511012200 12位
:param topic: kafka－topic名称
:type topic: basestring
:param start_time: 文件后缀名称, 为recv文件的时刻
:type start_time: basestring 201511012300 12位
:param data_type: 写入数据的类型，分为 orig,des,err
:type data_type: basestring
:return:
"""
try:
output_path = ServerConf.get("output_path", data_type)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pydotalog.error("initConfig fail:%s", str(e))
return ""
if "err" in data_type:
file_str = '{0}/{1}/{2}/err_{3}_{4}_{5}'.format(
str(output_path), log_time[0:4], log_time[4:6], log_time, topic,
start_time)
elif "des" == data_type:
if "_pv" in topic:
file_str = '{0}/{1}/{2}/{3}_pvrawdata_{4}_{5}'.format(
str(output_path), log_time[0:4], log_time[4:6], log_time,
topic, start_time)
elif "mglive" in topic:
file_str = '{0}/{1}/{2}/{3}_mgliverawdata_{4}_{5}'.format(
str(output_path), log_time[0:4], log_time[4:6], log_time,
topic, start_time)
else:
file_str = '{0}/{1}/{2}/{3}_playrawdata_{4}_{5}'.format(
str(output_path), log_time[0:4], log_time[4:6], log_time,
topic, start_time)
else:
file_str = '{0}/{1}/{2}/{3}_{4}_{5}'.format(str(output_path),
log_time[0:4],
log_time[4:6], log_time,
topic, start_time)
file_str = os.path.abspath(file_str)
dir_name = os.path.dirname(file_str)
if not os.path.exists(dir_name):
try:
os.makedirs(dir_name)
except OSError as e:
pydotalog.error("make dir failed:[%s]", str(e))
return ""
return file_str
def __output_to_files(log_time, line, topic, start_time, data_type):
str_log_file = __genOutputFileName(log_time, topic, start_time, data_type)
if str_log_file == "":
return False
if str_log_file in file_list:
log_file = file_list[str_log_file]
else:
try:
log_file = open(str_log_file, 'w')
file_list[str_log_file] = log_file
except IOError as e:
pydotalog.error("IOError: %s", str(e))
return False
log_file.write(line.strip('\n') + '\n')
return True
def __output_to_files(log_time, line, topic, start_time, data_type):
str_log_file = __genOutputFileName(log_time, topic, start_time, data_type)
if str_log_file == "":
return False
if str_log_file in file_list:
log_file = file_list[str_log_file]
else:
try:
log_file = open(str_log_file, 'w')
file_list[str_log_file] = log_file
except IOError as e:
pydotalog.error("IOError: %s", str(e))
return False
log_file.write(line.strip('\n') + '\n')
return True
def __genOutputFileName(log_time, topic, start_time, data_type):
"""
根据日志时间，topic，后缀，数据类型拼接写入文件
:param log_time: 需要写入的日志时间,精确到小时
:type log_time: basestring 201511012200 12位
:param topic: kafka－topic名称
:type topic: basestring
:param start_time: 文件后缀名称, 为recv文件的时刻
:type start_time: basestring 201511012300 12位
:param data_type: 写入数据的类型，分为 orig,des,err
:type data_type: basestring
:return:
"""
try:
output_path = ServerConf.get("output_path", data_type)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pydotalog.error("initConfig fail:%s", str(e))
return ""
if "err" in data_type:
file_str = '{0}/{1}/{2}/err_{3}_{4}_{5}'.format(str(output_path), log_time[0:4],
log_time[4:6], log_time, topic, start_time)
elif "des" == data_type:
if "_pv" in topic:
file_str = '{0}/{1}/{2}/{3}_pvrawdata_{4}_{5}'.format(str(output_path), log_time[0:4],
log_time[4:6], log_time, topic, start_time)
elif "mglive" in topic:
file_str = '{0}/{1}/{2}/{3}_mgliverawdata_{4}_{5}'.format(str(output_path), log_time[0:4],
log_time[4:6], log_time, topic, start_time)
else:
file_str = '{0}/{1}/{2}/{3}_playrawdata_{4}_{5}'.format(str(output_path), log_time[0:4],
log_time[4:6], log_time, topic, start_time)
else:
file_str = '{0}/{1}/{2}/{3}_{4}_{5}'.format(str(output_path), log_time[0:4],
log_time[4:6], log_time, topic, start_time)
file_str = os.path.abspath(file_str)
dir_name = os.path.dirname(file_str)
if not os.path.exists(dir_name):
try:
os.makedirs(dir_name)
except OSError as e:
pydotalog.error("make dir failed:[%s]", str(e))
return ""
return file_str
def LoadLiveMeizi(start_time):
"""
:summery: 加载直播媒资信息
:param start_time: 加载媒资信息的时刻，12为字符串，201511031700
:return: 媒资信息dict
"""
Meizi_info = {}
try:
output_path = ServerConf.get("meizi", "output_path")
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pydotalog.error("initConfig fail:%s", str(e))
sys.exit(-1)
file = output_path + '/' + start_time + '_live.csv'
if not os.path.exists(file):
pydotalog.error("live_meizi info file is not exists!")
sys.exit(-1)
with open(file) as fp:
for i, line in enumerate(fp):
try:
record = line.strip().split(',')
sourceid = record[0]
activityid = record[2]
cameraid = record[5]
timeid = record[7]
startTime = record[8]
endTime = record[9]
try:
Meizi_info[sourceid].append(
[activityid, cameraid, timeid, startTime, endTime])
except KeyError:
Meizi_info[sourceid] = []
Meizi_info[sourceid].append(
[activityid, cameraid, timeid, startTime, endTime])
except KeyError:
pydotalog.error("line[%d]: live meizi index err!", i)
continue
return Meizi_info
def LoadLiveMeizi(start_time):
"""
:summery: 加载直播媒资信息
:param start_time: 加载媒资信息的时刻，12为字符串，201511031700
:return: 媒资信息dict
"""
Meizi_info = {}
try:
output_path = ServerConf.get("meizi", "output_path")
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pydotalog.error("initConfig fail:%s", str(e))
sys.exit(-1)
file = output_path + '/' + start_time + '_live.csv'
if not os.path.exists(file):
pydotalog.error("live_meizi info file is not exists!")
sys.exit(-1)
with open(file) as fp:
for i, line in enumerate(fp):
try:
record = line.strip().split(',')
sourceid = record[0]
activityid = record[2]
cameraid = record[5]
timeid = record[7]
startTime = record[8]
endTime = record[9]
try:
Meizi_info[sourceid].append([activityid, cameraid, timeid, startTime, endTime])
except KeyError:
Meizi_info[sourceid] = []
Meizi_info[sourceid].append([activityid, cameraid, timeid, startTime, endTime])
except KeyError:
pydotalog.error("line[%d]: live meizi index err!", i)
continue
return Meizi_info
def process_main():
for line in sys.stdin:
format_client.processFormat(line)
if __name__ == '__main__':
log_dir = os.path.join(os.path.dirname(__file__), "./log/")
pydotalog.init_logger(log_dir + "/pydota_run.log")
if len(sys.argv) == 3:
(di, topic_name, start_time) = sys.argv
(rt, format_client) = get_class_name(topic_name, start_time)
if rt != 0:
pydotalog.error(format_client)
sys.exit(-1)
print len(start_time)
if len(start_time) != 12:
pydotalog.error("topic:[%s] start_time format error, start_time: %s", topic_name, start_time)
sys.exit(-1)
process_main()
else:
pydotalog.error('arg is topic_name date')
sys.exit(-1)
def process_main():
for line in sys.stdin:
format_client.processFormat(line)
if __name__ == '__main__':
log_dir = os.path.join(os.path.dirname(__file__), "./log/")
pydotalog.init_logger(log_dir + "/pydota_run.log")
if len(sys.argv) == 3:
(di, topic_name, start_time) = sys.argv
(rt, format_client) = get_class_name(topic_name, start_time)
if rt != 0:
pydotalog.error(format_client)
sys.exit(-1)
print len(start_time)
if len(start_time) != 12:
pydotalog.error(
"topic:[%s] start_time format error, start_time: %s",
topic_name, start_time)
sys.exit(-1)
process_main()
else:
pydotalog.error('arg is topic_name date')
sys.exit(-1)