def __genOutputFileName(log_time, topic, start_time, data_type):
"""
:write file
"""
try:
output_path = AppConfig['data_out_put'][data_type]
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pydotalog.error("initConfig fail:%s", str(e))
return ""
dir_name = '{0}/{1}/{2}/{3}/{4}/{5}'.format(str(output_path), topic,
log_time[0:4], log_time[4:6],
log_time[6:8], log_time[8:10])
if "err" in data_type:
file_str = '{0}/err_{1}_{2}_{3}'.format(str(dir_name), log_time, topic,
start_time)
elif "des" == data_type:
file_str = '{0}/{1}_dm_{2}_{3}'.format(str(dir_name), log_time, topic,
start_time)
else:
file_str = '{0}/{1}_{2}_{3}'.format(str(output_path), log_time, topic,
start_time)
file_str = os.path.abspath(file_str)
dir_name = os.path.dirname(file_str)
if not os.path.exists(dir_name):
try:
os.makedirs(dir_name)
except OSError as e:
pydotalog.error("make dir failed:[%s]", str(e))
return ""
return file_str
def __check_time(log_time):
if len(log_time) != 10:
return False
try:
return 2010000000 < int(log_time) < 2050000000
except ValueError:
pydotalog.error("log_time[%s] is overtime", str(log_time))
return False
def __output_to_files(log_time, line, topic, start_time, data_type):
str_log_file = __genOutputFileName(log_time, topic, start_time, data_type)
if str_log_file == "":
return False
if str_log_file in file_list:
log_file = file_list[str_log_file]
else:
try:
log_file = open(str_log_file, 'w')
file_list[str_log_file] = log_file
except IOError as e:
pydotalog.error("IOError: %s", str(e))
return False
log_file.write(line.strip('\n') + '\n')
return True
def CheckLiveTime(log_time, live_info=[]):
"""
:check log time is effect,if this time effect return action id and cameraid
"""
for live_info_one in live_info:
try:
startTime = time.strptime(live_info_one[3], "%Y-%m-%d %H:%M:%S")
endTime = time.strptime(live_info_one[4], "%Y-%m-%d %H:%M:%S")
startTime = int(time.mktime(startTime))
endTime = int(time.mktime(endTime))
if startTime <= int(log_time) <= endTime:
return 0, live_info_one[0], live_info_one[1]
else:
continue
except (KeyError, ValueError) as e:
pydotalog.error("check time failed:%s", str(e))
return -1, 0, 0
pydotalog.error("%s is not a valid time!", str(log_time))
return -2, 0, 0
log_dir = os.path.join(os.path.dirname(__file__), "./log/")
pydotalog.init_logger(log_dir + "/dm.log")
pydotalog.info("start dm wash")
from scheduling import scheduling
def process():
'''
:清洗数据
'''
# 开始时间填入全局,用于后续流程
global_var_client.set_global_process_time(start_time)
process_handler = scheduling(topicname, dm_name, start_time)
for line in sys.stdin:
process_handler.proccess_line(line)
if __name__ == '__main__':
if len(sys.argv) == 4:
(di, topicname, dm_name, start_time) = sys.argv
pydotalog.info("input parameter topicname:%s" % topicname)
pydotalog.info("input parameter dm_name:%s" % dm_name)
pydotalog.info("input parameter start_time:%s" % start_time)
process()
else:
pydotalog.error("args is error")
def proccess_line(self, line):
'''
:根据配置文件的情况处理数据
'''
if line is None or line == '':
return [-1, '']
#将数据转换成Dict
_res = self.getDict(line)
if _res[0] > 0:
#有字典转换错误
self.write_to_file('%s,%s' % (_res[1], line),
self.output_file_prefix, self.recv_time,
self.recv_time, "dm_err")
return
#定义需要输出的数据List
_output_list = {}
for _output_index in range(len(self.dm_config_output_key)):
#根据需要输入的字段获取Dict中的配置然后调用规则的脚本进行换算
_output_field = self.dm_config_output_key[_output_index]
_rule_config = self.dm_config_output_dict[_output_field]
if _rule_config is None or len(_rule_config) < 2:
pydotalog.error('%s,%s' % ("config_no", line))
return
#判断是否有字段，和有对应的规则
_rule_config_field = _rule_config[0]
_rule_config_class = _rule_config[1]
if _rule_config_field is None:
pydotalog.error('%s,%s' % ("config_field", line))
return
if _rule_config_class == 0:
_output_list[_output_field] = _res[1][_rule_config_field]
elif _rule_config_class == 1:
if str(_res[1][_rule_config_field]).isdigit():
_output_list[_output_field] = _res[1][_rule_config_field]
else:
_output_list[_output_field] = '\N'
elif _rule_config_class == 2:
if str(_res[1][_rule_config_field]).isdigit():
_output_list[_output_field] = _res[1][_rule_config_field]
else:
_output_list[_output_field] = '-1'
elif _rule_config_class == -1:
_output_list[_output_field] = _rule_config_field
else:
if _rule_config_field == "":
pydotalog.error('%s,%s' % ("config_field_null", line))
return
if _rule_config_class not in self.my_rule_dict.keys():
pydotalog.error(
'This rule_id[%s] is failed to init to my_rule_dict ' %
str(_rule_config_class))
return
_rule_class = self.my_rule_dict[_rule_config_class]
if _rule_config_class is None or not isinstance(
_rule_class, rule):
pydotalog.error('%s,%s' % ("config_class", line))
return
#如果config_field 包含了多个字段使用|进行分融
_field_list = _rule_config_field.strip().split('|')
#获取每个字段的值
_field_values = []
for _field_list_index in range(len(_field_list)):
_field_values.append(
_res[1][_field_list[_field_list_index]])
#将拼装的各字段的值转给规则。并获取返回值
_rule_res = _rule_class.convert(_field_values)
if len(_rule_res) != 2:
self.write_to_file('%s,%s' % ("rule_return", line),
self.output_file_prefix, self.recv_time,
self.recv_time, "dm_err")
return
if _rule_res[0] != 0:
self.write_to_file(
'rule_%s_%s,%s' %
(_rule_config_class, _rule_res[1], line),
self.output_file_prefix, self.recv_time,
self.recv_time, "dm_err")
return
_output_list[_output_field] = _rule_res[1]
#字段解析完成并换算完成
_write_res = self.gen_des_line(_output_list)
if _write_res[0] != 0:
self.write_to_file('analyze_%s,%s' % (_write_res[1], line),
self.output_file_prefix, self.recv_time,
self.recv_time, "dm_err")
else:
self.write_to_file(_write_res[1], self.output_file_prefix,
self.recv_time, self.recv_time, 'des')
def init_import_module(self):
for _output_index in range(len(self.dm_config_output_key)):
_output_field = self.dm_config_output_key[_output_index]
_rule_config = self.dm_config_output_dict[_output_field]
if _rule_config is None or len(_rule_config) < 2:
pydotalog.error('%s,%s' % ("config_err", str(_rule_config)))
sys.exit(-1)
_rule_import_num = _rule_config[1]
if _rule_import_num in [0, 1, 2, -1]:
continue
if _rule_import_num not in rule_dict.keys():
pydotalog.error('This rule_id[%s] is not in rule_dict' %
str(_rule_import_num))
sys.exit(-1)
_rule_import_list = rule_dict[_rule_import_num]
if _rule_import_list is None or len(_rule_import_list) != 2:
pydotalog.error(
'%s, rule_num:[%s]' %
("rule_dict_config_failed", str(_rule_import_num)))
sys.exit(-1)
_rule_import_module = _rule_import_list[0]
_rule_class_name = _rule_import_list[1]
try:
module_name = importlib.import_module(_rule_import_module)
except ImportError:
pydotalog.error(
'%s, rule_module:[%s]' %
("import module failed ", str(_rule_import_module)))
sys.exit(-1)
_rule_class_str = getattr(module_name, _rule_class_name,
"no_class")
if _rule_class_str == "no_class":
pydotalog.error('%s, rule_class:[%s]' %
("get module class failed ", _rule_class_name))
sys.exit(-1)
self.my_rule_dict[_rule_import_num] = _rule_class_str()
if not isinstance(self.my_rule_dict[_rule_import_num], rule):
pydotalog.error(
'%s,%s' %
("config_class_failed, not rule_class", _rule_class_name))
sys.exit(-1)