def event_func_mtime(self):
try:
pygtail = PygtailMtime(self._file_path,
self._file_name_pattern,
offset_file=None,
paranoid=True,
copytruncate=True)
message_array_to_send = []
for line in pygtail:
message_lenth = len(line)
if message_lenth > self._log_max_length:
comlog.warning("The message is too long:" +
str(message_lenth))
continue
message_array_to_send.append(line.strip('\n'))
if len(message_array_to_send) > int(self._batch_flush_counter):
message_final_str = self.gen_message_final_str(
message_array_to_send)
self.producer.send_messages(self._topic_name,
message_final_str)
message_array_to_send = []
if len(message_array_to_send) > 0:
message_final_str = self.gen_message_final_str(
message_array_to_send)
self.producer.send_messages(self._topic_name,
message_final_str)
message_array_to_send = []
except Exception, data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
def file2dict(self):
if not os.path.isfile(self.bin_diff_config):
comlog.fatal(self.bin_diff_config+" is not exist!")
return -1
conf_file = file(self.bin_diff_config)
line = conf_file.readline()
line_num = 0
while line:
line = self.f_line(line)
if len(line) <= 0 or line[0] == '#':
line = conf_file.readline()
else:
self.conf_dict[line_num] = line
line = conf_file.readline()
line_num += 1
return 0
def get_broker_list(self,zookeeper):
try:
zookeeper = KazooClient(zookeeper)
zookeeper.start()
broker_list=[]
for id in zookeeper.get_children("/brokers/ids"):
broker_info=json.loads(zookeeper.get("/brokers/ids/"+str(id))[0])
host=broker_info["host"]
port=broker_info["port"]
broker_list.append(str(host)+':'+str(port))
zookeeper.stop()
return broker_list
except Exception,data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
def get_broker_list(self, zookeeper):
try:
zookeeper = KazooClient(zookeeper)
zookeeper.start()
broker_list = []
for id in zookeeper.get_children("/brokers/ids"):
broker_info = json.loads(
zookeeper.get("/brokers/ids/" + str(id))[0])
host = broker_info["host"]
port = broker_info["port"]
broker_list.append(str(host) + ':' + str(port))
zookeeper.stop()
return broker_list
except Exception, data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
def event_func_mtime(self):
try:
pygtail = PygtailMtime(self._file_path, self._file_name_pattern, offset_file=None, paranoid=True, copytruncate=True)
message_array_to_send=[]
for line in pygtail:
message_lenth=len(line)
if message_lenth > self._log_max_length:
comlog.warning("The message is too long:" + str(message_lenth))
continue
message_array_to_send.append(line.strip('\n'))
if len(message_array_to_send)>int(self._batch_flush_counter):
message_final_str=self.gen_message_final_str(message_array_to_send)
self.producer.send_messages(self._topic_name, message_final_str)
message_array_to_send=[]
if len(message_array_to_send)>0:
message_final_str=self.gen_message_final_str(message_array_to_send)
self.producer.send_messages(self._topic_name, message_final_str)
message_array_to_send=[]
except Exception,data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
def format_check(self):
if self.conf_dict == {}:
comlog.fatal("Failed to load " + self.bin_diff_config)
return -1
if (self.conf_dict[0].find('[') != 0 or self.conf_dict[0].find('.') == 1 or self.conf_dict[0].find(']') == -1):
comlog.fatal("Wrong conf format: " + self.conf_dict[0])
return -1
base_level = 0
for i in range(0,len(self.conf_dict.keys())):
if self.conf_dict[i].find('[') == 0:
num = self.conf_dict[i].count('.')
if (num == 0 and base_level in [0,1,2]) or \
(num == 1 and base_level in [0,1,2]) or \
(num == 2 and base_level in [1,2]):
base_level = num
pass
else:
comlog.fatal("Wrong conf format: " + self.conf_dict[i])
return -1
return 0