def __init__(self):
comlog.init_logger("./../log/vdata.log")
self.ub_conf = UbConfig("./../conf/log_tail.conf")
self._conf_info = self.ub_conf.get_conf_info()
self._file_path = self._conf_info["[LOG_FILE_CONF]"]["file_path"]
self._file_name_pattern = self._conf_info["[LOG_FILE_CONF]"][
"file_name_pattern"]
self._log_max_length = int(
self._conf_info["[LOG_FILE_CONF]"]["log_max_length"])
self._batch_flush_counter = int(
self._conf_info["[LOG_FILE_CONF]"]["batch_flush_counter"])
self._topic_name = self._conf_info["[KAFKA]"]["topic_name"]
self._interval_time = self._conf_info["[TIME_INTERVAL]"]["interval"]
self.init_data_file()
self.s = sched.scheduler(time.time, time.sleep)
if self._conf_info["[KAFKA]"].has_key('broker_list'):
self.broker_list = self._conf_info["[KAFKA]"]["broker_list"]
elif self._conf_info["[KAFKA]"].has_key('zookeeper'):
self.broker_list = ','.join(
self.get_broker_list(self._conf_info["[KAFKA]"]['zookeeper']))
else:
raise ValueError, " zookeeper and broker_list are both null in config file"
self.client = KafkaClient(self.broker_list)
self.producer = SimpleProducer(self.client)
#tags
self.tags = {}
for key in self._conf_info["[TAGS]"]:
self.tags[key] = self._conf_info["[TAGS]"][key]
def __init__(self):
comlog.init_logger("./../log/vdata.log")
self.ub_conf = UbConfig("./../conf/log_tail.conf")
self._conf_info = self.ub_conf.get_conf_info()
self._file_path = self._conf_info["[LOG_FILE_CONF]"]["file_path"]
self._file_name_pattern = self._conf_info["[LOG_FILE_CONF]"]["file_name_pattern"]
self._log_max_length = int(self._conf_info["[LOG_FILE_CONF]"]["log_max_length"])
self._batch_flush_counter = int(self._conf_info["[LOG_FILE_CONF]"]["batch_flush_counter"])
self._topic_name = self._conf_info["[KAFKA]"]["topic_name"]
self._interval_time = self._conf_info["[TIME_INTERVAL]"]["interval"]
self.init_data_file()
self.s = sched.scheduler(time.time, time.sleep)
if self._conf_info["[KAFKA]"].has_key('broker_list'):
self.broker_list=self._conf_info["[KAFKA]"]["broker_list"]
elif self._conf_info["[KAFKA]"].has_key('zookeeper'):
self.broker_list=','.join(self.get_broker_list(self._conf_info["[KAFKA]"]['zookeeper']))
else:
raise ValueError, " zookeeper and broker_list are both null in config file"
self.client = KafkaClient(self.broker_list)
self.producer = SimpleProducer(self.client)
#tags
self.tags = {}
for key in self._conf_info["[TAGS]"]:
self.tags[key]=self._conf_info["[TAGS]"][key]
class Main(object):
def __init__(self):
comlog.init_logger("./../log/vdata.log")
self.ub_conf = UbConfig("./../conf/log_tail.conf")
self._conf_info = self.ub_conf.get_conf_info()
self._file_path = self._conf_info["[LOG_FILE_CONF]"]["file_path"]
self._file_name_pattern = self._conf_info["[LOG_FILE_CONF]"][
"file_name_pattern"]
self._log_max_length = int(
self._conf_info["[LOG_FILE_CONF]"]["log_max_length"])
self._batch_flush_counter = int(
self._conf_info["[LOG_FILE_CONF]"]["batch_flush_counter"])
self._topic_name = self._conf_info["[KAFKA]"]["topic_name"]
self._interval_time = self._conf_info["[TIME_INTERVAL]"]["interval"]
self.init_data_file()
self.s = sched.scheduler(time.time, time.sleep)
if self._conf_info["[KAFKA]"].has_key('broker_list'):
self.broker_list = self._conf_info["[KAFKA]"]["broker_list"]
elif self._conf_info["[KAFKA]"].has_key('zookeeper'):
self.broker_list = ','.join(
self.get_broker_list(self._conf_info["[KAFKA]"]['zookeeper']))
else:
raise ValueError, " zookeeper and broker_list are both null in config file"
self.client = KafkaClient(self.broker_list)
self.producer = SimpleProducer(self.client)
#tags
self.tags = {}
for key in self._conf_info["[TAGS]"]:
self.tags[key] = self._conf_info["[TAGS]"][key]
def init_data_file(self):
if not os.path.isdir("./../data"):
os.mkdir("./../data")
def gen_message_final_str(self, message_array=[]):
message_final = {}
message_final["message"] = message_array
message_final["tags"] = self.tags
message_final["tags"]["event_time"] = int(time.time() * 1000)
message_final_str = json.dumps(message_final)
return message_final_str
def event_func_mtime(self):
try:
pygtail = PygtailMtime(self._file_path,
self._file_name_pattern,
offset_file=None,
paranoid=True,
copytruncate=True)
message_array_to_send = []
for line in pygtail:
message_lenth = len(line)
if message_lenth > self._log_max_length:
comlog.warning("The message is too long:" +
str(message_lenth))
continue
message_array_to_send.append(line.strip('\n'))
if len(message_array_to_send) > int(self._batch_flush_counter):
message_final_str = self.gen_message_final_str(
message_array_to_send)
self.producer.send_messages(self._topic_name,
message_final_str)
message_array_to_send = []
if len(message_array_to_send) > 0:
message_final_str = self.gen_message_final_str(
message_array_to_send)
self.producer.send_messages(self._topic_name,
message_final_str)
message_array_to_send = []
except Exception, data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
class Main(object):
def __init__(self):
comlog.init_logger("./../log/vdata.log")
self.ub_conf = UbConfig("./../conf/log_tail.conf")
self._conf_info = self.ub_conf.get_conf_info()
self._file_path = self._conf_info["[LOG_FILE_CONF]"]["file_path"]
self._file_name_pattern = self._conf_info["[LOG_FILE_CONF]"]["file_name_pattern"]
self._log_max_length = int(self._conf_info["[LOG_FILE_CONF]"]["log_max_length"])
self._batch_flush_counter = int(self._conf_info["[LOG_FILE_CONF]"]["batch_flush_counter"])
self._topic_name = self._conf_info["[KAFKA]"]["topic_name"]
self._interval_time = self._conf_info["[TIME_INTERVAL]"]["interval"]
self.init_data_file()
self.s = sched.scheduler(time.time, time.sleep)
if self._conf_info["[KAFKA]"].has_key('broker_list'):
self.broker_list=self._conf_info["[KAFKA]"]["broker_list"]
elif self._conf_info["[KAFKA]"].has_key('zookeeper'):
self.broker_list=','.join(self.get_broker_list(self._conf_info["[KAFKA]"]['zookeeper']))
else:
raise ValueError, " zookeeper and broker_list are both null in config file"
self.client = KafkaClient(self.broker_list)
self.producer = SimpleProducer(self.client)
#tags
self.tags = {}
for key in self._conf_info["[TAGS]"]:
self.tags[key]=self._conf_info["[TAGS]"][key]
def init_data_file(self):
if not os.path.isdir("./../data"):
os.mkdir("./../data")
def gen_message_final_str(self,message_array=[]):
message_final={}
message_final["message"]=message_array
message_final["tags"]=self.tags
message_final["tags"]["event_time"]=int(time.time()*1000)
message_final_str=json.dumps(message_final)
return message_final_str
def event_func_mtime(self):
try:
pygtail = PygtailMtime(self._file_path, self._file_name_pattern, offset_file=None, paranoid=True, copytruncate=True)
message_array_to_send=[]
for line in pygtail:
message_lenth=len(line)
if message_lenth > self._log_max_length:
comlog.warning("The message is too long:" + str(message_lenth))
continue
message_array_to_send.append(line.strip('\n'))
if len(message_array_to_send)>int(self._batch_flush_counter):
message_final_str=self.gen_message_final_str(message_array_to_send)
self.producer.send_messages(self._topic_name, message_final_str)
message_array_to_send=[]
if len(message_array_to_send)>0:
message_final_str=self.gen_message_final_str(message_array_to_send)
self.producer.send_messages(self._topic_name, message_final_str)
message_array_to_send=[]
except Exception,data:
comlog.fatal(str(Exception))
comlog.fatal(str(data))
sys.exit()
