def __init__(self, index, conf): self.__load_conf(index, conf) self.q = [] self.meta = Meta(MetaUrl(self.cp_path, self.cp_name)) self.meta.startRecoding() self.meta_info = self.__load_from_meta() self.__check_meta() self.__init_sender() self.stopFlag = False self.stopevent = threading.Event() self.sleeptime = 0.1 threading.Thread.__init__(self, name="thread_tail_" + self.topic)
def __init__(self, index, conf): self.__load_conf(index, conf) self.q = [] self.meta = Meta(MetaUrl(self.cp_path, self.cp_name)) self.meta.startRecoding() self.meta_info = self.__load_from_meta() self.__check_meta() self.__init_sender() self.stopFlag = False self.stopevent = threading.Event() self.sleeptime = 0.1 threading.Thread.__init__(self, name="thread_tail_" + self.topic)
class Tail(Thread): def __init__(self, index, conf): self.__load_conf(index, conf) self.q = [] self.meta = Meta(MetaUrl(self.cp_path, self.cp_name)) self.meta.startRecoding() self.meta_info = self.__load_from_meta() self.__check_meta() self.__init_sender() self.stopFlag = False self.stopevent = threading.Event() self.sleeptime = 0.1 threading.Thread.__init__(self, name="thread_tail_" + self.topic) def __init_sender(self): if self.local_test_path != "null": self.sender = DiskSender(self.local_test_path, self.topic, self.encoding, self.max_buf_size) else: self.sender = TTSender(self.max_buf_size, self.topic, self.topic_timeout, self.encoding, self.topic_compress, self.topic_sequence) def __check_meta(self): self.scanner.scan() if len(self.q) == 0: return if self.tmp_path == "null": if self.meta_info.cur_file != "null" and self.meta_info.cur_file != self.q[0]: logger.error("cur file in meta: " + self.meta_info.cur_file + " not match: " + self.q[0]) self.offset = 0L return def __load_conf(self, i, c): self.path_regx = c.get_path_regx().split(",")[i] self.base_path = c.get_base_path().split(",")[i] tmp_paths = c.get_tmp_path() if tmp_paths == "null": self.tmp_path = "null" else: tmpath = c.get_tmp_path().split(",")[i] if tmpath is "" or tmpath is "None": self.tmp_path = "null" else: self.tmp_path = tmpath self.cp_path = c.get_cp_path() self.cp_name = c.get_cp_name().split(",")[i] self.local_test_path = c.get_test_path() self.topic = c.get_topic_name().split(",")[i] self.encoding = c.get_topic_encoding().split(",")[i] self.topic_timeout = c.get_topic_timeout().split(",")[i] self.topic_compress = c.get_topic_compress().split(",")[i] self.topic_sequence = c.get_topic_sequence().split(",")[i] self.max_buf_size = c.get_max_buf_size() pass def __load_from_meta(self): meta_info = self.meta.restore() self.offset = meta_info.offset history_files = meta_info.last_file.split(",") modify_time = 0 len_lasts=len(history_files) if len_lasts!=0: last_file=history_files[len_lasts-1] if os.path.exists(last_file) is True: modify_time = os.path.getmtime(last_file) else: modify_time = 0 logger.error("last file in meta: " + last_file + " not exist and set modify_time to 0") else: logger.error("no last files in meta: " + last_file + " and set modify_time to 0") self.scanner = DirScanner(self.base_path, self.tmp_path, modify_time, self.path_regx, self.q, history_files) return meta_info def run(self): self.stopped = False while not self.stopFlag: try: self.__process() except: logger.error("abnormal error happened, exit this thread") break logger.debug("one file done") if len(self.q) <= 3: self.stopevent.wait(self.sleeptime) self.stopevent.clear() self.stopped = True def join(self, timeout=None): self.stopFlag = True self.stopevent.set() while not self.stopped: logger.error("waiting for stopped") time.sleep(1) self.meta.flush(self.meta_info) self.meta.close() self.sender.close() threading.Thread.join(self, timeout) def __process(self): self.scanner.scan() if len(self.q) == 0: return try: filename = self.q[0] to_read = open(filename, "rb") if self.tmp_path != "null" and filename == self.tmp_path: self.scanner.scan(True) new_filename = self.q[0] while filename != new_filename: filename = new_filename to_read.close() to_read = open(filename, "rb") self.scanner.scan(True) new_filename = self.q[0] except IOError, e: logger.error(str(e)) logger.error(traceback.format_exc()) if filename != self.tmp_path and len(self.q) > 1: logger.error("CHECK_" + self.cp_name + ": file: " + filename + " no exist and skip it") self.__move_file() return self.meta_info.cur_file = filename self.meta.flush(self.meta_info) try: while self.stopFlag is False: self.offset = self.__handle_one_file(filename, to_read, self.offset) self.scanner.scan() if len(self.q) > 1: self.offset = self.__handle_one_file(filename, to_read, self.offset) to_read.close() self.__move_file() break sleep(0.1) continue except Exception, e: logger.error(str(e)) logger.error(traceback.format_exc()) to_read.close() raise e
class Tail(Thread): def __init__(self, index, conf): self.__load_conf(index, conf) self.q = [] self.meta = Meta(MetaUrl(self.cp_path, self.cp_name)) self.meta.startRecoding() self.meta_info = self.__load_from_meta() self.__check_meta() self.__init_sender() self.stopFlag = False self.stopevent = threading.Event() self.sleeptime = 0.1 threading.Thread.__init__(self, name="thread_tail_" + self.topic) def __init_sender(self): if self.local_test_path != "null": self.sender = DiskSender(self.local_test_path, self.topic, self.encoding, self.max_buf_size) else: self.sender = TTSender(self.max_buf_size, self.topic, self.topic_timeout, self.encoding, self.topic_compress, self.topic_sequence) def __check_meta(self): self.scanner.scan() if len(self.q) == 0: return if self.tmp_path == "null": if self.meta_info.cur_file != "null" and self.meta_info.cur_file != self.q[ 0]: logger.error("cur file in meta: " + self.meta_info.cur_file + " not match: " + self.q[0]) self.offset = 0L return def __load_conf(self, i, c): self.path_regx = c.get_path_regx().split(",")[i] self.base_path = c.get_base_path().split(",")[i] tmp_paths = c.get_tmp_path() if tmp_paths == "null": self.tmp_path = "null" else: tmpath = c.get_tmp_path().split(",")[i] if tmpath is "" or tmpath is "None": self.tmp_path = "null" else: self.tmp_path = tmpath self.cp_path = c.get_cp_path() self.cp_name = c.get_cp_name().split(",")[i] self.local_test_path = c.get_test_path() self.topic = c.get_topic_name().split(",")[i] self.encoding = c.get_topic_encoding().split(",")[i] self.topic_timeout = c.get_topic_timeout().split(",")[i] self.topic_compress = c.get_topic_compress().split(",")[i] self.topic_sequence = c.get_topic_sequence().split(",")[i] self.max_buf_size = c.get_max_buf_size() pass def __load_from_meta(self): meta_info = self.meta.restore() self.offset = meta_info.offset history_files = meta_info.last_file.split(",") modify_time = 0 len_lasts = len(history_files) if len_lasts != 0: last_file = history_files[len_lasts - 1] if os.path.exists(last_file) is True: modify_time = os.path.getmtime(last_file) else: modify_time = 0 logger.error("last file in meta: " + last_file + " not exist and set modify_time to 0") else: logger.error("no last files in meta: " + last_file + " and set modify_time to 0") self.scanner = DirScanner(self.base_path, self.tmp_path, modify_time, self.path_regx, self.q, history_files) return meta_info def run(self): self.stopped = False while not self.stopFlag: try: self.__process() except: logger.error("abnormal error happened, exit this thread") break logger.debug("one file done") if len(self.q) <= 3: self.stopevent.wait(self.sleeptime) self.stopevent.clear() self.stopped = True def join(self, timeout=None): self.stopFlag = True self.stopevent.set() while not self.stopped: logger.error("waiting for stopped") time.sleep(1) self.meta.flush(self.meta_info) self.meta.close() self.sender.close() threading.Thread.join(self, timeout) def __process(self): self.scanner.scan() if len(self.q) == 0: return try: filename = self.q[0] to_read = open(filename, "rb") if self.tmp_path != "null" and filename == self.tmp_path: self.scanner.scan(True) new_filename = self.q[0] while filename != new_filename: filename = new_filename to_read.close() to_read = open(filename, "rb") self.scanner.scan(True) new_filename = self.q[0] except IOError, e: logger.error(str(e)) logger.error(traceback.format_exc()) if filename != self.tmp_path and len(self.q) > 1: logger.error("CHECK_" + self.cp_name + ": file: " + filename + " no exist and skip it") self.__move_file() return self.meta_info.cur_file = filename self.meta.flush(self.meta_info) try: while self.stopFlag is False: self.offset = self.__handle_one_file(filename, to_read, self.offset) self.scanner.scan() if len(self.q) > 1: self.offset = self.__handle_one_file( filename, to_read, self.offset) to_read.close() self.__move_file() break sleep(0.1) continue except Exception, e: logger.error(str(e)) logger.error(traceback.format_exc()) to_read.close() raise e