def parse_cmd(args): logging.debug("Running command parse") if len(args) < 1: usage() dest_filename = args[-1] if os.path.isfile(dest_filename): usage() sources = list(set(args[:-1])) engine = DefaultEngine() dest_workset = workset.WorkSet() dest_workset.metadata["parse"] = "parse" dest_workset.metadata["file_name"] = dest_filename dest_workset.metadata["parse_source_files"] = sources for i in sources: mmap_file = util.open_mmap(i) while 1: line = mmap_file.readline() if not line: break duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line) dest_workset.hit(workset.cleanup_url(url), duration, node_name, http_code) engine.workset_manager.save(dest_workset, dest_filename)
def update_current_logfile(self, logfile, logfilename): new_size = os.stat(self.config.zxtm_root + '/' + logfilename)[stat.ST_SIZE] prev_size = logfile.size if (new_size == 0) or (new_size == prev_size): logging.debug("SyncEngine: no new data in current logfile [%s]", logfilename) if logfile.time_period < self.current_time_period: logging.debug("SyncEngine: logfile [%s] is done, marking as closable", logfilename) logfile.closable = True return logging.debug("SyncEngine: current logfile [%s] has new data, skipping %d", logfilename, prev_size) mmap_file = util.open_mmap(self.config.zxtm_root + '/' + logfilename) mmap_file.seek(prev_size) # look for possible services for this vserver possible_services = self.get_possible_services(logfile.zxtm_vserver) while 1: line = mmap_file.readline() if not line: break duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line) svc = self.find_svc_for_host(possible_services, host) if svc is None: continue app = self.find_app_for_url(svc, url) ws = self.get_workset(logfile.time_period, svc.svc_id, app) # should we ignore this hit ? if svc.ignore_re.search(url) is not None: ws.ignore_hit(url, duration, node_name, http_code) else: clean_url = workset.cleanup_url(url, svc.keep_params_re) ws.hit(clean_url, duration, node_name, http_code) if (app in svc.munin_apps): self.get_munin_workset(svc.svc_id, app).hit(clean_url, duration, node_name, http_code) logfile.size = new_size