def parse_cmd(args):
logging.debug("Running command parse")
if len(args) < 1:
usage()
dest_filename = args[-1]
if os.path.isfile(dest_filename):
usage()
sources = list(set(args[:-1]))
engine = DefaultEngine()
dest_workset = workset.WorkSet()
dest_workset.metadata["parse"] = "parse"
dest_workset.metadata["file_name"] = dest_filename
dest_workset.metadata["parse_source_files"] = sources
for i in sources:
mmap_file = util.open_mmap(i)
while 1:
line = mmap_file.readline()
if not line:
break
duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line)
dest_workset.hit(workset.cleanup_url(url), duration, node_name, http_code)
engine.workset_manager.save(dest_workset, dest_filename)
def update_current_logfile(self, logfile, logfilename):
new_size = os.stat(self.config.zxtm_root + '/' + logfilename)[stat.ST_SIZE]
prev_size = logfile.size
if (new_size == 0) or (new_size == prev_size):
logging.debug("SyncEngine: no new data in current logfile [%s]", logfilename)
if logfile.time_period < self.current_time_period:
logging.debug("SyncEngine: logfile [%s] is done, marking as closable", logfilename)
logfile.closable = True
return
logging.debug("SyncEngine: current logfile [%s] has new data, skipping %d", logfilename, prev_size)
mmap_file = util.open_mmap(self.config.zxtm_root + '/' + logfilename)
mmap_file.seek(prev_size)
# look for possible services for this vserver
possible_services = self.get_possible_services(logfile.zxtm_vserver)
while 1:
line = mmap_file.readline()
if not line:
break
duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line)
svc = self.find_svc_for_host(possible_services, host)
if svc is None:
continue
app = self.find_app_for_url(svc, url)
ws = self.get_workset(logfile.time_period, svc.svc_id, app)
# should we ignore this hit ?
if svc.ignore_re.search(url) is not None:
ws.ignore_hit(url, duration, node_name, http_code)
else:
clean_url = workset.cleanup_url(url, svc.keep_params_re)
ws.hit(clean_url, duration, node_name, http_code)
if (app in svc.munin_apps):
self.get_munin_workset(svc.svc_id, app).hit(clean_url, duration, node_name, http_code)
logfile.size = new_size
