Esempio n. 1
0
def parse_cmd(args):
    logging.debug("Running command parse")
    if len(args) < 1:
        usage()

    dest_filename = args[-1]
    if os.path.isfile(dest_filename):
        usage()
        
    sources = list(set(args[:-1]))

    engine = DefaultEngine()

    dest_workset = workset.WorkSet()
    dest_workset.metadata["parse"] = "parse"
    dest_workset.metadata["file_name"] = dest_filename
    dest_workset.metadata["parse_source_files"] = sources
    
    for i in sources:
        mmap_file = util.open_mmap(i)
        while 1:
            line = mmap_file.readline()
            if not line:
                break
            duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line)
            dest_workset.hit(workset.cleanup_url(url), duration, node_name, http_code)

    engine.workset_manager.save(dest_workset, dest_filename)
Esempio n. 2
0
File: sync.py Progetto: stfp/qostool
    def update_current_logfile(self, logfile, logfilename):
        new_size = os.stat(self.config.zxtm_root + '/' + logfilename)[stat.ST_SIZE]
        prev_size = logfile.size
        if (new_size == 0) or (new_size == prev_size):
            logging.debug("SyncEngine: no new data in current logfile [%s]", logfilename)
            
            if logfile.time_period < self.current_time_period:
                logging.debug("SyncEngine: logfile [%s] is done, marking as closable", logfilename)
                logfile.closable = True
            return

        logging.debug("SyncEngine: current logfile [%s] has new data, skipping %d", logfilename, prev_size)

        
        mmap_file = util.open_mmap(self.config.zxtm_root + '/' + logfilename)
        mmap_file.seek(prev_size)

        # look for possible services for this vserver
        possible_services = self.get_possible_services(logfile.zxtm_vserver)

        while 1:
            line = mmap_file.readline()
            if not line:
                break

            duration, host, url, node_name, http_code = zxtm.parse_zxtm_log_line(line)
            svc = self.find_svc_for_host(possible_services, host)
            if svc is None:
                continue

            app = self.find_app_for_url(svc, url)
            ws = self.get_workset(logfile.time_period, svc.svc_id, app)

            # should we ignore this hit ?
            if svc.ignore_re.search(url) is not None:
                ws.ignore_hit(url, duration, node_name, http_code)
            else:
                clean_url = workset.cleanup_url(url, svc.keep_params_re)
                ws.hit(clean_url, duration, node_name, http_code)
                if (app in svc.munin_apps):
                    self.get_munin_workset(svc.svc_id, app).hit(clean_url, duration, node_name, http_code)

        logfile.size = new_size