Esempio n. 1
0
def process_line(msg, ld, lp, ha, isnew_check = False, latest = None):
    """Add a log message to DB.
    
    Args:
        msg (str): A log message to process.
            Line feed code will be ignored.
        ld (LogData): An log database interface opened in edit mode.
            Needs to initialize template classifier with ld.init_ltmanager.
        lp (logparser.LogParser): An open message parser.
        latest (Optional[datetime.datetime]): If not None,
            Ignore messages that have later timestamp than 'latest'.

    Returns:
        LogMessage: An annotated log message instance.
            Same as lines given with LogData.iterlines.
    """
    line = None

    dt, org_host, l_w, l_s = lp.process_line(msg)
    if latest is not None and dt < latest: return None
    if l_w is None: return None
    l_w = [strutil.add_esc(w) for w in l_w]
    host = ha.resolve_host(org_host)
    if host is None: host = org_host

    _logger.debug("Processing [{0}]".format(" ".join(l_w)))
    ltline = ld.ltm.process_line(l_w, l_s)
    if ltline is None:
        ld.ltm.failure_output(msg)
    else:
        _logger.debug("Template [{0}]".format(ltline))
        line = ld.add_line(ltline.ltid, dt, host, l_w)
    return line
Esempio n. 2
0
def generate_lt_from_file(conf, fn):
    _logger.info("job for ({0}) start".format(fn))

    import logparser
    lp = logparser.LogParser(conf)
    table = lt_common.TemplateTable()
    sym = conf.get("log_template", "variable_symbol")
    d_symlist = {}
    ltgen = LTGenCRF(table, sym, conf)

    with open(fn, "r") as f:
        for line in f:
            dt, org_host, l_w, l_s = lp.process_line(line)
            if l_w is None: continue
            l_w = [strutil.add_esc(w) for w in l_w]
            tid, dummy = ltgen.process_line(l_w, l_s)
            d_symlist[tid] = l_s

    ret = []
    for tid in table.tids():
        tpl = table.get_template(tid)
        l_s = d_symlist[tid]
        ret.append((tpl, l_s))

    _logger.info("job for ({0}) done".format(fn))
    return ret
Esempio n. 3
0
def generate_lt_from_file(conf, fn):
    _logger.info("job for ({0}) start".format(fn))
    
    import logparser
    lp = logparser.LogParser(conf)
    table = lt_common.TemplateTable()
    sym = conf.get("log_template", "variable_symbol")
    d_symlist = {}
    ltgen = LTGenCRF(table, sym, conf)
    
    with open(fn, "r") as f:
        for line in f:
            dt, org_host, l_w, l_s = lp.process_line(line)
            if l_w is None: continue
            l_w = [strutil.add_esc(w) for w in l_w]
            tid, dummy = ltgen.process_line(l_w, l_s)
            d_symlist[tid] = l_s

    ret = []
    for tid in table.tids():
        tpl = table.get_template(tid)
        l_s = d_symlist[tid]
        ret.append((tpl, l_s))

    _logger.info("job for ({0}) done".format(fn))
    return ret
Esempio n. 4
0
def process_init_data(conf, targets, isnew_check = False):
    """Add log messages to DB from files. This function do NOT process
    messages incrementally. Use this to avoid bad-start problem of
    log template generation with clustering or training methods.

    Note:
        This function needs large memory space.

    Args:
        conf (config.ExtendedConfigParser): A common configuration object.
        targets (List[str]): A sequence of filepaths to process.
        isnew_check (Optional[bool]): If True, add message to DB
            only if its timestamp is newest of existing messages in DB.

    Raises:
        IOError: If a file in targets not found.
    """
    ld = LogData(conf, edit = True, reset_db = True)
    ld.init_ltmanager()
    lp = logparser.LogParser(conf)
    ha = host_alias.HostAlias(conf)
    latest = ld.dt_term()[1] if isnew_check else None
    drop_undefhost = conf.getboolean("database", "undefined_host")

    l_line = []
    l_data = []
    for line in _iter_line_from_files(targets):
        dt, org_host, l_w, l_s = lp.process_line(line)
        if latest is not None and dt < latest: continue
        if l_w is None: continue
        l_w = [strutil.add_esc(w) for w in l_w]
        host = ha.resolve_host(org_host)
        if host is None:
            if drop_undefhost:
                ld.ltm.failure_output(msg)
                return None
            else:
                host = org_host

        l_line.append((l_w, l_s))
        l_data.append((dt, host))

    for ltline, line, data in zip(ld.ltm.process_init_data(l_line),
                                  l_line, l_data):
        l_w, l_s = line
        dt, host = data
        ld.add_line(ltline.ltid, dt, host, l_w)

    ld.commit_db()
Esempio n. 5
0
def process_line(msg, ld, lp, ha, isnew_check = False, latest = None,
            drop_undefhost = False):
    """Add a log message to DB.
    
    Args:
        msg (str): A log message to process.
            Line feed code will be ignored.
        ld (LogData): An log database interface opened in edit mode.
            Needs to initialize template classifier with ld.init_ltmanager.
        lp (logparser.LogParser): An open message parser.
        latest (Optional[datetime.datetime]): If not None,
            Ignore messages that have later timestamp than 'latest'.

    Returns:
        LogMessage: An annotated log message instance.
            Same as lines given with LogData.iterlines.
    """
    line = None

    dt, org_host, l_w, l_s = lp.process_line(msg)
    if latest is not None and dt < latest: return None
    if l_w is None: return None
    l_w = [strutil.add_esc(w) for w in l_w]
    host = ha.resolve_host(org_host)
    #if host is None: host = org_host
    if host is None:
        #if conf.getboolean("database", "undefined_host"):
        if drop_undefhost:
            ld.ltm.failure_output(msg)
            return None
        else:
            host = org_host

    _logger.debug("Processing [{0}]".format(" ".join(l_w)))
    ltline = ld.ltm.process_line(l_w, l_s)
    if ltline is None:
        ld.ltm.failure_output(msg)
        return None
    else:
        _logger.debug("Template [{0}]".format(ltline))
        line = ld.add_line(ltline.ltid, dt, host, l_w)
    return line
Esempio n. 6
0
def process_init_data(conf, targets, isnew_check = False):
    """Add log messages to DB from files. This function do NOT process
    messages incrementally. Use this to avoid bad-start problem of
    log template generation with clustering or training methods.

    Note:
        This function needs large memory space.

    Args:
        conf (config.ExtendedConfigParser): A common configuration object.
        targets (List[str]): A sequence of filepaths to process.
        isnew_check (Optional[bool]): If True, add message to DB
            only if its timestamp is newest of existing messages in DB.

    Raises:
        IOError: If a file in targets not found.
    """
    ld = LogData(conf, edit = True, reset_db = True)
    ld.init_ltmanager()
    lp = logparser.LogParser(conf)
    ha = host_alias.HostAlias(conf)
    latest = ld.dt_term()[1] if isnew_check else None

    l_line = []
    l_data = []
    for line in _iter_line_from_files(targets):
        dt, org_host, l_w, l_s = lp.process_line(line)
        if latest is not None and dt < latest: continue
        if l_w is None: continue
        l_w = [strutil.add_esc(w) for w in l_w]
        host = ha.resolve_host(org_host)
        if host is None: host = org_host
        l_line.append((l_w, l_s))
        l_data.append((dt, host))

    for ltline, line, data in zip(ld.ltm.process_init_data(l_line),
                                  l_line, l_data):
        l_w, l_s = line
        dt, host = data
        ld.add_line(ltline.ltid, dt, host, l_w)

    ld.commit_db()
Esempio n. 7
0
def search_exception(conf, targets):
    
    import logparser
    import strutil
    lp = logparser.LogParser(conf)
    def_path = conf.get("log_template_import", "def_path")
    sym = conf.get("log_template", "variable_symbol")
    mode = conf.get("log_template_import", "mode")
    table = lt_common.TemplateTable()
    temp_lp = logparser.LogParser(conf, sep_variable = True)
    ltgen = LTGenImport(table, sym, def_path, mode, temp_lp)

    for fn in targets:
        _logger.info("lt_import job for ({0}) start".format(fn))
        with open(fn, "r") as f:
            for line in f:
                dt, org_host, l_w, l_s = lp.process_line(line)
                if l_w is None: continue
                l_w = [strutil.add_esc(w) for w in l_w]
                tid, dummy = ltgen.process_line(l_w, l_s)
                if tid is None:
                    print line.rstrip("\n")
        _logger.info("lt_import job for ({0}) done".format(fn))
Esempio n. 8
0
 def _key_template(self, template):
     l_word = [strutil.add_esc(w) for w in template]
     return "@".join(l_word)
Esempio n. 9
0
 def _key_template(self, template):
     l_word = [strutil.add_esc(w) for w in template]
     return "@".join(l_word)