def run(self, options): stdout = options.stdout newfilename = options.newfile if options.newfile == options.oldfile: print >>stdout, "modifying event file in place" newfilename = newfilename + ".tmp" if options.newfile.endswith(".bz2"): newfile = bz2.BZ2File(newfilename, "w") else: newfile = open(newfilename, "wb") newfile.write(flogfile.MAGIC) after = options['after'] if after is not None: print >>stdout, " --after: removing events before %s" % time.ctime(after) before = options['before'] if before is not None: print >>stdout, " --before: removing events after %s" % time.ctime(before) above = options['above'] if above: print >>stdout, " --above: removing events below level %d" % above from_tubid = options['from'] if from_tubid: print >>stdout, " --from: retaining events only from tubid prefix %s" % from_tubid strip_facility = options['strip-facility'] if strip_facility is not None: print >>stdout, "--strip-facility: removing events for %s and children" % strip_facility total = 0 copied = 0 for e in flogfile.get_events(options.oldfile): if options['verbose']: if "d" in e: print >>stdout, e['d']['num'] else: print >>stdout, "HEADER" total += 1 if "d" in e: if before is not None and e['d']['time'] >= before: continue if after is not None and e['d']['time'] <= after: continue if above is not None and e['d']['level'] < above: continue if from_tubid is not None and not e['from'].startswith(from_tubid): continue if (strip_facility is not None and e['d'].get('facility', "").startswith(strip_facility)): continue copied += 1 flogfile.serialize_raw_wrapper(newfile, e) newfile.close() if options.newfile == options.oldfile: if sys.platform == "win32": # Win32 can't do an atomic rename to an existing file. try: os.unlink(options.newfile) except OSError: pass move_into_place(newfilename, options.newfile) print >>stdout, "copied %d of %d events into new file" % (copied, total)
def get_incident_trigger(self, abs_fn): events = flogfile.get_events(abs_fn) try: header = next(iter(events)) except (EOFError, ValueError): return None assert header["header"]["type"] == "incident" trigger = header["header"]["trigger"] return trigger
def run(self, options): try: for e in flogfile.get_events(options.dumpfile): if "header" in e: self.print_header(e, options) if "d" in e: self.print_event(e, options) except EnvironmentError, e: # "flogtool dump FLOGFILE |less" is very common, and if you quit # it early with "q", the stdout pipe is broken and python dies # with a messy stacktrace. Catch and ignore that. if e.errno == errno.EPIPE: return 1 raise
def remote_get_incident(self, name): if not name.startswith("incident"): raise KeyError("bad incident name %s" % name) incident_dir = filepath.FilePath(self._logger.logdir) abs_fn = incident_dir.child(name).path + ".flog" try: fn = abs_fn + ".bz2" if not os.path.exists(fn): fn = abs_fn events = flogfile.get_events(fn) # note the generator isn't actually cycled yet, not until next() header = _keys_to_bytes(next(events)["header"]) except EnvironmentError: raise KeyError("no incident named %s" % name) wrapped_events = [_keys_to_bytes(event["d"]) for event in events] return (header, wrapped_events)
def run(self, options): try: for e in flogfile.get_events(options.dumpfile): if "header" in e: self.print_header(e, options) if "d" in e: self.print_event(e, options) except EnvironmentError as e: # "flogtool dump FLOGFILE |less" is very common, and if you quit # it early with "q", the stdout pipe is broken and python dies # with a messy stacktrace. Catch and ignore that. if e.errno == errno.EPIPE: return 1 raise except flogfile.ThisIsActuallyAFurlFileError: print(textwrap.dedent("""\ Error: %s appears to be a FURL file. Perhaps you meant to run 'flogtool tail' instead of 'flogtool dump'?""" % (options.dumpfile, )), file=options.stderr) return 1 except flogfile.EvilPickleFlogFile: print(textwrap.dedent("""\ Error: %s appears to be an old-style (pickle-based) flogfile, which cannot be loaded safely. If you wish to allow the author of the flogfile to take over your computer (and incidentally allow you to view the content), please use the flogtool from a copy of foolscap-0.12.7 or earlier.""" % (options.dumpfile, )), file=options.stderr) return 1 except flogfile.BadMagic as e: print(textwrap.dedent("""\ Error: %s does not appear to be a flogfile. """ % (options.dumpfile, )), file=options.stderr) return 1 except ValueError as ex: print("truncated pickle file? (%s): %s" % (options.dumpfile, ex), file=options.stderr) return 1
def run(self, options): stdout = options.stdout newfilename = options.newfile if options.newfile == options.oldfile: print("modifying event file in place", file=stdout) newfilename = newfilename + ".tmp" if options.newfile.endswith(".bz2"): newfile = bz2.BZ2File(newfilename, "w") else: newfile = open(newfilename, "wb") newfile.write(flogfile.MAGIC) after = options['after'] if after is not None: print(" --after: removing events before %s" % time.ctime(after), file=stdout) before = options['before'] if before is not None: print(" --before: removing events after %s" % time.ctime(before), file=stdout) above = options['above'] if above: print(" --above: removing events below level %d" % above, file=stdout) from_tubid = options['from'] if from_tubid: print(" --from: retaining events only from tubid prefix %s" % from_tubid, file=stdout) strip_facility = options['strip-facility'] if strip_facility is not None: print("--strip-facility: removing events for %s and children" % strip_facility, file=stdout) total = 0 copied = 0 for e in flogfile.get_events(options.oldfile): if options['verbose']: if "d" in e: print(e['d']['num'], file=stdout) else: print("HEADER", file=stdout) total += 1 if "d" in e: if before is not None and e['d']['time'] >= before: continue if after is not None and e['d']['time'] <= after: continue if above is not None and e['d']['level'] < above: continue if from_tubid is not None and not e['from'].startswith( from_tubid): continue if (strip_facility is not None and e['d'].get( 'facility', "").startswith(strip_facility)): continue copied += 1 flogfile.serialize_raw_wrapper(newfile, e) newfile.close() if options.newfile == options.oldfile: if sys.platform == "win32": # Win32 can't do an atomic rename to an existing file. try: os.unlink(options.newfile) except OSError: pass move_into_place(newfilename, options.newfile) print("copied %d of %d events into new file" % (copied, total), file=stdout)
def load_incident(self, abs_fn): assert abs_fn.endswith(".bz2") events = flogfile.get_events(abs_fn, ignore_value_error=True) header = next(events)["header"] wrapped_events = [event["d"] for event in events] return (header, wrapped_events)
def process_logfiles(self, logfiles): summaries = {} # build up a tree of events based upon parent/child relationships number_map = {} roots = [] trigger_numbers = [] first_event_from = None for lf in logfiles: (first_event_number, first_event_time) = (None, None) (last_event_number, last_event_time) = (None, None) num_events = 0 levels = {} pid = None for e in flogfile.get_events(lf): if "header" in e: h = e["header"] if h["type"] == "incident": t = h["trigger"] trigger_numbers.append(t["num"]) pid = h.get("pid") versions = h.get("versions", {}) if "d" not in e: continue # skip headers if not first_event_from: first_event_from = e['from'] le = LogEvent(e) if le.index: number_map[le.index] = le if le.parent_index in number_map: le.parent = number_map[le.parent_index] le.parent.children.append(le) else: roots.append(le) d = e['d'] level = d.get("level", "NORMAL") number = d.get("num", None) when = d.get("time") if number in trigger_numbers: le.is_trigger = True if False: # this is only meaningful if the logfile contains events # from just a single tub and incarnation, but our current # LogGatherer combines multiple processes' logs into a # single file. if first_event_number is None: first_event_number = number elif number is not None: first_event_number = min(first_event_number, number) if last_event_number is None: last_event_number = number elif number is not None: last_event_number = max(last_event_number, number) if first_event_time is None: first_event_time = when elif when is not None: first_event_time = min(first_event_time, when) if last_event_time is None: last_event_time = when elif when is not None: last_event_time = max(last_event_time, when) num_events += 1 if level not in levels: levels[level] = [] levels[level].append(le) summary = ( (first_event_number, first_event_time), (last_event_number, last_event_time), num_events, levels, pid, versions ) summaries[lf] = summary triggers = [(first_event_from, num) for num in trigger_numbers] return summaries, roots, number_map, triggers