def run(self, options):
stdout = options.stdout
newfilename = options.newfile
if options.newfile == options.oldfile:
print >>stdout, "modifying event file in place"
newfilename = newfilename + ".tmp"
if options.newfile.endswith(".bz2"):
newfile = bz2.BZ2File(newfilename, "w")
else:
newfile = open(newfilename, "wb")
newfile.write(flogfile.MAGIC)
after = options['after']
if after is not None:
print >>stdout, " --after: removing events before %s" % time.ctime(after)
before = options['before']
if before is not None:
print >>stdout, " --before: removing events after %s" % time.ctime(before)
above = options['above']
if above:
print >>stdout, " --above: removing events below level %d" % above
from_tubid = options['from']
if from_tubid:
print >>stdout, " --from: retaining events only from tubid prefix %s" % from_tubid
strip_facility = options['strip-facility']
if strip_facility is not None:
print >>stdout, "--strip-facility: removing events for %s and children" % strip_facility
total = 0
copied = 0
for e in flogfile.get_events(options.oldfile):
if options['verbose']:
if "d" in e:
print >>stdout, e['d']['num']
else:
print >>stdout, "HEADER"
total += 1
if "d" in e:
if before is not None and e['d']['time'] >= before:
continue
if after is not None and e['d']['time'] <= after:
continue
if above is not None and e['d']['level'] < above:
continue
if from_tubid is not None and not e['from'].startswith(from_tubid):
continue
if (strip_facility is not None
and e['d'].get('facility', "").startswith(strip_facility)):
continue
copied += 1
flogfile.serialize_raw_wrapper(newfile, e)
newfile.close()
if options.newfile == options.oldfile:
if sys.platform == "win32":
# Win32 can't do an atomic rename to an existing file.
try:
os.unlink(options.newfile)
except OSError:
pass
move_into_place(newfilename, options.newfile)
print >>stdout, "copied %d of %d events into new file" % (copied, total)
def get_incident_trigger(self, abs_fn):
events = flogfile.get_events(abs_fn)
try:
header = next(iter(events))
except (EOFError, ValueError):
return None
assert header["header"]["type"] == "incident"
trigger = header["header"]["trigger"]
return trigger
def get_incident_trigger(self, abs_fn):
events = flogfile.get_events(abs_fn)
try:
header = next(iter(events))
except (EOFError, ValueError):
return None
assert header["header"]["type"] == "incident"
trigger = header["header"]["trigger"]
return trigger
def run(self, options):
try:
for e in flogfile.get_events(options.dumpfile):
if "header" in e:
self.print_header(e, options)
if "d" in e:
self.print_event(e, options)
except EnvironmentError, e:
# "flogtool dump FLOGFILE |less" is very common, and if you quit
# it early with "q", the stdout pipe is broken and python dies
# with a messy stacktrace. Catch and ignore that.
if e.errno == errno.EPIPE:
return 1
raise
def run(self, options):
try:
for e in flogfile.get_events(options.dumpfile):
if "header" in e:
self.print_header(e, options)
if "d" in e:
self.print_event(e, options)
except EnvironmentError, e:
# "flogtool dump FLOGFILE |less" is very common, and if you quit
# it early with "q", the stdout pipe is broken and python dies
# with a messy stacktrace. Catch and ignore that.
if e.errno == errno.EPIPE:
return 1
raise
def remote_get_incident(self, name):
if not name.startswith("incident"):
raise KeyError("bad incident name %s" % name)
incident_dir = filepath.FilePath(self._logger.logdir)
abs_fn = incident_dir.child(name).path + ".flog"
try:
fn = abs_fn + ".bz2"
if not os.path.exists(fn):
fn = abs_fn
events = flogfile.get_events(fn)
# note the generator isn't actually cycled yet, not until next()
header = _keys_to_bytes(next(events)["header"])
except EnvironmentError:
raise KeyError("no incident named %s" % name)
wrapped_events = [_keys_to_bytes(event["d"]) for event in events]
return (header, wrapped_events)
def remote_get_incident(self, name):
if not name.startswith("incident"):
raise KeyError("bad incident name %s" % name)
incident_dir = filepath.FilePath(self._logger.logdir)
abs_fn = incident_dir.child(name).path + ".flog"
try:
fn = abs_fn + ".bz2"
if not os.path.exists(fn):
fn = abs_fn
events = flogfile.get_events(fn)
# note the generator isn't actually cycled yet, not until next()
header = _keys_to_bytes(next(events)["header"])
except EnvironmentError:
raise KeyError("no incident named %s" % name)
wrapped_events = [_keys_to_bytes(event["d"]) for event in events]
return (header, wrapped_events)
def run(self, options):
try:
for e in flogfile.get_events(options.dumpfile):
if "header" in e:
self.print_header(e, options)
if "d" in e:
self.print_event(e, options)
except EnvironmentError as e:
# "flogtool dump FLOGFILE |less" is very common, and if you quit
# it early with "q", the stdout pipe is broken and python dies
# with a messy stacktrace. Catch and ignore that.
if e.errno == errno.EPIPE:
return 1
raise
except flogfile.ThisIsActuallyAFurlFileError:
print(textwrap.dedent("""\
Error: %s appears to be a FURL file.
Perhaps you meant to run 'flogtool tail' instead of 'flogtool dump'?"""
% (options.dumpfile, )),
file=options.stderr)
return 1
except flogfile.EvilPickleFlogFile:
print(textwrap.dedent("""\
Error: %s appears to be an old-style
(pickle-based) flogfile, which cannot be loaded safely. If you
wish to allow the author of the flogfile to take over your
computer (and incidentally allow you to view the content), please
use the flogtool from a copy of foolscap-0.12.7 or earlier.""" %
(options.dumpfile, )),
file=options.stderr)
return 1
except flogfile.BadMagic as e:
print(textwrap.dedent("""\
Error: %s does not appear to be a flogfile.
""" % (options.dumpfile, )),
file=options.stderr)
return 1
except ValueError as ex:
print("truncated pickle file? (%s): %s" % (options.dumpfile, ex),
file=options.stderr)
return 1
def run(self, options):
stdout = options.stdout
newfilename = options.newfile
if options.newfile == options.oldfile:
print("modifying event file in place", file=stdout)
newfilename = newfilename + ".tmp"
if options.newfile.endswith(".bz2"):
newfile = bz2.BZ2File(newfilename, "w")
else:
newfile = open(newfilename, "wb")
newfile.write(flogfile.MAGIC)
after = options['after']
if after is not None:
print(" --after: removing events before %s" % time.ctime(after),
file=stdout)
before = options['before']
if before is not None:
print(" --before: removing events after %s" % time.ctime(before),
file=stdout)
above = options['above']
if above:
print(" --above: removing events below level %d" % above,
file=stdout)
from_tubid = options['from']
if from_tubid:
print(" --from: retaining events only from tubid prefix %s" %
from_tubid,
file=stdout)
strip_facility = options['strip-facility']
if strip_facility is not None:
print("--strip-facility: removing events for %s and children" %
strip_facility,
file=stdout)
total = 0
copied = 0
for e in flogfile.get_events(options.oldfile):
if options['verbose']:
if "d" in e:
print(e['d']['num'], file=stdout)
else:
print("HEADER", file=stdout)
total += 1
if "d" in e:
if before is not None and e['d']['time'] >= before:
continue
if after is not None and e['d']['time'] <= after:
continue
if above is not None and e['d']['level'] < above:
continue
if from_tubid is not None and not e['from'].startswith(
from_tubid):
continue
if (strip_facility is not None and e['d'].get(
'facility', "").startswith(strip_facility)):
continue
copied += 1
flogfile.serialize_raw_wrapper(newfile, e)
newfile.close()
if options.newfile == options.oldfile:
if sys.platform == "win32":
# Win32 can't do an atomic rename to an existing file.
try:
os.unlink(options.newfile)
except OSError:
pass
move_into_place(newfilename, options.newfile)
print("copied %d of %d events into new file" % (copied, total),
file=stdout)
def load_incident(self, abs_fn):
assert abs_fn.endswith(".bz2")
events = flogfile.get_events(abs_fn, ignore_value_error=True)
header = next(events)["header"]
wrapped_events = [event["d"] for event in events]
return (header, wrapped_events)
def process_logfiles(self, logfiles):
summaries = {}
# build up a tree of events based upon parent/child relationships
number_map = {}
roots = []
trigger_numbers = []
first_event_from = None
for lf in logfiles:
(first_event_number, first_event_time) = (None, None)
(last_event_number, last_event_time) = (None, None)
num_events = 0
levels = {}
pid = None
for e in flogfile.get_events(lf):
if "header" in e:
h = e["header"]
if h["type"] == "incident":
t = h["trigger"]
trigger_numbers.append(t["num"])
pid = h.get("pid")
versions = h.get("versions", {})
if "d" not in e:
continue # skip headers
if not first_event_from:
first_event_from = e['from']
le = LogEvent(e)
if le.index:
number_map[le.index] = le
if le.parent_index in number_map:
le.parent = number_map[le.parent_index]
le.parent.children.append(le)
else:
roots.append(le)
d = e['d']
level = d.get("level", "NORMAL")
number = d.get("num", None)
when = d.get("time")
if number in trigger_numbers:
le.is_trigger = True
if False:
# this is only meaningful if the logfile contains events
# from just a single tub and incarnation, but our current
# LogGatherer combines multiple processes' logs into a
# single file.
if first_event_number is None:
first_event_number = number
elif number is not None:
first_event_number = min(first_event_number, number)
if last_event_number is None:
last_event_number = number
elif number is not None:
last_event_number = max(last_event_number, number)
if first_event_time is None:
first_event_time = when
elif when is not None:
first_event_time = min(first_event_time, when)
if last_event_time is None:
last_event_time = when
elif when is not None:
last_event_time = max(last_event_time, when)
num_events += 1
if level not in levels:
levels[level] = []
levels[level].append(le)
summary = ( (first_event_number, first_event_time),
(last_event_number, last_event_time),
num_events, levels, pid, versions )
summaries[lf] = summary
triggers = [(first_event_from, num) for num in trigger_numbers]
return summaries, roots, number_map, triggers
def process_logfiles(self, logfiles):
summaries = {}
# build up a tree of events based upon parent/child relationships
number_map = {}
roots = []
trigger_numbers = []
first_event_from = None
for lf in logfiles:
(first_event_number, first_event_time) = (None, None)
(last_event_number, last_event_time) = (None, None)
num_events = 0
levels = {}
pid = None
for e in flogfile.get_events(lf):
if "header" in e:
h = e["header"]
if h["type"] == "incident":
t = h["trigger"]
trigger_numbers.append(t["num"])
pid = h.get("pid")
versions = h.get("versions", {})
if "d" not in e:
continue # skip headers
if not first_event_from:
first_event_from = e['from']
le = LogEvent(e)
if le.index:
number_map[le.index] = le
if le.parent_index in number_map:
le.parent = number_map[le.parent_index]
le.parent.children.append(le)
else:
roots.append(le)
d = e['d']
level = d.get("level", "NORMAL")
number = d.get("num", None)
when = d.get("time")
if number in trigger_numbers:
le.is_trigger = True
if False:
# this is only meaningful if the logfile contains events
# from just a single tub and incarnation, but our current
# LogGatherer combines multiple processes' logs into a
# single file.
if first_event_number is None:
first_event_number = number
elif number is not None:
first_event_number = min(first_event_number, number)
if last_event_number is None:
last_event_number = number
elif number is not None:
last_event_number = max(last_event_number, number)
if first_event_time is None:
first_event_time = when
elif when is not None:
first_event_time = min(first_event_time, when)
if last_event_time is None:
last_event_time = when
elif when is not None:
last_event_time = max(last_event_time, when)
num_events += 1
if level not in levels:
levels[level] = []
levels[level].append(le)
summary = ( (first_event_number, first_event_time),
(last_event_number, last_event_time),
num_events, levels, pid, versions )
summaries[lf] = summary
triggers = [(first_event_from, num) for num in trigger_numbers]
return summaries, roots, number_map, triggers