def plot_cpu(*, reports, filename):
"""Read reports and plot the CPU"""
fig = plt.figure()
ax = fig.add_axes((0.1, 0.2, 0.8, 0.7))
ax.set_ylabel("% of CPU utilized")
ax.set_xlabel("Seconds from start of run")
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
ax.set_ylim(ymin=0, ymax=400)
ymax = 100
lines = []
legends = []
for report in reports:
r = BulkReport(report)
xyvals = r.cpu_track()
# Extract timestamps and convert to seconds from beginning of run
xvals = [xy[0] for xy in xyvals]
xmin = min(xvals)
xvals = [(x - xmin).total_seconds() for x in xvals]
# Extract CPU% and find max val
yvals = [xy[1] for xy in xyvals]
ymax = max([ymax] + yvals)
# Plot and retain handle
line, = plt.plot(xvals, yvals, label='report')
lines.append(line)
ax.legend(handles=lines) # draw legends
ax.set_ylim(ymin=0, ymax=math.ceil(ymax / 100) * 100) # set ymax
plt.savefig(filename, format='pdf')
def main():
args = proc_args()
group_var = args.group_var
x_var = args.x_var
y_vars = tuple()
if args.show_memory:
y_vars += (AxisVar("peak_memory"), )
else:
y_vars += (AxisVar("clocktime"), )
if len(y_vars) < 1:
print("all y values suppressed", file=sys.stderr)
return 1
try:
reports = tuple(
sorted((BulkReport(x) for x in args.reports), key=group_var.of))
except RuntimeError as e:
print("failed to open reports: ", str(e), file=sys.stderr)
return 1
# group related data into datasets
datasets = tuple()
dataset = tuple()
for report in reports:
if len(dataset) < 1:
dataset += (report, )
continue
if group_var.of(report) != group_var.of(dataset[0]):
datasets += (sorted(dataset, key=x_var.of), )
dataset = (report, )
continue
dataset += (report, )
datasets += (sorted(dataset, key=x_var.of), )
del (dataset)
# check x uniqueness
for dataset in datasets:
if len(set([x_var.of(report) for report in dataset])) != len(dataset):
print("x value ({}) must be unique per grouping ({})".format(
x_var.name, group_var.name),
file=sys.stderr)
return 1
make_plot(datasets, x_var, y_vars, group_var, args.output_filename)
args = arg_parser.parse_args()
res = []
if args.calc:
for (dirpath, dirnames, filenames) in os.walk(args.output):
for filename in filenames:
if filename.endswith("~"): continue
fn = os.path.join(dirpath, filename)
r = calc_stats(fn)
print(r)
res.append(r)
print("{:20} {:8} {:8} {:4} {:8} {:8}".format("Feature", "Total",
"Sampled", "%",
"Accuracy", "Err Rate"))
for r in res:
print("{:20} {:8} {:8} {:4}% {:8} {:8}".format(
r['fn'], r['total'], r['sampled'],
r['sampled'] * 100.0 / r['total'], r['accuracy'],
r['error_rate']))
exit(0)
if args.sample:
(input, output) = args.sample
if os.path.exists(output):
raise RuntimeError(output + " exists")
os.mkdir(output)
report = BulkReport(input)
for fn in report.feature_files():
sample(output, fn)
if not os.path.exists(args.reportdir):
print("{} does not exist".format(args.reportdir))
exit(1)
explainfile = os.path.join(args.reportdir,"identified_blocks_explained.txt")
if not os.path.exists(explainfile) and args.explain:
ifname = os.path.join(args.reportdir,"identified_blocks.txt")
ofname = os.path.join(args.reportdir,"identified_blocks_explained.txt")
cmd = ['hashdb','explain_identified_blocks',args.explain,ifname]
print(" ".join(cmd))
call(cmd,stdout=open(ofname,"w"))
if not args.image:
from bulk_extractor_reader import BulkReport
b = BulkReport(args.reportdir)
args.image = b.image_filename()
dbname = os.path.join(args.reportdir,args.dbname)
if not os.path.exists(dbname) and args.run:
print("{} does not exist. Will try to run tsk_loaddb".format(dbname))
cmd=['tsk_loaddb','-d',dbname,args.image]
print(" ".join(cmd))
call(cmd)
# Add the indexes
print("Adding indexes to database")
import sqlite3
con = sqlite3.connect("file:{}".format(dbname),uri=True)
cur = con.cursor()
cur.execute("create index if not exists start1 on tsk_file_layout(byte_start)");
cur.execute("create index if not exists start2 on tsk_file_layout(byte_len)");
print("{} does not exist".format(args.reportdir))
exit(1)
explainfile = os.path.join(args.reportdir,
"identified_blocks_explained.txt")
if not os.path.exists(explainfile) and args.explain:
ifname = os.path.join(args.reportdir, "identified_blocks.txt")
ofname = os.path.join(args.reportdir,
"identified_blocks_explained.txt")
cmd = ['hashdb', 'explain_identified_blocks', args.explain, ifname]
print(" ".join(cmd))
call(cmd, stdout=open(ofname, "w"))
if not args.image:
from bulk_extractor_reader import BulkReport
b = BulkReport(args.reportdir)
args.image = b.image_filename()
dbname = os.path.join(args.reportdir, args.dbname)
if not os.path.exists(dbname) and args.run:
print("{} does not exist. Will try to run tsk_loaddb".format(dbname))
cmd = ['tsk_loaddb', '-d', dbname, args.image]
print(" ".join(cmd))
call(cmd)
# Add the indexes
print("Adding indexes to database")
import sqlite3
con = sqlite3.connect("file:{}".format(dbname), uri=True)
cur = con.cursor()
cur.execute(
"create index if not exists start1 on tsk_file_layout(byte_start)")
group_var = AxisVar("none")
if args.cpu:
plot_cpu(reports=args.reports, filename=args.output_filename)
exit(0)
y_vars = []
if args.show_memory:
y_vars.append(AxisVar("peak_memory"))
elif args.cpu:
y_vars.append(AxisVar("cpu"))
else:
y_vars.append(AxisVar("clocktime"))
try:
reports = sorted((BulkReport(x) for x in args.reports),
key=group_var.of)
except RuntimeError as e:
raise RuntimeError(f"failed to open reports: {e}")
# group related data into datasets
datasets = []
dataset = []
for report in reports:
if len(dataset) < 1:
dataset.append(report)
continue
if group_var.of(report) != group_var.of(dataset[0]):
datasets += (sorted(dataset, key=args.x_var.of), )
dataset.append(report)
continue
arg_parser.add_argument("--xpattern", type=str, help="Do not sample lines that include this pattern in the forensic path")
arg_parser.add_argument("--calc", help="Compute the statistics",action="store_true")
arg_parser.add_argument("--trials", type=int, default="5", help="Number of trials to divide into")
arg_parser.add_argument("--quiet",action='store_true',help='do not alert on lines with no classification')
args = arg_parser.parse_args()
res = []
if args.calc:
for (dirpath,dirnames,filenames) in os.walk(args.output):
for filename in filenames:
if filename.endswith("~"): continue
fn = os.path.join(dirpath,filename)
r = calc_stats(fn)
print(r)
res.append(r)
print("{:20} {:8} {:8} {:4} {:8} {:8}".format("Feature","Total","Sampled","%","Accuracy","Err Rate"))
for r in res:
print("{:20} {:8} {:8} {:4}% {:8} {:8}".format(
r['fn'],r['total'],r['sampled'],r['sampled']*100.0/r['total'],r['accuracy'],r['error_rate']))
exit(0)
if args.sample:
(input,output) = args.sample
if os.path.exists(output):
raise RuntimeError(output+" exists")
os.mkdir(output)
report = BulkReport(input)
for fn in report.feature_files():
sample(output,fn)