def load_bb_data(data_root):
print(yellow("Loading data ..."))
exp2data = {}
# TODO: change this to automatically extract names from a single exp. number
# extract each data file into a Pandas dataframe
exp_combos = list(itertools.product(STATES, WIDTHS, EXP_BASE_NAMES))
for num_states, width, exp_base_name in exp_combos:
for trial in TRIALS:
# Build complete path to data files
exp_name_wo_trialnum = exp_base_name % (num_states, width)
exp_name = "%s-%d" % (exp_name_wo_trialnum, trial)
data_path = os.path.join(data_root, exp_name)
# Extract experiment info.
exp_name_list = exp_name.split("-")
instr_type = exp_name_list[6]
if len(exp_name_list) > 9:
fs_opt = True
else:
fs_opt = False
# Load fuzzing data into an object
exp2data[exp_name_wo_trialnum] = FuzzingData(
num_states, width, instr_type, fs_opt, trial, data_path)
print(green("Done."))
print(LINE_SEP)
return exp2data
def _print_configs(self):
exp_config_table = prettytable.PrettyTable(header=False)
exp_config_table.title = "Experiment Parameters"
exp_config_table.field_names = ["Parameter", "Value"]
# Add main experiment parameters
exp_config_table.add_row(["Experiment Name", self.experiment_name])
exp_config_table.add_row(["Toplevel", self.toplevel])
exp_config_table.add_row(["Version", self.version])
exp_config_table.add_row(["Testbench Type", self.tb_type])
exp_config_table.add_row(["Testbench", self.tb])
exp_config_table.add_row(["Fuzzer", self.fuzzer])
exp_config_table.add_row(["Instrument DUT", self.instrument_dut])
exp_config_table.add_row(["Instrument TB", self.instrument_tb])
exp_config_table.add_row(["Instrument VLT-RT", self.instrument_vltrt])
exp_config_table.add_row(["Manual", self.manual])
exp_config_table.add_row(["Run on GCP", self.run_on_gcp])
# Add other parameters
for params in [self.gcp_params] + self.env_var_params:
for param, value in params.items():
param = param.replace("_", " ").title()
exp_config_table.add_row([param, value])
# Print table
exp_config_table.align = "l"
print(yellow(exp_config_table.get_string()))
def build_bbs_df(exp2data):
print(yellow("Building basic block stats dataframe ..."))
# Create empty dictionary that will be used to create Pandas
# a DataFrame that look like the following:
# +---------------------------------------------------+
# | # states | instrumentation level | # basic blocks |
# +---------------------------------------------------+
# | ... | ... | ... |
bbs_dict = {
NUM_STATES_LABEL: [],
INSTR_TYPE_LABEL: [],
NUM_BB_LABEL: [],
}
for exp_name, fd in exp2data.items():
bbs_dict[NUM_STATES_LABEL].extend([fd.num_states] * 2)
# bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine", "TB", "DUT"])
bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine + TB", "DUT"])
# bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs)
# bbs_dict[NUM_BB_LABEL].append(fd.tb_bbs)
bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs + fd.tb_bbs)
bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs)
# bbs_dict[NUM_STATES_LABEL].append(fd.num_states)
# bbs_dict[INSTR_TYPE_LABEL].append(INSTR_TYPE_MAPPINGS[fd.instr_type])
# if fd.instr_type == "full":
# bbs_dict[NUM_BB_LABEL].append(fd.full_bbs)
# elif fd.instr_type == "duttb":
# bbs_dict[NUM_BB_LABEL].append(fd.duttb_bbs)
# elif fd.instr_type == "dut":
# bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs)
# else:
# print(red("ERROR: unknown instrumentation type."))
# sys.exit(1)
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(bbs_dict)
def compute_instr_type_mann_whitney(instr_rts):
print(
yellow(
"Computing Mann-Whitney U-test on instrumentation complexity data ..."
))
for num_states in STATES:
sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states]
full_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] ==
INSTR_TYPE_MAPPINGS["full"]][RUN_TIME_LABEL]
duttb_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] ==
INSTR_TYPE_MAPPINGS["duttb"]][RUN_TIME_LABEL]
dut_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] ==
INSTR_TYPE_MAPPINGS["dut"]][RUN_TIME_LABEL]
# mw_full_duttb = stats.mannwhitneyu(full_instr_data, duttb_instr_data)
mw_full_dut = stats.mannwhitneyu(full_instr_data, dut_instr_data)
# mw_duttb_dut = stats.mannwhitneyu(duttb_instr_data, dut_instr_data)
print("%d States - Mann-Whitney:" % num_states)
# print(
# "\t%s vs. %s:" %
# (INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["duttb"]),
# mw_full_duttb)
print(
"\t%s vs. %s:" %
(INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["dut"]), mw_full_dut)
# print(
# "\t%s vs. %s:" %
# (INSTR_TYPE_MAPPINGS["duttb"], INSTR_TYPE_MAPPINGS["dut"]),
# mw_duttb_dut)
print(green("Done."))
print(LINE_SEP)
def build_fs_opt_rts_df(exp2data):
print(yellow("Building fork server optimization dataframe ..."))
FS_OPT_BASELINE = True
# Create empty dictionary that will be used to create Pandas
# a DataFrame that look like the following:
# +----------------------------------------------------+
# | # states | fork server optimization? | runtime (s) |
# +----------------------------------------------------+
# | ... | ... | ... |
runtimes_dict = {
NUM_STATES_LABEL: [],
OPT_TYPE_LABEL: [],
RUN_TIME_LABEL: [],
}
# Aggregate data into a dictionary
exp2rts = _aggregrate_fs_opt_rts(exp2data)
# Compute scale factors for each set of num_states experiments
states2scales = {}
for (num_states, instr_type, fs_opt), runtimes in exp2rts.items():
if instr_type == "full" and fs_opt is FS_OPT_BASELINE:
scale_factor = np.median(runtimes)
states2scales[num_states] = scale_factor
# Build the dataframe for plotting
for (num_states, instr_type, fs_opt), runtimes in exp2rts.items():
runtimes = list(map(lambda x: x / states2scales[num_states], runtimes))
runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes))
runtimes_dict[OPT_TYPE_LABEL].extend([OPT_TYPE_MAPPINGS[fs_opt]] *
len(runtimes))
runtimes_dict[RUN_TIME_LABEL].extend(runtimes)
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(runtimes_dict)
def build_instr_complex_rts_df(exp2data):
print(yellow("Building instruction complexity dataframe ..."))
INSTR_TYPE_BASELINE = "dut"
# Create empty dictionary that will be used to create Pandas
# a DataFrame that look like the following:
# +------------------------------------------------+
# | # states | instrumentation level | runtime (s) |
# +------------------------------------------------+
# | ... | ... | ... |
runtimes_dict = {
NUM_STATES_LABEL: [],
INSTR_TYPE_LABEL: [],
RUN_TIME_LABEL: [],
}
# Aggregate data into a dictionary
exp2rts = _aggregrate_instr_complex_rts(exp2data)
# Compute scale factors for each set of num_states experiments
states2scales = {}
for (num_states, instr_type, fs_opt), runtimes in exp2rts.items():
if instr_type == INSTR_TYPE_BASELINE and fs_opt is False:
scale_factor = np.median(runtimes)
states2scales[num_states] = scale_factor
# Build the dataframe for plotting
for (num_states, instr_type, fs_opt), runtimes in exp2rts.items():
runtimes = list(map(lambda x: x / states2scales[num_states], runtimes))
runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes))
runtimes_dict[INSTR_TYPE_LABEL].extend([INSTR_TYPE_MAPPINGS[instr_type]] *
len(runtimes))
runtimes_dict[RUN_TIME_LABEL].extend(runtimes)
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(runtimes_dict)
def check_num_active_vm_instances(config):
"""Checks number of active VM instances on GCE as a $$$ safety measure."""
if not config.args.silent:
print(LINE_SEP)
print("Checking number of active VMs on GCE ...")
print(LINE_SEP)
cmd = [
"gcloud", "compute", "instances", "list",
"--zones=%s" % config.gcp_params["zone"]
]
proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
num_active_vm_instances = -1 # first line is header
while True:
line = proc.stdout.readline()
if not line:
break
num_active_vm_instances += 1
if num_active_vm_instances < config.args.max_vm_instances:
if not config.args.silent:
print(green("%d active VM(s)" % num_active_vm_instances))
else:
if not config.args.silent:
print(red("%d active VM(s)" % num_active_vm_instances))
print(
yellow("waiting %d seconds and trying again ..." %
config.args.vm_launch_wait_time_s))
return num_active_vm_instances
def plot_avg_coverage_vs_time_broken(hwf_cov_df, rfuzz_cov_df, time_units="m"):
print(yellow("Generating plot ..."))
# Set plot style and extract only HDL line coverage
# sns.set_theme(context="notebook", style="darkgrid")
hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df])
# create subplots
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6, 4))
# create figure and plot the data
# fig, ax = plt.subplots(1, 1, figsize=(6, 4))
sns.lineplot(data=hdl_cov_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=TOPLEVEL_LABEL,
style=FUZZER_LABEL,
ax=ax1)
sns.lineplot(data=hdl_cov_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=TOPLEVEL_LABEL,
style=FUZZER_LABEL,
ax=ax2)
# set axis ranges
ax1.set_ylim(0.3, 1.0)
ax2.set_ylim(0, 0.05)
# hide the spines between the two axes
ax1.spines['bottom'].set_visible(False)
ax2.spines['top'].set_visible(False)
ax1.xaxis.tick_top()
ax1.tick_params(labeltop=False)
ax2.xaxis.tick_bottom()
# format the plot
if time_units == "m":
time_units_label = "min."
elif time_units == "h":
time_units_label = "hours"
else:
time_units_label = "s"
# ax1.set_xlabel(TIME_LABEL + " (%s)" % time_units_label,
# fontsize=LABEL_FONT_SIZE)
# ax1.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE)
# ax1.tick_params("x", labelsize=TICK_FONT_SIZE)
# ax1.tick_params("y", labelsize=TICK_FONT_SIZE)
# plt.legend(fontsize=LEGEND_FONT_SIZE,
# title_fontsize=LEGEND_TITLE_FONT_SIZE,
# bbox_to_anchor=(1.01, 0.75),
# loc='upper left')
plt.tight_layout()
# save the plot
plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT)
print(green("Done."))
print(LINE_SEP)
def run_cmd(cmd, error_str, silent=False, fail_silent=False):
"""Runs the provided command (list of strings) in a separate process."""
try:
if not silent:
print("Running command:")
print(yellow(subprocess.list2cmdline(cmd)))
subprocess.check_call(cmd)
else:
subprocess.check_call(cmd, stdout=subprocess.DEVNULL)
except subprocess.CalledProcessError:
if not fail_silent:
print(red(error_str), file=sys.stderr)
sys.exit(1)
def _verify_action(config, action, action_msg, abort_msg):
if config.args.fail_silently:
return True
elif config.args.no:
_abort(abort_msg)
elif config.args.yes:
action(config)
else:
ovw = input(yellow(action_msg))
if ovw in {"yes", "y", "Y", "YES", "Yes", ""}:
action(config)
else:
_abort(abort_msg)
return False
def compute_fs_opt_mann_whitney(instr_rts):
print(yellow("Computing Mann-Whitney U-test on fork server opt. data ..."))
for num_states in STATES:
sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states]
no_opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] ==
OPT_TYPE_MAPPINGS[False]][RUN_TIME_LABEL]
opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] ==
OPT_TYPE_MAPPINGS[True]][RUN_TIME_LABEL]
mw = stats.mannwhitneyu(no_opt_data, opt_data)
print("%d States - Mann-Whitney:" % num_states)
print("\t%s vs. %s:" % (OPT_TYPE_MAPPINGS[False], OPT_TYPE_MAPPINGS[True]),
mw.pvalue)
print(green("Done."))
print(LINE_SEP)
def push_docker_image_to_gcr(config):
"""Pushes docker image to GCR if it does not exist there yet."""
if not config.args.silent:
print(LINE_SEP)
print("Pushing Docker image to GCR ...")
print(LINE_SEP)
if config.args.update or not check_if_docker_image_exists_in_gcr(config):
cmd = ["docker", "push", config.docker_image]
error_str = "ERROR: pushing image to GCR FAILED. Terminating experiment!"
run_cmd(cmd, error_str, silent=config.args.silent)
if not config.args.silent:
print(green("IMAGE PUSH SUCCESSFUL -- Done!"))
else:
if not config.args.silent:
print(yellow("IMAGE ALREADY EXISTS IN GCR -- Done!"))
def load_fuzzing_data(hwf_exp_prefix, rfuzz_exp_prefix):
print(yellow("Loading data ..."))
exp2data = collections.defaultdict(list)
for toplevel in TOPLEVELS:
for trial in TRIALS:
# Build complete path to data files
exp_suffix = EXPERIMENT_SUFFIX % (toplevel.lower(), DURATION_MINS)
hwf_data_path = "{}-{}-{}".format(hwf_exp_prefix, exp_suffix,
trial)
rfuzz_data_path = "{}/rfuzz-{}-{}".format(rfuzz_exp_prefix,
exp_suffix, trial)
# Load fuzzing data into an object
exp2data[exp_suffix].append(
FuzzingData(toplevel, DURATION_MINS, trial, hwf_data_path,
rfuzz_data_path))
return exp2data
def plot_coverage_vs_time(coverage_dfs):
print(yellow("Generating plots ..."))
cov_metrics = [
SW_LINE_COVERAGE_LABEL, SW_REGION_COVERAGE_LABEL, HW_LINE_COVERAGE_LABEL
]
num_cores = len(TOPLEVELS)
num_cov_metrics = len(cov_metrics)
sns.set_theme(context="notebook", style="darkgrid")
fig, axes = plt.subplots(num_cov_metrics,
num_cores,
sharex=True,
sharey=True)
for trial in range(len(coverage_dfs)):
# Select experiment trial number
cov_df = coverage_dfs[trial]
for row in range(len(axes)):
# select portion of data corresponding to current COVERAGE METRIC
sub_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == cov_metrics[row]]
for col in range(len(axes[row])):
# select portion of data corresponding to current core
plt_df = sub_cov_df[sub_cov_df[TOPLEVEL_LABEL] == TOPLEVELS[col]]
# sns.set_context("paper")
curr_ax = sns.lineplot(data=plt_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=GRAMMAR_LABEL,
ax=axes[row][col],
legend=False)
if row == 0 and col == 0 and trial == 0:
lines = curr_ax.get_lines()
axes[row][col].set_title("Coverage = %s | Core = %s" %
(cov_metrics[row], TOPLEVELS[col]))
fig.legend(
lines,
[
"Const. Opcode & Variable Frame",
"Const. Opcode & Fixed Frame",
"Mapped Opcode & Variable Frame",
"Mapped Opcode & Fixed Frame",
],
loc="lower center",
ncol=4,
)
print(green("Done."))
print(LINE_SEP)
plt.show()
def _print_configs(args): # Create table to print configurations to STDIN config_table = prettytable.PrettyTable(header=False) config_table.title = "Seed Generation Parameters" config_table.field_names = ["Parameter", "Value"] # Add parameter values to table config_table.add_row(["Input (YAML) Filename", args.input_filename]) config_table.add_row(["Output Filename", args.output_filename]) config_table.add_row(["Frame Type", args.frame_type]) config_table.add_row(["Opcode Size (# bytes)", args.opcode_size]) config_table.add_row(["Address Size (# bytes)", args.address_size]) config_table.add_row(["Data Size (# bytes)", args.data_size]) # Print table config_table.align = "l" print(yellow(config_table.get_string()))
def build_coverage_df(exp2data, trial):
print(yellow("Building coverage dataframe ..."))
# Create empty dictionary that will be used to create a Pandas DataFrame that
# looks like the following:
# +--------------------------------------------------------------------+
# | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) |
# +--------------------------------------------------------------------+
# | ... | ... | ... | ... | ... |
coverage_dict = {
TOPLEVEL_LABEL: [],
GRAMMAR_LABEL: [],
COVERAGE_TYPE_LABEL: [],
TIME_LABEL: [],
COVERAGE_LABEL: [],
}
# Add rows to the dataframe
for exp_name, fd_list in exp2data.items():
fd = fd_list[trial]
for time, row in fd.afl_data.iterrows():
cov_df_idx = row["paths_total"] - 1
for _ in range(3):
coverage_dict[TOPLEVEL_LABEL].append(fd.toplevel)
coverage_dict[GRAMMAR_LABEL].append(fd.grammar)
coverage_dict[TIME_LABEL].append(time)
# Add kcov coverage
kcov = fd.kcov_data.loc[cov_df_idx, "Line-Coverage-(%)"]
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(kcov * 100.0)
# Add LLVM coverage
llvm_cov = fd.llvm_cov_data.loc[cov_df_idx, "Region-Coverage-(%)"]
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(llvm_cov * 100.0)
# Add Verilator coverage
vlt_cov = (float(fd.vlt_cov_data.loc[cov_df_idx, "Lines-Covered"]) /
float(fd.vlt_cov_data.loc[cov_df_idx, "Total-Lines"]))
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(vlt_cov * 100.0)
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(coverage_dict)
def check_for_data_locally(config):
"""Checks if experiment data already exists locally."""
exp_data_path = "%s/data/%s" % (config.root_path, config.experiment_name)
abort_str = "ABORT: re-run with different experiment name."
if glob.glob(exp_data_path):
if config.args.fail_silently:
return True
elif config.args.no:
_abort(abort_str)
elif config.args.yes:
shutil.rmtree(exp_data_path)
else:
ovw = input(yellow("WARNING: experiment data exists. Overwrite? [Yn]"))
if ovw in {"yes", "y", "Y", "YES", "Yes", ""}:
shutil.rmtree(exp_data_path)
else:
_abort(abort_str)
return False
def compute_stats(hwf_cov_dict, rfuzz_cov_dict):
print(yellow("Computing stats ..."))
# Compute HDL coverage % differences
cov_diffs_sum = 0
for toplevel, hwf_cov in hwf_cov_dict.items():
min_hwf_cov = min(hwf_cov)
max_rfuzz_cov = max(rfuzz_cov_dict[toplevel])
cov_diff = min_hwf_cov - max_rfuzz_cov
cov_diffs_sum += cov_diff
print("HWF vs. RFUZZ coverage (%15s): %.3f%%" % (toplevel, cov_diff))
cov_diffs_avg = float(cov_diffs_sum) / float(len(hwf_cov_dict.keys()))
print("Avg. coverage difference: %.3f%%" % (cov_diffs_avg))
print('-' * len(LINE_SEP))
for toplevel, hwf_cov in hwf_cov_dict.items():
rfuzz_cov = rfuzz_cov_dict[toplevel]
myu = stats.mannwhitneyu(hwf_cov, rfuzz_cov)
print("HWF vs. RFUZZ Mann-Whitney (%15s): %s" % (toplevel, myu))
print(green("Done."))
print(LINE_SEP)
def plot_bbs(instr_bbs, orientation="h"):
print(yellow("Generating plots ..."))
LABEL_FONT_SIZE = 14
sns.set()
if orientation == "h":
ax = sns.barplot(y=NUM_STATES_LABEL,
x=NUM_BB_LABEL,
hue=INSTR_TYPE_LABEL,
data=instr_bbs,
orient="h",
ci=None)
ax.set_ylabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE)
ax.set_xlabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE)
ax.tick_params("y", labelsize=LABEL_FONT_SIZE)
ax.tick_params("x", labelsize=LABEL_FONT_SIZE)
ax.set_xlim(10, 10000)
plt.xscale("log")
else:
ax = sns.barplot(x=NUM_STATES_LABEL,
y=NUM_BB_LABEL,
hue=INSTR_TYPE_LABEL,
data=instr_bbs,
ci=None)
ax.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE)
ax.set_ylabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE)
ax.tick_params("x", labelsize=LABEL_FONT_SIZE)
ax.tick_params("y", labelsize=LABEL_FONT_SIZE)
ax.set_ylim(10, 10000)
plt.yscale("log")
plt.legend(title=INSTR_TYPE_LABEL,
fontsize=LABEL_FONT_SIZE,
title_fontsize=LABEL_FONT_SIZE,
ncol=3,
loc="upper center",
bbox_to_anchor=(0.5, 1.25))
plt.tight_layout()
plt.savefig("hwf_components_bbs.png", format="PNG")
print(green("Done."))
print(LINE_SEP)
def load_fuzzing_data(afl_data_root, cov_data_root):
print(yellow("Loading data ..."))
exp2data = collections.defaultdict(list)
# TODO: change this to automatically extract names from a single exp. number
# extract each data file into a Pandas dataframe
isas = list(
itertools.product(TOPLEVELS, OPCODE_TYPES, INSTR_TYPES,
TERMINATE_TYPES))
for toplevel, opcode_type, instr_type, terminate_type in isas:
for trial in TRIALS:
# Build complete path to data files
exp_name_wo_trialnum = EXPERIMENT_BASE_NAME % (
toplevel, opcode_type, instr_type, terminate_type)
exp_name_wo_trialnum = exp_name_wo_trialnum.replace("_", "-")
exp_name = "%s-%d" % (exp_name_wo_trialnum, trial)
afl_data_path = os.path.join(afl_data_root, exp_name)
cov_data_path = os.path.join(cov_data_root, exp_name)
# Load fuzzing data into an object
exp2data[exp_name_wo_trialnum].append(
FuzzingData(toplevel, opcode_type, instr_type, terminate_type,
trial, afl_data_path, cov_data_path))
return exp2data
def plot_avg_coverage_vs_time(hwf_cov_df, rfuzz_cov_df, time_units="m"):
print(yellow("Generating plot ..."))
# Set plot style and extract only HDL line coverage
sns.set_theme(context="notebook", style="darkgrid")
hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df])
# create figure and plot the data
fig, ax = plt.subplots(1, 1, figsize=(6, 4))
sns.lineplot(data=hdl_cov_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=TOPLEVEL_LABEL,
style=FUZZER_LABEL,
ax=ax)
# format the plot
if time_units == "m":
time_units_label = "min."
elif time_units == "h":
time_units_label = "hours"
else:
time_units_label = "s"
ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label,
fontsize=LABEL_FONT_SIZE)
ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE)
ax.tick_params("x", labelsize=TICK_FONT_SIZE)
ax.tick_params("y", labelsize=TICK_FONT_SIZE)
plt.legend(fontsize=LEGEND_FONT_SIZE,
title_fontsize=LEGEND_TITLE_FONT_SIZE,
bbox_to_anchor=(1.01, 0.75),
loc='upper left')
plt.tight_layout()
# save the plot
plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT)
print(green("Done."))
print(LINE_SEP)
def plot_avg_coverage_vs_time(cov_df, time_units="m"):
print(yellow("Generating plot ..."))
# Set plot style and extract only HDL line coverage
sns.set_theme(context="notebook", style="darkgrid")
hdl_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == HW_LINE_COVERAGE_LABEL]
# create figure and plot the data
fig, ax = plt.subplots(1, 1, figsize=(4, 2))
sns.lineplot(data=hdl_cov_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=TOPLEVEL_LABEL,
ax=ax,
markers="x")
# format the plot
if time_units == "m":
time_units_label = "min."
elif time_units == "h":
time_units_label = "hours"
else:
time_units_label = "s"
ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label,
fontsize=LABEL_FONT_SIZE)
ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE)
ax.tick_params("x", labelsize=TICK_FONT_SIZE)
ax.tick_params("y", labelsize=TICK_FONT_SIZE)
plt.legend(title="Core",
fontsize=LEGEND_FONT_SIZE,
title_fontsize=LEGEND_TITLE_FONT_SIZE,
ncol=2)
plt.tight_layout()
# save the plot
plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT)
print(green("Done."))
print(LINE_SEP)
def build_min_hwf_coverage_df(exp2data,
time_units="m",
normalize_to_start=False,
consolidation="max"):
print(yellow("Building HWF coverage dataframe ..."))
# Create empty dictionary that will be used to create a Pandas DataFrame that
# looks like the following:
# +--------------------------------------------------------------------+
# | toplevel | fuzzer | coverage type | time | coverage (%) |
# +--------------------------------------------------------------------+
# | ... | ... | ... | ... | ... |
coverage_dict = {
TOPLEVEL_LABEL: [],
FUZZER_LABEL: [],
COVERAGE_TYPE_LABEL: [],
TIME_LABEL: [],
COVERAGE_LABEL: [],
}
cov_dict = collections.defaultdict(
list) # maps toplevel --> [coverage list]
for exp_name, fd_list in exp2data.items():
# get min coverage experiment
min_cov = get_max_vlt_cov(fd_list[0].hwf_cov_data)
min_cov_fd = fd_list[0]
for fd in fd_list:
cov = get_max_vlt_cov(fd.hwf_cov_data)
cov_dict[fd.toplevel].append(cov)
if cov < min_cov:
min_cov = cov
min_cov_fd = fd
# build data frame for plotting
for time, row in min_cov_fd.hwf_afl_data.iterrows():
# scale time
scaled_time = scale_time(time, time_units)
# add circuit, fuzzer, and time values to dataframe row
coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel)
coverage_dict[TIME_LABEL].append(scaled_time)
# get the AFL paths_total at the current time
paths_total = get_paths_total_at_time(time,
min_cov_fd.hwf_afl_data) - 1
# get HWF coverage data
hwf_vlt_cov = get_vlt_cov_at_time(paths_total,
min_cov_fd.hwf_cov_data)
# normalize to start time if requested
if time == 0:
hwf_vlt_cov_t0 = hwf_vlt_cov
if normalize_to_start:
hwf_vlt_cov /= hwf_vlt_cov_t0
# add to data frame
coverage_dict[FUZZER_LABEL].append("HWFP")
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(hwf_vlt_cov)
# extend lines to max time value
if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME:
coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel)
coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME)
coverage_dict[FUZZER_LABEL].append("HWFP")
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(
coverage_dict[COVERAGE_LABEL][-1])
print("Min. HW Line coverage (%15s): %.3f%%" %
(min_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1]))
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(coverage_dict), cov_dict
def plot_avg_coverage_vs_time(cov_df, time_units="m"):
print(yellow("Generating plots ..."))
cov_metrics = [
SW_LINE_COVERAGE_LABEL, SW_REGION_COVERAGE_LABEL, HW_LINE_COVERAGE_LABEL
]
axis_limits = _get_axis_limits()
sns.set_theme(context="notebook", style="darkgrid")
# create figure and subplot axes
fig, axes = plt.subplots(len(cov_metrics),
len(TOPLEVELS),
sharex="col",
figsize=(10, 4))
# plot coverage traces
for row in range(len(axes)):
# select portion of data corresponding to current COVERAGE METRIC
sub_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == cov_metrics[row]]
for col in range(len(axes[row])):
# select portion of data corresponding to current core
plt_df = sub_cov_df[sub_cov_df[TOPLEVEL_LABEL] == TOPLEVELS[col]]
# TODO(ttrippel): there has to be a better way to do the following:
# plot legend only for the first subplot so we can extract the labels
if row == 0 and col == 0:
plot_legend = True
else:
plot_legend = False
ax = sns.lineplot(data=plt_df,
x=TIME_LABEL,
y=COVERAGE_LABEL,
hue=GRAMMAR_LABEL,
ax=axes[row][col],
legend=plot_legend)
ax.axhline(y=1.0, color='r', linestyle='-')
# get legend info if we are plotting the first plot
if plot_legend:
lines = ax.get_lines()
labels = _get_legend_labels(ax)
# format each figure
for row in range(len(axes)):
for col in range(len(axes[row])):
# get corresponding axis_limits
if cov_metrics[row] == SW_LINE_COVERAGE_LABEL:
ax_lims = axis_limits[TOPLEVELS[col]].kcov_limits
elif cov_metrics[row] == SW_REGION_COVERAGE_LABEL:
ax_lims = axis_limits[TOPLEVELS[col]].llvm_cov_limits
else:
ax_lims = axis_limits[TOPLEVELS[col]].vlt_cov_limits
# format plot labels and axes
subplot_title = "%s | %s" % (TOPLEVELS[col], cov_metrics[row])
_format_plot(axes[row][col], ax_lims, time_units, subplot_title)
plt.tight_layout()
# set legend
# isa_label_mapping = {
# "constant-variable-never": "Const. Opcode | Variable Frame",
# "constant-fixed-never": "Const. Opcode | Fixed Frame",
# "mapped-variable-never": "Mapped Opcode | Variable Frame",
# "mapped-fixed-never": "Mapped Opcode | Fixed Frame",
# }
isa_label_mapping = {
"constant-variable-never": "Constant | Variable",
"constant-fixed-never": "Constant | Fixed",
"mapped-variable-never": "Mapped | Variable",
"mapped-fixed-never": "Mapped | Fixed",
}
clean_labels = [isa_label_mapping[isa_label] for isa_label in labels]
fig.legend(lines,
clean_labels,
fontsize=LEGEND_FONT_SIZE,
title_fontsize=LEGEND_TITLE_FONT_SIZE,
loc="upper center",
ncol=4,
borderaxespad=0.05,
bbox_to_anchor=(0.5, 0.12),
title=GRAMMAR_LABEL + " (Opcode Format | Frame Format)")
plt.subplots_adjust(bottom=0.23, wspace=0.25, hspace=0.25)
# adjust figure layout and save to file
plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT)
print(green("Done."))
print(LINE_SEP)
def build_avg_coverage_df(exp2data,
time_units="m",
normalize_to_start=False,
consolidation="max"):
print(yellow("Building average coverage dataframe ..."))
# Create empty dictionary that will be used to create a Pandas DataFrame that
# looks like the following:
# +--------------------------------------------------------------------+
# | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) |
# +--------------------------------------------------------------------+
# | ... | ... | ... | ... | ... |
coverage_dict = {
TOPLEVEL_LABEL: [],
GRAMMAR_LABEL: [],
COVERAGE_TYPE_LABEL: [],
TIME_LABEL: [],
COVERAGE_LABEL: [],
}
for exp_name, fd_list in exp2data.items():
anchor_fd = fd_list[0]
for time, row in anchor_fd.afl_data.iterrows():
# scale time
if time_units == "h":
scaled_time = float(time) / float(3600)
elif time_units == "m":
scaled_time = float(time) / float(60)
else:
scaled_time = time
# add circuit, grammar, and time values to dataframe row
for _ in range(3):
coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel)
coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar)
coverage_dict[TIME_LABEL].append(scaled_time)
# compute average coverage at all points in time
kcov_avg = 0
llvm_cov_avg = 0
vlt_cov_avg = 0
kcov_max = 0
llvm_cov_max = 0
vlt_cov_max = 0
i = 0
for fd in fd_list:
# get the paths_total at the current time
paths_total = get_paths_total_at_time(time, fd.afl_data) - 1
# get coverage data
# print(exp_name, i)
kcov = get_cov_at_time(paths_total, fd.kcov_data,
"Line-Coverage-(%)")
kcov_avg += kcov
kcov_max = max(kcov_max, kcov)
llvm_cov = get_cov_at_time(paths_total, fd.llvm_cov_data,
"Region-Coverage-(%)")
llvm_cov_avg += llvm_cov
llvm_cov_max = max(llvm_cov_max, llvm_cov)
vlt_cov = get_vlt_cov_at_time(paths_total, fd.vlt_cov_data)
vlt_cov_avg += vlt_cov
vlt_cov_max = max(vlt_cov_max, vlt_cov)
i += 1
kcov_avg /= float(len(fd_list))
llvm_cov_avg /= float(len(fd_list))
vlt_cov_avg /= float(len(fd_list))
# save time 0 coverage to normalize
if time == 0:
kcov_avg_t0 = kcov_avg
llvm_cov_avg_t0 = llvm_cov_avg
vlt_cov_avg_t0 = vlt_cov_avg
if normalize_to_start:
kcov_avg /= kcov_avg_t0
llvm_cov_avg /= llvm_cov_avg_t0
vlt_cov_avg /= vlt_cov_avg_t0
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL)
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
if consolidation == "avg":
coverage_dict[COVERAGE_LABEL].append(kcov_avg)
coverage_dict[COVERAGE_LABEL].append(llvm_cov_avg)
coverage_dict[COVERAGE_LABEL].append(vlt_cov_avg)
else:
coverage_dict[COVERAGE_LABEL].append(kcov_max)
coverage_dict[COVERAGE_LABEL].append(llvm_cov_max)
coverage_dict[COVERAGE_LABEL].append(vlt_cov_max)
# extend lines to max time value
if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME:
for _ in range(3):
coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel)
coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar)
coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME)
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL)
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].extend(
coverage_dict[COVERAGE_LABEL][-3:])
# print("Max SW Line coverage: ", coverage_dict[COVERAGE_LABEL][-3])
# print("Max SW Basic Block coverage:", coverage_dict[COVERAGE_LABEL][-2])
print("Max HW Line coverage: ",
coverage_dict[COVERAGE_LABEL][-1])
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(coverage_dict)
def plot_opt_strategies(instr_rts, fsopt_rts, plot_type="violin"):
print(yellow("Generating plots ..."))
LABEL_FONT_SIZE = 14
sns.set()
# HW fuzzing instrumentation levels
if plot_type == "violin":
ax1 = sns.violinplot(x=NUM_STATES_LABEL,
y=RUN_TIME_LABEL,
hue=INSTR_TYPE_LABEL,
data=instr_rts)
else:
ax1 = sns.stripplot(x=NUM_STATES_LABEL,
y=RUN_TIME_LABEL,
hue=INSTR_TYPE_LABEL,
data=instr_rts,
dodge=True,
jitter=0.3,
size=MARKER_SIZE)
ax1.axhline(y=1.0, color='r', linestyle='-')
ax1.set_ylim(0.5, 3)
ax1.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE)
ax1.set_ylabel(RUN_TIME_LABEL, fontsize=LABEL_FONT_SIZE)
ax1.tick_params("x", labelsize=LABEL_FONT_SIZE)
ax1.tick_params("y", labelsize=LABEL_FONT_SIZE)
plt.legend(title=INSTR_TYPE_LABEL,
fontsize=LABEL_FONT_SIZE,
title_fontsize=LABEL_FONT_SIZE)
plt.tight_layout()
# plt.savefig("hwf_instrumentation_levels.png", format="png")
plt.savefig("hwf_instrumentation_levels.pdf", format="pdf")
plt.close()
# HW fork server optimization
if plot_type == "violin":
ax2 = sns.violinplot(x=NUM_STATES_LABEL,
y=RUN_TIME_LABEL,
hue=OPT_TYPE_LABEL,
data=fsopt_rts)
else:
ax2 = sns.stripplot(x=NUM_STATES_LABEL,
y=RUN_TIME_LABEL,
hue=OPT_TYPE_LABEL,
data=fsopt_rts,
dodge=True,
jitter=0.3,
size=MARKER_SIZE)
ax2.axhline(y=1.0, color='r', linestyle='-')
ax1.set_ylim(0.5, 3)
ax2.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE)
ax2.set_ylabel(RUN_TIME_LABEL, fontsize=LABEL_FONT_SIZE)
ax2.tick_params("x", labelsize=LABEL_FONT_SIZE)
ax2.tick_params("y", labelsize=LABEL_FONT_SIZE)
plt.legend(title=OPT_TYPE_LABEL,
fontsize=LABEL_FONT_SIZE,
title_fontsize=LABEL_FONT_SIZE)
plt.tight_layout()
# plt.savefig("hwf_fs_opt.png", format="png")
plt.savefig("hwf_fs_opt.pdf", format="pdf")
print(green("Done."))
print(LINE_SEP)
def build_max_rfuzz_coverage_df(exp2data,
time_units="m",
normalize_to_start=False,
consolidation="max"):
print(yellow("Building RFUZZ coverage dataframe ..."))
# Create empty dictionary that will be used to create a Pandas DataFrame that
# looks like the following:
# +--------------------------------------------------------------------+
# | toplevel | fuzzer | coverage type | time | coverage (%) |
# +--------------------------------------------------------------------+
# | ... | ... | ... | ... | ... |
coverage_dict = {
TOPLEVEL_LABEL: [],
FUZZER_LABEL: [],
COVERAGE_TYPE_LABEL: [],
TIME_LABEL: [],
COVERAGE_LABEL: [],
}
cov_dict = collections.defaultdict(
list) # maps toplevel --> [coverage list]
for exp_name, fd_list in exp2data.items():
# get max coverage experiment
max_cov = get_max_vlt_cov(fd_list[0].rfuzz_cov_data)
max_cov_fd = fd_list[0]
for fd in fd_list:
cov = get_max_vlt_cov(fd.rfuzz_cov_data)
cov_dict[fd.toplevel].append(cov)
if cov > max_cov:
max_cov = cov
max_cov_fd = fd
for test_id, row in max_cov_fd.rfuzz_data.iterrows():
# scale time
scaled_time = scale_time(row["Time (s)"], time_units)
# add circuit, fuzzer, and time values to dataframe row
coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel)
coverage_dict[TIME_LABEL].append(scaled_time)
# compute average coverage at all points in time
rfuzz_vlt_cov = get_vlt_cov_at_time(test_id,
max_cov_fd.rfuzz_cov_data)
# save time 0 coverage to normalize if requested
if test_id == 0:
rfuzz_vlt_cov_t0 = rfuzz_vlt_cov
if normalize_to_start:
rfuzz_vlt_cov /= rfuzz_vlt_cov_t0
# add coverage to dataframe row
coverage_dict[FUZZER_LABEL].append("RFUZZ")
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(rfuzz_vlt_cov)
# extend lines to max time value
if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME:
coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel)
coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME)
coverage_dict[FUZZER_LABEL].append("RFUZZ")
coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL)
coverage_dict[COVERAGE_LABEL].append(
coverage_dict[COVERAGE_LABEL][-1])
print("Max HW Line coverage (%15s): %.3f%%" %
(max_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1]))
print(green("Done."))
print(LINE_SEP)
return pd.DataFrame.from_dict(coverage_dict), cov_dict
def fuzz(argv):
"""Runs fuzzing experiment with provided configuration filename."""
# Parse cmd args
module_description = "Hardware Fuzzing Pipeline"
parser = argparse.ArgumentParser(description=module_description)
parser.add_argument("--fail-silently",
action="store_true",
help="Fail silently if data/VM already exists.")
parser.add_argument("-s",
"--silent",
action="store_true",
help="Supress stdout messages printed by HWFP.")
parser.add_argument("--log-driver",
choices=[
"none",
"json-file",
"syslog",
"journald",
"gelf",
"fluentd",
"awslogs",
"splunk",
],
default="json-file",
help="Docker logging driver to use.")
parser.add_argument("-n",
"--no",
action="store_true",
help="No to all prompts. (overides -y)")
parser.add_argument("-y",
"--yes",
action="store_true",
help="Yes to all prompts.")
parser.add_argument("-u",
"--update",
action="store_true",
help="Update Docker image in GCR.")
parser.add_argument("--max-vm-instances",
default=32,
help="Max number of VM instances allowed on GCP zone.")
parser.add_argument("--vm-launch-wait-time-s",
default=30,
help="Max number of VM instances allowed on GCP zone.")
parser.add_argument("--gcp-config-filename",
default="gcp_config.hjson",
help="GCP vonfiguration file in the HJSON format.")
parser.add_argument("config_filename",
metavar="config.hjson",
help="Configuration file in the HJSON format.")
args = parser.parse_args(argv)
# Load experiment configurations
config = Config(args)
# Check if experiment data already exists
if config.run_on_gcp == 0:
if check_for_data_locally(config):
if not config.args.silent:
print(yellow("WARNING: experiment data exists locally... skipping."))
return
else:
if check_for_data_in_gcs(config):
if not config.args.silent:
print(yellow("WARNING: experiment data exists in GCS... skipping."))
return
if check_if_gce_vm_up(config):
if not config.args.silent:
print(yellow("WARNING: experiment VM is already running... skipping."))
return
# Build docker image to fuzz target toplevel
build_docker_image(config)
# Run Docker container to fuzz toplevel
if config.run_on_gcp == 0:
exp_data_path = create_local_experiment_data_dir(config)
run_docker_container_locally(config, exp_data_path)
else:
push_docker_image_to_gcr(config)
push_vm_management_scripts_to_gcs(config)
run_docker_container_on_gce(config)
