def infer_filetypes_via_coverage(self) -> [(str, str, int)]:
"""
For each inferred parameter, this function find the corresponding best fitting filetype.
:return: A list of triples: (parameter,filetype,coverage)
"""
self.cli_config_list = []
for param in self.parameters:
inference_result = self.infer_filetype_via_coverage_for_parameter_parallel(
param)
if inference_result is None: # Everything yielded the same coverage
continue # Next one
max_files, max_covs, took_max_file = inference_result
p = "None"
if param:
p = param
c = CliConfig(invocation=param,
filetypes=max_files,
coverage_list=self.coverage_lists[p],
coverages=max_covs,
binary_path=self.binary_path,
max_coverage=max(max_covs),
took_max_file=took_max_file)
if self.failed_invocations >= self.FAILED_INVOCATIONS_THRESHOLD:
c.invocation_always_possible = False
c.qemu = self.qemu
self.cli_config_list.append(c)
return sorted(self.cli_config_list,
key=operator.attrgetter("max_coverage"),
reverse=True)
def ui_command_saveconfig(self):
'''
Saves the whole configuration tree to disk so that it will be restored
on next boot. Unless you do that, changes are lost accross reboots.
'''
self.assert_root()
try:
input = raw_input("Save configuration? [Y/n]: ")
except EOFError:
input = None
self.shell.con.display('')
if input in ["y", "Y", ""]:
CliConfig.save_running_config()
else:
self.shell.log.warning("Configuration not saved.")
def ui_command_saveconfig(self):
'''
Saves the whole configuration tree to disk so that it will be restored
on next boot. Unless you do that, changes are lost accross reboots.
'''
self.assert_root()
try:
input = raw_input("Save configuration? [Y/n]: ")
except EOFError:
input = None
self.shell.con.display('')
if input in ["y", "Y", ""]:
CliConfig.save_running_config()
else:
self.shell.log.warning("Configuration not saved.")
def ui_command_saveconfig(self):
'''
Saves the whole configuration tree to disk so that it will be restored
on next boot. Unless you do that, changes are lost accross reboots.
'''
self.assert_root()
self.shell.con.display("WARNING: Saving %s current configuration to "
% gethostname()
+ "disk will overwrite your boot settings.")
self.shell.con.display("The current target configuration will become "
+ "the default boot config.")
try:
input = raw_input("Are you sure? Type 'yes': ")
except EOFError:
input = None
self.shell.con.display('')
if input == "yes":
CliConfig.save_running_config()
else:
self.shell.log.warning("Aborted, startup configuration left untouched.")
def minize(parameter: str,
seeds_dir: str,
binary_path: str,
package: str,
volume_path: str,
afl_config_file_name: str,
qemu: bool = False,
name: str = None,
tmin_total_time: int = None,
do_tmin: bool = True,
cores: int = 1) -> bool:
# if package:
# b = builder.Builder(package=package, qemu=qemu)
# if not b.install():
# print("Could not install package, exiting")
if not package:
package = os.path.basename(binary_path)
input_vector = CliConfig(invocation=parameter,
filetypes=seeds_dir.split(";"),
binary_path=binary_path)
out_dir = volume_path + "/" + package + "/" + os.path.basename(
input_vector.binary_path)
if not os.path.exists(out_dir):
os.makedirs(out_dir, exist_ok=True)
min_seeds_dir = os.path.join(out_dir, "minseeds_" + str(uuid.uuid4()))
file_list = []
print(seeds_dir)
for filetype_seeds in seeds_dir.split(";"):
for file in os.listdir(filetype_seeds):
file_path = os.path.join(filetype_seeds, file)
if os.path.isfile(file_path):
file_list.append(file_path)
else:
print("Ignored non-file at {}".format(file_path))
os.makedirs(min_seeds_dir, exist_ok=True)
for file in file_list:
if os.path.getsize(
file
) < 850 * 1000: # If file is smaller than 850 kb just copy it
copyfile(file, os.path.join(min_seeds_dir, os.path.basename(file)))
else: # Reduce the file to 1kb
helpers.utils.crop_file(
file, os.path.join(min_seeds_dir, os.path.basename(file)),
1 * 1000)
dump_into_json(input_vector=input_vector,
min_seeds_dir=min_seeds_dir,
package=package,
name=name,
volume_path=volume_path,
afl_config_file_name=afl_config_file_name)
use_qemu = helpers.utils.qemu_required_for_binary(
input_vector.binary_path) # TODO: There seems to be an issue here
# First: Minimize the seeds
cmin_dir = os.path.join(out_dir, "afl_cmin_" + str(uuid.uuid4()))
cmin_params = []
if use_qemu:
cmin_params.append("-Q")
cmin_params += [
"-I", "-i", min_seeds_dir, "-o", cmin_dir, "-m", "none", "-t",
str(config_settings.AFL_CMIN_INVOKE_TIMEOUT * 1000), "--",
input_vector.binary_path
]
if input_vector.parameter:
cmin_params += input_vector.parameter.split(" ")
print("Calling afl-cmin {0}".format(" ".join(cmin_params)), flush=True)
try:
afl_cmin(cmin_params,
_timeout=config_settings.AFL_CMIN_TIMEOUT,
_env=helpers.utils.get_fuzzing_env_for_invocation(parameter),
_out=sys.stdout,
_error=sys.stderr)
print("afl cmin done", flush=True)
except sh.ErrorReturnCode as e:
print("afl cmin failed for {0}".format(input_vector.binary_path))
print("STDOUT:\n", e.stdout.decode("utf-8"))
print("STDERR:\n", e.stderr.decode("utf-8"))
sys.exit(-1)
except sh.TimeoutException as e:
print("afl cmin timed out for {0}".format(input_vector.binary_path))
# print("STDOUT:\n", e.stdout.decode("utf-8"))
# print("STDERR:\n", e.stderr.decode("utf-8"))
for f in os.listdir(min_seeds_dir):
shutil.copyfile(os.path.join(min_seeds_dir, f),
os.path.join(cmin_dir, f))
dump_into_json(input_vector=input_vector,
min_seeds_dir=cmin_dir,
package=package,
name=name,
volume_path=volume_path,
afl_config_file_name=afl_config_file_name)
shutil.rmtree(
min_seeds_dir) # We do not want to store the minimized seeds again
if not do_tmin:
return True
tmin_dir = out_dir + "/afl_tmin_" + str(uuid.uuid4())
os.makedirs(tmin_dir, exist_ok=True)
for iter_num, file in enumerate(os.listdir(cmin_dir)):
tmin_params = []
if use_qemu:
tmin_params.append("-Q")
if os.path.isdir(os.path.join(
cmin_dir, file)): # Maybe .traces is still in there or so?
continue
tmin_params += [
"-t",
str(config_settings.AFL_TMIN_INVOKE_TIMEOUT * 1000), "-i",
cmin_dir + "/" + file, "-o", tmin_dir + "/" + file, "-m", "none",
"--", input_vector.binary_path
]
if input_vector.parameter:
tmin_params += input_vector.parameter.split(" ")
if iter_num == 0:
print("Minimizing seed file size", flush=True)
print("Calling afl-tmin {0}".format(" ".join(tmin_params)), flush=True)
if tmin_total_time:
afl_tmin_timeout_per_file = tmin_total_time / len(
os.listdir(cmin_dir)) # Divide time per file
else:
afl_tmin_timeout_per_file = config_settings.AFL_TMIN_TIMEOUT / len(
os.listdir(cmin_dir)) # Divide time per file
try:
helpers.utils.temp_print("Calling afl-tmin {0}".format(
" ".join(tmin_params)))
afl_tmin(
tmin_params,
_out=sys.stdout,
_timeout=afl_tmin_timeout_per_file,
_timeout_signal=signal.SIGTERM,
_env=config_settings.get_fuzzing_env_without_desock()
) # AFL tmin can be very slow. We should only use a limited amount of time on it. If we send SIGTERM though, the current progress is saved
except sh.ErrorReturnCode as e:
print("afl tmin failed for {0}".format(input_vector.binary_path))
print("STDOUT:\n", e.stdout.decode("utf-8"))
print("STDERR:\n", e.stderr.decode("utf-8"))
if aflerrors["AFL_TIMEOUT"] in e.stderr.decode("utf-8"):
print(
"afl-tmin timed out for {0}. Going to use current progress or raw file"
.format(package))
if not os.path.exists(os.path.join(tmin_dir, file)):
copyfile(os.path.join(cmin_dir, file),
os.path.join(tmin_dir, file))
else:
sys.exit(-1)
except sh.TimeoutException as e:
# print("STDOUT:\n", e.stdout.decode("utf-8"))
# print("STDERR:\n", e.stderr.decode("utf-8"))
print(
"afl-tmin timed out for {0}. Going to use current progress or raw file"
.format(package))
if not os.path.exists(os.path.join(tmin_dir, file)):
copyfile(os.path.join(cmin_dir, file),
os.path.join(tmin_dir, file))
# seeds_dir = cmin_dir
dump_into_json(input_vector=input_vector,
min_seeds_dir=tmin_dir,
package=package,
name=name,
volume_path=volume_path,
afl_config_file_name=afl_config_file_name)
return True
def prepare_and_start_fuzzer(parameter: str,
seeds_dir: str,
binary_path: str,
package: str,
volume_path: str,
afl_config_file_name: str,
qemu: bool = False,
name: str = None,
timeout: str = None,
wrapper_function=afl_fuzz_wrapper,
fuzz_duration: int = None,
log_dict=None,
file_types=None):
if file_types is None:
file_types = []
print("Now starting the fuzzing", flush=True)
package_str = ""
if package:
package_str = package
else:
package_str = os.path.basename(binary_path)
out_dir = os.path.join(
volume_path + "/",
os.path.join(package_str, os.path.basename(binary_path)))
os.makedirs(os.path.join(
volume_path + "/",
os.path.join(package_str, os.path.basename(binary_path))),
exist_ok=True)
afl_config_file_path = os.path.join(
volume_path + "/", os.path.join(package_str, afl_config_file_name))
afl_seeds_dir = seeds_dir
afl_out_dir = out_dir + "/afl_fuzz_" + str(uuid.uuid4())
dict_file = None
seeds_dir_usable = False
file_types = []
if os.path.exists(afl_config_file_path):
with open(afl_config_file_path) as fp:
input_vector_dict = json.load(fp)
if input_vector_dict.get("min_seeds_dir"):
afl_seeds_dir = input_vector_dict.get("min_seeds_dir")
for file in os.listdir(afl_seeds_dir):
if os.path.getsize(os.path.join(afl_seeds_dir, file)) > 0:
seeds_dir_usable = True
if not seeds_dir_usable:
print("The given seeds dir for {0}:{1} seems unusable".
format(package, binary_path))
return
if not seeds_dir_usable: # As a fallback option, just use the filetype
return
# afl_seeds_dir = input_vector_dict.get("file_type")
os.makedirs("mockseeds", exist_ok=True)
with open("mockseeds/mockseed", "w") as fp:
fp.write("0")
afl_seeds_dir = "mockseeds"
if input_vector_dict.get("file_types"):
# file_type_dir = input_vector_dict.get("file_types")
file_types = input_vector_dict.get(
"file_types"
) # os.path.basename(file_type_dir).split("_")[0] # Format is: seeds//jpg_samples
# if input_vector_dict.get("min_seeds_dir"):
# afl_seeds_dir = input_vector_dict.get("min_seeds_dir")
# else:
# afl_seeds_dir = input_vector_dict["seeds_dir"]
input_vector_dict["fuzzer_started"] = True
input_vector_dict["status"] = config_settings.Status.FUZZING
input_vector_dict["afl_out_dir"] = afl_out_dir
else:
input_vector = CliConfig(invocation=parameter,
filetypes=seeds_dir,
binary_path=binary_path)
input_vector_dict = input_vector.__dict__
input_vector_dict["afl_out_dir"] = afl_out_dir
input_vector_dict["package"] = package
afl_seeds_dir = seeds_dir
if name:
input_vector_dict["name"] = name
input_vector_dict["fuzzer_started"] = True
input_vector_dict["status"] = config_settings.Status.FUZZING
binary_path = binary_path
with open(afl_config_file_path, "w") as jsonfp:
json.dump(input_vector_dict, jsonfp)
if os.path.exists(
os.path.join(
"/fuzz_dictionaries/dictionaries/",
os.path.basename(afl_seeds_dir).split("_")[0] + ".dict")):
dict_file = os.path.join(
"/fuzz_dictionaries/dictionaries/",
os.path.basename(afl_seeds_dir).split("_")[0] + ".dict")
for file_type in file_types:
if file_type and os.path.exists(
os.path.join(
"/fuzz_dictionaries/dictionaries/",
os.path.basename(file_type).split("_")[0] + ".dict")):
dict_file = os.path.join(
"/fuzz_dictionaries/dictionaries/",
os.path.basename(file_type).split("_")[0] + ".dict")
break
res = start_fuzzer(input_dir=afl_seeds_dir,
afl_out_dir=afl_out_dir,
binary_path=binary_path,
parameter=parameter,
qemu=qemu,
timeout=timeout,
wrapper_function=wrapper_function,
fuzz_duration=fuzz_duration,
dict_file=dict_file,
log_dict=log_dict)
if res == False and not log_dict:
sys.exit(-1)
elif res == False and log_dict:
return False
else:
# with open(os.path.join(volume_path + "/", package + "/" + afl_config_file_name), "w") as jsonfp:
# json.dump(input_vector_dict, jsonfp)
return True
logfilename = os.path.join(
os.path.join(arguments.output_volume, arguments.package),
os.path.basename(arguments.binary) + ".fuzzlog")
else:
logfilename = os.path.join(
os.path.join(arguments.output_volume,
os.path.basename(arguments.binary) + ".fuzzlog"))
with_qemu = arguments.qemu
if arguments.logfile:
logfilename = arguments.logfile
logging.basicConfig(filename=logfilename,
level=logging.INFO,
format='%(levelname)s %(asctime)s: %(message)s')
config = CliConfig(invocation=parameter,
filetype=arguments.seeds,
binary_path=arguments.binary)
chmod = sh.Command("chmod")
chmod("-R", "0777", arguments.output_volume
) # Hacky fix for the problem that docker stores every as root
try:
if arguments.afl_out_file:
res = prepare_and_start_fuzzer(
parameter=parameter,
seeds_dir=arguments.seeds,
binary_path=arguments.binary,
package=arguments.package,
volume_path=arguments.output_volume,
afl_config_file_name=arguments.afl_out_file,
qemu=with_qemu,
name=arguments.name,