Beispiel #1
0
 def infer_filetypes_via_coverage(self) -> [(str, str, int)]:
     """
     For each inferred parameter, this function find the corresponding best fitting filetype.
     :return: A list of triples: (parameter,filetype,coverage)
     """
     self.cli_config_list = []
     for param in self.parameters:
         inference_result = self.infer_filetype_via_coverage_for_parameter_parallel(
             param)
         if inference_result is None:  # Everything yielded the same coverage
             continue  # Next one
         max_files, max_covs, took_max_file = inference_result
         p = "None"
         if param:
             p = param
         c = CliConfig(invocation=param,
                       filetypes=max_files,
                       coverage_list=self.coverage_lists[p],
                       coverages=max_covs,
                       binary_path=self.binary_path,
                       max_coverage=max(max_covs),
                       took_max_file=took_max_file)
         if self.failed_invocations >= self.FAILED_INVOCATIONS_THRESHOLD:
             c.invocation_always_possible = False
         c.qemu = self.qemu
         self.cli_config_list.append(c)
     return sorted(self.cli_config_list,
                   key=operator.attrgetter("max_coverage"),
                   reverse=True)
Beispiel #2
0
 def ui_command_saveconfig(self):
     '''
     Saves the whole configuration tree to disk so that it will be restored
     on next boot. Unless you do that, changes are lost accross reboots.
     '''
     self.assert_root()
     try:
         input = raw_input("Save configuration? [Y/n]: ")
     except EOFError:
         input = None
         self.shell.con.display('')
     if input in ["y", "Y", ""]:
         CliConfig.save_running_config()
     else:
         self.shell.log.warning("Configuration not saved.")
Beispiel #3
0
 def ui_command_saveconfig(self):
     '''
     Saves the whole configuration tree to disk so that it will be restored
     on next boot. Unless you do that, changes are lost accross reboots.
     '''
     self.assert_root()
     try:
         input = raw_input("Save configuration? [Y/n]: ")
     except EOFError:
         input = None
         self.shell.con.display('')
     if input in ["y", "Y", ""]:
         CliConfig.save_running_config()
     else:
         self.shell.log.warning("Configuration not saved.")
Beispiel #4
0
 def ui_command_saveconfig(self):
     '''
     Saves the whole configuration tree to disk so that it will be restored
     on next boot. Unless you do that, changes are lost accross reboots.
     '''
     self.assert_root()
     self.shell.con.display("WARNING: Saving %s current configuration to "
                            % gethostname()
                            + "disk will overwrite your boot settings.")
     self.shell.con.display("The current target configuration will become "
                            + "the default boot config.")
     try:
         input = raw_input("Are you sure? Type 'yes': ")
     except EOFError:
         input = None
         self.shell.con.display('')
     if input == "yes":
         CliConfig.save_running_config()
     else:
         self.shell.log.warning("Aborted, startup configuration left untouched.")
Beispiel #5
0
def minize(parameter: str,
           seeds_dir: str,
           binary_path: str,
           package: str,
           volume_path: str,
           afl_config_file_name: str,
           qemu: bool = False,
           name: str = None,
           tmin_total_time: int = None,
           do_tmin: bool = True,
           cores: int = 1) -> bool:
    # if package:
    #    b = builder.Builder(package=package, qemu=qemu)
    #    if not b.install():
    #        print("Could not install package, exiting")
    if not package:
        package = os.path.basename(binary_path)
    input_vector = CliConfig(invocation=parameter,
                             filetypes=seeds_dir.split(";"),
                             binary_path=binary_path)
    out_dir = volume_path + "/" + package + "/" + os.path.basename(
        input_vector.binary_path)
    if not os.path.exists(out_dir):
        os.makedirs(out_dir, exist_ok=True)
    min_seeds_dir = os.path.join(out_dir, "minseeds_" + str(uuid.uuid4()))
    file_list = []
    print(seeds_dir)
    for filetype_seeds in seeds_dir.split(";"):
        for file in os.listdir(filetype_seeds):
            file_path = os.path.join(filetype_seeds, file)
            if os.path.isfile(file_path):
                file_list.append(file_path)
            else:
                print("Ignored non-file at {}".format(file_path))

    os.makedirs(min_seeds_dir, exist_ok=True)
    for file in file_list:
        if os.path.getsize(
                file
        ) < 850 * 1000:  # If file is smaller than 850 kb just copy it
            copyfile(file, os.path.join(min_seeds_dir, os.path.basename(file)))
        else:  # Reduce the file to 1kb
            helpers.utils.crop_file(
                file, os.path.join(min_seeds_dir, os.path.basename(file)),
                1 * 1000)
    dump_into_json(input_vector=input_vector,
                   min_seeds_dir=min_seeds_dir,
                   package=package,
                   name=name,
                   volume_path=volume_path,
                   afl_config_file_name=afl_config_file_name)
    use_qemu = helpers.utils.qemu_required_for_binary(
        input_vector.binary_path)  # TODO: There seems to be an issue here
    # First: Minimize the seeds
    cmin_dir = os.path.join(out_dir, "afl_cmin_" + str(uuid.uuid4()))
    cmin_params = []
    if use_qemu:
        cmin_params.append("-Q")
    cmin_params += [
        "-I", "-i", min_seeds_dir, "-o", cmin_dir, "-m", "none", "-t",
        str(config_settings.AFL_CMIN_INVOKE_TIMEOUT * 1000), "--",
        input_vector.binary_path
    ]
    if input_vector.parameter:
        cmin_params += input_vector.parameter.split(" ")
    print("Calling afl-cmin {0}".format(" ".join(cmin_params)), flush=True)
    try:
        afl_cmin(cmin_params,
                 _timeout=config_settings.AFL_CMIN_TIMEOUT,
                 _env=helpers.utils.get_fuzzing_env_for_invocation(parameter),
                 _out=sys.stdout,
                 _error=sys.stderr)
        print("afl cmin done", flush=True)
    except sh.ErrorReturnCode as e:
        print("afl cmin failed for {0}".format(input_vector.binary_path))
        print("STDOUT:\n", e.stdout.decode("utf-8"))
        print("STDERR:\n", e.stderr.decode("utf-8"))
        sys.exit(-1)
    except sh.TimeoutException as e:
        print("afl cmin timed out for {0}".format(input_vector.binary_path))
        # print("STDOUT:\n", e.stdout.decode("utf-8"))
        # print("STDERR:\n", e.stderr.decode("utf-8"))
        for f in os.listdir(min_seeds_dir):
            shutil.copyfile(os.path.join(min_seeds_dir, f),
                            os.path.join(cmin_dir, f))
    dump_into_json(input_vector=input_vector,
                   min_seeds_dir=cmin_dir,
                   package=package,
                   name=name,
                   volume_path=volume_path,
                   afl_config_file_name=afl_config_file_name)
    shutil.rmtree(
        min_seeds_dir)  # We do not want to store the minimized seeds again
    if not do_tmin:
        return True
    tmin_dir = out_dir + "/afl_tmin_" + str(uuid.uuid4())
    os.makedirs(tmin_dir, exist_ok=True)
    for iter_num, file in enumerate(os.listdir(cmin_dir)):
        tmin_params = []
        if use_qemu:
            tmin_params.append("-Q")
        if os.path.isdir(os.path.join(
                cmin_dir, file)):  # Maybe .traces is still in there or so?
            continue
        tmin_params += [
            "-t",
            str(config_settings.AFL_TMIN_INVOKE_TIMEOUT * 1000), "-i",
            cmin_dir + "/" + file, "-o", tmin_dir + "/" + file, "-m", "none",
            "--", input_vector.binary_path
        ]
        if input_vector.parameter:
            tmin_params += input_vector.parameter.split(" ")
        if iter_num == 0:
            print("Minimizing seed file size", flush=True)
        print("Calling afl-tmin {0}".format(" ".join(tmin_params)), flush=True)
        if tmin_total_time:
            afl_tmin_timeout_per_file = tmin_total_time / len(
                os.listdir(cmin_dir))  # Divide time per file
        else:
            afl_tmin_timeout_per_file = config_settings.AFL_TMIN_TIMEOUT / len(
                os.listdir(cmin_dir))  # Divide time per file
        try:
            helpers.utils.temp_print("Calling afl-tmin {0}".format(
                " ".join(tmin_params)))
            afl_tmin(
                tmin_params,
                _out=sys.stdout,
                _timeout=afl_tmin_timeout_per_file,
                _timeout_signal=signal.SIGTERM,
                _env=config_settings.get_fuzzing_env_without_desock()
            )  # AFL tmin can be very slow. We should only use a limited amount of time on it. If we send SIGTERM though, the current progress is saved
        except sh.ErrorReturnCode as e:
            print("afl tmin failed for {0}".format(input_vector.binary_path))
            print("STDOUT:\n", e.stdout.decode("utf-8"))
            print("STDERR:\n", e.stderr.decode("utf-8"))
            if aflerrors["AFL_TIMEOUT"] in e.stderr.decode("utf-8"):
                print(
                    "afl-tmin timed out for {0}. Going to use current progress or raw file"
                    .format(package))
                if not os.path.exists(os.path.join(tmin_dir, file)):
                    copyfile(os.path.join(cmin_dir, file),
                             os.path.join(tmin_dir, file))
            else:
                sys.exit(-1)
        except sh.TimeoutException as e:
            # print("STDOUT:\n", e.stdout.decode("utf-8"))
            # print("STDERR:\n", e.stderr.decode("utf-8"))
            print(
                "afl-tmin timed out for {0}. Going to use current progress or raw file"
                .format(package))
            if not os.path.exists(os.path.join(tmin_dir, file)):
                copyfile(os.path.join(cmin_dir, file),
                         os.path.join(tmin_dir, file))
                # seeds_dir = cmin_dir
    dump_into_json(input_vector=input_vector,
                   min_seeds_dir=tmin_dir,
                   package=package,
                   name=name,
                   volume_path=volume_path,
                   afl_config_file_name=afl_config_file_name)
    return True
Beispiel #6
0
def prepare_and_start_fuzzer(parameter: str,
                             seeds_dir: str,
                             binary_path: str,
                             package: str,
                             volume_path: str,
                             afl_config_file_name: str,
                             qemu: bool = False,
                             name: str = None,
                             timeout: str = None,
                             wrapper_function=afl_fuzz_wrapper,
                             fuzz_duration: int = None,
                             log_dict=None,
                             file_types=None):
    if file_types is None:
        file_types = []
    print("Now starting the fuzzing", flush=True)
    package_str = ""
    if package:
        package_str = package
    else:
        package_str = os.path.basename(binary_path)
    out_dir = os.path.join(
        volume_path + "/",
        os.path.join(package_str, os.path.basename(binary_path)))
    os.makedirs(os.path.join(
        volume_path + "/",
        os.path.join(package_str, os.path.basename(binary_path))),
                exist_ok=True)
    afl_config_file_path = os.path.join(
        volume_path + "/", os.path.join(package_str, afl_config_file_name))
    afl_seeds_dir = seeds_dir
    afl_out_dir = out_dir + "/afl_fuzz_" + str(uuid.uuid4())
    dict_file = None
    seeds_dir_usable = False
    file_types = []
    if os.path.exists(afl_config_file_path):
        with open(afl_config_file_path) as fp:
            input_vector_dict = json.load(fp)
            if input_vector_dict.get("min_seeds_dir"):
                afl_seeds_dir = input_vector_dict.get("min_seeds_dir")
                for file in os.listdir(afl_seeds_dir):
                    if os.path.getsize(os.path.join(afl_seeds_dir, file)) > 0:
                        seeds_dir_usable = True
                if not seeds_dir_usable:
                    print("The given seeds dir for {0}:{1} seems unusable".
                          format(package, binary_path))
                    return
            if not seeds_dir_usable:  # As a fallback option, just use the filetype
                return
                # afl_seeds_dir = input_vector_dict.get("file_type")
                os.makedirs("mockseeds", exist_ok=True)
                with open("mockseeds/mockseed", "w") as fp:
                    fp.write("0")
                afl_seeds_dir = "mockseeds"

            if input_vector_dict.get("file_types"):
                # file_type_dir = input_vector_dict.get("file_types")
                file_types = input_vector_dict.get(
                    "file_types"
                )  # os.path.basename(file_type_dir).split("_")[0] # Format is: seeds//jpg_samples

            # if input_vector_dict.get("min_seeds_dir"):
            #    afl_seeds_dir = input_vector_dict.get("min_seeds_dir")
            # else:
            #    afl_seeds_dir = input_vector_dict["seeds_dir"]
            input_vector_dict["fuzzer_started"] = True
            input_vector_dict["status"] = config_settings.Status.FUZZING
            input_vector_dict["afl_out_dir"] = afl_out_dir

    else:
        input_vector = CliConfig(invocation=parameter,
                                 filetypes=seeds_dir,
                                 binary_path=binary_path)
        input_vector_dict = input_vector.__dict__
        input_vector_dict["afl_out_dir"] = afl_out_dir
        input_vector_dict["package"] = package
        afl_seeds_dir = seeds_dir
        if name:
            input_vector_dict["name"] = name
        input_vector_dict["fuzzer_started"] = True
        input_vector_dict["status"] = config_settings.Status.FUZZING
    binary_path = binary_path
    with open(afl_config_file_path, "w") as jsonfp:
        json.dump(input_vector_dict, jsonfp)
    if os.path.exists(
            os.path.join(
                "/fuzz_dictionaries/dictionaries/",
                os.path.basename(afl_seeds_dir).split("_")[0] + ".dict")):
        dict_file = os.path.join(
            "/fuzz_dictionaries/dictionaries/",
            os.path.basename(afl_seeds_dir).split("_")[0] + ".dict")
    for file_type in file_types:
        if file_type and os.path.exists(
                os.path.join(
                    "/fuzz_dictionaries/dictionaries/",
                    os.path.basename(file_type).split("_")[0] + ".dict")):
            dict_file = os.path.join(
                "/fuzz_dictionaries/dictionaries/",
                os.path.basename(file_type).split("_")[0] + ".dict")
            break
    res = start_fuzzer(input_dir=afl_seeds_dir,
                       afl_out_dir=afl_out_dir,
                       binary_path=binary_path,
                       parameter=parameter,
                       qemu=qemu,
                       timeout=timeout,
                       wrapper_function=wrapper_function,
                       fuzz_duration=fuzz_duration,
                       dict_file=dict_file,
                       log_dict=log_dict)
    if res == False and not log_dict:
        sys.exit(-1)
    elif res == False and log_dict:
        return False
    else:
        # with open(os.path.join(volume_path + "/", package + "/" + afl_config_file_name), "w") as jsonfp:
        #    json.dump(input_vector_dict, jsonfp)
        return True
Beispiel #7
0
        logfilename = os.path.join(
            os.path.join(arguments.output_volume, arguments.package),
            os.path.basename(arguments.binary) + ".fuzzlog")
    else:
        logfilename = os.path.join(
            os.path.join(arguments.output_volume,
                         os.path.basename(arguments.binary) + ".fuzzlog"))
        with_qemu = arguments.qemu
    if arguments.logfile:
        logfilename = arguments.logfile
    logging.basicConfig(filename=logfilename,
                        level=logging.INFO,
                        format='%(levelname)s %(asctime)s: %(message)s')

    config = CliConfig(invocation=parameter,
                       filetype=arguments.seeds,
                       binary_path=arguments.binary)
    chmod = sh.Command("chmod")
    chmod("-R", "0777", arguments.output_volume
          )  # Hacky fix for the problem that docker stores every as root
    try:
        if arguments.afl_out_file:
            res = prepare_and_start_fuzzer(
                parameter=parameter,
                seeds_dir=arguments.seeds,
                binary_path=arguments.binary,
                package=arguments.package,
                volume_path=arguments.output_volume,
                afl_config_file_name=arguments.afl_out_file,
                qemu=with_qemu,
                name=arguments.name,