def __build_base_image(self, config): # This does not seem to be the right way to get the container tag? self.container_tag = config.get_container_tag() all_images = docker.images(a=True) import pdb; pdb.set_trace() if self.container_tag in all_images: return print ("Container not found or changes in watched files. Rebuilding base container (%s)..." % self.container_tag) base_name = 'nose-docker-base-%s' % self.container_tag all_containers = docker.ps(a=True) if base_name in all_containers: docker.rm('-f', base_name) docker.run( '--name=%s' % base_name, '-v', '%s:/app' % abspath(os.curdir), config.base_image, '/bin/bash', c="cd /app && %s" % ( " && ".join(config.build_commands) ), _out=sys.stdout ) docker.commit('nose-docker-base-%s' % self.container_tag, 'nose-docker:%s' % self.container_tag)
def __build_base_image(self, config): # This does not seem to be the right way to get the container tag? self.container_tag = config.get_container_tag() all_images = docker.images(a=True) import pdb pdb.set_trace() if self.container_tag in all_images: return print( "Container not found or changes in watched files. Rebuilding base container (%s)..." % self.container_tag) base_name = 'nose-docker-base-%s' % self.container_tag all_containers = docker.ps(a=True) if base_name in all_containers: docker.rm('-f', base_name) docker.run('--name=%s' % base_name, '-v', '%s:/app' % abspath(os.curdir), config.base_image, '/bin/bash', c="cd /app && %s" % (" && ".join(config.build_commands)), _out=sys.stdout) docker.commit('nose-docker-base-%s' % self.container_tag, 'nose-docker:%s' % self.container_tag)
def main(dataset): tmpname="kolab/kolabtestcontainer:tmppopulated" imagename="kolab/kolabtestcontainer:populated-"+dataset basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config) print("Building tmpcontainer...") docker.build(settings.dockerCacheString(), "-t", tmpname, "{basedir}/.".format(basedir=basedir)) print("Starting tmpcontainer...") container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir, dataset=dataset), '-v', "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config), tmpname).rstrip() try: # Wait for imap to become available on imaps://localhost:993 time.sleep(5) print "Populate OU..." docker("exec", container, "python2", "/populate/populate_ou.py", _out=sys.stdout) print "Populate users..." docker("exec", container, "python2", "/populate/populate_users.py", _out=sys.stdout) print "Populate resources..." docker("exec", container, "python2", "/populate/populate_resources.py", _out=sys.stdout) print("Running populate.sh...") docker("exec", container, "/data/populate.sh", _out=sys.stdout) # Give kolabd some time to create all mailboxes time.sleep(5) docker("exec", container, "patch", "-R", "/etc/kolab/kolab.conf", "/data/kolab.conf.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/calendar.inc.php", "/data/calendar.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/config.inc.php", "/data/config.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_addressbook.inc.php", "/data/kolab_addressbook.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_auth.inc.php", "/data/kolab_auth.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/password.inc.php", "/data/password.inc.php.diff", _out=sys.stdout) print("Comitting results to: {}".format(imagename)) docker.commit(container, imagename) except: print("Unexpected error:", sys.exc_info()[0]) traceback.print_exc(file=sys.stdout) print("Failed to setup container") docker.stop(container) docker.rm(container)
def main(dataset): tmpname="kolab/kolabtestcontainer:tmppopulated" imagename="kolab/kolabtestcontainer:populated-"+dataset SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) print("Building tmpcontainer...") docker.build("-t", tmpname, SCRIPT_DIR+"/kolabpopulated/.") print("Starting tmpcontainer...") container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolabpopulated/"+dataset+"/:/data/", tmpname).rstrip() # Wait for imap to become available on imaps://localhost:993 time.sleep(5) print("Running populate.sh...") docker("exec", container, "/data/populate.sh", _out=process_output) print("Comitting results...") docker.commit(container, imagename) docker.stop(container) docker.rm(container)
def main(): tmpname = settings.kolabimagename("tmpbase") SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) print("Building tmpcontainer...") docker("build", "-t", tmpname, SCRIPT_DIR+"/kolab/") print("Starting tmpcontainer...") print(SCRIPT_DIR+"/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh") container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolab/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh", tmpname).rstrip() print("Setting up kolab") docker("exec", container, "setup-kolab", "--default", "--timezone="+settings.TIMEZONE, "--directory-manager-pwd="+settings.LDAPPW, "--mysqlserver=new", _out=process_output) print("Fixing roundcube") docker("exec", container, "bash", "/usr/share/roundcubemail/fixRoundcubeT243.sh", _out=process_output) docker("exec", container, "systemctl", "restart", "httpd", _out=process_output) print("Comitting results...") docker.commit(container, settings.kolabimagename("base")) docker.stop(container) docker.rm(container)
def main(dataset): tmpname = "kolab/kolabtestcontainer:tmppopulated" imagename = "kolab/kolabtestcontainer:populated-" + dataset basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config) print("Building tmpcontainer...") docker.build("-t", tmpname, "{basedir}/.".format(basedir=basedir)) print("Starting tmpcontainer...") container = docker.run( "-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir, dataset=dataset), '-v', "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config), tmpname).rstrip() try: # Wait for imap to become available on imaps://localhost:993 time.sleep(5) print "Populate OU..." docker("exec", container, "python2", "/populate/populate_ou.py", _out=sys.stdout) print "Populate users..." docker("exec", container, "python2", "/populate/populate_users.py", _out=sys.stdout) print "Populate resources..." docker("exec", container, "python2", "/populate/populate_resources.py", _out=sys.stdout) print("Running populate.sh...") docker("exec", container, "/data/populate.sh", _out=sys.stdout) # Give kolabd some time to create all mailboxes time.sleep(5) docker("exec", container, "patch", "-R", "/etc/kolab/kolab.conf", "/data/kolab.conf.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/calendar.inc.php", "/data/calendar.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/config.inc.php", "/data/config.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_addressbook.inc.php", "/data/kolab_addressbook.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_auth.inc.php", "/data/kolab_auth.inc.php.diff", _out=sys.stdout) docker("exec", container, "patch", "-R", "/etc/roundcubemail/password.inc.php", "/data/password.inc.php.diff", _out=sys.stdout) print("Comitting results to: {}".format(imagename)) docker.commit(container, imagename) except: print("Unexpected error:", sys.exc_info()[0]) traceback.print_exc(file=sys.stdout) print("Failed to setup container") docker.stop(container) docker.rm(container)
def run(package: str, configuration_dir: str, binary_name: str, qemu: bool, minimize: bool, timeout: float, fuzz_duration: float): if os.path.exists(configuration_dir): print( "Skipping {0}. Directory already exists".format(configuration_dir)) return reset(package) print("Starting qemu={0},minimize={1},Fuzzing timeout={2}".format( qemu, minimize, timeout)) start = time.time() container_name = package + "_build" # +"_"+str(uuid.uuid4())[:8] image_name = package + "_image" # +"_"+str(uuid.uuid4())[:8] timecommand = sh.Command("time") # type: sh.Command docker_args = [ "--name", container_name, "--entrypoint", "python", "pacmanfuzzer", "/inputinferer/configfinder/builder_wrapper.py", "-p", package ] if qemu: docker_args.append("-Q") with timecommand(_with=True) as timeprocess: print("Building") build_process = docker.run(docker_args, _ok_code=[0, 1, 2], _out=sys.stdout, _err=sys.stderr) # type: sh.RunningCommand if not qemu and build_process.exit_code == 2: print("WITHOUT QEMU: Failed") return with timecommand(_with=True): docker.commit([container_name, image_name], _out=sys.stdout, _err=sys.stderr) docker_args = [ "--rm", "--cap-add=SYS_PTRACE", "-v", configuration_dir + ":/results", "--entrypoint", "python", image_name, "/inputinferer/configfinder/config_finder_for_pacman_package.py", "-p", package, "-v", "/results/" ] if qemu: docker_args.append("-Q") with timecommand(_with=True): print("Finding the input vector") input_process = docker.run(docker_args, _out=sys.stdout, _err=sys.stderr) print(input_process.cmd) with open( os.path.join(configuration_dir, package + "/" + binary_name + ".json")) as binary_name_fp: config_dict = json.load(binary_name_fp)[0] seeds = config_dict["file_type"] parameter = config_dict["parameter"] binary_path = config_dict["binary_path"] if not os.path.exists( os.path.join(configuration_dir, package + "/" + binary_name)): os.mkdir(os.path.join(configuration_dir, package + "/" + binary_name)) if minimize: docker_args = [ "--rm", "--cap-add=SYS_PTRACE", "-v", configuration_dir + ":/results", "--entrypoint", "python", image_name, "/inputinferer/configfinder/controller.py", "minimize", "-p", package, "-v", "/results", "-s", seeds, "--parameter=" + parameter, "-b", binary_path, "-afile", binary_name + ".afl_config" ] if qemu: docker_args.append("-Q") with timecommand(_with=True): print("Minimizing") docker.run(docker_args, _out=sys.stdout, _err=sys.stderr) with open( os.path.join(configuration_dir, package + "/" + binary_name + ".afl_config")) as afl_config_fp: seeds = json.load(afl_config_fp)["min_seeds_dir"] docker_args = [ "--rm", "--cap-add=SYS_PTRACE", "-v", configuration_dir + ":/results", "--entrypoint", "python", image_name, "/inputinferer/configfinder/controller.py", "evalfuzz" ] if fuzz_duration: docker_args += ["-ft", fuzz_duration] if timeout: docker_args += ["-t", timeout] docker_args += [ "-p", package, "-v", "/results", "-s", seeds, "--parameter=" + parameter, "-b", binary_path, "-afile", binary_name + ".afl_config" ] if qemu: docker_args.append("-Q") with timecommand(_with=True): print("Fuzzing") docker.run(docker_args, _out=sys.stdout, _err=sys.stderr) print("Done") end = time.time() print("Time elapsed: ", str(end - start))
def build_and_commit(package: str, fuzzer_image: str, json_output_path: str = None, qemu=False, timeout=None) -> str: """ This builds a package inside a docker container and then commits the container to an image. :return: """ start = time.time() docker_image_name = package + "_" + str(uuid.uuid4())[:8] docker_container_name = str(uuid.uuid4()) try: if not qemu: build_process = docker.run( '--cpus=0.90', "--privileged", "--name", docker_container_name, "--entrypoint", "python", fuzzer_image, "/inputinferer/configfinder/builder_wrapper.py", "-p", package, _out=sys.stdout, _ok_code=[ config_settings.BUILDER_BUILD_NORMAL, config_settings.BUILDER_BUILD_FAILED, config_settings.BUILDER_BUILD_QEMU ], _timeout=timeout) # type: sh.RunningCommand else: build_process = docker.run( '--cpus=0.90', "--privileged", "--name", docker_container_name, "--entrypoint", "python", fuzzer_image, "/inputinferer/configfinder/builder_wrapper.py", "-p", package, "-Q", _out=sys.stdout, _ok_code=[ config_settings.BUILDER_BUILD_NORMAL, config_settings.BUILDER_BUILD_FAILED, config_settings.BUILDER_BUILD_QEMU ], _timeout=timeout) # type: sh.RunningCommand except sh.TimeoutException as e: print("Building {0} timed out!".format(package)) return None exit_code = build_process.exit_code if exit_code == -1: print("Failed to build image for package {0}, not commiting".format( package)) return None docker.commit(docker_container_name, docker_image_name, _out=sys.stdout) end = time.time() if json_output_path is not None: json_dict = {} json_dict["docker_image_name"] = docker_image_name if exit_code == config_settings.BUILDER_BUILD_NORMAL: json_dict["qemu"] = False elif exit_code == config_settings.BUILDER_BUILD_QEMU: json_dict["qemu"] = True json_dict["time"] = end - start with open(json_output_path, "w") as json_output_fp: json.dump(json_dict, json_output_fp) docker.rm(docker_container_name) # Remove the image after we commited return docker_image_name