def __build_base_image(self, config):
# This does not seem to be the right way to get the container tag?
self.container_tag = config.get_container_tag()
all_images = docker.images(a=True)
import pdb; pdb.set_trace()
if self.container_tag in all_images:
return
print ("Container not found or changes in watched files. Rebuilding base container (%s)..." % self.container_tag)
base_name = 'nose-docker-base-%s' % self.container_tag
all_containers = docker.ps(a=True)
if base_name in all_containers:
docker.rm('-f', base_name)
docker.run(
'--name=%s' % base_name,
'-v',
'%s:/app' % abspath(os.curdir),
config.base_image,
'/bin/bash',
c="cd /app && %s" % (
" && ".join(config.build_commands)
),
_out=sys.stdout
)
docker.commit('nose-docker-base-%s' % self.container_tag, 'nose-docker:%s' % self.container_tag)
def __build_base_image(self, config):
# This does not seem to be the right way to get the container tag?
self.container_tag = config.get_container_tag()
all_images = docker.images(a=True)
import pdb
pdb.set_trace()
if self.container_tag in all_images:
return
print(
"Container not found or changes in watched files. Rebuilding base container (%s)..."
% self.container_tag)
base_name = 'nose-docker-base-%s' % self.container_tag
all_containers = docker.ps(a=True)
if base_name in all_containers:
docker.rm('-f', base_name)
docker.run('--name=%s' % base_name,
'-v',
'%s:/app' % abspath(os.curdir),
config.base_image,
'/bin/bash',
c="cd /app && %s" % (" && ".join(config.build_commands)),
_out=sys.stdout)
docker.commit('nose-docker-base-%s' % self.container_tag,
'nose-docker:%s' % self.container_tag)
def main(dataset):
tmpname="kolab/kolabtestcontainer:tmppopulated"
imagename="kolab/kolabtestcontainer:populated-"+dataset
basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)
print("Building tmpcontainer...")
docker.build(settings.dockerCacheString(), "-t", tmpname, "{basedir}/.".format(basedir=basedir))
print("Starting tmpcontainer...")
container = docker.run("-d", "-h", settings.HOSTNAME,
"-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro",
"-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir, dataset=dataset),
'-v', "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
tmpname).rstrip()
try:
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print "Populate OU..."
docker("exec", container, "python2", "/populate/populate_ou.py", _out=sys.stdout)
print "Populate users..."
docker("exec", container, "python2", "/populate/populate_users.py", _out=sys.stdout)
print "Populate resources..."
docker("exec", container, "python2", "/populate/populate_resources.py", _out=sys.stdout)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=sys.stdout)
# Give kolabd some time to create all mailboxes
time.sleep(5)
docker("exec", container, "patch", "-R", "/etc/kolab/kolab.conf", "/data/kolab.conf.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/calendar.inc.php", "/data/calendar.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/config.inc.php", "/data/config.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_addressbook.inc.php", "/data/kolab_addressbook.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_auth.inc.php", "/data/kolab_auth.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/password.inc.php", "/data/password.inc.php.diff", _out=sys.stdout)
print("Comitting results to: {}".format(imagename))
docker.commit(container, imagename)
except:
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc(file=sys.stdout)
print("Failed to setup container")
docker.stop(container)
docker.rm(container)
def main(dataset):
tmpname="kolab/kolabtestcontainer:tmppopulated"
imagename="kolab/kolabtestcontainer:populated-"+dataset
SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
print("Building tmpcontainer...")
docker.build("-t", tmpname, SCRIPT_DIR+"/kolabpopulated/.")
print("Starting tmpcontainer...")
container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolabpopulated/"+dataset+"/:/data/", tmpname).rstrip()
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=process_output)
print("Comitting results...")
docker.commit(container, imagename)
docker.stop(container)
docker.rm(container)
def main():
tmpname = settings.kolabimagename("tmpbase")
SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
print("Building tmpcontainer...")
docker("build", "-t", tmpname, SCRIPT_DIR+"/kolab/")
print("Starting tmpcontainer...")
print(SCRIPT_DIR+"/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh")
container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolab/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh", tmpname).rstrip()
print("Setting up kolab")
docker("exec", container, "setup-kolab", "--default", "--timezone="+settings.TIMEZONE, "--directory-manager-pwd="+settings.LDAPPW, "--mysqlserver=new", _out=process_output)
print("Fixing roundcube")
docker("exec", container, "bash", "/usr/share/roundcubemail/fixRoundcubeT243.sh", _out=process_output)
docker("exec", container, "systemctl", "restart", "httpd", _out=process_output)
print("Comitting results...")
docker.commit(container, settings.kolabimagename("base"))
docker.stop(container)
docker.rm(container)
def main(dataset):
tmpname = "kolab/kolabtestcontainer:tmppopulated"
imagename = "kolab/kolabtestcontainer:populated-" + dataset
basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)
print("Building tmpcontainer...")
docker.build("-t", tmpname, "{basedir}/.".format(basedir=basedir))
print("Starting tmpcontainer...")
container = docker.run(
"-d", "-h", settings.HOSTNAME, "-v",
"/sys/fs/cgroup:/sys/fs/cgroup:ro",
"-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir,
dataset=dataset), '-v',
"{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
tmpname).rstrip()
try:
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print "Populate OU..."
docker("exec",
container,
"python2",
"/populate/populate_ou.py",
_out=sys.stdout)
print "Populate users..."
docker("exec",
container,
"python2",
"/populate/populate_users.py",
_out=sys.stdout)
print "Populate resources..."
docker("exec",
container,
"python2",
"/populate/populate_resources.py",
_out=sys.stdout)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=sys.stdout)
# Give kolabd some time to create all mailboxes
time.sleep(5)
docker("exec",
container,
"patch",
"-R",
"/etc/kolab/kolab.conf",
"/data/kolab.conf.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/calendar.inc.php",
"/data/calendar.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/config.inc.php",
"/data/config.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/kolab_addressbook.inc.php",
"/data/kolab_addressbook.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/kolab_auth.inc.php",
"/data/kolab_auth.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/password.inc.php",
"/data/password.inc.php.diff",
_out=sys.stdout)
print("Comitting results to: {}".format(imagename))
docker.commit(container, imagename)
except:
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc(file=sys.stdout)
print("Failed to setup container")
docker.stop(container)
docker.rm(container)
def run(package: str, configuration_dir: str, binary_name: str, qemu: bool,
minimize: bool, timeout: float, fuzz_duration: float):
if os.path.exists(configuration_dir):
print(
"Skipping {0}. Directory already exists".format(configuration_dir))
return
reset(package)
print("Starting qemu={0},minimize={1},Fuzzing timeout={2}".format(
qemu, minimize, timeout))
start = time.time()
container_name = package + "_build" # +"_"+str(uuid.uuid4())[:8]
image_name = package + "_image" # +"_"+str(uuid.uuid4())[:8]
timecommand = sh.Command("time") # type: sh.Command
docker_args = [
"--name", container_name, "--entrypoint", "python", "pacmanfuzzer",
"/inputinferer/configfinder/builder_wrapper.py", "-p", package
]
if qemu:
docker_args.append("-Q")
with timecommand(_with=True) as timeprocess:
print("Building")
build_process = docker.run(docker_args,
_ok_code=[0, 1, 2],
_out=sys.stdout,
_err=sys.stderr) # type: sh.RunningCommand
if not qemu and build_process.exit_code == 2:
print("WITHOUT QEMU: Failed")
return
with timecommand(_with=True):
docker.commit([container_name, image_name],
_out=sys.stdout,
_err=sys.stderr)
docker_args = [
"--rm", "--cap-add=SYS_PTRACE", "-v", configuration_dir + ":/results",
"--entrypoint", "python", image_name,
"/inputinferer/configfinder/config_finder_for_pacman_package.py", "-p",
package, "-v", "/results/"
]
if qemu:
docker_args.append("-Q")
with timecommand(_with=True):
print("Finding the input vector")
input_process = docker.run(docker_args,
_out=sys.stdout,
_err=sys.stderr)
print(input_process.cmd)
with open(
os.path.join(configuration_dir, package + "/" + binary_name +
".json")) as binary_name_fp:
config_dict = json.load(binary_name_fp)[0]
seeds = config_dict["file_type"]
parameter = config_dict["parameter"]
binary_path = config_dict["binary_path"]
if not os.path.exists(
os.path.join(configuration_dir, package + "/" + binary_name)):
os.mkdir(os.path.join(configuration_dir, package + "/" + binary_name))
if minimize:
docker_args = [
"--rm", "--cap-add=SYS_PTRACE", "-v",
configuration_dir + ":/results", "--entrypoint", "python",
image_name, "/inputinferer/configfinder/controller.py", "minimize",
"-p", package, "-v", "/results", "-s", seeds,
"--parameter=" + parameter, "-b", binary_path, "-afile",
binary_name + ".afl_config"
]
if qemu:
docker_args.append("-Q")
with timecommand(_with=True):
print("Minimizing")
docker.run(docker_args, _out=sys.stdout, _err=sys.stderr)
with open(
os.path.join(configuration_dir, package + "/" + binary_name +
".afl_config")) as afl_config_fp:
seeds = json.load(afl_config_fp)["min_seeds_dir"]
docker_args = [
"--rm", "--cap-add=SYS_PTRACE", "-v", configuration_dir + ":/results",
"--entrypoint", "python", image_name,
"/inputinferer/configfinder/controller.py", "evalfuzz"
]
if fuzz_duration:
docker_args += ["-ft", fuzz_duration]
if timeout:
docker_args += ["-t", timeout]
docker_args += [
"-p", package, "-v", "/results", "-s", seeds,
"--parameter=" + parameter, "-b", binary_path, "-afile",
binary_name + ".afl_config"
]
if qemu:
docker_args.append("-Q")
with timecommand(_with=True):
print("Fuzzing")
docker.run(docker_args, _out=sys.stdout, _err=sys.stderr)
print("Done")
end = time.time()
print("Time elapsed: ", str(end - start))
def build_and_commit(package: str,
fuzzer_image: str,
json_output_path: str = None,
qemu=False,
timeout=None) -> str:
"""
This builds a package inside a docker container and then commits the container to an image.
:return:
"""
start = time.time()
docker_image_name = package + "_" + str(uuid.uuid4())[:8]
docker_container_name = str(uuid.uuid4())
try:
if not qemu:
build_process = docker.run(
'--cpus=0.90',
"--privileged",
"--name",
docker_container_name,
"--entrypoint",
"python",
fuzzer_image,
"/inputinferer/configfinder/builder_wrapper.py",
"-p",
package,
_out=sys.stdout,
_ok_code=[
config_settings.BUILDER_BUILD_NORMAL,
config_settings.BUILDER_BUILD_FAILED,
config_settings.BUILDER_BUILD_QEMU
],
_timeout=timeout) # type: sh.RunningCommand
else:
build_process = docker.run(
'--cpus=0.90',
"--privileged",
"--name",
docker_container_name,
"--entrypoint",
"python",
fuzzer_image,
"/inputinferer/configfinder/builder_wrapper.py",
"-p",
package,
"-Q",
_out=sys.stdout,
_ok_code=[
config_settings.BUILDER_BUILD_NORMAL,
config_settings.BUILDER_BUILD_FAILED,
config_settings.BUILDER_BUILD_QEMU
],
_timeout=timeout) # type: sh.RunningCommand
except sh.TimeoutException as e:
print("Building {0} timed out!".format(package))
return None
exit_code = build_process.exit_code
if exit_code == -1:
print("Failed to build image for package {0}, not commiting".format(
package))
return None
docker.commit(docker_container_name, docker_image_name, _out=sys.stdout)
end = time.time()
if json_output_path is not None:
json_dict = {}
json_dict["docker_image_name"] = docker_image_name
if exit_code == config_settings.BUILDER_BUILD_NORMAL:
json_dict["qemu"] = False
elif exit_code == config_settings.BUILDER_BUILD_QEMU:
json_dict["qemu"] = True
json_dict["time"] = end - start
with open(json_output_path, "w") as json_output_fp:
json.dump(json_dict, json_output_fp)
docker.rm(docker_container_name) # Remove the image after we commited
return docker_image_name
