def __init__(self, name, start_calico=True, dind=True):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
if dind:
docker.rm("-f", self.name, _ok_code=[0, 1])
docker.run("--privileged", "-v", os.getcwd()+":/code", "--name",
self.name,
"-e", "DOCKER_DAEMON_ARGS="
"--kv-store=consul:%s:8500" % utils.get_ip(),
"-tid", "calico/dind")
self.ip = docker.inspect("--format", "{{ .NetworkSettings.IPAddress }}",
self.name).stdout.rstrip()
self.ip6 = docker.inspect("--format",
"{{ .NetworkSettings."
"GlobalIPv6Address }}",
self.name).stdout.rstrip()
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError)
self.execute("docker load --input /code/calico_containers/calico-node.tar && "
"docker load --input /code/calico_containers/busybox.tar")
else:
self.ip = get_ip()
if start_calico:
self.start_calico_node()
self.assert_driver_up()
def __build_base_image(self, config):
# This does not seem to be the right way to get the container tag?
self.container_tag = config.get_container_tag()
all_images = docker.images(a=True)
import pdb; pdb.set_trace()
if self.container_tag in all_images:
return
print ("Container not found or changes in watched files. Rebuilding base container (%s)..." % self.container_tag)
base_name = 'nose-docker-base-%s' % self.container_tag
all_containers = docker.ps(a=True)
if base_name in all_containers:
docker.rm('-f', base_name)
docker.run(
'--name=%s' % base_name,
'-v',
'%s:/app' % abspath(os.curdir),
config.base_image,
'/bin/bash',
c="cd /app && %s" % (
" && ".join(config.build_commands)
),
_out=sys.stdout
)
docker.commit('nose-docker-base-%s' % self.container_tag, 'nose-docker:%s' % self.container_tag)
def __build_base_image(self, config):
# This does not seem to be the right way to get the container tag?
self.container_tag = config.get_container_tag()
all_images = docker.images(a=True)
import pdb
pdb.set_trace()
if self.container_tag in all_images:
return
print(
"Container not found or changes in watched files. Rebuilding base container (%s)..."
% self.container_tag)
base_name = 'nose-docker-base-%s' % self.container_tag
all_containers = docker.ps(a=True)
if base_name in all_containers:
docker.rm('-f', base_name)
docker.run('--name=%s' % base_name,
'-v',
'%s:/app' % abspath(os.curdir),
config.base_image,
'/bin/bash',
c="cd /app && %s" % (" && ".join(config.build_commands)),
_out=sys.stdout)
docker.commit('nose-docker-base-%s' % self.container_tag,
'nose-docker:%s' % self.container_tag)
def __init__(self, name, start_calico=True, dind=True):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
if dind:
# TODO use pydocker
docker.rm("-f", self.name, _ok_code=[0, 1])
docker.run("--privileged", "-v", os.getcwd()+":/code", "--name",
self.name,
"-tid", "calico/dind")
self.ip = docker.inspect("--format", "{{ .NetworkSettings.IPAddress }}",
self.name).stdout.rstrip()
self.ip6 = docker.inspect("--format",
"{{ .NetworkSettings."
"GlobalIPv6Address }}",
self.name).stdout.rstrip()
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=100)
self.execute("docker load --input /code/calico_containers/calico-node.tar && "
"docker load --input /code/calico_containers/busybox.tar")
else:
self.ip = get_ip()
if start_calico:
self.start_calico_node()
def main(dataset):
tmpname="kolab/kolabtestcontainer:tmppopulated"
imagename="kolab/kolabtestcontainer:populated-"+dataset
basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)
print("Building tmpcontainer...")
docker.build(settings.dockerCacheString(), "-t", tmpname, "{basedir}/.".format(basedir=basedir))
print("Starting tmpcontainer...")
container = docker.run("-d", "-h", settings.HOSTNAME,
"-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro",
"-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir, dataset=dataset),
'-v', "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
tmpname).rstrip()
try:
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print "Populate OU..."
docker("exec", container, "python2", "/populate/populate_ou.py", _out=sys.stdout)
print "Populate users..."
docker("exec", container, "python2", "/populate/populate_users.py", _out=sys.stdout)
print "Populate resources..."
docker("exec", container, "python2", "/populate/populate_resources.py", _out=sys.stdout)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=sys.stdout)
# Give kolabd some time to create all mailboxes
time.sleep(5)
docker("exec", container, "patch", "-R", "/etc/kolab/kolab.conf", "/data/kolab.conf.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/calendar.inc.php", "/data/calendar.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/config.inc.php", "/data/config.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_addressbook.inc.php", "/data/kolab_addressbook.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_auth.inc.php", "/data/kolab_auth.inc.php.diff", _out=sys.stdout)
docker("exec", container, "patch", "-R", "/etc/roundcubemail/password.inc.php", "/data/password.inc.php.diff", _out=sys.stdout)
print("Comitting results to: {}".format(imagename))
docker.commit(container, imagename)
except:
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc(file=sys.stdout)
print("Failed to setup container")
docker.stop(container)
docker.rm(container)
def cleanup(self):
"""
Clean up this host, including removing any containers is created. This
is necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
self.remove_workloads()
# And remove the calico-node
docker.rm("-f", "calico-node", _ok_code=[0, 1])
if self.dind:
# For docker in docker, we need to remove images too.
self.remove_images()
# Remove the outer container for DinD.
docker.rm("-f", self.name, _ok_code=[0, 1])
self._cleaned = True
def containers(ctx, force, all):
"""clean docker containers which not running"""
all_containers = docker.ps('-a', '-q').stdout.split()
if all:
removed_containers = all_containers
else:
running_containers = docker.ps('-q').stdout.split()
removed_containers = set(all_containers) ^ set(running_containers)
for cid in removed_containers:
ctx.log('docker removing container %s' % cid)
if force:
print(docker.rm('-f', cid))
else:
print(docker.rm(cid))
ctx.log('list all containers')
print(docker.ps('-a'))
def containers(ctx, force, all):
"""clean docker containers which not running"""
all_containers = docker.ps('-a', '-q').stdout.split()
if all:
removed_containers = all_containers
else:
running_containers = docker.ps('-q').stdout.split()
removed_containers = set(all_containers) ^ set(running_containers)
for cid in removed_containers:
ctx.log('docker removing container %s' % cid)
if force:
print(docker.rm('-f', cid))
else:
print(docker.rm(cid))
ctx.log('list all containers')
print(docker.ps('-a'))
def cleanup(self):
"""
Clean up this host, including removing any containers is created. This
is necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
self.remove_workloads()
# And remove the calico-node
docker.rm("-f", "calico-node", _ok_code=[0, 1])
if self.dind:
# For docker in docker, we need to remove images too.
self.remove_images()
# Remove the outer container for DinD.
docker.rm("-f", self.name, _ok_code=[0, 1])
self._cleaned = True
def main(dataset):
tmpname="kolab/kolabtestcontainer:tmppopulated"
imagename="kolab/kolabtestcontainer:populated-"+dataset
SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
print("Building tmpcontainer...")
docker.build("-t", tmpname, SCRIPT_DIR+"/kolabpopulated/.")
print("Starting tmpcontainer...")
container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolabpopulated/"+dataset+"/:/data/", tmpname).rstrip()
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=process_output)
print("Comitting results...")
docker.commit(container, imagename)
docker.stop(container)
docker.rm(container)
def cleanup(self):
"""
Clean up this host, including removing any containers is created. This
is necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
docker.rm("-f", self.name, _ok_code=[0, 1])
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads and the calico node.
self.remove_workloads()
docker.rm("-f", "calico-node", _ok_code=[0, 1])
self._cleaned = True
def main():
tmpname = settings.kolabimagename("tmpbase")
SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
print("Building tmpcontainer...")
docker("build", "-t", tmpname, SCRIPT_DIR+"/kolab/")
print("Starting tmpcontainer...")
print(SCRIPT_DIR+"/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh")
container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolab/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh", tmpname).rstrip()
print("Setting up kolab")
docker("exec", container, "setup-kolab", "--default", "--timezone="+settings.TIMEZONE, "--directory-manager-pwd="+settings.LDAPPW, "--mysqlserver=new", _out=process_output)
print("Fixing roundcube")
docker("exec", container, "bash", "/usr/share/roundcubemail/fixRoundcubeT243.sh", _out=process_output)
docker("exec", container, "systemctl", "restart", "httpd", _out=process_output)
print("Comitting results...")
docker.commit(container, settings.kolabimagename("base"))
docker.stop(container)
docker.rm(container)
def main(dataset):
tmpname = "kolab/kolabtestcontainer:tmppopulated"
imagename = "kolab/kolabtestcontainer:populated-" + dataset
basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)
print("Building tmpcontainer...")
docker.build("-t", tmpname, "{basedir}/.".format(basedir=basedir))
print("Starting tmpcontainer...")
container = docker.run(
"-d", "-h", settings.HOSTNAME, "-v",
"/sys/fs/cgroup:/sys/fs/cgroup:ro",
"-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir,
dataset=dataset), '-v',
"{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
tmpname).rstrip()
try:
# Wait for imap to become available on imaps://localhost:993
time.sleep(5)
print "Populate OU..."
docker("exec",
container,
"python2",
"/populate/populate_ou.py",
_out=sys.stdout)
print "Populate users..."
docker("exec",
container,
"python2",
"/populate/populate_users.py",
_out=sys.stdout)
print "Populate resources..."
docker("exec",
container,
"python2",
"/populate/populate_resources.py",
_out=sys.stdout)
print("Running populate.sh...")
docker("exec", container, "/data/populate.sh", _out=sys.stdout)
# Give kolabd some time to create all mailboxes
time.sleep(5)
docker("exec",
container,
"patch",
"-R",
"/etc/kolab/kolab.conf",
"/data/kolab.conf.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/calendar.inc.php",
"/data/calendar.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/config.inc.php",
"/data/config.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/kolab_addressbook.inc.php",
"/data/kolab_addressbook.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/kolab_auth.inc.php",
"/data/kolab_auth.inc.php.diff",
_out=sys.stdout)
docker("exec",
container,
"patch",
"-R",
"/etc/roundcubemail/password.inc.php",
"/data/password.inc.php.diff",
_out=sys.stdout)
print("Comitting results to: {}".format(imagename))
docker.commit(container, imagename)
except:
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc(file=sys.stdout)
print("Failed to setup container")
docker.stop(container)
docker.rm(container)
def reset(package: str):
container_name = package + "_build"
image_name = package + "_image"
docker.rm("-f", container_name, _ok_code=[0, 1])
docker.rmi(image_name, _ok_code=[0, 1])
def build_and_commit(package: str,
fuzzer_image: str,
json_output_path: str = None,
qemu=False,
timeout=None) -> str:
"""
This builds a package inside a docker container and then commits the container to an image.
:return:
"""
start = time.time()
docker_image_name = package + "_" + str(uuid.uuid4())[:8]
docker_container_name = str(uuid.uuid4())
try:
if not qemu:
build_process = docker.run(
'--cpus=0.90',
"--privileged",
"--name",
docker_container_name,
"--entrypoint",
"python",
fuzzer_image,
"/inputinferer/configfinder/builder_wrapper.py",
"-p",
package,
_out=sys.stdout,
_ok_code=[
config_settings.BUILDER_BUILD_NORMAL,
config_settings.BUILDER_BUILD_FAILED,
config_settings.BUILDER_BUILD_QEMU
],
_timeout=timeout) # type: sh.RunningCommand
else:
build_process = docker.run(
'--cpus=0.90',
"--privileged",
"--name",
docker_container_name,
"--entrypoint",
"python",
fuzzer_image,
"/inputinferer/configfinder/builder_wrapper.py",
"-p",
package,
"-Q",
_out=sys.stdout,
_ok_code=[
config_settings.BUILDER_BUILD_NORMAL,
config_settings.BUILDER_BUILD_FAILED,
config_settings.BUILDER_BUILD_QEMU
],
_timeout=timeout) # type: sh.RunningCommand
except sh.TimeoutException as e:
print("Building {0} timed out!".format(package))
return None
exit_code = build_process.exit_code
if exit_code == -1:
print("Failed to build image for package {0}, not commiting".format(
package))
return None
docker.commit(docker_container_name, docker_image_name, _out=sys.stdout)
end = time.time()
if json_output_path is not None:
json_dict = {}
json_dict["docker_image_name"] = docker_image_name
if exit_code == config_settings.BUILDER_BUILD_NORMAL:
json_dict["qemu"] = False
elif exit_code == config_settings.BUILDER_BUILD_QEMU:
json_dict["qemu"] = True
json_dict["time"] = end - start
with open(json_output_path, "w") as json_output_fp:
json.dump(json_dict, json_output_fp)
docker.rm(docker_container_name) # Remove the image after we commited
return docker_image_name