Beispiel #1
0
    def __init__(self, name, start_calico=True, dind=True):
        self.name = name
        self.dind = dind
        self.workloads = set()

        # This variable is used to assert on destruction that this object was
        # cleaned up.  If not used as a context manager, users of this object
        self._cleaned = False

        if dind:
            docker.rm("-f", self.name, _ok_code=[0, 1])
            docker.run("--privileged", "-v", os.getcwd()+":/code", "--name",
                       self.name,
                       "-e", "DOCKER_DAEMON_ARGS="
                       "--kv-store=consul:%s:8500" % utils.get_ip(),
                       "-tid", "calico/dind")
            self.ip = docker.inspect("--format", "{{ .NetworkSettings.IPAddress }}",
                                     self.name).stdout.rstrip()

            self.ip6 = docker.inspect("--format",
                                      "{{ .NetworkSettings."
                                      "GlobalIPv6Address }}",
                                      self.name).stdout.rstrip()

            # Make sure docker is up
            docker_ps = partial(self.execute, "docker ps")
            retry_until_success(docker_ps, ex_class=CalledProcessError)
            self.execute("docker load --input /code/calico_containers/calico-node.tar && "
                         "docker load --input /code/calico_containers/busybox.tar")
        else:
            self.ip = get_ip()

        if start_calico:
            self.start_calico_node()
            self.assert_driver_up()
    def __build_base_image(self, config):
        # This does not seem to be the right way to get the container tag?
        self.container_tag = config.get_container_tag()
        all_images = docker.images(a=True)

        import pdb; pdb.set_trace()

        if self.container_tag in all_images:
            return

        print ("Container not found or changes in watched files. Rebuilding base container (%s)..." % self.container_tag)

        base_name = 'nose-docker-base-%s' % self.container_tag

        all_containers = docker.ps(a=True)
        if base_name in all_containers:
            docker.rm('-f', base_name)

        docker.run(
            '--name=%s' % base_name, 
            '-v',
            '%s:/app' % abspath(os.curdir),
            config.base_image,
            '/bin/bash',
            c="cd /app && %s" % (
                " && ".join(config.build_commands)
            ),
            _out=sys.stdout
        )
        docker.commit('nose-docker-base-%s' % self.container_tag, 'nose-docker:%s' % self.container_tag)
Beispiel #3
0
    def __build_base_image(self, config):
        # This does not seem to be the right way to get the container tag?
        self.container_tag = config.get_container_tag()
        all_images = docker.images(a=True)

        import pdb
        pdb.set_trace()

        if self.container_tag in all_images:
            return

        print(
            "Container not found or changes in watched files. Rebuilding base container (%s)..."
            % self.container_tag)

        base_name = 'nose-docker-base-%s' % self.container_tag

        all_containers = docker.ps(a=True)
        if base_name in all_containers:
            docker.rm('-f', base_name)

        docker.run('--name=%s' % base_name,
                   '-v',
                   '%s:/app' % abspath(os.curdir),
                   config.base_image,
                   '/bin/bash',
                   c="cd /app && %s" % (" && ".join(config.build_commands)),
                   _out=sys.stdout)
        docker.commit('nose-docker-base-%s' % self.container_tag,
                      'nose-docker:%s' % self.container_tag)
Beispiel #4
0
    def __init__(self, name, start_calico=True, dind=True):
        self.name = name
        self.dind = dind
        self.workloads = set()

        # This variable is used to assert on destruction that this object was
        # cleaned up.  If not used as a context manager, users of this object
        self._cleaned = False

        if dind:
            # TODO use pydocker
            docker.rm("-f", self.name, _ok_code=[0, 1])
            docker.run("--privileged", "-v", os.getcwd()+":/code", "--name",
                       self.name,
                       "-tid", "calico/dind")
            self.ip = docker.inspect("--format", "{{ .NetworkSettings.IPAddress }}",
                                     self.name).stdout.rstrip()

            self.ip6 = docker.inspect("--format",
                                      "{{ .NetworkSettings."
                                      "GlobalIPv6Address }}",
                                      self.name).stdout.rstrip()

            # Make sure docker is up
            docker_ps = partial(self.execute, "docker ps")
            retry_until_success(docker_ps, ex_class=CalledProcessError,
                                retries=100)
            self.execute("docker load --input /code/calico_containers/calico-node.tar && "
                         "docker load --input /code/calico_containers/busybox.tar")
        else:
            self.ip = get_ip()

        if start_calico:
            self.start_calico_node()
Beispiel #5
0
def main(dataset):
    tmpname="kolab/kolabtestcontainer:tmppopulated"
    imagename="kolab/kolabtestcontainer:populated-"+dataset
    basedir =  "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)

    print("Building tmpcontainer...")
    docker.build(settings.dockerCacheString(), "-t", tmpname, "{basedir}/.".format(basedir=basedir))

    print("Starting tmpcontainer...")
    container = docker.run("-d", "-h", settings.HOSTNAME,
            "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro",
            "-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir, dataset=dataset),
            '-v',  "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
            tmpname).rstrip()
    try:
        # Wait for imap to become available on imaps://localhost:993
        time.sleep(5)

        print "Populate OU..."
        docker("exec", container, "python2", "/populate/populate_ou.py", _out=sys.stdout)

        print "Populate users..."
        docker("exec", container, "python2", "/populate/populate_users.py", _out=sys.stdout)

        print "Populate resources..."
        docker("exec", container, "python2", "/populate/populate_resources.py", _out=sys.stdout)

        print("Running populate.sh...")
        docker("exec", container,  "/data/populate.sh", _out=sys.stdout)

        # Give kolabd some time to create all mailboxes
        time.sleep(5)

        docker("exec", container, "patch", "-R", "/etc/kolab/kolab.conf", "/data/kolab.conf.diff", _out=sys.stdout)
        docker("exec", container, "patch", "-R", "/etc/roundcubemail/calendar.inc.php", "/data/calendar.inc.php.diff", _out=sys.stdout)
        docker("exec", container, "patch", "-R", "/etc/roundcubemail/config.inc.php", "/data/config.inc.php.diff", _out=sys.stdout)
        docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_addressbook.inc.php", "/data/kolab_addressbook.inc.php.diff", _out=sys.stdout)
        docker("exec", container, "patch", "-R", "/etc/roundcubemail/kolab_auth.inc.php", "/data/kolab_auth.inc.php.diff", _out=sys.stdout)
        docker("exec", container, "patch", "-R", "/etc/roundcubemail/password.inc.php", "/data/password.inc.php.diff", _out=sys.stdout)

        print("Comitting results to: {}".format(imagename))
        docker.commit(container, imagename)
    except:
        print("Unexpected error:", sys.exc_info()[0])
        traceback.print_exc(file=sys.stdout)
        print("Failed to setup container")

    docker.stop(container)
    docker.rm(container)
Beispiel #6
0
    def cleanup(self):
        """
        Clean up this host, including removing any containers is created.  This
        is necessary especially for Docker-in-Docker so we don't leave dangling
        volumes.
        :return:
        """
        self.remove_workloads()
        # And remove the calico-node
        docker.rm("-f", "calico-node", _ok_code=[0, 1])

        if self.dind:
            # For docker in docker, we need to remove images too.
            self.remove_images()
            # Remove the outer container for DinD.
            docker.rm("-f", self.name, _ok_code=[0, 1])
        self._cleaned = True
Beispiel #7
0
def containers(ctx, force, all):
    """clean docker containers which not running"""
    all_containers = docker.ps('-a', '-q').stdout.split()
    if all:
        removed_containers = all_containers
    else:
        running_containers = docker.ps('-q').stdout.split()
        removed_containers = set(all_containers) ^ set(running_containers)
    for cid in removed_containers:
        ctx.log('docker removing container %s' % cid)
        if force:
            print(docker.rm('-f', cid))
        else:
            print(docker.rm(cid))

    ctx.log('list all containers')
    print(docker.ps('-a'))
Beispiel #8
0
def containers(ctx, force, all):
    """clean docker containers which not running"""
    all_containers = docker.ps('-a', '-q').stdout.split()
    if all:
        removed_containers = all_containers
    else:
        running_containers = docker.ps('-q').stdout.split()
        removed_containers = set(all_containers) ^ set(running_containers)
    for cid in removed_containers:
        ctx.log('docker removing container %s' % cid)
        if force:
            print(docker.rm('-f', cid))
        else:
            print(docker.rm(cid))

    ctx.log('list all containers')
    print(docker.ps('-a'))
Beispiel #9
0
    def cleanup(self):
        """
        Clean up this host, including removing any containers is created.  This
        is necessary especially for Docker-in-Docker so we don't leave dangling
        volumes.
        :return:
        """
        self.remove_workloads()
        # And remove the calico-node
        docker.rm("-f", "calico-node", _ok_code=[0, 1])

        if self.dind:
            # For docker in docker, we need to remove images too.
            self.remove_images()
            # Remove the outer container for DinD.
            docker.rm("-f", self.name, _ok_code=[0, 1])
        self._cleaned = True
Beispiel #10
0
def main(dataset):
    tmpname="kolab/kolabtestcontainer:tmppopulated"
    imagename="kolab/kolabtestcontainer:populated-"+dataset

    SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))

    print("Building tmpcontainer...")
    docker.build("-t", tmpname, SCRIPT_DIR+"/kolabpopulated/.")
    print("Starting tmpcontainer...")
    container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolabpopulated/"+dataset+"/:/data/", tmpname).rstrip()

    # Wait for imap to become available on imaps://localhost:993
    time.sleep(5)

    print("Running populate.sh...")
    docker("exec", container,  "/data/populate.sh", _out=process_output)

    print("Comitting results...")
    docker.commit(container, imagename)
    docker.stop(container)
    docker.rm(container)
Beispiel #11
0
    def cleanup(self):
        """
        Clean up this host, including removing any containers is created.  This
        is necessary especially for Docker-in-Docker so we don't leave dangling
        volumes.
        :return:
        """
        if self.dind:
            # For Docker-in-Docker, we need to remove all containers and
            # all images...
            self.remove_containers()
            self.remove_images()

            # ...and the outer container for DinD.
            docker.rm("-f", self.name, _ok_code=[0, 1])
        else:
            # For non Docker-in-Docker, we can only remove the containers we
            # created - so remove the workloads and the calico node.
            self.remove_workloads()
            docker.rm("-f", "calico-node", _ok_code=[0, 1])

        self._cleaned = True
Beispiel #12
0
def main():
    tmpname = settings.kolabimagename("tmpbase")

    SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))

    print("Building tmpcontainer...")
    docker("build", "-t", tmpname, SCRIPT_DIR+"/kolab/")
    print("Starting tmpcontainer...")
    print(SCRIPT_DIR+"/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh")
    container = docker.run("-d", "-h", settings.HOSTNAME, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", SCRIPT_DIR+"/kolab/fixRoundcubeT243.sh:/usr/share/roundcubemail/fixRoundcubeT243.sh", tmpname).rstrip()

    print("Setting up kolab")
    docker("exec", container,  "setup-kolab", "--default", "--timezone="+settings.TIMEZONE, "--directory-manager-pwd="+settings.LDAPPW, "--mysqlserver=new", _out=process_output)

    print("Fixing roundcube")
    docker("exec", container, "bash", "/usr/share/roundcubemail/fixRoundcubeT243.sh", _out=process_output)
    docker("exec", container, "systemctl", "restart", "httpd", _out=process_output)

    print("Comitting results...")
    docker.commit(container, settings.kolabimagename("base"))
    docker.stop(container)
    docker.rm(container)
Beispiel #13
0
def main(dataset):
    tmpname = "kolab/kolabtestcontainer:tmppopulated"
    imagename = "kolab/kolabtestcontainer:populated-" + dataset
    basedir = "{c.SCRIPT_DIR}/kolabpopulated".format(c=config)

    print("Building tmpcontainer...")
    docker.build("-t", tmpname, "{basedir}/.".format(basedir=basedir))

    print("Starting tmpcontainer...")
    container = docker.run(
        "-d", "-h", settings.HOSTNAME, "-v",
        "/sys/fs/cgroup:/sys/fs/cgroup:ro",
        "-v", "{basedir}/{dataset}/:/data/".format(basedir=basedir,
                                                   dataset=dataset), '-v',
        "{c.SCRIPT_DIR}/kolab/populate/:/populate".format(c=config),
        tmpname).rstrip()
    try:
        # Wait for imap to become available on imaps://localhost:993
        time.sleep(5)

        print "Populate OU..."
        docker("exec",
               container,
               "python2",
               "/populate/populate_ou.py",
               _out=sys.stdout)

        print "Populate users..."
        docker("exec",
               container,
               "python2",
               "/populate/populate_users.py",
               _out=sys.stdout)

        print "Populate resources..."
        docker("exec",
               container,
               "python2",
               "/populate/populate_resources.py",
               _out=sys.stdout)

        print("Running populate.sh...")
        docker("exec", container, "/data/populate.sh", _out=sys.stdout)

        # Give kolabd some time to create all mailboxes
        time.sleep(5)

        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/kolab/kolab.conf",
               "/data/kolab.conf.diff",
               _out=sys.stdout)
        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/roundcubemail/calendar.inc.php",
               "/data/calendar.inc.php.diff",
               _out=sys.stdout)
        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/roundcubemail/config.inc.php",
               "/data/config.inc.php.diff",
               _out=sys.stdout)
        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/roundcubemail/kolab_addressbook.inc.php",
               "/data/kolab_addressbook.inc.php.diff",
               _out=sys.stdout)
        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/roundcubemail/kolab_auth.inc.php",
               "/data/kolab_auth.inc.php.diff",
               _out=sys.stdout)
        docker("exec",
               container,
               "patch",
               "-R",
               "/etc/roundcubemail/password.inc.php",
               "/data/password.inc.php.diff",
               _out=sys.stdout)

        print("Comitting results to: {}".format(imagename))
        docker.commit(container, imagename)
    except:
        print("Unexpected error:", sys.exc_info()[0])
        traceback.print_exc(file=sys.stdout)
        print("Failed to setup container")

    docker.stop(container)
    docker.rm(container)
Beispiel #14
0
def reset(package: str):
    container_name = package + "_build"
    image_name = package + "_image"
    docker.rm("-f", container_name, _ok_code=[0, 1])
    docker.rmi(image_name, _ok_code=[0, 1])
Beispiel #15
0
def build_and_commit(package: str,
                     fuzzer_image: str,
                     json_output_path: str = None,
                     qemu=False,
                     timeout=None) -> str:
    """
    This builds a package inside a docker container and then commits the container to an image.
    :return: 
    """
    start = time.time()
    docker_image_name = package + "_" + str(uuid.uuid4())[:8]
    docker_container_name = str(uuid.uuid4())
    try:
        if not qemu:
            build_process = docker.run(
                '--cpus=0.90',
                "--privileged",
                "--name",
                docker_container_name,
                "--entrypoint",
                "python",
                fuzzer_image,
                "/inputinferer/configfinder/builder_wrapper.py",
                "-p",
                package,
                _out=sys.stdout,
                _ok_code=[
                    config_settings.BUILDER_BUILD_NORMAL,
                    config_settings.BUILDER_BUILD_FAILED,
                    config_settings.BUILDER_BUILD_QEMU
                ],
                _timeout=timeout)  # type: sh.RunningCommand
        else:
            build_process = docker.run(
                '--cpus=0.90',
                "--privileged",
                "--name",
                docker_container_name,
                "--entrypoint",
                "python",
                fuzzer_image,
                "/inputinferer/configfinder/builder_wrapper.py",
                "-p",
                package,
                "-Q",
                _out=sys.stdout,
                _ok_code=[
                    config_settings.BUILDER_BUILD_NORMAL,
                    config_settings.BUILDER_BUILD_FAILED,
                    config_settings.BUILDER_BUILD_QEMU
                ],
                _timeout=timeout)  # type: sh.RunningCommand
    except sh.TimeoutException as e:
        print("Building {0} timed out!".format(package))
        return None
    exit_code = build_process.exit_code
    if exit_code == -1:
        print("Failed to build image for package {0}, not commiting".format(
            package))
        return None
    docker.commit(docker_container_name, docker_image_name, _out=sys.stdout)
    end = time.time()
    if json_output_path is not None:
        json_dict = {}
        json_dict["docker_image_name"] = docker_image_name
        if exit_code == config_settings.BUILDER_BUILD_NORMAL:
            json_dict["qemu"] = False
        elif exit_code == config_settings.BUILDER_BUILD_QEMU:
            json_dict["qemu"] = True
        json_dict["time"] = end - start
        with open(json_output_path, "w") as json_output_fp:
            json.dump(json_dict, json_output_fp)
    docker.rm(docker_container_name)  # Remove the image after we commited
    return docker_image_name