class Docker(object):

    def __init__(self,base_url = ''):
        self.cli = Client(base_url=base_url)
        self.containers = []
        self.images = 'docker/whalesay'

    def build(self):
        self.cli.build(path='../../dockfiles/'+self.images,rm=True,tag=self.images)

    def create(self):
        container = self.cli.create_container(image=self.images + ':latest',
                                              command='cowsay boo',detach=True,
                                              name=(self.images + ": " + (len(self.containers)-1).__str__()))
        self.containers.append(container)
        return len(self.containers) - 1

    def remove(self,num):
        self.stop(num)
        container = self.containers[num]
        self.cli.remove_container(container=container)
        self.containers.remove(num)

    def start(self,num):
        self.cli.start(container=self.containers[num])

    def stop(self,num):
        self.cli.stop(container=self.containers[num])
    def get_image_id(self):
        image_name = 'container-%d:v%d' % (self.id, self.version)
        path = os.path.join(settings.DOCKER_ROOT, 'build', image_name)

        # create a client to communicate with docker
        client = Client(base_url='unix://var/run/docker.sock')

        # check if already built
        images = client.images(name=image_name)
        if images:
            print('docker image exists: %s with name %s' % (self.tag, image_name))
            return images[0]['Id']

        # build the docker file
        print('building new docker image: %s with name %s' % (self.tag, image_name))
        extract_zip(self.dockerfile_src, path)
        log = list(client.build(path=path, rm=True, tag=image_name))
        self.build_log = "".join([str(i)+': '+str(log[i]) for i in range(len(log))])
        self.save()

        images = client.images(name=image_name)
        if images:
            return images[0]['Id']
        else:
            raise LookupError('Docker image not found: "' + self.tag + '"')
Beispiel #3
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--nocache', action='store_true', default=False)
    parser.add_argument('--dockerfile', type=local)
    parser.add_argument('--label', choices=IMAGES.keys())
    arguments = parser.parse_args()
    content = ''

    client = Client(base_url='unix://var/run/docker.sock')

    stream = client.build(
        tag=IMAGES[arguments.label],
        dockerfile=arguments.dockerfile.basename,
        path=arguments.dockerfile.dirname,
        pull=True,
        decode=True,
        forcerm=True,
        nocache=arguments.nocache
    )

    for item in stream:
        buff = item.get('stream', item.get('status', ''))

        if not content or re.search('.+\[[. ]*$', content):
            content += buff

        if not re.search('.+\[[. ]*$', content):
            print(content)
            content = ''
Beispiel #4
0
class DockerLayer(object):

    def __init__(self, name, tag):
        self.__name__ = name
        self.__bases__ = tuple([])
        self.tag = tag
        self.client = Client(base_url='unix://var/run/docker.sock')
        try:
            self.client.ping()
        except ConnectionError as e:
            # http://docker-py.readthedocs.org/en/latest/boot2docker/
            kwargs = kwargs_from_env()
            kwargs['tls'].assert_hostname = False
            self.client = Client(**kwargs)

    def setUp(self):
        if self.client.ping() == u'OK':
            self.start()
        else:
            raise RuntimeError('Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.')

    def start(self):
        for line in self.client.build(
                path=os.path.abspath(os.path.join(DIR, '..')),
                tag=self.tag, rm=True, forcerm=True):
            sys.stdout.write(line)

    def tearDown(self):
        self.stop()

    def stop(self):
        self.client.close()
Beispiel #5
0
def main(token, commit):

  if token != os.getenv("TOKEN"):
    abort(401)  

  working_copy = "/repository/checkout"
  if not os.path.isdir(working_copy):
    os.mkdir(working_copy)
    Git().clone(os.getenv("GIT_REPOSITORY_URL"), working_copy)

  print "Refreshing git repository"
  repo = Repo(working_copy)
  origin = repo.remotes.origin
  origin.pull();
  repo.git.reset(commit, hard=True)  
  
  print "Building docker image"
  docker = Client(base_url='unix://mount/run/docker.sock')
  docker.login(username=os.getenv("DOCKER_REGISTRY_USERNAME"), password=os.getenv("DOCKER_REGISTRY_PASSWORD"), email=os.getenv("DOCKER_REGISTRY_EMAIL", "*****@*****.**"))
  before_build = os.getenv('BEFORE_BUILD_CMD')
  if before_build is not None:
    check_call(before_build, shell=True)  
  
  for line in docker.build(path=working_copy, tag=os.getenv("IMAGE_NAME"), stream=True):
    print line

  print "Pushing docker image"
  for line in docker.push(os.getenv("IMAGE_NAME"), stream=True):
    print line

  print "Listing docker image"
  check_call(os.getenv('DEPLOY_CMD'), shell=True)

  return "OK"
Beispiel #6
0
    def custom_docker_build(self):
        from io import BytesIO
        from docker import Client
        import json
        import re
        doc_client = Client(base_url='unix://var/run/docker.sock',
                            version='1.18')
        df1 = open(df_name, "r").read()
        f = BytesIO(df1.encode('utf-8'))
        resp = [
            line for line in doc_client.build(
                fileobj=f, rm=False, tag=cont_name + "-1")
        ]

        try:
            parsed_lines = [json.loads(e).get('stream', '') for e in resp]
        except ValueError:
            # sometimes all the data is sent on a single line ????
            #
            # ValueError: Extra data: line 1 column 87 - line 1 column
            # 33268 (char 86 - 33267)
            line = resp[0]
            # This ONLY works because every line is formatted as
            # {"stream": STRING}
            parsed_lines = [
                json.loads(obj).get('stream', '')
                for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
            ]
        # -----

        fmlogger.error(parsed_lines)
Beispiel #7
0
	def __init__(self,url_folder_language):
		cli = Client(base_url='unix://var/run/docker.sock')
		response = cli.build(path=url_folder_language,rm=True,nocache=True)
		for i in response:
			j=i[30:]
		id_container=j[:12]
		self.id_container=id_container		
Beispiel #8
0
def get_log():
    logs = []
    json_data = json.loads(request.get_data())
    project = ProjectModel.query.filter_by(proname=json_data['data']).first()
    if not project.is_build():
        os.chdir(app.config['CODE_FOLDER'] + '/' + json_data['data'])
        cli = Client(base_url=HOST)
        for line in cli.build(path=os.getcwd(), stream=True, decode=True,
                              tag=str(REGISTRY + json_data['data'])):
            send_log(line)
            logs.append(line)
        # 向私有仓库推送镜像, 没有log的打印
        for line in cli.push(REGISTRY + json_data['data'], stream=True):
            # 打印出上传的log
            print line
            assert line
        redis.hset(project.id, project.verify, logs)
        project.build = True
        if 'Successfully built' in logs[-1]['stream']:
            project.success = 1
        else:
            project.success = 2
        db.session.add(project)
        db.session.commit()
    else:
        lines = eval(redis.hget(project.id, project.verify))
        for line in lines:
            send_log(line)
    return json.dumps({'msg': 'ok'})
Beispiel #9
0
class DockerClient(object):
    def __init__(self):
        self.client = Client(base_url='unix://var/run/docker.sock')

    def build_images(self, dockerfile: str, tag: str):
        with open(dockerfile) as file:
            dkfile = BytesIO(file.read().encode('utf-8'))
        response = [line for line in self.client.build(
            fileobj=dkfile, rm=True, tag=tag)]
        return response

    def run_container(self, image, mem_limit=None, volume_binds: list = None, command=None):
        container = self.client.create_container(image=image,
                                                 host_config=self.client.create_host_config(
                                                     binds=volume_binds, mem_limit=mem_limit), command=command)
        self.client.start(container)
        try:
            self.client.wait(container, timeout=3)
        except ReadTimeout:
            print('time out')
        self.client.stop(container)
        self.client.remove_container(container)

    def exec_container(self, container, cmd):
        container_id = self.client.exec_create(container, cmd)['Id']
        return self.client.exec_start(container_id)
Beispiel #10
0
class DockerTest(object):
    def __init__(self):
        self.cli = Client(base_url='unix://var/run/docker.sock')

    def build(self, dockerfile):
        dockerfile = Path(dockerfile)
        tag = 'rf' + dockerfile.basename().replace('Dockerfile.', '')
        dockerfile.copy('redfish-client/tests/Dockerfile')
        response = [line for line in self.cli.build(
            path='redfish-client/tests',
            tag=tag,
            rm=True)]
        return(response)

    def run(self, image, command):
        container = self.cli.create_container(image=image,
                                              command=command,
                                              tty=True,
                                              stdin_open=True)
        self.cli.start(container=container.get('Id'))
        self.cli.wait(container=container.get('Id'))
        response = self.cli.logs(container=container.get('Id'),
                                 stdout=True)
        self.cli.remove_container(container=container.get('Id'))
        return(response.decode('utf8'))
Beispiel #11
0
def build_docker_image(docker_file_dir, docker_file, repository, log_file):
    """
    Build docker image from the given docker file

    Args:
        docker_file_dir (path): Docker file dir
        docker_file (path): Docker file
        repository (str): Repo name
        log_file (path): Log file path

    Returns:
        auth_config_payload (dict): AWS auth config
    """
    docker_client = Client(base_url='unix://var/run/docker.sock')
    write_to_debug_log(log_file,
                       "Creating Docker image: %s ..." % str(repository))

    info = docker_client.build(dockerfile=docker_file,
                               tag=repository,
                               path=docker_file_dir,
                               rm=True,
                               stream=True)
    with open(log_file, 'a') as f:
        for item in info:
            f.write("%s %s\n" % (" " * 10, str(item)))

    write_to_debug_log(
        log_file,
        "Docker image: %s has been created locally!!!" % str(repository))

    return docker_client
Beispiel #12
0
class DockerLayer(object):
    def __init__(self, name, tag):
        self.__name__ = name
        self.__bases__ = tuple([])
        self.tag = tag
        self.client = Client(base_url='unix://var/run/docker.sock')
        try:
            self.client.ping()
        except ConnectionError as e:
            # http://docker-py.readthedocs.org/en/latest/boot2docker/
            kwargs = kwargs_from_env()
            kwargs['tls'].assert_hostname = False
            self.client = Client(**kwargs)

    def setUp(self):
        if self.client.ping() == u'OK':
            self.start()
        else:
            raise RuntimeError(
                'Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.'
            )

    def start(self):
        for line in self.client.build(path=os.path.abspath(
                os.path.join(DIR, '..')),
                                      tag=self.tag,
                                      rm=True,
                                      forcerm=True):
            sys.stdout.write(line)

    def tearDown(self):
        self.stop()

    def stop(self):
        self.client.close()
Beispiel #13
0
class DockerTest(object):
    def __init__(self):
        self.cli = Client(base_url='unix://var/run/docker.sock')

    def build(self, dockerfile):
        dockerfile = Path(dockerfile)
        tag = 'rf-' + dockerfile.basename().replace('.dkf', '')
        dockerfile.copy('redfish-client/tests/Dockerfile')
        response = [line for line in self.cli.build(
            path='redfish-client/tests',
            tag=tag,
            rm=True)]
        return(response)

    def run(self, image, command):
        container = self.cli.create_container(image=image,
                                              command=command,
                                              tty=True,
                                              stdin_open=True)
        self.cli.start(container=container.get('Id'))
        self.cli.wait(container=container.get('Id'))
        response = self.cli.logs(container=container.get('Id'),
                                 stdout=True)
        self.cli.remove_container(container=container.get('Id'))
        return(response.decode('utf8'))
Beispiel #14
0
class DockerOpers(UtilOpers):
    def __init__(self):
        self.client = Client(base_url='unix://var/run/docker.sock',
                             version='1.12')

    @staticmethod
    def get_path_by__type(tp):
        return options.dockerfile_dir + "/%s" % tp

    def build(self, path, tag='', noCache=False):
        streams = self.client.build(tag=tag,
                                    path=path,
                                    nocache=noCache,
                                    rm=True)
        res = [line for line in streams]
        return res

    def push(self, repository, tag=''):
        res = self.client.push(repository, tag)
        return res

    def remove(self, repository, tag=''):
        image = '%s:%s' % (repository, 'latest' if tag == '' else tag)
        res = self.client.remove_image(image)
        return res
Beispiel #15
0
def _create_docker_image(dockerfilepath, repository, pacbot_installation):
    docker_client = Client(base_url='unix://var/run/docker.sock')
    for info in docker_client.build(path=dockerfilepath,
                                    tag=repository,
                                    rm=True,
                                    stream=True):
        _logs_display(info, pacbot_installation)
    return docker_client
Beispiel #16
0
def bake_main(options):
    """Munge a dockerfile from a cfg

    cake:
        layers: []
    """
    endpoint = os.environ.get("LAYERCAKE_API")
    if endpoint:
        options.layer_endpoint = endpoint

    config = yaml.load(open(options.config))['cake']
    df = dockerfile.Dockerfile(options.dockerfile)

    if options.layer_endpoint:
        df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint))

    # In this mode we are adding run cmds for each
    # layer in the cfg file (those may pull other layers)
    # then we output a new docker file and docker build the
    # new container.
    last_run = df.last("RUN")
    if not options.use_devel:
        df.add("RUN",
               ['pip3', 'install', '--upgrade',
                'layer_cake==%s' % VERSION],
               at=last_run)
    else:
        # For devel we ask the container to pull git master
        df.add("RUN", [
            'pip3', 'install', '--upgrade',
            "https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake"
        ],
               at=last_run)
    for layer_name in config['layers']:
        last_run = df.last("RUN")
        df.add("RUN", ["cake", "layer", layer_name, "-d", LAYERS_HOME],
               at=last_run)

    # we might have an entrypoint
    # or a command (or both)
    if df.entrypoint:
        df.entrypoint = ["disco"] + df.entrypoint['args']

    log.debug("Using Dockerfile\n%s", str(df))
    if not options.no_build:
        client = DockerClient()
        f = BytesIO(str(df).encode("utf-8"))
        response = client.build(fileobj=f, tag="layercake/disco", decode=True)
        for line in response:
            if 'errorDetail' in line:
                log.critical(line['errorDetail']['message'].strip())
            elif 'stream' in line:
                log.info(line['stream'].strip())
    else:
        return df
Beispiel #17
0
def build_docker_container():
    # Connect to docker
    docker = DockerClient(base_url='unix://var/run/docker.sock')
    # Build the container
    r = [line for line in docker.build(
            forcerm=True,
            tag="transmission",
            path=settings.BASE_DIR+'/dockerbuild/',
            decode=True
        )]
    return r[-1]['stream'].find('Successfully built')
Beispiel #18
0
def build(ctx, image_name='ghcr.io/Daplanet/terraform-localexecute:latest'):
    """
  Builds base docker image and pushes to <image_name>
  """

    try:

        cli = Client(base_url="unix:///var/run/docker.sock")

        cli.login(registry=image_name.split('/', 1)[0],
                  user="******",
                  password=getenv('GITHUB_TOKEN'))
        cli.build(path='.', tag=image_name)
        cli.push(image_name)

    except Exception as err:
        print(err)
        sys.exit(1)

    finally:
        del (cli)
Beispiel #19
0
def build_baton_docker(docker_client: Client, baton_docker_build: BatonImage):
    """
    Builds the baton Docker image.
    :param docker_client: the Docker client
    :param baton_docker_build: where the baton docker is built from
    """
    logging.info("Building baton Docker image - if this is not cached, it will take a few minutes")
    logging.debug("baton Docker build: %s" % baton_docker_build)
    # Note: reading the lines in this ways enforces that Python blocks - required
    for line in docker_client.build(tag=baton_docker_build.tag, path=baton_docker_build.path,
                                    dockerfile=baton_docker_build.docker_file, buildargs=baton_docker_build.build_args):
        logging.debug(line)
Beispiel #20
0
 def _build_images_from_dockerfiles(self):
     """
     Build Docker images for each local Dockerfile found in the package: self.local_docker_files
     """
     if GK_STANDALONE_MODE:
         return  # do not build anything in standalone mode
     dc = DockerClient()
     LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
     for k, v in self.local_docker_files.iteritems():
         for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
             LOG.debug("DOCKER BUILD: %s" % line)
         LOG.info("Docker image created: %s" % k)
Beispiel #21
0
def build_docker_image(nocache=False):
    docker_client = Client(base_url='unix://var/run/docker.sock')
    path = os.getcwd() + '/docker/'
    image = ImageFactory(docker_client=docker_client, path=path)
    return docker_client.build(
        path=image['path'],
        tag=image['tag'],
        dockerfile=image['dockerfile'],
        pull=True,
        decode=True,
        forcerm=True,
        nocache=nocache
    )
Beispiel #22
0
def build_docker_image(nocache=False, pull=True):
    version = os.environ.get('VERSION', 'sles12sp1')
    flavor = os.environ.get('FLAVOR') or 'products'
    docker_client = Client(base_url='unix://var/run/docker.sock')

    return docker_client.build(
        tag='registry.mgr.suse.de/toaster-{0}-{1}'.format(version, flavor),
        dockerfile='Dockerfile.{0}.{1}'.format(version, flavor),
        path=os.getcwd() + '/docker/',
        pull=pull,
        decode=True,
        forcerm=True,
        nocache=nocache)
Beispiel #23
0
def create_container(cname, user, giturl):
    try:
        error = False
        errmsg = ''
        cid = ''
        host_port = ''
        print('nodeJsBuild start')
        print('file creation')
        cli = Client(base_url='unix://var/run/docker.sock')
        #tls_config = tls.TLSConfig(
        #    client_cert=('/Users/kasi-mac/.docker/machine/machines/default/cert.pem', '/Users/kasi-mac/.docker/machine/machines/default/key.pem'),
        #    ca_cert='/Users/kasi-mac/.docker/machine/machines/default/ca.pem', verify=True)
        #cli = Client(base_url='tcp://192.168.99.100:2376', tls=tls_config)
        print('cli creation')
        response = [
            line
            for line in cli.build(path=giturl, rm=True, tag=user + '/' + cname)
        ]
        print(response)
        print('cli creation end')
        container = cli.create_container(
            image=user + '/' + cname,
            name=cname,
            host_config=cli.create_host_config(publish_all_ports=True))
        print(container)
        ccrres = json.loads(json.dumps(container))
        if 'Id' not in ccrres:
            error = True
            errmsg = 'Container creation failed'
        else:
            print('1')
            cid = ccrres['Id']
            print(cid)
            response = cli.start(container=container.get('Id'))
            print(response)
            if response != None:
                error = True
                errmsg = 'Container failed to start'
            else:
                error = False
                info = cli.inspect_container(container)
                print info
                host_port = 'http://' + str(
                    info['NetworkSettings']['Ports']['8080/tcp'][0]
                    ['HostIp']) + ':' + str(info['NetworkSettings']['Ports']
                                            ['8080/tcp'][0]['HostPort'])
                print host_port
        return error, errmsg, cid, host_port
    except Exception as inst:
        print inst.args
        return True, 'Exception', '', ''
Beispiel #24
0
def bake_main(options):
    """Munge a dockerfile from a cfg

    cake:
        layers: []
    """
    endpoint = os.environ.get("LAYERCAKE_API")
    if endpoint:
        options.layer_endpoint = endpoint

    config = yaml.load(open(options.config))['cake']
    df = dockerfile.Dockerfile(options.dockerfile)

    if options.layer_endpoint:
        df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint))

    # In this mode we are adding run cmds for each
    # layer in the cfg file (those may pull other layers)
    # then we output a new docker file and docker build the
    # new container.
    last_run = df.last("RUN")
    if not options.use_devel:
        df.add("RUN", ['pip3', 'install', '--upgrade', 'layer_cake==%s' % VERSION], at=last_run)
    else:
        # For devel we ask the container to pull git master
        df.add("RUN", ['pip3', 'install', '--upgrade',
            "https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake"],
                at=last_run)
    for layer_name in config['layers']:
        last_run = df.last("RUN")
        df.add("RUN", ["cake", "layer", layer_name,
               "-d", LAYERS_HOME],
               at=last_run)

    # we might have an entrypoint
    # or a command (or both)
    if df.entrypoint:
        df.entrypoint = ["disco"] + df.entrypoint['args']

    log.debug("Using Dockerfile\n%s", str(df))
    if not options.no_build:
        client = DockerClient()
        f = BytesIO(str(df).encode("utf-8"))
        response = client.build(fileobj=f, tag="layercake/disco", decode=True)
        for line in response:
            if 'errorDetail' in line:
                log.critical(line['errorDetail']['message'].strip())
            elif 'stream' in line:
                log.info(line['stream'].strip())
    else:
        return df
Beispiel #25
0
    def build_project(self, project_id, tag, **kwargs):
        DOCKER_HOST = os.environ['DOCKER_HOST']
        DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https')
        DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH']
        cert_dir = os.environ['DOCKER_CERT_PATH'] + '/'
        tls_config = TLSConfig(
            client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False
        )
        docker_client = Client(base_url=DOCKER_HOST, tls = tls_config)

        build = self.model()
        self.validate_tag(tag)
        build.tag = tag
        image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag
        build.project = Project.objects.get_project(project_id)
        Dockerfile = self.optimize_docker_text(project_id = project_id)

        try:
            Build.objects.get(project = build.project, is_active = True)
        except ObjectDoesNotExist:
            pass
        else:  
            old_build = Build.objects.get(project = build.project, is_active = True)
            docker_client.stop(container = old_build.container)


#        build.response = "".join([json.loads(line)["stream"] for line in 
        build.response = ([ line for line in 
#          docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ])
           docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name) ])
        container = docker_client.create_container(
            image=image_name,
            name= build.project.name+"_"+build.tag,
            detach=True,
            ports = [ 8080 ]
        )
        build.container = container.get('Id')
        build.port = 10000 + build.project.id
 
        try:
            Build.objects.get(project = build.project, is_active = True)
        except ObjectDoesNotExist:
            pass
        else:
            old_build = Build.objects.get(project = build.project, is_active = True)
            docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port })

        build.save(using = self.db)
        return build
Beispiel #26
0
 def build(self):
     from docker import Client
     from io import BytesIO
     dockerhost = os.environ.get("DOCKER_HOST", "tcp://127.0.0.1:2375")
     dockerfile = super(DockerBuild, self).build()
     dockerfile = BytesIO(dockerfile.encode('utf-8'))
     cli = Client(dockerhost)
     builder = cli.build(fileobj=dockerfile, rm=True, tag="bones")
     for line in builder:
         line = json.loads(line)
         if len(line.keys()) == 1 and "stream" in line:
             sys.stdout.write(line["stream"])
             sys.stdout.flush()
         else:
             print(line)
Beispiel #27
0
def build(repo, rev, image, hostname, location, socket='/var/run/docker.sock'):
    '''
    Build a docker image and install a salt minion, checkout the pull request
    an perform a function test against the new code.

    :param repo
    The repository you would like to test against

    :param rev
    The revision to check

    :param socket
    The docket socket to perform all communication accorss
    '''
    docker = '''
    FROM {0}

    RUN yum -y install git wget dmidecode pciutils
    RUN wget -O - https://bootstrap.saltstack.com | sh -s -- -X

    RUN salt-call --local network.mod_hostname {1}
    RUN git init /tmp/repo
    WORKDIR /tmp/repo

    RUN git config remote.origin.url {2}
    RUN git fetch --tags --progress {2} +refs/pull/*:refs/remotes/origin/pr/*
    RUN git pull
    RUN git checkout -f {3}

    RUN ln -s /tmp/repo/{4} /srv/salt
    RUN salt-call --local state.highstate
    '''.format(image, hostname, repo, rev, location)

    cli = Client(base_url='unix:/{0}'.format(socket))

    res = [i for i in cli.build(
        fileobj=BytesIO(docker.encode('utf-8')), rm=True, nocache=True
    )]

    ret = re.search('Failed:\s+([0-9]{1,5})', res[-4])

    if ret:
        print('\n'.join(res[-4].strip().split('\\n'))[:-2])
        return int(ret.group(1))
    else:
        print(res)
        print('FAILED!!')
        return 1
Beispiel #28
0
def build_docker_image(nocache=False, pull=True):
    docker_client = Client(base_url='unix://var/run/docker.sock')

    return docker_client.build(
        tag=os.environ['DOCKER_IMAGE'],
        dockerfile=os.environ['DOCKER_FILE'],
        path=os.getcwd() + '/docker/',
        pull=pull,
        decode=True,
        forcerm=True,
        # Allows to invalidate cache for certains steps in Dockerfile
        # https://github.com/docker/docker/issues/22832
        buildargs={
            'CACHE_DATE': datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
        },
        nocache=nocache)
Beispiel #29
0
    def build_project(self, project_id, tag, **kwargs):
#       Darwin is OS X
        if platform.system() == 'Darwin':
            DOCKER_HOST = os.environ['DOCKER_HOST']
            DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https')
            DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH']
            cert_dir = os.environ['DOCKER_CERT_PATH'] + '/'
            tls_config = TLSConfig(
                 client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False
            )
            docker_client = Client(base_url=DOCKER_HOST, tls = tls_config)
        if platform.system() == 'Linux':
            docker_client = Client(base_url='unix://var/run/docker.sock')

        build = self.model()
        self.validate_tag(tag)
        build.tag = tag
        image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag
        build.project = Project.objects.get_project(project_id)
        Dockerfile = self.optimize_docker_text(project_id = project_id)

        try:
            Build.objects.get(project = build.project, is_active = True)
        except ObjectDoesNotExist:
            pass
        else:  
            old_build = Build.objects.get(project = build.project, is_active = True)
            docker_client.stop(container = old_build.container)

#       build.response = "".join([json.loads(line)["stream"] for line in 
        build.response = ([ line for line in 
#          docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ])
           docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name, nocache=True) ])
 
        try:
            Build.objects.get(project = build.project, is_active = True)
        except ObjectDoesNotExist:
            pass
        else:
            old_build = Build.objects.get(project = build.project, is_active = True)
            docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port })

        build.image_name = image_name
        docker_client.push(repository=image_name, insecure_registry=True)
        build.save(using = self.db)
        return build
Beispiel #30
0
def create_image(name,deploy_settings):
    cli = Client(base_url='unix://var/run/docker.sock')

    docker_path = os.path.join(deploy_settings['base_dir'], 'demokratikollen/deployment/docker/' + name)
    full_name = 'demokratikollen/' + name

    deploy_settings['log'].info("Creating image: {0}".format(name))
    for line in cli.build(path=docker_path, rm=True, tag=full_name):
        log_data = misc_utils.decode_docker_log(line)
        if "error" in log_data:
            deploy_settings['log'].error("Something went wrong with docker: {0}".format(log_data['error']))
            raise Exception
        else:
            if 'stream' in log_data:
                msg = log_data['stream'].strip()
            if 'status' in log_data:
                msg = log_data['status'].strip()
            deploy_settings['log'].debug("Docker: {0}".format(msg))
Beispiel #31
0
def build_docker_image(docker_file_dir, docker_file, repository, log_file):
    docker_client = Client(base_url='unix://var/run/docker.sock')
    write_to_debug_log(log_file,
                       "Creating Docker image: %s ..." % str(repository))

    info = docker_client.build(dockerfile=docker_file,
                               tag=repository,
                               path=docker_file_dir,
                               rm=True,
                               stream=True)
    with open(log_file, 'a') as f:
        for item in info:
            f.write("%s %s\n" % (" " * 10, str(item)))

    write_to_debug_log(
        log_file,
        "Docker image: %s has been created locally!!!" % str(repository))
    return docker_client
Beispiel #32
0
def dock_image(dockerfile_path):
    """generate docker image"""
    status = True
    dc = DockerClient(base_url=CONF.clients_docker.url)
    dc.info()
    with open(dockerfile_path) as dockerfile:
        tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip()
        LOG.debug("Generating %s from %s" % (tag, dockerfile_path))
    if tag:
        resp = dc.build(path=os.path.dirname(dockerfile_path),
                        dockerfile=os.path.basename(dockerfile_path),
                        rm=True,
                        tag=tag)
        for l in resp:
            if "error" in l.lower():
                status = False
            LOG.debug(l.strip())
    return status
Beispiel #33
0
def dock_image(df):
    """generate docker image"""
    status = True
    dc = DockerClient(base_url=CONF.clients_docker.url)
    dc.info()
    with open(df) as dockerfile:
        tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip()
        LOG.debug("Generating %s from %s" % (tag, df))
        resp = dc.build(
            fileobj=dockerfile,
            rm=True,
            tag=tag
        )
    for l in resp:
        if "error" in l.lower():
            status = False
        LOG.debug(l)
    return status
Beispiel #34
0
def docker_build( build_dir, tag="cyledge/base", dockerfile="Dockerfile", pull_first=True, use_cache=True, quiet=False):

  logger = logging.getLogger("docker")
  logging.info("Building docker image %s" % tag)
  
  if pull_first:
    logging.debug("Pulling FROM image first")
  if not use_cache:
    logging.debug("Not using cache")
  
  
  #dockerfile = "%s/%s" % (build_dir, dockerfile)
  
  c = Client(base_url='unix://var/run/docker.sock')
  build_output = c.build(
    path=build_dir,
    dockerfile=dockerfile,
    tag=tag,
    stream=True,
    quiet=quiet,
    pull=pull_first,
    nocache= not use_cache
    )

  last_line = None
  for line in build_output:
    output = json.loads(line.decode('UTF-8'))
    if "stream" in output:
      logger.debug(output["stream"].rstrip())
      last_line = output["stream"]
    if "error" in output:
      logger.error(output["error"].rstrip())
      #logger.error(output["errorDetail"])
    
  if quiet:
    srch = r'sha256:([0-9a-f]{12})[0-9a-f]+'
  else:
    srch = r'Successfully built ([0-9a-f]{12})'
  match = re.search(srch, last_line)
  if not match:
    raise RuntimeError()
  else:
    return match.group(1)
Beispiel #35
0
 def build(self, path, tag):
     """
     Similar to the `docker interpreter`, interpreter a docker image
     :param path: context path include Dockerfile
     :param tag: image's tag
     :return: None
     """
     self.read_port()
     version = self.app['docker']['api']['version']
     cli = Client(base_url=self.url, version=str(version))
     response = cli.build(path, tag, rm=True)
     for line in response:
         rp = {
             key: str(item.strip().decode('unicode_escape'))
             for key, item in ast.literal_eval(line).items()
         }
         log.info(rp)
     log.info("successful build image with dockerImageTag=%s",
              str(tag).split(':')[1])
Beispiel #36
0
def build_image(git_auth,
                image_name,
                git_repo,
                image_tag='latest',
                git_branch=None,
                git_directory=None):
    cli = Client(base_url='unix://var/run/docker.sock',
                 version=docker_api_version)

    if not git_repo.endswith('.git'):
        logger.debug('Adding .git to {}'.format(git_repo))
        git_repo += '.git'
    git_repo = 'https://' + git_auth + '@' + git_repo[8:]
    if git_directory:
        logger.debug('Adding : to {}'.format(git_directory))
        git_directory = ':' + git_directory
        if git_branch is None:
            logger.debug('Adding # to {}'.format(git_branch))
            git_branch = '#'
        else:
            logger.debug('Adding # to {}'.format(git_branch))
            git_branch = '#' + git_branch
    else:
        git_directory = ''
    if not git_branch.startswith('#'):
        logger.debug('Adding # to {}'.format(git_branch))
        git_branch = '#' + git_branch

    logger.debug(git_repo + git_branch + git_directory)
    logger.debug(image_name + ':' + image_tag)

    try:
        response = [
            line for line in cli.build(git_repo + git_branch + git_directory,
                                       rm=True,
                                       tag=image_name + ':' + image_tag)
        ]
    except docker_error as msg:
        logger.error('Something went wrong:\n{}'.format(msg))
        return msg
    else:
        logger.debug(response)
        return None
def bootstrap(_: Namespace) -> None:
    """Create the images used by son-analyze in the current host"""
    cli = Client(base_url='unix://var/run/docker.sock')
    root_context = os.path.realpath(
        resource_filename('son_analyze.cli', '../../..'))
    _LOGGER.info('The root context path is: %s', root_context)
    path = resource_filename('son_analyze.cli.resources',
                             'anaconda.Dockerfile')
    path = os.path.relpath(path, root_context)
    _LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path)
    # import pdb; pdb.set_trace()
    for line in cli.build(path=root_context, tag=_IMAGE_TAG,
                          dockerfile=path, rm=True, decode=True):
        if "stream" in line:
            print('> ', line["stream"], end="")
        else:
            print(line)
            sys.exit(1)
    sys.exit(0)
Beispiel #38
0
def build_image( df, **options ):

	debug = False
	test = False

	input_data = df

	if 'debug' in options and options['debug'] in [True, False]: debug = options['debug']
	if 'test' in options and options['test'] in [True, False]: test = options['test']

	cli = Client( )

	try:

		f = BytesIO( df.as_string().encode('utf-8'))
		response = [line for line in cli.build( fileobj=f, rm=True, tag=df.get_tag() )]

		if debug:
			print("=========================================")
			pprint( response )
			print("=========================================")


		xty = eval( response[-1].decode() )
		if 'stream' in xty and re.match( r"^Successfully", xty['stream'] ):
			print("Completed build")
			return True

		elif 'errorDetail' in xty:
			print("Error in build: %(err)s" % {'err': xty['errorDetail']['message'] })
			return False

		else:
			print("Unexpected result: ")
			pprint( xty )
			return False


	except Exception as error:
		pprint( error )
		return False

	return True
Beispiel #39
0
    def _build_image_on_docker_host(self, base_url, build_file, dockerfile,
                                    image_name, image_version):
        """
        Build image on the selected docker host by Dockerfile.

        'base_url': the url of docker host.
        'build_file': the name of the build file in absolute path.
        'dockerfile': Dockerfile path in build_file.
        'image_name': the name of the image, containing registry address, user
        name and image name.
        'image_version': the version of the image.

        Returns:
        'token': the image token
        """
        self._delete_image_on_docker_host(base_url, image_name, image_version)

        client = Client(base_url=base_url)
        fileobj = open(build_file, 'rb')
        image_complete_name = '%s:%s' % (image_name, image_version)
        try:
            response = [
                line for line in client.build(fileobj=fileobj,
                                              custom_context=True,
                                              dockerfile=dockerfile,
                                              rm=True,
                                              tag=image_complete_name)
            ]
        except APIError as error:
            logger.debug(error)
            logger.error('Cannot locate specified Dockerfile: %s.' %
                         (self.dockerfile))
            fileobj.close()
            return None
        except Exception as error:
            logger.debug(error)
            logger.error('Build image %s failed.' % image_complete_name)
            fileobj.close()
            return None
        fileobj.close()

        return self._get_image_token(base_url, image_complete_name)
Beispiel #40
0
    def _build_image_on_docker_host(self, base_url, build_file, dockerfile,
            image_name, image_version):
        """
        Build image on the selected docker host by Dockerfile.

        'base_url': the url of docker host.
        'build_file': the name of the build file in absolute path.
        'dockerfile': Dockerfile path in build_file.
        'image_name': the name of the image, containing registry address, user
        name and image name.
        'image_version': the version of the image.

        Returns:
        'token': the image token
        """
        self._delete_image_on_docker_host(base_url, image_name, image_version)

        client = Client(base_url=base_url)
        fileobj = open(build_file, 'rb')
        image_complete_name = '%s:%s' % (image_name, image_version)
        try:
            response = [line for line in client.build(
                fileobj=fileobj,
                custom_context=True,
                dockerfile=dockerfile,
                rm=True,
                tag=image_complete_name)]
        except APIError as error:
            logger.debug(error)
            logger.error('Cannot locate specified Dockerfile: %s.' %
                (self.dockerfile))
            fileobj.close()
            return None
        except Exception as error:
            logger.debug(error)
            logger.error('Build image %s failed.' % image_complete_name)
            fileobj.close()
            return None
        fileobj.close()

        return self._get_image_token(base_url, image_complete_name)
Beispiel #41
0
def build_image(client: docker.Client, path='./remote-docker', name='noodles-remote'):
    """Build the Docker image as per Dockerfile present in the 'remote-docker'
    sub-directory. This Docker image should suffice to emulate a remote machine
    with SLURM queue manager and ssh/sftp connection installed. If the docker image
    with given name is newer than the Dockerfile, nothing is done.

    :param client:
        Docker client
    :param path:
        Location of Dockerfile
    :param name:
        Name of the image
    """
    assert os.path.exists(path + '/Dockerfile')
    time = os.stat(path + '/Dockerfile').st_mtime

    il = client.images(name=name)
    if len(il) == 0 or il[0]['Created'] < time:
        response = client.build(path, tag=name, rm=True)
        for json_bytes in response:
            line = json.loads(json_bytes.decode())['stream']
            print(line, end='', file=sys.stderr, flush=True)
def dock_image():
    """generate docker image"""
    import sys
    import os
    status = True
    dc = DockerClient(base_url=CONF.clients_docker.url)
    # inject config files dir to syspath
    print CONF.config_dir
    wp = os.path.abspath(CONF.config_dir)
    sys.path.insert(0, wp)
    os.chdir(wp)
    with open("ChefImage.docker") as dockerfile:
        resp = dc.build(
            fileobj=dockerfile,
            rm=True,
            tag=CONF.clients_docker.image
        )
    for l in resp:
        if "error" in l.lower():
            status = False
        LOG.debug(l)
    return status
Beispiel #43
0
def build_docker_image(request):
    assert isinstance(request,HttpRequest)

    # filter other type of request
    if request.method != "POST" and request.POST:
        return HttpResponseBadRequest()

    # dummy check just to supress the error when
    # accesing POST directly, that is Django's pitfall
    if len(request.POST) == 0:
        pass

    str_json = ""
    try:
        str_json = request.body.decode("utf-8")
    except Exception as e:
        return HttpResponseBadRequest(str(e))

    indata = json.loads(str_json)
    indata['user_id'] = request.user.id

    get_resp = json.loads(GetDockerImageLogic().execute(indata))
    docker_image = get_resp['data']

    try:
        cl = Client(settings.DOCKER_CLIENT)
        f = BytesIO(docker_image['content'].encode('utf-8'))

        if settings.DOCKER_SINGLE_HOST == True:
            image_tag = "%s/%s/%s:%s" % ("127.0.0.1:5000",docker_image['user'],docker_image['name'],docker_image['version'])
        else:
            image_tag = "%s/%s/%s:%s" % (settings.DOCKER_REGISTRY,docker_image['user'],docker_image['name'],docker_image['version'])

        return StreamingHttpResponse(output for output in cl.build(fileobj=f, rm=True, tag=image_tag))

    except Exception as e:
        res = LogicReturn(LogicStatus.FAILED,str(e))
        return HttpResponse(res.to_JSON(), content_type="application/json")
Beispiel #44
0
    def build(self):
        """
        runs docker build with the tarfile context
        """
        docker = Client(version='auto')
        status = docker.build(
            fileobj=self.tarfile,
            custom_context=True,
            tag=self.tag,
            pull=True,
            nocache=True,
            rm=True,
        )

        for line in status:  # This effectively blocks on `docker build`
            try:
                current_app.logger.debug(line)
            except RuntimeError:  # Outside of application context
                print line
        if "successfully built" not in line.lower():
            raise BuildError("Failed to build {}: {}".format(self.tag, line))

        self.built = True
Beispiel #45
0
    def build(self):
        """
        runs docker build with the tarfile context
        """
        docker = Client(version='auto')
        status = docker.build(
            fileobj=self.tarfile,
            custom_context=True,
            tag=self.tag,
            pull=True,
            nocache=True,
            rm=True,
        )

        for line in status:  # This effectively blocks on `docker build`
            try:
                current_app.logger.debug(line)
            except RuntimeError:  # Outside of application context
                print line
        if "successfully built" not in line.lower():
            raise BuildError("Failed to build {}: {}".format(self.tag, line))

        self.built = True
Beispiel #46
0
def build_image(client: docker.Client,
                path='./remote-docker',
                name='noodles-remote'):
    """Build the Docker image as per Dockerfile present in the 'remote-docker'
    sub-directory. This Docker image should suffice to emulate a remote machine
    with SLURM queue manager and ssh/sftp connection installed. If the docker image
    with given name is newer than the Dockerfile, nothing is done.

    :param client:
        Docker client
    :param path:
        Location of Dockerfile
    :param name:
        Name of the image
    """
    assert os.path.exists(path + '/Dockerfile')
    time = os.stat(path + '/Dockerfile').st_mtime

    il = client.images(name=name)
    if len(il) == 0 or il[0]['Created'] < time:
        response = client.build(path, tag=name, rm=True)
        for json_bytes in response:
            line = json.loads(json_bytes.decode())['stream']
            print(line, end='', file=sys.stderr, flush=True)
class DockerHelper(object):
    def __init__(self, logger=None, base_url=""):
        self.logger = logger or create_file_logger()
        self.docker = Client(base_url=base_url or "unix:///var/run/docker.sock")

    def image_exists(self, name):
        """Checks whether a docker image exists.

        :param name: Image name
        :returns: ``True`` if image exists, otherwise ``False``
        """
        images = self.docker.images(name)
        return True if images else False

    def build_image(self, path, tag):
        """Builds a docker image.

        :param path: Path to a directory where ``Dockerfile`` is located.
        :param tag: Desired tag name.
        :returns: ``True`` if image successfully built, otherwise ``False``
        """
        self.logger.info("building {} image".format(tag))
        resp = self.docker.build(path, tag=tag, quiet=True,
                                 rm=True, forcerm=True)

        output = ""
        while True:
            try:
                output = resp.next()
                self.logger.info(output)
            except StopIteration:
                break

        result = json.loads(output)
        if "errorDetail" in result:
            return False
        return True

    def run_container(self, name, image):
        """Runs a docker container in detached mode.

        This is a two-steps operation:

        1. Creates container
        2. Starts container

        :param name: Desired container name.
        :param image: Existing image name.
        :returns: A string of container ID in long format if container
                is running successfully, otherwise an empty string.
        """
        container_id = ""

        self.logger.info("creating container {!r}".format(name))
        env = {
            "SALT_MASTER_IPADDR": os.environ.get("SALT_MASTER_IPADDR"),
        }
        container = self.docker.create_container(
            image=image, name=name, detach=True, environment=env,
        )
        container_id = container["Id"]
        self.logger.info("container {!r} has been created".format(name))

        if container_id:
            self.docker.start(container=container_id)
            self.logger.info("container {!r} with ID {!r} "
                             "has been started".format(name, container_id))
        return container_id

    def get_remote_files(self, *files):
        """Retrieves files from remote paths.

        All retrieved files will be stored under a same temporary directory.

        :param files: List of files.
        :returns: Absolute path to temporary directory where all files
                were downloaded to.
        """
        local_dir = tempfile.mkdtemp()

        for file_ in files:
            local_path = os.path.join(local_dir, os.path.basename(file_))
            self.logger.info("downloading {!r}".format(file_))

            resp = requests.get(file_)
            if resp.status_code == 200:
                with open(local_path, "w") as fp:
                    fp.write(resp.text)
        return local_dir

    def _build_gluubase(self):
        """Builds gluubase image.

        :param salt_master_ipaddr: IP address of salt-master.
        :returns: ``True`` if image successfully built, otherwise ``False``
        """
        build_succeed = True

        if not self.image_exists("gluubase"):
            # There must be a better way than to hard code every file one by one
            DOCKER_REPO = 'https://raw.githubusercontent.com/GluuFederation' \
                          '/gluu-docker/master/ubuntu/14.04'
            minion_file = DOCKER_REPO + '/gluubase/minion'
            supervisor_conf = DOCKER_REPO + '/gluubase/supervisord.conf'
            render = DOCKER_REPO + '/gluubase/render.sh'
            dockerfile = DOCKER_REPO + '/gluubase/Dockerfile'
            files = [minion_file, supervisor_conf, render, dockerfile]
            build_dir = self.get_remote_files(*files)
            build_succeed = self.build_image(build_dir, "gluubase")
            shutil.rmtree(build_dir)
        return build_succeed

    def setup_container(self, name, image, dockerfile, salt_master_ipaddr):
        """Builds and runs a container.

        :param name: Container name.
        :param image: Image name.
        :param dockerfile: Path to remote Dockerfile. Used to build the image
                        if image is not exist.
        :returns: Container ID in long format if container running successfully,
                otherwise an empty string.
        """
        if not self._build_gluubase():
            return ""

        # a flag to determine whether build image process is succeed
        build_succeed = True

        if not self.image_exists(image):
            build_dir = self.get_remote_files(dockerfile)
            build_succeed = self.build_image(build_dir, image)
            shutil.rmtree(build_dir)

        if build_succeed:
            return self.run_container(name, image)
        return ""

    def get_container_ip(self, container_id):
        """Gets container IP.

        :param container_id: Container ID; ideally the short format.
        :returns: Container's IP address.
        """
        info = self.docker.inspect_container(container_id)
        return info["NetworkSettings"]["IPAddress"]

    def remove_container(self, container_id):
        """Removes container.
        """
        try:
            return self.docker.remove_container(container_id, force=True)
        except docker.errors.APIError as exc:
            err_code = exc.response.status_code
            if err_code == 404:
                self.logger.warn(
                    "container {!r} does not exist".format(container_id))
Beispiel #48
0
def set_container():
    global container_id
    global cli
    global render_path
    global config_path
    global watch_path
    log = logging.getLogger(__name__)

    ### Connecting to Docker Client API
    if os.name == 'nt':
        # Windows
        cli = Client(base_url='tcp://127.0.0.1:2375')
    else:
        # Unix
        cli = Client(base_url='unix://var/run/docker.sock')

    ### Just one instance running at a time
    running = cli.containers(all=True, filters={"name": CONTAINER_NAME})
    log.info("{} containers in host: {}".format(CONTAINER_NAME, len(running)))
    if len(running) > 0:
        container_id = running[0].get('Id')
        log.info("Stopping or removing old container {}".format(container_id))
        #response = cli.kill(container=container_id, signal=signal.SIGTERM)
        #print("EXITED")
        #exit(1)
        cli.stop(container=container_id)
        cli.remove_container(container=container_id)

    ### Build docker image from Dockerfile
    log.info("Building container...")
    try:
        response = [
            line for line in cli.build(
                path='./image', tag=IMAGE_NAME, rm=True, forcerm=True)
        ]
    except Exception as error:
        log.error("ERROR: Missing the Dockerfile to build the container %s" %
                  error)
        exit(1)

    for line in response:
        if "error" in line:
            log.error("ERROR: Missing /app folder inside the image.")
            exit(2)

    log.info("Container built succesfully.")

    ### Creating render structure for mounting on /app/render
    uri = [
        "AppData", "Roaming", "com.dedosmedia.SkinRetouching", "Local Store",
        "content", "Keshot", "compositions", "master", "input"
    ]
    render_path = os.path.join(os.path.expanduser("~"), *uri)
    if not os.path.exists(render_path):
        log.info("Creating rendering path on local disk.")
        try:
            os.makedirs(render_path)
        except OSError as error:
            log.error(
                "ERROR: creating rendering structure on local disk. {}".format(
                    error))
            exit(1)

    render_path = os.path.realpath(os.path.dirname(render_path))
    #print("Render path is: {} and exists: {}".format(render_path, os.path.exists(render_path)))

    ### Config path to mount on /app/config
    config_path = os.path.realpath(os.path.dirname(CONFIG_FILENAME))
    log.info("Config path is: {} and exists: {}".format(
        config_path, os.path.exists(config_path)))

    ### Watch path to mount on /app/watch
    watch_path = os.path.realpath(config["watch-folder-" + os.name])
    log.info("Watch path is: {} and exists: {}".format(
        watch_path, os.path.exists(watch_path)))

    try:
        folder = os.path.realpath(
            os.path.join(watch_path, config['output-subfolder']))
        os.makedirs(folder)
    except OSError as error:
        if os.path.exists(folder) == False:
            log.error(
                "ERROR: creating watch output structure on local disk. {}".
                format(error))
            exit(1)
    ### Setting up volumes and port bindings
    host_config = cli.create_host_config(
        binds={
            config_path: {
                'bind': posixpath.normpath(
                    posixpath.join(APP_PATH, CONFIG_PATH)),  #'/app/config',
                'mode': 'rw'
            },
            render_path: {
                'bind': posixpath.normpath(
                    posixpath.join(APP_PATH, RENDER_PATH)),  #'/app/render',
                'mode': 'rw'
            },
            watch_path: {
                'bind':
                posixpath.normpath(posixpath.join(APP_PATH,
                                                  WATCH_PATH)),  #'/app/watch',
                'mode': 'rw'
            }
        },
        port_bindings={PORT: PORT})

    ###  Creating and starting the container
    container = cli.create_container(
        image=IMAGE_NAME,
        #volumes= ['/app/config','/app/render','/app/watch'],
        #ports=[4999],
        detach=True,
        environment={
            "CONFIG": CONFIG_FILENAME,
            "PORT": PORT,
            "CORE": CORE_FILENAME,
            "WATCH": WATCH_PATH,
            "RENDER": RENDER_PATH,
            "HOST_RENDER_PATH": render_path,
            "OUTPUT_NAME": OUTPUT_NAME
        },
        host_config=host_config,
        name=CONTAINER_NAME)
    log.info("Container created: {}".format(container))

    container_id = container.get('Id')
    response = cli.start(container=container_id)
    log.info("Container started correctly. Monitoring: {}".format(
        os.path.realpath(os.path.join(watch_path, config["input-file"]))))
Beispiel #49
0
#Get the port the docker host is listening on
port = get_docker_port(docker)

######## Connect to the docker host ########

#Connect to the remote docker host in the LXC container
dockerclient = Dclient(base_url="tcp://{}:{}".format(host,port), version="auto")

print "Connected to Docker host on {}:{}".format(host,port)

######## Create an docker image ########

print "\nCreating image from Dockerfile\n"

#Create a docker container image from the Dockerfile in this directory
response = dockerclient.build(path=".", dockerfile="Dockerfile", tag="pyne/helloworld")

#Show the output from the build process
for line in response:
    print line

#Show the images now avalible on the docker host
print "Images on docker host:"
pprint(dockerclient.images())

print "\nCreating container from hello world image:\n"

#Create a config object containing the port mappings from the containers port to the host port
#The container port should match the one that is exposed in the Dockerfile
container_port = 5000
host_port = 4000
#Get the hostname
hostname = socket.gethostname()
imagename = hostname+'/'+app

#Connect to docker socket
cli = Client(base_url='unix://docker.sock')

#Define port binding
if dockerized:
	config=cli.create_host_config(network_mode='container:'+yunohostid)
else:
	config=cli.create_host_config(port_bindings={8000: ('127.0.0.1',8000)})

#Build docker image with the Dockerfile and disply the output
for line in cli.build(path='../build/', tag=imagename):	
	out=json.loads(line)
	#sys.stdout.write('\r')
	#print(out['stream'])
	#sys.stdout.flush()

#Create the container and display result
container = cli.create_container(
			image=imagename, 
			detach=True,  
			tty=True,
			name=containername,
			host_config=config
)		

#Start the container and display result
Beispiel #51
0
#!/usr/bin/python


#Se importa el cliente docker.
from docker import Client
from io import BytesIO
dockerfile = '''
 # Shared Volume
 FROM busybox
 MAINTAINER Ernesto Crespo, [email protected]
 VOLUME /data
 CMD ["/bin/sh"]
 '''
f = BytesIO(dockerfile.encode('utf-8'))
cli = Client(base_url='unix://var/run/docker.sock')
response = [line for line in cli.build(
	fileobj=f, rm=True, tag='yourname/volume'
 )]

for i in response:
	print (i)

print (cli.containers())
class Docker(object):
    def __init__(self,
                 socket,
                 remove_intermediate=False,
                 registry=None,
                 insecure_registry=False):
        log.debug('DockerClient connecting to {}'.format(socket))
        self._socket = socket
        self._client = Client(base_url=socket, timeout=300)
        self._remove_intermediate = remove_intermediate
        self._registry = registry
        self._insecure_registry = insecure_registry

    @exception_safe(ConnectionError, False)
    def build(self, dockerfile, image):
        """/
        :param dockerfile: The dockerfile full path
        :param image: the image name (eg: midolman:1.9)
        """
        log.info('Now building {}'.format(image))
        log.debug('Invoking docker build on {}'.format(dockerfile))

        response = self._client.build(path=os.path.dirname(dockerfile),
                                      tag=image,
                                      pull=False,
                                      rm=self._remove_intermediate,
                                      dockerfile=os.path.basename(dockerfile))

        last_line = None
        for line in response:
            eval_line = eval(line)
            if 'stream' in eval_line:
                print(eval_line['stream']),
                last_line = eval_line['stream']
        # Check for ensure success after building
        if 'Successfully built' not in last_line:
            log.error('Error building the image {}.'.format(image))
            return False
        return True

    @exception_safe(ConnectionError, None)
    def pull(self, image):
        """
        :param image: The image name with its tag
        :param repository: The repository where to pull the image
                          (dockerhub if none defined)
        """
        name, tag = image.split(':')
        repository = '{}/{}'.format(self._registry, name) \
            if self._registry else name
        log.info('Now Pulling {}:{}'.format(repository, tag))
        response = self._client.pull(repository=repository,
                                     tag=tag,
                                     insecure_registry=self._insecure_registry,
                                     stream=True)
        for line in response:
            eval_line = eval(line)
            if 'error' in eval_line:
                log.error('Error pulling image: {}'.format(eval_line['error']))
                return False
            if 'status' in eval_line:
                log.info('[{}:{}] Status: {}'.format(repository, tag,
                                                     eval_line['status']))
        if self._registry:
            # If pulling from an external repo, we need to tag with the
            # actual image name used in the flavours definition.
            images = self.list_images(repository)
            for image in images:
                for repotag in image['RepoTags']:
                    if '{}:{}'.format(repository, tag) == repotag:
                        self._client.tag(image['Id'], name, tag, force=True)
        return True

    @exception_safe(ConnectionError, None)
    def push(self, image):
        """
        :param image: The image name with its tag
        :param repository: The repository where to push the image
                           (dockerhub if none defined)
        """
        name, tag = image.split(':')
        if self._registry:
            # First tag the local image to map to the new registry
            repository = '{}/{}'.format(self._registry, name)
            images = self.list_images(name)
            for image in images:
                for repotag in image['RepoTags']:
                    if '{}:{}'.format(name, tag) == repotag:
                        self._client.tag(image['Id'],
                                         repository,
                                         tag,
                                         force=True)
        else:
            repository = name
        log.info('Now pushing {}:{}'.format(repository, tag))
        response = self._client.push(repository=repository,
                                     tag=tag,
                                     insecure_registry=self._insecure_registry,
                                     stream=True)
        for line in response:
            eval_line = eval(line)
            if 'error' in eval_line:
                log.error('Error pushing image: {}'.format(eval_line['error']))
                return False
            if 'status' in eval_line:
                if 'Pushing' not in eval_line['status'] \
                        and 'Buffering' not in eval_line['status']:
                    log.info('[{}:{}] Status: {}'.format(
                        repository, tag, eval_line['status']))
        return True

    @exception_safe(ConnectionError, [])
    def list_images(self, prefix=None):
        """
        List the available images
        :param prefix: Filter the images by a prefix (eg: "sandbox/")
        :return: the images list
        """
        images = self._client.images()

        if prefix:
            filtered = list()
            for image in images:
                for tag in image['RepoTags']:
                    if tag.startswith(prefix):
                        filtered.append(image)

            images = filtered

        return images

    def list_containers(self, prefix=None):
        """
        List the running containers, prefixed with prefix
        :param prefix: The container's name prefix
        :return: The list of containers
        """
        containers = self._client.containers()
        filtered = list()
        if prefix:
            for container_ref in containers:
                if prefix in container_ref['Names'][0]:
                    filtered.append(container_ref)

            containers = filtered

        return containers

    def container_by_name(self, name):
        containers = self.list_containers()
        for container_ref in containers:
            if name == self.principal_container_name(container_ref):
                return container_ref

    @staticmethod
    def principal_container_name(container_ref):
        for name in container_ref['Names']:
            if '/' not in name[1:]:
                return name[1:]

    def container_ip(self, container_ref):
        return self._client.inspect_container(
            container_ref)['NetworkSettings']['IPAddress']

    def stop_container(self, container_ref):
        self._client.stop(container_ref)

    def kill_container(self, container_ref):
        self._client.kill(container_ref)

    def remove_container(self, container_ref):
        self._client.remove_container(container_ref)

    @exception_safe(OSError, False)
    def execute(self, container_ref, command):
        """
        Execute a command inside the container.

        NOTE: Needs the 'docker' binary installed in the host
        """
        cmd = [
            'docker', 'exec', '-it',
            self.principal_container_name(container_ref), 'env', 'TERM=xterm',
            command
        ]
        log.debug('Running command: "{}"'.format(' '.join(cmd)))
        p = subprocess.Popen(cmd, stderr=subprocess.STDOUT)
        p.wait()

    def ssh(self, container_ref):
        self.execute(container_ref, 'bash')
Beispiel #53
0
class TestRunner(object):
    """Badwolf test runner"""
    def __init__(self, context, lock):
        self.context = context
        self.lock = lock
        self.repo_full_name = context.repository
        self.repo_name = context.repository.split('/')[-1]
        self.task_id = str(uuid.uuid4())
        self.commit_hash = context.source['commit']['hash']
        self.build_status = BuildStatus(
            bitbucket, context.source['repository']['full_name'],
            self.commit_hash, 'badwolf/test',
            url_for('log.build_log', sha=self.commit_hash, _external=True))

        self.docker = Client(
            base_url=current_app.config['DOCKER_HOST'],
            timeout=current_app.config['DOCKER_API_TIMEOUT'],
        )

    def run(self):
        start_time = time.time()
        self.branch = self.context.source['branch']['name']

        try:
            self.clone_repository()
        except git.GitCommandError as e:
            logger.exception('Git command error')
            self.update_build_status('FAILED', 'Git clone repository failed')
            content = ':broken_heart: **Git error**: {}'.format(to_text(e))
            if self.context.pr_id:
                pr = PullRequest(bitbucket, self.repo_full_name)
                pr.comment(self.context.pr_id, content)
            else:
                cs = Changesets(bitbucket, self.repo_full_name)
                cs.comment(self.commit_hash, content)

            self.cleanup()
            return

        if not self.validate_settings():
            self.cleanup()
            return

        context = {
            'context':
            self.context,
            'task_id':
            self.task_id,
            'build_log_url':
            url_for('log.build_log', sha=self.commit_hash, _external=True),
            'branch':
            self.branch,
            'scripts':
            self.spec.scripts,
        }

        if self.spec.scripts:
            self.update_build_status('INPROGRESS', 'Test in progress')
            docker_image_name, build_output = self.get_docker_image()
            context['build_logs'] = to_text(build_output)
            context.update({
                'build_logs': to_text(build_output),
                'elapsed_time': int(time.time() - start_time),
            })
            if not docker_image_name:
                self.update_build_status('FAILED',
                                         'Build or get Docker image failed')
                context['exit_code'] = -1
                self.send_notifications(context)
                self.cleanup()
                return

            exit_code, output = self.run_tests_in_container(docker_image_name)
            if exit_code == 0:
                # Success
                logger.info('Test succeed for repo: %s', self.repo_full_name)
                self.update_build_status('SUCCESSFUL', '1 of 1 test succeed')
            else:
                # Failed
                logger.info('Test failed for repo: %s, exit code: %s',
                            self.repo_full_name, exit_code)
                self.update_build_status('FAILED', '1 of 1 test failed')

            context.update({
                'logs': to_text(output),
                'exit_code': exit_code,
                'elapsed_time': int(time.time() - start_time),
            })
            self.send_notifications(context)

        # Code linting
        if self.context.pr_id and self.spec.linters:
            lint = LintProcessor(self.context, self.spec, self.clone_path)
            lint.process()

        self.cleanup()

    def clone_repository(self):
        self.clone_path = os.path.join(tempfile.gettempdir(), 'badwolf',
                                       self.task_id, self.repo_name)
        source_repo = self.context.source['repository']['full_name']
        # Use shallow clone to speed up
        bitbucket.clone(source_repo,
                        self.clone_path,
                        depth=50,
                        branch=self.branch)
        gitcmd = git.Git(self.clone_path)
        if self.context.target:
            # Pull Request
            target_repo = self.context.target['repository']['full_name']
            target_branch = self.context.target['branch']['name']
            if source_repo == target_repo:
                target_remote = 'origin'
            else:
                # Pull Reuqest across forks
                target_remote = target_repo.split('/', 1)[0]
                gitcmd.remote('add', target_remote,
                              bitbucket.get_git_url(target_repo))
            gitcmd.fetch(target_remote, target_branch)
            gitcmd.checkout('FETCH_HEAD')
            gitcmd.merge('origin/{}'.format(self.branch))
        else:
            # Push to branch or ci retry comment on some commit
            logger.info('Checkout commit %s', self.commit_hash)
            gitcmd.checkout(self.commit_hash)

        gitmodules = os.path.join(self.clone_path, '.gitmodules')
        if os.path.exists(gitmodules):
            gitcmd.submodule('update', '--init', '--recursive')

    def validate_settings(self):
        conf_file = os.path.join(self.clone_path,
                                 current_app.config['BADWOLF_PROJECT_CONF'])
        if not os.path.exists(conf_file):
            logger.warning('No project configuration file found for repo: %s',
                           self.repo_full_name)
            return False

        self.spec = spec = Specification.parse_file(conf_file)
        if self.context.type == 'commit' and spec.branch and self.branch not in spec.branch:
            logger.info(
                'Ignore tests since branch %s test is not enabled. Allowed branches: %s',
                self.branch, spec.branch)
            return False

        if not spec.scripts and not spec.linters:
            logger.warning('No script(s) or linter(s) to run')
            return False
        return True

    def get_docker_image(self):
        docker_image_name = self.repo_full_name.replace('/', '-')
        output = []
        with self.lock:
            docker_image = self.docker.images(docker_image_name)
            if not docker_image or self.context.rebuild:
                dockerfile = os.path.join(self.clone_path,
                                          self.spec.dockerfile)
                build_options = {
                    'tag': docker_image_name,
                    'rm': True,
                }
                if not os.path.exists(dockerfile):
                    logger.warning(
                        'No Dockerfile: %s found for repo: %s, using simple runner image',
                        dockerfile, self.repo_full_name)
                    dockerfile_content = 'FROM messense/badwolf-test-runner\n'
                    fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
                    build_options['fileobj'] = fileobj
                else:
                    build_options['dockerfile'] = self.spec.dockerfile

                build_success = False
                logger.info('Building Docker image %s', docker_image_name)
                self.update_build_status('INPROGRESS', 'Building Docker image')
                res = self.docker.build(self.clone_path, **build_options)
                for line in res:
                    if b'Successfully built' in line:
                        build_success = True
                    log = to_text(json.loads(to_text(line))['stream'])
                    output.append(log)
                    logger.info('`docker build` : %s', log.strip())
                if not build_success:
                    return None, ''.join(output)

        return docker_image_name, ''.join(output)

    def run_tests_in_container(self, docker_image_name):
        command = '/bin/sh -c badwolf-run'
        environment = {}
        if self.spec.environments:
            # TODO: Support run in multiple environments
            environment = self.spec.environments[0]

        # TODO: Add more test context related env vars
        environment.update({
            'DEBIAN_FRONTEND': 'noninteractive',
            'CI': 'true',
            'CI_NAME': 'badwolf',
            'BADWOLF_BRANCH': self.branch,
            'BADWOLF_COMMIT': self.commit_hash,
            'BADWOLF_BUILD_DIR': '/mnt/src',
            'BADWOLF_REPO_SLUG': self.repo_full_name,
        })
        if self.context.pr_id:
            environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id)

        container = self.docker.create_container(
            docker_image_name,
            command=command,
            environment=environment,
            working_dir='/mnt/src',
            volumes=['/mnt/src'],
            host_config=self.docker.create_host_config(
                privileged=self.spec.privileged,
                binds={
                    self.clone_path: {
                        'bind': '/mnt/src',
                        'mode': 'rw',
                    },
                }))
        container_id = container['Id']
        logger.info('Created container %s from image %s', container_id,
                    docker_image_name)

        output = []
        try:
            self.docker.start(container_id)
            self.update_build_status('INPROGRESS',
                                     'Running tests in Docker container')
            for line in self.docker.logs(container_id, stream=True):
                output.append(to_text(line))
            exit_code = self.docker.wait(
                container_id, current_app.config['DOCKER_RUN_TIMEOUT'])
        except (APIError, DockerException, ReadTimeout) as e:
            exit_code = -1
            output.append(to_text(e))
            logger.exception('Docker error')
        finally:
            try:
                self.docker.remove_container(container_id, force=True)
            except (APIError, DockerException):
                logger.exception('Error removing docker container')

        return exit_code, ''.join(output)

    def update_build_status(self, state, description=None):
        try:
            self.build_status.update(state, description=description)
        except BitbucketAPIError:
            logger.exception('Error calling Bitbucket API')

    def send_notifications(self, context):
        exit_code = context['exit_code']
        template = 'test_success' if exit_code == 0 else 'test_failure'
        html = render_template('mail/' + template + '.html', **context)
        html = sanitize_sensitive_data(html)

        # Save log html
        log_dir = os.path.join(current_app.config['BADWOLF_LOG_DIR'],
                               self.commit_hash)
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        log_file = os.path.join(log_dir, 'build.html')
        with open(log_file, 'wb') as f:
            f.write(to_binary(html))

        if exit_code == 0:
            subject = 'Test succeed for repository {}'.format(
                self.repo_full_name)
        else:
            subject = 'Test failed for repository {}'.format(
                self.repo_full_name)
        notification = self.spec.notification
        emails = notification['emails']
        if emails:
            send_mail(emails, subject, html)

        slack_webhooks = notification['slack_webhooks']
        if slack_webhooks:
            message = render_template('slack_webhook/' + template + '.md',
                                      **context)
            trigger_slack_webhook(slack_webhooks, message)

    def cleanup(self):
        shutil.rmtree(os.path.dirname(self.clone_path), ignore_errors=True)
Beispiel #54
0
class Docker_interface:
    def __init__(self,
                 net_name='tosker_net',
                 tmp_dir='/tmp',
                 socket='unix://var/run/docker.sock'):
        self._log = Logger.get(__name__)
        self._net_name = net_name
        self._cli = Client(base_url=os.environ.get('DOCKER_HOST') or socket)
        self._tmp_dir = tmp_dir

    # TODO: aggiungere un parametro per eliminare i container se esistono gia'!
    def create(self, con, cmd=None, entrypoint=None, saved_image=False):
        def create_container():
            tmp_dir = path.join(self._tmp_dir, con.name)
            try:
                os.makedirs(tmp_dir)
            except:
                pass
            saved_img_name = '{}/{}'.format(self._net_name, con.name)
            img_name = con.image
            if saved_image and self.inspect(saved_img_name):
                img_name = saved_img_name

            self._log.debug('container: {}'.format(con.get_str_obj()))

            con.id = self._cli.create_container(
                name=con.name,
                image=img_name,
                entrypoint=entrypoint if entrypoint else con.entrypoint,
                command=cmd if cmd else con.cmd,
                environment=con.env,
                detach=True,
                # stdin_open=True,
                ports=[key for key in con.ports.keys()] if con.ports else None,
                volumes=['/tmp/dt'] +
                ([k for k, v in con.volume.items()] if con.volume else []),
                networking_config=self._cli.create_networking_config({
                    self._net_name:
                    self._cli.create_endpoint_config(links=con.link
                                                     # ,aliases=['db']
                                                     )
                }),
                host_config=self._cli.create_host_config(
                    port_bindings=con.ports,
                    # links=con.link,
                    binds=[tmp_dir + ':/tmp/dt'] +
                    ([v + ':' + k
                      for k, v in con.volume.items()] if con.volume else []),
                )).get('Id')

        assert isinstance(con, Container)

        if con.to_build:
            self._log.debug('start building..')
            # utility.print_json(
            self._cli.build(path='/'.join(con.dockerfile.split('/')[0:-1]),
                            dockerfile='./' + con.dockerfile.split('/')[-1],
                            tag=con.image,
                            pull=True,
                            quiet=True)
            # )
            self._log.debug('stop building..')
        elif not saved_image:
            # TODO: da evitare se si deve utilizzare un'immagine custom
            self._log.debug('start pulling.. {}'.format(con.image))
            utility.print_json(self._cli.pull(con.image, stream=True),
                               self._log.debug)
            self._log.debug('end pulling..')

        try:
            create_container()
        except errors.APIError as e:
            self._log.debug(e)
            # self.stop(con)
            self.delete(con)
            create_container()
            # raise e

    def stop(self, container):
        name = self._get_name(container)
        try:
            return self._cli.stop(name)
        except errors.NotFound as e:
            self._log.error(e)

    def start(self, container, wait=False):
        name = self._get_name(container)
        self._cli.start(name)
        if wait:
            self._log.debug('wait container..')
            self._cli.wait(name)
            utility.print_byte(self._cli.logs(name, stream=True),
                               self._log.debug)

    def delete(self, container):
        name = self._get_name(container)
        try:
            self._cli.remove_container(name, v=True)
        except (errors.NotFound, errors.APIError) as e:
            self._log.error(e)
            raise e

    def exec_cmd(self, container, cmd):
        name = self._get_name(container)
        if not self.is_running(name):
            return False
        try:
            exec_id = self._cli.exec_create(name, cmd)
            status = self._cli.exec_start(exec_id)

            # TODO: verificare attendibilita' di questo check!
            check = 'rpc error:' != status[:10].decode("utf-8")
            self._log.debug('check: {}'.format(check))
            return check
        except errors.APIError as e:
            self._log.error(e)
            return False
        except requests.exceptions.ConnectionError as e:
            # TODO: questo errore arriva dopo un timeout di 10 secodi
            self._log.error(e)
            return False

    def create_volume(self, volume):
        assert isinstance(volume, Volume)
        self._log.debug('volume opt: {}'.format(volume.get_all_opt()))
        return self._cli.create_volume(volume.name, volume.driver,
                                       volume.get_all_opt())

    def delete_volume(self, volume):
        name = self._get_name(volume)
        return self._cli.remove_volume(name)

    def get_containers(self, all=False):
        return self._cli.containers(all=all)

    def get_volumes(self):
        volumes = self._cli.volumes()
        return volumes['Volumes'] or []

    def inspect(self, item):
        name = self._get_name(item)
        try:
            return self._cli.inspect_container(name)
        except errors.NotFound:
            pass
        try:
            return self._cli.inspect_image(name)
        except errors.NotFound:
            pass
        try:
            return self._cli.inspect_volume(name)
        except errors.NotFound:
            return None

    def remove_all_containers(self):
        for c in self.get_containers(all=True):
            self.stop(c['Id'])
            self.delete(c['Id'])

    def remove_all_volumes(self):
        for v in self.get_volumes():
            self.delete_volume(v['Name'])

    def create_network(self, name, subnet='172.25.0.0/16'):
        # docker network create -d bridge --subnet 172.25.0.0/16 isolated_nw
        # self.delete_network(name)
        try:
            self._cli.create_network(name=name,
                                     driver='bridge',
                                     ipam={'subnet': subnet},
                                     check_duplicate=True)
        except errors.APIError:
            self._log.debug('network already exists!')

    def delete_network(self, name):
        assert isinstance(name, str)
        try:
            self._cli.remove_network(name)
        except errors.APIError:
            self._log.debug('network not exists!')

    def delete_image(self, name):
        assert isinstance(name, str)
        try:
            self._cli.remove_image(name)
        except errors.NotFound:
            pass

    # TODO: splittare questo metodo in due, semantica non chiara!
    def update_container(self, node, cmd, saved_image=True):
        assert isinstance(node, Container)
        # self._log.debug('container_conf: {}'.format(node.host_container))
        stat = self.inspect(node.image)
        old_cmd = stat['Config']['Cmd'] or None
        old_entry = stat['Config']['Entrypoint'] or None

        if self.inspect(node):
            self.stop(node)
            self.delete(node)
        self.create(node, cmd=cmd, entrypoint='', saved_image=saved_image)

        self.start(node.id, wait=True)
        self.stop(node.id)

        name = '{}/{}'.format(self._net_name, node.name)

        self._cli.commit(node.id, name)

        self.stop(node)
        self.delete(node)
        self.create(node,
                    cmd=node.cmd or old_cmd,
                    entrypoint=node.entrypoint or old_entry,
                    saved_image=True)

        self._cli.commit(node.id, name)

    def is_running(self, container):
        name = self._get_name(container)
        stat = self.inspect(name)
        stat = stat is not None and stat['State']['Running'] is True
        self._log.debug('State: {}'.format(stat))
        return stat

    def _get_name(self, name):
        if isinstance(name, six.string_types):
            return name
        else:
            assert isinstance(name, (Container, Volume))
            return name.name
Beispiel #55
0
 def buildDockerfile(self):
     encDockerfile = BytesIO(self.dockerfile.encode('utf-8'))
     cli = Client(base_url='tcp://192.168.1.20:2375')
     response = [line for line in cli.build(fileobj=encDockerfile, rm=True, tag='webpy' )]
     print response
class Deploy():

    def __init__(self):
        self.registry_username = self.get_env("KS_DOCKER_REGISTRY_USERNAME")
        self.registry_password = self.get_env("KS_DOCKER_REGISTRY_PASSWORD")
        self.registry = self.get_env('KS_DOCKER_REGISTRY')
        self.project_status = self.get_env("KS_PROJECT_STATUS")
        self.project_name = self.get_env('KS_PROJECT_NAME')
        self.project_group = self.get_env('KS_PROJECT_GROUP')
        self.go_pipeline_label = self.get_env("GO_PIPELINE_LABEL")
        self.project_replicas = int(self.get_env("KS_PROJECT_REPLICAS",1))
        self.project_port = int(self.get_env('KS_PROJECT_PORT', 80))
        self.docker_server = self.get_env("KS_DOCKER_SERVER")
        self.k8s_host = self.get_env('K8S_HOST')
        self.k8s_username = self.get_env('K8S_USERNAME')
        self.k8s_password = self.get_env('K8S_PASSWORD')
        self.container_cpu = self.get_env('KS_CONTAINER_CPU', '2')
        self.container_cpu = self.get_env('KS_CONTAINER_MEMORY', '512M')
        self.k8s_namespace = self.get_env('KS_PROJECT_GROUP',None)

        self.go_environment_name = self.get_env('GO_ENVIRONMENT_NAME')

        self.project_branch = self.get_project_branch()
        self.project_commit  = self.get_project_commit()

        self.project_version = '{branch}-{label}.git{commit}'.format(
            branch = self.project_branch,
            label = self.go_pipeline_label,
            commit = self.project_commit
        )

        self.image_name = self.project_name
        self.image_tag = self.project_version
        self.image_tag_name='{image}:{tag}'.format(
            image = self.image_name,
            tag = self.image_tag
        )

        self.k8s_deployment_name = self.project_name + '-' + self.project_status
        self.k8s_image = self.registry + '/' + self.image_name + ':' + self.image_tag

    def connect_docker(self):
        self.dockerClient = Client(base_url=self.docker_server)
        self.dockerClient.login(username=self.registry_username, password=self.registry_password,registry=self.registry)

    def get_project_commit(self):
        commit = commands.getoutput('git log  --pretty=format:"%h" -n 1')
        return commit

    def get_project_branch(self):
        branch = commands.getoutput('git branch | grep "*" | head -n 1')
        branch = re.sub('\*','',branch)
        branch = re.sub("/",".",branch)
        branch = branch.strip()
        log.info(branch)
        return branch

    def get_env(self,key,value=None):
        value = os.getenv(key,value)
        if value == None:
            raise Exception('{key} not be null.'.format(key=key))
        return value

    def build(self, dockerfile='.'):
        self.connect_docker()
        image_tag = self.image_tag_name
        status = True
        log.info('Start To Build Image %s' % image_tag)
        msgs = ''
        for msg in self.dockerClient.build(path=dockerfile, tag=image_tag, forcerm=True,buildargs={'KS_PROJECT_NAME':self.project_name},nocache=True):
            log.info(msg)
            msgs+=msg
            if re.search("error", msg, re.IGNORECASE):
                status = False
        if not re.search("Successfully", msgs, re.IGNORECASE):
            status = False
        else:
            status = True

        if status == True:
            log.info("Build Docker Image[SUCCESS].")
        else:
            log.error("Build Docker Image[FAILD].")
            exit(1)
        return status
    def push(self,image_tag_src=None,image_tag_dest=None):
        self.connect_docker()
        if self.registry=="" or self.registry==None:return
        if image_tag_src == None:image_tag_src= self.image_tag_name
        if image_tag_dest == None:image_tag_dest= self.image_tag_name
        self.dockerClient.login(self.registry_username,password=self.registry_password,registry=self.registry)
        self.dockerClient.tag(image_tag_src , repository=self.registry+"/"+self.image_name,tag=self.image_tag)
        response = self.dockerClient.push(repository=self.registry+"/"+image_tag_dest)
        if re.search("ERROR",response,re.IGNORECASE):
            log.error("PUSH IMAGE[%s] TO REPOSITORY FAILD."%image_tag_dest)
            log.error(response)
            exit(1)
        else:
            log.info(response)



    def  deployment_action(self):


        cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host)
        deployment = K8sDeployment(config=cfg_cert, name=self.k8s_deployment_name)
        deployment.add_label(k='project',v=self.project_name)
        if not deployment._exists():
            container = K8sContainer(name=self.k8s_deployment_name, image=self.k8s_image)
            container.add_env('KS_PROJECT_VERSION',self.project_version)
            container.add_image_pull_policy(policy='Always')
            container.add_volume_mount(K8sVolumeMount(name="{project}-resources-volume".format(project=self.k8s_deployment_name),
                                                      mount_path="/data/{project}/resources".format(
                                                          project=self.project_name)))
            #container.add_volume_mount(K8sVolumeMount(name='localtime-volumn',mount_path='/etc/localtime'))
            container.add_volume_mount(K8sVolumeMount(name='{project}-log-volume'.format(project=self.k8s_deployment_name),
                                                      mount_path='/data/logs/'))
            container.add_container_resources_limits(cpu='2', memory='1024Mi')
            #container.add_container_resources_requests(cpu='200m',memory='512M')
            deployment.add_container(container)
            deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image))
            deployment.add_image_pull_secrets([{'name': 'ksszregistrykey'}])

            vol = K8sVolume(name="{project}-resources-volume".format(project=self.k8s_deployment_name), type='hostPath')
            vol.path = "/data/docker/resources/{env}/{project}".format(env=self.project_status,project=self.project_name)
            deployment.add_volume(vol)

            #vol_localtime = K8sVolume(name='localtime-volumn', type='hostPath')
            #vol_localtime.path = "/etc/localtime"
            #deployment.add_volume(vol_localtime)

            vol_log = K8sVolume(name="{project}-log-volume".format(project=self.k8s_deployment_name), type='hostPath')
            vol_log.path = "/data/docker/logs/{env}/{project}".format(env=self.project_status,project=self.project_name)
            deployment.add_volume(vol_log)
            deployment.create()
        else:
            deployment.change_container_resources_limits(cpu='2', memory='1024Mi')
            deployment.set_container_image(name=self.k8s_deployment_name, image=self.k8s_image)
            deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image))
            deployment.update()
        deployment.scale(self.project_replicas)

    def service_action(self):
        self.k8s_svc_name = self.project_name+'-'+self.project_status
        cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host)
        service = K8sService(config=cfg_cert, name=self.k8s_svc_name)
        if not service._exists():
            service.add_selector(selector={"name": self.k8s_deployment_name})
            service.type = 'NodePort'
            service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP')
            service.create()
        else:
            service.add_selector(selector={"name": self.k8s_deployment_name})
            service.type = 'NodePort'
            if not service._exists_port(self.project_port):
                service.del_port()
                service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP')
            service.update()