class Docker(object): def __init__(self,base_url = ''): self.cli = Client(base_url=base_url) self.containers = [] self.images = 'docker/whalesay' def build(self): self.cli.build(path='../../dockfiles/'+self.images,rm=True,tag=self.images) def create(self): container = self.cli.create_container(image=self.images + ':latest', command='cowsay boo',detach=True, name=(self.images + ": " + (len(self.containers)-1).__str__())) self.containers.append(container) return len(self.containers) - 1 def remove(self,num): self.stop(num) container = self.containers[num] self.cli.remove_container(container=container) self.containers.remove(num) def start(self,num): self.cli.start(container=self.containers[num]) def stop(self,num): self.cli.stop(container=self.containers[num])
def get_image_id(self): image_name = 'container-%d:v%d' % (self.id, self.version) path = os.path.join(settings.DOCKER_ROOT, 'build', image_name) # create a client to communicate with docker client = Client(base_url='unix://var/run/docker.sock') # check if already built images = client.images(name=image_name) if images: print('docker image exists: %s with name %s' % (self.tag, image_name)) return images[0]['Id'] # build the docker file print('building new docker image: %s with name %s' % (self.tag, image_name)) extract_zip(self.dockerfile_src, path) log = list(client.build(path=path, rm=True, tag=image_name)) self.build_log = "".join([str(i)+': '+str(log[i]) for i in range(len(log))]) self.save() images = client.images(name=image_name) if images: return images[0]['Id'] else: raise LookupError('Docker image not found: "' + self.tag + '"')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--nocache', action='store_true', default=False) parser.add_argument('--dockerfile', type=local) parser.add_argument('--label', choices=IMAGES.keys()) arguments = parser.parse_args() content = '' client = Client(base_url='unix://var/run/docker.sock') stream = client.build( tag=IMAGES[arguments.label], dockerfile=arguments.dockerfile.basename, path=arguments.dockerfile.dirname, pull=True, decode=True, forcerm=True, nocache=arguments.nocache ) for item in stream: buff = item.get('stream', item.get('status', '')) if not content or re.search('.+\[[. ]*$', content): content += buff if not re.search('.+\[[. ]*$', content): print(content) content = ''
class DockerLayer(object): def __init__(self, name, tag): self.__name__ = name self.__bases__ = tuple([]) self.tag = tag self.client = Client(base_url='unix://var/run/docker.sock') try: self.client.ping() except ConnectionError as e: # http://docker-py.readthedocs.org/en/latest/boot2docker/ kwargs = kwargs_from_env() kwargs['tls'].assert_hostname = False self.client = Client(**kwargs) def setUp(self): if self.client.ping() == u'OK': self.start() else: raise RuntimeError('Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.') def start(self): for line in self.client.build( path=os.path.abspath(os.path.join(DIR, '..')), tag=self.tag, rm=True, forcerm=True): sys.stdout.write(line) def tearDown(self): self.stop() def stop(self): self.client.close()
def main(token, commit): if token != os.getenv("TOKEN"): abort(401) working_copy = "/repository/checkout" if not os.path.isdir(working_copy): os.mkdir(working_copy) Git().clone(os.getenv("GIT_REPOSITORY_URL"), working_copy) print "Refreshing git repository" repo = Repo(working_copy) origin = repo.remotes.origin origin.pull(); repo.git.reset(commit, hard=True) print "Building docker image" docker = Client(base_url='unix://mount/run/docker.sock') docker.login(username=os.getenv("DOCKER_REGISTRY_USERNAME"), password=os.getenv("DOCKER_REGISTRY_PASSWORD"), email=os.getenv("DOCKER_REGISTRY_EMAIL", "*****@*****.**")) before_build = os.getenv('BEFORE_BUILD_CMD') if before_build is not None: check_call(before_build, shell=True) for line in docker.build(path=working_copy, tag=os.getenv("IMAGE_NAME"), stream=True): print line print "Pushing docker image" for line in docker.push(os.getenv("IMAGE_NAME"), stream=True): print line print "Listing docker image" check_call(os.getenv('DEPLOY_CMD'), shell=True) return "OK"
def custom_docker_build(self): from io import BytesIO from docker import Client import json import re doc_client = Client(base_url='unix://var/run/docker.sock', version='1.18') df1 = open(df_name, "r").read() f = BytesIO(df1.encode('utf-8')) resp = [ line for line in doc_client.build( fileobj=f, rm=False, tag=cont_name + "-1") ] try: parsed_lines = [json.loads(e).get('stream', '') for e in resp] except ValueError: # sometimes all the data is sent on a single line ???? # # ValueError: Extra data: line 1 column 87 - line 1 column # 33268 (char 86 - 33267) line = resp[0] # This ONLY works because every line is formatted as # {"stream": STRING} parsed_lines = [ json.loads(obj).get('stream', '') for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line) ] # ----- fmlogger.error(parsed_lines)
def __init__(self,url_folder_language): cli = Client(base_url='unix://var/run/docker.sock') response = cli.build(path=url_folder_language,rm=True,nocache=True) for i in response: j=i[30:] id_container=j[:12] self.id_container=id_container
def get_log(): logs = [] json_data = json.loads(request.get_data()) project = ProjectModel.query.filter_by(proname=json_data['data']).first() if not project.is_build(): os.chdir(app.config['CODE_FOLDER'] + '/' + json_data['data']) cli = Client(base_url=HOST) for line in cli.build(path=os.getcwd(), stream=True, decode=True, tag=str(REGISTRY + json_data['data'])): send_log(line) logs.append(line) # 向私有仓库推送镜像, 没有log的打印 for line in cli.push(REGISTRY + json_data['data'], stream=True): # 打印出上传的log print line assert line redis.hset(project.id, project.verify, logs) project.build = True if 'Successfully built' in logs[-1]['stream']: project.success = 1 else: project.success = 2 db.session.add(project) db.session.commit() else: lines = eval(redis.hget(project.id, project.verify)) for line in lines: send_log(line) return json.dumps({'msg': 'ok'})
class DockerClient(object): def __init__(self): self.client = Client(base_url='unix://var/run/docker.sock') def build_images(self, dockerfile: str, tag: str): with open(dockerfile) as file: dkfile = BytesIO(file.read().encode('utf-8')) response = [line for line in self.client.build( fileobj=dkfile, rm=True, tag=tag)] return response def run_container(self, image, mem_limit=None, volume_binds: list = None, command=None): container = self.client.create_container(image=image, host_config=self.client.create_host_config( binds=volume_binds, mem_limit=mem_limit), command=command) self.client.start(container) try: self.client.wait(container, timeout=3) except ReadTimeout: print('time out') self.client.stop(container) self.client.remove_container(container) def exec_container(self, container, cmd): container_id = self.client.exec_create(container, cmd)['Id'] return self.client.exec_start(container_id)
class DockerTest(object): def __init__(self): self.cli = Client(base_url='unix://var/run/docker.sock') def build(self, dockerfile): dockerfile = Path(dockerfile) tag = 'rf' + dockerfile.basename().replace('Dockerfile.', '') dockerfile.copy('redfish-client/tests/Dockerfile') response = [line for line in self.cli.build( path='redfish-client/tests', tag=tag, rm=True)] return(response) def run(self, image, command): container = self.cli.create_container(image=image, command=command, tty=True, stdin_open=True) self.cli.start(container=container.get('Id')) self.cli.wait(container=container.get('Id')) response = self.cli.logs(container=container.get('Id'), stdout=True) self.cli.remove_container(container=container.get('Id')) return(response.decode('utf8'))
def build_docker_image(docker_file_dir, docker_file, repository, log_file): """ Build docker image from the given docker file Args: docker_file_dir (path): Docker file dir docker_file (path): Docker file repository (str): Repo name log_file (path): Log file path Returns: auth_config_payload (dict): AWS auth config """ docker_client = Client(base_url='unix://var/run/docker.sock') write_to_debug_log(log_file, "Creating Docker image: %s ..." % str(repository)) info = docker_client.build(dockerfile=docker_file, tag=repository, path=docker_file_dir, rm=True, stream=True) with open(log_file, 'a') as f: for item in info: f.write("%s %s\n" % (" " * 10, str(item))) write_to_debug_log( log_file, "Docker image: %s has been created locally!!!" % str(repository)) return docker_client
class DockerLayer(object): def __init__(self, name, tag): self.__name__ = name self.__bases__ = tuple([]) self.tag = tag self.client = Client(base_url='unix://var/run/docker.sock') try: self.client.ping() except ConnectionError as e: # http://docker-py.readthedocs.org/en/latest/boot2docker/ kwargs = kwargs_from_env() kwargs['tls'].assert_hostname = False self.client = Client(**kwargs) def setUp(self): if self.client.ping() == u'OK': self.start() else: raise RuntimeError( 'Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.' ) def start(self): for line in self.client.build(path=os.path.abspath( os.path.join(DIR, '..')), tag=self.tag, rm=True, forcerm=True): sys.stdout.write(line) def tearDown(self): self.stop() def stop(self): self.client.close()
class DockerTest(object): def __init__(self): self.cli = Client(base_url='unix://var/run/docker.sock') def build(self, dockerfile): dockerfile = Path(dockerfile) tag = 'rf-' + dockerfile.basename().replace('.dkf', '') dockerfile.copy('redfish-client/tests/Dockerfile') response = [line for line in self.cli.build( path='redfish-client/tests', tag=tag, rm=True)] return(response) def run(self, image, command): container = self.cli.create_container(image=image, command=command, tty=True, stdin_open=True) self.cli.start(container=container.get('Id')) self.cli.wait(container=container.get('Id')) response = self.cli.logs(container=container.get('Id'), stdout=True) self.cli.remove_container(container=container.get('Id')) return(response.decode('utf8'))
class DockerOpers(UtilOpers): def __init__(self): self.client = Client(base_url='unix://var/run/docker.sock', version='1.12') @staticmethod def get_path_by__type(tp): return options.dockerfile_dir + "/%s" % tp def build(self, path, tag='', noCache=False): streams = self.client.build(tag=tag, path=path, nocache=noCache, rm=True) res = [line for line in streams] return res def push(self, repository, tag=''): res = self.client.push(repository, tag) return res def remove(self, repository, tag=''): image = '%s:%s' % (repository, 'latest' if tag == '' else tag) res = self.client.remove_image(image) return res
def _create_docker_image(dockerfilepath, repository, pacbot_installation): docker_client = Client(base_url='unix://var/run/docker.sock') for info in docker_client.build(path=dockerfilepath, tag=repository, rm=True, stream=True): _logs_display(info, pacbot_installation) return docker_client
def bake_main(options): """Munge a dockerfile from a cfg cake: layers: [] """ endpoint = os.environ.get("LAYERCAKE_API") if endpoint: options.layer_endpoint = endpoint config = yaml.load(open(options.config))['cake'] df = dockerfile.Dockerfile(options.dockerfile) if options.layer_endpoint: df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint)) # In this mode we are adding run cmds for each # layer in the cfg file (those may pull other layers) # then we output a new docker file and docker build the # new container. last_run = df.last("RUN") if not options.use_devel: df.add("RUN", ['pip3', 'install', '--upgrade', 'layer_cake==%s' % VERSION], at=last_run) else: # For devel we ask the container to pull git master df.add("RUN", [ 'pip3', 'install', '--upgrade', "https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake" ], at=last_run) for layer_name in config['layers']: last_run = df.last("RUN") df.add("RUN", ["cake", "layer", layer_name, "-d", LAYERS_HOME], at=last_run) # we might have an entrypoint # or a command (or both) if df.entrypoint: df.entrypoint = ["disco"] + df.entrypoint['args'] log.debug("Using Dockerfile\n%s", str(df)) if not options.no_build: client = DockerClient() f = BytesIO(str(df).encode("utf-8")) response = client.build(fileobj=f, tag="layercake/disco", decode=True) for line in response: if 'errorDetail' in line: log.critical(line['errorDetail']['message'].strip()) elif 'stream' in line: log.info(line['stream'].strip()) else: return df
def build_docker_container(): # Connect to docker docker = DockerClient(base_url='unix://var/run/docker.sock') # Build the container r = [line for line in docker.build( forcerm=True, tag="transmission", path=settings.BASE_DIR+'/dockerbuild/', decode=True )] return r[-1]['stream'].find('Successfully built')
def build(ctx, image_name='ghcr.io/Daplanet/terraform-localexecute:latest'): """ Builds base docker image and pushes to <image_name> """ try: cli = Client(base_url="unix:///var/run/docker.sock") cli.login(registry=image_name.split('/', 1)[0], user="******", password=getenv('GITHUB_TOKEN')) cli.build(path='.', tag=image_name) cli.push(image_name) except Exception as err: print(err) sys.exit(1) finally: del (cli)
def build_baton_docker(docker_client: Client, baton_docker_build: BatonImage): """ Builds the baton Docker image. :param docker_client: the Docker client :param baton_docker_build: where the baton docker is built from """ logging.info("Building baton Docker image - if this is not cached, it will take a few minutes") logging.debug("baton Docker build: %s" % baton_docker_build) # Note: reading the lines in this ways enforces that Python blocks - required for line in docker_client.build(tag=baton_docker_build.tag, path=baton_docker_build.path, dockerfile=baton_docker_build.docker_file, buildargs=baton_docker_build.build_args): logging.debug(line)
def _build_images_from_dockerfiles(self): """ Build Docker images for each local Dockerfile found in the package: self.local_docker_files """ if GK_STANDALONE_MODE: return # do not build anything in standalone mode dc = DockerClient() LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files)) for k, v in self.local_docker_files.iteritems(): for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False): LOG.debug("DOCKER BUILD: %s" % line) LOG.info("Docker image created: %s" % k)
def build_docker_image(nocache=False): docker_client = Client(base_url='unix://var/run/docker.sock') path = os.getcwd() + '/docker/' image = ImageFactory(docker_client=docker_client, path=path) return docker_client.build( path=image['path'], tag=image['tag'], dockerfile=image['dockerfile'], pull=True, decode=True, forcerm=True, nocache=nocache )
def build_docker_image(nocache=False, pull=True): version = os.environ.get('VERSION', 'sles12sp1') flavor = os.environ.get('FLAVOR') or 'products' docker_client = Client(base_url='unix://var/run/docker.sock') return docker_client.build( tag='registry.mgr.suse.de/toaster-{0}-{1}'.format(version, flavor), dockerfile='Dockerfile.{0}.{1}'.format(version, flavor), path=os.getcwd() + '/docker/', pull=pull, decode=True, forcerm=True, nocache=nocache)
def create_container(cname, user, giturl): try: error = False errmsg = '' cid = '' host_port = '' print('nodeJsBuild start') print('file creation') cli = Client(base_url='unix://var/run/docker.sock') #tls_config = tls.TLSConfig( # client_cert=('/Users/kasi-mac/.docker/machine/machines/default/cert.pem', '/Users/kasi-mac/.docker/machine/machines/default/key.pem'), # ca_cert='/Users/kasi-mac/.docker/machine/machines/default/ca.pem', verify=True) #cli = Client(base_url='tcp://192.168.99.100:2376', tls=tls_config) print('cli creation') response = [ line for line in cli.build(path=giturl, rm=True, tag=user + '/' + cname) ] print(response) print('cli creation end') container = cli.create_container( image=user + '/' + cname, name=cname, host_config=cli.create_host_config(publish_all_ports=True)) print(container) ccrres = json.loads(json.dumps(container)) if 'Id' not in ccrres: error = True errmsg = 'Container creation failed' else: print('1') cid = ccrres['Id'] print(cid) response = cli.start(container=container.get('Id')) print(response) if response != None: error = True errmsg = 'Container failed to start' else: error = False info = cli.inspect_container(container) print info host_port = 'http://' + str( info['NetworkSettings']['Ports']['8080/tcp'][0] ['HostIp']) + ':' + str(info['NetworkSettings']['Ports'] ['8080/tcp'][0]['HostPort']) print host_port return error, errmsg, cid, host_port except Exception as inst: print inst.args return True, 'Exception', '', ''
def bake_main(options): """Munge a dockerfile from a cfg cake: layers: [] """ endpoint = os.environ.get("LAYERCAKE_API") if endpoint: options.layer_endpoint = endpoint config = yaml.load(open(options.config))['cake'] df = dockerfile.Dockerfile(options.dockerfile) if options.layer_endpoint: df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint)) # In this mode we are adding run cmds for each # layer in the cfg file (those may pull other layers) # then we output a new docker file and docker build the # new container. last_run = df.last("RUN") if not options.use_devel: df.add("RUN", ['pip3', 'install', '--upgrade', 'layer_cake==%s' % VERSION], at=last_run) else: # For devel we ask the container to pull git master df.add("RUN", ['pip3', 'install', '--upgrade', "https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake"], at=last_run) for layer_name in config['layers']: last_run = df.last("RUN") df.add("RUN", ["cake", "layer", layer_name, "-d", LAYERS_HOME], at=last_run) # we might have an entrypoint # or a command (or both) if df.entrypoint: df.entrypoint = ["disco"] + df.entrypoint['args'] log.debug("Using Dockerfile\n%s", str(df)) if not options.no_build: client = DockerClient() f = BytesIO(str(df).encode("utf-8")) response = client.build(fileobj=f, tag="layercake/disco", decode=True) for line in response: if 'errorDetail' in line: log.critical(line['errorDetail']['message'].strip()) elif 'stream' in line: log.info(line['stream'].strip()) else: return df
def build_project(self, project_id, tag, **kwargs): DOCKER_HOST = os.environ['DOCKER_HOST'] DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https') DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH'] cert_dir = os.environ['DOCKER_CERT_PATH'] + '/' tls_config = TLSConfig( client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False ) docker_client = Client(base_url=DOCKER_HOST, tls = tls_config) build = self.model() self.validate_tag(tag) build.tag = tag image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag build.project = Project.objects.get_project(project_id) Dockerfile = self.optimize_docker_text(project_id = project_id) try: Build.objects.get(project = build.project, is_active = True) except ObjectDoesNotExist: pass else: old_build = Build.objects.get(project = build.project, is_active = True) docker_client.stop(container = old_build.container) # build.response = "".join([json.loads(line)["stream"] for line in build.response = ([ line for line in # docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ]) docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name) ]) container = docker_client.create_container( image=image_name, name= build.project.name+"_"+build.tag, detach=True, ports = [ 8080 ] ) build.container = container.get('Id') build.port = 10000 + build.project.id try: Build.objects.get(project = build.project, is_active = True) except ObjectDoesNotExist: pass else: old_build = Build.objects.get(project = build.project, is_active = True) docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port }) build.save(using = self.db) return build
def build(self): from docker import Client from io import BytesIO dockerhost = os.environ.get("DOCKER_HOST", "tcp://127.0.0.1:2375") dockerfile = super(DockerBuild, self).build() dockerfile = BytesIO(dockerfile.encode('utf-8')) cli = Client(dockerhost) builder = cli.build(fileobj=dockerfile, rm=True, tag="bones") for line in builder: line = json.loads(line) if len(line.keys()) == 1 and "stream" in line: sys.stdout.write(line["stream"]) sys.stdout.flush() else: print(line)
def build(repo, rev, image, hostname, location, socket='/var/run/docker.sock'): ''' Build a docker image and install a salt minion, checkout the pull request an perform a function test against the new code. :param repo The repository you would like to test against :param rev The revision to check :param socket The docket socket to perform all communication accorss ''' docker = ''' FROM {0} RUN yum -y install git wget dmidecode pciutils RUN wget -O - https://bootstrap.saltstack.com | sh -s -- -X RUN salt-call --local network.mod_hostname {1} RUN git init /tmp/repo WORKDIR /tmp/repo RUN git config remote.origin.url {2} RUN git fetch --tags --progress {2} +refs/pull/*:refs/remotes/origin/pr/* RUN git pull RUN git checkout -f {3} RUN ln -s /tmp/repo/{4} /srv/salt RUN salt-call --local state.highstate '''.format(image, hostname, repo, rev, location) cli = Client(base_url='unix:/{0}'.format(socket)) res = [i for i in cli.build( fileobj=BytesIO(docker.encode('utf-8')), rm=True, nocache=True )] ret = re.search('Failed:\s+([0-9]{1,5})', res[-4]) if ret: print('\n'.join(res[-4].strip().split('\\n'))[:-2]) return int(ret.group(1)) else: print(res) print('FAILED!!') return 1
def build_docker_image(nocache=False, pull=True): docker_client = Client(base_url='unix://var/run/docker.sock') return docker_client.build( tag=os.environ['DOCKER_IMAGE'], dockerfile=os.environ['DOCKER_FILE'], path=os.getcwd() + '/docker/', pull=pull, decode=True, forcerm=True, # Allows to invalidate cache for certains steps in Dockerfile # https://github.com/docker/docker/issues/22832 buildargs={ 'CACHE_DATE': datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") }, nocache=nocache)
def build_project(self, project_id, tag, **kwargs): # Darwin is OS X if platform.system() == 'Darwin': DOCKER_HOST = os.environ['DOCKER_HOST'] DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https') DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH'] cert_dir = os.environ['DOCKER_CERT_PATH'] + '/' tls_config = TLSConfig( client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False ) docker_client = Client(base_url=DOCKER_HOST, tls = tls_config) if platform.system() == 'Linux': docker_client = Client(base_url='unix://var/run/docker.sock') build = self.model() self.validate_tag(tag) build.tag = tag image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag build.project = Project.objects.get_project(project_id) Dockerfile = self.optimize_docker_text(project_id = project_id) try: Build.objects.get(project = build.project, is_active = True) except ObjectDoesNotExist: pass else: old_build = Build.objects.get(project = build.project, is_active = True) docker_client.stop(container = old_build.container) # build.response = "".join([json.loads(line)["stream"] for line in build.response = ([ line for line in # docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ]) docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name, nocache=True) ]) try: Build.objects.get(project = build.project, is_active = True) except ObjectDoesNotExist: pass else: old_build = Build.objects.get(project = build.project, is_active = True) docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port }) build.image_name = image_name docker_client.push(repository=image_name, insecure_registry=True) build.save(using = self.db) return build
def create_image(name,deploy_settings): cli = Client(base_url='unix://var/run/docker.sock') docker_path = os.path.join(deploy_settings['base_dir'], 'demokratikollen/deployment/docker/' + name) full_name = 'demokratikollen/' + name deploy_settings['log'].info("Creating image: {0}".format(name)) for line in cli.build(path=docker_path, rm=True, tag=full_name): log_data = misc_utils.decode_docker_log(line) if "error" in log_data: deploy_settings['log'].error("Something went wrong with docker: {0}".format(log_data['error'])) raise Exception else: if 'stream' in log_data: msg = log_data['stream'].strip() if 'status' in log_data: msg = log_data['status'].strip() deploy_settings['log'].debug("Docker: {0}".format(msg))
def build_docker_image(docker_file_dir, docker_file, repository, log_file): docker_client = Client(base_url='unix://var/run/docker.sock') write_to_debug_log(log_file, "Creating Docker image: %s ..." % str(repository)) info = docker_client.build(dockerfile=docker_file, tag=repository, path=docker_file_dir, rm=True, stream=True) with open(log_file, 'a') as f: for item in info: f.write("%s %s\n" % (" " * 10, str(item))) write_to_debug_log( log_file, "Docker image: %s has been created locally!!!" % str(repository)) return docker_client
def dock_image(dockerfile_path): """generate docker image""" status = True dc = DockerClient(base_url=CONF.clients_docker.url) dc.info() with open(dockerfile_path) as dockerfile: tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip() LOG.debug("Generating %s from %s" % (tag, dockerfile_path)) if tag: resp = dc.build(path=os.path.dirname(dockerfile_path), dockerfile=os.path.basename(dockerfile_path), rm=True, tag=tag) for l in resp: if "error" in l.lower(): status = False LOG.debug(l.strip()) return status
def dock_image(df): """generate docker image""" status = True dc = DockerClient(base_url=CONF.clients_docker.url) dc.info() with open(df) as dockerfile: tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip() LOG.debug("Generating %s from %s" % (tag, df)) resp = dc.build( fileobj=dockerfile, rm=True, tag=tag ) for l in resp: if "error" in l.lower(): status = False LOG.debug(l) return status
def docker_build( build_dir, tag="cyledge/base", dockerfile="Dockerfile", pull_first=True, use_cache=True, quiet=False): logger = logging.getLogger("docker") logging.info("Building docker image %s" % tag) if pull_first: logging.debug("Pulling FROM image first") if not use_cache: logging.debug("Not using cache") #dockerfile = "%s/%s" % (build_dir, dockerfile) c = Client(base_url='unix://var/run/docker.sock') build_output = c.build( path=build_dir, dockerfile=dockerfile, tag=tag, stream=True, quiet=quiet, pull=pull_first, nocache= not use_cache ) last_line = None for line in build_output: output = json.loads(line.decode('UTF-8')) if "stream" in output: logger.debug(output["stream"].rstrip()) last_line = output["stream"] if "error" in output: logger.error(output["error"].rstrip()) #logger.error(output["errorDetail"]) if quiet: srch = r'sha256:([0-9a-f]{12})[0-9a-f]+' else: srch = r'Successfully built ([0-9a-f]{12})' match = re.search(srch, last_line) if not match: raise RuntimeError() else: return match.group(1)
def build(self, path, tag): """ Similar to the `docker interpreter`, interpreter a docker image :param path: context path include Dockerfile :param tag: image's tag :return: None """ self.read_port() version = self.app['docker']['api']['version'] cli = Client(base_url=self.url, version=str(version)) response = cli.build(path, tag, rm=True) for line in response: rp = { key: str(item.strip().decode('unicode_escape')) for key, item in ast.literal_eval(line).items() } log.info(rp) log.info("successful build image with dockerImageTag=%s", str(tag).split(':')[1])
def build_image(git_auth, image_name, git_repo, image_tag='latest', git_branch=None, git_directory=None): cli = Client(base_url='unix://var/run/docker.sock', version=docker_api_version) if not git_repo.endswith('.git'): logger.debug('Adding .git to {}'.format(git_repo)) git_repo += '.git' git_repo = 'https://' + git_auth + '@' + git_repo[8:] if git_directory: logger.debug('Adding : to {}'.format(git_directory)) git_directory = ':' + git_directory if git_branch is None: logger.debug('Adding # to {}'.format(git_branch)) git_branch = '#' else: logger.debug('Adding # to {}'.format(git_branch)) git_branch = '#' + git_branch else: git_directory = '' if not git_branch.startswith('#'): logger.debug('Adding # to {}'.format(git_branch)) git_branch = '#' + git_branch logger.debug(git_repo + git_branch + git_directory) logger.debug(image_name + ':' + image_tag) try: response = [ line for line in cli.build(git_repo + git_branch + git_directory, rm=True, tag=image_name + ':' + image_tag) ] except docker_error as msg: logger.error('Something went wrong:\n{}'.format(msg)) return msg else: logger.debug(response) return None
def bootstrap(_: Namespace) -> None: """Create the images used by son-analyze in the current host""" cli = Client(base_url='unix://var/run/docker.sock') root_context = os.path.realpath( resource_filename('son_analyze.cli', '../../..')) _LOGGER.info('The root context path is: %s', root_context) path = resource_filename('son_analyze.cli.resources', 'anaconda.Dockerfile') path = os.path.relpath(path, root_context) _LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path) # import pdb; pdb.set_trace() for line in cli.build(path=root_context, tag=_IMAGE_TAG, dockerfile=path, rm=True, decode=True): if "stream" in line: print('> ', line["stream"], end="") else: print(line) sys.exit(1) sys.exit(0)
def build_image( df, **options ): debug = False test = False input_data = df if 'debug' in options and options['debug'] in [True, False]: debug = options['debug'] if 'test' in options and options['test'] in [True, False]: test = options['test'] cli = Client( ) try: f = BytesIO( df.as_string().encode('utf-8')) response = [line for line in cli.build( fileobj=f, rm=True, tag=df.get_tag() )] if debug: print("=========================================") pprint( response ) print("=========================================") xty = eval( response[-1].decode() ) if 'stream' in xty and re.match( r"^Successfully", xty['stream'] ): print("Completed build") return True elif 'errorDetail' in xty: print("Error in build: %(err)s" % {'err': xty['errorDetail']['message'] }) return False else: print("Unexpected result: ") pprint( xty ) return False except Exception as error: pprint( error ) return False return True
def _build_image_on_docker_host(self, base_url, build_file, dockerfile, image_name, image_version): """ Build image on the selected docker host by Dockerfile. 'base_url': the url of docker host. 'build_file': the name of the build file in absolute path. 'dockerfile': Dockerfile path in build_file. 'image_name': the name of the image, containing registry address, user name and image name. 'image_version': the version of the image. Returns: 'token': the image token """ self._delete_image_on_docker_host(base_url, image_name, image_version) client = Client(base_url=base_url) fileobj = open(build_file, 'rb') image_complete_name = '%s:%s' % (image_name, image_version) try: response = [ line for line in client.build(fileobj=fileobj, custom_context=True, dockerfile=dockerfile, rm=True, tag=image_complete_name) ] except APIError as error: logger.debug(error) logger.error('Cannot locate specified Dockerfile: %s.' % (self.dockerfile)) fileobj.close() return None except Exception as error: logger.debug(error) logger.error('Build image %s failed.' % image_complete_name) fileobj.close() return None fileobj.close() return self._get_image_token(base_url, image_complete_name)
def _build_image_on_docker_host(self, base_url, build_file, dockerfile, image_name, image_version): """ Build image on the selected docker host by Dockerfile. 'base_url': the url of docker host. 'build_file': the name of the build file in absolute path. 'dockerfile': Dockerfile path in build_file. 'image_name': the name of the image, containing registry address, user name and image name. 'image_version': the version of the image. Returns: 'token': the image token """ self._delete_image_on_docker_host(base_url, image_name, image_version) client = Client(base_url=base_url) fileobj = open(build_file, 'rb') image_complete_name = '%s:%s' % (image_name, image_version) try: response = [line for line in client.build( fileobj=fileobj, custom_context=True, dockerfile=dockerfile, rm=True, tag=image_complete_name)] except APIError as error: logger.debug(error) logger.error('Cannot locate specified Dockerfile: %s.' % (self.dockerfile)) fileobj.close() return None except Exception as error: logger.debug(error) logger.error('Build image %s failed.' % image_complete_name) fileobj.close() return None fileobj.close() return self._get_image_token(base_url, image_complete_name)
def build_image(client: docker.Client, path='./remote-docker', name='noodles-remote'): """Build the Docker image as per Dockerfile present in the 'remote-docker' sub-directory. This Docker image should suffice to emulate a remote machine with SLURM queue manager and ssh/sftp connection installed. If the docker image with given name is newer than the Dockerfile, nothing is done. :param client: Docker client :param path: Location of Dockerfile :param name: Name of the image """ assert os.path.exists(path + '/Dockerfile') time = os.stat(path + '/Dockerfile').st_mtime il = client.images(name=name) if len(il) == 0 or il[0]['Created'] < time: response = client.build(path, tag=name, rm=True) for json_bytes in response: line = json.loads(json_bytes.decode())['stream'] print(line, end='', file=sys.stderr, flush=True)
def dock_image(): """generate docker image""" import sys import os status = True dc = DockerClient(base_url=CONF.clients_docker.url) # inject config files dir to syspath print CONF.config_dir wp = os.path.abspath(CONF.config_dir) sys.path.insert(0, wp) os.chdir(wp) with open("ChefImage.docker") as dockerfile: resp = dc.build( fileobj=dockerfile, rm=True, tag=CONF.clients_docker.image ) for l in resp: if "error" in l.lower(): status = False LOG.debug(l) return status
def build_docker_image(request): assert isinstance(request,HttpRequest) # filter other type of request if request.method != "POST" and request.POST: return HttpResponseBadRequest() # dummy check just to supress the error when # accesing POST directly, that is Django's pitfall if len(request.POST) == 0: pass str_json = "" try: str_json = request.body.decode("utf-8") except Exception as e: return HttpResponseBadRequest(str(e)) indata = json.loads(str_json) indata['user_id'] = request.user.id get_resp = json.loads(GetDockerImageLogic().execute(indata)) docker_image = get_resp['data'] try: cl = Client(settings.DOCKER_CLIENT) f = BytesIO(docker_image['content'].encode('utf-8')) if settings.DOCKER_SINGLE_HOST == True: image_tag = "%s/%s/%s:%s" % ("127.0.0.1:5000",docker_image['user'],docker_image['name'],docker_image['version']) else: image_tag = "%s/%s/%s:%s" % (settings.DOCKER_REGISTRY,docker_image['user'],docker_image['name'],docker_image['version']) return StreamingHttpResponse(output for output in cl.build(fileobj=f, rm=True, tag=image_tag)) except Exception as e: res = LogicReturn(LogicStatus.FAILED,str(e)) return HttpResponse(res.to_JSON(), content_type="application/json")
def build(self): """ runs docker build with the tarfile context """ docker = Client(version='auto') status = docker.build( fileobj=self.tarfile, custom_context=True, tag=self.tag, pull=True, nocache=True, rm=True, ) for line in status: # This effectively blocks on `docker build` try: current_app.logger.debug(line) except RuntimeError: # Outside of application context print line if "successfully built" not in line.lower(): raise BuildError("Failed to build {}: {}".format(self.tag, line)) self.built = True
class DockerHelper(object): def __init__(self, logger=None, base_url=""): self.logger = logger or create_file_logger() self.docker = Client(base_url=base_url or "unix:///var/run/docker.sock") def image_exists(self, name): """Checks whether a docker image exists. :param name: Image name :returns: ``True`` if image exists, otherwise ``False`` """ images = self.docker.images(name) return True if images else False def build_image(self, path, tag): """Builds a docker image. :param path: Path to a directory where ``Dockerfile`` is located. :param tag: Desired tag name. :returns: ``True`` if image successfully built, otherwise ``False`` """ self.logger.info("building {} image".format(tag)) resp = self.docker.build(path, tag=tag, quiet=True, rm=True, forcerm=True) output = "" while True: try: output = resp.next() self.logger.info(output) except StopIteration: break result = json.loads(output) if "errorDetail" in result: return False return True def run_container(self, name, image): """Runs a docker container in detached mode. This is a two-steps operation: 1. Creates container 2. Starts container :param name: Desired container name. :param image: Existing image name. :returns: A string of container ID in long format if container is running successfully, otherwise an empty string. """ container_id = "" self.logger.info("creating container {!r}".format(name)) env = { "SALT_MASTER_IPADDR": os.environ.get("SALT_MASTER_IPADDR"), } container = self.docker.create_container( image=image, name=name, detach=True, environment=env, ) container_id = container["Id"] self.logger.info("container {!r} has been created".format(name)) if container_id: self.docker.start(container=container_id) self.logger.info("container {!r} with ID {!r} " "has been started".format(name, container_id)) return container_id def get_remote_files(self, *files): """Retrieves files from remote paths. All retrieved files will be stored under a same temporary directory. :param files: List of files. :returns: Absolute path to temporary directory where all files were downloaded to. """ local_dir = tempfile.mkdtemp() for file_ in files: local_path = os.path.join(local_dir, os.path.basename(file_)) self.logger.info("downloading {!r}".format(file_)) resp = requests.get(file_) if resp.status_code == 200: with open(local_path, "w") as fp: fp.write(resp.text) return local_dir def _build_gluubase(self): """Builds gluubase image. :param salt_master_ipaddr: IP address of salt-master. :returns: ``True`` if image successfully built, otherwise ``False`` """ build_succeed = True if not self.image_exists("gluubase"): # There must be a better way than to hard code every file one by one DOCKER_REPO = 'https://raw.githubusercontent.com/GluuFederation' \ '/gluu-docker/master/ubuntu/14.04' minion_file = DOCKER_REPO + '/gluubase/minion' supervisor_conf = DOCKER_REPO + '/gluubase/supervisord.conf' render = DOCKER_REPO + '/gluubase/render.sh' dockerfile = DOCKER_REPO + '/gluubase/Dockerfile' files = [minion_file, supervisor_conf, render, dockerfile] build_dir = self.get_remote_files(*files) build_succeed = self.build_image(build_dir, "gluubase") shutil.rmtree(build_dir) return build_succeed def setup_container(self, name, image, dockerfile, salt_master_ipaddr): """Builds and runs a container. :param name: Container name. :param image: Image name. :param dockerfile: Path to remote Dockerfile. Used to build the image if image is not exist. :returns: Container ID in long format if container running successfully, otherwise an empty string. """ if not self._build_gluubase(): return "" # a flag to determine whether build image process is succeed build_succeed = True if not self.image_exists(image): build_dir = self.get_remote_files(dockerfile) build_succeed = self.build_image(build_dir, image) shutil.rmtree(build_dir) if build_succeed: return self.run_container(name, image) return "" def get_container_ip(self, container_id): """Gets container IP. :param container_id: Container ID; ideally the short format. :returns: Container's IP address. """ info = self.docker.inspect_container(container_id) return info["NetworkSettings"]["IPAddress"] def remove_container(self, container_id): """Removes container. """ try: return self.docker.remove_container(container_id, force=True) except docker.errors.APIError as exc: err_code = exc.response.status_code if err_code == 404: self.logger.warn( "container {!r} does not exist".format(container_id))
def set_container(): global container_id global cli global render_path global config_path global watch_path log = logging.getLogger(__name__) ### Connecting to Docker Client API if os.name == 'nt': # Windows cli = Client(base_url='tcp://127.0.0.1:2375') else: # Unix cli = Client(base_url='unix://var/run/docker.sock') ### Just one instance running at a time running = cli.containers(all=True, filters={"name": CONTAINER_NAME}) log.info("{} containers in host: {}".format(CONTAINER_NAME, len(running))) if len(running) > 0: container_id = running[0].get('Id') log.info("Stopping or removing old container {}".format(container_id)) #response = cli.kill(container=container_id, signal=signal.SIGTERM) #print("EXITED") #exit(1) cli.stop(container=container_id) cli.remove_container(container=container_id) ### Build docker image from Dockerfile log.info("Building container...") try: response = [ line for line in cli.build( path='./image', tag=IMAGE_NAME, rm=True, forcerm=True) ] except Exception as error: log.error("ERROR: Missing the Dockerfile to build the container %s" % error) exit(1) for line in response: if "error" in line: log.error("ERROR: Missing /app folder inside the image.") exit(2) log.info("Container built succesfully.") ### Creating render structure for mounting on /app/render uri = [ "AppData", "Roaming", "com.dedosmedia.SkinRetouching", "Local Store", "content", "Keshot", "compositions", "master", "input" ] render_path = os.path.join(os.path.expanduser("~"), *uri) if not os.path.exists(render_path): log.info("Creating rendering path on local disk.") try: os.makedirs(render_path) except OSError as error: log.error( "ERROR: creating rendering structure on local disk. {}".format( error)) exit(1) render_path = os.path.realpath(os.path.dirname(render_path)) #print("Render path is: {} and exists: {}".format(render_path, os.path.exists(render_path))) ### Config path to mount on /app/config config_path = os.path.realpath(os.path.dirname(CONFIG_FILENAME)) log.info("Config path is: {} and exists: {}".format( config_path, os.path.exists(config_path))) ### Watch path to mount on /app/watch watch_path = os.path.realpath(config["watch-folder-" + os.name]) log.info("Watch path is: {} and exists: {}".format( watch_path, os.path.exists(watch_path))) try: folder = os.path.realpath( os.path.join(watch_path, config['output-subfolder'])) os.makedirs(folder) except OSError as error: if os.path.exists(folder) == False: log.error( "ERROR: creating watch output structure on local disk. {}". format(error)) exit(1) ### Setting up volumes and port bindings host_config = cli.create_host_config( binds={ config_path: { 'bind': posixpath.normpath( posixpath.join(APP_PATH, CONFIG_PATH)), #'/app/config', 'mode': 'rw' }, render_path: { 'bind': posixpath.normpath( posixpath.join(APP_PATH, RENDER_PATH)), #'/app/render', 'mode': 'rw' }, watch_path: { 'bind': posixpath.normpath(posixpath.join(APP_PATH, WATCH_PATH)), #'/app/watch', 'mode': 'rw' } }, port_bindings={PORT: PORT}) ### Creating and starting the container container = cli.create_container( image=IMAGE_NAME, #volumes= ['/app/config','/app/render','/app/watch'], #ports=[4999], detach=True, environment={ "CONFIG": CONFIG_FILENAME, "PORT": PORT, "CORE": CORE_FILENAME, "WATCH": WATCH_PATH, "RENDER": RENDER_PATH, "HOST_RENDER_PATH": render_path, "OUTPUT_NAME": OUTPUT_NAME }, host_config=host_config, name=CONTAINER_NAME) log.info("Container created: {}".format(container)) container_id = container.get('Id') response = cli.start(container=container_id) log.info("Container started correctly. Monitoring: {}".format( os.path.realpath(os.path.join(watch_path, config["input-file"]))))
#Get the port the docker host is listening on port = get_docker_port(docker) ######## Connect to the docker host ######## #Connect to the remote docker host in the LXC container dockerclient = Dclient(base_url="tcp://{}:{}".format(host,port), version="auto") print "Connected to Docker host on {}:{}".format(host,port) ######## Create an docker image ######## print "\nCreating image from Dockerfile\n" #Create a docker container image from the Dockerfile in this directory response = dockerclient.build(path=".", dockerfile="Dockerfile", tag="pyne/helloworld") #Show the output from the build process for line in response: print line #Show the images now avalible on the docker host print "Images on docker host:" pprint(dockerclient.images()) print "\nCreating container from hello world image:\n" #Create a config object containing the port mappings from the containers port to the host port #The container port should match the one that is exposed in the Dockerfile container_port = 5000 host_port = 4000
#Get the hostname hostname = socket.gethostname() imagename = hostname+'/'+app #Connect to docker socket cli = Client(base_url='unix://docker.sock') #Define port binding if dockerized: config=cli.create_host_config(network_mode='container:'+yunohostid) else: config=cli.create_host_config(port_bindings={8000: ('127.0.0.1',8000)}) #Build docker image with the Dockerfile and disply the output for line in cli.build(path='../build/', tag=imagename): out=json.loads(line) #sys.stdout.write('\r') #print(out['stream']) #sys.stdout.flush() #Create the container and display result container = cli.create_container( image=imagename, detach=True, tty=True, name=containername, host_config=config ) #Start the container and display result
#!/usr/bin/python #Se importa el cliente docker. from docker import Client from io import BytesIO dockerfile = ''' # Shared Volume FROM busybox MAINTAINER Ernesto Crespo, [email protected] VOLUME /data CMD ["/bin/sh"] ''' f = BytesIO(dockerfile.encode('utf-8')) cli = Client(base_url='unix://var/run/docker.sock') response = [line for line in cli.build( fileobj=f, rm=True, tag='yourname/volume' )] for i in response: print (i) print (cli.containers())
class Docker(object): def __init__(self, socket, remove_intermediate=False, registry=None, insecure_registry=False): log.debug('DockerClient connecting to {}'.format(socket)) self._socket = socket self._client = Client(base_url=socket, timeout=300) self._remove_intermediate = remove_intermediate self._registry = registry self._insecure_registry = insecure_registry @exception_safe(ConnectionError, False) def build(self, dockerfile, image): """/ :param dockerfile: The dockerfile full path :param image: the image name (eg: midolman:1.9) """ log.info('Now building {}'.format(image)) log.debug('Invoking docker build on {}'.format(dockerfile)) response = self._client.build(path=os.path.dirname(dockerfile), tag=image, pull=False, rm=self._remove_intermediate, dockerfile=os.path.basename(dockerfile)) last_line = None for line in response: eval_line = eval(line) if 'stream' in eval_line: print(eval_line['stream']), last_line = eval_line['stream'] # Check for ensure success after building if 'Successfully built' not in last_line: log.error('Error building the image {}.'.format(image)) return False return True @exception_safe(ConnectionError, None) def pull(self, image): """ :param image: The image name with its tag :param repository: The repository where to pull the image (dockerhub if none defined) """ name, tag = image.split(':') repository = '{}/{}'.format(self._registry, name) \ if self._registry else name log.info('Now Pulling {}:{}'.format(repository, tag)) response = self._client.pull(repository=repository, tag=tag, insecure_registry=self._insecure_registry, stream=True) for line in response: eval_line = eval(line) if 'error' in eval_line: log.error('Error pulling image: {}'.format(eval_line['error'])) return False if 'status' in eval_line: log.info('[{}:{}] Status: {}'.format(repository, tag, eval_line['status'])) if self._registry: # If pulling from an external repo, we need to tag with the # actual image name used in the flavours definition. images = self.list_images(repository) for image in images: for repotag in image['RepoTags']: if '{}:{}'.format(repository, tag) == repotag: self._client.tag(image['Id'], name, tag, force=True) return True @exception_safe(ConnectionError, None) def push(self, image): """ :param image: The image name with its tag :param repository: The repository where to push the image (dockerhub if none defined) """ name, tag = image.split(':') if self._registry: # First tag the local image to map to the new registry repository = '{}/{}'.format(self._registry, name) images = self.list_images(name) for image in images: for repotag in image['RepoTags']: if '{}:{}'.format(name, tag) == repotag: self._client.tag(image['Id'], repository, tag, force=True) else: repository = name log.info('Now pushing {}:{}'.format(repository, tag)) response = self._client.push(repository=repository, tag=tag, insecure_registry=self._insecure_registry, stream=True) for line in response: eval_line = eval(line) if 'error' in eval_line: log.error('Error pushing image: {}'.format(eval_line['error'])) return False if 'status' in eval_line: if 'Pushing' not in eval_line['status'] \ and 'Buffering' not in eval_line['status']: log.info('[{}:{}] Status: {}'.format( repository, tag, eval_line['status'])) return True @exception_safe(ConnectionError, []) def list_images(self, prefix=None): """ List the available images :param prefix: Filter the images by a prefix (eg: "sandbox/") :return: the images list """ images = self._client.images() if prefix: filtered = list() for image in images: for tag in image['RepoTags']: if tag.startswith(prefix): filtered.append(image) images = filtered return images def list_containers(self, prefix=None): """ List the running containers, prefixed with prefix :param prefix: The container's name prefix :return: The list of containers """ containers = self._client.containers() filtered = list() if prefix: for container_ref in containers: if prefix in container_ref['Names'][0]: filtered.append(container_ref) containers = filtered return containers def container_by_name(self, name): containers = self.list_containers() for container_ref in containers: if name == self.principal_container_name(container_ref): return container_ref @staticmethod def principal_container_name(container_ref): for name in container_ref['Names']: if '/' not in name[1:]: return name[1:] def container_ip(self, container_ref): return self._client.inspect_container( container_ref)['NetworkSettings']['IPAddress'] def stop_container(self, container_ref): self._client.stop(container_ref) def kill_container(self, container_ref): self._client.kill(container_ref) def remove_container(self, container_ref): self._client.remove_container(container_ref) @exception_safe(OSError, False) def execute(self, container_ref, command): """ Execute a command inside the container. NOTE: Needs the 'docker' binary installed in the host """ cmd = [ 'docker', 'exec', '-it', self.principal_container_name(container_ref), 'env', 'TERM=xterm', command ] log.debug('Running command: "{}"'.format(' '.join(cmd))) p = subprocess.Popen(cmd, stderr=subprocess.STDOUT) p.wait() def ssh(self, container_ref): self.execute(container_ref, 'bash')
class TestRunner(object): """Badwolf test runner""" def __init__(self, context, lock): self.context = context self.lock = lock self.repo_full_name = context.repository self.repo_name = context.repository.split('/')[-1] self.task_id = str(uuid.uuid4()) self.commit_hash = context.source['commit']['hash'] self.build_status = BuildStatus( bitbucket, context.source['repository']['full_name'], self.commit_hash, 'badwolf/test', url_for('log.build_log', sha=self.commit_hash, _external=True)) self.docker = Client( base_url=current_app.config['DOCKER_HOST'], timeout=current_app.config['DOCKER_API_TIMEOUT'], ) def run(self): start_time = time.time() self.branch = self.context.source['branch']['name'] try: self.clone_repository() except git.GitCommandError as e: logger.exception('Git command error') self.update_build_status('FAILED', 'Git clone repository failed') content = ':broken_heart: **Git error**: {}'.format(to_text(e)) if self.context.pr_id: pr = PullRequest(bitbucket, self.repo_full_name) pr.comment(self.context.pr_id, content) else: cs = Changesets(bitbucket, self.repo_full_name) cs.comment(self.commit_hash, content) self.cleanup() return if not self.validate_settings(): self.cleanup() return context = { 'context': self.context, 'task_id': self.task_id, 'build_log_url': url_for('log.build_log', sha=self.commit_hash, _external=True), 'branch': self.branch, 'scripts': self.spec.scripts, } if self.spec.scripts: self.update_build_status('INPROGRESS', 'Test in progress') docker_image_name, build_output = self.get_docker_image() context['build_logs'] = to_text(build_output) context.update({ 'build_logs': to_text(build_output), 'elapsed_time': int(time.time() - start_time), }) if not docker_image_name: self.update_build_status('FAILED', 'Build or get Docker image failed') context['exit_code'] = -1 self.send_notifications(context) self.cleanup() return exit_code, output = self.run_tests_in_container(docker_image_name) if exit_code == 0: # Success logger.info('Test succeed for repo: %s', self.repo_full_name) self.update_build_status('SUCCESSFUL', '1 of 1 test succeed') else: # Failed logger.info('Test failed for repo: %s, exit code: %s', self.repo_full_name, exit_code) self.update_build_status('FAILED', '1 of 1 test failed') context.update({ 'logs': to_text(output), 'exit_code': exit_code, 'elapsed_time': int(time.time() - start_time), }) self.send_notifications(context) # Code linting if self.context.pr_id and self.spec.linters: lint = LintProcessor(self.context, self.spec, self.clone_path) lint.process() self.cleanup() def clone_repository(self): self.clone_path = os.path.join(tempfile.gettempdir(), 'badwolf', self.task_id, self.repo_name) source_repo = self.context.source['repository']['full_name'] # Use shallow clone to speed up bitbucket.clone(source_repo, self.clone_path, depth=50, branch=self.branch) gitcmd = git.Git(self.clone_path) if self.context.target: # Pull Request target_repo = self.context.target['repository']['full_name'] target_branch = self.context.target['branch']['name'] if source_repo == target_repo: target_remote = 'origin' else: # Pull Reuqest across forks target_remote = target_repo.split('/', 1)[0] gitcmd.remote('add', target_remote, bitbucket.get_git_url(target_repo)) gitcmd.fetch(target_remote, target_branch) gitcmd.checkout('FETCH_HEAD') gitcmd.merge('origin/{}'.format(self.branch)) else: # Push to branch or ci retry comment on some commit logger.info('Checkout commit %s', self.commit_hash) gitcmd.checkout(self.commit_hash) gitmodules = os.path.join(self.clone_path, '.gitmodules') if os.path.exists(gitmodules): gitcmd.submodule('update', '--init', '--recursive') def validate_settings(self): conf_file = os.path.join(self.clone_path, current_app.config['BADWOLF_PROJECT_CONF']) if not os.path.exists(conf_file): logger.warning('No project configuration file found for repo: %s', self.repo_full_name) return False self.spec = spec = Specification.parse_file(conf_file) if self.context.type == 'commit' and spec.branch and self.branch not in spec.branch: logger.info( 'Ignore tests since branch %s test is not enabled. Allowed branches: %s', self.branch, spec.branch) return False if not spec.scripts and not spec.linters: logger.warning('No script(s) or linter(s) to run') return False return True def get_docker_image(self): docker_image_name = self.repo_full_name.replace('/', '-') output = [] with self.lock: docker_image = self.docker.images(docker_image_name) if not docker_image or self.context.rebuild: dockerfile = os.path.join(self.clone_path, self.spec.dockerfile) build_options = { 'tag': docker_image_name, 'rm': True, } if not os.path.exists(dockerfile): logger.warning( 'No Dockerfile: %s found for repo: %s, using simple runner image', dockerfile, self.repo_full_name) dockerfile_content = 'FROM messense/badwolf-test-runner\n' fileobj = io.BytesIO(dockerfile_content.encode('utf-8')) build_options['fileobj'] = fileobj else: build_options['dockerfile'] = self.spec.dockerfile build_success = False logger.info('Building Docker image %s', docker_image_name) self.update_build_status('INPROGRESS', 'Building Docker image') res = self.docker.build(self.clone_path, **build_options) for line in res: if b'Successfully built' in line: build_success = True log = to_text(json.loads(to_text(line))['stream']) output.append(log) logger.info('`docker build` : %s', log.strip()) if not build_success: return None, ''.join(output) return docker_image_name, ''.join(output) def run_tests_in_container(self, docker_image_name): command = '/bin/sh -c badwolf-run' environment = {} if self.spec.environments: # TODO: Support run in multiple environments environment = self.spec.environments[0] # TODO: Add more test context related env vars environment.update({ 'DEBIAN_FRONTEND': 'noninteractive', 'CI': 'true', 'CI_NAME': 'badwolf', 'BADWOLF_BRANCH': self.branch, 'BADWOLF_COMMIT': self.commit_hash, 'BADWOLF_BUILD_DIR': '/mnt/src', 'BADWOLF_REPO_SLUG': self.repo_full_name, }) if self.context.pr_id: environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id) container = self.docker.create_container( docker_image_name, command=command, environment=environment, working_dir='/mnt/src', volumes=['/mnt/src'], host_config=self.docker.create_host_config( privileged=self.spec.privileged, binds={ self.clone_path: { 'bind': '/mnt/src', 'mode': 'rw', }, })) container_id = container['Id'] logger.info('Created container %s from image %s', container_id, docker_image_name) output = [] try: self.docker.start(container_id) self.update_build_status('INPROGRESS', 'Running tests in Docker container') for line in self.docker.logs(container_id, stream=True): output.append(to_text(line)) exit_code = self.docker.wait( container_id, current_app.config['DOCKER_RUN_TIMEOUT']) except (APIError, DockerException, ReadTimeout) as e: exit_code = -1 output.append(to_text(e)) logger.exception('Docker error') finally: try: self.docker.remove_container(container_id, force=True) except (APIError, DockerException): logger.exception('Error removing docker container') return exit_code, ''.join(output) def update_build_status(self, state, description=None): try: self.build_status.update(state, description=description) except BitbucketAPIError: logger.exception('Error calling Bitbucket API') def send_notifications(self, context): exit_code = context['exit_code'] template = 'test_success' if exit_code == 0 else 'test_failure' html = render_template('mail/' + template + '.html', **context) html = sanitize_sensitive_data(html) # Save log html log_dir = os.path.join(current_app.config['BADWOLF_LOG_DIR'], self.commit_hash) if not os.path.exists(log_dir): os.makedirs(log_dir) log_file = os.path.join(log_dir, 'build.html') with open(log_file, 'wb') as f: f.write(to_binary(html)) if exit_code == 0: subject = 'Test succeed for repository {}'.format( self.repo_full_name) else: subject = 'Test failed for repository {}'.format( self.repo_full_name) notification = self.spec.notification emails = notification['emails'] if emails: send_mail(emails, subject, html) slack_webhooks = notification['slack_webhooks'] if slack_webhooks: message = render_template('slack_webhook/' + template + '.md', **context) trigger_slack_webhook(slack_webhooks, message) def cleanup(self): shutil.rmtree(os.path.dirname(self.clone_path), ignore_errors=True)
class Docker_interface: def __init__(self, net_name='tosker_net', tmp_dir='/tmp', socket='unix://var/run/docker.sock'): self._log = Logger.get(__name__) self._net_name = net_name self._cli = Client(base_url=os.environ.get('DOCKER_HOST') or socket) self._tmp_dir = tmp_dir # TODO: aggiungere un parametro per eliminare i container se esistono gia'! def create(self, con, cmd=None, entrypoint=None, saved_image=False): def create_container(): tmp_dir = path.join(self._tmp_dir, con.name) try: os.makedirs(tmp_dir) except: pass saved_img_name = '{}/{}'.format(self._net_name, con.name) img_name = con.image if saved_image and self.inspect(saved_img_name): img_name = saved_img_name self._log.debug('container: {}'.format(con.get_str_obj())) con.id = self._cli.create_container( name=con.name, image=img_name, entrypoint=entrypoint if entrypoint else con.entrypoint, command=cmd if cmd else con.cmd, environment=con.env, detach=True, # stdin_open=True, ports=[key for key in con.ports.keys()] if con.ports else None, volumes=['/tmp/dt'] + ([k for k, v in con.volume.items()] if con.volume else []), networking_config=self._cli.create_networking_config({ self._net_name: self._cli.create_endpoint_config(links=con.link # ,aliases=['db'] ) }), host_config=self._cli.create_host_config( port_bindings=con.ports, # links=con.link, binds=[tmp_dir + ':/tmp/dt'] + ([v + ':' + k for k, v in con.volume.items()] if con.volume else []), )).get('Id') assert isinstance(con, Container) if con.to_build: self._log.debug('start building..') # utility.print_json( self._cli.build(path='/'.join(con.dockerfile.split('/')[0:-1]), dockerfile='./' + con.dockerfile.split('/')[-1], tag=con.image, pull=True, quiet=True) # ) self._log.debug('stop building..') elif not saved_image: # TODO: da evitare se si deve utilizzare un'immagine custom self._log.debug('start pulling.. {}'.format(con.image)) utility.print_json(self._cli.pull(con.image, stream=True), self._log.debug) self._log.debug('end pulling..') try: create_container() except errors.APIError as e: self._log.debug(e) # self.stop(con) self.delete(con) create_container() # raise e def stop(self, container): name = self._get_name(container) try: return self._cli.stop(name) except errors.NotFound as e: self._log.error(e) def start(self, container, wait=False): name = self._get_name(container) self._cli.start(name) if wait: self._log.debug('wait container..') self._cli.wait(name) utility.print_byte(self._cli.logs(name, stream=True), self._log.debug) def delete(self, container): name = self._get_name(container) try: self._cli.remove_container(name, v=True) except (errors.NotFound, errors.APIError) as e: self._log.error(e) raise e def exec_cmd(self, container, cmd): name = self._get_name(container) if not self.is_running(name): return False try: exec_id = self._cli.exec_create(name, cmd) status = self._cli.exec_start(exec_id) # TODO: verificare attendibilita' di questo check! check = 'rpc error:' != status[:10].decode("utf-8") self._log.debug('check: {}'.format(check)) return check except errors.APIError as e: self._log.error(e) return False except requests.exceptions.ConnectionError as e: # TODO: questo errore arriva dopo un timeout di 10 secodi self._log.error(e) return False def create_volume(self, volume): assert isinstance(volume, Volume) self._log.debug('volume opt: {}'.format(volume.get_all_opt())) return self._cli.create_volume(volume.name, volume.driver, volume.get_all_opt()) def delete_volume(self, volume): name = self._get_name(volume) return self._cli.remove_volume(name) def get_containers(self, all=False): return self._cli.containers(all=all) def get_volumes(self): volumes = self._cli.volumes() return volumes['Volumes'] or [] def inspect(self, item): name = self._get_name(item) try: return self._cli.inspect_container(name) except errors.NotFound: pass try: return self._cli.inspect_image(name) except errors.NotFound: pass try: return self._cli.inspect_volume(name) except errors.NotFound: return None def remove_all_containers(self): for c in self.get_containers(all=True): self.stop(c['Id']) self.delete(c['Id']) def remove_all_volumes(self): for v in self.get_volumes(): self.delete_volume(v['Name']) def create_network(self, name, subnet='172.25.0.0/16'): # docker network create -d bridge --subnet 172.25.0.0/16 isolated_nw # self.delete_network(name) try: self._cli.create_network(name=name, driver='bridge', ipam={'subnet': subnet}, check_duplicate=True) except errors.APIError: self._log.debug('network already exists!') def delete_network(self, name): assert isinstance(name, str) try: self._cli.remove_network(name) except errors.APIError: self._log.debug('network not exists!') def delete_image(self, name): assert isinstance(name, str) try: self._cli.remove_image(name) except errors.NotFound: pass # TODO: splittare questo metodo in due, semantica non chiara! def update_container(self, node, cmd, saved_image=True): assert isinstance(node, Container) # self._log.debug('container_conf: {}'.format(node.host_container)) stat = self.inspect(node.image) old_cmd = stat['Config']['Cmd'] or None old_entry = stat['Config']['Entrypoint'] or None if self.inspect(node): self.stop(node) self.delete(node) self.create(node, cmd=cmd, entrypoint='', saved_image=saved_image) self.start(node.id, wait=True) self.stop(node.id) name = '{}/{}'.format(self._net_name, node.name) self._cli.commit(node.id, name) self.stop(node) self.delete(node) self.create(node, cmd=node.cmd or old_cmd, entrypoint=node.entrypoint or old_entry, saved_image=True) self._cli.commit(node.id, name) def is_running(self, container): name = self._get_name(container) stat = self.inspect(name) stat = stat is not None and stat['State']['Running'] is True self._log.debug('State: {}'.format(stat)) return stat def _get_name(self, name): if isinstance(name, six.string_types): return name else: assert isinstance(name, (Container, Volume)) return name.name
def buildDockerfile(self): encDockerfile = BytesIO(self.dockerfile.encode('utf-8')) cli = Client(base_url='tcp://192.168.1.20:2375') response = [line for line in cli.build(fileobj=encDockerfile, rm=True, tag='webpy' )] print response
class Deploy(): def __init__(self): self.registry_username = self.get_env("KS_DOCKER_REGISTRY_USERNAME") self.registry_password = self.get_env("KS_DOCKER_REGISTRY_PASSWORD") self.registry = self.get_env('KS_DOCKER_REGISTRY') self.project_status = self.get_env("KS_PROJECT_STATUS") self.project_name = self.get_env('KS_PROJECT_NAME') self.project_group = self.get_env('KS_PROJECT_GROUP') self.go_pipeline_label = self.get_env("GO_PIPELINE_LABEL") self.project_replicas = int(self.get_env("KS_PROJECT_REPLICAS",1)) self.project_port = int(self.get_env('KS_PROJECT_PORT', 80)) self.docker_server = self.get_env("KS_DOCKER_SERVER") self.k8s_host = self.get_env('K8S_HOST') self.k8s_username = self.get_env('K8S_USERNAME') self.k8s_password = self.get_env('K8S_PASSWORD') self.container_cpu = self.get_env('KS_CONTAINER_CPU', '2') self.container_cpu = self.get_env('KS_CONTAINER_MEMORY', '512M') self.k8s_namespace = self.get_env('KS_PROJECT_GROUP',None) self.go_environment_name = self.get_env('GO_ENVIRONMENT_NAME') self.project_branch = self.get_project_branch() self.project_commit = self.get_project_commit() self.project_version = '{branch}-{label}.git{commit}'.format( branch = self.project_branch, label = self.go_pipeline_label, commit = self.project_commit ) self.image_name = self.project_name self.image_tag = self.project_version self.image_tag_name='{image}:{tag}'.format( image = self.image_name, tag = self.image_tag ) self.k8s_deployment_name = self.project_name + '-' + self.project_status self.k8s_image = self.registry + '/' + self.image_name + ':' + self.image_tag def connect_docker(self): self.dockerClient = Client(base_url=self.docker_server) self.dockerClient.login(username=self.registry_username, password=self.registry_password,registry=self.registry) def get_project_commit(self): commit = commands.getoutput('git log --pretty=format:"%h" -n 1') return commit def get_project_branch(self): branch = commands.getoutput('git branch | grep "*" | head -n 1') branch = re.sub('\*','',branch) branch = re.sub("/",".",branch) branch = branch.strip() log.info(branch) return branch def get_env(self,key,value=None): value = os.getenv(key,value) if value == None: raise Exception('{key} not be null.'.format(key=key)) return value def build(self, dockerfile='.'): self.connect_docker() image_tag = self.image_tag_name status = True log.info('Start To Build Image %s' % image_tag) msgs = '' for msg in self.dockerClient.build(path=dockerfile, tag=image_tag, forcerm=True,buildargs={'KS_PROJECT_NAME':self.project_name},nocache=True): log.info(msg) msgs+=msg if re.search("error", msg, re.IGNORECASE): status = False if not re.search("Successfully", msgs, re.IGNORECASE): status = False else: status = True if status == True: log.info("Build Docker Image[SUCCESS].") else: log.error("Build Docker Image[FAILD].") exit(1) return status def push(self,image_tag_src=None,image_tag_dest=None): self.connect_docker() if self.registry=="" or self.registry==None:return if image_tag_src == None:image_tag_src= self.image_tag_name if image_tag_dest == None:image_tag_dest= self.image_tag_name self.dockerClient.login(self.registry_username,password=self.registry_password,registry=self.registry) self.dockerClient.tag(image_tag_src , repository=self.registry+"/"+self.image_name,tag=self.image_tag) response = self.dockerClient.push(repository=self.registry+"/"+image_tag_dest) if re.search("ERROR",response,re.IGNORECASE): log.error("PUSH IMAGE[%s] TO REPOSITORY FAILD."%image_tag_dest) log.error(response) exit(1) else: log.info(response) def deployment_action(self): cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host) deployment = K8sDeployment(config=cfg_cert, name=self.k8s_deployment_name) deployment.add_label(k='project',v=self.project_name) if not deployment._exists(): container = K8sContainer(name=self.k8s_deployment_name, image=self.k8s_image) container.add_env('KS_PROJECT_VERSION',self.project_version) container.add_image_pull_policy(policy='Always') container.add_volume_mount(K8sVolumeMount(name="{project}-resources-volume".format(project=self.k8s_deployment_name), mount_path="/data/{project}/resources".format( project=self.project_name))) #container.add_volume_mount(K8sVolumeMount(name='localtime-volumn',mount_path='/etc/localtime')) container.add_volume_mount(K8sVolumeMount(name='{project}-log-volume'.format(project=self.k8s_deployment_name), mount_path='/data/logs/')) container.add_container_resources_limits(cpu='2', memory='1024Mi') #container.add_container_resources_requests(cpu='200m',memory='512M') deployment.add_container(container) deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image)) deployment.add_image_pull_secrets([{'name': 'ksszregistrykey'}]) vol = K8sVolume(name="{project}-resources-volume".format(project=self.k8s_deployment_name), type='hostPath') vol.path = "/data/docker/resources/{env}/{project}".format(env=self.project_status,project=self.project_name) deployment.add_volume(vol) #vol_localtime = K8sVolume(name='localtime-volumn', type='hostPath') #vol_localtime.path = "/etc/localtime" #deployment.add_volume(vol_localtime) vol_log = K8sVolume(name="{project}-log-volume".format(project=self.k8s_deployment_name), type='hostPath') vol_log.path = "/data/docker/logs/{env}/{project}".format(env=self.project_status,project=self.project_name) deployment.add_volume(vol_log) deployment.create() else: deployment.change_container_resources_limits(cpu='2', memory='1024Mi') deployment.set_container_image(name=self.k8s_deployment_name, image=self.k8s_image) deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image)) deployment.update() deployment.scale(self.project_replicas) def service_action(self): self.k8s_svc_name = self.project_name+'-'+self.project_status cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host) service = K8sService(config=cfg_cert, name=self.k8s_svc_name) if not service._exists(): service.add_selector(selector={"name": self.k8s_deployment_name}) service.type = 'NodePort' service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP') service.create() else: service.add_selector(selector={"name": self.k8s_deployment_name}) service.type = 'NodePort' if not service._exists_port(self.project_port): service.del_port() service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP') service.update()