def build_image(self, dockerfileobj): response = [] print(" Docker build output:") # extend for multiplatform for line in self.builder.api.build( fileobj=dockerfileobj, rm=True, custom_context=True, platform= "[linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x]" ): response.append(line) event = list(json_stream([line]))[0] if 'stream' in event: print(" " + event['stream'].rstrip()) elif 'status' in event: print(" " + event['status'].rstrip()) elif 'error' in event: raise BuildError(event['error'], json_stream(response)) events = list(json_stream(response)) if not events: raise BuildError('Unknown build error', events) event = events[-1] if 'stream' in event: match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', '')) if match: image_id = match.group(1) if image_id: return image_id raise BuildError(event, events)
def main(): client = docker.from_env() args = parse_args() head = BytesIO() body = BytesIO() body.write("FROM {}\n\n".format(args.base).encode('utf-8')) if 'core' not in args.components: args.components.insert(0, 'core') for component in args.components: dockerfile_head = Path('components', component, 'Dockerfile.head') if dockerfile_head.exists(): with dockerfile_head.open() as component_dockerfile: head.write((dockerfile_head.read()).encode('utf-8')) dockerfile_body = Path('components', component, 'Dockerfile.body') if dockerfile_body.exists(): with dockerfile_body.open() as component_dockerfile: body.write((component_dockerfile.read()).encode('utf-8')) dockerfile = BytesIO() dockerfile.write(head.getvalue()) dockerfile.write(body.getvalue()) # we need the " character around the command or else it doesn't work properly dockerfile.write("CMD [\"/bin/bash\"]\n".encode('utf-8')) parts = args.tag.split(':') name_parts = parts[0].split('/') metadata = { 'hostname': name_parts[0], 'name': name_parts[1], 'tag': parts[1], 'components': args.components } build_dir = Path('dockerfiles', metadata['hostname'], metadata['name'], metadata['tag']) build_dir.mkdir(parents=True, exist_ok=True) dockerfile.seek(0) Path(build_dir, 'Dockerfile').write_bytes(dockerfile.read()) json.dump(metadata, Path(build_dir, 'metadata.json').open('w'), indent=2) return_status = 0 if args.build: image_id = None stream = json_stream( client.api.build(fileobj=dockerfile, tag=args.tag, rm=True, forcerm=True)) for line in stream: if 'stream' in line and not args.quiet: print(line['stream'], end='') elif 'errorDetail' in line: print(line['errorDetail']['message'], file=sys.stderr) if 'code' in line['errorDetail']: return_status = line['errorDetail']['code'] else: return_status = -1 elif 'aux' in line: image_id = line['aux']['ID']
def build_image(docker_client: DockerClient, dockerfile: Path) -> str: image_name = "pyqt5-stubs" # Using low-level API so that we can log as it occurs instead of only # after build has finished/failed resp = docker_client.api.build(path=str(dockerfile.parent), rm=True, tag=image_name) image_id: str = typing.cast(str, None) for chunk in json_stream(resp): if 'error' in chunk: message = f"Error while building Dockerfile for " \ f"{image_name}:\n{chunk['error']}" print(message) raise DockerBuildError(message) elif 'stream' in chunk: print(chunk['stream'].rstrip('\n')) # Taken from the high level API implementation of build match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream']) if match: image_id = match.group(2) if not image_id: message = f"Unknown Error while building Dockerfile for " \ f"{image_name}. Build did not return an image ID" raise DockerBuildError(message) return image_id
def docker_build(self): self.build_update.emit("Starting Docker image build.") try: resp = self.__client.api.build(path="./core/Docker", dockerfile=self.dockerfile, tag=self.tag, quiet=False, rm=True) if isinstance(resp, str): return self.docker_run(self.__client.images.get(resp)) last_event = None image_id = None result_stream, internal_stream = itertools.tee(json_stream(resp)) for chunk in internal_stream: if 'error' in chunk: self.build_error.emit(chunk['error']) raise BuildError(chunk['error'], result_stream) if 'stream' in chunk: self.build_progress.emit(chunk['stream']) match = re.search( r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream'] ) if match: image_id = match.group(2) last_event = chunk if image_id: return self.docker_run(image_id) raise BuildError(last_event or 'Unknown', result_stream) except Exception as e: self.build_error.emit("An exception occurred while starting Docker image building process: {}".format(e)) self.build_finished.emit(1) return
def build_image(self, **kwargs): resp = self.client.api.build(path=self.config.context, dockerfile=os.path.basename( self.config.dockerfile.name), tag=self.config.tag, buildargs=self.config.args, rm=True) if isinstance(resp, str): return self.client.images.get(resp) events = [] for event in json_stream(resp): # TODO: Redirect image pull logs line = event.get('stream', '') self.redirect_output(line) events.append(event) if not events: raise BuildError('Unknown') event = events[-1] if 'stream' in event: match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', event.get('stream', '')) if match: image_id = match.group(2) return self.client.images.get(image_id) raise BuildError(event.get('error') or event)
def build_image(docker_client: DockerClient) -> Image: resp = docker_client.api.build(path=".", rm=True, tag='pyfixm') # noinspection PyTypeChecker image_id: str = None for chunk in json_stream(resp): if 'error' in chunk: message = f"Error while building Dockerfile for pyfixm:\n" \ f"{chunk['error']}" logger.error(message) raise DockerBuildError(message) elif 'stream' in chunk: logger.info(chunk['stream'].rstrip('\n')) # Taken from the high level API implementation of build match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream']) if match: image_id = match.group(2) if not image_id: message = f"Unknown Error while building Dockerfile for pyfixm. " \ f"Build did not return an image ID." raise DockerBuildError(message) image: Image = docker_client.images.get(image_id) return image
def push_image(self, repository, tag=None): print(" Docker push output:") for line in self.builder.api.push(repository=repository, tag=tag, stream=True): event = list(json_stream([line]))[0] if 'status' in event: print(" " + event['status']) elif 'error' in event: raise DockerException(event['error'])
def test_with_leading_whitespace(self): stream = [ '\n \r\n {"one": "two"}{"x": 1}', ' {"three": "four"}\t\t{"x": 2}' ] output = list(json_stream(stream)) assert output == [ {'one': 'two'}, {'x': 1}, {'three': 'four'}, {'x': 2} ]
def test_with_falsy_entries(self): stream = [ '{"one": "two"}\n{}\n', "[1, 2, 3]\n[]\n", ] output = list(json_stream(stream)) assert output == [ {'one': 'two'}, {}, [1, 2, 3], [], ]
def test_with_falsy_entries(self): stream = [ '{"one": "two"}\n{}\n', "[1, 2, 3]\n[]\n", ] output = list(json_stream(stream)) assert output == [ { 'one': 'two' }, {}, [1, 2, 3], [], ]
def test_with_leading_whitespace(self): stream = [ '\n \r\n {"one": "two"}{"x": 1}', ' {"three": "four"}\t\t{"x": 2}' ] output = list(json_stream(stream)) assert output == [{ 'one': 'two' }, { 'x': 1 }, { 'three': 'four' }, { 'x': 2 }]
def build(image_name, version, workspace, stream_handler=None): tag = ":".join([image_name, version]) log_result = '' img = None try: resp = client.images.client.api.build(path=workspace, tag=tag) if isinstance(resp, six.string_types): return client.images.get(resp) last_event = None image_id = None result_stream, internal_stream = itertools.tee(json_stream(resp)) for chunk in internal_stream: if stream_handler: stream_handler(chunk) if 'error' in chunk: raise BuildError(chunk['error'], result_stream) if 'stream' in chunk: match = re.search( r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream'] ) if match: image_id = match.group(2) last_event = chunk if image_id: img = client.images.get(image_id) else: raise BuildError(last_event or 'Unknown', result_stream) except BuildError as e: log_result = ''.join([chunk.get('stream') or chunk.get('error') for chunk in e.build_log if 'stream' in chunk or 'error' in chunk]) logger.error("BuildError while building {}@{} :\n{}".format(image_name, version, log_result)) except APIError as e: log_result = str(e) logger.error("APIError while building {}@{} :\n{}".format(image_name, version, log_result)) except ConnectionError as e: log_result = str(e) logger.error("ConnectionError while building {}@{} :\n{}".format(image_name, version, log_result)) except Exception as e: log_result = str(e) logger.error("Exception while building {}@{} :\n{}".format(image_name, version, log_result)) return img, log_result
def _build(client, **kwargs): resp = client.api.build(**kwargs) if isinstance(resp, six.string_types): return client.images.get(resp), [] last_event = None image_id = None output = [] for chunk in json_stream(resp): if 'error' in chunk: msg = chunk['error'] + '\n' + ''.join(output) raise docker.errors.BuildError(msg, chunk) if 'stream' in chunk: output.append(chunk['stream']) match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream']) if match: image_id = match.group(2) last_event = chunk if image_id: return client.images.get(image_id), output raise docker.errors.BuildError(last_event or 'Unknown', '\n'.join(output))
def _process_stream(self, stream): # TODO Safe dictionary key access result_stream, internal_stream = itertools.tee(json_stream(stream)) image_ids = set() progress_bars = {} for chunk in internal_stream: _logger.debug(chunk) if 'error' in chunk: raise DockerException(chunk['error'], result_stream) elif 'status' in chunk: status = chunk['status'] id = chunk.get('id', None) if status in ['Downloading', 'Extracting', 'Pushing']: current = chunk['progressDetail']['current'] total = chunk['progressDetail']['total'] if id in progress_bars: bar, prev_current, total = progress_bars[id] update = current - prev_current bar.update(1 if total < current else update) bar.set_description_str('{0}: {1}'.format(id, status)) else: bar = tqdm(desc='{0}: {1}'.format(id, status), total=total, unit='B', unit_scale=True, unit_divisor=1024, position=len(progress_bars)) bar.update(current) progress_bars.update({id: (bar, current, total)}) elif status in ['Verifying Checksum']: bar, prev_current, total = progress_bars[id] update = bar.total - prev_current bar.update(1 if update < 0 else update) bar.set_description_str('{0}: {1}'.format(id, status)) elif status in ['Download complete']: bar, prev_current, total = progress_bars[id] bar.clear() # TODO Progress bar can not be reset progress_bars.update({id: (bar, 0, total)}) bar.set_description_str('{0}: {1}'.format(id, status)) elif status in ['Pull complete', 'Pushed']: bar, prev_current, total = progress_bars[id] update = bar.total - prev_current bar.update(1 if update < 0 else update) bar.set_description_str('{0}: {1}'.format(id, status)) else: if 'id' in chunk: click.echo('{0}: {1}'.format(id, status)) else: click.echo(status) elif 'aux' in chunk: aux = chunk['aux'] if 'Digest' in aux: image_ids.add(aux['Digest']) elif 'ID' in aux: image_ids.add(aux['ID']) elif 'stream' in chunk: click.echo(chunk['stream'].strip('\n')) else: click.echo(chunk) for bar, _, _ in progress_bars.values(): bar.close() return tuple(image_ids)
def dbuild(path: str, image_tag: str, build_args=None, docker_client=docker.from_env(), verbose=True, nocache=False): """ Adopted from https://github.com/docker/docker-py/blob/master/docker/models/images.py """ try: logger.info(f"Begin build for {pjoin(path, 'Dockerfile')}") logger.info( f"dbuild(path={path}, image_tag={image_tag}, build_args={build_args}, " f"verbose={True}, nocache={nocache})") resp = docker_client.api.build( path=path, dockerfile='Dockerfile', tag=image_tag, buildargs=build_args, nocache=nocache, ) meta = [] result_stream, internal_stream = itertools.tee(json_stream(resp)) last_event = None image_id = None last_layer_cmd = None last_layer_start_time = None last_layer_finish_time = None last_layer_id = None last_layer_logs = [] ptrn_step = r'^Step (\d+\/\d+) : (.+)' ptrn_layer = r'^ ---> ([a-z0-9]+)' ptrn_success = r'(^Successfully built |sha256:)([0-9a-f]+)$' for chunk in internal_stream: logger.info(chunk) if 'error' in chunk: raise BuildError(chunk['error'], result_stream) if 'stream' in chunk: if verbose: print(chunk['stream'], end='') raw_output = chunk['stream'] if (match_step := re.search(ptrn_step, raw_output)) is not None: if last_layer_cmd: meta.append( LayerMeta( CMD=last_layer_cmd, START_T=last_layer_start_time, FINISH_T=last_layer_finish_time, LOGS=last_layer_logs, ID=last_layer_id, )._asdict()) last_layer_cmd = None last_layer_start_time = None last_layer_finish_time = None last_layer_id = None last_layer_logs = [] last_layer_cmd = match_step.group(2) last_layer_start_time = time.time() elif (match_layer := re.search(ptrn_layer, raw_output)) is not None: last_layer_id = match_layer.group(1) last_layer_finish_time = time.time() elif (match_success := re.search(ptrn_success, raw_output)) is not None: meta.append( LayerMeta( CMD=last_layer_cmd, START_T=last_layer_start_time, FINISH_T=last_layer_finish_time, LOGS=last_layer_logs, ID=last_layer_id, )._asdict()) image_id = match_success.group(2) else:
def run(self): from docker.utils.json_stream import json_stream for line in json_stream(self.generator): self.logger.debug(line) self.logs.append(line)