def build_image(self, dockerfileobj):
response = []
print(" Docker build output:")
# extend for multiplatform
for line in self.builder.api.build(
fileobj=dockerfileobj,
rm=True,
custom_context=True,
platform=
"[linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x]"
):
response.append(line)
event = list(json_stream([line]))[0]
if 'stream' in event:
print(" " + event['stream'].rstrip())
elif 'status' in event:
print(" " + event['status'].rstrip())
elif 'error' in event:
raise BuildError(event['error'], json_stream(response))
events = list(json_stream(response))
if not events:
raise BuildError('Unknown build error', events)
event = events[-1]
if 'stream' in event:
match = re.search(r'Successfully built ([0-9a-f]+)',
event.get('stream', ''))
if match:
image_id = match.group(1)
if image_id:
return image_id
raise BuildError(event, events)
def main():
client = docker.from_env()
args = parse_args()
head = BytesIO()
body = BytesIO()
body.write("FROM {}\n\n".format(args.base).encode('utf-8'))
if 'core' not in args.components:
args.components.insert(0, 'core')
for component in args.components:
dockerfile_head = Path('components', component, 'Dockerfile.head')
if dockerfile_head.exists():
with dockerfile_head.open() as component_dockerfile:
head.write((dockerfile_head.read()).encode('utf-8'))
dockerfile_body = Path('components', component, 'Dockerfile.body')
if dockerfile_body.exists():
with dockerfile_body.open() as component_dockerfile:
body.write((component_dockerfile.read()).encode('utf-8'))
dockerfile = BytesIO()
dockerfile.write(head.getvalue())
dockerfile.write(body.getvalue())
# we need the " character around the command or else it doesn't work properly
dockerfile.write("CMD [\"/bin/bash\"]\n".encode('utf-8'))
parts = args.tag.split(':')
name_parts = parts[0].split('/')
metadata = {
'hostname': name_parts[0],
'name': name_parts[1],
'tag': parts[1],
'components': args.components
}
build_dir = Path('dockerfiles', metadata['hostname'], metadata['name'],
metadata['tag'])
build_dir.mkdir(parents=True, exist_ok=True)
dockerfile.seek(0)
Path(build_dir, 'Dockerfile').write_bytes(dockerfile.read())
json.dump(metadata, Path(build_dir, 'metadata.json').open('w'), indent=2)
return_status = 0
if args.build:
image_id = None
stream = json_stream(
client.api.build(fileobj=dockerfile,
tag=args.tag,
rm=True,
forcerm=True))
for line in stream:
if 'stream' in line and not args.quiet:
print(line['stream'], end='')
elif 'errorDetail' in line:
print(line['errorDetail']['message'], file=sys.stderr)
if 'code' in line['errorDetail']:
return_status = line['errorDetail']['code']
else:
return_status = -1
elif 'aux' in line:
image_id = line['aux']['ID']
def build_image(docker_client: DockerClient, dockerfile: Path) -> str:
image_name = "pyqt5-stubs"
# Using low-level API so that we can log as it occurs instead of only
# after build has finished/failed
resp = docker_client.api.build(path=str(dockerfile.parent),
rm=True,
tag=image_name)
image_id: str = typing.cast(str, None)
for chunk in json_stream(resp):
if 'error' in chunk:
message = f"Error while building Dockerfile for " \
f"{image_name}:\n{chunk['error']}"
print(message)
raise DockerBuildError(message)
elif 'stream' in chunk:
print(chunk['stream'].rstrip('\n'))
# Taken from the high level API implementation of build
match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream'])
if match:
image_id = match.group(2)
if not image_id:
message = f"Unknown Error while building Dockerfile for " \
f"{image_name}. Build did not return an image ID"
raise DockerBuildError(message)
return image_id
def docker_build(self):
self.build_update.emit("Starting Docker image build.")
try:
resp = self.__client.api.build(path="./core/Docker", dockerfile=self.dockerfile,
tag=self.tag, quiet=False, rm=True)
if isinstance(resp, str):
return self.docker_run(self.__client.images.get(resp))
last_event = None
image_id = None
result_stream, internal_stream = itertools.tee(json_stream(resp))
for chunk in internal_stream:
if 'error' in chunk:
self.build_error.emit(chunk['error'])
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
self.build_progress.emit(chunk['stream'])
match = re.search(
r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream']
)
if match:
image_id = match.group(2)
last_event = chunk
if image_id:
return self.docker_run(image_id)
raise BuildError(last_event or 'Unknown', result_stream)
except Exception as e:
self.build_error.emit("An exception occurred while starting Docker image building process: {}".format(e))
self.build_finished.emit(1)
return
def build_image(self, **kwargs):
resp = self.client.api.build(path=self.config.context,
dockerfile=os.path.basename(
self.config.dockerfile.name),
tag=self.config.tag,
buildargs=self.config.args,
rm=True)
if isinstance(resp, str):
return self.client.images.get(resp)
events = []
for event in json_stream(resp):
# TODO: Redirect image pull logs
line = event.get('stream', '')
self.redirect_output(line)
events.append(event)
if not events:
raise BuildError('Unknown')
event = events[-1]
if 'stream' in event:
match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)',
event.get('stream', ''))
if match:
image_id = match.group(2)
return self.client.images.get(image_id)
raise BuildError(event.get('error') or event)
def build_image(docker_client: DockerClient) -> Image:
resp = docker_client.api.build(path=".", rm=True, tag='pyfixm')
# noinspection PyTypeChecker
image_id: str = None
for chunk in json_stream(resp):
if 'error' in chunk:
message = f"Error while building Dockerfile for pyfixm:\n" \
f"{chunk['error']}"
logger.error(message)
raise DockerBuildError(message)
elif 'stream' in chunk:
logger.info(chunk['stream'].rstrip('\n'))
# Taken from the high level API implementation of build
match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream'])
if match:
image_id = match.group(2)
if not image_id:
message = f"Unknown Error while building Dockerfile for pyfixm. " \
f"Build did not return an image ID."
raise DockerBuildError(message)
image: Image = docker_client.images.get(image_id)
return image
def push_image(self, repository, tag=None):
print(" Docker push output:")
for line in self.builder.api.push(repository=repository,
tag=tag,
stream=True):
event = list(json_stream([line]))[0]
if 'status' in event:
print(" " + event['status'])
elif 'error' in event:
raise DockerException(event['error'])
def test_with_leading_whitespace(self):
stream = [
'\n \r\n {"one": "two"}{"x": 1}',
' {"three": "four"}\t\t{"x": 2}'
]
output = list(json_stream(stream))
assert output == [
{'one': 'two'},
{'x': 1},
{'three': 'four'},
{'x': 2}
]
def test_with_falsy_entries(self):
stream = [
'{"one": "two"}\n{}\n',
"[1, 2, 3]\n[]\n",
]
output = list(json_stream(stream))
assert output == [
{'one': 'two'},
{},
[1, 2, 3],
[],
]
def test_with_falsy_entries(self):
stream = [
'{"one": "two"}\n{}\n',
"[1, 2, 3]\n[]\n",
]
output = list(json_stream(stream))
assert output == [
{
'one': 'two'
},
{},
[1, 2, 3],
[],
]
def test_with_leading_whitespace(self):
stream = [
'\n \r\n {"one": "two"}{"x": 1}',
' {"three": "four"}\t\t{"x": 2}'
]
output = list(json_stream(stream))
assert output == [{
'one': 'two'
}, {
'x': 1
}, {
'three': 'four'
}, {
'x': 2
}]
def build(image_name, version, workspace, stream_handler=None):
tag = ":".join([image_name, version])
log_result = ''
img = None
try:
resp = client.images.client.api.build(path=workspace, tag=tag)
if isinstance(resp, six.string_types):
return client.images.get(resp)
last_event = None
image_id = None
result_stream, internal_stream = itertools.tee(json_stream(resp))
for chunk in internal_stream:
if stream_handler:
stream_handler(chunk)
if 'error' in chunk:
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
match = re.search(
r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream']
)
if match:
image_id = match.group(2)
last_event = chunk
if image_id:
img = client.images.get(image_id)
else:
raise BuildError(last_event or 'Unknown', result_stream)
except BuildError as e:
log_result = ''.join([chunk.get('stream') or chunk.get('error') for chunk in e.build_log if
'stream' in chunk or 'error' in chunk])
logger.error("BuildError while building {}@{} :\n{}".format(image_name, version, log_result))
except APIError as e:
log_result = str(e)
logger.error("APIError while building {}@{} :\n{}".format(image_name, version, log_result))
except ConnectionError as e:
log_result = str(e)
logger.error("ConnectionError while building {}@{} :\n{}".format(image_name, version, log_result))
except Exception as e:
log_result = str(e)
logger.error("Exception while building {}@{} :\n{}".format(image_name, version, log_result))
return img, log_result
def _build(client, **kwargs):
resp = client.api.build(**kwargs)
if isinstance(resp, six.string_types):
return client.images.get(resp), []
last_event = None
image_id = None
output = []
for chunk in json_stream(resp):
if 'error' in chunk:
msg = chunk['error'] + '\n' + ''.join(output)
raise docker.errors.BuildError(msg, chunk)
if 'stream' in chunk:
output.append(chunk['stream'])
match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream'])
if match:
image_id = match.group(2)
last_event = chunk
if image_id:
return client.images.get(image_id), output
raise docker.errors.BuildError(last_event or 'Unknown', '\n'.join(output))
def _process_stream(self, stream):
# TODO Safe dictionary key access
result_stream, internal_stream = itertools.tee(json_stream(stream))
image_ids = set()
progress_bars = {}
for chunk in internal_stream:
_logger.debug(chunk)
if 'error' in chunk:
raise DockerException(chunk['error'], result_stream)
elif 'status' in chunk:
status = chunk['status']
id = chunk.get('id', None)
if status in ['Downloading', 'Extracting', 'Pushing']:
current = chunk['progressDetail']['current']
total = chunk['progressDetail']['total']
if id in progress_bars:
bar, prev_current, total = progress_bars[id]
update = current - prev_current
bar.update(1 if total < current else update)
bar.set_description_str('{0}: {1}'.format(id, status))
else:
bar = tqdm(desc='{0}: {1}'.format(id, status),
total=total,
unit='B',
unit_scale=True,
unit_divisor=1024,
position=len(progress_bars))
bar.update(current)
progress_bars.update({id: (bar, current, total)})
elif status in ['Verifying Checksum']:
bar, prev_current, total = progress_bars[id]
update = bar.total - prev_current
bar.update(1 if update < 0 else update)
bar.set_description_str('{0}: {1}'.format(id, status))
elif status in ['Download complete']:
bar, prev_current, total = progress_bars[id]
bar.clear() # TODO Progress bar can not be reset
progress_bars.update({id: (bar, 0, total)})
bar.set_description_str('{0}: {1}'.format(id, status))
elif status in ['Pull complete', 'Pushed']:
bar, prev_current, total = progress_bars[id]
update = bar.total - prev_current
bar.update(1 if update < 0 else update)
bar.set_description_str('{0}: {1}'.format(id, status))
else:
if 'id' in chunk:
click.echo('{0}: {1}'.format(id, status))
else:
click.echo(status)
elif 'aux' in chunk:
aux = chunk['aux']
if 'Digest' in aux:
image_ids.add(aux['Digest'])
elif 'ID' in aux:
image_ids.add(aux['ID'])
elif 'stream' in chunk:
click.echo(chunk['stream'].strip('\n'))
else:
click.echo(chunk)
for bar, _, _ in progress_bars.values():
bar.close()
return tuple(image_ids)
def dbuild(path: str,
image_tag: str,
build_args=None,
docker_client=docker.from_env(),
verbose=True,
nocache=False):
"""
Adopted from https://github.com/docker/docker-py/blob/master/docker/models/images.py
"""
try:
logger.info(f"Begin build for {pjoin(path, 'Dockerfile')}")
logger.info(
f"dbuild(path={path}, image_tag={image_tag}, build_args={build_args}, "
f"verbose={True}, nocache={nocache})")
resp = docker_client.api.build(
path=path,
dockerfile='Dockerfile',
tag=image_tag,
buildargs=build_args,
nocache=nocache,
)
meta = []
result_stream, internal_stream = itertools.tee(json_stream(resp))
last_event = None
image_id = None
last_layer_cmd = None
last_layer_start_time = None
last_layer_finish_time = None
last_layer_id = None
last_layer_logs = []
ptrn_step = r'^Step (\d+\/\d+) : (.+)'
ptrn_layer = r'^ ---> ([a-z0-9]+)'
ptrn_success = r'(^Successfully built |sha256:)([0-9a-f]+)$'
for chunk in internal_stream:
logger.info(chunk)
if 'error' in chunk:
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
if verbose:
print(chunk['stream'], end='')
raw_output = chunk['stream']
if (match_step := re.search(ptrn_step,
raw_output)) is not None:
if last_layer_cmd:
meta.append(
LayerMeta(
CMD=last_layer_cmd,
START_T=last_layer_start_time,
FINISH_T=last_layer_finish_time,
LOGS=last_layer_logs,
ID=last_layer_id,
)._asdict())
last_layer_cmd = None
last_layer_start_time = None
last_layer_finish_time = None
last_layer_id = None
last_layer_logs = []
last_layer_cmd = match_step.group(2)
last_layer_start_time = time.time()
elif (match_layer := re.search(ptrn_layer,
raw_output)) is not None:
last_layer_id = match_layer.group(1)
last_layer_finish_time = time.time()
elif (match_success := re.search(ptrn_success,
raw_output)) is not None:
meta.append(
LayerMeta(
CMD=last_layer_cmd,
START_T=last_layer_start_time,
FINISH_T=last_layer_finish_time,
LOGS=last_layer_logs,
ID=last_layer_id,
)._asdict())
image_id = match_success.group(2)
else:
def run(self):
from docker.utils.json_stream import json_stream
for line in json_stream(self.generator):
self.logger.debug(line)
self.logs.append(line)