def docker_build(self):
self.build_update.emit("Starting Docker image build.")
try:
resp = self.__client.api.build(path="./core/Docker", dockerfile=self.dockerfile,
tag=self.tag, quiet=False, rm=True)
if isinstance(resp, str):
return self.docker_run(self.__client.images.get(resp))
last_event = None
image_id = None
result_stream, internal_stream = itertools.tee(json_stream(resp))
for chunk in internal_stream:
if 'error' in chunk:
self.build_error.emit(chunk['error'])
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
self.build_progress.emit(chunk['stream'])
match = re.search(
r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream']
)
if match:
image_id = match.group(2)
last_event = chunk
if image_id:
return self.docker_run(image_id)
raise BuildError(last_event or 'Unknown', result_stream)
except Exception as e:
self.build_error.emit("An exception occurred while starting Docker image building process: {}".format(e))
self.build_finished.emit(1)
return
def build_image(self, dockerfileobj):
response = []
print(" Docker build output:")
# extend for multiplatform
for line in self.builder.api.build(
fileobj=dockerfileobj,
rm=True,
custom_context=True,
platform=
"[linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x]"
):
response.append(line)
event = list(json_stream([line]))[0]
if 'stream' in event:
print(" " + event['stream'].rstrip())
elif 'status' in event:
print(" " + event['status'].rstrip())
elif 'error' in event:
raise BuildError(event['error'], json_stream(response))
events = list(json_stream(response))
if not events:
raise BuildError('Unknown build error', events)
event = events[-1]
if 'stream' in event:
match = re.search(r'Successfully built ([0-9a-f]+)',
event.get('stream', ''))
if match:
image_id = match.group(1)
if image_id:
return image_id
raise BuildError(event, events)
def build_image(self, **kwargs):
resp = self.client.api.build(path=self.config.context,
dockerfile=os.path.basename(
self.config.dockerfile.name),
tag=self.config.tag,
buildargs=self.config.args,
rm=True)
if isinstance(resp, str):
return self.client.images.get(resp)
events = []
for event in json_stream(resp):
# TODO: Redirect image pull logs
line = event.get('stream', '')
self.redirect_output(line)
events.append(event)
if not events:
raise BuildError('Unknown')
event = events[-1]
if 'stream' in event:
match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)',
event.get('stream', ''))
if match:
image_id = match.group(2)
return self.client.images.get(image_id)
raise BuildError(event.get('error') or event)
def _prepare_log_lines(self, log_line):
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
raise BuildError(str(json_line.get('error', json_line)))
else:
if json_line.get('stream'):
log_lines.append('Build: {}'.format(
json_line['stream'].strip()))
elif json_line.get('status'):
log_lines.append('Push: {} {}'.format(
json_line['status'], json_line.get('progress')))
elif json_line.get('aux'):
log_lines.append('Push finished: {}'.format(
json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines
def build(image_name, version, workspace, stream_handler=None):
tag = ":".join([image_name, version])
log_result = ''
img = None
try:
resp = client.images.client.api.build(path=workspace, tag=tag)
if isinstance(resp, six.string_types):
return client.images.get(resp)
last_event = None
image_id = None
result_stream, internal_stream = itertools.tee(json_stream(resp))
for chunk in internal_stream:
if stream_handler:
stream_handler(chunk)
if 'error' in chunk:
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
match = re.search(
r'(^Successfully built |sha256:)([0-9a-f]+)$',
chunk['stream']
)
if match:
image_id = match.group(2)
last_event = chunk
if image_id:
img = client.images.get(image_id)
else:
raise BuildError(last_event or 'Unknown', result_stream)
except BuildError as e:
log_result = ''.join([chunk.get('stream') or chunk.get('error') for chunk in e.build_log if
'stream' in chunk or 'error' in chunk])
logger.error("BuildError while building {}@{} :\n{}".format(image_name, version, log_result))
except APIError as e:
log_result = str(e)
logger.error("APIError while building {}@{} :\n{}".format(image_name, version, log_result))
except ConnectionError as e:
log_result = str(e)
logger.error("ConnectionError while building {}@{} :\n{}".format(image_name, version, log_result))
except Exception as e:
log_result = str(e)
logger.error("Exception while building {}@{} :\n{}".format(image_name, version, log_result))
return img, log_result
def build_logs(self, resp: t.Iterator, image_tag: str) -> None:
"""
Stream build logs to stderr.
Args:
resp (:obj:`t.Iterator`):
blocking generator from docker.api.build
image_tag (:obj:`str`):
given model server tags.
Ex: bento-server:0.13.0-python3.8-debian-runtime
Raises:
docker.errors.BuildErrors:
When errors occurs during build process. Usually
this comes when generated Dockerfile are incorrect.
"""
last_event: str = ""
image_id: str = ""
output: str = ""
logs: t.List = []
built_regex = re.compile(r"(^Successfully built |sha256:)([0-9a-f]+)$")
try:
while True:
try:
# output logs to stdout
# https://docker-py.readthedocs.io/en/stable/user_guides/multiplex.html
output = next(resp).decode("utf-8")
json_output: t.Dict = json.loads(output.strip("\r\n"))
# output to stderr when running in docker
if "stream" in json_output:
sprint(json_output["stream"])
matched = built_regex.search(json_output["stream"])
if matched:
image_id = matched.group(2)
last_event = json_output["stream"]
logs.append(json_output)
except StopIteration:
log.info(f"Successfully built {image_tag}.")
break
except ValueError:
log.error(f"Errors while building image:\n{output}")
if image_id:
self._push_context[image_tag] = docker_client.images.get(
image_id)
else:
raise BuildError(last_event or "Unknown", logs)
except BuildError as e:
log.error(f"Failed to build {image_tag} :\n{e.msg}")
for line in e.build_log:
if "stream" in line:
sprint(line["stream"].strip())
log.fatal("ABORTING due to failure!")
def _get_image(build_logger_obj):
"""Helper to check for build errors and return image. This method is
in the DockerImage class so that errors are raised in the main thread.
This method borrows from the higher-level API of docker-py.
See https://github.com/docker/docker-py/pull/1581.
"""
import re
from docker.errors import BuildError
if isinstance(build_logger_obj.generator, str):
return client.images.get(build_logger_obj.generator)
if not build_logger_obj.logs:
return BuildError('Unknown')
for event in build_logger_obj.logs:
if 'stream' in event:
match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)',
event.get('stream', ''))
if match:
image_id = match.group(2)
return client.images.get(image_id)
last_event = build_logger_obj.logs[-1]
raise BuildError(last_event.get('error') or last_event)
def test_build_image_fails_with_BuildError(self, generate_dockerfile_patch,
path_patch, uuid_patch,
create_tarball_patch):
uuid_patch.uuid4.return_value = "uuid"
generate_dockerfile_patch.return_value = "Dockerfile content"
docker_full_path_mock = Mock()
docker_full_path_mock.exists.return_value = False
path_patch.return_value = docker_full_path_mock
docker_client_mock = Mock()
docker_client_mock.api.build.side_effect = BuildError(
"buildError", "buildlog")
layer_downloader_mock = Mock()
layer_downloader_mock.layer_cache = "cached layers"
tarball_fileobj = Mock()
create_tarball_patch.return_value.__enter__.return_value = tarball_fileobj
layer_version1 = Mock()
layer_version1.codeuri = "somevalue"
layer_version1.name = "name"
dockerfile_mock = Mock()
m = mock_open(dockerfile_mock)
with patch("samcli.local.docker.lambda_image.open", m):
with self.assertRaises(ImageBuildException):
LambdaImage(layer_downloader_mock,
True,
False,
docker_client=docker_client_mock)._build_image(
"base_image", "docker_tag", [layer_version1],
True)
handle = m()
handle.write.assert_called_with("Dockerfile content")
path_patch.assert_called_once_with("cached layers", "dockerfile_uuid")
docker_client_mock.api.build.assert_called_once_with(
fileobj=tarball_fileobj,
rm=True,
tag="docker_tag",
pull=False,
custom_context=True)
docker_full_path_mock.unlink.assert_not_called()
def dbuild(path: str,
image_tag: str,
build_args=None,
docker_client=docker.from_env(),
verbose=True,
nocache=False):
"""
Adopted from https://github.com/docker/docker-py/blob/master/docker/models/images.py
"""
try:
logger.info(f"Begin build for {pjoin(path, 'Dockerfile')}")
logger.info(
f"dbuild(path={path}, image_tag={image_tag}, build_args={build_args}, "
f"verbose={True}, nocache={nocache})")
resp = docker_client.api.build(
path=path,
dockerfile='Dockerfile',
tag=image_tag,
buildargs=build_args,
nocache=nocache,
)
meta = []
result_stream, internal_stream = itertools.tee(json_stream(resp))
last_event = None
image_id = None
last_layer_cmd = None
last_layer_start_time = None
last_layer_finish_time = None
last_layer_id = None
last_layer_logs = []
ptrn_step = r'^Step (\d+\/\d+) : (.+)'
ptrn_layer = r'^ ---> ([a-z0-9]+)'
ptrn_success = r'(^Successfully built |sha256:)([0-9a-f]+)$'
for chunk in internal_stream:
logger.info(chunk)
if 'error' in chunk:
raise BuildError(chunk['error'], result_stream)
if 'stream' in chunk:
if verbose:
print(chunk['stream'], end='')
raw_output = chunk['stream']
if (match_step := re.search(ptrn_step,
raw_output)) is not None:
if last_layer_cmd:
meta.append(
LayerMeta(
CMD=last_layer_cmd,
START_T=last_layer_start_time,
FINISH_T=last_layer_finish_time,
LOGS=last_layer_logs,
ID=last_layer_id,
)._asdict())
last_layer_cmd = None
last_layer_start_time = None
last_layer_finish_time = None
last_layer_id = None
last_layer_logs = []
last_layer_cmd = match_step.group(2)
last_layer_start_time = time.time()
elif (match_layer := re.search(ptrn_layer,
raw_output)) is not None:
last_layer_id = match_layer.group(1)
last_layer_finish_time = time.time()
elif (match_success := re.search(ptrn_success,
raw_output)) is not None:
meta.append(
LayerMeta(
CMD=last_layer_cmd,
START_T=last_layer_start_time,
FINISH_T=last_layer_finish_time,
LOGS=last_layer_logs,
ID=last_layer_id,
)._asdict())
image_id = match_success.group(2)
else:
FINISH_T=last_layer_finish_time,
LOGS=last_layer_logs,
ID=last_layer_id,
)._asdict())
image_id = match_success.group(2)
else:
last_layer_logs.append(raw_output)
last_event = chunk
if image_id:
image = docker_client.images.get(image_id)
logger.info(f"Successfully finish build for {image}")
return image, meta
raise BuildError(last_event or 'Unknown', result_stream)
except BuildError as err:
print('!!! Build failed !!!')
# for l in err.build_log:
# if 'stream' in l:
# print(' ', l['stream'], end='')
logger.error(f"Build failed when building {image_tag}: {err.msg}")
raise err
except KeyboardInterrupt:
logger.error('Interrupted')
raise KeyboardInterrupt(f'{image_tag}')
class DockerStackBuilder:
def build_image(docker_client: DockerClient,
image_name: str,
remove_image: bool = True,
file: Optional[TextIO] = sys.stderr,
spinner: bool = True,
**kwargs):
"""
Create a docker image (similar to docker build command)
At the end, deletes the image (using rmi command)
Args:
docker_client: DockerClient to be used to create the image
image_name: Name of the image to be created
remove_image: boolean, whether or not to delete the image at the end, default as True
file: a file-like object (stream); defaults to the current sys.stderr. if set to None, will disable printing
spinner: boolean, whether or not to use spinner (default as True), note that this param is set to False in
case `file` param is not None
"""
if file is None:
file = open(os.devnull, 'w')
else:
spinner = False # spinner splits into multiple lines in case stream is being printed at the same time
image_tag = f'{image_name}:test'
yaspin_spinner = _get_spinner(spinner)
with yaspin_spinner(f'Creating image {image_tag}...'):
kwargs = {'tag': image_tag, 'rm': True, 'forcerm': True, **kwargs}
build_log = docker_client.api.build(**kwargs)
for msg_b in build_log:
msgs = str(msg_b, 'utf-8').splitlines()
for msg in msgs:
try:
parse_msg = json.loads(msg)
except JSONDecodeError:
raise DockerException('error at build logs')
s = parse_msg.get('stream')
if s:
print(s, end='', flush=True, file=file)
else:
# runtime errors
error_detail = parse_msg.get('errorDetail')
# parse errors
error_msg = parse_msg.get('message')
# steps of the image creation
status = parse_msg.get('status')
# end of process, will contain the ID of the temporary container created at the end
aux = parse_msg.get('aux')
if error_detail is not None:
raise BuildError(reason=error_detail, build_log=None)
elif error_msg is not None:
raise DockerfileParseException(reason=error_msg,
build_log=None)
elif status is not None:
print(status, end='', flush=True, file=file)
elif aux is not None:
print(aux, end='', flush=True, file=file)
else:
raise DockerException(parse_msg)
yield image_tag
if remove_image:
try:
docker_client.api.remove_image(image_tag)
except ImageNotFound:
# if the image was already deleted
pass
def execute_plan(plan):
plan.status.blocking = False
module_path = os.path.join(plan.arguments['base_path'], plan.module)
images = [tag.full_interp for tag in plan.arguments['tags']]
first_image = images.pop(0)
plan.status.description = 'build %s' % first_image
client = docker.from_env(version='auto')
logger.debug('building: path=%s, tag=%s, args=%r', module_path,
first_image, plan.arguments['build_args'])
build_log = plan.arguments['build_log']
if plan.arguments['log_file']:
log_file = io.open(plan.arguments['log_file'], 'w', encoding='UTF-8')
else:
log_file = None
# build phase
stream = client.api.build(buildargs=plan.arguments['build_args'],
path=module_path,
rm=True,
tag=first_image,
decode=True)
last_events = deque(maxlen=2)
for event in stream:
last_events.append(event)
if 'error' in event:
logger.error(event['error'])
plan.status.description = 'error'
if 'stream' in event:
m = REGEX_DOCKER_BUILD_STEP.match(event['stream'])
for line in event['stream'].strip().splitlines():
if build_log:
logger.info('build %s: %s', plan.module, line)
if log_file:
log_file.write(line)
if not line.endswith('\n'):
log_file.write(u'\n')
log_file.flush()
if m:
step = m.group(1)
plan.status.current = int(step)
start, end = m.span()
cmd_snippet = event['stream'][end:20].strip()
plan.status.description = 'build %s %s %s' % (
first_image, m.group(3), cmd_snippet)
if log_file:
log_file.close()
# grabbed from docker-py/docker/models/images.py:ImageCollection.build
if not last_events[-1]:
raise BuildError('Unknown')
# the last line must say success, otherwise the build failed
image_id = None
build_errors = []
for event in last_events:
m = REGEX_DOCKER_BUILD_SUCCESS.match(event.get('stream') or '')
if event.get('error'):
build_errors.append(event.get('error'))
if m:
image_id = m.group(2)
if build_errors:
raise BuildError(build_errors)
if not image_id:
if build_errors:
raise BuildError('Build failed, errors: %r' % build_errors)
else:
raise BuildError('Build did not succeed. Last '
'line: %s' % last_events[-1])
image = client.images.get(image_id)
plan.artifacts.append(first_image)
# tagging phase
plan.status.current = plan.status.total
for extra_tag in images:
repo, tag = extra_tag.rsplit(':', 1)
image.tag(repo, tag=tag)
plan.artifacts.append(extra_tag)
def deploy_demo(demo_id, demo_dir):
"""
Checks for the existence of demo container and redploy it if it exist
Args:
demo_id: Demo ID for the demo to be deployed(this is a unique ID from
origami database)
demo_dir: Absolute path to the demo directory where it was unzipped.
"""
logging.info('Starting task to deploy demo with id : {}'.format(demo_id))
# Before doing anything get the previously created image if any.
try:
remove_demo_instance_if_exist(demo_id, 'redeploying')
except OrigamiDockerConnectionError as e:
logging.error(e)
return
demo = Demos.get_or_none(Demos.demo_id == demo_id)
if not demo:
demo_logs_uid = uuid.uuid4().hex
demo = Demos(demo_id=demo_id, log_id=demo_logs_uid, status='deploying')
# Get the dockerfile from the demo dir from ORIGAMI_CONFIG_HOME
dockerfile_dir = os.path.join(os.environ['HOME'], ORIGAMI_CONFIG_DIR,
ORIGAMI_DEMOS_DIRNAME, demo_id)
try:
# Here we are using low level API bindings provided by docker-py to
# interact with docker daemon. This enables us to collect image build
# Logs and provide them to user for debugging purposes.
logging.info('Trying to build image for demo.')
cli = APIClient(base_url=DOCKER_UNIX_SOCKET)
response = [
json.loads(line.decode().strip())
for line in cli.build(path=dockerfile_dir)
]
# Write build logs to log file.
logfile = os.path.join(get_origami_static_dir(),
ORIGAMI_DEPLOY_LOGS_DIR, demo.log_id)
with open(logfile, LOGS_FILE_MODE_REQ) as fp:
json.dump(response, fp)
final_res = response[-1]['stream'].strip()
match_obj = re.match(r'Successfully built (.*)', final_res, re.M | re.I)
image_id = None
try:
match_obj.group(1)
# SHA256 of the built image.
image_id = response[-2]['aux']['ID'][7:]
except IndexError as e:
raise BuildError(e)
except Exception as e:
logging.error(
"Error while parsing SHA256 ID of the image : {}".format(e))
raise BuildError(e)
# This was without using low level dockerpy client, it did not provide
# logs for the build process.
# image = docker_client.images.build(path=dockerfile_dir)[0]
logging.info('Image built : ID: {}'.format(image_id))
demo.image_id = image_id
# Run a new container instance for the demo.
if not demo.port:
port = get_a_free_port()
logging.info('New port for demo is {}'.format(port))
demo.port = port
port_map = '{}/tcp'.format(ORIGAMI_WRAPPED_DEMO_PORT)
cont = docker_client.containers.run(
image_id,
detach=True,
name=demo_id,
ports={port_map: demo.port},
remove=True)
logging.info('Demo deployed with container id : {}'.format(cont.id))
demo.container_id = cont.id
demo.status = 'running'
except BuildError as e:
logging.error('Error while building image for {} : {}'.format(
demo_id, e))
demo.status = 'error'
except APIError as e:
logging.error(
'Error while communicating to to docker API: {}'.format(e))
demo.status = 'error'
demo.save()
