def build_delay(task: int,
host,
build_type,
tag,
admin,
pt=None,
dockerfile=None):
"""
编译镜像
"""
task = db.session.query(TaskList).get(task)
instance = db.session.query(Host).filter(Host.id == host).one_or_none()
cli = APIClient(base_url='tcp://{}'.format(instance.addr))
if build_type == 'tar':
f = open(pt, 'rb')
for line in cli.build(fileobj=f, rm=True, tag=tag,
custom_context=True):
print(line)
task_add_log(task.id, line)
task.status = task.STATUS_DONE
elif build_type == 'pull':
for line in cli.pull(tag, stream=True, decode=True):
task_add_log(task.id, line)
task.status = task.STATUS_DONE
else:
try:
f = BytesIO(dockerfile.encode('utf-8'))
for line in cli.build(fileobj=f, rm=True, tag=tag):
task_add_log(task.id, line)
task.status = task.STATUS_DONE
except docker_error.DockerException as e:
task.status = task.STATUS_ERROR
task.remark = str(e)
db.session.commit()
class DockerImage(object):
def __init__(self, base_url):
self.client = APIClient(base_url=base_url, version='auto')
self.full_name = None
def build(self, path, name, tag):
self.full_name = '{0}/{1}:{2}'.format(app.config['DOCKER_REGISTRY_SERVER'], name, tag)
for item in self.client.build(path=path, tag=self.full_name, forcerm=False):
detail = json.loads(item.decode().strip())
if 'errorDetail' in detail:
raise Exception('Build image error: ' + detail['errorDetail'].get('message', '未知错误'))
def push(self, image=None):
repository = image or self.full_name
if repository is None:
raise Exception('Push image error: argument <image> is missing.')
for item in self.client.push(repository, auth_config=app.config['DOCKER_REGISTRY_AUTH'], stream=True):
detail = json.loads(item.decode().strip())
if 'errorDetail' in detail:
raise Exception('Push image error: ' + detail['errorDetail'].get('message', '未知错误'))
if 'aux' in detail:
return detail['aux']['Digest']
raise Exception('Push image error: 未知错误')
def remove(self, image=None):
repository = image or self.full_name
if repository is None:
raise Exception('Remove image error: argument <image> is missing.')
self.client.remove_image(repository)
def main(build_dict):
logger.info('BUILD STARTED BY %s\n', os.getenv('USER', 'unknown'))
cli = APIClient(base_url='unix://var/run/docker.sock')
dry_run = build_dict.pop('dry_run')
build_dict.update({
'rm': True,
'labels': {
'builder': os.getenv('USER'),
'build_time': time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
}
})
for image in build_order:
d = build_dict.copy()
d['tag'] = image + ':' + build_dict['tag']
d['path'] = op.join(_base_dir, image)
if dry_run:
logger.info('DRY RUN: Building image %s', d['tag'])
else:
try:
build_stream = cli.build(**d)
for line in build_stream:
msg = '{}: {}'.format(image,
json.loads(line).get('stream', ''))
logger.info(msg)
except Exception as e:
logger.error(e, exc_info=True)
class DockerBuilder(Builder[DockerBuildConfiguration, str,
DockerChecksumCalculator]):
"""
Builder of Docker images.
"""
def __init__(
self,
managed_build_configurations: Iterable[BuildConfigurationType] = None,
checksum_retriever: ChecksumRetriever = None,
checksum_calculator_factory: Callable[
[], DockerChecksumCalculator] = DockerChecksumCalculator):
super().__init__(managed_build_configurations, checksum_retriever,
checksum_calculator_factory)
self.checksum_calculator.managed_build_configurations = self.managed_build_configurations
self._docker_client = APIClient()
def __del__(self):
self._docker_client.close()
def _build(self, build_configuration: DockerBuildConfiguration) -> str:
logger.info(f"Building Docker image: {build_configuration.identifier}")
logger.debug(
f"{build_configuration.identifier} to be built using dockerfile "
f"\"{build_configuration.dockerfile_location}\" in context \"{build_configuration.context}\""
)
log_generator = self._docker_client.build(
path=build_configuration.context,
tag=build_configuration.identifier,
dockerfile=build_configuration.dockerfile_location,
decode=True)
log = {}
try:
for log in log_generator:
details = log.get("stream", "").strip()
if len(details) > 0:
logger.debug(details)
except APIError as e:
if e.status_code == 400 and "parse error" in e.explanation:
dockerfile_location = build_configuration.dockerfile_location
with open(dockerfile_location, "r") as file:
raise InvalidDockerfileBuildError(build_configuration.name,
dockerfile_location,
file.read())
raise e
if "error" in log:
error_details = log["errorDetail"]
raise BuildStepError(build_configuration.name,
error_details["message"],
error_details.get("code"))
return build_configuration.identifier
def build_image(path, tag):
cli = APIClient(base_url=os.environ.get('DOCKER_HOST'))
response = cli.build(path=path, forcerm=True, tag=tag)
for r in response:
for part in r.decode('utf-8').split('\r\n'):
if part == '':
continue
part = json.loads(part)
if 'stream' in part:
print(part['stream'].replace('\n', ''))
else:
print(part)
def build_partition_container(image_name: str, partition: str):
cli = APIClient()
out = ""
for output in cli.build(path=".",
rm=True,
tag=image_name,
dockerfile=f"Dockerfile.{partition}",
network_mode="host",
buildargs={"NOCACHE": str(int(time.time()))}):
out = json.loads(output.decode()).get("stream", "")
click.echo(out, nl=False)
if not out.startswith("Successfully tagged"):
raise Exception("Error building container")
def build_images(path, dockerfile, tag):
cli = APIClient()
os.environ['BUILD_PATH'] = path
for line_dict in cli.build(path=path,
dockerfile=dockerfile,
tag=tag,
decode=True):
line = line_dict.get('stream')
if line:
stdout = line.strip()
click.echo(stdout)
logger.debug(stdout)
class DockerBuilder:
def __init__(self):
self.docker_client = APIClient(version='auto')
def build(self, img, path='.'):
bld = self.docker_client.build(
path=path,
tag=img,
encoding='utf-8'
)
for line in bld:
self._process_stream(line)
def publish(self, img):
# TODO: do we need to set tag?
for line in self.docker_client.push(img, stream=True):
self._process_stream(line)
def _process_stream(self, line):
ln = line.decode('utf-8').strip()
# try to decode json
try:
ljson = json.loads(ln)
if ljson.get('error'):
msg = str(ljson.get('error', ljson))
logger.error('Build failed: ' + msg)
raise Exception('Image build failed: ' + msg)
else:
if ljson.get('stream'):
msg = 'Build output: {}'.format(ljson['stream'].strip())
elif ljson.get('status'):
msg = 'Push output: {} {}'.format(
ljson['status'],
ljson.get('progress')
)
elif ljson.get('aux'):
msg = 'Push finished: {}'.format(ljson.get('aux'))
else:
msg = str(ljson)
logger.debug(msg)
except json.JSONDecodeError:
logger.warning('JSON decode error: {}'.format(ln))
def build_image(client: docker.APIClient, path: str, name: str):
"""Build the Docker image as per Dockerfile present in <path>.
If the docker image with given name is newer than the Dockerfile,
nothing is done.
:param client:
Docker client
:param path:
Location of Dockerfile
:param name:
Name of the image
"""
assert os.path.exists(os.path.join(path, 'Dockerfile'))
time = os.stat(os.path.join(path, 'Dockerfile')).st_mtime
il = client.images(name=name)
if len(il) == 0 or il[0]['Created'] < time:
response = client.build(path, tag=name, rm=True)
for json_bytes in response:
line = json.loads(json_bytes.decode())['stream']
print(line, end='', file=sys.stderr, flush=True)
def create_image(request, model, path=None):
if not path:
path = BASE_DIR
try:
with open(path, 'r') as d:
dockerfile = [x.strip() for x in d.readlines()]
dockerfile = ' '.join(dockerfile)
dockerfile = bytes(dockerfile.encode('utf-8'))
f = BytesIO(dockerfile)
# Point to the Docker instance
cli = APIClient(base_url='tcp://192.168.99.100:2376')
response = [line for line in cli.build(fileobj=f, rm=True, tag=model)]
return JsonResponse({'image': response})
except:
return HttpResponseServerError()
def build_images(id):
# Dockerfile
t = Topic.objects.get(id=id)
docker_file_dir = os.path.join(os.path.dirname(t.zip_file.path),
t.build_name)
docker_client = APIClient(base_url='tcp://127.0.0.1:2375')
t.build_log = ""
try:
generator = docker_client.build(path=docker_file_dir,
tag=t.build_name,
rm=True)
except TypeError as e:
t.build_log += "请检查压缩包中是否存在Dockerfile文件\n错误信息："
t.build_log += str(e)
t.build_status = "fail"
t.save()
return
t.build_status = "building"
t.save()
status = True
for g in generator:
g = str(g, encoding="utf-8")
g = json.loads(g)
if 'stream' in g:
g = g['stream']
elif 'errorDetail' in g:
status = False
g = g['errorDetail']['message']
elif 'aux' in g:
# 容器ＩＤ
g = g['aux']['ID']
t.image_id = re.match(r'sha256:([A-Za-z0-9]{15})', g).group(1)
t.build_log += g
t.save()
if status:
t.build_status = 'success'
else:
t.build_status = 'fail'
t.save()
def bootstrap(args: Namespace) -> None:
"""Create the images used by son-analyze in the current host"""
cli = APIClient(base_url=args.docker_socket)
root_context = os.path.realpath(
resource_filename('son_analyze.cli', '../../..'))
_LOGGER.info('The root context path is: %s', root_context)
path = resource_filename('son_analyze.cli.resources',
'anaconda.Dockerfile')
path = os.path.relpath(path, root_context)
_LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path)
# import pdb; pdb.set_trace()
for line in cli.build(path=root_context,
tag=_IMAGE_TAG,
dockerfile=path,
rm=True,
decode=True):
if "stream" in line:
print('> ', line["stream"], end="")
else:
print(line)
sys.exit(1)
sys.exit(0)
def __add_node():
docker_client = APIClient(base_url=TwinHub.DOCKER_CLIENT_URI)
responses = docker_client.build(
dockerfile=TwinHub.DOCKERFILE_PATH,
path=TwinHub.TWIN_DOCKER_SOURCES_DIR,
encoding=TwinHub.DEFAULT_DOCKERFILE_ENCODING,
rm=True,
tag=TwinHub.NODE_IMAGE_TAG)
for msg in responses:
print(msg)
# Creating network if not exists.
network = docker_client.networks(names=[TwinHub.NETWORK_NAME])
if not network:
docker_client.create_network(TwinHub.NETWORK_NAME, driver="bridge")
# Creating new container
TwinHub.NODE_COUNTER += 1
node_name = 'twin_node_{}'.format(TwinHub.NODE_COUNTER)
container = docker_client.create_container(TwinHub.NODE_IMAGE_TAG,
name=node_name,
ports=[5000],
tty=True,
stdin_open=True,
detach=True,
hostname=node_name)
docker_client.start(container['Id'])
docker_client.connect_container_to_network(container=node_name,
net_id=TwinHub.NETWORK_NAME)
return "<html><h1>Hello world</h1></html>"
class DockerBuilder(object):
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
HEART_BEAT_INTERVAL = 60
def __init__(self,
build_job,
repo_path,
from_image,
copy_code=True,
build_steps=None,
env_vars=None,
dockerfile_name='Dockerfile'):
self.build_job = build_job
self.job_uuid = build_job.uuid.hex
self.job_name = build_job.unique_name
self.from_image = from_image
self.image_name = get_image_name(self.build_job)
self.image_tag = self.job_uuid
self.folder_name = repo_path.split('/')[-1]
self.repo_path = repo_path
self.copy_code = copy_code
self.build_path = '/'.join(self.repo_path.split('/')[:-1])
self.build_steps = to_list(build_steps, check_none=True)
self.env_vars = to_list(env_vars, check_none=True)
self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
self.polyaxon_requirements_path = self._get_requirements_path()
self.polyaxon_setup_path = self._get_setup_path()
self.docker = APIClient(version='auto')
self.registry_host = None
self.docker_url = None
self.is_pushing = False
def get_tagged_image(self):
return get_tagged_image(self.build_job)
def check_image(self):
return self.docker.images(self.get_tagged_image())
def clean(self):
# Clean dockerfile
delete_path(self.dockerfile_path)
def login_internal_registry(self):
try:
self.docker.login(username=conf.get('REGISTRY_USER'),
password=conf.get('REGISTRY_PASSWORD'),
registry=conf.get('REGISTRY_HOST'),
reauth=True)
except DockerException as e:
_logger.exception('Failed to connect to registry %s\n', e)
def login_private_registries(self):
if not conf.get('PRIVATE_REGISTRIES'):
return
for registry in conf.get('PRIVATE_REGISTRIES'):
self.docker.login(username=registry.user,
password=registry.password,
registry=registry.host,
reauth=True)
def _prepare_log_lines(self, log_line):
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
status = True
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
log_lines.append('{}: {}'.format(
LogLevels.ERROR, str(json_line.get('error',
json_line))))
status = False
else:
if json_line.get('stream'):
log_lines.append('Building: {}'.format(
json_line['stream'].strip()))
elif json_line.get('status'):
if not self.is_pushing:
self.is_pushing = True
log_lines.append('Pushing ...')
elif json_line.get('aux'):
log_lines.append('Pushing finished: {}'.format(
json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines, status
def _handle_logs(self, log_lines):
for log_line in log_lines:
print(log_line)
def _handle_log_stream(self, stream):
log_lines = []
last_heart_beat = time.time()
status = True
try:
for log_line in stream:
new_log_lines, new_status = self._prepare_log_lines(log_line)
log_lines += new_log_lines
if not new_status:
status = new_status
self._handle_logs(log_lines)
log_lines = []
if time.time() - last_heart_beat > self.HEART_BEAT_INTERVAL:
last_heart_beat = time.time()
RedisHeartBeat.build_ping(build_id=self.build_job.id)
if log_lines:
self._handle_logs(log_lines)
except (BuildError, APIError) as e:
self._handle_logs('{}: Could not build the image, '
'encountered {}'.format(LogLevels.ERROR, e))
return False
return status
def _get_requirements_path(self):
def get_requirements(requirements_file):
requirements_path = os.path.join(self.repo_path, requirements_file)
if os.path.isfile(requirements_path):
return os.path.join(self.folder_name, requirements_file)
requirements = get_requirements('polyaxon_requirements.txt')
if requirements:
return requirements
requirements = get_requirements('requirements.txt')
if requirements:
return requirements
return None
def _get_setup_path(self):
def get_setup(setup_file):
setup_file_path = os.path.join(self.repo_path, setup_file)
has_setup = os.path.isfile(setup_file_path)
if has_setup:
st = os.stat(setup_file_path)
os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
return os.path.join(self.folder_name, setup_file)
setup_file = get_setup('polyaxon_setup.sh')
if setup_file:
return setup_file
setup_file = get_setup('setup.sh')
if setup_file:
return setup_file
return None
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
from_image=self.from_image,
polyaxon_requirements_path=self.polyaxon_requirements_path,
polyaxon_setup_path=self.polyaxon_setup_path,
build_steps=self.build_steps,
env_vars=self.env_vars,
folder_name=self.folder_name,
workdir=self.WORKDIR,
nvidia_bin=conf.get('MOUNT_PATHS_NVIDIA').get('bin'),
copy_code=self.copy_code)
def build(self, nocache=False, memory_limit=None):
_logger.debug('Starting build for `%s`', self.repo_path)
# Checkout to the correct commit
# if self.image_tag != self.LATEST_IMAGE_TAG:
# git.checkout_commit(repo_path=self.repo_path, commit=self.image_tag)
limits = {
# Disable memory swap for building
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
rendered_dockerfile = self.render()
celery_app.send_task(
SchedulerCeleryTasks.BUILD_JOBS_SET_DOCKERFILE,
kwargs={
'build_job_uuid': self.job_uuid,
'dockerfile': rendered_dockerfile
})
dockerfile.write(rendered_dockerfile)
stream = self.docker.build(path=self.build_path,
tag=self.get_tagged_image(),
forcerm=True,
rm=True,
pull=True,
nocache=nocache,
container_limits=limits)
return self._handle_log_stream(stream=stream)
def push(self):
stream = self.docker.push(self.image_name,
tag=self.image_tag,
stream=True)
return self._handle_log_stream(stream=stream)
class HubIO:
""" :class:`HubIO` provides the way to interact with Jina Hub registry.
You can use it with CLI to package a directory into a Jina Hub image and publish it to the world.
Examples:
- :command:`jina hub build my_pod/` build the image
- :command:`jina hub build my_pod/ --push` build the image and push to the public registry
- :command:`jina hub pull jinahub/pod.dummy_mwu_encoder:0.0.6` to download the image
"""
def __init__(self, args: 'argparse.Namespace'):
self.logger = get_logger(self.__class__.__name__, **vars(args))
self.args = args
try:
import docker
from docker import APIClient
self._client = docker.from_env()
# low-level client
self._raw_client = APIClient(base_url='unix://var/run/docker.sock')
except (ImportError, ModuleNotFoundError):
self.logger.critical(
'requires "docker" dependency, please install it via "pip install jina[docker]"'
)
raise
def new(self) -> None:
"""Create a new executor using cookiecutter template """
try:
from cookiecutter.main import cookiecutter
except (ImportError, ModuleNotFoundError):
self.logger.critical(
'requires "cookiecutter" dependency, please install it via "pip install cookiecutter"'
)
raise
import click
cookiecutter_template = self.args.template
if self.args.type == 'app':
cookiecutter_template = 'https://github.com/jina-ai/cookiecutter-jina.git'
elif self.args.type == 'pod':
cookiecutter_template = 'https://github.com/jina-ai/cookiecutter-jina-hub.git'
cookiecutter(cookiecutter_template,
overwrite_if_exists=self.args.overwrite,
output_dir=self.args.output_dir)
try:
cookiecutter(cookiecutter_template,
overwrite_if_exists=self.args.overwrite,
output_dir=self.args.output_dir)
except click.exceptions.Abort:
self.logger.info('nothing is created, bye!')
def push(self, name: str = None, readme_path: str = None) -> None:
""" A wrapper of docker push
- Checks for the tempfile, returns without push if it cannot find
- Pushes to docker hub, returns withput writing to db if it fails
- Writes to the db
"""
name = name or self.args.name
file_path = get_summary_path(name)
if not os.path.isfile(file_path):
self.logger.error(f'can not find the build summary file')
return
try:
self._push_docker_hub(name, readme_path)
except:
self.logger.error('can not push to the docker hub registry')
return
with open(file_path) as f:
result = json.load(f)
if result['is_build_success']:
self._write_summary_to_db(summary=result)
def _push_docker_hub(self,
name: str = None,
readme_path: str = None) -> None:
""" Helper push function """
check_registry(self.args.registry, name, _repo_prefix)
self._check_docker_image(name)
self.login()
with ProgressBar(task_name=f'pushing {name}', batch_unit='') as t:
for line in self._client.images.push(name,
stream=True,
decode=True):
t.update(1)
self.logger.debug(line)
self.logger.success(f'🎉 {name} is now published!')
if False and readme_path:
# unfortunately Docker Hub Personal Access Tokens cannot be used as they are not supported by the API
_volumes = {
os.path.dirname(os.path.abspath(readme_path)): {
'bind': '/workspace'
}
}
_env = get_default_login()
_env = {
'DOCKERHUB_USERNAME': _env['username'],
'DOCKERHUB_PASSWORD': _env['password'],
'DOCKERHUB_REPOSITORY': name.split(':')[0],
'README_FILEPATH': '/workspace/README.md',
}
self._client.containers.run('peterevans/dockerhub-description:2.1',
auto_remove=True,
volumes=_volumes,
environment=_env)
share_link = f'https://api.jina.ai/hub/?jh={urllib.parse.quote_plus(name)}'
try:
webbrowser.open(share_link, new=2)
except:
pass
finally:
self.logger.info(
f'Check out the usage {colored(share_link, "cyan", attrs=["underline"])} and share it with others!'
)
def pull(self) -> None:
"""A wrapper of docker pull """
check_registry(self.args.registry, self.args.name, _repo_prefix)
self.login()
try:
with TimeContext(f'pulling {self.args.name}', self.logger):
image = self._client.images.pull(self.args.name)
if isinstance(image, list):
image = image[0]
image_tag = image.tags[0] if image.tags else ''
self.logger.success(
f'🎉 pulled {image_tag} ({image.short_id}) uncompressed size: {get_readable_size(image.attrs["Size"])}'
)
except:
self.logger.error(
f'can not pull image {self.args.name} from {self.args.registry}'
)
def _check_docker_image(self, name: str) -> None:
# check local image
image = self._client.images.get(name)
for r in _allowed:
if f'{_label_prefix}{r}' not in image.labels.keys():
self.logger.warning(
f'{r} is missing in your docker image labels, you may want to check it'
)
try:
if name != safe_url_name(f'{_repo_prefix}' +
'{type}.{kind}.{name}:{version}'.format(
**{
k.replace(_label_prefix, ''): v
for k, v in image.labels.items()
})):
raise ValueError(
f'image {name} does not match with label info in the image'
)
except KeyError:
self.logger.error('missing key in the label of the image')
raise
self.logger.info(
f'✅ {name} is a valid Jina Hub image, ready to publish')
def login(self) -> None:
"""A wrapper of docker login """
try:
password = self.args.password # or (self.args.password_stdin and self.args.password_stdin.read())
except ValueError:
password = ''
if self.args.username and password:
self._client.login(username=self.args.username,
password=password,
registry=self.args.registry)
else:
# use default login
self._client.login(**get_default_login(),
registry=self.args.registry)
def build(self) -> Dict:
"""A wrapper of docker build """
if self.args.dry_run:
result = self.dry_run()
else:
is_build_success, is_push_success = True, False
_logs = []
_excepts = []
with TimeContext(f'building {colored(self.args.path, "green")}',
self.logger) as tc:
try:
self._check_completeness()
streamer = self._raw_client.build(
decode=True,
path=self.args.path,
tag=self.tag,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
for chunk in streamer:
if 'stream' in chunk:
for line in chunk['stream'].splitlines():
if is_error_message(line):
self.logger.critical(line)
_excepts.append(line)
elif 'warning' in line.lower():
self.logger.warning(line)
else:
self.logger.info(line)
_logs.append(line)
except Exception as ex:
# if pytest fails it should end up here as well
is_build_success = False
_excepts.append(str(ex))
if is_build_success:
# compile it again, but this time don't show the log
image, log = self._client.images.build(
path=self.args.path,
tag=self.tag,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
# success
_details = {
'inspect': self._raw_client.inspect_image(image.tags[0]),
'tag': image.tags[0],
'hash': image.short_id,
'size': get_readable_size(image.attrs['Size']),
}
self.logger.success(
'🎉 built {tag} ({hash}) uncompressed size: {size}'.
format_map(_details))
else:
self.logger.error(
f'can not build the image, please double check the log')
_details = {}
if is_build_success:
if self.args.test_uses:
try:
is_build_success = False
from jina.flow import Flow
p_name = random_name()
with Flow().add(name=p_name,
uses=image.tags[0],
daemon=self.args.daemon):
pass
if self.args.daemon:
self._raw_client.stop(p_name)
self._raw_client.prune_containers()
is_build_success = True
except PeaFailToStart:
self.logger.error(
f'can not use it in the Flow, please check your file bundle'
)
except Exception as ex:
self.logger.error(
f'something wrong but it is probably not your fault. {repr(ex)}'
)
_version = self.manifest[
'version'] if 'version' in self.manifest else '0.0.1'
info, env_info = get_full_version()
_host_info = {
'jina': info,
'jina_envs': env_info,
'docker': self._raw_client.info(),
'build_args': vars(self.args)
}
_build_history = {
'time':
get_now_timestamp(),
'host_info':
_host_info if is_build_success and self.args.host_info else '',
'duration':
tc.readable_duration,
'logs':
_logs,
'exception':
_excepts
}
if self.args.prune_images:
self.logger.info('deleting unused images')
self._raw_client.prune_images()
result = {
'name':
getattr(self, 'canonical_name', ''),
'version':
self.manifest['version'] if is_build_success
and 'version' in self.manifest else '0.0.1',
'path':
self.args.path,
'manifest_info':
self.manifest if is_build_success else '',
'details':
_details,
'is_build_success':
is_build_success,
'build_history': [_build_history]
}
# only successful build (NOT dry run) writes the summary to disk
if result['is_build_success']:
self._write_summary_to_file(summary=result)
if self.args.push:
try:
self._push_docker_hub(image.tags[0], self.readme_path)
self._write_summary_to_db(summary=result)
self._write_slack_message(result, _details,
_build_history)
except Exception as ex:
self.logger.error(
f'can not complete the push due to {repr(ex)}')
if not result['is_build_success'] and self.args.raise_error:
# remove the very verbose build log when throw error
result['build_history'][0].pop('logs')
raise RuntimeError(result)
return result
def dry_run(self) -> Dict:
try:
s = self._check_completeness()
s['is_build_success'] = True
except Exception as ex:
s = {'is_build_success': False, 'exception': str(ex)}
return s
def _write_summary_to_db(self, summary: Dict) -> None:
""" Inserts / Updates summary document in mongodb """
if not is_db_envs_set():
self.logger.warning(
'MongoDB environment vars are not set! bookkeeping skipped.')
return
build_summary = handle_dot_in_keys(document=summary)
_build_query = {
'name': build_summary['name'],
'version': build_summary['version']
}
_current_build_history = build_summary['build_history']
with MongoDBHandler(
hostname=os.environ['JINA_DB_HOSTNAME'],
username=os.environ['JINA_DB_USERNAME'],
password=os.environ['JINA_DB_PASSWORD'],
database_name=os.environ['JINA_DB_NAME'],
collection_name=os.environ['JINA_DB_COLLECTION']) as db:
existing_doc = db.find(query=_build_query)
if existing_doc:
build_summary['build_history'] = existing_doc[
'build_history'] + _current_build_history
_modified_count = db.replace(document=build_summary,
query=_build_query)
self.logger.debug(
f'Updated the build + push summary in db. {_modified_count} documents modified'
)
else:
_inserted_id = db.insert(document=build_summary)
self.logger.debug(
f'Inserted the build + push summary in db with id {_inserted_id}'
)
def _write_summary_to_file(self, summary: Dict) -> None:
file_path = get_summary_path(f'{summary["name"]}:{summary["version"]}')
with open(file_path, 'w+') as f:
json.dump(summary, f)
self.logger.debug(f'stored the summary from build to {file_path}')
def _check_completeness(self) -> Dict:
self.dockerfile_path = get_exist_path(self.args.path, 'Dockerfile')
self.manifest_path = get_exist_path(self.args.path, 'manifest.yml')
self.readme_path = get_exist_path(self.args.path, 'README.md')
self.requirements_path = get_exist_path(self.args.path,
'requirements.txt')
yaml_glob = glob.glob(os.path.join(self.args.path, '*.yml'))
if yaml_glob:
yaml_glob.remove(self.manifest_path)
py_glob = glob.glob(os.path.join(self.args.path, '*.py'))
test_glob = glob.glob(os.path.join(self.args.path, 'tests/test_*.py'))
completeness = {
'Dockerfile': self.dockerfile_path,
'manifest.yml': self.manifest_path,
'README.md': self.readme_path,
'requirements.txt': self.requirements_path,
'*.yml': yaml_glob,
'*.py': py_glob,
'tests': test_glob
}
self.logger.info(f'completeness check\n' + '\n'.join(
'%4s %-20s %s' %
(colored('✓', 'green') if v else colored('✗', 'red'), k, v)
for k, v in completeness.items()) + '\n')
if completeness['Dockerfile'] and completeness['manifest.yml']:
pass
else:
self.logger.critical(
'Dockerfile or manifest.yml is not given, can not build')
raise FileNotFoundError(
'Dockerfile or manifest.yml is not given, can not build')
self.manifest = self._read_manifest(self.manifest_path)
self.dockerfile_path_revised = self._get_revised_dockerfile(
self.dockerfile_path, self.manifest)
self.tag = safe_url_name(f'{_repo_prefix}' +
'{type}.{kind}.{name}:{version}'.format(
**self.manifest))
self.canonical_name = safe_url_name(f'{_repo_prefix}' +
'{type}.{kind}.{name}'.format(
**self.manifest))
return completeness
def _read_manifest(self, path: str, validate: bool = True) -> Dict:
with resource_stream(
'jina', '/'.join(
('resources', 'hub-builder', 'manifest.yml'))) as fp:
tmp = yaml.load(
fp
) # do not expand variables at here, i.e. DO NOT USE expand_dict(yaml.load(fp))
with open(path) as fp:
tmp.update(yaml.load(fp))
if validate:
self._validate_manifest(tmp)
return tmp
def _validate_manifest(self, manifest: Dict) -> None:
required = {'name', 'type', 'version'}
# check the required field in manifest
for r in required:
if r not in manifest:
raise ValueError(
f'{r} is missing in the manifest.yaml, it is required')
# check if all fields are there
for r in _allowed:
if r not in manifest:
self.logger.warning(
f'{r} is missing in your manifest.yml, you may want to check it'
)
# check name
check_name(manifest['name'])
# check_image_type
check_image_type(manifest['type'])
# check version number
check_version(manifest['version'])
# check version number
check_license(manifest['license'])
# check platform
if not isinstance(manifest['platform'], list):
manifest['platform'] = list(manifest['platform'])
check_platform(manifest['platform'])
# replace all chars in value to safe chars
for k, v in manifest.items():
if v and isinstance(v, str):
manifest[k] = remove_control_characters(v)
# show manifest key-values
for k, v in manifest.items():
self.logger.debug(f'{k}: {v}')
def _get_revised_dockerfile(self, dockerfile_path: str,
manifest: Dict) -> str:
# modify dockerfile
revised_dockerfile = []
with open(dockerfile_path) as fp:
for l in fp:
revised_dockerfile.append(l)
if l.startswith('FROM'):
revised_dockerfile.append('LABEL ')
revised_dockerfile.append(' \\ \n'.join(
f'{_label_prefix}{k}="{v}"'
for k, v in manifest.items()))
f = tempfile.NamedTemporaryFile('w', delete=False).name
with open(f, 'w', encoding='utf8') as fp:
fp.writelines(revised_dockerfile)
for k in revised_dockerfile:
self.logger.debug(k)
return f
def _write_slack_message(self, *args):
def _expand_fn(v):
if isinstance(v, str):
for d in args:
try:
v = v.format(**d)
except KeyError:
pass
return v
if 'JINAHUB_SLACK_WEBHOOK' in os.environ:
with resource_stream(
'jina', '/'.join(('resources', 'hub-builder-success',
'slack-jinahub.json'))) as fp:
tmp = expand_dict(json.load(fp),
_expand_fn,
resolve_cycle_ref=False)
req = urllib.request.Request(
os.environ['JINAHUB_SLACK_WEBHOOK'])
req.add_header('Content-Type',
'application/json; charset=utf-8')
jdb = json.dumps(tmp).encode('utf-8') # needs to be bytes
req.add_header('Content-Length', str(len(jdb)))
with urllib.request.urlopen(req, jdb) as f:
res = f.read()
self.logger.info(f'push to Slack: {res}')
# alias of "new" in cli
create = new
init = new
from io import BytesIO
from docker import APIClient
dockerfile = '''
FROM microsoft/dotnet-framework:4.7.1-windowsservercore-1709
ADD . /app
WORKDIR /app
ENTRYPOINT ["cmd.exe", "/k", "DockerAutonomProject.exe"]
'''
f = BytesIO(dockerfile.encode('utf-8'))
cli = APIClient(base_url='tcp://127.0.0.1:2375')
response = [
line for line in cli.build(fileobj=f, rm=True, tag='baseareaimage')
]
print(response)
class DockerBuilder(object):
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
def __init__(self,
build_job,
repo_path,
from_image,
copy_code=True,
build_steps=None,
env_vars=None,
dockerfile_name='Dockerfile'):
self.build_job = build_job
self.job_uuid = build_job.uuid.hex
self.job_name = build_job.unique_name
self.from_image = from_image
self.image_name = get_image_name(self.build_job)
self.image_tag = self.job_uuid
self.folder_name = repo_path.split('/')[-1]
self.repo_path = repo_path
self.copy_code = copy_code
self.build_path = '/'.join(self.repo_path.split('/')[:-1])
self.build_steps = get_list(build_steps)
self.env_vars = get_list(env_vars)
self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
self.polyaxon_requirements_path = self._get_requirements_path()
self.polyaxon_setup_path = self._get_setup_path()
self.docker = APIClient(version='auto')
self.registry_host = None
self.docker_url = None
def get_tagged_image(self):
return get_tagged_image(self.build_job)
def check_image(self):
return self.docker.images(self.get_tagged_image())
def clean(self):
# Clean dockerfile
delete_path(self.dockerfile_path)
def login(self, registry_user, registry_password, registry_host):
try:
self.docker.login(username=registry_user,
password=registry_password,
registry=registry_host,
reauth=True)
except DockerException as e:
_logger.exception('Failed to connect to registry %s\n', e)
@staticmethod
def _prepare_log_lines(log_line):
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
raise DockerBuilderError(str(json_line.get('error', json_line)))
else:
if json_line.get('stream'):
log_lines.append('Build: {}'.format(json_line['stream'].strip()))
elif json_line.get('status'):
log_lines.append('Push: {} {}'.format(
json_line['status'],
json_line.get('progress')
))
elif json_line.get('aux'):
log_lines.append('Push finished: {}'.format(json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines
def _handle_logs(self, log_lines):
publisher.publish_build_job_log(
log_lines=log_lines,
job_uuid=self.job_uuid,
job_name=self.job_name
)
def _handle_log_stream(self, stream):
log_lines = []
last_emit_time = time.time()
try:
for log_line in stream:
log_lines += self._prepare_log_lines(log_line)
publish_cond = (
len(log_lines) == publisher.MESSAGES_COUNT or
(log_lines and time.time() - last_emit_time > publisher.MESSAGES_TIMEOUT)
)
if publish_cond:
self._handle_logs(log_lines)
log_lines = []
last_emit_time = time.time()
if log_lines:
self._handle_logs(log_lines)
except (BuildError, APIError, DockerBuilderError) as e:
self._handle_logs('Build Error {}'.format(e))
return False
return True
def _get_requirements_path(self):
requirements_path = os.path.join(self.repo_path, 'polyaxon_requirements.txt')
if os.path.isfile(requirements_path):
return os.path.join(self.folder_name, 'polyaxon_requirements.txt')
return None
def _get_setup_path(self):
setup_file_path = os.path.join(self.repo_path, 'polyaxon_setup.sh')
has_setup = os.path.isfile(setup_file_path)
if has_setup:
st = os.stat(setup_file_path)
os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
return os.path.join(self.folder_name, 'polyaxon_setup.sh')
return None
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
from_image=self.from_image,
polyaxon_requirements_path=self.polyaxon_requirements_path,
polyaxon_setup_path=self.polyaxon_setup_path,
build_steps=self.build_steps,
env_vars=self.env_vars,
folder_name=self.folder_name,
workdir=self.WORKDIR,
nvidia_bin=settings.MOUNT_PATHS_NVIDIA.get('bin'),
copy_code=self.copy_code
)
def build(self, nocache=False, memory_limit=None):
_logger.debug('Starting build in `%s`', self.repo_path)
# Checkout to the correct commit
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.repo_path, commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
rendered_dockerfile = self.render()
celery_app.send_task(
SchedulerCeleryTasks.BUILD_JOBS_SET_DOCKERFILE,
kwargs={'build_job_uuid': self.job_uuid, 'dockerfile': rendered_dockerfile})
dockerfile.write(rendered_dockerfile)
stream = self.docker.build(
path=self.build_path,
tag=self.get_tagged_image(),
forcerm=True,
rm=True,
pull=True,
nocache=nocache,
container_limits=limits)
return self._handle_log_stream(stream=stream)
def push(self):
stream = self.docker.push(self.image_name, tag=self.image_tag, stream=True)
return self._handle_log_stream(stream=stream)
class BaseDockerBuilder(object):
CHECK_INTERVAL = 10
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
def __init__(self,
repo_path,
from_image,
image_name,
image_tag,
copy_code=True,
in_tmp_repo=True,
steps=None,
env_vars=None,
dockerfile_name='Dockerfile'):
self.from_image = from_image
self.image_name = image_name
self.image_tag = image_tag
self.repo_path = repo_path
self.folder_name = repo_path.split('/')[-1]
self.copy_code = copy_code
self.in_tmp_repo = in_tmp_repo
if in_tmp_repo and copy_code:
self.build_repo_path = self.create_tmp_repo()
else:
self.build_repo_path = self.repo_path
self.build_path = '/'.join(self.build_repo_path.split('/')[:-1])
self.steps = steps or []
self.env_vars = env_vars or []
self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
self.polyaxon_requirements_path = self._get_requirements_path()
self.polyaxon_setup_path = self._get_setup_path()
self.docker = None
def create_tmp_repo(self):
# Create a tmp copy of the repo before starting the build
return copy_to_tmp_dir(self.repo_path,
os.path.join(self.image_tag, self.folder_name))
def clean(self):
# Clean dockerfile
delete_path(self.dockerfile_path)
# Clean tmp dir if created
if self.in_tmp_repo and self.copy_code:
delete_tmp_dir(self.image_tag)
def connect(self):
if not self.docker:
self.docker = APIClient(version='auto')
def login(self, registry_user, registry_password, registry_host):
self.connect()
try:
self.docker.login(username=registry_user,
password=registry_password,
registry=registry_host,
reauth=True)
except DockerException as e:
logger.exception('Failed to connect to registry %s\n' % e)
def _handle_logs(self, log_line):
raise NotImplementedError
def _check_pulse(self, check_pulse):
"""Checks if the job/experiment is still running.
returns:
* int: the updated check_pulse (+1) value
* boolean: if the docker process should stop
"""
raise NotImplementedError
def _get_requirements_path(self):
requirements_path = os.path.join(self.build_repo_path,
'polyaxon_requirements.txt')
if os.path.isfile(requirements_path):
return os.path.join(self.folder_name, 'polyaxon_requirements.txt')
return None
def _get_setup_path(self):
setup_file_path = os.path.join(self.build_repo_path,
'polyaxon_setup.sh')
has_setup = os.path.isfile(setup_file_path)
if has_setup:
st = os.stat(setup_file_path)
os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
return os.path.join(self.folder_name, 'polyaxon_setup.sh')
return None
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
from_image=self.from_image,
polyaxon_requirements_path=self.polyaxon_requirements_path,
polyaxon_setup_path=self.polyaxon_setup_path,
steps=self.steps,
env_vars=self.env_vars,
folder_name=self.folder_name,
workdir=self.WORKDIR,
nvidia_bin=settings.MOUNT_PATHS_NVIDIA.get('bin'),
copy_code=self.copy_code)
def build(self, memory_limit=None):
logger.debug('Starting build in `{}`'.format(self.build_repo_path))
# Checkout to the correct commit
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path,
commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
dockerfile.write(self.render())
self.connect()
check_pulse = 0
for log_line in self.docker.build(
path=self.build_path,
tag='{}:{}'.format(self.image_name, self.image_tag),
buildargs={},
decode=True,
forcerm=True,
rm=True,
pull=True,
nocache=False,
container_limits=limits,
stream=True,
):
self._handle_logs(log_line)
# Check if we need to stop this process
check_pulse, should_stop = self._check_pulse(check_pulse)
if should_stop:
return False
# Checkout back to master
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path)
return True
def push(self):
# Build a progress setup for each layer, and only emit per-layer info every 1.5s
layers = {}
last_emit_time = time.time()
self.connect()
check_pulse = 0
for log_line in self.docker.push(self.image_name,
tag=self.image_tag,
stream=True):
lines = [l for l in log_line.decode('utf-8').split('\r\n') if l]
lines = [json.loads(l) for l in lines]
for progress in lines:
if 'error' in progress:
logger.error(progress['error'], extra=dict(phase='failed'))
return
if 'id' not in progress:
continue
if 'progressDetail' in progress and progress['progressDetail']:
layers[progress['id']] = progress['progressDetail']
else:
layers[progress['id']] = progress['status']
if time.time() - last_emit_time > 1.5:
logger.debug('Pushing image\n',
extra=dict(progress=layers, phase='pushing'))
last_emit_time = time.time()
self._handle_logs(log_line)
# Check if we need to stop this process
check_pulse, should_stop = self._check_pulse(check_pulse)
if should_stop:
return False
return True
class DockerProxy:
""" A wrapper over docker-py and some utility methods and classes. """
LOG_TAG = "Docker "
shell_commands = ["source"]
class ImageBuildException(Exception):
def __init__(self, message=None):
super("Something went wrong while building docker container image.\n{0}".format(message))
def __init__(self):
self.client = Client(base_url=Constants.DOCKER_BASE_URL)
self.build_count = 0
logging.basicConfig(level=logging.DEBUG)
@staticmethod
def get_container_volume_from_working_dir(working_directory):
import os
return os.path.join("/home/ubuntu/", os.path.basename(working_directory))
def create_container(self, image_str, working_directory=None, name=None,
port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: ('127.0.0.1', 8080),
Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: ('127.0.0.1', 8081)}):
"""Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container
to 8080 of locahost by default"""
docker_image = DockerImage.from_string(image_str)
volume_dir = DockerProxy.get_container_volume_from_working_dir(working_directory)
if name is None:
import uuid
random_str = str(uuid.uuid4())
name = constants.Constants.MolnsDockerContainerNamePrefix + random_str[:8]
image = docker_image.image_id if docker_image.image_id is not Constants.DockerNonExistentTag \
else docker_image.image_tag
logging.info("Using image {0}".format(image))
import os
if DockerProxy._verify_directory(working_directory) is False:
if working_directory is not None:
raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the "
"controller environment, and unfortunately the provided name for working "
"directory of the controller cannot be one of these. Please configure this "
"controller again with a different volume name and retry. "
"Here is the list of forbidden names: \n{0}"
.format(Constants.ForbiddenVolumeNames))
logging.warning(DockerProxy.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT "
"be created.")
hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings)
container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True,
ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT,
Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT],
host_config=hc,
environment={"PYTHONPATH": "/usr/local/"})
else:
container_mount_point = '/home/ubuntu/{0}'.format(os.path.basename(working_directory))
hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings,
binds={working_directory: {'bind': container_mount_point,
'mode': 'rw'}})
container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True,
ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT,
Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT],
volumes=container_mount_point, host_config=hc,
working_dir=volume_dir,
environment={"PYTHONPATH": "/usr/local/"})
container_id = container.get("Id")
return container_id
# noinspection PyBroadException
@staticmethod
def _verify_directory(working_directory):
import os
if working_directory is None or os.path.basename(working_directory) in Constants.ForbiddenVolumeNames:
return False
try:
if not os.path.exists(working_directory):
os.makedirs(working_directory)
return True
except:
return False
def stop_containers(self, container_ids):
"""Stops given containers."""
for container_id in container_ids:
self.stop_container(container_id)
def stop_container(self, container_id):
"""Stops the container with given ID."""
self.client.stop(container_id)
def container_status(self, container_id):
"""Checks if container with given ID running."""
status = ProviderBase.STATUS_TERMINATED
try:
ret_val = str(self.client.inspect_container(container_id).get('State').get('Status'))
if ret_val.startswith("running"):
status = ProviderBase.STATUS_RUNNING
else:
status = ProviderBase.STATUS_STOPPED
except NotFound:
pass
return status
def start_containers(self, container_ids):
"""Starts each container in given list of container IDs."""
for container_id in container_ids:
self.start_container(container_id)
def start_container(self, container_id):
""" Start the container with given ID."""
logging.info(DockerProxy.LOG_TAG + " Starting container " + container_id)
try:
self.client.start(container=container_id)
except (NotFound, NullResource) as e:
print (DockerProxy.LOG_TAG + "Something went wrong while starting container.", e)
return False
return True
def execute_command(self, container_id, command):
"""Executes given command as a shell command in the given container. Returns None is anything goes wrong."""
run_command = "/bin/bash -c \"" + command + "\""
# print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command))
if self.start_container(container_id) is False:
print (DockerProxy.LOG_TAG + "Could not start container.")
return None
try:
exec_instance = self.client.exec_create(container_id, run_command)
response = self.client.exec_start(exec_instance)
return [self.client.exec_inspect(exec_instance), response]
except (NotFound, APIError) as e:
print (DockerProxy.LOG_TAG + " Could not execute command.", e)
return None
def build_image(self, dockerfile):
""" Build image from given Dockerfile object and return ID of the image created. """
import uuid
logging.info("Building image...")
random_string = str(uuid.uuid4())
image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(random_string[:])
last_line = ""
try:
for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag):
print(DockerProxy._decorate(line))
if "errorDetail" in line:
raise DockerProxy.ImageBuildException()
last_line = line
# Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image
# id.
image_id = get_docker_image_id_from_string(str(last_line))
logging.info("Image ID: {0}".format(image_id))
return str(DockerImage(image_id, image_tag))
except (DockerProxy.ImageBuildException, IndexError) as e:
raise DockerProxy.ImageBuildException(e)
@staticmethod
def _decorate(some_line):
return some_line[11:-4].rstrip()
def image_exists(self, image_str):
"""Checks if an image with the given ID/tag exists locally."""
docker_image = DockerImage.from_string(image_str)
if docker_image.image_id is Constants.DockerNonExistentTag \
and docker_image.image_tag is Constants.DockerNonExistentTag:
raise InvalidDockerImageException("Neither image_id nor image_tag provided.")
for image in self.client.images():
some_id = image["Id"]
some_tags = image["RepoTags"] or [None]
if docker_image.image_id in \
some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]:
return True
if docker_image.image_tag in some_tags:
return True
return False
def terminate_containers(self, container_ids):
""" Terminates containers with given container ids."""
for container_id in container_ids:
try:
if self.container_status(container_id) == ProviderBase.STATUS_RUNNING:
self.stop_container(container_id)
self.terminate_container(container_id)
except NotFound:
pass
def terminate_container(self, container_id):
self.client.remove_container(container_id)
def get_mapped_ports(self, container_id):
container_ins = self.client.inspect_container(container_id)
mapped_ports = container_ins['HostConfig']['PortBindings']
ret_val = []
if mapped_ports is None:
logging.info("No mapped ports for {0}".format(container_id))
return
for k, v in mapped_ports.iteritems():
host_port = v[0]['HostPort']
ret_val.append(host_port)
return ret_val
def get_working_directory(self, container_id):
return self.client.inspect_container(container_id)["Config"]["WorkingDir"]
def get_home_directory(self, container_id):
env_vars = self.client.inspect_container(container_id)["Config"]["Env"]
home = [i for i in env_vars if i.startswith("HOME")]
return home[0].split("=")[1]
def put_archive(self, container_id, tar_file_bytes, target_path_in_container):
""" Copies and unpacks a given tarfile in the container at specified location.
Location must exist in container."""
if self.start_container(container_id) is False:
raise Exception("Could not start container.")
# Prepend file path with /home/ubuntu/. TODO Should be refined.
if not target_path_in_container.startswith("/home/ubuntu/"):
import os
target_path_in_container = os.path.join("/home/ubuntu/", target_path_in_container)
logging.info("target path in container: {0}".format(target_path_in_container))
if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes):
logging.error(DockerProxy.LOG_TAG + "Failed to copy.")
def get_container_ip_address(self, container_id):
""" Returns the IP Address of given container."""
self.start_container(container_id)
ins = self.client.inspect_container(container_id)
ip_address = str(ins.get("NetworkSettings").get("IPAddress"))
while True:
ip_address = str(ins.get("NetworkSettings").get("IPAddress"))
if ip_address == "":
time.sleep(3)
if ip_address.startswith("1") is True:
break
return ip_address
class HubIO:
""" :class:`HubIO` provides the way to interact with Jina Hub registry.
You can use it with CLI to package a directory into a Jina Hub image and publish it to the world.
Examples:
- :command:`jina hub build my_pod/` build the image
- :command:`jina hub build my_pod/ --push` build the image and push to the public registry
- :command:`jina hub pull jinahub/pod.dummy_mwu_encoder:0.0.6` to download the image
"""
def __init__(self, args: 'argparse.Namespace'):
self.logger = JinaLogger(self.__class__.__name__, **vars(args))
self.args = args
self._load_docker_client()
def _load_docker_client(self):
with ImportExtensions(
required=False,
help_text=
'missing "docker" dependency, available CLIs limited to "jina hub [list, new]"'
'to enable full CLI, please do pip install "jina[docker]"'):
import docker
from docker import APIClient
self._client: DockerClient = docker.from_env()
# low-level client
self._raw_client = APIClient(base_url='unix://var/run/docker.sock')
def new(self) -> None:
"""Create a new executor using cookiecutter template """
with ImportExtensions(required=True):
from cookiecutter.main import cookiecutter
import click # part of cookiecutter
cookiecutter_template = self.args.template
if self.args.type == 'app':
cookiecutter_template = 'https://github.com/jina-ai/cookiecutter-jina.git'
elif self.args.type == 'pod':
cookiecutter_template = 'https://github.com/jina-ai/cookiecutter-jina-hub.git'
try:
cookiecutter(cookiecutter_template,
overwrite_if_exists=self.args.overwrite,
output_dir=self.args.output_dir)
except click.exceptions.Abort:
self.logger.info('nothing is created, bye!')
def login(self) -> None:
"""Login using Github Device flow to allow push access to Jina Hub Registry"""
import requests
with resource_stream('jina', '/'.join(
('resources', 'hubapi.yml'))) as fp:
hubapi_yml = JAML.load(fp)
client_id = hubapi_yml['github']['client_id']
scope = hubapi_yml['github']['scope']
device_code_url = hubapi_yml['github']['device_code_url']
access_token_url = hubapi_yml['github']['access_token_url']
grant_type = hubapi_yml['github']['grant_type']
login_max_retry = hubapi_yml['github']['login_max_retry']
headers = {'Accept': 'application/json'}
code_request_body = {'client_id': client_id, 'scope': scope}
try:
self.logger.info(
'Jina Hub login will use Github Device to generate one time token'
)
response = requests.post(url=device_code_url,
headers=headers,
data=code_request_body)
if response.status_code != requests.codes.ok:
self.logger.error(
'cannot reach github server. please make sure you\'re connected to internet'
)
code_response = response.json()
device_code = code_response['device_code']
user_code = code_response['user_code']
verification_uri = code_response['verification_uri']
try:
webbrowser.open(verification_uri, new=2)
except:
pass # intentional pass, browser support isn't cross-platform
finally:
self.logger.info(
f'You should see a "Device Activation" page open in your browser. '
f'If not, please go to {colored(verification_uri, "cyan", attrs=["underline"])}'
)
self.logger.info(
'Please follow the steps:\n'
f'1. Enter the following code to that page: {colored(user_code, "cyan", attrs=["bold"])}\n'
'2. Click "Continue"\n'
'3. Come back to this terminal\n')
access_request_body = {
'client_id': client_id,
'device_code': device_code,
'grant_type': grant_type
}
for _ in range(login_max_retry):
access_token_response = requests.post(
url=access_token_url,
headers=headers,
data=access_request_body).json()
if access_token_response.get('error',
None) == 'authorization_pending':
self.logger.warning('still waiting for authorization')
countdown(10,
reason=colored('re-fetch access token',
'cyan',
attrs=['bold', 'reverse']))
elif 'access_token' in access_token_response:
token = {
'access_token': access_token_response['access_token']
}
with open(credentials_file(), 'w') as cf:
JAML.dump(token, cf)
self.logger.success(f'successfully logged in!')
break
else:
self.logger.error(f'max retries {login_max_retry} reached')
except KeyError as exp:
self.logger.error(f'can not read the key in response: {exp}')
def list(self) -> Dict[str, Any]:
""" List all hub images given a filter specified by CLI """
if self.args.local_only:
return _list_local(self.logger)
else:
return _list(logger=self.logger,
image_name=self.args.name,
image_kind=self.args.kind,
image_type=self.args.type,
image_keywords=self.args.keywords)
def push(self,
name: str = None,
readme_path: str = None,
build_result: Dict = None) -> None:
""" A wrapper of docker push
- Checks for the tempfile, returns without push if it cannot find
- Pushes to docker hub, returns withput writing to db if it fails
- Writes to the db
"""
name = name or self.args.name
try:
# check if image exists
# fail if it does
if self.args.no_overwrite and self._image_version_exists(
build_result['manifest_info']['name'],
build_result['manifest_info']['version'], jina_version):
raise ImageAlreadyExists(
f'Image with name {name} already exists. Will NOT overwrite.'
)
else:
self.logger.debug(
f'Image with name {name} does not exist. Pushing now...')
self._push_docker_hub(name, readme_path)
if not build_result:
file_path = get_summary_path(name)
if os.path.isfile(file_path):
with open(file_path) as f:
build_result = json.load(f)
else:
self.logger.error(
f'can not find the build summary file.'
f'please use "jina hub build" to build the image first '
f'before pushing.')
if build_result:
if build_result.get('is_build_success', False):
_register_to_mongodb(logger=self.logger,
summary=build_result)
if build_result.get('details', None) and build_result.get(
'build_history', None):
self._write_slack_message(build_result,
build_result['details'],
build_result['build_history'])
except Exception as e:
self.logger.error(
f'Error when trying to push image {name}: {repr(e)}')
if isinstance(e, ImageAlreadyExists):
raise e
def _push_docker_hub(self,
name: str = None,
readme_path: str = None) -> None:
""" Helper push function """
check_registry(self.args.registry, name, self.args.repository)
self._check_docker_image(name)
self._docker_login()
with ProgressBar(task_name=f'pushing {name}', batch_unit='') as t:
for line in self._client.images.push(name,
stream=True,
decode=True):
t.update(1)
if 'error' in line and 'authentication required' in line[
'error']:
raise DockerLoginFailed('user not logged in to docker.')
self.logger.debug(line)
self.logger.success(f'🎉 {name} is now published!')
if False and readme_path:
# unfortunately Docker Hub Personal Access Tokens cannot be used as they are not supported by the API
_volumes = {
os.path.dirname(os.path.abspath(readme_path)): {
'bind': '/workspace'
}
}
_env = {
'DOCKERHUB_USERNAME': self.args.username,
'DOCKERHUB_PASSWORD': self.args.password,
'DOCKERHUB_REPOSITORY': self.args.repository,
'README_FILEPATH': '/workspace/README.md',
}
self._client.containers.run('peterevans/dockerhub-description:2.1',
auto_remove=True,
volumes=_volumes,
environment=_env)
share_link = f'https://api.jina.ai/hub/?jh={urllib.parse.quote_plus(name)}'
try:
webbrowser.open(share_link, new=2)
except:
# pass intentionally, dont want to bother users on opening browser failure
pass
finally:
self.logger.info(
f'Check out the usage {colored(share_link, "cyan", attrs=["underline"])} and share it with others!'
)
def pull(self) -> None:
"""A wrapper of docker pull """
check_registry(self.args.registry, self.args.name,
self.args.repository)
try:
self._docker_login()
with TimeContext(f'pulling {self.args.name}', self.logger):
image = self._client.images.pull(self.args.name)
if isinstance(image, list):
image = image[0]
image_tag = image.tags[0] if image.tags else ''
self.logger.success(
f'🎉 pulled {image_tag} ({image.short_id}) uncompressed size: {get_readable_size(image.attrs["Size"])}'
)
except Exception as ex:
self.logger.error(
f'can not pull image {self.args.name} from {self.args.registry} due to {repr(ex)}'
)
def _check_docker_image(self, name: str) -> None:
# check local image
image = self._client.images.get(name)
for r in _allowed:
if f'{_label_prefix}{r}' not in image.labels.keys():
self.logger.warning(
f'{r} is missing in your docker image labels, you may want to check it'
)
try:
image.labels['ai.jina.hub.jina_version'] = jina_version
if name != safe_url_name(
f'{self.args.repository}/' +
'{type}.{kind}.{name}:{version}-{jina_version}'.format(
**{
k.replace(_label_prefix, ''): v
for k, v in image.labels.items()
})):
raise ValueError(
f'image {name} does not match with label info in the image'
)
except KeyError:
self.logger.error('missing key in the label of the image')
raise
self.logger.info(
f'✅ {name} is a valid Jina Hub image, ready to publish')
def _docker_login(self) -> None:
"""A wrapper of docker login """
from docker.errors import APIError
if self.args.username and self.args.password:
try:
self._client.login(username=self.args.username,
password=self.args.password,
registry=self.args.registry)
self.logger.debug(f'successfully logged in to docker hub')
except APIError:
raise DockerLoginFailed(
f'invalid credentials passed. docker login failed')
def build(self) -> Dict:
"""A wrapper of docker build """
if self.args.dry_run:
result = self.dry_run()
else:
is_build_success, is_push_success = True, False
_logs = []
_except_strs = []
_excepts = []
with TimeContext(f'building {colored(self.args.path, "green")}',
self.logger) as tc:
try:
self._check_completeness()
streamer = self._raw_client.build(
decode=True,
path=self.args.path,
tag=self.tag,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
for chunk in streamer:
if 'stream' in chunk:
for line in chunk['stream'].splitlines():
if is_error_message(line):
self.logger.critical(line)
_except_strs.append(line)
elif 'warning' in line.lower():
self.logger.warning(line)
else:
self.logger.info(line)
_logs.append(line)
except Exception as ex:
# if pytest fails it should end up here as well
is_build_success = False
ex = HubBuilderBuildError(ex)
_except_strs.append(repr(ex))
_excepts.append(ex)
if is_build_success:
# compile it again, but this time don't show the log
image, log = self._client.images.build(
path=self.args.path,
tag=self.tag,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
# success
_details = {
'inspect': self._raw_client.inspect_image(image.tags[0]),
'tag': image.tags[0],
'hash': image.short_id,
'size': get_readable_size(image.attrs['Size']),
}
self.logger.success(
'🎉 built {tag} ({hash}) uncompressed size: {size}'.
format_map(_details))
else:
self.logger.error(
f'can not build the image, please double check the log')
_details = {}
if is_build_success:
if self.args.test_uses:
p_names = []
try:
is_build_success = False
p_names, failed_test_levels = HubIO._test_build(
image, self.args.test_level, self.config_yaml_path,
self.args.timeout_ready, self.args.daemon)
if any(
test_level in failed_test_levels
for test_level in
[BuildTestLevel.POD_DOCKER, BuildTestLevel.FLOW]):
is_build_success = False
self.logger.error(
f'build unsuccessful, failed at {str(failed_test_levels)} level'
)
else:
is_build_success = True
self.logger.warning(
f'Build successful. Tests failed at : {str(failed_test_levels)} levels. This could be due to the fact that the executor has non-installed external dependencies'
)
except Exception as ex:
self.logger.error(
f'something wrong while testing the build: {repr(ex)}'
)
ex = HubBuilderTestError(ex)
_except_strs.append(repr(ex))
_excepts.append(ex)
finally:
if self.args.daemon:
try:
for p in p_names:
self._raw_client.stop(p)
except:
pass # suppress on purpose
self._raw_client.prune_containers()
_version = self.manifest[
'version'] if 'version' in self.manifest else '0.0.1'
info, env_info = get_full_version()
_host_info = {
'jina': info,
'jina_envs': env_info,
'docker': self._raw_client.info(),
'build_args': vars(self.args)
}
_build_history = {
'time':
get_now_timestamp(),
'host_info':
_host_info if is_build_success and self.args.host_info else '',
'duration':
tc.readable_duration,
'logs':
_logs,
'exception':
_except_strs
}
if self.args.prune_images:
self.logger.info('deleting unused images')
self._raw_client.prune_images()
result = {
'name':
getattr(self, 'canonical_name', ''),
'version':
self.manifest['version'] if is_build_success
and 'version' in self.manifest else '0.0.1',
'path':
self.args.path,
'manifest_info':
self.manifest if is_build_success else '',
'details':
_details,
'is_build_success':
is_build_success,
'build_history':
_build_history
}
# only successful build (NOT dry run) writes the summary to disk
if result['is_build_success']:
self._write_summary_to_file(summary=result)
if self.args.push:
self.push(image.tags[0], self.readme_path, result)
if not result['is_build_success'] and self.args.raise_error:
# remove the very verbose build log when throw error
result['build_history'].pop('logs')
raise HubBuilderError(_excepts)
return result
@staticmethod
def _test_build(
image, # type docker image object
test_level: 'BuildTestLevel',
config_yaml_path: str,
timeout_ready: int,
daemon_arg: bool):
p_names = []
failed_levels = []
# test uses at executor level
if test_level >= BuildTestLevel.EXECUTOR:
try:
with BaseExecutor.load_config(config_yaml_path):
pass
except:
failed_levels.append(BuildTestLevel.EXECUTOR)
# test uses at Pod level (no docker)
if test_level >= BuildTestLevel.POD_NONDOCKER:
try:
with Pod(set_pod_parser().parse_args([
'--uses', config_yaml_path, '--timeout-ready',
str(timeout_ready)
])):
pass
except:
failed_levels.append(BuildTestLevel.POD_NONDOCKER)
# test uses at Pod level (with docker)
if test_level >= BuildTestLevel.POD_DOCKER:
p_name = random_name()
try:
with Pod(set_pod_parser().parse_args([
'--uses', f'docker://{image.tags[0]}', '--name',
p_name, '--timeout-ready',
str(timeout_ready)
] + ['--daemon'] if daemon_arg else [])):
pass
p_names.append(p_name)
except:
failed_levels.append(BuildTestLevel.POD_DOCKER)
# test uses at Flow level
if test_level >= BuildTestLevel.FLOW:
p_name = random_name()
try:
with Flow().add(name=random_name(),
uses=f'docker://{image.tags[0]}',
daemon=daemon_arg,
timeout_ready=timeout_ready):
pass
p_names.append(p_name)
except:
failed_levels.append(BuildTestLevel.FLOW)
return p_names, failed_levels
def dry_run(self) -> Dict:
try:
s = self._check_completeness()
s['is_build_success'] = True
except Exception as ex:
s = {'is_build_success': False, 'exception': str(ex)}
return s
def _write_summary_to_file(self, summary: Dict) -> None:
file_path = get_summary_path(f'{summary["name"]}:{summary["version"]}')
with open(file_path, 'w+') as f:
json.dump(summary, f)
self.logger.debug(f'stored the summary from build to {file_path}')
def _check_completeness(self) -> Dict:
self.dockerfile_path = get_exist_path(self.args.path, 'Dockerfile')
self.manifest_path = get_exist_path(self.args.path, 'manifest.yml')
self.config_yaml_path = get_exist_path(self.args.path, 'config.yml')
self.readme_path = get_exist_path(self.args.path, 'README.md')
self.requirements_path = get_exist_path(self.args.path,
'requirements.txt')
yaml_glob = set(glob.glob(os.path.join(self.args.path, '*.yml')))
yaml_glob.difference_update(
{self.manifest_path, self.config_yaml_path})
if not self.config_yaml_path:
self.config_yaml_path = yaml_glob.pop()
py_glob = glob.glob(os.path.join(self.args.path, '*.py'))
test_glob = glob.glob(os.path.join(self.args.path, 'tests/test_*.py'))
completeness = {
'Dockerfile': self.dockerfile_path,
'manifest.yml': self.manifest_path,
'config.yml': self.config_yaml_path,
'README.md': self.readme_path,
'requirements.txt': self.requirements_path,
'*.yml': yaml_glob,
'*.py': py_glob,
'tests': test_glob
}
self.logger.info(f'completeness check\n' + '\n'.join(
f'{colored("✓", "green") if v else colored("✗", "red"):>4} {k:<20} {v}'
for k, v in completeness.items()) + '\n')
if completeness['Dockerfile'] and completeness['manifest.yml']:
pass
else:
self.logger.critical(
'Dockerfile or manifest.yml is not given, can not build')
raise FileNotFoundError(
'Dockerfile or manifest.yml is not given, can not build')
self.manifest = self._read_manifest(self.manifest_path)
self.manifest['jina_version'] = jina_version
self.dockerfile_path_revised = self._get_revised_dockerfile(
self.dockerfile_path, self.manifest)
tag_name = safe_url_name(
f'{self.args.repository}/' +
f'{self.manifest["type"]}.{self.manifest["kind"]}.{self.manifest["name"]}:{self.manifest["version"]}-{jina_version}'
)
self.tag = tag_name
self.canonical_name = tag_name
return completeness
def _read_manifest(self, path: str, validate: bool = True) -> Dict:
with resource_stream(
'jina', '/'.join(
('resources', 'hub-builder', 'manifest.yml'))) as fp:
tmp = JAML.load(
fp
) # do not expand variables at here, i.e. DO NOT USE expand_dict(yaml.load(fp))
with open(path) as fp:
tmp.update(JAML.load(fp))
if validate:
self._validate_manifest(tmp)
return tmp
def _validate_manifest(self, manifest: Dict) -> None:
required = {'name', 'type', 'version'}
# check the required field in manifest
for r in required:
if r not in manifest:
raise ValueError(
f'{r} is missing in the manifest.yaml, it is required')
# check if all fields are there
for r in _allowed:
if r not in manifest:
self.logger.warning(
f'{r} is missing in your manifest.yml, you may want to check it'
)
# check name
check_name(manifest['name'])
# check_image_type
check_image_type(manifest['type'])
# check version number
check_version(manifest['version'])
# check version number
check_license(manifest['license'])
# check platform
if not isinstance(manifest['platform'], list):
manifest['platform'] = list(manifest['platform'])
check_platform(manifest['platform'])
# replace all chars in value to safe chars
for k, v in manifest.items():
if v and isinstance(v, str):
manifest[k] = remove_control_characters(v)
# show manifest key-values
for k, v in manifest.items():
self.logger.debug(f'{k}: {v}')
def _get_revised_dockerfile(self, dockerfile_path: str,
manifest: Dict) -> str:
# modify dockerfile
revised_dockerfile = []
with open(dockerfile_path) as fp:
for l in fp:
revised_dockerfile.append(l)
if l.startswith('FROM'):
revised_dockerfile.append('LABEL ')
revised_dockerfile.append(' \\ \n'.join(
f'{_label_prefix}{k}="{v}"'
for k, v in manifest.items()))
f = tempfile.NamedTemporaryFile('w', delete=False).name
with open(f, 'w', encoding='utf8') as fp:
fp.writelines(revised_dockerfile)
for k in revised_dockerfile:
self.logger.debug(k)
return f
def _write_slack_message(self, *args):
def _expand_fn(v):
if isinstance(v, str):
for d in args:
try:
v = v.format(**d)
except KeyError:
pass
return v
if 'JINAHUB_SLACK_WEBHOOK' in os.environ:
with resource_stream(
'jina', '/'.join(('resources', 'hub-builder-success',
'slack-jinahub.json'))) as fp:
tmp = expand_dict(json.load(fp),
_expand_fn,
resolve_cycle_ref=False)
req = urllib.request.Request(
os.environ['JINAHUB_SLACK_WEBHOOK'])
req.add_header('Content-Type',
'application/json; charset=utf-8')
jdb = json.dumps(tmp).encode('utf-8') # needs to be bytes
req.add_header('Content-Length', str(len(jdb)))
with urllib.request.urlopen(req, jdb) as f:
res = f.read()
self.logger.info(f'push to Slack: {res}')
# alias of "new" in cli
create = new
init = new
def _image_version_exists(self, name, module_version, req_jina_version):
manifests = _list(self.logger, name)
# check if matching module version and jina version exists
if manifests:
matching = [
m for m in manifests
if m['version'] == module_version and 'jina_version' in
m.keys() and m['jina_version'] == req_jina_version
]
return len(matching) > 0
return False
class LocalContainerBuilder(ContainerBuilder):
"""Container builder that uses local docker daemon process."""
def get_docker_image(
self, max_status_check_attempts=None, delay_between_status_checks=None
):
"""Builds, publishes and returns a docker image.
Args:
max_status_check_attempts: Maximum number of times allowed to check
build status. Not applicable to this builder.
delay_between_status_checks: Time is seconds to wait between status
checks. Not applicable to this builder.
"""
self.docker_client = APIClient(version="auto")
self._get_tar_file_path()
# create docker image from tarball
image_uri = self._build_docker_image()
# push to the registry
self._publish_docker_image(image_uri)
return image_uri
def _build_docker_image(self):
"""Builds docker image."""
image_uri = self._generate_name()
logger.info("Building docker image: {}".format(image_uri))
# `fileobj` is generally set to the Dockerfile file path. If a tar file
# is used for docker build context (ones that includes a Dockerfile)
# then `custom_context` should be enabled.
with open(self.tar_file_path, "rb") as fileobj:
bld_logs_generator = self.docker_client.build(
path=".",
custom_context=True,
fileobj=fileobj,
tag=image_uri,
encoding="utf-8",
decode=True,
)
self._get_logs(bld_logs_generator, "build", image_uri)
return image_uri
def _publish_docker_image(self, image_uri):
"""Publishes docker image.
Args:
image_uri: String, the registry name and tag.
"""
logger.info("Publishing docker image: {}".format(image_uri))
pb_logs_generator = self.docker_client.push(image_uri, stream=True, decode=True)
self._get_logs(pb_logs_generator, "publish", image_uri)
def _get_logs(self, logs_generator, name, image_uri):
"""Decodes logs from docker and generates user friendly logs.
Args:
logs_generator: Generator returned from docker build/push APIs.
name: String, 'build' or 'publish' used to identify where the
generator came from.
image_uri: String, the docker image URI.
Raises:
RuntimeError: if there are any errors when building or publishing a
docker image.
"""
for chunk in logs_generator:
if "stream" in chunk:
for line in chunk["stream"].splitlines():
logger.info(line)
if "error" in chunk:
raise RuntimeError(
"Docker image {} failed: {}\nImage URI: {}".format(
name, str(chunk["error"]), image_uri
)
)
class BaseDockerBuilder(object):
CHECK_INTERVAL = 10
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
def __init__(self,
repo_path,
from_image,
image_name,
image_tag,
copy_code=True,
in_tmp_repo=True,
build_steps=None,
env_vars=None,
dockerfile_name='Dockerfile'):
# This will help create a unique tmp folder for dockerizer in case of concurrent jobs
self.uuid = uuid.uuid4().hex
self.from_image = from_image
self.image_name = image_name
self.image_tag = image_tag
self.repo_path = repo_path
self.folder_name = repo_path.split('/')[-1]
self.copy_code = copy_code
self.in_tmp_repo = in_tmp_repo
if in_tmp_repo and copy_code:
self.build_repo_path = self.create_tmp_repo()
else:
self.build_repo_path = self.repo_path
self.build_path = '/'.join(self.build_repo_path.split('/')[:-1])
self.build_steps = get_list(build_steps)
self.env_vars = get_list(env_vars)
self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
self.polyaxon_requirements_path = self._get_requirements_path()
self.polyaxon_setup_path = self._get_setup_path()
self.docker = APIClient(version='auto')
self.registry_host = None
self.docker_url = None
def get_tagged_image(self):
return '{}:{}'.format(self.image_name, self.image_tag)
def create_tmp_repo(self):
# Create a tmp copy of the repo before starting the build
return copy_to_tmp_dir(path=self.repo_path,
dir_name=os.path.join(self.uuid, self.image_tag, self.folder_name))
def check_image(self):
return self.docker.images(self.get_tagged_image())
def clean(self):
# Clean dockerfile
delete_path(self.dockerfile_path)
# Clean tmp dir if created
if self.in_tmp_repo and self.copy_code:
delete_tmp_dir(self.image_tag)
def login(self, registry_user, registry_password, registry_host):
try:
self.docker.login(username=registry_user,
password=registry_password,
registry=registry_host,
reauth=True)
except DockerException as e:
_logger.exception('Failed to connect to registry %s\n', e)
def _handle_logs(self, log_line):
raise NotImplementedError
def _check_pulse(self, check_pulse):
"""Checks if the job/experiment is still running.
returns:
* int: the updated check_pulse (+1) value
* boolean: if the docker process should stop
"""
raise NotImplementedError
def _get_requirements_path(self):
requirements_path = os.path.join(self.build_repo_path, 'polyaxon_requirements.txt')
if os.path.isfile(requirements_path):
return os.path.join(self.folder_name, 'polyaxon_requirements.txt')
return None
def _get_setup_path(self):
setup_file_path = os.path.join(self.build_repo_path, 'polyaxon_setup.sh')
has_setup = os.path.isfile(setup_file_path)
if has_setup:
st = os.stat(setup_file_path)
os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
return os.path.join(self.folder_name, 'polyaxon_setup.sh')
return None
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
from_image=self.from_image,
polyaxon_requirements_path=self.polyaxon_requirements_path,
polyaxon_setup_path=self.polyaxon_setup_path,
build_steps=self.build_steps,
env_vars=self.env_vars,
folder_name=self.folder_name,
workdir=self.WORKDIR,
nvidia_bin=settings.MOUNT_PATHS_NVIDIA.get('bin'),
copy_code=self.copy_code
)
def build(self, memory_limit=None):
_logger.debug('Starting build in `%s`', self.build_repo_path)
# Checkout to the correct commit
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path, commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
dockerfile.write(self.render())
check_pulse = 0
for log_line in self.docker.build(
path=self.build_path,
tag=self.get_tagged_image(),
buildargs={},
decode=True,
forcerm=True,
rm=True,
pull=True,
nocache=False,
container_limits=limits,
):
self._handle_logs(log_line)
# Check if we need to stop this process
check_pulse, should_stop = self._check_pulse(check_pulse)
if should_stop:
return False
# Checkout back to master
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.build_repo_path)
return True
def push(self):
# Build a progress setup for each layer, and only emit per-layer info every 1.5s
layers = {}
last_emit_time = time.time()
check_pulse = 0
for log_line in self.docker.push(self.image_name, tag=self.image_tag, stream=True):
lines = [l for l in log_line.decode('utf-8').split('\r\n') if l]
lines = [json.loads(l) for l in lines]
for progress in lines:
if 'error' in progress:
_logger.error(progress['error'], extra=dict(phase='failed'))
return
if 'id' not in progress:
continue
if 'progressDetail' in progress and progress['progressDetail']:
layers[progress['id']] = progress['progressDetail']
else:
layers[progress['id']] = progress['status']
if time.time() - last_emit_time > 1.5:
_logger.debug('Pushing image\n', extra=dict(progress=layers, phase='pushing'))
last_emit_time = time.time()
self._handle_logs(log_line)
# Check if we need to stop this process
check_pulse, should_stop = self._check_pulse(check_pulse)
if should_stop:
return False
return True
class DockerImage:
"""
The DockerImage class has the functions and attributes for building the dockerimage
"""
def __init__(
self,
info,
dockerfile,
repository,
tag,
to_build,
context=None,
):
# Meta-data about the image should go to info.
# All keys in info are accessible as attributes
# of this class
self.info = info
self.summary = {}
self.build_args = {}
self.labels = {}
self.dockerfile = dockerfile
self.context = context
# TODO: Add ability to tag image with multiple tags
self.repository = repository
self.tag = tag
self.ecr_url = f"{self.repository}:{self.tag}"
if not isinstance(to_build, bool):
to_build = True if to_build == "true" else False
self.to_build = to_build
self.build_status = None
self.client = APIClient(base_url=constants.DOCKER_URL)
self.log = []
def __getattr__(self, name):
return self.info[name]
def collect_installed_packages_information(self):
"""
Returns an array with outcomes of the commands listed in the 'commands' array
"""
docker_client = DockerClient(base_url=constants.DOCKER_URL)
command_responses = []
commands = [
"pip list", "dpkg-query -Wf '${Installed-Size}\\t${Package}\\n'",
"apt list --installed"
]
for command in commands:
command_responses.append(f"\n{command}")
command_responses.append(
bytes.decode(
docker_client.containers.run(self.ecr_url, command)))
docker_client.containers.prune()
return command_responses
def build(self):
"""
The build function builds the specified docker image
"""
self.summary["start_time"] = datetime.now()
if not self.to_build:
self.log = ["Not built"]
self.build_status = constants.NOT_BUILT
self.summary["status"] = constants.STATUS_MESSAGE[
self.build_status]
return self.build_status
if self.info.get("base_image_uri"):
self.build_args["BASE_IMAGE"] = self.info["base_image_uri"]
if self.info.get("extra_build_args"):
self.build_args.update(self.info.get("extra_build_args"))
if self.info.get("labels"):
self.labels.update(self.info.get("labels"))
with open(self.context.context_path, "rb") as context_file:
response = []
for line in self.client.build(fileobj=context_file,
path=self.dockerfile,
custom_context=True,
rm=True,
decode=True,
tag=self.ecr_url,
buildargs=self.build_args,
labels=self.labels):
if line.get("error") is not None:
self.context.remove()
response.append(line["error"])
self.log = response
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[
self.build_status]
self.summary["end_time"] = datetime.now()
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
elif line.get("status") is not None:
response.append(line["status"])
else:
response.append(str(line))
self.context.remove()
self.summary["image_size"] = int(
self.client.inspect_image(
self.ecr_url)["Size"]) / (1024 * 1024)
if self.summary[
"image_size"] > self.info["image_size_baseline"] * 1.20:
response.append("Image size baseline exceeded")
response.append(
f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}"
)
response += self.collect_installed_packages_information()
self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT
else:
self.build_status = constants.SUCCESS
for line in self.client.push(self.repository,
self.tag,
stream=True,
decode=True):
if line.get("error") is not None:
response.append(line["error"])
self.log = response
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[
self.build_status]
self.summary["end_time"] = datetime.now()
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
else:
response.append(str(line))
self.summary["status"] = constants.STATUS_MESSAGE[
self.build_status]
self.summary["end_time"] = datetime.now()
self.summary["ecr_url"] = self.ecr_url
self.log = response
return self.build_status
class DockerImage:
"""
The DockerImage class has the functions and attributes for building the dockerimage
"""
def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True, additional_tags=[], target=None):
# Meta-data about the image should go to info.
# All keys in info are accessible as attributes
# of this class
self.info = info
self.summary = {}
self.build_args = {}
self.labels = {}
self.stage = stage
self.dockerfile = dockerfile
self.context = context
self.to_push = to_push
# TODO: Add ability to tag image with multiple tags
self.repository = repository
self.tag = tag
self.additional_tags = additional_tags
self.ecr_url = f"{self.repository}:{self.tag}"
if not isinstance(to_build, bool):
to_build = True if to_build == "true" else False
self.to_build = to_build
self.build_status = None
self.client = APIClient(base_url=constants.DOCKER_URL, timeout=constants.API_CLIENT_TIMEOUT)
self.log = []
self._corresponding_common_stage_image = None
self.target = target
def __getattr__(self, name):
return self.info[name]
@property
def is_child_image(self):
"""
If we require a base image URI, the image is a child image (where the base image is the parent)
"""
return bool(self.info.get('base_image_uri'))
@property
def is_test_promotion_enabled(self):
return bool(self.info.get('enable_test_promotion'))
@property
def corresponding_common_stage_image(self):
"""
Retrieve the corresponding common stage image for a given image.
"""
return self._corresponding_common_stage_image
@corresponding_common_stage_image.setter
def corresponding_common_stage_image(self, docker_image_object):
"""
For a pre-push stage image, it sets the value for the corresponding_common_stage_image variable.
"""
if self.to_push:
raise ValueError(
"For any pre-push stage image, corresponding common stage image should only exist if the pre-push stage image is non-pushable."
)
self._corresponding_common_stage_image = docker_image_object
def collect_installed_packages_information(self):
"""
Returns an array with outcomes of the commands listed in the 'commands' array
"""
docker_client = DockerClient(base_url=constants.DOCKER_URL)
command_responses = []
commands = ["pip list", "dpkg-query -Wf '${Installed-Size}\\t${Package}\\n'", "apt list --installed"]
for command in commands:
command_responses.append(f"\n{command}")
command_responses.append(bytes.decode(docker_client.containers.run(self.ecr_url, command)))
docker_client.containers.prune()
return command_responses
def get_tail_logs_in_pretty_format(self, number_of_lines=10):
"""
Displays the tail of the logs.
:param number_of_lines: int, number of ending lines to be printed
:return: str, last number_of_lines of the logs concatenated with a new line
"""
return "\n".join(self.log[-1][-number_of_lines:])
def update_pre_build_configuration(self):
"""
Updates image configuration before the docker client starts building the image.
"""
if self.info.get("base_image_uri"):
self.build_args["BASE_IMAGE"] = self.info["base_image_uri"]
if self.info.get("extra_build_args"):
self.build_args.update(self.info.get("extra_build_args"))
if self.info.get("labels"):
self.labels.update(self.info.get("labels"))
def build(self):
"""
The build function sets the stage for starting the docker build process for a given image.
:return: int, Build Status
"""
self.summary["start_time"] = datetime.now()
# Confirm if building the image is required or not
if not self.to_build:
self.log.append(["Not built"])
self.build_status = constants.NOT_BUILT
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
return self.build_status
# Conduct some preprocessing before building the image
self.update_pre_build_configuration()
# Start building the image
with open(self.context.context_path, "rb") as context_file:
self.docker_build(fileobj=context_file, custom_context=True)
self.context.remove()
if self.build_status != constants.SUCCESS:
return self.build_status
if not self.to_push:
# If this image is not supposed to be pushed, in that case, we are already done
# with building the image and do not need to conduct any further processing.
self.summary["end_time"] = datetime.now()
# check the size after image is built.
self.image_size_check()
# This return is necessary. Otherwise FORMATTER fails while displaying the status.
return self.build_status
def docker_build(self, fileobj=None, custom_context=False):
"""
Uses low level Docker API Client to actually start the process of building the image.
:param fileobj: FileObject, a readable file-like object pointing to the context tarfile.
:param custom_context: bool
:return: int, Build Status
"""
response = [f"Starting the Build Process for {self.repository}:{self.tag}"]
for line in self.client.build(
fileobj=fileobj,
path=self.dockerfile,
custom_context=custom_context,
rm=True,
decode=True,
tag=self.ecr_url,
buildargs=self.build_args,
labels=self.labels,
target=self.target,
):
if line.get("error") is not None:
response.append(line["error"])
self.log.append(response)
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
LOGGER.info(f"Docker Build Logs: \n {self.get_tail_logs_in_pretty_format(100)}")
LOGGER.error("ERROR during Docker BUILD")
LOGGER.error(f"Error message received for {self.dockerfile} while docker build: {line}")
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
elif line.get("status") is not None:
response.append(line["status"])
else:
response.append(str(line))
self.log.append(response)
LOGGER.info(f"DOCKER BUILD LOGS: \n{self.get_tail_logs_in_pretty_format()}")
LOGGER.info(f"Completed Build for {self.repository}:{self.tag}")
self.build_status = constants.SUCCESS
return self.build_status
def image_size_check(self):
"""
Checks if the size of the image is not greater than the baseline.
:return: int, Build Status
"""
response = [f"Starting image size check for {self.repository}:{self.tag}"]
self.summary["image_size"] = int(self.client.inspect_image(self.ecr_url)["Size"]) / (1024 * 1024)
if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20:
response.append("Image size baseline exceeded")
response.append(f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}")
response += self.collect_installed_packages_information()
self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT
else:
response.append(f"Image Size Check Succeeded for {self.repository}:{self.tag}")
self.build_status = constants.SUCCESS
self.log.append(response)
LOGGER.info(f"{self.get_tail_logs_in_pretty_format()}")
return self.build_status
def push_image(self, tag_value=None):
"""
Pushes the Docker image to ECR using Docker low-level API client for docker.
:param tag_value: str, an optional variable to provide a different tag
:return: int, states if the Push was successful or not
"""
tag = tag_value
if tag_value is None:
tag = self.tag
response = [f"Starting image Push for {self.repository}:{tag}"]
for line in self.client.push(self.repository, tag, stream=True, decode=True):
if line.get("error") is not None:
response.append(line["error"])
self.log.append(response)
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
LOGGER.info(f"Docker Build Logs: \n {self.get_tail_logs_in_pretty_format(100)}")
LOGGER.error("ERROR during Docker PUSH")
LOGGER.error(f"Error message received for {self.repository}:{tag} while docker push: {line}")
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
else:
response.append(str(line))
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
self.summary["ecr_url"] = self.ecr_url
if "pushed_uris" not in self.summary:
self.summary["pushed_uris"] = []
self.summary["pushed_uris"].append(f"{self.repository}:{tag}")
response.append(f"Completed Push for {self.repository}:{tag}")
self.log.append(response)
LOGGER.info(f"DOCKER PUSH LOGS: \n {self.get_tail_logs_in_pretty_format(2)}")
return self.build_status
def push_image_with_additional_tags(self):
"""
Pushes an already built Docker image by applying additional tags to it.
:return: int, states if the Push was successful or not
"""
self.log.append([f"Started Tagging for {self.ecr_url}"])
for additional_tag in self.additional_tags:
response = [f"Tagging {self.ecr_url} as {self.repository}:{additional_tag}"]
tagging_successful = self.client.tag(self.ecr_url, self.repository, additional_tag)
if not tagging_successful:
response.append(f"Tagging {self.ecr_url} with {additional_tag} unsuccessful.")
self.log.append(response)
LOGGER.error("ERROR during Tagging")
LOGGER.error(f"Tagging {self.ecr_url} with {additional_tag} unsuccessful.")
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
return self.build_status
response.append(f"Tagged {self.ecr_url} succussefully as {self.repository}:{additional_tag}")
self.log.append(response)
self.build_status = self.push_image(tag_value=additional_tag)
if self.build_status != constants.SUCCESS:
return self.build_status
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
self.log.append([f"Completed Tagging for {self.ecr_url}"])
LOGGER.info(f"DOCKER TAG and PUSH LOGS: \n {self.get_tail_logs_in_pretty_format(5)}")
return self.build_status
def main(self, user, dir, cache, pull, rm, detach, env, wait, push, run,
build, repo_name, port, volume, network, dockerfile, verbose):
client = docker.from_env()
api_client = APIClient(base_url='unix://var/run/docker.sock')
name = f'{user}_{env}'
tag = f'{user}:{env}'
if build:
dockerfile = f'Dockerfile.{env}' if dockerfile is None else dockerfile
build_kwargs = {
'path': dir,
'tag': tag,
'dockerfile': dockerfile,
'buildargs': {
'USER': user,
},
'nocache': not cache,
'pull': pull,
'rm': rm,
'decode': True,
}
if verbose >= 2:
print('Build Args:')
[
print(f'\t{key}: {build_kwargs[key]}')
for key in build_kwargs.keys()
]
image = api_client.build(**build_kwargs)
build_success = False
if verbose >= 1:
print('\nBuild Log:')
for i in image:
if 'stream' in i:
if 'Successfully built' in i['stream']:
build_success = True
[
print(f'\t{line.strip()}')
for line in i['stream'].splitlines()
if line.strip().strip('\n') != ''
]
if not build_success:
raise BuildRunError(101, 'Build Failed')
if run:
run_kwargs = {
'image': tag,
'name': name,
'user': user,
'ports': {p.split(':')[0]: p.split(':')[1]
for p in port},
'volumes':
{v.split(':')[0]: {
'bind': v.split(':')[1]
}
for v in volume},
'detach': detach,
'network': network,
}
if verbose >= 2:
print('\nRun Args:')
[
print(f'\t{key}: {run_kwargs[key]}')
for key in run_kwargs.keys()
]
all_containers = api_client.containers(
all=True,
filters={
'name': f'^{name}$',
},
)
if all_containers:
if verbose >= 1:
print(f'\nContainers exist for {name}...')
for container in all_containers:
container_id = container['Id']
if container['State'].lower() == 'running':
if verbose >= 1:
print(
f'\tStopping running container: {container_id}'
)
api_client.stop(container_id)
rename = f'{name}_{datetime.fromtimestamp(container["Created"])}'.replace(
' ', '_').replace(':', '_')
count = 1
while api_client.containers(
all=True, filters={'name': f'^{rename}$'}):
rename = f'{name}_{datetime.fromtimestamp(container["Created"])}_{count}'.replace(
' ', '_').replace(':', '_')
count += 1
if verbose >= 1:
print(f'\tRenaming to {rename} (id: {container_id})')
api_client.rename(container_id, rename)
else:
if verbose >= 1:
print(f'No existing containers for {name}.')
container = client.containers.run(**run_kwargs)
if verbose >= 1:
print(f'\nStarted container: {container.id}')
print('\nConnect with:')
print(f'\tdocker exec -it {name} bash')
print('\nView logs with:')
print(f'\tdocker logs {name}')
print(
f'\nWaiting {str(wait)} seconds to ensure container stays up...'
)
run_success = True
while wait >= 0:
if api_client.containers(filters={'name': f'^{name}$'}):
print(f'\r\t{wait}: Running ', end='', flush=True)
wait -= 1
sleep(1)
else:
print(f'\r\t{wait}: Died ', flush=True)
run_success = False
break
if run_success:
print(f'\r\tSuccess! ', flush=True)
if push:
if api_client.tag(tag, f'{repo_name}/{user}', tag=env):
if verbose >= 1:
print(f'Tagged image as {repo_name}/{user}:{env}')
else:
raise BuildRunError(
102, f'Failed to tag image as {repo_name}/{user}:{env}')
try:
[
print(f'\t{line["status"]}: {line["progressDetail"]}')
for line in api_client.push(f'{repo_name}/{user}',
tag=env,
stream=True,
decode=True) if verbose >= 2
and 'status' in line and 'progressDetail' in line
]
except docker.errors.APIError:
raise BuildRunError(
103, f'Failed to push {repo_name}/{user}:{env}')
if verbose >= 1:
print(f'Pushed image: {repo_name}/{user}:{env}')
def build(self):
f = BytesIO(self.dockerfile.encode('utf-8'))
cli = APIClient(base_url=self.__daemon)
return cli.build(fileobj=f, rm=True, tag=self.__tag)
class DockerBuilder(object):
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
def __init__(self,
build_context: str,
image_name: str,
image_tag: str,
copy_code: bool = True,
dockerfile_name: str = 'Dockerfile') -> None:
self.image_name = image_name
self.image_tag = image_tag
self.copy_code = copy_code
self.build_context = build_context
self.dockerfile_path = os.path.join(self.build_context, dockerfile_name)
self.docker = APIClient(version='auto')
self.is_pushing = False
def get_tagged_image(self) -> str:
return '{}:{}'.format(self.image_name, self.image_tag)
def check_image(self) -> Any:
return self.docker.images(self.get_tagged_image())
def clean(self) -> None:
pass
def login_internal_registry(self) -> None:
try:
self.docker.login(username=settings.REGISTRY_USER,
password=settings.REGISTRY_PASSWORD,
registry=settings.REGISTRY_URI,
reauth=True)
except DockerException as e:
_logger.exception('Failed to connect to registry %s\n', e)
def login_private_registries(self) -> None:
if not settings.PRIVATE_REGISTRIES:
return
for registry in settings.PRIVATE_REGISTRIES:
self.docker.login(username=registry.user,
password=registry.password,
registry=registry.host,
reauth=True)
def _prepare_log_lines(self, # pylint:disable=too-many-branches
log_line) -> Tuple[List[str], bool]:
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
status = True
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
log_lines.append('{}: {}'.format(
LogLevels.ERROR, str(json_line.get('error', json_line))))
status = False
else:
if json_line.get('stream'):
log_lines.append('Building: {}'.format(json_line['stream'].strip()))
elif json_line.get('status'):
if not self.is_pushing:
self.is_pushing = True
log_lines.append('Pushing ...')
elif json_line.get('aux'):
log_lines.append('Pushing finished: {}'.format(json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines, status
def _handle_logs(self, log_lines) -> None:
for log_line in log_lines:
print(log_line)
def _handle_log_stream(self, stream) -> bool:
log_lines = []
status = True
try:
for log_line in stream:
new_log_lines, new_status = self._prepare_log_lines(log_line)
log_lines += new_log_lines
if not new_status:
status = new_status
self._handle_logs(log_lines)
log_lines = []
if log_lines:
self._handle_logs(log_lines)
except (BuildError, APIError) as e:
self._handle_logs('{}: Could not build the image, '
'encountered {}'.format(LogLevels.ERROR, e))
return False
return status
def build(self, nocache: bool = False, memory_limit: Any = None) -> bool:
limits = {
# Disable memory swap for building
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
stream = self.docker.build(
path=self.build_context,
tag=self.get_tagged_image(),
forcerm=True,
rm=True,
pull=True,
nocache=nocache,
container_limits=limits)
return self._handle_log_stream(stream=stream)
def push(self) -> bool:
stream = self.docker.push(self.image_name, tag=self.image_tag, stream=True)
return self._handle_log_stream(stream=stream)
class HubIO:
""" :class:`HubIO` provides the way to interact with Jina Hub registry.
You can use it with CLI to package a directory into a Jina Hub image and publish it to the world.
Examples:
- :command:`jina hub build my_pod/` build the image
- :command:`jina hub build my_pod/ --push` build the image and push to the public registry
- :command:`jina hub pull jinahub/pod.dummy_mwu_encoder:0.0.6` to download the image
"""
def __init__(self, args: 'argparse.Namespace'):
self.logger = get_logger(self.__class__.__name__, **vars(args))
self.args = args
try:
import docker
from docker import APIClient
self._client = docker.from_env()
# low-level client
self._raw_client = APIClient(base_url='unix://var/run/docker.sock')
except (ImportError, ModuleNotFoundError):
self.logger.critical(
'requires "docker" dependency, please install it via "pip install jina[docker]"'
)
raise
def new(self):
"""Create a new executor using cookiecutter template """
try:
from cookiecutter.main import cookiecutter
except (ImportError, ModuleNotFoundError):
self.logger.critical(
'requires "cookiecutter" dependency, please install it via "pip install cookiecutter"'
)
raise
cookiecutter(self.args.template,
overwrite_if_exists=self.args.overwrite,
output_dir=self.args.output_dir)
def push(self, name: str = None, readme_path: str = None):
"""A wrapper of docker push """
name = name or self.args.name
check_registry(self.args.registry, name, _repo_prefix)
self._check_docker_image(name)
self.login()
with ProgressBar(task_name=f'pushing {name}', batch_unit='') as t:
for line in self._client.images.push(name,
stream=True,
decode=True):
t.update(1)
self.logger.debug(line)
self.logger.success(f'🎉 {name} is now published!')
if False and readme_path:
# unfortunately Docker Hub Personal Access Tokens cannot be used as they are not supported by the API
_volumes = {
os.path.dirname(os.path.abspath(readme_path)): {
'bind': '/workspace'
}
}
_env = get_default_login()
_env = {
'DOCKERHUB_USERNAME': _env['username'],
'DOCKERHUB_PASSWORD': _env['password'],
'DOCKERHUB_REPOSITORY': name.split(':')[0],
'README_FILEPATH': '/workspace/README.md',
}
self._client.containers.run('peterevans/dockerhub-description:2.1',
auto_remove=True,
volumes=_volumes,
environment=_env)
share_link = f'https://api.jina.ai/hub/?jh={urllib.parse.quote_plus(name)}'
try:
webbrowser.open(share_link, new=2)
except:
pass
finally:
self.logger.info(
f'Check out the usage {colored(share_link, "cyan", attrs=["underline"])} and share it with others!'
)
def pull(self):
"""A wrapper of docker pull """
check_registry(self.args.registry, self.args.name, _repo_prefix)
self.login()
with TimeContext(f'pulling {self.args.name}', self.logger):
image = self._client.images.pull(self.args.name)
if isinstance(image, list):
image = image[0]
image_tag = image.tags[0] if image.tags else ""
self.logger.success(
f'🎉 pulled {image_tag} ({image.short_id}) uncompressed size: {get_readable_size(image.attrs["Size"])}'
)
def _check_docker_image(self, name: str):
# check local image
image = self._client.images.get(name)
for r in _allowed:
if f'{_label_prefix}{r}' not in image.labels.keys():
self.logger.warning(
f'{r} is missing in your docker image labels, you may want to check it'
)
try:
if name != safe_url_name(f'{_repo_prefix}' +
'{type}.{kind}.{name}:{version}'.format(
**{
k.replace(_label_prefix, ''): v
for k, v in image.labels.items()
})):
raise ValueError(
f'image {name} does not match with label info in the image'
)
except KeyError:
self.logger.error('missing key in the label of the image')
raise
self.logger.info(
f'✅ {name} is a valid Jina Hub image, ready to publish')
def login(self):
"""A wrapper of docker login """
try:
password = self.args.password # or (self.args.password_stdin and self.args.password_stdin.read())
except ValueError:
password = ''
if self.args.username and password:
self._client.login(username=self.args.username,
password=password,
registry=self.args.registry)
else:
# use default login
self._client.login(**get_default_login(),
registry=self.args.registry)
def build(self):
"""A wrapper of docker build """
self._check_completeness()
is_build_success = True
with TimeContext(f'building {colored(self.canonical_name, "green")}',
self.logger):
streamer = self._raw_client.build(
decode=True,
path=self.args.path,
tag=self.canonical_name,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
for chunk in streamer:
if 'stream' in chunk:
for line in chunk['stream'].splitlines():
if 'error' in line.lower():
self.logger.critical(line)
is_build_success = False
elif 'warning' in line.lower():
self.logger.warning(line)
else:
self.logger.info(line)
if is_build_success:
# compile it again, but this time don't show the log
image, log = self._client.images.build(
path=self.args.path,
tag=self.canonical_name,
pull=self.args.pull,
dockerfile=self.dockerfile_path_revised,
rm=True)
# success
self.logger.success(
f'🎉 built {image.tags[0]} ({image.short_id}) uncompressed size: {get_readable_size(image.attrs["Size"])}'
)
if self.args.push:
self.push(image.tags[0], self.readme_path)
else:
self.logger.error(
f'can not build the image, please double check the log')
def _check_completeness(self):
self.dockerfile_path = get_exist_path(self.args.path, 'Dockerfile')
self.manifest_path = get_exist_path(self.args.path, 'manifest.yml')
self.readme_path = get_exist_path(self.args.path, 'README.md')
self.requirements_path = get_exist_path(self.args.path,
'requirements.txt')
yaml_glob = glob.glob(os.path.join(self.args.path, '*.yml'))
if yaml_glob:
yaml_glob.remove(self.manifest_path)
py_glob = glob.glob(os.path.join(self.args.path, '*.py'))
test_glob = glob.glob(os.path.join(self.args.path, 'tests/test_*.py'))
completeness = {
'Dockerfile': self.dockerfile_path,
'manifest.yml': self.manifest_path,
'README.md': os.path.exists(self.readme_path),
'requirements.txt': os.path.exists(self.readme_path),
'*.yml': yaml_glob,
'*.py': py_glob,
'tests': test_glob
}
self.logger.info(f'a completeness check\n' + '\n'.join(
'%4s %-20s %s' %
(colored('✓', 'green') if v else colored('✗', 'red'), k, v)
for k, v in completeness.items()) + '\n')
if completeness['Dockerfile'] and completeness['manifest.yml']:
pass
else:
self.logger.critical(
'Dockerfile or manifest.yml is not given, can not build')
raise FileNotFoundError(
'Dockerfile or manifest.yml is not given, can not build')
tmp = self._read_manifest(self.manifest_path)
self.dockerfile_path_revised = self._get_revised_dockerfile(
self.dockerfile_path, tmp)
self.canonical_name = safe_url_name(
f'{_repo_prefix}' + '{type}.{kind}.{name}:{version}'.format(**tmp))
def _read_manifest(self, path: str, validate: bool = True):
with resource_stream(
'jina', '/'.join(
('resources', 'hub-builder', 'manifest.yml'))) as fp:
tmp = yaml.load(
fp
) # do not expand variables at here, i.e. DO NOT USE expand_dict(yaml.load(fp))
with open(path) as fp:
tmp.update(yaml.load(fp))
if validate:
self._validate_manifest(tmp)
return tmp
def _validate_manifest(self, manifest: Dict):
required = {'name', 'type', 'version'}
# check the required field in manifest
for r in required:
if r not in manifest:
raise ValueError(
f'{r} is missing in the manifest.yaml, it is required')
# check if all fields are there
for r in _allowed:
if r not in manifest:
self.logger.warning(
f'{r} is missing in your manifest.yml, you may want to check it'
)
# check name
check_name(manifest['name'])
# check_image_type
check_image_type(manifest['type'])
# check version number
check_version(manifest['version'])
# check version number
check_license(manifest['license'])
# check platform
if not isinstance(manifest['platform'], list):
manifest['platform'] = list(manifest['platform'])
check_platform(manifest['platform'])
# replace all chars in value to safe chars
for k, v in manifest.items():
if v and isinstance(v, str):
manifest[k] = remove_control_characters(v)
elif v and isinstance(v, list):
manifest[k] = ','.join(v)
# show manifest key-values
for k, v in manifest.items():
self.logger.debug(f'{k}: {v}')
def _get_revised_dockerfile(self, dockerfile_path: str, manifest: Dict):
# modify dockerfile
revised_dockerfile = []
with open(dockerfile_path) as fp:
for l in fp:
revised_dockerfile.append(l)
if l.startswith('FROM'):
revised_dockerfile.append('LABEL ')
revised_dockerfile.append(' \\ \n'.join(
f'{_label_prefix}{k}="{v}"'
for k, v in manifest.items()))
f = tempfile.NamedTemporaryFile('w', delete=False).name
with open(f, 'w', encoding='utf8') as fp:
fp.writelines(revised_dockerfile)
for k in revised_dockerfile:
self.logger.debug(k)
return f
class DockerBuilder():
def __init__(self, repo_manifest):
global buildable_images
global pull_only_images
self.rm = repo_manifest
self.dc = None # Docker Client object
self.images = []
# arrays of images, used for write_actions
self.preexisting = []
self.obsolete = []
self.pulled = []
self.failed_pull = []
self.obsolete_pull = []
self.built = []
self.failed_build = []
# create dict of images, setting defaults
for image in buildable_images:
repo_d = self.rm.get_repo(image['repo'])
if "components" in image:
components = []
for component in image['components']:
comp = {}
comp['repo_name'] = component['repo']
comp['repo_d'] = self.rm.get_repo(component['repo'])
comp['dest'] = component['dest']
comp['path'] = component.get('path', '.')
components.append(comp)
else:
components = None
# set the full name in case this is pulled
full_name = "%s:%s" % (image['name'], build_tag)
img_o = DockerImage(full_name, image['repo'], repo_d,
image.get('path', '.'),
image.get('context', '.'),
image.get('dockerfile', 'Dockerfile'),
components=components)
self.images.append(img_o)
# add misc images
for misc_image in pull_only_images:
img_o = DockerImage(misc_image)
self.images.append(img_o)
if not args.dry_run:
self._docker_connect()
self.create_dependency()
if not args.build: # if forcing build, don't use preexisting
self.find_preexisting()
if args.graph is not None:
self.dependency_graph(args.graph)
self.process_images()
if args.actions_taken is not None:
self.write_actions_file(args.actions_taken)
def _docker_connect(self):
""" Connect to docker daemon """
try:
self.dc = DockerClient()
except requests.ConnectionError:
LOG.debug("Docker connection not available")
sys.exit(1)
if self.dc.ping():
LOG.debug("Docker server is responding")
else:
LOG.error("Unable to ping docker server")
sys.exit(1)
def find_preexisting(self):
""" find images that already exist in Docker and mark """
if self.dc:
LOG.debug("Evaluating already built/fetched Docker images")
# get list of images from docker
pe_images = self.dc.images()
for pe_image in pe_images:
raw_tags = pe_image['RepoTags']
if raw_tags:
LOG.info("Preexisting Image - ID: %s, tags: %s" %
(pe_image['Id'], ",".join(raw_tags)))
has_build_tag = False
for tag in raw_tags:
if build_tag in tag:
LOG.debug(" image has build_tag: %s" % build_tag)
has_build_tag = True
base_name = raw_tags[0].split(":")[0]
image = self.find_image(base_name)
# only evaluate images in the list of desired images
if image is not None:
good_labels = image.compare_labels(pe_image['Labels'])
if good_labels:
if has_build_tag:
LOG.info(" Image %s has up-to-date labels and"
" build_tag" % pe_image['Id'])
else:
LOG.info(" Image %s has up-to-date labels but"
" missing build_tag. Tagging image"
" with build_tag: %s" %
(pe_image['Id'], build_tag))
self.dc.tag(pe_image['Id'], image.name,
tag=build_tag)
self.preexisting.append({
'id': pe_image['Id'],
'tags': raw_tags,
'base': image.name.split(":")[0],
})
image.image_id = pe_image['Id']
image.status = DI_EXISTS
else: # doesn't have good labels
if has_build_tag:
LOG.info(" Image %s has obsolete labels and"
" build_tag, remove" % pe_image['Id'])
# remove build_tag from image
name_bt = "%s:%s" % (base_name, build_tag)
self.dc.remove_image(name_bt, False, True)
else:
LOG.info(" Image %s has obsolete labels, lacks"
" build_tag, ignore" % pe_image['Id'])
self.obsolete.append({
'id': pe_image['Id'],
'tags': raw_tags,
})
def find_image(self, image_name):
""" return image object matching name """
LOG.debug(" attempting to find image for: %s" % image_name)
for image in self.images:
if image.same_name(image_name):
LOG.debug(" found a match: %s" % image.raw_name)
return image
return None
def create_dependency(self):
""" set parent/child links for images """
# List of lists of parents images. Done in two steps for clarity
lol_of_parents = [img.parent_names for img in self.images
if img.parent_names is not []]
# flat list of all parent image names, with dupes
parents_with_dupes = [parent for parent_sublist in lol_of_parents
for parent in parent_sublist]
# remove duplicates
parents = list(set(parents_with_dupes))
LOG.info("All parent images: %s" % ", ".join(parents))
# list of "external parents", ones not built internally
external_parents = []
for parent_name in parents:
LOG.debug("Evaluating parent image: %s" % parent_name)
internal_parent = False
# match on p_name, without tag
(p_name, p_tag) = split_name(parent_name)
for image in self.images:
if image.same_name(p_name): # internal image is a parent
internal_parent = True
LOG.debug(" Internal parent: %s" % image.name)
break
if not internal_parent: # parent is external
LOG.debug(" External parent: %s" % parent_name)
external_parents.append(parent_name)
# add unique external parents to image list
for e_p_name in set(external_parents):
LOG.debug(" Creating external parent image object: %s" % e_p_name)
img_o = DockerImage(e_p_name)
self.images.append(img_o)
# now that all images (including parents) are in list, associate them
for image in filter(lambda img: img.parent_names is not [],
self.images):
LOG.debug("Associating image: %s" % image.name)
for parent_name in image.parent_names:
parent = self.find_image(parent_name)
image.parents.append(parent)
if parent is not None:
LOG.debug(" internal image '%s' is parent of '%s'" %
(parent.name, image.name))
parent.children.append(image)
else:
LOG.debug(" external image '%s' is parent of '%s'" %
(image.parent_name, image.name))
# loop again now that parents are linked to create labels
for image in self.images:
image.create_labels()
image.create_tags()
# if image has parent, get labels from parent(s)
if image.parents is not None:
for parent in image.parents:
LOG.debug("Adding parent labels from %s to child %s" %
(parent.name, image.name))
# don't create component labels for same repo as image
repo_list = [image.repo_name]
image.labels.update(parent.child_labels(repo_list))
def dependency_graph(self, graph_fn):
""" save a DOT dependency graph to a file """
graph_fn_abs = os.path.abspath(graph_fn)
LOG.info("Saving DOT dependency graph to: %s" % graph_fn_abs)
try:
import graphviz
except ImportError:
LOG.error('graphviz pip module not found')
raise
dg = graphviz.Digraph(comment='Image Dependency Graph',
graph_attr={'rankdir': 'LR'})
component_nodes = []
# Use raw names, so they match with what's in Dockerfiles
# delete colons as python graphviz module breaks with them
for image in self.images:
name_g = image.raw_name.replace(':', '\n')
dg.node(name_g)
if image.parents is not None:
for parent in image.parents:
name_p = parent.raw_name.replace(':', '\n')
dg.edge(name_p, name_g)
if image.components is not None:
for component in image.components:
name_c = "component - %s" % component['repo_name']
if name_c not in component_nodes:
dg.node(name_c)
component_nodes.append(name_c)
dg.edge(name_c, name_g, "", {'style': 'dashed'})
with open(graph_fn_abs, 'w') as g_fh:
g_fh.write(dg.source)
def write_actions_file(self, actions_fn):
actions_fn_abs = os.path.abspath(actions_fn)
LOG.info("Saving actions as YAML to: %s" % actions_fn_abs)
actions = {
"ib_pulled": self.pulled,
"ib_built": self.built,
"ib_preexisting_images": self.preexisting,
"ib_obsolete_images": self.obsolete,
"ib_failed_pull": self.failed_pull,
"ib_obsolete_pull": self.obsolete_pull,
"ib_failed_build": self.failed_build,
}
with open(actions_fn_abs, 'w') as a_fh:
yaml.safe_dump(actions, a_fh)
LOG.debug(yaml.safe_dump(actions))
def process_images(self):
""" determine whether to build/fetch images """
# upstream images (have no parents), must be fetched
must_fetch_a = filter(lambda img: not img.parents, self.images)
for image in must_fetch_a:
if image.status is not DI_EXISTS:
image.status = DI_FETCH
# images that can be built or fetched (have parents)
b_or_f_a = filter(lambda img: img.parents, self.images)
for image in b_or_f_a:
if not image.parents_clean() or args.build:
# must be built if not clean
image.status = DI_BUILD
elif image.status is not DI_EXISTS:
# try to fetch if clean and doesn't exist
image.status = DI_FETCH
# otherwise, image is clean and exists (image.status == DI_EXISTS)
c_and_e_a = filter(lambda img: img.status is DI_EXISTS, self.images)
LOG.info("Preexisting and clean images: %s" %
", ".join(c.name for c in c_and_e_a))
upstream_a = filter(lambda img: (img.status is DI_FETCH and
not img.parents), self.images)
LOG.info("Upstream images that must be fetched: %s" %
", ".join(u.raw_name for u in upstream_a))
fetch_a = filter(lambda img: (img.status is DI_FETCH and
img.parents), self.images)
LOG.info("Clean, buildable images to attempt to fetch: %s" %
", ".join(f.raw_name for f in fetch_a))
build_a = filter(lambda img: img.status is DI_BUILD, self.images)
LOG.info("Buildable images, due to unclean context or parents: %s" %
", ".join(b.raw_name for b in build_a))
# OK to fetch upstream in any case as they should reduce number of
# layers pulled/built later
for image in upstream_a:
if not self._fetch_image(image):
LOG.error("Unable to fetch upstream image: %s" %
image.raw_name)
sys.exit(1)
# fetch if not forcing the build of all images
if not args.build:
fetch_sort = sorted(fetch_a, key=(lambda img: len(img.children)),
reverse=True)
for image in fetch_sort:
if not self._fetch_image(image):
# if didn't fetch, build
image.status = DI_BUILD
while True:
buildable_images = self.get_buildable()
if buildable_images and args.pull:
LOG.error("Images must be built, but --pull is specified")
exit(1)
if buildable_images:
for image in buildable_images:
self._build_image(image)
else:
LOG.debug("No more images to build, ending build loop")
break
def get_buildable(self):
""" Returns list of images that can be built"""
buildable = []
for image in filter(lambda img: img.status is DI_BUILD, self.images):
for parent in image.parents:
if parent.status is DI_EXISTS:
if image not in buildable: # build once if two parents
buildable.append(image)
LOG.debug("Buildable images: %s" %
', '.join(image.name for image in buildable))
return buildable
def tag_image(self, image):
""" Applies tags to an image """
for tag in image.tags:
LOG.info("Tagging id: '%s', repo: '%s', tag: '%s'" %
(image.image_id, image.name, tag))
if self.dc is not None:
self.dc.tag(image.image_id, image.name, tag=tag)
def _fetch_image(self, image):
LOG.info("Attempting to fetch docker image: %s" % image.raw_name)
if self.dc is not None:
try:
for stat_json in self.dc.pull(image.raw_name,
stream=True):
# sometimes Docker's JSON is dirty, per:
# https://github.com/docker/docker-py/pull/1081/
stat_s = stat_json.strip()
stat_list = stat_s.split("\r\n")
for s_j in stat_list:
stat_d = json.loads(s_j)
if 'stream' in stat_d:
for stat_l in stat_d['stream'].split('\n'):
LOG.debug(stat_l)
if 'status' in stat_d:
for stat_l in stat_d['status'].split('\n'):
noisy = ["Extracting", "Downloading",
"Waiting", "Download complete",
"Pulling fs layer", "Pull complete",
"Verifying Checksum",
"Already exists"]
if stat_l in noisy:
LOG.debug(stat_l)
else:
LOG.info(stat_l)
if 'error' in stat_d:
LOG.error(stat_d['error'])
sys.exit(1)
except (DockerErrors.NotFound, DockerErrors.ImageNotFound) as e:
LOG.warning("Image could not be pulled: %s , %s" %
(e.errno, e.strerror))
self.failed_pull.append({
"tags": [image.raw_name, ],
})
if not image.parents:
LOG.error("Pulled image required to build, not available!")
sys.exit(1)
return False
except:
LOG.exception("Error pulling docker image")
self.failed_pull.append({
"tags": [image.raw_name, ],
})
return False
# obtain the image_id by inspecting the pulled image. Seems unusual
# that the Docker API `pull` method doesn't provide it when the
# `build` method does
pulled_image = self.dc.inspect_image(image.raw_name)
# check to make sure that image that was downloaded has the labels
# that we expect it to have, otherwise return false, trigger build
if not image.compare_labels(
pulled_image['ContainerConfig']['Labels']):
LOG.info("Tried fetching image %s, but labels didn't match" %
image.raw_name)
self.obsolete_pull.append({
"id": pulled_image['Id'],
"tags": pulled_image['RepoTags'],
})
return False
image.image_id = pulled_image['Id']
LOG.info("Fetched image %s, id: %s" %
(image.raw_name, image.image_id))
self.pulled.append({
"id": pulled_image['Id'],
"tags": pulled_image['RepoTags'],
"base": image.name.split(":")[0],
})
self.tag_image(image)
image.status = DI_EXISTS
return True
def _build_image(self, image):
LOG.info("Building docker image for %s" % image.raw_name)
if self.dc is not None:
build_tag = "%s:%s" % (image.name, image.tags[0])
buildargs = image.buildargs()
context_tar = image.context_tarball()
dockerfile = image.dockerfile_rel_path()
for key, val in buildargs.iteritems():
LOG.debug("Buildarg - %s : %s" % (key, val))
bl_path = ""
start_time = datetime.datetime.utcnow()
if(args.build_log_dir):
bl_name = "%s_%s" % (start_time.strftime("%Y%m%dT%H%M%SZ"),
re.sub(r'\W', '_', image.name))
bl_path = os.path.abspath(
os.path.join(args.build_log_dir, bl_name))
LOG.info("Build log: %s" % bl_path)
bl_fh = open(bl_path, 'w+', 0) # 0 = unbuffered writes
else:
bl_fh = None
try:
LOG.info("Building image: %s" % image)
for stat_d in self.dc.build(tag=build_tag,
buildargs=buildargs,
nocache=args.build,
custom_context=True,
fileobj=context_tar,
dockerfile=dockerfile,
rm=True,
forcerm=True,
pull=False,
stream=True,
decode=True):
if 'stream' in stat_d:
if bl_fh:
bl_fh.write(stat_d['stream'].encode('utf-8'))
for stat_l in stat_d['stream'].split('\n'):
if(stat_l):
LOG.debug(stat_l)
if stat_d['stream'].startswith("Successfully built "):
siid = stat_d['stream'].split(' ')[2]
short_image_id = siid.strip()
LOG.debug("Short Image ID: %s" % short_image_id)
if 'status' in stat_d:
for stat_l in stat_d['status'].split('\n'):
if(stat_l):
LOG.info(stat_l)
if 'error' in stat_d:
LOG.error(stat_d['error'])
image.status = DI_ERROR
sys.exit(1)
except:
LOG.exception("Error building docker image")
self.failed_build.append({
"tags": [build_tag, ],
})
return
finally:
if(bl_fh):
bl_fh.close()
# the image ID given by output isn't the full SHA256 id, so find
# and set it to the full one
built_image = self.dc.inspect_image(short_image_id)
image.image_id = built_image['Id']
end_time = datetime.datetime.utcnow()
duration = end_time - start_time # duration is a timedelta
LOG.info("Built Image: %s, duration: %s, id: %s" %
(image.name, duration, image.image_id))
self.built.append({
"id": image.image_id,
"tags": [build_tag, ],
"push_name": image.raw_name,
"build_log": bl_path,
"duration": duration.total_seconds(),
"base": image.name.split(":")[0],
})
self.tag_image(image)
image.status = DI_EXISTS
class DockerBuilder(BaseBuilder):
"""A builder using the local Docker client"""
def __init__(self,
registry=None,
image_name=constants.DEFAULT_IMAGE_NAME,
base_image=constants.DEFAULT_BASE_IMAGE,
preprocessor=None,
push=True,
dockerfile_path=None):
super().__init__(
registry=registry,
image_name=image_name,
push=push,
base_image=base_image,
preprocessor=preprocessor,
)
def build(self):
logging.info("Building image using docker")
self.docker_client = APIClient(version='auto')
self._build()
if self.push:
self.publish()
def _build(self):
docker_command = self.preprocessor.get_command()
logger.warning("Docker command: {}".format(docker_command))
if not docker_command:
logger.warning(
"Not setting a command for the output docker image.")
install_reqs_before_copy = self.preprocessor.is_requirements_txt_file_present(
)
dockerfile_path = dockerfile.write_dockerfile(
docker_command=docker_command,
dockerfile_path=self.dockerfile_path,
path_prefix=self.preprocessor.path_prefix,
base_image=self.base_image,
install_reqs_before_copy=install_reqs_before_copy)
self.preprocessor.output_map[dockerfile_path] = 'Dockerfile'
context_file, context_hash = self.preprocessor.context_tar_gz()
self.image_tag = self.full_image_name(context_hash)
logger.warning('Building docker image {}...'.format(self.image_tag))
with open(context_file, 'rb') as fileobj:
bld = self.docker_client.build(path='.',
custom_context=True,
fileobj=fileobj,
tag=self.image_tag,
encoding='utf-8')
for line in bld:
self._process_stream(line)
def publish(self):
logger.warning('Publishing image {}...'.format(self.image_tag))
for line in self.docker_client.push(self.image_tag, stream=True):
self._process_stream(line)
def _process_stream(self, line):
raw = line.decode('utf-8').strip()
lns = raw.split('\n')
for ln in lns:
try:
ljson = json.loads(ln)
if ljson.get('error'):
msg = str(ljson.get('error', ljson))
logger.error('Build failed: %s', msg)
raise Exception('Image build failed: ' + msg)
else:
if ljson.get('stream'):
msg = 'Build output: {}'.format(
ljson['stream'].strip())
elif ljson.get('status'):
msg = 'Push output: {} {}'.format(
ljson['status'], ljson.get('progress'))
elif ljson.get('aux'):
msg = 'Push finished: {}'.format(ljson.get('aux'))
else:
msg = str(ljson)
logger.info(msg)
except json.JSONDecodeError:
logger.warning('JSON decode error: {}'.format(ln))
class DockerBuilder(object):
LATEST_IMAGE_TAG = 'latest'
def __init__(self,
build_context,
image_name,
image_tag,
copy_code=True,
dockerfile_name='Dockerfile',
credstore_env=None,
registries=None):
self.image_name = image_name
self.image_tag = image_tag
self.copy_code = copy_code
self.build_context = build_context
self.dockerfile_path = os.path.join(self.build_context,
dockerfile_name)
self._validate_registries(registries)
self.registries = registries
self.docker = APIClient(version='auto', credstore_env=credstore_env)
self.is_pushing = False
@staticmethod
def _validate_registries(registries):
if not registries or isinstance(registries, UriSpec):
return True
for registry in registries:
if not isinstance(registry, UriSpec):
raise BuildException(
'A registry `{}` is not valid Urispec.'.format(registry))
return True
def get_tagged_image(self):
return '{}:{}'.format(self.image_name, self.image_tag)
def check_image(self):
return self.docker.images(self.get_tagged_image())
def clean(self):
pass
def login_private_registries(self):
if not self.registries:
return
for registry in self.registries:
self.docker.login(username=registry.user,
password=registry.password,
registry=registry.host,
reauth=True)
def _prepare_log_lines(self, log_line): # pylint:disable=too-many-branches
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
status = True
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
log_lines.append('{}: {}'.format(
LogLevels.ERROR, str(json_line.get('error',
json_line))))
status = False
else:
if json_line.get('stream'):
log_lines.append('Building: {}'.format(
json_line['stream'].strip()))
elif json_line.get('status'):
if not self.is_pushing:
self.is_pushing = True
log_lines.append('Pushing ...')
elif json_line.get('aux'):
log_lines.append('Pushing finished: {}'.format(
json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines, status
def _handle_logs(self, log_lines):
for log_line in log_lines:
print(log_line) # pylint:disable=superfluous-parens
def _handle_log_stream(self, stream):
log_lines = []
status = True
try:
for log_line in stream:
new_log_lines, new_status = self._prepare_log_lines(log_line)
log_lines += new_log_lines
if not new_status:
status = new_status
self._handle_logs(log_lines)
log_lines = []
if log_lines:
self._handle_logs(log_lines)
except (BuildError, APIError) as e:
self._handle_logs([
'{}: Could not build the image, encountered {}'.format(
LogLevels.ERROR, e)
])
return False
return status
def build(self, nocache=False, memory_limit=None):
limits = {
# Disable memory swap for building
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
stream = self.docker.build(path=self.build_context,
tag=self.get_tagged_image(),
forcerm=True,
rm=True,
pull=True,
nocache=nocache,
container_limits=limits)
return self._handle_log_stream(stream=stream)
def push(self):
stream = self.docker.push(self.image_name,
tag=self.image_tag,
stream=True)
return self._handle_log_stream(stream=stream)
class DockerBuilder(object):
LATEST_IMAGE_TAG = 'latest'
WORKDIR = '/code'
def __init__(self,
build_job,
repo_path,
from_image,
copy_code=True,
build_steps=None,
env_vars=None,
dockerfile_name='Dockerfile'):
self.build_job = build_job
self.job_uuid = build_job.uuid.hex
self.job_name = build_job.unique_name
self.from_image = from_image
self.image_name = get_image_name(self.build_job)
self.image_tag = self.job_uuid
self.folder_name = repo_path.split('/')[-1]
self.repo_path = repo_path
self.copy_code = copy_code
self.build_path = '/'.join(self.repo_path.split('/')[:-1])
self.build_steps = get_list(build_steps)
self.env_vars = get_list(env_vars)
self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
self.polyaxon_requirements_path = self._get_requirements_path()
self.polyaxon_setup_path = self._get_setup_path()
self.docker = APIClient(version='auto')
self.registry_host = None
self.docker_url = None
def get_tagged_image(self):
return get_tagged_image(self.build_job)
def check_image(self):
return self.docker.images(self.get_tagged_image())
def clean(self):
# Clean dockerfile
delete_path(self.dockerfile_path)
def login(self, registry_user, registry_password, registry_host):
try:
self.docker.login(username=registry_user,
password=registry_password,
registry=registry_host,
reauth=True)
except DockerException as e:
_logger.exception('Failed to connect to registry %s\n', e)
@staticmethod
def _prepare_log_lines(log_line):
raw = log_line.decode('utf-8').strip()
raw_lines = raw.split('\n')
log_lines = []
for raw_line in raw_lines:
try:
json_line = json.loads(raw_line)
if json_line.get('error'):
raise DockerBuilderError(
str(json_line.get('error', json_line)))
else:
if json_line.get('stream'):
log_lines.append('Build: {}'.format(
json_line['stream'].strip()))
elif json_line.get('status'):
log_lines.append('Push: {} {}'.format(
json_line['status'], json_line.get('progress')))
elif json_line.get('aux'):
log_lines.append('Push finished: {}'.format(
json_line.get('aux')))
else:
log_lines.append(str(json_line))
except json.JSONDecodeError:
log_lines.append('JSON decode error: {}'.format(raw_line))
return log_lines
def _handle_logs(self, log_lines):
publisher.publish_build_job_log(log_lines=log_lines,
job_uuid=self.job_uuid,
job_name=self.job_name)
def _handle_log_stream(self, stream):
log_lines = []
last_emit_time = time.time()
try:
for log_line in stream:
log_lines += self._prepare_log_lines(log_line)
publish_cond = (len(log_lines) == publisher.MESSAGES_COUNT
or (log_lines and time.time() - last_emit_time
> publisher.MESSAGES_TIMEOUT))
if publish_cond:
self._handle_logs(log_lines)
log_lines = []
last_emit_time = time.time()
if log_lines:
self._handle_logs(log_lines)
except (BuildError, APIError, DockerBuilderError) as e:
self._handle_logs('Build Error {}'.format(e))
return False
return True
def _get_requirements_path(self):
requirements_path = os.path.join(self.repo_path,
'polyaxon_requirements.txt')
if os.path.isfile(requirements_path):
return os.path.join(self.folder_name, 'polyaxon_requirements.txt')
return None
def _get_setup_path(self):
setup_file_path = os.path.join(self.repo_path, 'polyaxon_setup.sh')
has_setup = os.path.isfile(setup_file_path)
if has_setup:
st = os.stat(setup_file_path)
os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
return os.path.join(self.folder_name, 'polyaxon_setup.sh')
return None
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
from_image=self.from_image,
polyaxon_requirements_path=self.polyaxon_requirements_path,
polyaxon_setup_path=self.polyaxon_setup_path,
build_steps=self.build_steps,
env_vars=self.env_vars,
folder_name=self.folder_name,
workdir=self.WORKDIR,
nvidia_bin=settings.MOUNT_PATHS_NVIDIA.get('bin'),
copy_code=self.copy_code)
def build(self, nocache=False, memory_limit=None):
_logger.debug('Starting build in `%s`', self.repo_path)
# Checkout to the correct commit
if self.image_tag != self.LATEST_IMAGE_TAG:
git.checkout_commit(repo_path=self.repo_path,
commit=self.image_tag)
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
# Create DockerFile
with open(self.dockerfile_path, 'w') as dockerfile:
rendered_dockerfile = self.render()
celery_app.send_task(
SchedulerCeleryTasks.BUILD_JOBS_SET_DOCKERFILE,
kwargs={
'build_job_uuid': self.job_uuid,
'dockerfile': rendered_dockerfile
})
dockerfile.write(rendered_dockerfile)
stream = self.docker.build(path=self.build_path,
tag=self.get_tagged_image(),
forcerm=True,
rm=True,
pull=True,
nocache=nocache,
container_limits=limits)
return self._handle_log_stream(stream=stream)
def push(self):
stream = self.docker.push(self.image_name,
tag=self.image_tag,
stream=True)
return self._handle_log_stream(stream=stream)
class DockerImage:
"""
The DockerImage class has the functions and attributes for building the dockerimage
"""
def __init__(
self, info, dockerfile, repository, tag, to_build, context=None,
):
# Meta-data about the image should go to info.
# All keys in info are accessible as attributes
# of this class
self.info = info
self.summary = {}
self.dockerfile = dockerfile
self.context = context
# TODO: Add ability to tag image with multiple tags
self.repository = repository
self.tag = tag
self.ecr_url = f"{self.repository}:{self.tag}"
if not isinstance(to_build, bool):
to_build = True if to_build == "true" else False
self.to_build = to_build
self.build_status = None
self.client = APIClient(base_url=constants.DOCKER_URL)
self.log = []
def __getattr__(self, name):
return self.info[name]
def build(self):
"""
The build function builds the specified docker image
"""
self.summary["start_time"] = datetime.now()
if not self.to_build:
self.log = ["Not built"]
self.build_status = constants.NOT_BUILT
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
return self.build_status
build_args = {}
if self.info.get("base_image_uri"):
build_args["BASE_IMAGE"] = self.info["base_image_uri"]
with open(self.context.context_path, "rb") as context_file:
response = []
for line in self.client.build(
fileobj=context_file,
path=self.dockerfile,
custom_context=True,
rm=True,
decode=True,
tag=self.ecr_url,
buildargs=build_args
):
if line.get("error") is not None:
self.context.remove()
response.append(line["error"])
self.log = response
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
elif line.get("status") is not None:
response.append(line["status"])
else:
response.append(str(line))
self.context.remove()
self.summary["image_size"] = int(
self.client.inspect_image(self.ecr_url)["Size"]
) / (1024 * 1024)
if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20:
response.append("Image size baseline exceeded")
response.append(f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}")
self.log = response
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
return self.build_status
for line in self.client.push(
self.repository, self.tag, stream=True, decode=True
):
if line.get("error") is not None:
response.append(line["error"])
self.log = response
self.build_status = constants.FAIL
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
return self.build_status
if line.get("stream") is not None:
response.append(line["stream"])
else:
response.append(str(line))
self.build_status = constants.SUCCESS
self.summary["status"] = constants.STATUS_MESSAGE[self.build_status]
self.summary["end_time"] = datetime.now()
self.summary["ecr_url"] = self.ecr_url
self.log = response
return self.build_status
