def _do_build_docker(name, path):
path_sha = pkgpanda.build.hash_folder_abs(path, os.path.dirname(path))
container_name = 'dcos/dcos-builder:{}_dockerdir-{}'.format(name, path_sha)
log.debug("Attempting to pull docker: %s", container_name)
pulled = False
try:
# TODO(cmaloney): Rather than pushing / pulling from Docker Hub upload as a build artifact.
# the exported tarball.
subprocess.check_call(['docker', 'pull', container_name])
pulled = True
# TODO(cmaloney): Differentiate different failures of running the process better here
except subprocess.CalledProcessError:
pulled = False
if not pulled:
log.debug("Pull failed, building instead: %s", container_name)
# Pull failed, build it
subprocess.check_call(['docker', 'build', '-t', container_name, path])
# TODO(cmaloney): Push the built docker image on successful package build to both
# 1) commit-<commit_id>
# 2) Dockerfile-<file_contents_sha1>
# 3) bootstrap-<bootstrap_id>
# So we can track back the builder id for a given commit or bootstrap id, and reproduce whatever
# we need. The Dockerfile-<sha1> is useful for making sure we don't rebuild more than
# necessary.
try:
subprocess.check_call(['docker', 'push', container_name])
except subprocess.CalledProcessError:
logger.warning(
"docker push of dcos-builder failed. This means it will be very difficult "
"for this build to be reproduced (others will have a different / non-identical "
"base docker for most packages.")
pass
# mark as latest so it will be used when building packages
# extract the docker client version string
try:
docker_version = subprocess.check_output(
['docker', 'version', '-f', '{{.Client.Version}}']).decode()
except subprocess.CalledProcessError:
# If the above command fails then we know we have an older version of docker
# Older versions of docker spit out an entirely different format
docker_version = subprocess.check_output(
['docker', 'version']).decode().split("\n")[0].split()[2]
# only use force tag if using docker version 1.9 or earlier
container_name_t = 'dcos/dcos-builder:{}_dockerdir-latest'.format(name)
if LooseVersion(docker_version) < LooseVersion('1.10'):
args = ['docker', 'tag', '-f', container_name, container_name_t]
else:
args = ['docker', 'tag', container_name, container_name_t]
subprocess.check_call(args)
def add_user(self, username, groupname):
UserManagement.validate_username(username)
if not self._manage_users:
return
# Check if the user already exists and exit.
try:
if not is_windows:
UserManagement.validate_user_group(username, groupname)
self._users.add(username)
return
except KeyError as ex:
# Doesn't exist, fall through
log.warning("User [%s:%s] already exists", username, groupname)
# If we're not allowed to manage users, error
if not self._add_users:
raise ValidationError(
"User {} doesn't exist but is required by a DC/OS Component, and "
"automatic user addition is disabled".format(username))
log.info("Add the user")
add_user_cmd = [
'useradd',
'--system',
'--home-dir',
'/opt/mesosphere',
'--shell',
'/sbin/nologin',
'-c',
'DCOS System User',
]
# A group matching the username will be created by the adduser command.
# Any other group that the user is added to needs to exist prior to executing the
# adduser command.
if groupname is not None and groupname != username:
UserManagement.validate_group(groupname)
add_user_cmd += ['-g', groupname]
else:
add_user_cmd += ['--user-group']
add_user_cmd += [username]
try:
log.debug(" ".join(add_user_cmd))
check_output(add_user_cmd)
self._users.add(username)
except CalledProcessError as ex:
raise ValidationError(
"User {} doesn't exist and couldn't be created because of: {}".
format(username, ex.output))
def get_git_sha1(bare_folder, ref):
try:
return check_output(
["git", "--git-dir", bare_folder, "rev-parse",
ref + "^{commit}"]).decode('ascii').strip()
except CalledProcessError as ex:
raise ValidationError("Unable to find ref '{}' in '{}': {}".format(
ref, bare_folder, ex)) from ex
def __init__(self, src_info, cache_dir, working_directory):
super().__init__(src_info)
assert self.kind == 'git_local'
if src_info.keys() > {'kind', 'rel_path'}:
raise ValidationError(
"Only kind, rel_path can be specified for git_local")
if os.path.isabs(src_info['rel_path']):
raise ValidationError(
"rel_path must be a relative path to the current directory "
"when used with git_local. Using a relative path means others "
"that clone the repository will have things just work rather "
"than a path.")
self.src_repo_path = os.path.normpath(working_directory + '/' +
src_info['rel_path']).rstrip('/')
# Make sure there are no local changes, we can't `git clone` local changes.
try:
git_status = check_output([
'git', '-C', self.src_repo_path, 'status', '--porcelain',
'-uno', '-z'
]).decode()
if len(git_status):
raise ValidationError(
"No local changse are allowed in the git_local_work base repository. "
"Use `git -C {0} status` to see local changes. "
"All local changes must be committed or stashed before the "
"package can be built. One workflow (temporary commit): `git -C {0} "
"commit -am TMP` to commit everything, build the package, "
"`git -C {0} reset --soft HEAD^` to get back to where you were.\n\n"
"Found changes: {1}".format(self.src_repo_path,
git_status))
except CalledProcessError:
raise ValidationError(
"Unable to check status of git_local_work checkout {}. Is the "
"rel_path correct?".format(src_info['rel_path']))
self.commit = get_git_sha1(self.src_repo_path + "/.git", "HEAD")