def _do_login(self):
response = self.ecr.get_authorization_token(registryIds=[self.account])
data = response['authorizationData'][0]
token = data['authorizationToken']
user, password = base64.decodestring(token).split(":")
proxy = data['proxyEndpoint']
sh("docker", "login", "-u", user, "-p", Secret(password), proxy)
def delete(self, labels):
# never try to delete namespaces or storage classes because they are shared resources
all = ",".join(r for r in ALL if r not in ('ns', 'sc'))
lines = sh(
"kubectl", "get", all, '--all-namespaces', selector(labels),
'-ogo-template={{range .items}}{{.kind}} {{.metadata.namespace}} {{.metadata.name}}{{"\\n"}}{{end}}'
).output.splitlines()
byns = {}
for line in lines:
parts = line.split()
if len(parts) == 2:
kind, name = parts
namespace = None
else:
kind, namespace, name = parts
if namespace not in byns:
byns[namespace] = []
byns[namespace].append((kind, name))
for ns in sorted(byns.keys()):
names = sorted("%s/%s" % (k, n) for k, n in byns[ns])
if ns is None:
sh("kubectl", "delete", *names)
else:
sh("kubectl", "delete", "-n", ns, *names)
def delete(self, labels):
lines = sh("kubectl", "get", "ns", "-oname").output.splitlines()
namespaces = (l.strip().split("/")[-1] for l in lines)
for ns in namespaces:
# never try to delete namespaces because they are shared resources
sh("kubectl", "delete", "-n", ns,
",".join(r for r in ALL if r != 'ns'), selector(labels))
def push(self, name, version):
self._login()
self._create_repo(name)
img = self.image(name, version)
self.image_cache.pop(img, None)
sh("docker", "push", img)
return img
def build(self, directory, dockerfile, name, version, args=None):
args = args or {}
buildargs = []
for k, v in args.items():
buildargs.append("--build-arg")
buildargs.append("%s=%s" % (k, v))
img = image(self.registry, self.namespace, name, version)
sh("docker", "build", directory, "-f", dockerfile, "-t", img, *buildargs)
def build(self, directory, dockerfile, name, version, args):
args = args or {}
buildargs = []
for k, v in args.items():
buildargs.append("--build-arg")
buildargs.append("%s=%s" % (k, v))
img = self.image(name, version)
sh("docker", "build", directory, "-f", dockerfile, "-t", img,
*buildargs)
return img
def find_builders(self, name):
builder_prefix = self.builder_prefix(name)
containers = sh("docker", "ps", "-qaf", "name=%s" % builder_prefix,
"--format", "{{.ID}} {{.Names}}")
for line in containers.output.splitlines():
id, builder_name = line.split()
yield id, builder_name
def resources(self, yaml_dir):
if is_yaml_empty(yaml_dir):
return []
cmd = "kubectl", "apply", "--dry-run", "-R", "-f", yaml_dir, "-o", "name"
if self.namespace:
cmd += "--namespace", self.namespace
return sh(*cmd).output.split()
def commit(self, name, version):
args = []
for change in self.changes:
args.append("-c")
args.append(change)
args.extend((self.cid, self.docker.image(name, version)))
return sh("docker", "commit", *args)
def builder(self,
directory,
dockerfile,
name,
version,
args,
builder=None):
# We hash the buildargs and Dockerfile so that we reconstruct
# the builder container if anything changes. This might want
# to be extended to cover other files the Dockerfile
# references somehow at some point. (Maybe we could use the
# spec stuff we use in .forgeignore?)
builder_name = "%s_%s" % (self.builder_prefix(name),
self.builder_hash(dockerfile, args))
cid = None
for id, bname in self.find_builders(name):
if bname == builder_name:
cid = id
else:
Builder(self, id).kill()
if not cid:
image = self.build(directory,
dockerfile,
name,
version,
args,
builder=None)
cid = sh("docker", "run", "--rm", "--name", builder_name, "-dit",
"--entrypoint", "/bin/sh", image).output.strip()
return Builder(self, cid, self.get_changes(dockerfile))
def build(self, directory, dockerfile, name, version, args, builder=None):
args = args or {}
builder = builder or DockerImageBuilder.DOCKER
buildargs = []
for k, v in args.items():
buildargs.append("--build-arg")
buildargs.append("%s=%s" % (k, v))
img = self.image(name, version)
cmd = DockerImageBuilder.get_cmd_from_name(builder)
sh(*cmd(directory, dockerfile, img, buildargs))
return img
def _do_login(self):
if self.user == "_token":
self.password = sh(
"gcloud",
"auth",
"print-access-token",
output_transform=lambda x: "<OUTPUT_ELIDED>").output.strip()
Docker._do_login(self)
def apply(self, yaml_dir):
if is_yaml_empty(yaml_dir):
return SHResult("", 0, "")
cmd = "kubectl", "apply", "-f", yaml_dir
if self.namespace:
cmd += "--namespace", self.namespace
if self.dry_run:
cmd += "--dry-run",
result = sh(*cmd)
return result
def apply(self, yaml_dir):
if is_yaml_empty(yaml_dir):
return Result((), 0, "")
cmd = "kubectl", "apply", "-f", yaml_dir
if self.namespace:
cmd += "--namespace", self.namespace
if self.dry_run:
cmd += "--dry-run",
result = sh(*cmd, expected=xrange(256))
return result
def _do_login(self):
if self._run_login:
sh("docker", "login", "-u", self.user, "-p", Secret(self.password),
self.registry)
def kill(self):
sh("docker", "kill", self.cid, expected=(0, 1))
def _login(self):
if not self.logged_in:
sh("docker", "login", "-u", self.user, "-p", Secret(self.password), self.registry)
self.logged_in = True
def tag(self, source, name, version):
img = self.image(name, version)
sh("docker", "tag", source, img)
def list(self):
"""
Return a structured view of all forge deployed resources in a kubernetes cluster.
"""
output = sh("kubectl", "get", "--all-namespaces", ",".join(ALL),
"-oyaml", "-lforge.service").output
repos = {}
endpoints = {}
for nd in yamlutil.load("kubectl-get", output):
items = nd["items"]
for i in items:
kind = i["kind"].lower()
md = i["metadata"]
name = md["name"]
namespace = md["namespace"]
status = i.get("status", {})
ann = md.get("annotations", {})
repo = ann.get("forge.repo", "(none)")
descriptor = ann.get("forge.descriptor", "(none)")
version = ann.get("forge.version", "(none)")
labels = md.get("labels", {})
service = labels["forge.service"]
profile = labels["forge.profile"]
if kind == "endpoints":
endpoints[(namespace, name)] = i.get("subsets", ())
continue
if repo not in repos:
repos[repo] = {}
if service not in repos[repo]:
repos[repo][service] = {}
if profile not in repos[repo][service]:
repos[repo][service][profile] = []
repos[repo][service][profile].append({
"kind": kind,
"namespace": namespace,
"name": name,
"version": version,
"descriptor": descriptor,
"status": status
})
for repo, services in repos.items():
for service, profiles in services.items():
for profile, resources in profiles.items():
for resource in resources:
kind = resource["kind"]
if kind == "service":
status = status_summary(
kind,
endpoints.get(
(resource["namespace"], resource["name"])))
else:
status = status_summary(kind, resource["status"])
resource["status"] = status
return repos
def push(self, name, version):
self._login()
img = image(self.registry, self.namespace, name, version)
self.image_cache.pop(img, None)
sh("docker", "push", img)
return img
def tag(self, source, name, version):
img = image(self.registry, self.namespace, name, version)
sh("docker", "tag", source, img)
def local_exists(self, name, version):
return bool(sh("docker", "images", "-q", image(self.registry, self.namespace, name, version)).output)
def local_exists(self, name, version):
return bool(
sh("docker", "images", "-q", self.image(name, version)).output)
def run(self, *args):
# XXX: for some reason when we put a -t here it messes up the
# terminal output
return sh("docker", "exec", "-i", self.cid, *args)
def pull(self, image):
self._login()
sh("docker", "pull", image)
def cp(self, source, target):
return sh("docker", "cp", source, "{0}:{1}".format(self.cid, target))
def run(self, name, version, cmd, *args):
return sh("docker", "run", "--rm", "-it", "--entrypoint", cmd,
self.image(name, version), *args)
def build(self, directory, name, version):
sh("docker", "build", ".", "-t", image(self.registry, self.namespace, name, version), cwd=directory)