def channel_map(self):
"""Gets the channel map for a snap"""
output = capture([
"surl_cli.py", "-a", self.creds, "-X", "GET",
f"{self.api}/channel-map"
]).stdout.decode()
return json.loads(output)
def arn():
log("Adding AWS IAM Role KubernetesAdmin")
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Resource": os.environ["AWSIAMARN"],
"Action": "sts:AssumeRole",
"Condition": {},
}
],
}
log(f"Role Policy {policy}")
arn = capture(
[
"aws",
"iam",
"create-role",
"--role-name",
"KubernetesAdmin",
"--description",
"Kubernetes administrator role (for AWS IAM Authenticator for Kubernetes).",
"--assume-role-policy-document",
json.dumps(policy),
"--output",
"text",
"--query",
"Role.Arn",
]
)
log(f"Created arn: {arn}")
yield arn.stdout.decode().strip()
def push(self):
"""Pushes a built charm to Charmstore"""
if "override-push" in self.opts:
self.echo("Override push found, running in place of charm push.")
script(
self.opts["override-push"],
cwd=self.src_path,
charm=self.name,
namespace=self.namespace,
echo=self.echo,
)
return
self.echo(f"Pushing built {self.dst_path} to {self.entity}")
out = capture(["charm", "push", self.dst_path, self.entity])
self.echo(f"Charm push returned: {out}")
# Output includes lots of ansi escape sequences from the docker push,
# and we only care about the first line, which contains the url as yaml.
out = yaml.safe_load(out.stdout.decode().strip().splitlines()[0])
self.new_entity = out["url"]
self.echo(f"Setting {self.new_entity} metadata: {self.commit}")
cmd_ok(
["charm", "set", self.new_entity, f"commit={self.commit}"], echo=self.echo
)
def download(self, layer_name):
out = capture(
f"charm pull-source -i {self.layer_index} -b {self.layer_branch} {layer_name}"
)
click.echo(f"- {out.stdout.decode()}")
rev = re.compile("rev: ([a-zA-Z0-9]+)")
layer_manifest = {
"rev": rev.search(out.stdout.decode()).group(1),
"url": layer_name,
}
return layer_manifest
def download(self, layer_name):
"""Pull layer source from the charm store."""
out = capture(
f"charm pull-source -i {self.layer_index} -b {self.layer_branch} {layer_name}"
)
self.echo(f"- {out.stdout.decode()}")
layer_manifest = {
"rev": self.REV.search(out.stdout.decode()).group(1),
"url": layer_name,
}
return layer_manifest
def get_charmstore_rev_url(self):
# Grab charmstore revision for channels charm
response = capture([
"charm",
"show",
self.entity,
"--channel",
self.build.db["build_args"]["to_channel"],
"id",
])
if not response.ok:
return None
response = yaml.safe_load(response.stdout.decode().strip())
return response["id"]["Id"]
def all_published(snap):
"""Get all known published snap versions, tracks, arch"""
re_comp = re.compile("[ \t+]{2,}")
revision_list = capture(["snapcraft", "revisions", snap])
revision_list = revision_list.stdout.decode().splitlines()[1:]
revision_list = [re_comp.split(line) for line in revision_list]
publish_map = {"arm64": {}, "ppc64el": {}, "amd64": {}, "s390x": {}}
for line in revision_list:
rev, uploaded, arch, version, channels = line
channels = channels.split(",")
for chan in channels:
if chan.endswith("*") and version in publish_map[arch]:
publish_map[arch][version].append(chan)
elif chan.endswith("*"):
publish_map[arch][version] = [chan]
return publish_map
def __run_git(args):
username, password, layer_name, upstream, downstream = args
log.info(f"Syncing {layer_name} :: {upstream} -> {downstream}")
downstream = f"https://{username}:{password}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
ret = capture(f"git clone {downstream} {identifier}")
if not ret.ok:
log.info(f"Failed to clone repo: {ret.stderr.decode()}")
sys.exit(1)
cmd_ok("git config user.email '*****@*****.**'", cwd=identifier)
cmd_ok("git config user.name cdkbot", cwd=identifier)
cmd_ok("git config push.default simple", cwd=identifier)
cmd_ok(f"git remote add upstream {upstream}", cwd=identifier)
cmd_ok("git fetch upstream", cwd=identifier)
cmd_ok("git checkout master", cwd=identifier)
cmd_ok("git merge upstream/master", cwd=identifier)
cmd_ok("git push origin", cwd=identifier)
cmd_ok("rm -rf {identifier}")
def _create_branch(repo, from_branch, to_branch, dry_run, force, patches):
""" Creates a git branch based on the upstream snap repo and a version to branch as. This will also update
the snapcraft.yaml with the correct version to build the snap from in that particular branch.
These branches must already exist in https://github.com/kubernetes/kubernetes.
Usage:
snap.py branch --repo git+ssh://[email protected]/snap-kubectl \
--from-branch master \
--to-branch 1.13.2
"""
env = os.environ.copy()
if branch_exists(repo, to_branch, env) and not force:
click.echo(f"{to_branch} already exists, skipping...")
sys.exit(0)
snap_basename = urlparse(repo)
snap_basename = Path(snap_basename.path).name
if snap_basename.endswith(".git"):
snap_basename = snap_basename.rstrip(".git")
tmpdir = tempfile.TemporaryDirectory()
snap_basename = tmpdir.name
capture(["git", "clone", repo, snap_basename])
capture(["git", "remote", "prune", "origin"], cwd=snap_basename)
capture(["git", "config" "user.email", "*****@*****.**"], cwd=snap_basename)
capture(["git", "config", "user.name", "cdkbot"], cwd=snap_basename)
capture(["git", "checkout", "-b", to_branch], cwd=snap_basename)
snapcraft_fn = Path(snap_basename) / "snapcraft.yaml"
snapcraft_fn_tpl = Path(snap_basename) / "snapcraft.yaml.in"
if not snapcraft_fn_tpl.exists():
click.echo(f"{snapcraft_fn_tpl} not found")
sys.exit(1)
# Apply patches
patches_list = []
if patches:
patches_path = Path(patches)
if patches_path.exists():
click.echo("Patches found, applying.")
patches_map = yaml.safe_load(patches_path.read_text(encoding="utf8"))
# TODO: cleanup
if "all" in patches_map:
for patch_fn in patches_map["all"]:
patch_fn = Path(patch_fn).absolute()
shared_path = str(Path("shared") / patch_fn.parts[-1])
sh.cp(str(patch_fn), str(shared_path), _cwd=snap_basename)
patches_list.append(shared_path)
git.add(shared_path, _cwd=snap_basename)
if to_branch.lstrip("v") in patches_map:
for patch_fn in patches_map[to_branch.lstrip("v")]:
patch_fn = Path(patch_fn).absolute()
shared_path = str(Path("shared") / patch_fn.parts[-1])
sh.cp(str(patch_fn), str(shared_path), _cwd=snap_basename)
patches_list.append(shared_path)
git.add(shared_path, _cwd=snap_basename)
k8s_major_minor = semver.parse(to_branch.lstrip("v"))
k8s_major_minor = f"{k8s_major_minor['major']}.{k8s_major_minor['minor']}"
snapcraft_yml = snapcraft_fn_tpl.read_text()
snapcraft_yml = _render(
snapcraft_fn_tpl,
{
"snap_version": to_branch.lstrip("v"),
"patches": patches_list,
"go_version": K8S_GO_MAP.get(k8s_major_minor, "go/1.12/stable"),
},
)
snapcraft_fn.write_text(snapcraft_yml)
if not dry_run:
cmd_ok("git add .", cwd=snap_basename)
cmd_ok(f"git commit -m 'Creating branch {to_branch}'", cwd=snap_basename)
cmd_ok(f"git push --force {repo} {to_branch}", cwd=snap_basename)
def attach_resource(self, from_channel):
resource_builder = self.opts.get("resource_build_sh", None)
if not resource_builder:
return
builder = Path(self.src_path) / resource_builder
out_path = Path(self.src_path) / "tmp"
resource_spec = yaml.safe_load(
Path(self.build.resource_spec).read_text())
resource_spec_fragment = resource_spec.get(self.entity, None)
click.echo(resource_spec_fragment)
if not resource_spec_fragment:
raise SystemExit("Unable to determine resource spec for entity")
os.makedirs(str(out_path), exist_ok=True)
charm_id = capture(
["charm", "show", self.entity, "--channel", from_channel, "id"])
charm_id = yaml.safe_load(charm_id.stdout.decode())
resources = capture([
"charm",
"list-resources",
charm_id["id"]["Id"],
"--channel",
from_channel,
"--format",
"yaml",
])
if not resources.ok:
click.echo("No resources found for {}".format(charm_id))
return
resources = yaml.safe_load(resources.stdout.decode())
builder_sh = builder.absolute()
click.echo(f"Running {builder_sh} from {self.dst_path}")
# Grab a list of all file extensions to lookout for
known_resource_extensions = list(
set("".join(Path(k).suffixes)
for k in resource_spec_fragment.keys()))
click.echo(
f" attaching resources with known extensions: {', '.join(known_resource_extensions)}"
)
ret = cmd_ok(["bash", str(builder_sh)], cwd=out_path)
if not ret.ok:
raise SystemExit("Unable to build resources")
for line in glob("{}/*".format(out_path)):
click.echo(f" verifying {line}")
resource_path = Path(line)
resource_fn = resource_path.parts[-1]
resource_key = resource_spec_fragment.get(resource_fn, None)
if resource_key:
retry_call(
cmd_ok,
fargs=[[
"charm",
"attach",
self.entity,
"--channel",
from_channel,
f"{resource_key}={resource_path}",
]],
fkwargs={"check": True},
delay=2,
backoff=2,
tries=15,
exceptions=CalledProcessError,
)
def get_charmstore_rev_url(entity, channel):
# Grab charmstore revision for channels charm
response = capture(["charm", "show", entity, "--channel", channel, "id"])
response = yaml.safe_load(response.stdout.decode().strip())
return response["id"]["Id"]