class BuildEnv:
"""Charm or Bundle build data class."""
REV = re.compile("rev: ([a-zA-Z0-9]+)")
def __new__(cls, *args, **kwargs):
"""Initialize class variables used during the build from the CI environment."""
try:
cls.base_dir = Path(os.environ.get("CHARM_BASE_DIR"))
cls.build_dir = Path(os.environ.get("CHARM_BUILD_DIR"))
cls.layers_dir = Path(os.environ.get("CHARM_LAYERS_DIR"))
cls.interfaces_dir = Path(os.environ.get("CHARM_INTERFACES_DIR"))
cls.charms_dir = Path(os.environ.get("CHARM_CHARMS_DIR"))
cls.work_dir = Path(os.environ.get("WORKSPACE"))
cls.tmp_dir = cls.work_dir / "tmp"
cls.home_dir = Path(os.environ.get("HOME"))
except TypeError:
raise BuildException(
"CHARM_BUILD_DIR, CHARM_LAYERS_DIR, CHARM_INTERFACES_DIR, WORKSPACE, HOME: "
"Unable to find some or all of these charm build environment variables."
)
return super(BuildEnv, cls).__new__(cls)
def __init__(self, build_type):
"""Create a BuildEnv to hold/save build metadata."""
self.store = Store("BuildCharms")
self.now = datetime.utcnow()
self.build_type = build_type
self.db = {}
self.clean_dirs = tuple()
# poison base_dir to prevent `git rev-parse` from working in this subdirectory
(self.base_dir / ".git").touch(0o664, exist_ok=True)
if self.build_type == BuildType.CHARM:
self.db_json = Path("buildcharms.json")
self.repos_dir = None
self.clean_dirs = (self.layers_dir, self.interfaces_dir,
self.charms_dir)
elif self.build_type == BuildType.BUNDLE:
self.db_json = Path("buildbundles.json")
self.repos_dir = self.tmp_dir / "repos"
self.bundles_dir = self.tmp_dir / "bundles"
self.default_repo_dir = self.repos_dir / "bundles-kubernetes"
self.clean_dirs = (self.repos_dir, self.bundles_dir)
if not self.db.get("build_datetime", None):
self.db["build_datetime"] = self.now.strftime("%Y/%m/%d")
# Reload data from current day
response = self.store.get_item(
Key={"build_datetime": self.db["build_datetime"]})
if response and "Item" in response:
self.db = response["Item"]
def clean(self):
for each in self.clean_dirs:
if each.exists():
shutil.rmtree(each)
each.mkdir(parents=True)
@property
def layers(self):
"""List of layers defined in our jobs/includes/charm-layer-list.inc."""
return yaml.safe_load(
Path(self.db["build_args"]["layer_list"]).read_text(
encoding="utf8"))
@property
def artifacts(self):
"""List of charms or bundles to process."""
return yaml.safe_load(
Path(self.db["build_args"]["artifact_list"]).read_text(
encoding="utf8"))
@property
def layer_index(self):
"""Remote Layer index."""
return self.db["build_args"].get("layer_index", None)
@property
def layer_branch(self):
"""Remote Layer branch."""
return self.db["build_args"].get("layer_branch", None)
@property
def filter_by_tag(self):
"""Filter by tag."""
return self.db["build_args"].get("filter_by_tag", None)
@property
def resource_spec(self):
"""Get Resource specs."""
return self.db["build_args"].get("resource_spec", None)
@property
def to_channels(self) -> List[str]:
"""
Returns destination channels.
Based on the build_args for historical reasons a *risk*
can be returned in the list of channels which implies
latest/<risk> when necessary.
Numerical channels will always be in the format i.ii/risk
"""
chan = self.db["build_args"].get("to_channel", None)
numerical = matched_numerical_channel(chan, SNAP_K8S_TRACK_MAP)
return list(filter(None, [chan, numerical]))
@property
def from_channel(self):
"""Get source channel."""
return self.db["build_args"].get("from_channel", None)
@property
def force(self):
"""Get if we should force a build."""
return self.db["build_args"].get("force", None)
def echo(self, msg):
"""Click echo wrapper."""
click.echo(f"[BuildEnv] {msg}")
def save(self):
"""Store build metadata into stateful db."""
self.echo("Saving build")
self.echo(dict(self.db))
self.db_json.write_text(json.dumps(dict(self.db)))
self.store.put_item(Item=dict(self.db))
def promote_all(self,
from_channel="unpublished",
to_channels=("edge", ),
store="cs"):
"""Promote set of charm artifacts in the store."""
for charm_map in self.artifacts:
for charm_name, charm_opts in charm_map.items():
if not any(match in self.filter_by_tag
for match in charm_opts["tags"]):
continue
cs_store = charm_opts.get("store") or store
if cs_store == "cs":
charm_entity = f"cs:~{charm_opts['namespace']}/{charm_name}"
assert len(to_channels
) == 1, "Charmstore only supports one channel"
assert (
to_channels[0]
in RISKS), "Charmstore supports risk for its channel"
_CharmStore(self).promote(charm_entity, from_channel,
to_channels[0])
elif cs_store == "ch":
_CharmHub(self).promote(charm_name, from_channel,
to_channels)
def download(self, layer_name):
"""Pull layer source from the charm store."""
out = capture(
f"charm pull-source -i {self.layer_index} -b {self.layer_branch} {layer_name}"
)
self.echo(f"- {out.stdout.decode()}")
layer_manifest = {
"rev": self.REV.search(out.stdout.decode()).group(1),
"url": layer_name,
}
return layer_manifest
def pull_layers(self):
"""Clone all downstream layers to be processed locally when doing charm builds."""
layers_to_pull = []
for layer_map in self.layers:
layer_name = list(layer_map.keys())[0]
if layer_name == "layer:index":
continue
layers_to_pull.append(layer_name)
pool = ThreadPool()
results = pool.map(self.download, layers_to_pull)
self.db["pull_layer_manifest"] = [result for result in results]
class BuildEnv:
"""Charm or Bundle build data class"""
try:
build_dir = Path(os.environ.get("CHARM_BUILD_DIR"))
layers_dir = Path(os.environ.get("CHARM_LAYERS_DIR"))
interfaces_dir = Path(os.environ.get("CHARM_INTERFACES_DIR"))
charms_dir = Path(os.environ.get("CHARM_CHARMS_DIR"))
work_dir = Path(os.environ.get("WORKSPACE"))
tmp_dir = work_dir / "tmp"
home_dir = Path(os.environ.get("HOME"))
except TypeError:
raise BuildException(
"CHARM_BUILD_DIR, CHARM_LAYERS_DIR, CHARM_INTERFACES_DIR, WORKSPACE, HOME: "
"Unable to find some or all of these charm build environment variables."
)
def __init__(self, build_type):
self.store = Store("BuildCharms")
self.now = datetime.utcnow()
self.build_type = build_type
self.db = {}
if self.build_type == BuildType.CHARM:
self.db_json = Path("buildcharms.json")
elif self.build_type == BuildType.BUNDLE:
self.db_json = Path("buildbundles.json")
if not self.db.get("build_datetime", None):
self.db["build_datetime"] = self.now.strftime("%Y/%m/%d")
# Reload data from current day
response = self.store.get_item(
Key={"build_datetime": self.db["build_datetime"]})
if response and "Item" in response:
self.db = response["Item"]
@property
def layers(self):
"""List of layers defined in our jobs/includes/charm-layer-list.inc"""
return yaml.safe_load(
Path(self.db["build_args"]["layer_list"]).read_text(
encoding="utf8"))
@property
def artifacts(self):
"""List of charms or bundles to process"""
return yaml.safe_load(
Path(self.db["build_args"]["artifact_list"]).read_text(
encoding="utf8"))
@property
def layer_index(self):
"""Remote Layer index"""
return self.db["build_args"].get("layer_index", None)
@property
def layer_branch(self):
"""Remote Layer branch"""
return self.db["build_args"].get("layer_branch", None)
@property
def filter_by_tag(self):
"""filter tag"""
return self.db["build_args"].get("filter_by_tag", None)
@property
def resource_spec(self):
return self.db["build_args"].get("resource_spec", None)
@property
def to_channel(self):
return self.db["build_args"].get("to_channel", None)
@property
def from_channel(self):
return self.db["build_args"].get("from_channel", None)
@property
def force(self):
return self.db["build_args"].get("force", None)
def _layer_type(self, ltype):
"""Check the type of an individual layer set in the layer list"""
if ltype == "layer":
return LayerType.LAYER
elif ltype == "interface":
return LayerType.INTERFACE
raise BuildException(f"Unknown layer type for {ltype}")
def build_path(self, layer):
ltype, name = layer.split(":")
if self._layer_type(ltype) == LayerType.LAYER:
return str(self.layers_dir / name)
elif self._layer_type(ltype) == LayerType.INTERFACE:
return str(self.interfaces_dir / name)
else:
return None
def save(self):
click.echo("Saving build")
click.echo(dict(self.db))
self.db_json.write_text(json.dumps(dict(self.db)))
self.store.put_item(Item=dict(self.db))
def promote_all(self, from_channel="unpublished", to_channel="edge"):
for charm_map in self.artifacts:
for charm_name, charm_opts in charm_map.items():
if not any(match in self.filter_by_tag
for match in charm_opts["tags"]):
continue
charm_entity = f"cs:~{charm_opts['namespace']}/{charm_name}"
click.echo(
f"Promoting :: {charm_entity:^35} :: from:{from_channel} to: {to_channel}"
)
charm_id = sh.charm.show(charm_entity, "--channel",
from_channel, "id")
charm_id = yaml.safe_load(charm_id.stdout.decode())
resources_args = []
try:
resources = sh.charm(
"list-resources",
charm_id["id"]["Id"],
channel=from_channel,
format="yaml",
)
resources = yaml.safe_load(resources.stdout.decode())
if resources:
resources_args = [(
"--resource",
"{}-{}".format(resource["name"],
resource["revision"]),
) for resource in resources]
except sh.ErrorReturnCode_1:
click.echo("No resources for {}".format(charm_id))
sh.charm.release(charm_id["id"]["Id"], "--channel", to_channel,
*resources_args)
def download(self, layer_name):
out = capture(
f"charm pull-source -i {self.layer_index} -b {self.layer_branch} {layer_name}"
)
click.echo(f"- {out.stdout.decode()}")
rev = re.compile("rev: ([a-zA-Z0-9]+)")
layer_manifest = {
"rev": rev.search(out.stdout.decode()).group(1),
"url": layer_name,
}
return layer_manifest
def pull_layers(self):
"""clone all downstream layers to be processed locally when doing charm builds"""
layers_to_pull = []
for layer_map in self.layers:
layer_name = list(layer_map.keys())[0]
if layer_name == "layer:index":
continue
layers_to_pull.append(layer_name)
pool = ThreadPool()
results = pool.map(self.download, layers_to_pull)
self.db["pull_layer_manifest"] = [result for result in results]
class BuildEnv:
""" Charm or Bundle build data class
"""
try:
build_dir = Path(os.environ.get("CHARM_BUILD_DIR"))
layers_dir = Path(os.environ.get("CHARM_LAYERS_DIR"))
interfaces_dir = Path(os.environ.get("CHARM_INTERFACES_DIR"))
tmp_dir = Path(os.environ.get("WORKSPACE"))
except TypeError:
raise BuildException(
"CHARM_BUILD_DIR, CHARM_LAYERS_DIR, CHARM_INTERFACES_DIR, WORKSPACE: "
"Unable to find some or all of these charm build environment variables."
)
def __init__(self, build_type):
self.store = Store("BuildCharms")
self.now = datetime.utcnow()
self.build_type = build_type
self.db = {}
if self.build_type == BuildType.CHARM:
self.db_json = Path("buildcharms.json")
elif self.build_type == BuildType.BUNDLE:
self.db_json = Path("buildbundles.json")
if not self.db.get("build_datetime", None):
self.db["build_datetime"] = self.now.strftime("%Y/%m/%d")
# Reload data from current day
response = self.store.get_item(
Key={"build_datetime": self.db["build_datetime"]})
if response and "Item" in response:
self.db = response["Item"]
@property
def layers(self):
""" List of layers defined in our jobs/includes/charm-layer-list.inc
"""
return yaml.safe_load(
Path(self.db["build_args"]["layer_list"]).read_text(
encoding="utf8"))
@property
def artifacts(self):
""" List of charms or bundles to process
"""
return yaml.safe_load(
Path(self.db["build_args"]["artifact_list"]).read_text(
encoding="utf8"))
@property
def layer_index(self):
""" Remote Layer index
"""
return self.db["build_args"].get("layer_index", None)
@property
def layer_branch(self):
""" Remote Layer branch
"""
return self.db["build_args"].get("layer_branch", None)
@property
def filter_by_tag(self):
""" filter tag
"""
return self.db["build_args"].get("filter_by_tag", None)
@property
def resource_spec(self):
return self.db["build_args"].get("resource_spec", None)
@property
def to_channel(self):
return self.db["build_args"].get("to_channel", None)
@property
def from_channel(self):
return self.db["build_args"].get("from_channel", None)
@property
def rebuild_cache(self):
return self.db["build_args"].get("rebuild_cache", None)
def _layer_type(self, ltype):
""" Check the type of an individual layer set in the layer list
"""
if ltype == "layer":
return LayerType.LAYER
elif ltype == "interface":
return LayerType.INTERFACE
raise BuildException(f"Unknown layer type for {ltype}")
def build_path(self, layer):
ltype, name = layer.split(":")
if self._layer_type(ltype) == LayerType.LAYER:
return str(self.layers_dir / name)
elif self._layer_type(ltype) == LayerType.INTERFACE:
return str(self.interfaces_dir / name)
else:
return None
def save(self):
click.echo("Saving build")
click.echo(dict(self.db))
self.db_json.write_text(json.dumps(dict(self.db)))
self.store.put_item(Item=dict(self.db))
def promote_all(self, from_channel="unpublished", to_channel="edge"):
for charm_map in self.artifacts:
for charm_name, charm_opts in charm_map.items():
if not any(match in self.filter_by_tag
for match in charm_opts["tags"]):
continue
charm_entity = f"cs:~{charm_opts['namespace']}/{charm_name}"
click.echo(
f"Promoting :: {charm_entity:^35} :: from:{from_channel} to: {to_channel}"
)
charm_id = sh.charm.show(charm_entity, "--channel",
from_channel, "id")
charm_id = yaml.safe_load(charm_id.stdout.decode())
resources_args = []
try:
resources = sh.charm(
"list-resources",
charm_id["id"]["Id"],
channel=from_channel,
format="yaml",
)
resources = yaml.safe_load(resources.stdout.decode())
if resources:
resources_args = [(
"--resource",
"{}-{}".format(resource["name"],
resource["revision"]),
) for resource in resources]
except sh.ErrorReturnCode_1:
click.echo("No resources for {}".format(charm_id))
sh.charm.release(charm_id["id"]["Id"], "--channel", to_channel,
*resources_args)
def download(self, layer_name):
if Path(self.build_path(layer_name)).exists():
click.echo(f"- Refreshing {layer_name} cache.")
cmd_ok(f"git checkout {self.layer_branch}",
cwd=self.build_path(layer_name))
cmd_ok(
f"git.pull origin {self.layer_branch}",
cwd=self.build_path(layer_name),
)
else:
click.echo(f"- Downloading {layer_name}")
cmd_ok(f"charm pull-source -i {self.layer_index} {layer_name}")
return True
def pull_layers(self):
""" clone all downstream layers to be processed locally when doing charm builds
"""
if self.rebuild_cache:
click.echo("- rebuild cache triggered, cleaning out cache.")
shutil.rmtree(str(self.layers_dir))
shutil.rmtree(str(self.interfaces_dir))
os.mkdir(str(self.layers_dir))
os.mkdir(str(self.interfaces_dir))
layers_to_pull = []
for layer_map in self.layers:
layer_name = list(layer_map.keys())[0]
if layer_name == "layer:index":
continue
layers_to_pull.append(layer_name)
pool = ThreadPool()
pool.map(self.download, layers_to_pull)
self.db["pull_layer_manifest"] = []
_paths_to_process = {
"layer": glob("{}/*".format(str(self.layers_dir))),
"interface": glob("{}/*".format(str(self.interfaces_dir))),
}
for prefix, paths in _paths_to_process.items():
for _path in paths:
build_path = _path
if not build_path:
raise BuildException(
f"Could not determine build path for {_path}")
git.checkout(self.layer_branch, _cwd=build_path)
layer_manifest = {
"rev":
git("rev-parse", "HEAD",
_cwd=build_path).stdout.decode().strip(),
"url":
f"{prefix}:{Path(build_path).stem}",
}
self.db["pull_layer_manifest"].append(layer_manifest)
click.echo(
f"- {layer_manifest['url']} at commit: {layer_manifest['rev']}"
)