def _check_path(self, path_to_check, need_write=False):
if not path_to_check:
return
self._check_home(path_to_check)
if not os.access(path_to_check, os.R_OK):
raise BuildError('Unable to read from: {}'.format(path_to_check))
if need_write and not os.access(path_to_check, os.W_OK):
raise BuildError('Unable to write to: {}'.format(path_to_check))
def plan_interfaces(self, layers, output_files, plan):
# Interface includes don't directly map to output files
# as they are computed in combination with the metadata.yaml
if not layers.get('interfaces'):
return
metadata_tactic = [
tactic for tactic in plan
if isinstance(tactic, charmtools.build.tactics.MetadataYAML)
]
if not metadata_tactic:
raise BuildError('At least one layer must provide '
'metadata.yaml')
meta = metadata_tactic[0].process()
if not meta and layers.get('interfaces'):
raise BuildError(
'Includes interfaces but no metadata.yaml to bind them')
elif self.HOOK_TEMPLATE_FILE not in output_files:
raise BuildError('At least one layer must provide %s',
self.HOOK_TEMPLATE_FILE)
elif not meta:
log.warn('Empty metadata.yaml')
template_file = self.target / self.HOOK_TEMPLATE_FILE
target_config = layers["layers"][-1].config
specs = []
used_interfaces = set()
for role in ("provides", "requires", "peers"):
for k, v in meta.get(role, {}).items():
# ex: ["provides", "db", "mysql"]
specs.append([role, k, v["interface"]])
used_interfaces.add(v["interface"])
for iface in layers["interfaces"]:
if iface.name not in used_interfaces:
# we shouldn't include something the charm doesn't use
log.warn("layer.yaml includes {} which isn't "
"used in metadata.yaml".format(iface.name))
continue
for role, relation_name, interface_name in specs:
if interface_name != iface.name:
continue
log.info(
"Processing interface: %s%s", interface_name,
"" if 'deps' in iface.directory.splitall() else
" (from %s)" % iface.directory.relpath())
# COPY phase
plan.append(
charmtools.build.tactics.InterfaceCopy(
iface, relation_name, role, self.target,
target_config))
# Link Phase
plan.append(
charmtools.build.tactics.InterfaceBind(
relation_name, iface.url, self.target, target_config,
template_file))
def load(self, fn):
try:
return yaml.danger_load(fn, Loader=yaml.RoundTripLoader)
except yaml.YAMLError as e:
log.debug(e)
raise BuildError("Failed to process {0}. "
"Ensure the YAML is valid".format(fn.name))
def __call__(self):
""" When the tactic is called, download puppet dependencies and remove
useless folders."""
try:
os.makedirs(self.dest)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
copy_tree(self.entity, self.dest)
try:
check_call(['librarian-puppet', 'install', '--verbose'],
cwd=self.dest)
except OSError as exception:
if exception.errno == 2:
print(exception)
raise BuildError(
"ERROR: File not found. "
"Is 'librarian-puppet' installed? Install with:\n"
"sudo apt install ruby; "
"sudo gem install librarian-puppet\n"
"make sure to use the pip version of charm-build\n"
"(run charm-build instead of charm build)")
else:
raise
make_tarfile(self.dest / "modules.tgz", self.dest / "modules")
shutil.rmtree(self.dest / 'modules')
shutil.rmtree(self.dest / '.tmp')
shutil.rmtree(self.dest / '.librarian')
def exec_plan(self, plan=None, layers=None):
signatures = {}
cont = True
for phase in self.PHASES:
for tactic in plan:
if phase == "lint":
cont &= tactic.lint()
if cont is False and self.force is not True:
# no message, reason will already have been logged
raise BuildError()
elif phase == "read":
# We use a read (into memory phase to make layer comps
# simpler)
tactic.read()
elif phase == "call":
tactic()
elif phase == "sign":
sig = tactic.sign()
if sig:
signatures.update(sig)
new_repo = not self.manifest.exists()
if new_repo:
added, changed, removed = set(), set(), set()
else:
added, changed, _ = utils.delta_signatures(self.manifest)
removed = self.clean_removed(signatures)
# write out the sigs
if "sign" in self.PHASES:
self.write_signatures(signatures, layers)
if self.report:
self.write_report(new_repo, added, changed, removed)
def _check_path(self, path_to_check, need_write=False, can_create=False):
if not path_to_check:
return
if not os.path.exists(path_to_check):
if not can_create:
raise BuildError('Missing required path: '
'{}'.format(path_to_check))
try:
path_to_check.makedirs_p()
except Exception:
raise BuildError('Unable to create required path: '
'{}'.format(path_to_check))
if not os.access(path_to_check, os.R_OK):
raise BuildError('Unable to read from: {}'.format(path_to_check))
if need_write and not os.access(path_to_check, os.W_OK):
raise BuildError('Unable to write to: {}'.format(path_to_check))
def validate(self):
self._validate_charm_repo()
if not self.manifest.exists():
return [], [], []
a, c, d = utils.delta_signatures(self.manifest)
for f in a:
log.warn(
"Conflict: File in destination directory "
"was added after charm build: %s", f)
for f in c:
log.warn(
"Conflict: File in destination directory "
"was modified after charm build: %s", f)
for f in d:
log.warn(
"Conflict: File in destination directory "
"was deleted after charm build: %s", f)
if a or c or d:
if self.force is True:
log.info("Continuing with known changes to target layer. "
"Changes will be overwritten")
else:
raise BuildError(
"Unable to continue due to unexpected modifications "
"(try --force)")
return a, c, d
def fetch(self):
try:
fetcher = get_fetcher(self.url)
except FetchError:
# We might be passing a local dir path directly
# which fetchers don't currently support
self.directory = path(self.url)
else:
if hasattr(fetcher, "path") and fetcher.path.exists():
self.directory = path(fetcher.path)
else:
if not self.target_repo.exists():
self.target_repo.makedirs_p()
self.directory = path(fetcher.fetch(self.target_repo))
if not self.directory.exists():
raise BuildError("Unable to locate {}. "
"Do you need to set {}?".format(
self.url, self.ENVIRON))
self.config_file = self.directory / self.CONFIG_FILE
if not self.config_file.exists():
if self.OLD_CONFIG and (self.directory / self.OLD_CONFIG).exists():
self.config_file = (self.directory / self.OLD_CONFIG)
self._name = self.config.name
return self
def get(cls, entity, target, layer, next_config, current_config,
existing_tactic):
"""
Factory method to get an instance of the correct Tactic to handle the
given entity.
"""
for candidate in current_config.tactics + DEFAULT_TACTICS:
argspec = getargspec(candidate.trigger)
if len(argspec.args) == 2:
# old calling convention
name = candidate.__name__
if name not in Tactic._warnings:
Tactic._warnings[name] = True
log.warn('Deprecated method signature for trigger in %s',
name)
args = [entity.relpath(layer.directory)]
else:
# new calling convention
args = [entity, target, layer, next_config]
if candidate.trigger(*args):
tactic = candidate(entity, target, layer, next_config)
if existing_tactic is not None:
tactic = tactic.combine(existing_tactic)
return tactic
raise BuildError('Unable to process file: {} '
'(no tactics matched)'.format(entity))
def validate(self):
self._validate_charm_repo()
if not self.manifest.exists():
return [], [], []
a, c, d = utils.delta_signatures(self.manifest)
for f in a:
log.warn(
"Added unexpected file, should be in a base layer: %s", f)
for f in c:
log.warn(
"Changed file owned by another layer: %s", f)
for f in d:
log.warn(
"Deleted a file owned by another layer: %s", f)
if a or c or d:
if self.force is True:
log.info(
"Continuing with known changes to target layer. "
"Changes will be overwritten")
else:
raise BuildError(
"Unable to continue due to unexpected modifications "
"(try --force)")
return a, c, d
def load(self, fn):
"""Load the yaml file and return the contents as objects."""
try:
return yaml.load(fn, Loader=yaml.RoundTripLoader)
except yaml.YAMLError as e:
log.debug(e)
raise BuildError("Failed to process {0}. "
"Ensure the YAML is valid".format(fn.name))
def _check_path(self, path_to_check, need_write=False):
home_dir = utils.get_home()
home_msg = ('For security reasons, only paths under your '
'home directory can be accessed, including '
'the build output dir, JUJU_REPOSITORY, '
'LAYER_PATH, INTERFACE_PATH, and any '
'wheelhouse overrides.')
if not home_dir: # expansion failed
log.warn('Could not determine home directory.')
log.warn(home_msg)
elif os.path.abspath(path_to_check).startswith(home_dir):
log.warn('The path {} is not under your '
'home directory.'.format(home_dir))
log.warn(home_msg)
if not os.access(path_to_check, os.R_OK):
raise BuildError('Unable to read from: {}'.format(path_to_check))
if need_write and not os.access(path_to_check, os.W_OK):
raise BuildError('Unable to write to: {}'.format(path_to_check))
def _run_in_venv(self, *args):
assert self._venv is not None
# have to use bash to activate the venv properly first
res = utils.Process(
('bash', '-c',
' '.join(('.', self._venv / 'bin' / 'activate', ';') + args)))()
if res.exit_code != 0:
raise BuildError(res.output)
return res
def create_repo(self):
# Generated output will go into this directory
base = path(self.output_dir)
self.repo = (base / (self.series if self.series else 'builds'))
# And anything it includes from will be placed here
# outside the series
self.deps = (base / "deps")
if not (self.name and str(self.name)[0] in string.ascii_lowercase):
raise BuildError('Charm name must start with a lower-case letter')
self.target_dir = (self.repo / self.name)
def charm_metadata(self):
if not hasattr(self, '_charm_metadata'):
md = path(self.charm) / "metadata.yaml"
try:
setattr(self, '_charm_metadata',
yaml.load(md.open()) if md.exists() else None)
except yaml.YAMLError as e:
log.debug(e)
raise BuildError("Failed to process {0}. "
"Ensure the YAML is valid".format(md))
return self._charm_metadata
def plan_storage(self, layers, output_files, plan):
# Storage hooks don't directly map to output files
# as they are computed in combination with the metadata.yaml
metadata_tactic = [tactic for tactic in plan if isinstance(
tactic, charmtools.build.tactics.MetadataYAML)]
if not metadata_tactic:
raise BuildError('At least one layer must provide metadata.yaml')
meta_tac = metadata_tactic[0]
meta_tac.process()
if not meta_tac.storage:
return
if self.HOOK_TEMPLATE_FILE not in output_files:
raise BuildError('At least one layer must provide %s',
self.HOOK_TEMPLATE_FILE)
template_file = self.target / self.HOOK_TEMPLATE_FILE
target_config = layers["layers"][-1].config
for name, owner in meta_tac.storage.items():
plan.append(
charmtools.build.tactics.StorageBind(
name, owner, self.target,
target_config, template_file))
def get(cls, entity, target, layer, next_config, existing_tactic):
"""
Factory method to get an instance of the correct Tactic to handle the
given entity.
"""
for candidate in next_config.tactics + DEFAULT_TACTICS:
if candidate.trigger(entity, target, layer, next_config):
tactic = candidate(entity, target, layer, next_config)
if existing_tactic is not None:
tactic = tactic.combine(existing_tactic)
return tactic
raise BuildError('Unable to process file: {} '
'(no tactics matched)'.format(entity))
def validate(self):
if not (self.name and str(self.name)[0] in string.ascii_lowercase):
raise BuildError('Charm name must start with a lower-case letter')
for cls in (InterfaceFetcher, LayerFetcher):
if cls.OLD_ENVIRON in os.environ and cls.ENVIRON not in os.environ:
log.warning('DEPRECATED: {} environment variable; '
'please use {} instead'.format(
cls.OLD_ENVIRON, cls.ENVIRON))
self._validate_charm_repo()
if not self.manifest.exists():
return [], [], []
a, c, d = utils.delta_signatures(self.manifest)
for f in a:
log.warn(
"Conflict: File in destination directory "
"was added after charm build: %s", f)
for f in c:
log.warn(
"Conflict: File in destination directory "
"was modified after charm build: %s", f)
for f in d:
log.warn(
"Conflict: File in destination directory "
"was deleted after charm build: %s", f)
if a or c or d:
if self.force is True:
log.info("Continuing with known changes to target layer. "
"Changes will be overwritten")
else:
raise BuildError(
"Unable to continue due to unexpected modifications "
"(try --force)")
return a, c, d
def plan_storage(self, layers, output_files, plan):
"""
Add Tactics to the plan for each storage endpoint to render hooks
for that storage endpoint from the template.
:param layers: Data structure containing all of the layers composing
this charm, along with some overall metadata.
:type layers: dict
:param output_files: Mapping of file Paths to Tactics that should
process those files.
:type output_files: dict
:param plan: List of all Tactics that need to be invoked.
:type plan: list
"""
# Storage hooks don't directly map to output files
# as they are computed in combination with the metadata.yaml
metadata_tactic = [
tactic for tactic in plan
if isinstance(tactic, charmtools.build.tactics.MetadataYAML)
]
if not metadata_tactic:
raise BuildError('At least one layer must provide metadata.yaml')
meta_tac = metadata_tactic[0]
meta_tac.process()
if not meta_tac.storage:
return
if self.HOOK_TEMPLATE_FILE not in output_files:
raise BuildError('At least one layer must provide %s',
self.HOOK_TEMPLATE_FILE)
template_file = path(self.target.directory) / self.HOOK_TEMPLATE_FILE
target_config = layers["layers"][-1].config
for name, owner in meta_tac.storage.items():
plan.append(
charmtools.build.tactics.StorageBind(name, owner, self.target,
target_config,
output_files,
template_file))
def normalize_cache_dir(self):
charm_cache_dir = os.environ.get('CHARM_CACHE_DIR')
if not self.cache_dir:
if charm_cache_dir:
self.cache_dir = path(charm_cache_dir)
else:
self.cache_dir = path('~/.cache/charm').expanduser()
self.cache_dir = self.cache_dir.abspath()
if self.cache_dir.startswith(path(self.charm).abspath()):
raise BuildError('Cache directory nested under source directory. '
'This will cause recursive nesting of build '
'artifacts and can fill up your disk. Please '
'specify a different build directory with '
'--cache-dir or $CHARM_CACHE_DIR')
def check_paths(self):
paths_to_check = [
self.output_dir,
self.wheelhouse_overrides,
os.environ.get('JUJU_REPOSITORY'),
os.environ.get('LAYER_PATH'),
os.environ.get('INTERACE_PATH'),
]
for path_to_check in paths_to_check:
if path_to_check and not self._check_path(path_to_check):
raise BuildError('For security reasons, only paths under your '
'home directory can be accessed, including '
'the build output dir, JUJU_REPOSITORY, '
'LAYER_PATH, INTERFACE_PATH, and any '
'wheelhouse overrides')
def read(self):
if self.lines is None:
src = path(self.entity)
if src.exists():
for req in requirements.parse(src.text()):
if req.name is None:
raise BuildError(
'Unable to determine package name for "{}"; '
'did you forget "#egg=..."?'.format(
req.line.strip()))
self._layer_refs[safe_name(req.name)] = self.layer.url
self.lines = (['# ' + self.layer.url] +
src.lines(retain=False) + [''])
else:
self.lines = []
def fetch(self):
try:
# In order to lock the fetcher we need to adjust the self.url
# to get the right thing. Curiously, self.url is actually
# "layer:something" here, and so we can match on that.
if self.lock:
url = make_url_from_lock_for_layer(self.lock,
self.use_branches)
else:
url = self.url
fetcher = get_fetcher(url)
except FetchError:
# We might be passing a local dir path directly
# which fetchers don't currently support
self.directory = path(self.url)
self.revision = RepoFetcher(self.url).get_revision(self.url)
else:
if hasattr(fetcher, "path") and fetcher.path.exists():
self.directory = path(fetcher.path)
else:
if not self.target_repo.exists():
self.target_repo.makedirs_p()
self.directory = path(fetcher.fetch(self.target_repo))
self.fetched = True
self.fetched_url = getattr(fetcher, "fetched_url", None)
self.vcs = getattr(fetcher, "vcs", None)
self.revision = fetcher.get_revision(self.directory)
self.branch = fetcher.get_branch_for_revision(
self.directory, self.revision)
if not self.directory.exists():
raise BuildError("Unable to locate {}. "
"Do you need to set {}?".format(
self.url, self.ENVIRON))
if self.config_file:
self.config_file = self.directory / self.config_file
if self.old_config_file:
self.old_config_file = self.directory / self.old_config_file
self._name = self.config.name
return self
def normalize_build_dir(self):
charm_build_dir = os.environ.get('CHARM_BUILD_DIR')
juju_repo_dir = os.environ.get('JUJU_REPOSITORY')
series = self.series or 'builds'
if not self.build_dir:
if self.output_dir:
self.build_dir = self.output_dir / series
elif charm_build_dir:
self.build_dir = path(charm_build_dir)
elif juju_repo_dir:
self.build_dir = path(juju_repo_dir) / series
else:
log.warn('Build dir not specified via command-line or '
'environment; defaulting to /tmp/charm-builds')
self.build_dir = path('/tmp/charm-builds')
self.build_dir = self.build_dir.abspath()
if self.build_dir.startswith(path(self.charm).abspath()):
raise BuildError('Build directory nested under source directory. '
'This will cause recursive nesting of build '
'artifacts and can fill up your disk. Please '
'specify a different build directory with '
'--build-dir or $CHARM_BUILD_DIR')
def plan_hooks(self, layers, output_files, plan):
"""
Add a Tactic to the plan to handle rendering all standard hooks from
the template, if not explicitly overridden.
:param layers: Data structure containing all of the layers composing
this charm, along with some overall metadata.
:type layers: dict
:param output_files: Mapping of file Paths to Tactics that should
process those files.
:type output_files: dict
:param plan: List of all Tactics that need to be invoked.
:type plan: list
"""
if self.HOOK_TEMPLATE_FILE not in output_files:
raise BuildError('At least one layer must provide %s',
self.HOOK_TEMPLATE_FILE)
template_file = self.target.directory / self.HOOK_TEMPLATE_FILE
target_config = layers["layers"][-1].config
source_layer = output_files[self.HOOK_TEMPLATE_FILE].layer
plan.append(
charmtools.build.tactics.StandardHooksBind(
'hook', source_layer.url, self.target, target_config,
output_files, template_file))
def inspect(self):
self.charm = path(self.charm).abspath()
if not self._check_path(self.charm):
raise BuildError('For security reasons, only paths under '
'your home directory can be accessed')
inspector.inspect(self.charm, force_styling=self.force_raw)
def _check_path(self, path_to_check):
home_dir = utils.get_home()
if not home_dir: # expansion failed
raise BuildError('Could not determine home directory')
return os.path.abspath(path_to_check).startswith(home_dir)
def plan_interfaces(self, layers, output_files, plan):
"""
Add Tactics to the plan for each relation endpoint to render hooks
for that relation endpoint from the template, as well as a tactic
to pull in the interface layer's code (if there is an interface layer).
:param layers: Data structure containing all of the layers composing
this charm, along with some overall metadata.
:type layers: dict
:param output_files: Mapping of file Paths to Tactics that should
process those files.
:type output_files: dict
:param plan: List of all Tactics that need to be invoked.
:type plan: list
"""
# Interface includes don't directly map to output files
# as they are computed in combination with the metadata.yaml
metadata_tactic = [
tactic for tactic in plan
if isinstance(tactic, charmtools.build.tactics.MetadataYAML)
]
if not metadata_tactic:
raise BuildError('At least one layer must provide '
'metadata.yaml')
meta = metadata_tactic[0].process()
if not meta and layers.get('interfaces'):
raise BuildError(
'Includes interfaces but no metadata.yaml to bind them')
elif self.HOOK_TEMPLATE_FILE not in output_files:
raise BuildError('At least one layer must provide %s',
self.HOOK_TEMPLATE_FILE)
elif not meta:
log.warn('Empty metadata.yaml')
template_file = path(self.target.directory) / self.HOOK_TEMPLATE_FILE
target_config = layers["layers"][-1].config
specs = []
used_interfaces = set()
for role in ("provides", "requires", "peers"):
for k, v in meta.get(role, {}).items():
# ex: ["provides", "db", "mysql"]
specs.append([role, k, v["interface"]])
used_interfaces.add(v["interface"])
for iface in layers.get("interfaces", []):
if iface.name not in used_interfaces:
# we shouldn't include something the charm doesn't use
log.warn("layer.yaml includes {} which isn't "
"used in metadata.yaml".format(iface.name))
continue
for role, relation_name, interface_name in specs:
if interface_name != iface.name:
continue
log.info(
"Processing interface: %s%s", interface_name,
"" if iface.directory.startswith(self.cache_dir) else
" (from %s)" % iface.directory.relpath())
# COPY phase
plan.append(
charmtools.build.tactics.InterfaceCopy(
iface, relation_name, role, self.target,
target_config))
# Link Phase
# owner = metadata_tactic[0].layer.url
owner = output_files[self.HOOK_TEMPLATE_FILE].layer.url
for role, relation_name, interface_name in specs:
plan.append(
charmtools.build.tactics.InterfaceBind(relation_name, owner,
self.target,
target_config,
output_files,
template_file))