def _check_path(self, path_to_check, need_write=False): if not path_to_check: return self._check_home(path_to_check) if not os.access(path_to_check, os.R_OK): raise BuildError('Unable to read from: {}'.format(path_to_check)) if need_write and not os.access(path_to_check, os.W_OK): raise BuildError('Unable to write to: {}'.format(path_to_check))
def plan_interfaces(self, layers, output_files, plan): # Interface includes don't directly map to output files # as they are computed in combination with the metadata.yaml if not layers.get('interfaces'): return metadata_tactic = [ tactic for tactic in plan if isinstance(tactic, charmtools.build.tactics.MetadataYAML) ] if not metadata_tactic: raise BuildError('At least one layer must provide ' 'metadata.yaml') meta = metadata_tactic[0].process() if not meta and layers.get('interfaces'): raise BuildError( 'Includes interfaces but no metadata.yaml to bind them') elif self.HOOK_TEMPLATE_FILE not in output_files: raise BuildError('At least one layer must provide %s', self.HOOK_TEMPLATE_FILE) elif not meta: log.warn('Empty metadata.yaml') template_file = self.target / self.HOOK_TEMPLATE_FILE target_config = layers["layers"][-1].config specs = [] used_interfaces = set() for role in ("provides", "requires", "peers"): for k, v in meta.get(role, {}).items(): # ex: ["provides", "db", "mysql"] specs.append([role, k, v["interface"]]) used_interfaces.add(v["interface"]) for iface in layers["interfaces"]: if iface.name not in used_interfaces: # we shouldn't include something the charm doesn't use log.warn("layer.yaml includes {} which isn't " "used in metadata.yaml".format(iface.name)) continue for role, relation_name, interface_name in specs: if interface_name != iface.name: continue log.info( "Processing interface: %s%s", interface_name, "" if 'deps' in iface.directory.splitall() else " (from %s)" % iface.directory.relpath()) # COPY phase plan.append( charmtools.build.tactics.InterfaceCopy( iface, relation_name, role, self.target, target_config)) # Link Phase plan.append( charmtools.build.tactics.InterfaceBind( relation_name, iface.url, self.target, target_config, template_file))
def load(self, fn): try: return yaml.danger_load(fn, Loader=yaml.RoundTripLoader) except yaml.YAMLError as e: log.debug(e) raise BuildError("Failed to process {0}. " "Ensure the YAML is valid".format(fn.name))
def __call__(self): """ When the tactic is called, download puppet dependencies and remove useless folders.""" try: os.makedirs(self.dest) except OSError as exception: if exception.errno != errno.EEXIST: raise copy_tree(self.entity, self.dest) try: check_call(['librarian-puppet', 'install', '--verbose'], cwd=self.dest) except OSError as exception: if exception.errno == 2: print(exception) raise BuildError( "ERROR: File not found. " "Is 'librarian-puppet' installed? Install with:\n" "sudo apt install ruby; " "sudo gem install librarian-puppet\n" "make sure to use the pip version of charm-build\n" "(run charm-build instead of charm build)") else: raise make_tarfile(self.dest / "modules.tgz", self.dest / "modules") shutil.rmtree(self.dest / 'modules') shutil.rmtree(self.dest / '.tmp') shutil.rmtree(self.dest / '.librarian')
def exec_plan(self, plan=None, layers=None): signatures = {} cont = True for phase in self.PHASES: for tactic in plan: if phase == "lint": cont &= tactic.lint() if cont is False and self.force is not True: # no message, reason will already have been logged raise BuildError() elif phase == "read": # We use a read (into memory phase to make layer comps # simpler) tactic.read() elif phase == "call": tactic() elif phase == "sign": sig = tactic.sign() if sig: signatures.update(sig) new_repo = not self.manifest.exists() if new_repo: added, changed, removed = set(), set(), set() else: added, changed, _ = utils.delta_signatures(self.manifest) removed = self.clean_removed(signatures) # write out the sigs if "sign" in self.PHASES: self.write_signatures(signatures, layers) if self.report: self.write_report(new_repo, added, changed, removed)
def _check_path(self, path_to_check, need_write=False, can_create=False): if not path_to_check: return if not os.path.exists(path_to_check): if not can_create: raise BuildError('Missing required path: ' '{}'.format(path_to_check)) try: path_to_check.makedirs_p() except Exception: raise BuildError('Unable to create required path: ' '{}'.format(path_to_check)) if not os.access(path_to_check, os.R_OK): raise BuildError('Unable to read from: {}'.format(path_to_check)) if need_write and not os.access(path_to_check, os.W_OK): raise BuildError('Unable to write to: {}'.format(path_to_check))
def validate(self): self._validate_charm_repo() if not self.manifest.exists(): return [], [], [] a, c, d = utils.delta_signatures(self.manifest) for f in a: log.warn( "Conflict: File in destination directory " "was added after charm build: %s", f) for f in c: log.warn( "Conflict: File in destination directory " "was modified after charm build: %s", f) for f in d: log.warn( "Conflict: File in destination directory " "was deleted after charm build: %s", f) if a or c or d: if self.force is True: log.info("Continuing with known changes to target layer. " "Changes will be overwritten") else: raise BuildError( "Unable to continue due to unexpected modifications " "(try --force)") return a, c, d
def fetch(self): try: fetcher = get_fetcher(self.url) except FetchError: # We might be passing a local dir path directly # which fetchers don't currently support self.directory = path(self.url) else: if hasattr(fetcher, "path") and fetcher.path.exists(): self.directory = path(fetcher.path) else: if not self.target_repo.exists(): self.target_repo.makedirs_p() self.directory = path(fetcher.fetch(self.target_repo)) if not self.directory.exists(): raise BuildError("Unable to locate {}. " "Do you need to set {}?".format( self.url, self.ENVIRON)) self.config_file = self.directory / self.CONFIG_FILE if not self.config_file.exists(): if self.OLD_CONFIG and (self.directory / self.OLD_CONFIG).exists(): self.config_file = (self.directory / self.OLD_CONFIG) self._name = self.config.name return self
def get(cls, entity, target, layer, next_config, current_config, existing_tactic): """ Factory method to get an instance of the correct Tactic to handle the given entity. """ for candidate in current_config.tactics + DEFAULT_TACTICS: argspec = getargspec(candidate.trigger) if len(argspec.args) == 2: # old calling convention name = candidate.__name__ if name not in Tactic._warnings: Tactic._warnings[name] = True log.warn('Deprecated method signature for trigger in %s', name) args = [entity.relpath(layer.directory)] else: # new calling convention args = [entity, target, layer, next_config] if candidate.trigger(*args): tactic = candidate(entity, target, layer, next_config) if existing_tactic is not None: tactic = tactic.combine(existing_tactic) return tactic raise BuildError('Unable to process file: {} ' '(no tactics matched)'.format(entity))
def validate(self): self._validate_charm_repo() if not self.manifest.exists(): return [], [], [] a, c, d = utils.delta_signatures(self.manifest) for f in a: log.warn( "Added unexpected file, should be in a base layer: %s", f) for f in c: log.warn( "Changed file owned by another layer: %s", f) for f in d: log.warn( "Deleted a file owned by another layer: %s", f) if a or c or d: if self.force is True: log.info( "Continuing with known changes to target layer. " "Changes will be overwritten") else: raise BuildError( "Unable to continue due to unexpected modifications " "(try --force)") return a, c, d
def load(self, fn): """Load the yaml file and return the contents as objects.""" try: return yaml.load(fn, Loader=yaml.RoundTripLoader) except yaml.YAMLError as e: log.debug(e) raise BuildError("Failed to process {0}. " "Ensure the YAML is valid".format(fn.name))
def _check_path(self, path_to_check, need_write=False): home_dir = utils.get_home() home_msg = ('For security reasons, only paths under your ' 'home directory can be accessed, including ' 'the build output dir, JUJU_REPOSITORY, ' 'LAYER_PATH, INTERFACE_PATH, and any ' 'wheelhouse overrides.') if not home_dir: # expansion failed log.warn('Could not determine home directory.') log.warn(home_msg) elif os.path.abspath(path_to_check).startswith(home_dir): log.warn('The path {} is not under your ' 'home directory.'.format(home_dir)) log.warn(home_msg) if not os.access(path_to_check, os.R_OK): raise BuildError('Unable to read from: {}'.format(path_to_check)) if need_write and not os.access(path_to_check, os.W_OK): raise BuildError('Unable to write to: {}'.format(path_to_check))
def _run_in_venv(self, *args): assert self._venv is not None # have to use bash to activate the venv properly first res = utils.Process( ('bash', '-c', ' '.join(('.', self._venv / 'bin' / 'activate', ';') + args)))() if res.exit_code != 0: raise BuildError(res.output) return res
def create_repo(self): # Generated output will go into this directory base = path(self.output_dir) self.repo = (base / (self.series if self.series else 'builds')) # And anything it includes from will be placed here # outside the series self.deps = (base / "deps") if not (self.name and str(self.name)[0] in string.ascii_lowercase): raise BuildError('Charm name must start with a lower-case letter') self.target_dir = (self.repo / self.name)
def charm_metadata(self): if not hasattr(self, '_charm_metadata'): md = path(self.charm) / "metadata.yaml" try: setattr(self, '_charm_metadata', yaml.load(md.open()) if md.exists() else None) except yaml.YAMLError as e: log.debug(e) raise BuildError("Failed to process {0}. " "Ensure the YAML is valid".format(md)) return self._charm_metadata
def plan_storage(self, layers, output_files, plan): # Storage hooks don't directly map to output files # as they are computed in combination with the metadata.yaml metadata_tactic = [tactic for tactic in plan if isinstance( tactic, charmtools.build.tactics.MetadataYAML)] if not metadata_tactic: raise BuildError('At least one layer must provide metadata.yaml') meta_tac = metadata_tactic[0] meta_tac.process() if not meta_tac.storage: return if self.HOOK_TEMPLATE_FILE not in output_files: raise BuildError('At least one layer must provide %s', self.HOOK_TEMPLATE_FILE) template_file = self.target / self.HOOK_TEMPLATE_FILE target_config = layers["layers"][-1].config for name, owner in meta_tac.storage.items(): plan.append( charmtools.build.tactics.StorageBind( name, owner, self.target, target_config, template_file))
def get(cls, entity, target, layer, next_config, existing_tactic): """ Factory method to get an instance of the correct Tactic to handle the given entity. """ for candidate in next_config.tactics + DEFAULT_TACTICS: if candidate.trigger(entity, target, layer, next_config): tactic = candidate(entity, target, layer, next_config) if existing_tactic is not None: tactic = tactic.combine(existing_tactic) return tactic raise BuildError('Unable to process file: {} ' '(no tactics matched)'.format(entity))
def validate(self): if not (self.name and str(self.name)[0] in string.ascii_lowercase): raise BuildError('Charm name must start with a lower-case letter') for cls in (InterfaceFetcher, LayerFetcher): if cls.OLD_ENVIRON in os.environ and cls.ENVIRON not in os.environ: log.warning('DEPRECATED: {} environment variable; ' 'please use {} instead'.format( cls.OLD_ENVIRON, cls.ENVIRON)) self._validate_charm_repo() if not self.manifest.exists(): return [], [], [] a, c, d = utils.delta_signatures(self.manifest) for f in a: log.warn( "Conflict: File in destination directory " "was added after charm build: %s", f) for f in c: log.warn( "Conflict: File in destination directory " "was modified after charm build: %s", f) for f in d: log.warn( "Conflict: File in destination directory " "was deleted after charm build: %s", f) if a or c or d: if self.force is True: log.info("Continuing with known changes to target layer. " "Changes will be overwritten") else: raise BuildError( "Unable to continue due to unexpected modifications " "(try --force)") return a, c, d
def plan_storage(self, layers, output_files, plan): """ Add Tactics to the plan for each storage endpoint to render hooks for that storage endpoint from the template. :param layers: Data structure containing all of the layers composing this charm, along with some overall metadata. :type layers: dict :param output_files: Mapping of file Paths to Tactics that should process those files. :type output_files: dict :param plan: List of all Tactics that need to be invoked. :type plan: list """ # Storage hooks don't directly map to output files # as they are computed in combination with the metadata.yaml metadata_tactic = [ tactic for tactic in plan if isinstance(tactic, charmtools.build.tactics.MetadataYAML) ] if not metadata_tactic: raise BuildError('At least one layer must provide metadata.yaml') meta_tac = metadata_tactic[0] meta_tac.process() if not meta_tac.storage: return if self.HOOK_TEMPLATE_FILE not in output_files: raise BuildError('At least one layer must provide %s', self.HOOK_TEMPLATE_FILE) template_file = path(self.target.directory) / self.HOOK_TEMPLATE_FILE target_config = layers["layers"][-1].config for name, owner in meta_tac.storage.items(): plan.append( charmtools.build.tactics.StorageBind(name, owner, self.target, target_config, output_files, template_file))
def normalize_cache_dir(self): charm_cache_dir = os.environ.get('CHARM_CACHE_DIR') if not self.cache_dir: if charm_cache_dir: self.cache_dir = path(charm_cache_dir) else: self.cache_dir = path('~/.cache/charm').expanduser() self.cache_dir = self.cache_dir.abspath() if self.cache_dir.startswith(path(self.charm).abspath()): raise BuildError('Cache directory nested under source directory. ' 'This will cause recursive nesting of build ' 'artifacts and can fill up your disk. Please ' 'specify a different build directory with ' '--cache-dir or $CHARM_CACHE_DIR')
def check_paths(self): paths_to_check = [ self.output_dir, self.wheelhouse_overrides, os.environ.get('JUJU_REPOSITORY'), os.environ.get('LAYER_PATH'), os.environ.get('INTERACE_PATH'), ] for path_to_check in paths_to_check: if path_to_check and not self._check_path(path_to_check): raise BuildError('For security reasons, only paths under your ' 'home directory can be accessed, including ' 'the build output dir, JUJU_REPOSITORY, ' 'LAYER_PATH, INTERFACE_PATH, and any ' 'wheelhouse overrides')
def read(self): if self.lines is None: src = path(self.entity) if src.exists(): for req in requirements.parse(src.text()): if req.name is None: raise BuildError( 'Unable to determine package name for "{}"; ' 'did you forget "#egg=..."?'.format( req.line.strip())) self._layer_refs[safe_name(req.name)] = self.layer.url self.lines = (['# ' + self.layer.url] + src.lines(retain=False) + ['']) else: self.lines = []
def fetch(self): try: # In order to lock the fetcher we need to adjust the self.url # to get the right thing. Curiously, self.url is actually # "layer:something" here, and so we can match on that. if self.lock: url = make_url_from_lock_for_layer(self.lock, self.use_branches) else: url = self.url fetcher = get_fetcher(url) except FetchError: # We might be passing a local dir path directly # which fetchers don't currently support self.directory = path(self.url) self.revision = RepoFetcher(self.url).get_revision(self.url) else: if hasattr(fetcher, "path") and fetcher.path.exists(): self.directory = path(fetcher.path) else: if not self.target_repo.exists(): self.target_repo.makedirs_p() self.directory = path(fetcher.fetch(self.target_repo)) self.fetched = True self.fetched_url = getattr(fetcher, "fetched_url", None) self.vcs = getattr(fetcher, "vcs", None) self.revision = fetcher.get_revision(self.directory) self.branch = fetcher.get_branch_for_revision( self.directory, self.revision) if not self.directory.exists(): raise BuildError("Unable to locate {}. " "Do you need to set {}?".format( self.url, self.ENVIRON)) if self.config_file: self.config_file = self.directory / self.config_file if self.old_config_file: self.old_config_file = self.directory / self.old_config_file self._name = self.config.name return self
def normalize_build_dir(self): charm_build_dir = os.environ.get('CHARM_BUILD_DIR') juju_repo_dir = os.environ.get('JUJU_REPOSITORY') series = self.series or 'builds' if not self.build_dir: if self.output_dir: self.build_dir = self.output_dir / series elif charm_build_dir: self.build_dir = path(charm_build_dir) elif juju_repo_dir: self.build_dir = path(juju_repo_dir) / series else: log.warn('Build dir not specified via command-line or ' 'environment; defaulting to /tmp/charm-builds') self.build_dir = path('/tmp/charm-builds') self.build_dir = self.build_dir.abspath() if self.build_dir.startswith(path(self.charm).abspath()): raise BuildError('Build directory nested under source directory. ' 'This will cause recursive nesting of build ' 'artifacts and can fill up your disk. Please ' 'specify a different build directory with ' '--build-dir or $CHARM_BUILD_DIR')
def plan_hooks(self, layers, output_files, plan): """ Add a Tactic to the plan to handle rendering all standard hooks from the template, if not explicitly overridden. :param layers: Data structure containing all of the layers composing this charm, along with some overall metadata. :type layers: dict :param output_files: Mapping of file Paths to Tactics that should process those files. :type output_files: dict :param plan: List of all Tactics that need to be invoked. :type plan: list """ if self.HOOK_TEMPLATE_FILE not in output_files: raise BuildError('At least one layer must provide %s', self.HOOK_TEMPLATE_FILE) template_file = self.target.directory / self.HOOK_TEMPLATE_FILE target_config = layers["layers"][-1].config source_layer = output_files[self.HOOK_TEMPLATE_FILE].layer plan.append( charmtools.build.tactics.StandardHooksBind( 'hook', source_layer.url, self.target, target_config, output_files, template_file))
def inspect(self): self.charm = path(self.charm).abspath() if not self._check_path(self.charm): raise BuildError('For security reasons, only paths under ' 'your home directory can be accessed') inspector.inspect(self.charm, force_styling=self.force_raw)
def _check_path(self, path_to_check): home_dir = utils.get_home() if not home_dir: # expansion failed raise BuildError('Could not determine home directory') return os.path.abspath(path_to_check).startswith(home_dir)
def plan_interfaces(self, layers, output_files, plan): """ Add Tactics to the plan for each relation endpoint to render hooks for that relation endpoint from the template, as well as a tactic to pull in the interface layer's code (if there is an interface layer). :param layers: Data structure containing all of the layers composing this charm, along with some overall metadata. :type layers: dict :param output_files: Mapping of file Paths to Tactics that should process those files. :type output_files: dict :param plan: List of all Tactics that need to be invoked. :type plan: list """ # Interface includes don't directly map to output files # as they are computed in combination with the metadata.yaml metadata_tactic = [ tactic for tactic in plan if isinstance(tactic, charmtools.build.tactics.MetadataYAML) ] if not metadata_tactic: raise BuildError('At least one layer must provide ' 'metadata.yaml') meta = metadata_tactic[0].process() if not meta and layers.get('interfaces'): raise BuildError( 'Includes interfaces but no metadata.yaml to bind them') elif self.HOOK_TEMPLATE_FILE not in output_files: raise BuildError('At least one layer must provide %s', self.HOOK_TEMPLATE_FILE) elif not meta: log.warn('Empty metadata.yaml') template_file = path(self.target.directory) / self.HOOK_TEMPLATE_FILE target_config = layers["layers"][-1].config specs = [] used_interfaces = set() for role in ("provides", "requires", "peers"): for k, v in meta.get(role, {}).items(): # ex: ["provides", "db", "mysql"] specs.append([role, k, v["interface"]]) used_interfaces.add(v["interface"]) for iface in layers.get("interfaces", []): if iface.name not in used_interfaces: # we shouldn't include something the charm doesn't use log.warn("layer.yaml includes {} which isn't " "used in metadata.yaml".format(iface.name)) continue for role, relation_name, interface_name in specs: if interface_name != iface.name: continue log.info( "Processing interface: %s%s", interface_name, "" if iface.directory.startswith(self.cache_dir) else " (from %s)" % iface.directory.relpath()) # COPY phase plan.append( charmtools.build.tactics.InterfaceCopy( iface, relation_name, role, self.target, target_config)) # Link Phase # owner = metadata_tactic[0].layer.url owner = output_files[self.HOOK_TEMPLATE_FILE].layer.url for role, relation_name, interface_name in specs: plan.append( charmtools.build.tactics.InterfaceBind(relation_name, owner, self.target, target_config, output_files, template_file))