def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format(spec, target)) cwd = path.getcwd() with utils.tempdir() as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() utils.Process(("pip", "install", "-t", temp_dir, spec)).throw_on_error()() dirs = temp_dir.listdir() self._tracked = [] for d in dirs: rp = d.relpath(temp_dir) dst = cwd / target / rp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not target.exists(): target.makedirs_p() logging.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format(spec, target)) cwd = path.getcwd() with utils.tempdir() as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() utils.Process( ("pip", "install", "-t", temp_dir, spec)).throw_on_error()() dirs = temp_dir.listdir() self._tracked = [] for d in dirs: rp = d.relpath(temp_dir) dst = cwd / target / rp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not target.exists(): target.makedirs_p() logging.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def _process_combined(self, wheelhouse): log.debug('Processing wheelhouse:') self.read() for line in self.lines: log.debug(' ' + line.strip()) with utils.tempdir(chdir=False) as temp_dir: wh_file = temp_dir / 'wheelhouse.txt' wh_file.write_lines(self.lines) self._add(wheelhouse, '-r', wh_file) wh_file.move(self.target.directory / 'wheelhouse.txt')
def _add(self, pip, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files utils.Process((pip, 'install', '--no-binary', ':all:', '-d', temp_dir) + reqs).exit_on_error()() for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files self._pip('download', '--no-binary', ':all:', '-d', temp_dir, *reqs) for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files self._pip('download', '--no-binary', ':all:', '-d', temp_dir, *reqs) for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def _add(self, pip, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files utils.Process((pip, 'install', '--no-binary', ':all:', '-d', temp_dir) + reqs).exit_on_error()() for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format( spec, target)) cwd = path.getcwd() with utils.tempdir() as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() localenv = os.environ.copy() localenv['PYTHONUSERBASE'] = temp_dir utils.Process(("pip", "install", "--user", "--ignore-installed", spec), env=localenv).throw_on_error()() self._tracked = [] # We now manage two classes of explicit mappings # When python packages are installed into a prefix # we know that bin/* should map to <charmdir>/bin/ # and lib/python*/site-packages/* should map to # <target>/* src_paths = ["bin/*", "lib/python*/site-packages/*"] temp_dir = path(temp_dir) for p in src_paths: for d in temp_dir.glob(p): if not d.exists(): continue bp = d.relpath(temp_dir) if bp.startswith("bin/"): dst = self.target / bp elif bp.startswith("lib"): dst = cwd / target / d.name else: dst = cwd / target / bp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not dst.parent.exists(): dst.parent.makedirs_p() log.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files self._pip('download', '--no-binary', ':all:', '-d', temp_dir, *reqs) for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() if self.purge_wheels: unversioned_wheel = wheel.basename().split('-')[0] for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'): old_wheel.remove() else: dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format( spec, target)) with utils.tempdir(chdir=False) as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() localenv = os.environ.copy() localenv['PYTHONUSERBASE'] = temp_dir utils.Process(("pip3", "install", "--user", "--ignore-installed", spec), env=localenv).exit_on_error()() self._tracked = [] # We now manage two classes of explicit mappings # When python packages are installed into a prefix # we know that bin/* should map to <charmdir>/bin/ # and lib/python*/site-packages/* should map to # <target>/* src_paths = ["bin/*", "lib/python*/site-packages/*"] for p in src_paths: for d in temp_dir.glob(p): if not d.exists(): continue bp = d.relpath(temp_dir) if bp.startswith("bin/"): dst = self.target / bp elif bp.startswith("lib"): dst = target / d.name else: dst = target / bp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not dst.parent.exists(): dst.parent.makedirs_p() log.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def __call__(self): """ Process the wheelhouse.txt file. This gets called once on the tactic instance for the highest level layer which contains a `wheelhouse.txt` file (e.g., the charm layer). It then iterates the instances representing all of the previous layers, in order, so that higher layers take precedence over base layers. For each layer, its `wheelhouse.txt` is pip installed into a temp directory and all files that end up in that temp dir are recorded for tracking ownership in the build reporting, and then the temp dir's contents are copied into the charm's `lib/` dir. The installation happens separately for each layer's `wheelhouse.txt` (rather than just combining them into a single `wheelhouse.txt`) because files created by a given layer's `wheelhouse.txt` should be signed by that layer in the report to properly detect changes, etc. """ # recursively process previous layers, depth-first for tactic in self.previous: tactic() # process this layer self.dest.mkdir_p() with utils.tempdir(chdir=False) as temp_dir: # install into a temp dir first to track new and updated files utils.Process( ('pip3', 'install', '-t', str(temp_dir), '-r', self.entity) ).exit_on_error()() # clear out cached compiled files (there shouldn't really be a # reason to include these in the charms; they'll just be # recompiled on first run) for path in temp_dir.walk(): if path.isdir() and ( path.basename() == '__pycache__' or path.basename().endswith('.dist-info') ): path.rmtree() elif path.isfile() and path.basename().endswith('.pyc'): path.remove() # track all the files that were created by this layer self.tracked.extend([self.dest / file.relpath(temp_dir) for file in temp_dir.walkfiles()]) # copy everything over from temp_dir to charm's /lib temp_dir.merge_tree(self.dest)
def _extract_pkg_vcs(wheel, req): with utils.tempdir(chdir=False) as temp_dir: dst_file = temp_dir / wheel.name dst_dir = temp_dir / 'unarchive' wheel.copy(dst_file) if dst_file.endswith('.zip'): with zipfile.ZipFile(dst_file, 'r') as zip_ref: zip_ref.extractall(dst_dir) elif dst_file.endswith('.tar.gz'): with tarfile.open(dst_file, 'r') as archive: archive.extractall(path=dst_dir) else: raise RuntimeError("Can only handle zips or tar.gz?") # now use Fetcher to extract the branch and revision vcs_dir = dst_dir / req.name fetcher = fetchers.Fetcher(vcs_dir) revision = fetcher.get_revision(vcs_dir) branch = fetcher.get_branch_for_revision(vcs_dir, revision) return (branch, revision)
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files self._pip('download', '--no-binary', ':all:', '-d', temp_dir, *reqs) for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() if dest in self.tracked: return if self.purge_wheels: unversioned_wheel = wheel.basename().split('-')[0] for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'): old_wheel.remove() if old_wheel != dest: self.removed.append(old_wheel) else: dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def __call__(self): target = self.dest with utils.tempdir(chdir=False) as temp_dir: # Use the GithubFetcher to pull in the cloud-on-k8s repo. gh = GithubFetcher(url=self.url, repo=self.repo, revision=self.revision) gh_repo_dir = gh.fetch(dir_=str(temp_dir)) # The cloud-on-k8s/config/operator dir contains the yaml we will # use in this charm, so copy them over. shutil.copytree( os.path.join(os.path.abspath(gh_repo_dir), 'config', 'operator'), os.path.join(target, 'operator')) # Generate and include crds in target kustomize_cmd = [ "kubectl", "kustomize", os.path.join(os.path.abspath(gh_repo_dir), 'config', 'crds-flavor-default'), ] data = str() for document in yaml.load_all(check_output(kustomize_cmd)): data += yaml.dump(document) target_crd_path = Path( os.path.join(target, 'crds-flavor-default.yaml')) target_crd_path.write_text(data) # Remove the files we dont need. [ path.remove() for path in target.walk() if path.isfile() and not path.basename().endswith('.yaml') ] # Add the newly added files to self.tracked. self.tracked.extend( [target / file.relpath(target) for file in target.walkfiles()])
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # install into a temp dir first to track new and updated files utils.Process(('pip3', 'install', '-t', str(temp_dir), *reqs)).exit_on_error()() # clear out cached compiled files (there shouldn't really be a # reason to include these in the charms; they'll just be # recompiled on first run) for path in temp_dir.walk(): if path.isdir() and (path.basename() == '__pycache__' or path.basename().endswith('.dist-info')): path.rmtree() elif path.isfile() and path.basename().endswith('.pyc'): path.remove() # track all the files that were created by this layer self.tracked.extend([ self.dest / file.relpath(temp_dir) for file in temp_dir.walkfiles() ]) # copy everything over from temp_dir to charm's /lib temp_dir.merge_tree(self.dest)
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files self._pip('download', '--no-binary', ':all:', '-d', temp_dir, *reqs) log.debug('Copying wheels:') for wheel in temp_dir.files(): log.debug(' ' + wheel.name) dest = wheelhouse / wheel.basename() if dest in self.tracked: return if self.purge_wheels: unversioned_wheel = wheel.basename().split('-')[0] for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'): old_wheel.remove() if old_wheel != dest: self.removed.append(old_wheel) else: dest.remove_p() # extract the version from the wheelhouse name name = None if wheel.name.endswith(".zip"): name = wheel.name[:-4] elif wheel.name.endswith(".tar.gz"): name = wheel.name[:-7] if name is not None: ns = name.split('-') version = ns[-1] package = '-'.join(ns[:-1]) log.debug("Version extracted is: %s version %s", package, version) # we also need to determine if it was a git repo and # extract the branch/commit if package in self.modules: req = self.modules[package] log.debug("module: %s - is vcs: %s", package, req.vcs) if req.vcs: (branch, version) = self._extract_pkg_vcs(wheel, req) log.debug("branch: %s, version=%s", branch, version) self.lock_info.append({ "type": "python_module", "package": package, "url": req.uri, "branch": branch, "version": version, "vcs": req.vcs }) else: # keep the python module and version we build log.debug("not a vcs, therefore %s==%s", package, version) self.lock_info.append({ "type": "python_module", "package": package, "vcs": None, "version": version, }) else: # keep the python module and version we build log.debug("Just keeping %s==%s", package, version) self.lock_info.append({ "type": "python_module", "package": package, "vcs": None, "version": version, }) wheel.move(wheelhouse) self.tracked.append(dest)