def _run_in_venv(self, *args): assert self._venv is not None # have to use bash to activate the venv properly first return utils.Process( ('bash', '-c', ' '.join(('.', self._venv / 'bin' / 'activate', ';') + args))).exit_on_error()()
def __call__(self, venv=None): create_venv = venv is None venv = venv or path(tempfile.mkdtemp()) pip = venv / 'bin' / 'pip3' wheelhouse = self.target.directory / 'wheelhouse' wheelhouse.mkdir_p() if create_venv: utils.Process( ('virtualenv', '--python', 'python3', venv)).exit_on_error()() utils.Process( (pip, 'install', '-U', 'pip', 'wheel')).exit_on_error()() for tactic in self.previous: tactic(venv) self._add(pip, wheelhouse, '-r', self.entity) if create_venv: venv.rmtree_p()
def __call__(self): create_venv = self._venv is None self._venv = self._venv or path(tempfile.mkdtemp()) wheelhouse = self.target.directory / 'wheelhouse' wheelhouse.mkdir_p() if create_venv: # create venv without pip and use easy_install to install newer # version; use patched version if running in snap to include: # https://github.com/pypa/pip/blob/master/news/4320.bugfix utils.Process( ('virtualenv', '--python', 'python3', '--no-pip', self._venv) ).exit_on_error()() self._run_in_venv('easy_install', 'pip' if 'SNAP' not in os.environ else os.path.join(os.environ['SNAP'], 'pip-10.0.0.dev0.zip')) # we are the top layer; process all lower layers first for tactic in self.previous: tactic() # process this layer self._add(wheelhouse, '-r', self.entity) # clean up if create_venv: self._venv.rmtree_p() self._venv = None
def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format(spec, target)) cwd = path.getcwd() with utils.tempdir() as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() utils.Process( ("pip", "install", "-t", temp_dir, spec)).throw_on_error()() dirs = temp_dir.listdir() self._tracked = [] for d in dirs: rp = d.relpath(temp_dir) dst = cwd / target / rp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not target.exists(): target.makedirs_p() logging.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def _pip(self, *args): assert self._venv is not None # have to use bash to call pip to activate the venv properly first utils.Process( ('bash', '-c', ' '.join(('.', self._venv / 'bin' / 'activate', ';', 'pip3') + args))).exit_on_error()()
def _run_in_venv(self, *args): assert self._venv is not None # have to use bash to activate the venv properly first res = utils.Process( ('bash', '-c', ' '.join(('.', self._venv / 'bin' / 'activate', ';') + args)))() if res.exit_code != 0: raise BuildError(res.output) return res
def _add(self, pip, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # put in a temp dir first to ensure we track all of the files utils.Process((pip, 'install', '--no-binary', ':all:', '-d', temp_dir) + reqs).exit_on_error()() for wheel in temp_dir.files(): dest = wheelhouse / wheel.basename() dest.remove_p() wheel.move(wheelhouse) self.tracked.append(dest)
def __call__(self): # install package reference in trigger file # in place directory of target # XXX: Should this map multiline to "-r", self.entity spec = self.entity.text().strip() target = self.target_file.dirname() log.debug("pip installing {} as {}".format( spec, target)) with utils.tempdir(chdir=False) as temp_dir: # We do this dance so we don't have # to guess package and .egg file names # we move everything in the tempdir to the target # and track it for later use in sign() localenv = os.environ.copy() localenv['PYTHONUSERBASE'] = temp_dir utils.Process(("pip3", "install", "--user", "--ignore-installed", spec), env=localenv).exit_on_error()() self._tracked = [] # We now manage two classes of explicit mappings # When python packages are installed into a prefix # we know that bin/* should map to <charmdir>/bin/ # and lib/python*/site-packages/* should map to # <target>/* src_paths = ["bin/*", "lib/python*/site-packages/*"] for p in src_paths: for d in temp_dir.glob(p): if not d.exists(): continue bp = d.relpath(temp_dir) if bp.startswith("bin/"): dst = self.target / bp elif bp.startswith("lib"): dst = target / d.name else: dst = target / bp if dst.exists(): if dst.isdir(): dst.rmtree_p() elif dst.isfile(): dst.remove() if not dst.parent.exists(): dst.parent.makedirs_p() log.debug("Installer moving {} to {}".format(d, dst)) d.move(dst) self._tracked.append(dst)
def __call__(self): create_venv = self._venv is None self._venv = self._venv or path(tempfile.mkdtemp()) wheelhouse = self.target.directory / 'wheelhouse' wheelhouse.mkdir_p() if create_venv: utils.Process(('virtualenv', '--python', 'python3', self._venv)).exit_on_error()() if self.per_layer: self._process_per_layer(wheelhouse) else: self._process_combined(wheelhouse) # clean up if create_venv: self._venv.rmtree_p() self._venv = None
def _try_to_get_current_sha(self): cmds = ( ('git', 'describe', '--dirty', '--always'), ('bzr', 'version-info'), ('hg', 'id', '-n'), ) with utils.cd(str(self.charm)): for cmd in cmds: try: sha = utils.Process(cmd)() if sha: return sha.output except FileNotFoundError as e: log.debug(e) continue return ""
def __call__(self): create_venv = self._venv is None self._venv = self._venv or path(tempfile.mkdtemp()) wheelhouse = self.target.directory / 'wheelhouse' wheelhouse.mkdir_p() if create_venv: utils.Process(('virtualenv', '--python', 'python3', self._venv)).exit_on_error()() # we are the top layer; process all lower layers first for tactic in self.previous: tactic() # process this layer self._add(wheelhouse, '-r', self.entity) # clean up if create_venv: self._venv.rmtree_p() self._venv = None
def __call__(self): """ Process the wheelhouse.txt file. This gets called once on the tactic instance for the highest level layer which contains a `wheelhouse.txt` file (e.g., the charm layer). It then iterates the instances representing all of the previous layers, in order, so that higher layers take precedence over base layers. For each layer, its `wheelhouse.txt` is pip installed into a temp directory and all files that end up in that temp dir are recorded for tracking ownership in the build reporting, and then the temp dir's contents are copied into the charm's `lib/` dir. The installation happens separately for each layer's `wheelhouse.txt` (rather than just combining them into a single `wheelhouse.txt`) because files created by a given layer's `wheelhouse.txt` should be signed by that layer in the report to properly detect changes, etc. """ # recursively process previous layers, depth-first for tactic in self.previous: tactic() # process this layer self.dest.mkdir_p() with utils.tempdir(chdir=False) as temp_dir: # install into a temp dir first to track new and updated files utils.Process( ('pip3', 'install', '-t', str(temp_dir), '-r', self.entity) ).exit_on_error()() # clear out cached compiled files (there shouldn't really be a # reason to include these in the charms; they'll just be # recompiled on first run) for path in temp_dir.walk(): if path.isdir() and ( path.basename() == '__pycache__' or path.basename().endswith('.dist-info') ): path.rmtree() elif path.isfile() and path.basename().endswith('.pyc'): path.remove() # track all the files that were created by this layer self.tracked.extend([self.dest / file.relpath(temp_dir) for file in temp_dir.walkfiles()]) # copy everything over from temp_dir to charm's /lib temp_dir.merge_tree(self.dest)
def _try_to_get_current_sha(self): with utils.cd(str(self.charm)): for cmd in self.CMDS: try: log.debug('Trying to determine version with: ' '{}'.format(cmd[0])) sha = utils.Process(cmd)() if sha and sha.exit_code == 0 and sha.output: log.debug('Got version: {}'.format(sha.output)) return sha.output else: log.debug('Failed to get version{}'.format( ': {}'.format(sha.output) if sha else '')) continue except OSError as e: if e.errno != errno.ENOENT: raise log.debug(e) continue return ""
def _add(self, wheelhouse, *reqs): with utils.tempdir(chdir=False) as temp_dir: # install into a temp dir first to track new and updated files utils.Process(('pip3', 'install', '-t', str(temp_dir), *reqs)).exit_on_error()() # clear out cached compiled files (there shouldn't really be a # reason to include these in the charms; they'll just be # recompiled on first run) for path in temp_dir.walk(): if path.isdir() and (path.basename() == '__pycache__' or path.basename().endswith('.dist-info')): path.rmtree() elif path.isfile() and path.basename().endswith('.pyc'): path.remove() # track all the files that were created by this layer self.tracked.extend([ self.dest / file.relpath(temp_dir) for file in temp_dir.walkfiles() ]) # copy everything over from temp_dir to charm's /lib temp_dir.merge_tree(self.dest)