def _run_in_venv(self, *args):
assert self._venv is not None
# have to use bash to activate the venv properly first
return utils.Process(
('bash', '-c',
' '.join(('.', self._venv / 'bin' / 'activate', ';') +
args))).exit_on_error()()
def __call__(self, venv=None):
create_venv = venv is None
venv = venv or path(tempfile.mkdtemp())
pip = venv / 'bin' / 'pip3'
wheelhouse = self.target.directory / 'wheelhouse'
wheelhouse.mkdir_p()
if create_venv:
utils.Process(
('virtualenv', '--python', 'python3', venv)).exit_on_error()()
utils.Process(
(pip, 'install', '-U', 'pip', 'wheel')).exit_on_error()()
for tactic in self.previous:
tactic(venv)
self._add(pip, wheelhouse, '-r', self.entity)
if create_venv:
venv.rmtree_p()
def __call__(self):
create_venv = self._venv is None
self._venv = self._venv or path(tempfile.mkdtemp())
wheelhouse = self.target.directory / 'wheelhouse'
wheelhouse.mkdir_p()
if create_venv:
# create venv without pip and use easy_install to install newer
# version; use patched version if running in snap to include:
# https://github.com/pypa/pip/blob/master/news/4320.bugfix
utils.Process(
('virtualenv', '--python', 'python3', '--no-pip', self._venv)
).exit_on_error()()
self._run_in_venv('easy_install',
'pip' if 'SNAP' not in os.environ else
os.path.join(os.environ['SNAP'],
'pip-10.0.0.dev0.zip'))
# we are the top layer; process all lower layers first
for tactic in self.previous:
tactic()
# process this layer
self._add(wheelhouse, '-r', self.entity)
# clean up
if create_venv:
self._venv.rmtree_p()
self._venv = None
def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(spec, target))
cwd = path.getcwd()
with utils.tempdir() as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
utils.Process(
("pip", "install", "-t", temp_dir, spec)).throw_on_error()()
dirs = temp_dir.listdir()
self._tracked = []
for d in dirs:
rp = d.relpath(temp_dir)
dst = cwd / target / rp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not target.exists():
target.makedirs_p()
logging.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def _pip(self, *args):
assert self._venv is not None
# have to use bash to call pip to activate the venv properly first
utils.Process(
('bash', '-c',
' '.join(('.', self._venv / 'bin' / 'activate', ';', 'pip3') +
args))).exit_on_error()()
def _run_in_venv(self, *args):
assert self._venv is not None
# have to use bash to activate the venv properly first
res = utils.Process(
('bash', '-c',
' '.join(('.', self._venv / 'bin' / 'activate', ';') + args)))()
if res.exit_code != 0:
raise BuildError(res.output)
return res
def _add(self, pip, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
utils.Process((pip, 'install', '--no-binary', ':all:', '-d',
temp_dir) + reqs).exit_on_error()()
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(
spec, target))
with utils.tempdir(chdir=False) as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
localenv = os.environ.copy()
localenv['PYTHONUSERBASE'] = temp_dir
utils.Process(("pip3",
"install",
"--user",
"--ignore-installed",
spec), env=localenv).exit_on_error()()
self._tracked = []
# We now manage two classes of explicit mappings
# When python packages are installed into a prefix
# we know that bin/* should map to <charmdir>/bin/
# and lib/python*/site-packages/* should map to
# <target>/*
src_paths = ["bin/*", "lib/python*/site-packages/*"]
for p in src_paths:
for d in temp_dir.glob(p):
if not d.exists():
continue
bp = d.relpath(temp_dir)
if bp.startswith("bin/"):
dst = self.target / bp
elif bp.startswith("lib"):
dst = target / d.name
else:
dst = target / bp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not dst.parent.exists():
dst.parent.makedirs_p()
log.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def __call__(self):
create_venv = self._venv is None
self._venv = self._venv or path(tempfile.mkdtemp())
wheelhouse = self.target.directory / 'wheelhouse'
wheelhouse.mkdir_p()
if create_venv:
utils.Process(('virtualenv', '--python', 'python3',
self._venv)).exit_on_error()()
if self.per_layer:
self._process_per_layer(wheelhouse)
else:
self._process_combined(wheelhouse)
# clean up
if create_venv:
self._venv.rmtree_p()
self._venv = None
def _try_to_get_current_sha(self):
cmds = (
('git', 'describe', '--dirty', '--always'),
('bzr', 'version-info'),
('hg', 'id', '-n'),
)
with utils.cd(str(self.charm)):
for cmd in cmds:
try:
sha = utils.Process(cmd)()
if sha:
return sha.output
except FileNotFoundError as e:
log.debug(e)
continue
return ""
def __call__(self):
create_venv = self._venv is None
self._venv = self._venv or path(tempfile.mkdtemp())
wheelhouse = self.target.directory / 'wheelhouse'
wheelhouse.mkdir_p()
if create_venv:
utils.Process(('virtualenv', '--python', 'python3',
self._venv)).exit_on_error()()
# we are the top layer; process all lower layers first
for tactic in self.previous:
tactic()
# process this layer
self._add(wheelhouse, '-r', self.entity)
# clean up
if create_venv:
self._venv.rmtree_p()
self._venv = None
def __call__(self):
"""
Process the wheelhouse.txt file.
This gets called once on the tactic instance for the highest level
layer which contains a `wheelhouse.txt` file (e.g., the charm layer).
It then iterates the instances representing all of the previous
layers, in order, so that higher layers take precedence over base
layers.
For each layer, its `wheelhouse.txt` is pip installed into a temp
directory and all files that end up in that temp dir are recorded
for tracking ownership in the build reporting, and then the temp
dir's contents are copied into the charm's `lib/` dir.
The installation happens separately for each layer's `wheelhouse.txt`
(rather than just combining them into a single `wheelhouse.txt`)
because files created by a given layer's `wheelhouse.txt` should be
signed by that layer in the report to properly detect changes, etc.
"""
# recursively process previous layers, depth-first
for tactic in self.previous:
tactic()
# process this layer
self.dest.mkdir_p()
with utils.tempdir(chdir=False) as temp_dir:
# install into a temp dir first to track new and updated files
utils.Process(
('pip3', 'install', '-t', str(temp_dir), '-r', self.entity)
).exit_on_error()()
# clear out cached compiled files (there shouldn't really be a
# reason to include these in the charms; they'll just be
# recompiled on first run)
for path in temp_dir.walk():
if path.isdir() and (
path.basename() == '__pycache__' or
path.basename().endswith('.dist-info')
):
path.rmtree()
elif path.isfile() and path.basename().endswith('.pyc'):
path.remove()
# track all the files that were created by this layer
self.tracked.extend([self.dest / file.relpath(temp_dir)
for file in temp_dir.walkfiles()])
# copy everything over from temp_dir to charm's /lib
temp_dir.merge_tree(self.dest)
def _try_to_get_current_sha(self):
with utils.cd(str(self.charm)):
for cmd in self.CMDS:
try:
log.debug('Trying to determine version with: '
'{}'.format(cmd[0]))
sha = utils.Process(cmd)()
if sha and sha.exit_code == 0 and sha.output:
log.debug('Got version: {}'.format(sha.output))
return sha.output
else:
log.debug('Failed to get version{}'.format(
': {}'.format(sha.output) if sha else ''))
continue
except OSError as e:
if e.errno != errno.ENOENT:
raise
log.debug(e)
continue
return ""
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# install into a temp dir first to track new and updated files
utils.Process(('pip3', 'install', '-t', str(temp_dir),
*reqs)).exit_on_error()()
# clear out cached compiled files (there shouldn't really be a
# reason to include these in the charms; they'll just be
# recompiled on first run)
for path in temp_dir.walk():
if path.isdir() and (path.basename() == '__pycache__' or
path.basename().endswith('.dist-info')):
path.rmtree()
elif path.isfile() and path.basename().endswith('.pyc'):
path.remove()
# track all the files that were created by this layer
self.tracked.extend([
self.dest / file.relpath(temp_dir)
for file in temp_dir.walkfiles()
])
# copy everything over from temp_dir to charm's /lib
temp_dir.merge_tree(self.dest)
