def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(spec, target))
cwd = path.getcwd()
with utils.tempdir() as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
utils.Process(("pip", "install", "-t", temp_dir, spec)).throw_on_error()()
dirs = temp_dir.listdir()
self._tracked = []
for d in dirs:
rp = d.relpath(temp_dir)
dst = cwd / target / rp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not target.exists():
target.makedirs_p()
logging.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(spec, target))
cwd = path.getcwd()
with utils.tempdir() as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
utils.Process(
("pip", "install", "-t", temp_dir, spec)).throw_on_error()()
dirs = temp_dir.listdir()
self._tracked = []
for d in dirs:
rp = d.relpath(temp_dir)
dst = cwd / target / rp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not target.exists():
target.makedirs_p()
logging.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def _process_combined(self, wheelhouse):
log.debug('Processing wheelhouse:')
self.read()
for line in self.lines:
log.debug(' ' + line.strip())
with utils.tempdir(chdir=False) as temp_dir:
wh_file = temp_dir / 'wheelhouse.txt'
wh_file.write_lines(self.lines)
self._add(wheelhouse, '-r', wh_file)
wh_file.move(self.target.directory / 'wheelhouse.txt')
def _add(self, pip, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
utils.Process((pip, 'install', '--no-binary', ':all:', '-d',
temp_dir) + reqs).exit_on_error()()
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
self._pip('download', '--no-binary', ':all:', '-d', temp_dir,
*reqs)
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
self._pip('download', '--no-binary', ':all:', '-d', temp_dir,
*reqs)
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def _add(self, pip, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
utils.Process((pip, 'install',
'--no-binary', ':all:',
'-d', temp_dir) +
reqs).exit_on_error()()
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(
spec, target))
cwd = path.getcwd()
with utils.tempdir() as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
localenv = os.environ.copy()
localenv['PYTHONUSERBASE'] = temp_dir
utils.Process(("pip",
"install",
"--user",
"--ignore-installed",
spec), env=localenv).throw_on_error()()
self._tracked = []
# We now manage two classes of explicit mappings
# When python packages are installed into a prefix
# we know that bin/* should map to <charmdir>/bin/
# and lib/python*/site-packages/* should map to
# <target>/*
src_paths = ["bin/*", "lib/python*/site-packages/*"]
temp_dir = path(temp_dir)
for p in src_paths:
for d in temp_dir.glob(p):
if not d.exists():
continue
bp = d.relpath(temp_dir)
if bp.startswith("bin/"):
dst = self.target / bp
elif bp.startswith("lib"):
dst = cwd / target / d.name
else:
dst = cwd / target / bp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not dst.parent.exists():
dst.parent.makedirs_p()
log.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
self._pip('download', '--no-binary', ':all:', '-d', temp_dir,
*reqs)
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
if self.purge_wheels:
unversioned_wheel = wheel.basename().split('-')[0]
for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'):
old_wheel.remove()
else:
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def __call__(self):
# install package reference in trigger file
# in place directory of target
# XXX: Should this map multiline to "-r", self.entity
spec = self.entity.text().strip()
target = self.target_file.dirname()
log.debug("pip installing {} as {}".format(
spec, target))
with utils.tempdir(chdir=False) as temp_dir:
# We do this dance so we don't have
# to guess package and .egg file names
# we move everything in the tempdir to the target
# and track it for later use in sign()
localenv = os.environ.copy()
localenv['PYTHONUSERBASE'] = temp_dir
utils.Process(("pip3",
"install",
"--user",
"--ignore-installed",
spec), env=localenv).exit_on_error()()
self._tracked = []
# We now manage two classes of explicit mappings
# When python packages are installed into a prefix
# we know that bin/* should map to <charmdir>/bin/
# and lib/python*/site-packages/* should map to
# <target>/*
src_paths = ["bin/*", "lib/python*/site-packages/*"]
for p in src_paths:
for d in temp_dir.glob(p):
if not d.exists():
continue
bp = d.relpath(temp_dir)
if bp.startswith("bin/"):
dst = self.target / bp
elif bp.startswith("lib"):
dst = target / d.name
else:
dst = target / bp
if dst.exists():
if dst.isdir():
dst.rmtree_p()
elif dst.isfile():
dst.remove()
if not dst.parent.exists():
dst.parent.makedirs_p()
log.debug("Installer moving {} to {}".format(d, dst))
d.move(dst)
self._tracked.append(dst)
def __call__(self):
"""
Process the wheelhouse.txt file.
This gets called once on the tactic instance for the highest level
layer which contains a `wheelhouse.txt` file (e.g., the charm layer).
It then iterates the instances representing all of the previous
layers, in order, so that higher layers take precedence over base
layers.
For each layer, its `wheelhouse.txt` is pip installed into a temp
directory and all files that end up in that temp dir are recorded
for tracking ownership in the build reporting, and then the temp
dir's contents are copied into the charm's `lib/` dir.
The installation happens separately for each layer's `wheelhouse.txt`
(rather than just combining them into a single `wheelhouse.txt`)
because files created by a given layer's `wheelhouse.txt` should be
signed by that layer in the report to properly detect changes, etc.
"""
# recursively process previous layers, depth-first
for tactic in self.previous:
tactic()
# process this layer
self.dest.mkdir_p()
with utils.tempdir(chdir=False) as temp_dir:
# install into a temp dir first to track new and updated files
utils.Process(
('pip3', 'install', '-t', str(temp_dir), '-r', self.entity)
).exit_on_error()()
# clear out cached compiled files (there shouldn't really be a
# reason to include these in the charms; they'll just be
# recompiled on first run)
for path in temp_dir.walk():
if path.isdir() and (
path.basename() == '__pycache__' or
path.basename().endswith('.dist-info')
):
path.rmtree()
elif path.isfile() and path.basename().endswith('.pyc'):
path.remove()
# track all the files that were created by this layer
self.tracked.extend([self.dest / file.relpath(temp_dir)
for file in temp_dir.walkfiles()])
# copy everything over from temp_dir to charm's /lib
temp_dir.merge_tree(self.dest)
def _extract_pkg_vcs(wheel, req):
with utils.tempdir(chdir=False) as temp_dir:
dst_file = temp_dir / wheel.name
dst_dir = temp_dir / 'unarchive'
wheel.copy(dst_file)
if dst_file.endswith('.zip'):
with zipfile.ZipFile(dst_file, 'r') as zip_ref:
zip_ref.extractall(dst_dir)
elif dst_file.endswith('.tar.gz'):
with tarfile.open(dst_file, 'r') as archive:
archive.extractall(path=dst_dir)
else:
raise RuntimeError("Can only handle zips or tar.gz?")
# now use Fetcher to extract the branch and revision
vcs_dir = dst_dir / req.name
fetcher = fetchers.Fetcher(vcs_dir)
revision = fetcher.get_revision(vcs_dir)
branch = fetcher.get_branch_for_revision(vcs_dir, revision)
return (branch, revision)
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
self._pip('download', '--no-binary', ':all:', '-d', temp_dir,
*reqs)
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
if dest in self.tracked:
return
if self.purge_wheels:
unversioned_wheel = wheel.basename().split('-')[0]
for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'):
old_wheel.remove()
if old_wheel != dest:
self.removed.append(old_wheel)
else:
dest.remove_p()
wheel.move(wheelhouse)
self.tracked.append(dest)
def __call__(self):
target = self.dest
with utils.tempdir(chdir=False) as temp_dir:
# Use the GithubFetcher to pull in the cloud-on-k8s repo.
gh = GithubFetcher(url=self.url,
repo=self.repo,
revision=self.revision)
gh_repo_dir = gh.fetch(dir_=str(temp_dir))
# The cloud-on-k8s/config/operator dir contains the yaml we will
# use in this charm, so copy them over.
shutil.copytree(
os.path.join(os.path.abspath(gh_repo_dir), 'config',
'operator'), os.path.join(target, 'operator'))
# Generate and include crds in target
kustomize_cmd = [
"kubectl",
"kustomize",
os.path.join(os.path.abspath(gh_repo_dir), 'config',
'crds-flavor-default'),
]
data = str()
for document in yaml.load_all(check_output(kustomize_cmd)):
data += yaml.dump(document)
target_crd_path = Path(
os.path.join(target, 'crds-flavor-default.yaml'))
target_crd_path.write_text(data)
# Remove the files we dont need.
[
path.remove() for path in target.walk()
if path.isfile() and not path.basename().endswith('.yaml')
]
# Add the newly added files to self.tracked.
self.tracked.extend(
[target / file.relpath(target) for file in target.walkfiles()])
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# install into a temp dir first to track new and updated files
utils.Process(('pip3', 'install', '-t', str(temp_dir),
*reqs)).exit_on_error()()
# clear out cached compiled files (there shouldn't really be a
# reason to include these in the charms; they'll just be
# recompiled on first run)
for path in temp_dir.walk():
if path.isdir() and (path.basename() == '__pycache__' or
path.basename().endswith('.dist-info')):
path.rmtree()
elif path.isfile() and path.basename().endswith('.pyc'):
path.remove()
# track all the files that were created by this layer
self.tracked.extend([
self.dest / file.relpath(temp_dir)
for file in temp_dir.walkfiles()
])
# copy everything over from temp_dir to charm's /lib
temp_dir.merge_tree(self.dest)
def _add(self, wheelhouse, *reqs):
with utils.tempdir(chdir=False) as temp_dir:
# put in a temp dir first to ensure we track all of the files
self._pip('download', '--no-binary', ':all:', '-d', temp_dir,
*reqs)
log.debug('Copying wheels:')
for wheel in temp_dir.files():
log.debug(' ' + wheel.name)
dest = wheelhouse / wheel.basename()
if dest in self.tracked:
return
if self.purge_wheels:
unversioned_wheel = wheel.basename().split('-')[0]
for old_wheel in wheelhouse.glob(unversioned_wheel + '-*'):
old_wheel.remove()
if old_wheel != dest:
self.removed.append(old_wheel)
else:
dest.remove_p()
# extract the version from the wheelhouse name
name = None
if wheel.name.endswith(".zip"):
name = wheel.name[:-4]
elif wheel.name.endswith(".tar.gz"):
name = wheel.name[:-7]
if name is not None:
ns = name.split('-')
version = ns[-1]
package = '-'.join(ns[:-1])
log.debug("Version extracted is: %s version %s", package,
version)
# we also need to determine if it was a git repo and
# extract the branch/commit
if package in self.modules:
req = self.modules[package]
log.debug("module: %s - is vcs: %s", package, req.vcs)
if req.vcs:
(branch,
version) = self._extract_pkg_vcs(wheel, req)
log.debug("branch: %s, version=%s", branch,
version)
self.lock_info.append({
"type": "python_module",
"package": package,
"url": req.uri,
"branch": branch,
"version": version,
"vcs": req.vcs
})
else:
# keep the python module and version we build
log.debug("not a vcs, therefore %s==%s", package,
version)
self.lock_info.append({
"type": "python_module",
"package": package,
"vcs": None,
"version": version,
})
else:
# keep the python module and version we build
log.debug("Just keeping %s==%s", package, version)
self.lock_info.append({
"type": "python_module",
"package": package,
"vcs": None,
"version": version,
})
wheel.move(wheelhouse)
self.tracked.append(dest)
