def _get_pkgs_install_date(self, pkg_dicts):
# Convert package names to dpkg list filenames
queries = [self._pkg_name_to_dpkg_list_file(p["name"])
for p in pkg_dicts]
# Call stat in batches
exec_gen = execute_command_batch(
self._session, ['stat', '-c', '%n: %Y'], queries,
cmd_err_filter('No such file or directory'))
# Parse and accumulate stat results in a dict
results = {}
for (out, _, exc) in exec_gen:
if exc:
out = exc.stdout # One file not found, so continue
# Parse the output and store by filename
for outlines in out.splitlines():
(fname, ftime) = outlines.split(": ")
results[fname] = str(
pytz.utc.localize(
datetime.utcfromtimestamp(float(ftime))))
# Now lookup the packages in the results
for p in pkg_dicts:
fname = self._pkg_name_to_dpkg_list_file(p["name"])
if fname in results:
p["install_date"] = results[fname]
def _get_pkgs_details_from_apt_cache_show(self, pkg_dicts):
# Convert package names to name:arch=version format
queries = ["%(name)s:%(architecture)s=%(version)s" % p
for p in pkg_dicts]
# Call "apt-cache show" in batches
exec_gen = execute_command_batch(self._session, ['apt-cache', 'show'],
queries)
# Parse and accumulate "apt-cache show" results
results = (parse_apt_cache_show_pkgs_output(out)
for (out, _, _) in exec_gen)
# Combine sequence of lists
results = itertools.chain.from_iterable(results)
# Turn apt-cache show results into a lookup table by package name
results = self.create_lookup_from_apt_cache_show(results)
# Loop through each package and find the respective apt-cache results
for p in pkg_dicts:
r = results.get("%(name)s:%(architecture)s" % p)
if not r:
lgr.warning("Was unable to run apt-cache show for %s" %
p["name"])
continue
# Update the dictionary with found results (if present)
for f in ("source_name", "source_version", "size", "md5",
"sha1", "sha256"):
if f in r:
p[f] = r[f]
def _get_pkgs_arch_and_version(self, pkg_dicts):
# Convert package names to name:arch format
# Use "dpkg -s pkg" to get the installed version and arch
# Note: "architecture" is in the dict, but may be null
queries = [(p["name"] if not p["architecture"]
else "%(name)s:%(architecture)s" % p)
for p in pkg_dicts]
# Call "dpkg -s" in batches
exec_gen = execute_command_batch(self._session, ['dpkg', '-s'],
queries)
# Parse and accumulate "dpkg -s" results
# dpkg -s uses the same output as apt-cache show pkg
results = (parse_apt_cache_show_pkgs_output(out)
for (out, _, _) in exec_gen)
# Combine sequence of lists
results = itertools.chain.from_iterable(results)
# Turn dpkg -s results into a lookup table by package name
results = self.create_lookup_from_apt_cache_show(results)
# Loop through each package and find the respective dpkg results
for p in pkg_dicts:
r = results.get(p["name"] if not p["architecture"]
else "%(name)s:%(architecture)s" % p)
if not r:
lgr.warning("Was unable to run dpkg -s for %s" %
p["name"])
continue
# Update the dictionary with found results
p["architecture"] = r["architecture"]
p["version"] = r["version"]
def _get_packagefields_for_files(self, files):
# Call dpkg query in batches
exec_gen = execute_command_batch(
self._session, ['dpkg-query', '-S'], files,
cmd_err_filter('no path found matching pattern'))
# Parse and accumulate stat results in a dict
file_to_package_dict = {}
for (out, _, exc) in exec_gen:
if exc:
out = exc.stdout # One file not found, so continue
# Now go through the output and assign packages to files
for outline in out.splitlines():
# Parse package name (architecture) and path
# TODO: Handle query of /bin/sh better
outdict = self._parse_dpkgquery_line(outline)
if not outdict:
lgr.debug("Skipping line %s", outline)
continue
# Pull the found path from the dictionary
found_name = outdict.pop('path')
if not found_name:
raise ValueError(
"Record %s got no path defined... skipping"
% repr(outdict)
)
# Associate the file to the package name (and architecture)
pkg = outdict
lgr.debug("Identified file %r to belong to package %s",
found_name, pkg)
file_to_package_dict[found_name] = pkg
return file_to_package_dict
def install_packages(self, session=None):
session = session or get_local_session()
for env in self.environments:
# TODO: Deal with system and editable packages.
to_install = [
"{p.name}=={p.version}".format(p=p) for p in env.packages
if p.local and not p.editable
]
if not to_install:
lgr.info("No local, non-editable packages found")
continue
# TODO: Right now we just use the python to invoke "virtualenv
# --python=..." when the directory doesn't exist, but we should
# eventually use the yet-to-exist "satisfies" functionality to
# check whether an existing virtual environment has the right
# python (and maybe other things).
pyver = "{v.major}.{v.minor}".format(
v=parse_semantic_version(env.python_version))
if not session.exists(env.path):
# The location and version of virtualenv are recorded at the
# time of tracing, but should we use these values? For now,
# use a plain "virtualenv" below on the basis that we just use
# "apt-get" and "git" elsewhere.
session.execute_command([
"virtualenv", "--python=python{}".format(pyver), env.path
])
list(
execute_command_batch(session,
[env.path + "/bin/pip", "install"],
to_install))
def _pip_batched_show(session, which_pip, pkgs):
cmd = [which_pip, "show", "-f"]
batch = execute_command_batch(session, cmd, pkgs)
sep_re = re.compile("^---$", flags=re.MULTILINE)
entries = (sep_re.split(stacked) for stacked, _, _ in batch)
for pkg, entry in zip(pkgs, itertools.chain(*entries)):
info = parse_pip_show(entry)
yield pkg, info
def _get_pkgs_versions_and_sources(self, pkg_dicts):
# Convert package names to name:arch format
queries = ["%(name)s:%(architecture)s" % p for p in pkg_dicts]
# Call apt-cache policy in batches
exec_gen = execute_command_batch(self._session,
['apt-cache', 'policy'],
queries)
# Parse results into a single generator
results = (parse_apt_cache_policy_pkgs_output(out)
for (out, _, _) in exec_gen)
# Combine sequence of dicts
results = join_sequence_of_dicts(results)
# Loop through each package and find the respective apt-cache results
for p in pkg_dicts:
ver = results.get("%(name)s:%(architecture)s" % p)
if not ver:
ver = results.get(p["name"])
if not ver:
lgr.warning("Was unable to get version table for %s" %
p["name"])
continue
# Now construct the version table
ver_dict = {}
for v in ver.get("versions"):
key = v["version"]
ver_dict[key] = []
for s in v.get("sources"):
s = s["source"]
# If we haven't named the source yet, name it
if s not in self._source_line_to_name_map:
# Make sure we can find the source
if s not in self._all_apt_sources:
lgr.warning("Cannot find source %s" % s)
continue
# Grab and name the source
source = self._all_apt_sources[s]
src_name = self._get_apt_source_name(source)
source.name = src_name
# Now add the source to our used sources
self._apt_sources[src_name] = source
# add the name for easy future lookup
self._source_line_to_name_map[s] = src_name
# Look up and add the short name for the source
ver_dict[key].append(
self._source_line_to_name_map[s])
p["versions"] = ver_dict
