def get_data(self):
"""Return a C{dict} mapping APT preferences files to their contents.
If no APT preferences configuration is set at all on the system, then
simply return C{None}
"""
data = {}
preferences_filename = os.path.join(self._etc_apt_directory,
u"preferences")
if os.path.exists(preferences_filename):
data[preferences_filename] = read_text_file(preferences_filename)
preferences_directory = os.path.join(self._etc_apt_directory,
u"preferences.d")
if os.path.isdir(preferences_directory):
for entry in os.listdir(preferences_directory):
filename = os.path.join(preferences_directory, entry)
if os.path.isfile(filename):
data[filename] = read_text_file(filename)
if data == {}:
return None
item_size_limit = self.size_limit // len(data.keys())
for filename, contents in iteritems(data):
if len(filename) + len(contents) > item_size_limit:
truncated_contents_size = item_size_limit - len(filename)
data[filename] = data[filename][0:truncated_contents_size]
return data
def test_read_text_file_with_limit_bigger_than_file(self):
"""
If the limit is bigger than the file L{read_text_file} reads the entire
file.
"""
utf8_content = codecs.encode(u"foo \N{SNOWMAN} bar", "utf-8")
path = self.makeFile(utf8_content, mode="wb")
self.assertEqual(read_text_file(path, limit=100), u"foo ☃ bar")
self.assertEqual(read_text_file(path, limit=-100), u"foo ☃ bar")
def test_read_text_file_with_broken_utf8(self):
"""
A text file containing broken UTF-8 shouldn't cause an error, just
return some sensible replacement chars.
"""
not_quite_utf8_content = b'foo \xca\xff bar'
path = self.makeFile(not_quite_utf8_content, mode='wb')
self.assertEqual(read_text_file(path), u'foo \ufffd\ufffd bar')
self.assertEqual(read_text_file(path, limit=5), u'foo \ufffd')
self.assertEqual(read_text_file(path, limit=-3), u'bar')
def test_write_command_line_options(self):
self.reset_config(configuration_class=BabbleConfiguration)
self.write_config_file()
self.config.load(["--whatever", "spam"])
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[babble]\nwhatever = spam\n")
def add_channel_apt_deb(self,
url,
codename,
components=None,
trusted=None):
"""Add a deb URL which points to a repository.
@param url: The base URL of the repository.
@param codename: The dist in the repository.
@param components: The components to be included.
@param trusted: Whether validation should be skipped (if local).
"""
sources_file_path = self._get_internal_sources_list()
source_options = ""
if trusted is not None and url.startswith("file:"):
trusted_val = "yes" if trusted else "no"
source_options = "[ trusted={} ] ".format(trusted_val)
sources_line = "deb {}{} {}".format(source_options, url, codename)
if components:
sources_line += " %s" % " ".join(components)
if os.path.exists(sources_file_path):
current_content = read_text_file(sources_file_path).split("\n")
if sources_line in current_content:
return
sources_line += "\n"
append_text_file(sources_file_path, sources_line)
def test_write_existing_empty_file(self):
self.config_filename = self.makeFile("")
self.config.default_config_filenames[:] = [self.config_filename]
self.config.whatever = "eggs"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = eggs")
def _get_vm_by_vendor(sys_vendor_path):
"""Return the VM type byte string (possibly empty) based on the vendor."""
vendor = read_text_file(sys_vendor_path).lower()
# Use lower-key string for vendors, since we do case-insentive match.
# We need bytes here as required by the message schema.
# 2018-01: AWS and DO are now returning custom sys_vendor names
# instead of qemu. If this becomes a trend, it may be worth also checking
# dmi/id/chassis_vendor which seems to unchanged (bochs).
content_vendors_map = (
("amazon ec2", b"kvm"),
("bochs", b"kvm"),
("digitalocean", b"kvm"),
("google", b"gce"),
("innotek", b"virtualbox"),
("microsoft", b"hyperv"),
("nutanix", b"kvm"),
("openstack", b"kvm"),
("qemu", b"kvm"),
("vmware", b"vmware"))
for name, vm_type in content_vendors_map:
if name in vendor:
return vm_type
return b""
def _get_packages(self):
"""Return the list of packages that required a reboot, if any."""
if not os.path.exists(self._packages_filename):
return []
lines = read_text_file(self._packages_filename).splitlines()
packages = set(line.strip() for line in lines if line)
return sorted(packages)
def test_read_text_file(self):
"""
With no options L{read_text_file} reads the whole file passed as
argument as string decoded with utf-8.
"""
utf8_content = codecs.encode(u"foo \N{SNOWMAN}", "utf-8")
path = self.makeFile(utf8_content, mode="wb")
self.assertEqual(read_text_file(path), u"foo ☃")
def test_read_text_file_with_limit(self):
"""
With a positive limit L{read_text_file} returns up to L{limit}
characters from the start of the file.
"""
utf8_content = codecs.encode(u"foo \N{SNOWMAN}", "utf-8")
path = self.makeFile(utf8_content, mode="wb")
self.assertEqual(read_text_file(path, limit=3), u"foo")
def test_read_text_file_with_negative_limit(self):
"""
With a negative limit L{read_text_file} reads only the tail characters
of the string.
"""
utf8_content = codecs.encode(u"foo \N{SNOWMAN} bar", "utf-8")
path = self.makeFile(utf8_content, mode="wb")
self.assertEqual(read_text_file(path, limit=-5), u"☃ bar")
def test_write_on_the_right_default_config_file(self):
self.write_config_file(whatever="spam")
self.config_class.default_config_filenames.insert(0, "/non/existent")
self.config.load([])
self.config.whatever = "eggs"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = eggs\n")
def test_write_to_given_config_file(self):
self.reset_config(configuration_class=cfg_class(whatever="spam"))
filename = self.makeFile(content="")
self.config.load(["--whatever", "eggs", "--config", filename])
self.config.whatever = "ham"
self.config.write()
data = read_text_file(filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = ham\n")
def test_read_text_file_with_limit(self):
"""
With a positive limit L{read_text_file} returns only the characters
after the first L{limit} characters as string.
"""
utf8_content = codecs.encode(u"foo \N{SNOWMAN}", "utf-8")
path = self.makeFile(utf8_content, mode="wb")
self.assertEqual(read_text_file(path, limit=3), u" ☃")
def test_write_existing_file(self):
self.config_filename = self.makeFile(
"\n[other]\nfoo = bar\n[again]\nx = y\n")
self.config.default_config_filenames[:] = [self.config_filename]
self.config.whatever = "eggs"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, ("[other]\nfoo = bar\n"
"[again]\nx = y\n"
"[my-config]\nwhatever = eggs"))
def test_write_command_line_precedence(self):
"""
Command line options take precedence over config file when writing.
"""
self.reset_config(configuration_class=cfg_class(whatever="spam"))
self.write_config_file(whatever="eggs")
self.config.load(["--whatever", "ham"])
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = ham\n")
def get_container_info(run_path="/run"):
"""
Return a string with the type of container the client is running in, if
any, an empty string otherwise.
"""
for filename in ("container_type", "systemd/container"):
path = os.path.join(run_path, filename)
if os.path.exists(path):
return read_text_file(path).strip()
return ""
def _get_vm_legacy(root_path):
"""Check if the host is virtualized looking at /proc/cpuinfo content."""
try:
cpuinfo = read_text_file(os.path.join(root_path, "proc/cpuinfo"))
except (IOError, OSError):
return b""
if "qemu" in cpuinfo:
return b"kvm"
return b""
def test_dont_write_unspecified_default_options(self):
"""
Don't write options to the file if the value exactly matches the
default and the value did not exist in the original config file.
"""
self.reset_config(configuration_class=cfg_class(whatever="spam"))
self.write_config_file()
self.config.whatever = "spam"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]")
def test_dont_write_client_section_default_options(self):
"""
Don't write options to the file if they exactly match the default and
didn't already exist in the file.
"""
self.reset_config(configuration_class=cfg_class(whatever="spam"))
self.write_config_file(whatever="eggs")
self.config.whatever = "spam"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]")
def test_dont_delete_explicitly_set_default_options(self):
"""
If the user explicitly sets a configuration option to its default
value, we shouldn't delete that option from the conf file when we
write it, just to be nice.
"""
self.reset_config(configuration_class=cfg_class(whatever="spam"))
self.write_config_file(whatever="spam")
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = spam")
def test_write_empty_list_values_instead_of_double_quotes(self):
"""
Since list values are strings, an empty string such as "" will be
written to the config file as an option with a empty value instead of
"".
"""
self.write_config_file(spam="42")
self.config.load([])
self.config.spam = ""
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[my-config]\nspam = \n")
def test_do_write_preexisting_default_options(self):
"""
If the value of an option matches the default, but the option was
already written in the file, then write it back to the file.
"""
self.reset_config(configuration_class=cfg_class(whatever="spam"))
config = "[my-config]\nwhatever = spam\n"
config_filename = self.makeFile(config)
self.config.load_configuration_file(config_filename)
self.config.whatever = "spam"
self.config.write()
data = read_text_file(config_filename)
self.assertConfigEqual(data, "[my-config]\nwhatever = spam\n")
def test_write_unrelated_configuration_back(self):
"""
If a configuration file has a section that isn't processed by a
particular configuration object, that unrelated configuration section
will be maintained even when written back.
"""
config = "[my-config]\nwhatever = zoot\n[goojy]\nunrelated = yes"
config_filename = self.makeFile(config)
self.config.load_configuration_file(config_filename)
self.config.whatever = "boo"
self.config.write()
data = read_text_file(config_filename)
self.assertConfigEqual(
data, "[my-config]\nwhatever = boo\n\n[goojy]\nunrelated = yes")
def test_comments_are_maintained(self):
"""
When we write an updated config file, comments that existed previously
are maintained.
"""
config = "[my-config]\n# Comment 1\nwhatever = spam\n#Comment 2\n"
filename = self.makeFile(config)
self.config.load_configuration_file(filename)
self.config.whatever = "eggs"
self.config.write()
new_config = read_text_file(filename)
self.assertConfigEqual(
new_config,
"[my-config]\n# Comment 1\nwhatever = eggs\n#Comment 2\n")
def add_channel_apt_deb(self, url, codename, components=None):
"""Add a deb URL which points to a repository.
@param url: The base URL of the repository.
@param codename: The dist in the repository.
@param components: The components to be included.
"""
sources_file_path = self._get_internal_sources_list()
sources_line = "deb %s %s" % (url, codename)
if components:
sources_line += " %s" % " ".join(components)
if os.path.exists(sources_file_path):
current_content = read_text_file(sources_file_path).split("\n")
if sources_line in current_content:
return
sources_line += "\n"
append_text_file(sources_file_path, sources_line)
def get_juju_info(config):
"""
Returns available Juju info or C{None} if the path referenced from
L{config} is not a valid file.
"""
if not os.path.exists(config.juju_filename):
return
json_contents = read_text_file(config.juju_filename)
try:
juju_info = json.loads(json_contents)
# Catch any error the json lib could throw, because we don't know or
# care what goes wrong - we'll display a generic error message and
# return None in any case.
except Exception:
logging.exception("Error attempting to read JSON from %s" %
config.juju_filename)
return None
juju_info["api-addresses"] = juju_info["api-addresses"].split()
return juju_info
def make_operation_result_text(self, out, err):
"""Return the operation result text to be sent to the server.
@param out: The standard output of the upgrade-tool process.
@param err: The standard error of the upgrade-tool process.
@return: A text aggregating the process output, error and log files.
"""
buf = io.StringIO()
for label, content in [("output", out), ("error", err)]:
if content:
buf.write(u"=== Standard %s ===\n\n%s\n\n" % (label, content))
for basename in sorted(os.listdir(self.logs_directory)):
if not basename.endswith(".log"):
continue
filename = os.path.join(self.logs_directory, basename)
content = read_text_file(filename, -self.logs_limit)
buf.write(u"=== %s ===\n\n%s\n\n" % (basename, content))
return buf.getvalue()
def test_write_configuration(self):
self.write_config_file(log_level="debug")
self.config.log_level = "warning"
self.config.write()
data = read_text_file(self.config_filename)
self.assertConfigEqual(data, "[client]\nlog_level = warning")
def _commit_package_changes(self):
"""
Commit cached APT operations and give feedback on the results as a
string.
"""
# XXX we cannot use io.StringIO() here with Python 2 as there is a
# string literal written in apt.progress.text.TextProgress._write()
# which is not recognized as unicode by io.StringIO() with Python 2.
fetch_output = StringIO()
# Redirect stdout and stderr to a file. We need to work with the
# file descriptors, rather than sys.stdout/stderr, since dpkg is
# run in a subprocess.
fd, install_output_path = tempfile.mkstemp()
old_stdout = os.dup(1)
old_stderr = os.dup(2)
os.dup2(fd, 1)
os.dup2(fd, 2)
install_progress = LandscapeInstallProgress()
try:
# Since others (charms) might be installing packages on this system
# We need to retry a bit in case dpkg is locked in progress
dpkg_tries = 0
while dpkg_tries <= self.max_dpkg_retries:
error = None
if dpkg_tries > 0:
# Yeah, sleeping isn't kosher according to Twisted, but
# this code is run in the package-changer, which doesn't
# have any concurrency going on.
time.sleep(self.dpkg_retry_sleep)
logging.warning(
"dpkg process might be in use. "
"Retrying package changes. %d retries remaining." %
(self.max_dpkg_retries - dpkg_tries))
dpkg_tries += 1
try:
self._cache.commit(
fetch_progress=LandscapeAcquireProgress(fetch_output),
install_progress=install_progress)
if not install_progress.dpkg_exited:
raise SystemError("dpkg didn't exit cleanly.")
except SystemError as exc:
result_text = (fetch_output.getvalue() +
read_text_file(install_output_path))
error = TransactionError(exc.args[0] +
"\n\nPackage operation log:\n" +
result_text)
# No need to retry SystemError, since it's most
# likely a permanent error.
break
except apt.cache.LockFailedException as exception:
result_text = (fetch_output.getvalue() +
read_text_file(install_output_path))
error = TransactionError(exception.args[0] +
"\n\nPackage operation log:\n" +
result_text)
else:
result_text = (fetch_output.getvalue() +
read_text_file(install_output_path))
break
if error is not None:
raise error
finally:
# Restore stdout and stderr.
os.dup2(old_stdout, 1)
os.dup2(old_stderr, 2)
os.remove(install_output_path)
return result_text
