def test_read_binary_file_with_limit_bigger_than_file(self):
"""
If the limit is bigger than the file L{read_binary_file} reads the
entire file.
"""
path = self.makeFile("foo bar from end")
self.assertEqual(read_binary_file(path, limit=100),
b"foo bar from end")
self.assertEqual(read_binary_file(path, limit=-100),
b"foo bar from end")
def _get_vm_by_vendor(sys_vendor_path):
"""Return the VM type byte string (possibly empty) based on the vendor."""
# Use lower-key string for vendors, since we do case-insentive match.
# We need bytes here as required by the message schema.
vendor = read_binary_file(sys_vendor_path, limit=1024).lower()
# 2018-01: AWS and DO are now returning custom sys_vendor names
# instead of qemu. If this becomes a trend, it may be worth also checking
# dmi/id/chassis_vendor which seems to unchanged (bochs).
content_vendors_map = (
(b"amazon ec2", b"kvm"),
(b"bochs", b"kvm"),
(b"digitalocean", b"kvm"),
(b"google", b"gce"),
(b"innotek", b"virtualbox"),
(b"microsoft", b"hyperv"),
(b"nutanix", b"kvm"),
(b"openstack", b"kvm"),
(b"qemu", b"kvm"),
(b"kvm", b"kvm"),
(b"vmware", b"vmware"),
)
for name, vm_type in content_vendors_map:
if name in vendor:
return vm_type
return b""
def get_data(self):
"""
Return the Keystone administrative token.
"""
if not os.path.exists(self._keystone_config_file):
return None
config = ConfigParser()
if _PY3:
# We need to use the surrogateescape error handler as the
# admin_token my contain arbitrary bytes. The ConfigParser in
# Python 2 on the other hand does not support read_string.
config_str = read_binary_file(
self._keystone_config_file).decode("utf-8", "surrogateescape")
config.read_string(config_str)
else:
config.read(self._keystone_config_file)
try:
admin_token = config.get("DEFAULT", "admin_token")
except NoOptionError:
logging.error("KeystoneToken: No admin_token found in %s"
% (self._keystone_config_file))
return None
# There is no support for surrogateescape in Python 2, but we actually
# have bytes in this case anyway.
if _PY3:
admin_token = admin_token.encode("utf-8", "surrogateescape")
return admin_token
def write_package_stanza(self, deb_path, dest):
"""Write a stanza for the package to a Packages file.
@param deb_path: The path to the deb package.
@param dest: A writable package file.
"""
deb_file = open(deb_path)
deb = apt_inst.DebFile(deb_file)
control = deb.control.extractdata("control")
deb_file.close()
filename = os.path.basename(deb_path)
size = os.path.getsize(deb_path)
contents = read_binary_file(deb_path)
md5 = hashlib.md5(contents).hexdigest()
sha1 = hashlib.sha1(contents).hexdigest()
sha256 = hashlib.sha256(contents).hexdigest()
tag_section = apt_pkg.TagSection(control)
new_tags = [("Filename", filename), ("Size", str(size)),
("MD5sum", md5), ("SHA1", sha1), ("SHA256", sha256)]
try:
tag_section.write(dest, apt_pkg.REWRITE_PACKAGE_ORDER,
[apt_pkg.TagRewrite(k, v) for k, v in new_tags])
except AttributeError:
# support for python-apt < 1.9
section = apt_pkg.rewrite_section(tag_section,
apt_pkg.REWRITE_PACKAGE_ORDER,
new_tags)
dest.write(section.encode("utf-8"))
def test_read_binary_file_with_limit(self):
"""
With a positive limit L{read_binary_file} reads up to L{limit} bytes
from the start of the file.
"""
path = self.makeFile("foo bar")
self.assertEqual(read_binary_file(path, limit=3), b"foo")
def test_read_binary_file_with_negative_limit(self):
"""
With a negative limit L{read_binary_file} reads only the tail of the
file.
"""
path = self.makeFile("foo bar from end")
self.assertEqual(read_binary_file(path, limit=-3), b"end")
def test_read_binary_file(self):
"""
With no options L{read_binary_file} reads the whole file passed as
argument.
"""
path = self.makeFile("foo")
self.assertEqual(read_binary_file(path), b"foo")
def _reprocess_holding(self):
"""
Unhold accepted messages left behind, and hold unaccepted
pending messages.
"""
offset = 0
pending_offset = self.get_pending_offset()
accepted_types = self.get_accepted_types()
for old_filename in self._walk_messages():
flags = self._get_flags(old_filename)
try:
message = bpickle.loads(read_binary_file(old_filename))
except ValueError as e:
logging.exception(e)
if HELD not in flags:
offset += 1
else:
accepted = message["type"] in accepted_types
if HELD in flags:
if accepted:
new_filename = self._get_next_message_filename()
os.rename(old_filename, new_filename)
self._set_flags(new_filename, set(flags) - set(HELD))
else:
if not accepted and offset >= pending_offset:
self._set_flags(old_filename, set(flags) | set(HELD))
offset += 1
def get_pending_messages(self, max=None):
"""Get any pending messages that aren't being held, up to max."""
accepted_types = self.get_accepted_types()
server_api = self.get_server_api()
messages = []
for filename in self._walk_pending_messages():
if max is not None and len(messages) >= max:
break
data = read_binary_file(self._message_dir(filename))
try:
# don't reinterpret messages that are meant to be sent out
message = bpickle.loads(data, as_is=True)
except ValueError as e:
logging.exception(e)
self._add_flags(filename, BROKEN)
else:
if u"type" not in message:
# Special case to decode keys for messages which were
# serialized by py27 prior to py3 upgrade, and having
# implicit byte message keys. Message may still get
# rejected by the server, but it won't block the client
# broker. (lp: #1718689)
message = {
(k if isinstance(k, str) else k.decode("ascii")): v
for k, v in message.items()}
message[u"type"] = message[u"type"].decode("ascii")
unknown_type = message["type"] not in accepted_types
unknown_api = not is_version_higher(server_api, message["api"])
if unknown_type or unknown_api:
self._add_flags(filename, HELD)
else:
messages.append(message)
return messages
def test_read_binary_file_with_limit(self):
"""
With a positive limit L{read_binary_file} reads only the bytes after
the given limit.
"""
path = self.makeFile("foo bar")
self.assertEqual(read_binary_file(path, limit=3), b" bar")
def _get_vm_by_vendor(sys_vendor_path):
"""Return the VM type byte string (possibly empty) based on the vendor."""
# Use lower-key string for vendors, since we do case-insentive match.
# We need bytes here as required by the message schema.
vendor = read_binary_file(sys_vendor_path, limit=1024).lower()
content_vendors_map = (
(b"amazon ec2", b"kvm"),
(b"bochs", b"kvm"),
(b"digitalocean", b"kvm"),
(b"google", b"gce"),
(b"innotek", b"virtualbox"),
(b"microsoft", b"hyperv"),
(b"nutanix", b"kvm"),
(b"openstack", b"kvm"),
(b"qemu", b"kvm"),
(b"kvm", b"kvm"),
(b"vmware", b"vmware"),
(b"rhev", b"kvm")
)
for name, vm_type in content_vendors_map:
if name in vendor:
return vm_type
return b""
def get_package_stanza(self, deb_path):
"""Return a stanza for the package to be included in a Packages file.
@param deb_path: The path to the deb package.
"""
deb_file = open(deb_path)
deb = apt_inst.DebFile(deb_file)
control = deb.control.extractdata("control")
deb_file.close()
filename = os.path.basename(deb_path)
size = os.path.getsize(deb_path)
contents = read_binary_file(deb_path)
md5 = hashlib.md5(contents).hexdigest()
sha1 = hashlib.sha1(contents).hexdigest()
sha256 = hashlib.sha256(contents).hexdigest()
# Use rewrite_section to ensure that the field order is correct.
return apt_pkg.rewrite_section(apt_pkg.TagSection(control),
apt_pkg.REWRITE_PACKAGE_ORDER,
[("Filename", filename),
("Size", str(size)), ("MD5sum", md5),
("SHA1", sha1), ("SHA256", sha256)])
