def test_read_binary_file_with_limit_bigger_than_file(self): """ If the limit is bigger than the file L{read_binary_file} reads the entire file. """ path = self.makeFile("foo bar from end") self.assertEqual(read_binary_file(path, limit=100), b"foo bar from end") self.assertEqual(read_binary_file(path, limit=-100), b"foo bar from end")
def _get_vm_by_vendor(sys_vendor_path): """Return the VM type byte string (possibly empty) based on the vendor.""" # Use lower-key string for vendors, since we do case-insentive match. # We need bytes here as required by the message schema. vendor = read_binary_file(sys_vendor_path, limit=1024).lower() # 2018-01: AWS and DO are now returning custom sys_vendor names # instead of qemu. If this becomes a trend, it may be worth also checking # dmi/id/chassis_vendor which seems to unchanged (bochs). content_vendors_map = ( (b"amazon ec2", b"kvm"), (b"bochs", b"kvm"), (b"digitalocean", b"kvm"), (b"google", b"gce"), (b"innotek", b"virtualbox"), (b"microsoft", b"hyperv"), (b"nutanix", b"kvm"), (b"openstack", b"kvm"), (b"qemu", b"kvm"), (b"kvm", b"kvm"), (b"vmware", b"vmware"), ) for name, vm_type in content_vendors_map: if name in vendor: return vm_type return b""
def get_data(self): """ Return the Keystone administrative token. """ if not os.path.exists(self._keystone_config_file): return None config = ConfigParser() if _PY3: # We need to use the surrogateescape error handler as the # admin_token my contain arbitrary bytes. The ConfigParser in # Python 2 on the other hand does not support read_string. config_str = read_binary_file( self._keystone_config_file).decode("utf-8", "surrogateescape") config.read_string(config_str) else: config.read(self._keystone_config_file) try: admin_token = config.get("DEFAULT", "admin_token") except NoOptionError: logging.error("KeystoneToken: No admin_token found in %s" % (self._keystone_config_file)) return None # There is no support for surrogateescape in Python 2, but we actually # have bytes in this case anyway. if _PY3: admin_token = admin_token.encode("utf-8", "surrogateescape") return admin_token
def write_package_stanza(self, deb_path, dest): """Write a stanza for the package to a Packages file. @param deb_path: The path to the deb package. @param dest: A writable package file. """ deb_file = open(deb_path) deb = apt_inst.DebFile(deb_file) control = deb.control.extractdata("control") deb_file.close() filename = os.path.basename(deb_path) size = os.path.getsize(deb_path) contents = read_binary_file(deb_path) md5 = hashlib.md5(contents).hexdigest() sha1 = hashlib.sha1(contents).hexdigest() sha256 = hashlib.sha256(contents).hexdigest() tag_section = apt_pkg.TagSection(control) new_tags = [("Filename", filename), ("Size", str(size)), ("MD5sum", md5), ("SHA1", sha1), ("SHA256", sha256)] try: tag_section.write(dest, apt_pkg.REWRITE_PACKAGE_ORDER, [apt_pkg.TagRewrite(k, v) for k, v in new_tags]) except AttributeError: # support for python-apt < 1.9 section = apt_pkg.rewrite_section(tag_section, apt_pkg.REWRITE_PACKAGE_ORDER, new_tags) dest.write(section.encode("utf-8"))
def test_read_binary_file_with_limit(self): """ With a positive limit L{read_binary_file} reads up to L{limit} bytes from the start of the file. """ path = self.makeFile("foo bar") self.assertEqual(read_binary_file(path, limit=3), b"foo")
def test_read_binary_file_with_negative_limit(self): """ With a negative limit L{read_binary_file} reads only the tail of the file. """ path = self.makeFile("foo bar from end") self.assertEqual(read_binary_file(path, limit=-3), b"end")
def test_read_binary_file(self): """ With no options L{read_binary_file} reads the whole file passed as argument. """ path = self.makeFile("foo") self.assertEqual(read_binary_file(path), b"foo")
def _reprocess_holding(self): """ Unhold accepted messages left behind, and hold unaccepted pending messages. """ offset = 0 pending_offset = self.get_pending_offset() accepted_types = self.get_accepted_types() for old_filename in self._walk_messages(): flags = self._get_flags(old_filename) try: message = bpickle.loads(read_binary_file(old_filename)) except ValueError as e: logging.exception(e) if HELD not in flags: offset += 1 else: accepted = message["type"] in accepted_types if HELD in flags: if accepted: new_filename = self._get_next_message_filename() os.rename(old_filename, new_filename) self._set_flags(new_filename, set(flags) - set(HELD)) else: if not accepted and offset >= pending_offset: self._set_flags(old_filename, set(flags) | set(HELD)) offset += 1
def get_pending_messages(self, max=None): """Get any pending messages that aren't being held, up to max.""" accepted_types = self.get_accepted_types() server_api = self.get_server_api() messages = [] for filename in self._walk_pending_messages(): if max is not None and len(messages) >= max: break data = read_binary_file(self._message_dir(filename)) try: # don't reinterpret messages that are meant to be sent out message = bpickle.loads(data, as_is=True) except ValueError as e: logging.exception(e) self._add_flags(filename, BROKEN) else: if u"type" not in message: # Special case to decode keys for messages which were # serialized by py27 prior to py3 upgrade, and having # implicit byte message keys. Message may still get # rejected by the server, but it won't block the client # broker. (lp: #1718689) message = { (k if isinstance(k, str) else k.decode("ascii")): v for k, v in message.items()} message[u"type"] = message[u"type"].decode("ascii") unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) if unknown_type or unknown_api: self._add_flags(filename, HELD) else: messages.append(message) return messages
def test_read_binary_file_with_limit(self): """ With a positive limit L{read_binary_file} reads only the bytes after the given limit. """ path = self.makeFile("foo bar") self.assertEqual(read_binary_file(path, limit=3), b" bar")
def _get_vm_by_vendor(sys_vendor_path): """Return the VM type byte string (possibly empty) based on the vendor.""" # Use lower-key string for vendors, since we do case-insentive match. # We need bytes here as required by the message schema. vendor = read_binary_file(sys_vendor_path, limit=1024).lower() content_vendors_map = ( (b"amazon ec2", b"kvm"), (b"bochs", b"kvm"), (b"digitalocean", b"kvm"), (b"google", b"gce"), (b"innotek", b"virtualbox"), (b"microsoft", b"hyperv"), (b"nutanix", b"kvm"), (b"openstack", b"kvm"), (b"qemu", b"kvm"), (b"kvm", b"kvm"), (b"vmware", b"vmware"), (b"rhev", b"kvm") ) for name, vm_type in content_vendors_map: if name in vendor: return vm_type return b""
def get_package_stanza(self, deb_path): """Return a stanza for the package to be included in a Packages file. @param deb_path: The path to the deb package. """ deb_file = open(deb_path) deb = apt_inst.DebFile(deb_file) control = deb.control.extractdata("control") deb_file.close() filename = os.path.basename(deb_path) size = os.path.getsize(deb_path) contents = read_binary_file(deb_path) md5 = hashlib.md5(contents).hexdigest() sha1 = hashlib.sha1(contents).hexdigest() sha256 = hashlib.sha256(contents).hexdigest() # Use rewrite_section to ensure that the field order is correct. return apt_pkg.rewrite_section(apt_pkg.TagSection(control), apt_pkg.REWRITE_PACKAGE_ORDER, [("Filename", filename), ("Size", str(size)), ("MD5sum", md5), ("SHA1", sha1), ("SHA256", sha256)])