def remove_test_vms(self, xmlpath=XMLPATH, prefix=VMPREFIX):
'''Aggressively remove any domain that has name in testing namespace.
:param prefix: name prefix of VMs to remove, can be a list of prefixes
'''
if isinstance(prefix, str):
prefixes = [prefix]
else:
prefixes = prefix
del prefix
# first, remove them Qubes-way
if os.path.exists(xmlpath):
try:
try:
app = self.app
except AttributeError:
app = qubes.Qubes(xmlpath)
try:
host_app = self.host_app
except AttributeError:
host_app = qubes.Qubes()
self.remove_vms([vm for vm in app.domains
if any(vm.name.startswith(prefix) for prefix in prefixes) or
(isinstance(vm, qubes.vm.dispvm.DispVM) and vm.name
not in host_app.domains)])
if not hasattr(self, 'host_app'):
host_app.close()
del host_app
if not hasattr(self, 'app'):
app.close()
del app
except qubes.exc.QubesException:
pass
os.unlink(xmlpath)
# now remove what was only in libvirt
conn = libvirt.open(qubes.config.defaults['libvirt_uri'])
for dom in conn.listAllDomains():
if any(dom.name().startswith(prefix) for prefix in prefixes):
self._remove_vm_libvirt(dom)
conn.close()
# finally remove anything that is left on disk
vmnames = set()
for dirspec in (
'qubes_appvms_dir',
'qubes_servicevms_dir',
'qubes_templates_dir'):
dirpath = os.path.join(qubes.config.qubes_base_dir,
qubes.config.system_path[dirspec])
if not os.path.exists(dirpath):
continue
for name in os.listdir(dirpath):
if any(name.startswith(prefix) for prefix in prefixes):
vmnames.add(name)
for vmname in vmnames:
self._remove_vm_disk(vmname)
for prefix in prefixes:
self._remove_vm_disk_lvm(prefix)
def setUp(self):
if not in_dom0:
self.skipTest('outside dom0')
super(SystemTestCase, self).setUp()
self.remove_test_vms()
# need some information from the real qubes.xml - at least installed
# templates; should not be used for testing, only to initialize self.app
self.host_app = qubes.Qubes(
os.path.join(qubes.config.qubes_base_dir,
qubes.config.system_path['qubes_store_filename']))
if os.path.exists(CLASS_XMLPATH):
shutil.copy(CLASS_XMLPATH, XMLPATH)
else:
shutil.copy(self.host_app.store, XMLPATH)
self.app = qubes.Qubes(XMLPATH)
os.environ['QUBES_XML_PATH'] = XMLPATH
self.app.register_event_handlers()
self.qubesd = self.loop.run_until_complete(
qubes.api.create_servers(qubes.api.admin.QubesAdminAPI,
qubes.api.internal.QubesInternalAPI,
app=self.app,
debug=True))
self.addCleanup(self.cleanup_app)
self.app.add_handler('domain-delete', self.close_qdb_on_remove)
def clear_outdated_error_markers(self):
# Clear outdated errors
for i in self.domdict.keys():
if self.domdict[i].slow_memset_react and \
self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
if dom_name is not None:
try:
qubes.Qubes().domains[str(dom_name)].fire_event(
'status:no-error', status='no-error',
msg=slow_memset_react_msg)
except LookupError:
pass
self.domdict[i].slow_memset_react = False
if self.domdict[i].no_progress and \
self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
if dom_name is not None:
try:
qubes.Qubes().domains[str(dom_name)].fire_event(
'status:no-error', status='no-error',
msg=no_progress_msg)
except LookupError:
pass
self.domdict[i].no_progress = False
def parse_args(self, args=None, namespace=None):
namespace = super(QubesArgumentParser,
self).parse_args(args, namespace)
if self._want_app and not self._want_app_no_instance:
self.set_qubes_verbosity(namespace)
namespace.app = qubes.Qubes(namespace.app,
offline_mode=namespace.offline_mode)
if self._want_force_root:
self.dont_run_as_root(namespace)
for action in self._actions:
# pylint: disable=protected-access
if issubclass(action.__class__, QubesAction):
action.parse_qubes_app(self, namespace)
elif issubclass(action.__class__, argparse._SubParsersAction): # pylint: disable=no-member
assert hasattr(namespace, 'command')
command = namespace.command
subparser = action._name_parser_map[command]
for subaction in subparser._actions:
if issubclass(subaction.__class__, QubesAction):
subaction.parse_qubes_app(self, namespace)
return namespace
def test_001_property(self):
self.assertEqual(0, qubes.tools.qubes_create.main([
'--qubesxml', qubes.tests.XMLPATH,
'--property', 'default_kernel=testkernel']))
self.assertEqual('testkernel',
qubes.Qubes(qubes.tests.XMLPATH).default_kernel)
def setUp(self):
super(TC_90_Qubes, self).setUp()
self.app = qubes.Qubes('/tmp/qubestest.xml', load=False,
offline_mode=True)
self.addCleanup(self.cleanup_qubes)
self.app.load_initial_values()
self.template = self.app.add_new_vm('TemplateVM', name='test-template',
label='green')
def with_qubes():
log.debug("Opening Qubes connection")
q = qubes.Qubes()
try:
yield q
finally:
q.close()
log.debug("Closed Qubes connection")
def setUp(self):
super().setUp()
self.app = qubes.Qubes('/tmp/qubestest.xml', load=False,
offline_mode=True)
self.test_dir = '/var/tmp/test-varlibqubes'
self.test_patch = mock.patch.dict(
qubes.config.defaults['pool_configs']['varlibqubes'],
{'dir_path': self.test_dir})
self.test_patch.start()
def vm(self, value): # pylint: disable=C0103
'''
Get Qubes VM object from qvm.collection and set it here.
'''
if value:
app = qubes.Qubes()
try:
self._vm = app.domains[value]
except KeyError:
self._vm = None # pylint: disable=W0212
def test_101_property_migrate_label(self):
xml_template = """<?xml version="1.0" encoding="utf-8" ?>
<qubes version="3.0">
<labels>
<label id="label-1" color="{old_gray}">gray</label>
</labels>
<pools>
<pool driver="file" dir_path="/tmp/qubes-test" name="default"/>
</pools>
<domains>
<domain class="StandaloneVM" id="domain-1">
<properties>
<property name="qid">1</property>
<property name="name">sys-net</property>
<property name="provides_network">True</property>
<property name="label" ref="label-1" />
<property name="netvm"></property>
<property name="uuid">2fcfc1f4-b2fe-4361-931a-c5294b35edfa</property>
</properties>
<features/>
<devices class="pci"/>
</domain>
</domains>
</qubes>
"""
with self.subTest('replace_label'):
with open('/tmp/qubestest.xml', 'w') as xml_file:
xml_file.write(xml_template.format(old_gray='0x555753'))
self.app = qubes.Qubes('/tmp/qubestest.xml', offline_mode=True)
self.assertEqual(self.app.get_label('gray').color, '0x555555')
self.app.close()
del self.app
with self.subTest('dont_replace_label'):
with open('/tmp/qubestest.xml', 'w') as xml_file:
xml_file.write(xml_template.format(old_gray='0x123456'))
self.app = qubes.Qubes('/tmp/qubestest.xml', offline_mode=True)
self.assertEqual(self.app.get_label('gray').color, '0x123456')
self.app.close()
del self.app
def list_templates():
'''Returns tuple of template names available in the system.'''
global _templates
if _templates is None:
try:
app = qubes.Qubes()
_templates = tuple(vm.name for vm in app.domains
if isinstance(vm, qubes.vm.templatevm.TemplateVM))
app.close()
del app
except OSError:
_templates = ()
return _templates
def load_tests(loader, tests, pattern):
try:
app = qubes.Qubes()
templates = [
vm.name for vm in app.domains
if isinstance(vm, qubes.vm.templatevm.TemplateVM)
]
except OSError:
templates = []
for template in templates:
tests.addTests(
loader.loadTestsFromTestCase(
type('TC_00_Dom0Upgrade_' + template,
(TC_00_Dom0UpgradeMixin, qubes.tests.QubesTestCase),
{'template': template})))
return tests
def list_templates():
'''Returns tuple of template names available in the system.'''
global _templates
if _templates is None:
if 'QUBES_TEST_TEMPLATES' in os.environ:
_templates = os.environ['QUBES_TEST_TEMPLATES'].split()
if _templates is None:
try:
app = qubes.Qubes()
_templates = tuple(vm.name for vm in app.domains
if isinstance(vm, qubes.vm.templatevm.TemplateVM) and
vm.features.get('os', None) != 'Windows')
app.close()
del app
except OSError:
_templates = ()
return _templates
def do_balance(self):
self.log.debug('do_balance()')
if os.path.isfile('/var/run/qubes/do-not-membalance'):
self.log.debug('do-not-membalance file preset, returning')
return
self.refresh_memactual()
self.clear_outdated_error_markers()
xenfree = self.get_free_xen_memory()
memset_reqs = qubes.qmemman.algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
if not self.is_balance_req_significant(memset_reqs, xenfree):
return
self.print_stats(xenfree, memset_reqs)
prev_memactual = {}
for i in self.domdict.keys():
prev_memactual[i] = self.domdict[i].memory_actual
for rq in memset_reqs:
dom, mem = rq
# Force to always have at least 0.9*self.XEN_FREE_MEM_LEFT (some
# margin for rounding errors). Before giving memory to
# domain, ensure that others have gived it back.
# If not - wait a little.
ntries = 5
while self.get_free_xen_memory() - (mem - self.domdict[dom].memory_actual) < 0.9*self.XEN_FREE_MEM_LEFT:
self.log.debug('do_balance dom={!r} sleeping ntries={}'.format(
dom, ntries))
time.sleep(self.BALOON_DELAY)
self.refresh_memactual()
ntries -= 1
if ntries <= 0:
# Waiting haven't helped; Find which domain get stuck and
# abort balance (after distributing what we have)
for rq2 in memset_reqs:
dom2, mem2 = rq2
if dom2 == dom:
# All donors have been procesed
break
# allow some small margin
if self.domdict[dom2].memory_actual > self.domdict[dom2].last_target + self.XEN_FREE_MEM_LEFT/4:
# VM didn't react to memory request at all, remove from donors
if prev_memactual[dom2] == self.domdict[dom2].memory_actual:
self.log.warning(
'dom {!r} didnt react to memory request'
' (holds {}, requested balloon down to {})'
.format(dom2,
self.domdict[dom2].memory_actual,
mem2))
self.domdict[dom2].no_progress = True
dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
if dom_name is not None:
try:
qubes.Qubes().domains[str(
dom_name)].fire_event(
'status:error', status='error',
msg=no_progress_msg)
except LookupError:
pass
else:
self.log.warning('dom {!r} still hold more'
' memory than have assigned ({} > {})'
.format(dom2,
self.domdict[dom2].memory_actual,
mem2))
self.domdict[dom2].slow_memset_react = True
dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
if dom_name is not None:
try:
qubes.Qubes().domains[str(
dom_name)].fire_event(
'status:error', status='error',
msg=slow_memset_react_msg)
except LookupError:
pass
self.mem_set(dom, self.get_free_xen_memory() + self.domdict[dom].memory_actual - self.XEN_FREE_MEM_LEFT)
return
self.mem_set(dom, mem)
async def _ask_password(self, receiving_cmd):
"""
Call out to QRexec qubes.AskPassword and passes the resulting stdout
to :receiving_cmd: when successful.
"""
if not self.app_reference:
# TODO this sucks, is there an easier way to get a reference to the
# global 'app' qubes.Qubes() instance?
self.log.warning("had no REFERENCE ETC to APP VARIABLE!!")
self.app_reference = qubes.Qubes()
pw_vm = self.app_reference.domains[self.ask_password_domain]
if not pw_vm:
raise qubes.storage.StoragePoolException(
"unable to find handle for ask_password_domain={}".format(
self.ask_password_domain
)
)
pw_pipe_in, pw_pipe_out = os.pipe()
try:
# TODO how do we pass $1 to this stuff? we can pass **kwargs to
# asyncio.create_subprocess_exec, but we can't influence command
# await pw_vm.run_service_for_stdio(
# TODO THIS used to work, now it f*****g broke.
# 'qubes.AskPassword',
# autostart=True, gui=True,
# user='******',
# input=self.name.encode()+b'\n', # context for the prompt
# stdout=pw_pipe_out)
# TODO instead for now:
environ = os.environ.copy()
environ["QREXEC_REMOTE_DOMAIN"] = "dom0"
environ["DISPLAY"] = ":0"
proc = await asyncio.create_subprocess_exec(
*['sudo','-Eu','user', '/etc/qubes-rpc/qubes.AskPassword'],
stdout=pw_pipe_out,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
close_fds=True,
env=environ,
)
proc.stdin.write(self.name.encode()+b'\n')
await proc.stdin.drain()
# TODO flush aka drain+write_eof() apparently, wtf python
proc.stdin.write_eof()
await proc.wait()
except subprocess.CalledProcessError as e:
os.close(pw_pipe_in)
os.close(pw_pipe_out)
self.log.warning(
"zfs ask_password: exception while trying to get pw: {}".format(
e
)
)
raise e
environ = os.environ.copy()
environ["LC_ALL"] = "C.utf8"
p = await asyncio.create_subprocess_exec(
*receiving_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=pw_pipe_in,
close_fds=True,
env=environ
)
zfsout, zfserr = await p.communicate()
self.log.warning("ZFS key consumer: ".format(
p.returncode, zfsout, zfserr))
os.close(pw_pipe_in)
os.close(pw_pipe_out)
# TODO make sure zfs get keystat foo == 'available'
# and zfs get encryptionroot foo == foo
return (p, zfsout, zfserr)
def test_100_property_migrate_default_fw_netvm(self):
xml_template = '''<?xml version="1.0" encoding="utf-8" ?>
<qubes version="3.0">
<properties>
<property name="default_netvm">{default_netvm}</property>
<property name="default_fw_netvm">{default_fw_netvm}</property>
</properties>
<labels>
<label id="label-1" color="#cc0000">red</label>
</labels>
<pools>
<pool driver="file" dir_path="/tmp/qubes-test" name="default"/>
</pools>
<domains>
<domain class="StandaloneVM" id="domain-1">
<properties>
<property name="qid">1</property>
<property name="name">sys-net</property>
<property name="provides_network">True</property>
<property name="label" ref="label-1" />
<property name="netvm"></property>
<property name="uuid">2fcfc1f4-b2fe-4361-931a-c5294b35edfa</property>
</properties>
<features/>
<devices class="pci"/>
</domain>
<domain class="StandaloneVM" id="domain-2">
<properties>
<property name="qid">2</property>
<property name="name">sys-firewall</property>
<property name="provides_network">True</property>
<property name="label" ref="label-1" />
<property name="uuid">9a6d9689-25f7-48c9-a15f-8205d6c5b7c6</property>
</properties>
</domain>
<domain class="StandaloneVM" id="domain-3">
<properties>
<property name="qid">3</property>
<property name="name">appvm</property>
<property name="label" ref="label-1" />
<property name="uuid">1d6aab41-3262-400a-b3d3-21aae8fdbec8</property>
</properties>
</domain>
</domains>
</qubes>
'''
with self.subTest('default_setup'):
with open('/tmp/qubestest.xml', 'w') as xml_file:
xml_file.write(
xml_template.format(default_netvm='sys-firewall',
default_fw_netvm='sys-net'))
self.app = qubes.Qubes('/tmp/qubestest.xml', offline_mode=True)
self.assertEqual(self.app.domains['sys-net'].netvm, None)
self.assertEqual(self.app.domains['sys-firewall'].netvm,
self.app.domains['sys-net'])
# property is no longer "default"
self.assertFalse(
self.app.domains['sys-firewall'].property_is_default('netvm'))
# verify that appvm.netvm is unaffected
self.assertTrue(
self.app.domains['appvm'].property_is_default('netvm'))
self.assertEqual(self.app.domains['appvm'].netvm,
self.app.domains['sys-firewall'])
with self.assertRaises(AttributeError):
self.app.default_fw_netvm
self.app.close()
del self.app
with self.subTest('same'):
with open('/tmp/qubestest.xml', 'w') as xml_file:
xml_file.write(
xml_template.format(default_netvm='sys-net',
default_fw_netvm='sys-net'))
self.app = qubes.Qubes('/tmp/qubestest.xml', offline_mode=True)
self.assertEqual(self.app.domains['sys-net'].netvm, None)
self.assertEqual(self.app.domains['sys-firewall'].netvm,
self.app.domains['sys-net'])
self.assertTrue(
self.app.domains['sys-firewall'].property_is_default('netvm'))
# verify that appvm.netvm is unaffected
self.assertTrue(
self.app.domains['appvm'].property_is_default('netvm'))
self.assertEqual(self.app.domains['appvm'].netvm,
self.app.domains['sys-net'])
with self.assertRaises(AttributeError):
self.app.default_fw_netvm
with self.subTest('loop'):
with open('/tmp/qubestest.xml', 'w') as xml_file:
xml_file.write(
xml_template.format(default_netvm='sys-firewall',
default_fw_netvm='sys-firewall'))
self.app = qubes.Qubes('/tmp/qubestest.xml', offline_mode=True)
self.assertEqual(self.app.domains['sys-net'].netvm, None)
# this was netvm loop, better set to none, to not crash qubesd
self.assertEqual(self.app.domains['sys-firewall'].netvm, None)
self.assertFalse(
self.app.domains['sys-firewall'].property_is_default('netvm'))
# verify that appvm.netvm is unaffected
self.assertTrue(
self.app.domains['appvm'].property_is_default('netvm'))
self.assertEqual(self.app.domains['appvm'].netvm,
self.app.domains['sys-firewall'])
with self.assertRaises(AttributeError):
self.app.default_fw_netvm
def backup_do(self):
# pylint: disable=too-many-statements
if self.passphrase is None:
raise qubes.exc.QubesException("No passphrase set")
if not isinstance(self.passphrase, bytes):
self.passphrase = self.passphrase.encode('utf-8')
qubes_xml = self.app.store
self.tmpdir = tempfile.mkdtemp()
shutil.copy(qubes_xml, os.path.join(self.tmpdir, 'qubes.xml'))
qubes_xml = os.path.join(self.tmpdir, 'qubes.xml')
backup_app = qubes.Qubes(qubes_xml, offline_mode=True)
backup_app.events_enabled = False
files_to_backup = self._files_to_backup
# make sure backup_content isn't set initially
for vm in backup_app.domains:
vm.events_enabled = False
vm.features['backup-content'] = False
for qid, vm_info in files_to_backup.items():
# VM is included in the backup
backup_app.domains[qid].features['backup-content'] = True
backup_app.domains[qid].features['backup-path'] = vm_info.subdir
backup_app.domains[qid].features['backup-size'] = vm_info.size
backup_app.save()
del backup_app
vmproc = None
if self.target_vm is not None:
# Prepare the backup target (Qubes service call)
# If APPVM, STDOUT is a PIPE
read_fd, write_fd = os.pipe()
vmproc = yield from self.target_vm.run_service(
'qubes.Backup',
stdin=read_fd,
stderr=subprocess.PIPE,
stdout=subprocess.DEVNULL)
os.close(read_fd)
os.write(write_fd,
(self.target_dir.replace("\r", "").replace("\n", "") +
"\n").encode())
backup_stdout = write_fd
else:
# Prepare the backup target (local file)
if os.path.isdir(self.target_dir):
backup_target = self.target_dir + "/qubes-{0}". \
format(time.strftime("%Y-%m-%dT%H%M%S"))
else:
backup_target = self.target_dir
# Create the target directory
if not os.path.exists(os.path.dirname(self.target_dir)):
raise qubes.exc.QubesException(
"ERROR: the backup directory for {0} does not exists".
format(self.target_dir))
# If not APPVM, STDOUT is a local file
backup_stdout = open(backup_target, 'wb')
# Tar with tape length does not deals well with stdout
# (close stdout between two tapes)
# For this reason, we will use named pipes instead
self.log.debug("Working in {}".format(self.tmpdir))
self.log.debug("Will backup: {}".format(files_to_backup))
header_files = yield from self._prepare_backup_header()
# Setup worker to send encrypted data chunks to the backup_target
to_send = asyncio.Queue(10)
send_proc = SendWorker(to_send, self.tmpdir, backup_stdout)
send_task = asyncio.ensure_future(send_proc.run())
vmproc_task = None
if vmproc is not None:
vmproc_task = asyncio.ensure_future(
self._monitor_process(
vmproc, 'Writing backup to VM {} failed'.format(
self.target_vm.name)))
asyncio.ensure_future(self._cancel_on_error(
vmproc_task, send_task))
for file_name in header_files:
yield from to_send.put(file_name)
qubes_xml_info = self.VMToBackup(None,
[self.FileToBackup(qubes_xml, '')],
'')
inner_archive_task = asyncio.ensure_future(
self._wrap_and_send_files(
itertools.chain([qubes_xml_info], files_to_backup.values()),
to_send))
asyncio.ensure_future(
self._cancel_on_error(send_task, inner_archive_task))
try:
try:
yield from inner_archive_task
except:
yield from to_send.put(QUEUE_ERROR)
# in fact we may be handling CancelledError, induced by
# exception in send_task or vmproc_task (and propagated by
# self._cancel_on_error call above); in such a case this
# yield from will raise exception, covering CancelledError -
# this is intended behaviour
if vmproc_task:
yield from vmproc_task
yield from send_task
raise
yield from send_task
finally:
if isinstance(backup_stdout, int):
os.close(backup_stdout)
else:
backup_stdout.close()
try:
if vmproc_task:
yield from vmproc_task
finally:
shutil.rmtree(self.tmpdir)
# Save date of last backup, only when backup succeeded
for qid, vm_info in files_to_backup.items():
if vm_info.vm:
vm_info.vm.backup_timestamp = \
int(datetime.datetime.now().strftime('%s'))
self.app.save()