def test(self):
"""Test debmarshal.privops.call."""
self.mox.StubOutWithMock(dbus, 'SystemBus', use_mock_anything=True)
bus = self.mox.CreateMock(dbus.bus.BusConnection)
proxy = self.mox.CreateMockAnything()
method = self.mox.CreateMockAnything()
dbus.SystemBus().AndReturn(bus)
bus.get_object(privops.DBUS_BUS_NAME,
privops.DBUS_OBJECT_PATH).AndReturn(proxy)
proxy.get_dbus_method(
'createNetwork',
dbus_interface=privops.DBUS_INTERFACE).AndReturn(method)
method(['www.company.com', 'login.company.com'])
self.mox.ReplayAll()
privops.call('createNetwork', ['www.company.com', 'login.company.com'])
def test(self):
"""Test debmarshal.privops.call."""
self.mox.StubOutWithMock(dbus, 'SystemBus', use_mock_anything=True)
bus = self.mox.CreateMock(dbus.bus.BusConnection)
proxy = self.mox.CreateMockAnything()
method = self.mox.CreateMockAnything()
dbus.SystemBus().AndReturn(bus)
bus.get_object(privops.DBUS_BUS_NAME, privops.DBUS_OBJECT_PATH).AndReturn(
proxy)
proxy.get_dbus_method(
'createNetwork', dbus_interface=privops.DBUS_INTERFACE).AndReturn(
method)
method(['www.company.com', 'login.company.com'])
self.mox.ReplayAll()
privops.call('createNetwork', ['www.company.com', 'login.company.com'])
def runTest(test):
"""Actually run a debmarshal test.
Args:
Path to the test to run.
"""
start = time.time()
# First, load the configuration
config = yaml.safe_load(open(os.path.join(test, 'config.yml')))
# Next, network configuration. Configure the network as it will be
# for the test run.
vms = config['vms'].keys()
net_name, net_gate, net_mask, net_vms = privops.call('createNetwork',
vms)
try:
# Then, spawn the web server for serving the test configuration.
httpd = subprocess.Popen(['python', '-m', 'debmarshal.web', test],
stdout=subprocess.PIPE)
web_port = httpd.stdout.read()
domain = config['domain']
try:
images = {}
domains = {}
for vm in vms:
vm_config = config['vms'][vm]
dist = base.findDistribution(vm_config['distribution'])
pristine_disk_path = dist.diskPath(vm, domain, test, vm_config)
fd, disk_path = tempfile.mkstemp()
os.close(fd)
subprocess.check_call(['cp',
'--sparse=always',
pristine_disk_path,
disk_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
images[vm] = disk_path
for vm in vms:
vm_config = config['vms'][vm]
memory = vm_config.get('memory', '128M')
arch = vm_config.get('arch', '')
domains[vm] = privops.call('createDomain',
memory,
[images[vm]],
net_name,
net_vms[vm][1],
'qemu',
arch,
{})
# Wait for the master VM to boot up, then run the test
key_path = os.path.join(test, 'id_rsa')
master = vms[0]
master_ip = net_vms[master][0]
while True:
if not subprocess.call(['ssh',
'-i', key_path,
'-o', 'ConnectTimeout=1',
'-l', 'root',
master_ip,
'true'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE):
break
else:
time.sleep(5)
subprocess.check_call(['ssh',
'-i', key_path,
'-l', 'root',
master_ip,
'wget',
'-N',
'-O', '/root/script',
'http://%s:%s/script' % (net_gate, web_port)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret = subprocess.call(['ssh',
'-i', key_path,
'-l', 'root',
master_ip,
'/root/script',
str(web_port)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
print ret
finally:
for domain in domains.values():
privops.call('destroyDomain', domain, 'qemu')
for image in images.values():
os.unlink(image)
os.kill(httpd.pid, signal.SIGTERM)
finally:
privops.call('destroyNetwork', net_name)
def prepareTest(test):
"""Prepare the disk image for a single test.
Args:
Path to a debmarshal test.
"""
start = time.time()
# First, load the configuration
config = yaml.safe_load(open(os.path.join(test, 'config.yml')))
# Next, network configuration. Configure the network as it will be
# for the test run.
vms = config['vms'].keys()
net_name, net_gate, net_mask, net_vms = privops.call('createNetwork',
vms)
# If there's not already an ssh key packaged with the test to use,
# generate one.
key_path = os.path.join(test, 'id_rsa')
if not os.path.exists(key_path):
subprocess.check_call(['ssh-keygen',
'-N', '',
'-f', key_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
# Then, spawn the web server for serving the test configuration.
httpd = subprocess.Popen(['python', '-m', 'debmarshal.web', test],
stdout=subprocess.PIPE)
web_port = httpd.stdout.read()
try:
results_queue = Queue.Queue()
threads = []
# For each VM that needs to be installed, create a thread to run
# the installer
for vm in vms:
dist = base.findDistribution(config['vms'][vm]['distribution'])
threads.append(threading.Thread(
target=dist.doInstall,
args=(test,
vm,
net_name,
net_gate,
net_vms[vm][1],
web_port,
results_queue)))
# Ready, set, go!
for t in threads:
t.start()
# Wait until all of the threads have completed execution
for t in threads:
t.join()
return prepareSummary(results_queue, start)
finally:
os.kill(httpd.pid, signal.SIGTERM)
finally:
privops.call('destroyNetwork', net_name)
def doInstall(test, vm, net_name, net_gateway, mac, web_port, results_queue):
"""Start and monitor an unattended Ubuntu install.
This function will start an unattended Ubuntu install, running in
a VM, as part of a debmarshal test. It will also monitor that
install to completion.
This is intended to be run as a separate thread of execution, so
that many installs can run in parallel.
Its success or failure is reported back to the spawning process
through the results_queue.
Args:
test: A path to a debmarshal test
vm: The hostname of the vm within the test to install
net_name: The name of a debmarshal network configured for this VM.
net_gateway: The gateway of the debmarshal network.
mac: The MAC address of this VM.
web_port: The port the test is being served over. The spawning
process should be serving the directory containing the
debmarshal test over HTTP.
results_queue: A Queue.Queue object that doInstall will store its
success or failure into that
"""
try:
config = yaml.safe_load(open(os.path.join(test, 'config.yml')))
domain = config['domain']
vm_config = config['vms'][vm]
memory, suite, arch, disk_size, preseed_path = parseConfig(
test, vm, vm_config)
deb_arch = arch if arch != 'x86_64' else 'amd64'
disk_path = Ubuntu.diskPath(vm, domain, test, vm_config)
disk_dir = os.path.dirname(disk_path)
if not os.path.exists(disk_dir):
os.makedirs(disk_dir)
disk_lock = open(disk_path + '.lock', 'w')
fcntl.lockf(disk_lock, fcntl.LOCK_EX)
if os.path.exists(disk_path):
results_queue.put((test, vm, True, 'cached'))
return
base.createSparseFile(disk_path, disk_size)
try:
kernel, initrd = loadKernel(suite, deb_arch)
cmdline = genCommandLine(preseed_path)
cmdline += ' preseed/url=http://%s:%s/%s.preseed' % (
net_gateway, web_port, vm)
cmdline += ' mirror/http/hostname=%s:9999' % net_gateway
cmdline += ' mirror/http/directory=/ubuntu'
cmdline += ' mirror/http/proxy='
dom_name = privops.call('createDomain',
memory,
[disk_path],
net_name,
mac,
'qemu',
arch,
{'kernel': kernel,
'initrd': initrd,
'cmdline': cmdline,
'on_reboot': 'destroy'})
# Now wait for the install to finish...
#
# libvirt has an API for integration with somebody else's main
# loop. Unfortunately, they forgot to make it usable by
# humans. libvirt-glib in Debian experimental might be a good
# jumping off point.
#
# TODO(ebroder): Figure out how to use some sort of select()
# loop instead of a while-sleep loop.
while True:
time.sleep(10)
if dom_name not in qemu.QEMU.listDomains():
break
results_queue.put((test, vm, True, None))
except:
os.remove(disk_path)
raise
except:
results_queue.put((test, vm, False, traceback.format_exc()))
def runTest(test):
"""Actually run a debmarshal test.
Args:
Path to the test to run.
"""
start = time.time()
# First, load the configuration
config = yaml.safe_load(open(os.path.join(test, "config.yml")))
# Next, network configuration. Configure the network as it will be
# for the test run.
vms = config["vms"].keys()
net_name, net_gate, net_mask, net_vms = privops.call("createNetwork", vms)
try:
# Then, spawn the web server for serving the test configuration.
httpd = subprocess.Popen(["python", "-m", "debmarshal.web", test], stdout=subprocess.PIPE)
web_port = httpd.stdout.read()
domain = config["domain"]
try:
images = {}
domains = {}
for vm in vms:
vm_config = config["vms"][vm]
dist = base.findDistribution(vm_config["distribution"])
pristine_disk_path = dist.diskPath(vm, domain, test, vm_config)
fd, disk_path = tempfile.mkstemp()
os.close(fd)
subprocess.check_call(
["cp", "--sparse=always", pristine_disk_path, disk_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
images[vm] = disk_path
for vm in vms:
vm_config = config["vms"][vm]
memory = vm_config.get("memory", "128M")
arch = vm_config.get("arch", "")
domains[vm] = privops.call(
"createDomain", memory, [images[vm]], net_name, net_vms[vm][1], "qemu", arch, {}
)
# Wait for the master VM to boot up, then run the test
key_path = os.path.join(test, "id_rsa")
master = vms[0]
master_ip = net_vms[master][0]
while True:
if not subprocess.call(
["ssh", "-i", key_path, "-o", "ConnectTimeout=1", "-l", "root", master_ip, "true"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
):
break
else:
time.sleep(5)
subprocess.check_call(
[
"ssh",
"-i",
key_path,
"-l",
"root",
master_ip,
"wget",
"-N",
"-O",
"/root/script",
"http://%s:%s/script" % (net_gate, web_port),
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
ret = subprocess.call(
["ssh", "-i", key_path, "-l", "root", master_ip, "/root/script", str(web_port)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
print ret
finally:
for domain in domains.values():
privops.call("destroyDomain", domain, "qemu")
for image in images.values():
os.unlink(image)
os.kill(httpd.pid, signal.SIGTERM)
finally:
privops.call("destroyNetwork", net_name)
def prepareTest(test):
"""Prepare the disk image for a single test.
Args:
Path to a debmarshal test.
"""
start = time.time()
# First, load the configuration
config = yaml.safe_load(open(os.path.join(test, "config.yml")))
# Next, network configuration. Configure the network as it will be
# for the test run.
vms = config["vms"].keys()
net_name, net_gate, net_mask, net_vms = privops.call("createNetwork", vms)
# If there's not already an ssh key packaged with the test to use,
# generate one.
key_path = os.path.join(test, "id_rsa")
if not os.path.exists(key_path):
subprocess.check_call(
["ssh-keygen", "-N", "", "-f", key_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
# Then, spawn the web server for serving the test configuration.
httpd = subprocess.Popen(["python", "-m", "debmarshal.web", test], stdout=subprocess.PIPE)
web_port = httpd.stdout.read()
try:
results_queue = Queue.Queue()
threads = []
# For each VM that needs to be installed, create a thread to run
# the installer
for vm in vms:
dist = base.findDistribution(config["vms"][vm]["distribution"])
threads.append(
threading.Thread(
target=dist.doInstall,
args=(test, vm, net_name, net_gate, net_vms[vm][1], web_port, results_queue),
)
)
# Ready, set, go!
for t in threads:
t.start()
# Wait until all of the threads have completed execution
for t in threads:
t.join()
return prepareSummary(results_queue, start)
finally:
os.kill(httpd.pid, signal.SIGTERM)
finally:
privops.call("destroyNetwork", net_name)
def doInstall(test, vm, net_name, net_gateway, mac, web_port,
results_queue):
"""Start and monitor an unattended Ubuntu install.
This function will start an unattended Ubuntu install, running in
a VM, as part of a debmarshal test. It will also monitor that
install to completion.
This is intended to be run as a separate thread of execution, so
that many installs can run in parallel.
Its success or failure is reported back to the spawning process
through the results_queue.
Args:
test: A path to a debmarshal test
vm: The hostname of the vm within the test to install
net_name: The name of a debmarshal network configured for this VM.
net_gateway: The gateway of the debmarshal network.
mac: The MAC address of this VM.
web_port: The port the test is being served over. The spawning
process should be serving the directory containing the
debmarshal test over HTTP.
results_queue: A Queue.Queue object that doInstall will store its
success or failure into that
"""
try:
config = yaml.safe_load(open(os.path.join(test, 'config.yml')))
domain = config['domain']
vm_config = config['vms'][vm]
memory, suite, arch, disk_size, preseed_path = parseConfig(
test, vm, vm_config)
deb_arch = arch if arch != 'x86_64' else 'amd64'
disk_path = Ubuntu.diskPath(vm, domain, test, vm_config)
disk_dir = os.path.dirname(disk_path)
if not os.path.exists(disk_dir):
os.makedirs(disk_dir)
disk_lock = open(disk_path + '.lock', 'w')
fcntl.lockf(disk_lock, fcntl.LOCK_EX)
if os.path.exists(disk_path):
results_queue.put((test, vm, True, 'cached'))
return
base.createSparseFile(disk_path, disk_size)
try:
kernel, initrd = loadKernel(suite, deb_arch)
cmdline = genCommandLine(preseed_path)
cmdline += ' preseed/url=http://%s:%s/%s.preseed' % (
net_gateway, web_port, vm)
cmdline += ' mirror/http/hostname=%s:9999' % net_gateway
cmdline += ' mirror/http/directory=/ubuntu'
cmdline += ' mirror/http/proxy='
dom_name = privops.call(
'createDomain', memory, [disk_path], net_name, mac, 'qemu',
arch, {
'kernel': kernel,
'initrd': initrd,
'cmdline': cmdline,
'on_reboot': 'destroy'
})
# Now wait for the install to finish...
#
# libvirt has an API for integration with somebody else's main
# loop. Unfortunately, they forgot to make it usable by
# humans. libvirt-glib in Debian experimental might be a good
# jumping off point.
#
# TODO(ebroder): Figure out how to use some sort of select()
# loop instead of a while-sleep loop.
while True:
time.sleep(10)
if dom_name not in qemu.QEMU.listDomains():
break
results_queue.put((test, vm, True, None))
except:
os.remove(disk_path)
raise
except:
results_queue.put((test, vm, False, traceback.format_exc()))
