def test_power_commands_are_not_run_twice(self):
# We will make the dummy power script sleep for this long:
power_sleep = 4
# To reproduce this bug, we need to queue up three commands for the
# same system (so they are run in sequence by beaker-provision), where
# the commands take enough time that the second one will still be
# running on the next iteration of the polling loop. The third command
# will be run twice.
assert power_sleep < get_conf().get('SLEEP_TIME')
assert 2 * power_sleep > get_conf().get('SLEEP_TIME')
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = power_sleep # make power script sleep
system.action_power(action=u'off', service=u'testdata')
system.action_power(action=u'off', service=u'testdata')
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system, timeout=5 * power_sleep)
with session.begin():
session.expire_all()
self.assertEquals(system.command_queue[0].status, CommandStatus.completed)
self.assertEquals(system.command_queue[1].status, CommandStatus.completed)
self.assertEquals(system.command_queue[2].status, CommandStatus.completed)
# The bug manifests as two "Completed" records for the power
# command which ran twice
self.assertEquals(system.dyn_activity
.filter_by(field_name=u'Power', new_value=u'Completed')
.count(), 3)
def test_power_commands_are_not_run_twice(self):
# We will make the dummy power script sleep for this long:
power_sleep = 4
# To reproduce this bug, we need to queue up three commands for the
# same system (so they are run in sequence by beaker-provision), where
# the commands take enough time that the second one will still be
# running on the next iteration of the polling loop. The third command
# will be run twice.
assert power_sleep < get_conf().get('SLEEP_TIME')
assert 2 * power_sleep > get_conf().get('SLEEP_TIME')
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = power_sleep # make power script sleep
system.action_power(action=u'off', service=u'testdata')
system.action_power(action=u'off', service=u'testdata')
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system, timeout=5 * power_sleep)
with session.begin():
session.expire_all()
self.assertEquals(system.command_queue[0].status, CommandStatus.completed)
self.assertEquals(system.command_queue[1].status, CommandStatus.completed)
self.assertEquals(system.command_queue[2].status, CommandStatus.completed)
# The bug manifests as two "Completed" records for the power
# command which ran twice
self.assertEquals(system.dyn_activity
.filter_by(field_name=u'Power', new_value=u'Completed')
.count(), 3)
def test_power_quiescent_period(self):
# Test that we do in fact wait for the quiescent period to pass
# before running a command
if daemons_running_externally():
raise SkipTest('cannot examine logs of remote beaker-provision')
provision_process, = [p for p in processes if p.name == \
'beaker-provision']
# These times are needed to guarantee that we are actually waiting for
# the quiescent period and not waiting for another poll loop
quiescent_period = get_conf().get('SLEEP_TIME') * 3
timeout = get_conf().get('SLEEP_TIME') * 2
try:
provision_process.start_output_capture()
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_quiescent_period = quiescent_period
system.power.power_id = u'' # make power script not sleep
system.power.delay_until = None
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_completed(system, timeout=timeout)
self.fail('The quiescent period is not being respected')
except AssertionError:
# wait_for_commands() should timeout if the quiescent period is
#respected
pass
finally:
provision_output = provision_process.finish_output_capture()
# The initial command seen for a system will always wait for the full
# quiescent period
self.assertIn('Entering quiescent period, delaying %s seconds for '
'command %s' % (quiescent_period, system.command_queue[0].id),
provision_output)
def setUp(self):
with session.begin():
self.system = data_setup.create_system(lab_controller=self.get_lc())
self.recipe = data_setup.create_recipe()
data_setup.create_job_for_recipes([self.recipe])
data_setup.mark_recipe_installing(self.recipe, system=self.system)
self.console_log = os.path.join(get_conf().get('CONSOLE_LOGS'), self.system.fqdn)
self.cached_console_log = os.path.join(get_conf().get('CACHEPATH'), 'recipes',
str(self.recipe.id // 1000) + '+', str(self.recipe.id), 'console.log')
def setUp(self):
with session.begin():
self.system = data_setup.create_system(lab_controller=self.get_lc())
self.recipe = data_setup.create_recipe()
job = data_setup.create_job_for_recipes([self.recipe])
self.addCleanup(self.cleanup_job, job)
data_setup.mark_recipe_installing(self.recipe, system=self.system)
self.console_log = os.path.join(get_conf().get('CONSOLE_LOGS'), self.system.fqdn)
self.cached_console_log = os.path.join(get_conf().get('CACHEPATH'), 'recipes',
str(self.recipe.id // 1000) + '+', str(self.recipe.id), 'console.log')
def test_timeout_is_enforced_for_fetching_images(self):
with session.begin():
lc = self.get_lc()
system = data_setup.create_system(arch=u'x86_64', lab_controller=lc)
distro_tree = data_setup.create_distro_tree(arch=u'x86_64',
lab_controllers=[lc],
# /slow/600 means the response will be delayed 10 minutes
urls=['http://localhost:19998/slow/600/'])
installation = Installation(distro_tree=distro_tree, system=system,
kernel_options=u'')
system.configure_netboot(installation=installation, service=u'testdata')
wait_for_commands_to_finish(system, timeout=(2 * get_conf().get('SLEEP_TIME')
+ get_conf().get('IMAGE_FETCH_TIMEOUT')))
self.assertEquals(system.command_queue[0].action, u'configure_netboot')
self.assertEquals(system.command_queue[0].status, CommandStatus.failed)
def test_power_passwords_are_not_logged(self):
if daemons_running_externally():
raise SkipTest('cannot examine logs of remote beaker-provision')
provision_process, = [
p for p in processes if p.name == 'beaker-provision'
]
try:
provision_process.start_output_capture()
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
self.addCleanup(self.cleanup_system, system)
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = u'' # make power script not sleep
system.power.power_passwd = u'dontleakmebro'
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system,
timeout=2 *
get_conf().get('SLEEP_TIME'))
finally:
provision_output = provision_process.finish_output_capture()
self.assert_('Handling command' in provision_output, provision_output)
self.assert_('Launching power script' in provision_output,
provision_output)
self.assert_(system.power.power_passwd not in provision_output,
provision_output)
def main():
parser = OptionParser()
parser.add_option("-c", "--config", help="Full path to config file to use")
parser.add_option("-f",
"--foreground",
default=False,
action="store_true",
help="run in foreground (do not spawn a daemon)")
parser.add_option("-p", "--pid-file", help="specify a pid file")
(opts, args) = parser.parse_args()
if opts.config:
load_conf(opts.config)
logging.getLogger().setLevel(logging.DEBUG)
conf = get_conf()
pid_file = opts.pid_file
if pid_file is None:
pid_file = conf.get(
"WATCHDOG_PID_FILE",
"/var/run/beaker-lab-controller/beaker-watchdog.pid")
# HubProxy will try to log some stuff, even though we
# haven't configured our logging handlers yet. So we send logs to stderr
# temporarily here, and configure it again below.
log_to_stream(sys.stderr, level=logging.WARNING)
try:
watchdog = Watchdog(conf=conf)
except Exception, ex:
sys.stderr.write("Error starting beaker-watchdog: %s\n" % ex)
sys.exit(1)
def test_quiescent_period_only_applies_between_power_commands(self):
# The purpose of the quiescent period is for power supplies with
# peculiar characteristics that need time to discharge or similar.
# But the quiescent period should not count any other commands like
# clear_logs or configure_netboot, because those are not touching the
# power supply.
quiescent_period = get_conf().get('SLEEP_TIME') * 2.0
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_quiescent_period = quiescent_period
system.power.power_id = u'' # make power script not sleep
system.action_power(action=u'off', service=u'testdata')
system.enqueue_command(action=u'clear_netboot', service=u'testdata')
commands = system.command_queue[:2]
assert_command_is_delayed(commands[1], quiescent_period - 0.5, timeout=quiescent_period / 2)
wait_for_command_to_finish(commands[0], timeout=quiescent_period / 2)
time.sleep(quiescent_period)
# Now there should be no delays because the quiescent period has
# already elapsed since the 'off' command above.
with session.begin():
system.enqueue_command(action=u'clear_logs', service=u'testdata')
system.action_power(action=u'on', service=u'testdata')
commands = system.command_queue[:2]
wait_for_command_to_finish(commands[1], timeout=quiescent_period / 2)
wait_for_command_to_finish(commands[0], timeout=quiescent_period / 2)
def main():
parser = OptionParser()
parser.add_option("-c", "--config",
help="Full path to config file to use")
parser.add_option("-f", "--foreground", default=False, action="store_true",
help="run in foreground (do not spawn a daemon)")
parser.add_option("-p", "--pid-file",
help="specify a pid file")
(opts, args) = parser.parse_args()
if opts.config:
load_conf(opts.config)
conf = get_conf()
pid_file = opts.pid_file
if pid_file is None:
pid_file = conf.get("WPID_FILE", "/var/run/beaker-lab-controller/beaker-transfer.pid")
if not conf.get('ARCHIVE_SERVER'):
sys.stderr.write('Archive server settings are missing from config file\n')
sys.exit(1)
try:
transfer = Watchdog(conf=conf)
except Exception, ex:
sys.stderr.write("Error initializing Watchdog: %s\n" % ex)
sys.exit(1)
def test_timeout_is_enforced_for_fetching_images(self):
with session.begin():
lc = self.get_lc()
system = data_setup.create_system(arch=u'x86_64', lab_controller=lc)
distro_tree = data_setup.create_distro_tree(arch=u'x86_64',
lab_controllers=[lc],
# /slow/600 means the response will be delayed 10 minutes
urls=['http://localhost:19998/slow/600/'])
installation = Installation(distro_tree=distro_tree, system=system,
kernel_options=u'')
system.configure_netboot(installation=installation, service=u'testdata')
wait_for_commands_to_finish(system, timeout=(2 * get_conf().get('SLEEP_TIME')
+ get_conf().get('IMAGE_FETCH_TIMEOUT')))
self.assertEquals(system.command_queue[0].action, u'configure_netboot')
self.assertEquals(system.command_queue[0].status, CommandStatus.failed)
self.assertIn(u'timed out', system.command_queue[0].new_value)
def test_unrelated_Oops_string_is_not_detected_as_panic():
# Sounds implausible, but this really happened...
line = "2013-11-19 05:47:48,109 backend __init__: INFO RPMTest some-test-rpm-name - /mnt/testarea/tmpnOopsn.sh ['some-test-rpm-name'] \n"
detector = PanicDetector(get_conf().get('PANIC_REGEX'))
failure_found = detector.feed(line)
if failure_found:
raise AssertionError('False panic detection: %s' % failure_found)
def configure_zpxe(fqdn, kernel_url, initrd_url, kernel_options, basedir):
"""
Creates bootloader files for ZPXE
<get_tftp_root()>/s390x/s_<fqdn>
<get_tftp_root()>/s390x/s_<fqdn>_parm
<get_tftp_root()>/s390x/s_<fqdn>_conf
"""
zpxe_dir = os.path.join(basedir, 's390x')
makedirs_ignore(zpxe_dir, mode=0755)
kernel_options = "%s netboot_method=zpxe" % kernel_options
# The structure of these files is dictated by zpxe.rexx,
# Cobbler's "pseudo-PXE" for zVM on s390(x).
# XXX I don't think multiple initrds are supported?
logger.debug('Writing zpxe index file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir, 's_%s' % fqdn)) as f:
if get_conf().get('ZPXE_USE_FTP', True):
if not kernel_url.startswith('ftp://') or not initrd_url.startswith('ftp://'):
raise ValueError('zPXE only supports FTP for downloading images')
f.write('%s\n%s\n\n' % (kernel_url, initrd_url))
else:
f.write('/images/%s/kernel\n/images/%s/initrd\n\n' % (fqdn, fqdn))
logger.debug('Writing zpxe parm file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir, 's_%s_parm' % fqdn)) as f:
# must be wrapped at 80 columns
rest = kernel_options
while rest:
f.write(rest[:80] + '\n')
rest = rest[80:]
logger.debug('Writing zpxe conf file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir, 's_%s_conf' % fqdn)) as f:
pass # unused, but zpxe.rexx fetches it anyway
def test_unrelated_Oops_string_is_not_detected_as_panic():
# Sounds implausible, but this really happened...
line = "2013-11-19 05:47:48,109 backend __init__: INFO RPMTest some-test-rpm-name - /mnt/testarea/tmpnOopsn.sh ['some-test-rpm-name'] \n"
detector = PanicDetector(get_conf().get('PANIC_REGEX'))
failure_found = detector.feed(line)
if failure_found:
raise AssertionError('False panic detection: %s' % failure_found)
def test_quiescent_period_only_applies_between_power_commands(self):
# The purpose of the quiescent period is for power supplies with
# peculiar characteristics that need time to discharge or similar.
# But the quiescent period should not count any other commands like
# clear_logs or configure_netboot, because those are not touching the
# power supply.
loop_interval = get_conf().get('SLEEP_TIME')
quiescent_period = loop_interval * 3.0
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
self.addCleanup(self.cleanup_system, system)
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_quiescent_period = quiescent_period
system.power.power_id = u'' # make power script not sleep
system.action_power(action=u'off', service=u'testdata')
system.enqueue_command(action=u'clear_netboot',
service=u'testdata')
commands = system.command_queue[:2]
assert_command_is_delayed(commands[1],
quiescent_period - 0.5,
timeout=2 * loop_interval)
wait_for_command_to_finish(commands[0], timeout=2 * loop_interval)
time.sleep(quiescent_period)
# Now there should be no delays because the quiescent period has
# already elapsed since the 'off' command above.
with session.begin():
system.enqueue_command(action=u'clear_logs', service=u'testdata')
system.action_power(action=u'on', service=u'testdata')
commands = system.command_queue[:2]
wait_for_command_to_finish(commands[1], timeout=2 * loop_interval)
wait_for_command_to_finish(commands[0], timeout=2 * loop_interval)
def test_PUT_result_log(self):
with session.begin():
task = self.recipe.tasks[0]
task.pass_(u'', 0, u'Pass')
result = self.recipe.tasks[0].results[0]
upload_url = '%srecipes/%s/tasks/%s/results/%s/logs/PUT-result-log' % (
self.get_proxy_url(), self.recipe.id, task.id, result.id)
response = requests.put(upload_url, data='a' * 10)
self.assertEquals(response.status_code, 204)
local_log_dir = '%s/results/%s+/%s/' % (get_conf().get('CACHEPATH'),
result.id // 1000, result.id)
with session.begin():
self.assertEquals(result.logs[0].path, '/')
self.assertEquals(result.logs[0].filename, 'PUT-result-log')
self.assertEquals(result.logs[0].server,
'%s/beaker/logs/results/%s+/%s/'
% (self.get_log_base_url(), result.id // 1000, result.id))
self.assertEquals(result.logs[0].basepath, local_log_dir)
self.assertEquals(
open(os.path.join(local_log_dir, 'PUT-result-log'), 'r').read(),
'aaaaaaaaaa')
response = requests.put(upload_url, data='b' * 10,
headers={'Content-Range': 'bytes 10-19/20'})
self.assertEquals(response.status_code, 204)
with session.begin():
self.assertEquals(
open(os.path.join(local_log_dir, 'PUT-result-log'), 'r').read(),
'aaaaaaaaaabbbbbbbbbb')
response = requests.get(upload_url)
response.raise_for_status()
self.assertEquals(response.content, 'aaaaaaaaaabbbbbbbbbb')
def test_xmlrpc_result_log(self):
with session.begin():
self.recipe.tasks[0].pass_(u'', 0, u'Pass')
result = self.recipe.tasks[0].results[0]
s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True)
s.result_upload_file(result.id, '/', 'result-log', 10, None, 0,
b64encode('a' * 10))
local_log_dir = '%s/results/%s+/%s/' % (get_conf().get('CACHEPATH'),
result.id // 1000, result.id)
with session.begin():
self.assertEquals(result.logs[0].path, '/')
self.assertEquals(result.logs[0].filename, 'result-log')
self.assertEquals(result.logs[0].server,
'%s/beaker/logs/results/%s+/%s/'
% (self.get_log_base_url(), result.id // 1000, result.id))
self.assertEquals(result.logs[0].basepath, local_log_dir)
self.assertEquals(
open(os.path.join(local_log_dir, 'result-log'), 'r').read(),
'aaaaaaaaaa')
s.result_upload_file(result.id, '/', 'result-log', 10, None, 10,
b64encode('b' * 10))
with session.begin():
self.assertEquals(
open(os.path.join(local_log_dir, 'result-log'), 'r').read(),
'aaaaaaaaaabbbbbbbbbb')
def main():
parser = OptionParser()
parser.add_option("-c", "--config",
help="Full path to config file to use")
parser.add_option("-f", "--foreground", default=False, action="store_true",
help="run in foreground (do not spawn a daemon)")
parser.add_option("-p", "--pid-file",
help="specify a pid file")
(opts, args) = parser.parse_args()
if opts.config:
load_conf(opts.config)
conf = get_conf()
pid_file = opts.pid_file
if pid_file is None:
pid_file = conf.get("WATCHDOG_PID_FILE", "/var/run/beaker-lab-controller/beaker-watchdog.pid")
if opts.foreground:
main_loop(conf=conf, foreground=True)
else:
with daemon.DaemonContext(pidfile=pidfile.TimeoutPIDLockFile(
pid_file, acquire_timeout=0)):
main_loop(conf=conf, foreground=False)
print 'exiting program'
def main():
parser = OptionParser()
parser.add_option("-c", "--config",
help="Full path to config file to use")
parser.add_option("-f", "--foreground", default=False, action="store_true",
help="run in foreground (do not spawn a daemon)")
parser.add_option("-p", "--pid-file",
help="specify a pid file")
(opts, args) = parser.parse_args()
if opts.config:
load_conf(opts.config)
conf = get_conf()
logging.getLogger().setLevel(logging.DEBUG)
pid_file = opts.pid_file
if pid_file is None:
pid_file = conf.get("PROXY_PID_FILE", "/var/run/beaker-lab-controller/beaker-proxy.pid")
# HubProxy will try to log some stuff, even though we
# haven't configured our logging handlers yet. So we send logs to stderr
# temporarily here, and configure it again below.
log_to_stream(sys.stderr, level=logging.WARNING)
try:
proxy = Proxy(conf=conf)
except Exception, ex:
sys.stderr.write("Error initializing Proxy: %s\n" % ex)
sys.exit(1)
def __init__(self, conf=None, hub=None, **kwargs):
self.conf = get_conf()
# update data from another config
if conf is not None:
self.conf.load_from_conf(conf)
# update data from config specified in os.environ
conf_environ_key = "BEAKER_PROXY_CONFIG_FILE"
if conf_environ_key in os.environ:
self.conf.load_from_file(os.environ[conf_environ_key])
self.conf.load_from_dict(kwargs)
# self.hub is created here
self.hub = hub
if self.hub is None:
self.hub = HubProxy(
logger=logging.getLogger('bkr.common.hub.HubProxy'),
conf=self.conf,
**kwargs)
self.log_storage = LogStorage(
self.conf.get("CACHEPATH"), "%s://%s/beaker/logs" %
(self.conf.get('URL_SCHEME', 'http'), self.conf.get_url_domain()),
self.hub)
def test_xmlrpc_result_log(self):
with session.begin():
self.recipe.tasks[0].pass_(u'', 0, u'Pass')
result = self.recipe.tasks[0].results[0]
s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True)
s.result_upload_file(result.id, '/', 'result-log', 10, None, 0,
b64encode('a' * 10))
local_log_dir = '%s/results/%s+/%s/' % (get_conf().get('CACHEPATH'),
result.id // 1000, result.id)
with session.begin():
self.assertEquals(result.logs[0].path, '/')
self.assertEquals(result.logs[0].filename, 'result-log')
self.assertEquals(
result.logs[0].server, '%s/beaker/logs/results/%s+/%s/' %
(self.get_log_base_url(), result.id // 1000, result.id))
self.assertEquals(result.logs[0].basepath, local_log_dir)
self.assertEquals(
open(os.path.join(local_log_dir, 'result-log'), 'r').read(),
'aaaaaaaaaa')
s.result_upload_file(result.id, '/', 'result-log', 10, None, 10,
b64encode('b' * 10))
with session.begin():
self.assertEquals(
open(os.path.join(local_log_dir, 'result-log'), 'r').read(),
'aaaaaaaaaabbbbbbbbbb')
def test_PUT_result_log(self):
with session.begin():
task = self.recipe.tasks[0]
task.pass_(u'', 0, u'Pass')
result = self.recipe.tasks[0].results[0]
upload_url = '%srecipes/%s/tasks/%s/results/%s/logs/PUT-result-log' % (
self.get_proxy_url(), self.recipe.id, task.id, result.id)
response = requests.put(upload_url, data='a' * 10)
self.assertEquals(response.status_code, 204)
local_log_dir = '%s/results/%s+/%s/' % (get_conf().get('CACHEPATH'),
result.id // 1000, result.id)
with session.begin():
self.assertEquals(result.logs[0].path, '/')
self.assertEquals(result.logs[0].filename, 'PUT-result-log')
self.assertEquals(
result.logs[0].server, '%s/beaker/logs/results/%s+/%s/' %
(self.get_log_base_url(), result.id // 1000, result.id))
self.assertEquals(result.logs[0].basepath, local_log_dir)
self.assertEquals(
open(os.path.join(local_log_dir, 'PUT-result-log'),
'r').read(), 'aaaaaaaaaa')
response = requests.put(upload_url,
data='b' * 10,
headers={'Content-Range': 'bytes 10-19/20'})
self.assertEquals(response.status_code, 204)
with session.begin():
self.assertEquals(
open(os.path.join(local_log_dir, 'PUT-result-log'),
'r').read(), 'aaaaaaaaaabbbbbbbbbb')
response = requests.get(upload_url)
response.raise_for_status()
self.assertEquals(response.content, 'aaaaaaaaaabbbbbbbbbb')
def test_power_passwords_are_not_reported_in_failure_message(self):
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
self.addCleanup(self.cleanup_system, system)
system.power.power_type = PowerType.lazy_create(
name=u'testing-bz1358063')
system.power.power_passwd = u'dontleakmebro'
system.power.quiescent_period = 0
system.action_power(action=u'off', service=u'testdata')
timeout = (2 * get_conf().get('SLEEP_TIME') +
get_conf().get('POWER_ATTEMPTS') *
2**get_conf().get('POWER_ATTEMPTS'))
wait_for_commands_to_finish(system, timeout=timeout)
self.assertEqual(system.command_queue[0].status, CommandStatus.failed)
self.assertIn(
u'failed after 2 attempts with exit status 1:\npassword is ********',
system.command_queue[0].error_message)
def add_rotating_file_logger(*args, **kw):
conf = get_conf()
max_bytes = conf.get('LOG_MAXBYTES')
backup_count = conf.get('LOG_BACKUPCOUNT')
file_logger_kw = kw
if backup_count:
file_logger_kw.update({'backupCount' : backup_count})
if max_bytes:
file_logger_kw.update({'maxBytes' : max_bytes})
return arfl(*args, **file_logger_kw)
def test_stops_collecting_console_log_when_recipe_aborted(self):
first_line = 'Here is the first line of the log file.\n'
open(self.console_log, 'w').write(first_line)
wait_for_condition(self.check_console_log_registered)
wait_for_condition(lambda: self.check_cached_log_contents(first_line))
# Abort the recipe
with session.begin():
data_setup.mark_recipe_complete(self.recipe, only=True)
# Wait for the watchdog to determine recipe aborted
# Give beaker-watchdog a chance to notice
time.sleep(get_conf().get('SLEEP_TIME') * 2)
second_line = 'Here is the second line of the log file. FNORD FNORD FNORD\n'
open(self.console_log, 'a').write(second_line)
time.sleep(get_conf().get('SLEEP_TIME') * 2)
self.assertTrue(self.check_cached_log_contents(first_line),
"Log should just contain first log line")
def test_PUT_empty_log(self):
upload_url = '%srecipes/%s/logs/empty-log' % (self.get_proxy_url(),
self.recipe.id)
local_log_dir = '%s/recipes/%s+/%s/' % (get_conf().get('CACHEPATH'),
self.recipe.id // 1000, self.recipe.id)
response = requests.put(upload_url, data='')
self.assertEquals(response.status_code, 204)
self.assertEquals(
open(os.path.join(local_log_dir, 'empty-log'), 'r').read(),
'')
def test_PUT_empty_log(self):
upload_url = '%srecipes/%s/logs/empty-log' % (self.get_proxy_url(),
self.recipe.id)
local_log_dir = '%s/recipes/%s+/%s/' % (get_conf().get('CACHEPATH'),
self.recipe.id // 1000,
self.recipe.id)
response = requests.put(upload_url, data='')
self.assertEquals(response.status_code, 204)
self.assertEquals(
open(os.path.join(local_log_dir, 'empty-log'), 'r').read(), '')
def test_stops_collecting_console_log_when_recipe_aborted(self):
first_line = 'Here is the first line of the log file.\n'
open(self.console_log, 'w').write(first_line)
wait_for_condition(self.check_console_log_registered)
wait_for_condition(lambda: self.check_cached_log_contents(first_line))
# Abort the recipe
with session.begin():
data_setup.mark_recipe_complete(self.recipe, only=True)
# Wait for the watchdog to determine recipe aborted
# Give beaker-watchdog a chance to notice
time.sleep(get_conf().get('SLEEP_TIME') * 2)
second_line = 'Here is the second line of the log file. FNORD FNORD FNORD\n'
open(self.console_log, 'a').write(second_line)
time.sleep(get_conf().get('SLEEP_TIME') * 2)
self.assertTrue(self.check_cached_log_contents(first_line),
"Log should just contain first log line")
def setUp(self):
with session.begin():
self.watchdog = Watchdog()
self.recipe = data_setup.create_recipe()
data_setup.create_job_for_recipes([self.recipe])
data_setup.mark_recipe_running(self.recipe, virt=True, lab_controller=self.get_lc())
self.cached_console_log = os.path.join(get_conf().get('CACHEPATH'),
'recipes',
str(self.recipe.id // 1000) + '+',
str(self.recipe.id), 'console.log')
def setUp(self):
with session.begin():
self.watchdog = Watchdog()
self.recipe = data_setup.create_recipe()
job = data_setup.create_job_for_recipes([self.recipe])
self.addCleanup(self.cleanup_job, job)
data_setup.mark_recipe_running(self.recipe, virt=True, lab_controller=self.get_lc())
self.cached_console_log = os.path.join(get_conf().get('CACHEPATH'),
'recipes',
str(self.recipe.id // 1000) + '+',
str(self.recipe.id), 'console.log')
def setUp(self):
with session.begin():
self.recipe = data_setup.create_recipe()
job = data_setup.create_job_for_recipes([self.recipe])
self.addCleanup(self.cleanup_job, job)
data_setup.mark_recipe_running(self.recipe, virt=True, lab_controller=self.get_lc())
self.cached_console_log = os.path.join(get_conf().get('CACHEPATH'),
'recipes',
str(self.recipe.id // 1000) + '+',
str(self.recipe.id), 'console.log')
self.watchdog = Watchdog()
self.monitor = Monitor({'recipe_id': self.recipe.id, 'is_virt_recipe': True}, self.watchdog)
def setUp(self):
self.conf = get_conf()
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.propagate = False
self.logger.setLevel(logging.DEBUG)
prefix = unique_name(u"watchdogtest%s")
self.log_file = tempfile.NamedTemporaryFile(prefix=prefix)
add_rotating_file_logger(
self.logger,
self.log_file.name,
maxBytes=_log_maxbytes,
backupCount=_log_backupcount,
log_level=logging.DEBUG,
)
def write_menus(tftp_root, tags, xml_filter):
conf = get_conf()
# The order of steps for cleaning images is important,
# to avoid races and to avoid deleting stuff we shouldn't:
# first read the directory,
# then fetch the list of trees,
# and then remove any which aren't in the list.
try:
existing_tree_ids = os.listdir(os.path.join(tftp_root, 'distrotrees'))
except OSError, e:
if e.errno != errno.ENOENT:
raise
existing_tree_ids = []
def setUp(self):
with session.begin():
self.system = data_setup.create_system(
lab_controller=self.get_lc())
self.recipe = data_setup.create_recipe()
self.guest_recipe = data_setup.create_guestrecipe(self.recipe)
job = data_setup.create_job_for_recipes(
[self.recipe, self.guest_recipe])
self.addCleanup(self.cleanup_job, job)
data_setup.mark_recipe_running(self.recipe, system=self.system)
data_setup.mark_recipe_installing(self.guest_recipe,
system=self.system)
self.console_log = os.path.join(get_conf().get('CONSOLE_LOGS'),
self.system.fqdn)
self.cached_console_log = os.path.join(
get_conf().get('CACHEPATH'), 'recipes',
str(self.recipe.id // 1000) + '+', str(self.recipe.id),
'console.log')
self.first_line = 'Here is the first line of the log file.\n'
open(self.console_log, 'w').write(self.first_line)
self.watchdog = Watchdog()
self.monitor = Monitor(
{
'recipe_id': self.recipe.id,
'is_virt_recipe': False,
'system': self.system.fqdn
}, self.watchdog)
self.monitor_guest = Monitor(
{
'recipe_id': self.guest_recipe.id,
'is_virt_recipe': False,
'system': None
}, self.watchdog)
def setup_package():
global lc_fqdn, _daemons_running_externally
conf = get_conf()
if not 'BEAKER_LABCONTROLLER_HOSTNAME' in os.environ:
# Need to start the lab controller daemons ourselves
with session.begin():
user = data_setup.create_user(user_name=conf.get('USERNAME').decode('utf8'), password=conf.get('PASSWORD'))
lc = data_setup.create_labcontroller(fqdn=u'localhost', user=user)
processes.extend([
Process('beaker-proxy',
args=['python', '../LabController/src/bkr/labcontroller/main.py',
'-c', config_file, '-f'],
listen_port=8000,
stop_signal=signal.SIGTERM),
Process('beaker-provision',
args=['python', '../LabController/src/bkr/labcontroller/provision.py',
'-c', config_file, '-f'],
stop_signal=signal.SIGTERM),
Process('beaker-watchdog',
args=['python', '../LabController/src/bkr/labcontroller/watchdog.py',
'-c', config_file, '-f'],
stop_signal=signal.SIGTERM),
])
lc_fqdn = u'localhost'
else:
_daemons_running_externally = True
# We have been passed a space seperated list of LCs
lab_controllers = os.environ.get('BEAKER_LABCONTROLLER_HOSTNAME').decode('utf8')
lab_controllers_list = lab_controllers.split()
# Just get the last one, it shouldn't matter to us
lab_controller = lab_controllers_list.pop()
# Make sure that the LC is in the DB
data_setup.create_labcontroller(fqdn=lab_controller)
lc_fqdn = lab_controller
# Clear out any existing job logs, so that they are registered correctly
# when first created.
# If we've been passed a remote hostname for the LC, we assume it's been
# freshly provisioned and the dir will already be empty.
shutil.rmtree(conf.get('CACHEPATH'), ignore_errors=True)
try:
for process in processes:
process.start()
except:
for process in processes:
process.stop()
raise
def setup_package():
global lc_fqdn, _daemons_running_externally
conf = get_conf()
if not 'BEAKER_LABCONTROLLER_HOSTNAME' in os.environ:
# Need to start the lab controller daemons ourselves
with session.begin():
user = data_setup.create_user(user_name=conf.get('USERNAME').decode('utf8'), password=conf.get('PASSWORD'))
lc = data_setup.create_labcontroller(fqdn=u'localhost', user=user)
processes.extend([
Process('beaker-proxy',
args=['python', '../LabController/src/bkr/labcontroller/main.py',
'-c', config_file, '-f'],
listen_port=8000,
stop_signal=signal.SIGTERM),
Process('beaker-provision',
args=['python', '../LabController/src/bkr/labcontroller/provision.py',
'-c', config_file, '-f'],
stop_signal=signal.SIGTERM),
Process('beaker-watchdog',
args=['python', '../LabController/src/bkr/labcontroller/watchdog.py',
'-c', config_file, '-f'],
stop_signal=signal.SIGTERM),
])
lc_fqdn = u'localhost'
else:
_daemons_running_externally = True
# We have been passed a space seperated list of LCs
lab_controllers = os.environ.get('BEAKER_LABCONTROLLER_HOSTNAME').decode('utf8')
lab_controllers_list = lab_controllers.split()
# Just get the last one, it shouldn't matter to us
lab_controller = lab_controllers_list.pop()
# Make sure that the LC is in the DB
data_setup.create_labcontroller(fqdn=lab_controller)
lc_fqdn = lab_controller
# Clear out any existing job logs, so that they are registered correctly
# when first created.
# If we've been passed a remote hostname for the LC, we assume it's been
# freshly provisioned and the dir will already be empty.
shutil.rmtree(conf.get('CACHEPATH'), ignore_errors=True)
try:
for process in processes:
process.start()
except:
for process in processes:
process.stop()
raise
def test_power_quiescent_period(self):
# Test that we do in fact wait for the quiescent period to pass
# before running a command.
# This time is needed to guarantee that we are actually waiting for
# the quiescent period and not waiting for another poll loop:
quiescent_period = get_conf().get('SLEEP_TIME') * 3
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_quiescent_period = quiescent_period
system.power.power_id = u'' # make power script not sleep
system.power.delay_until = None
system.action_power(action=u'off', service=u'testdata')
command = system.command_queue[0]
assert_command_is_delayed(command, quiescent_period - 0.5, 10)
def test_system_not_marked_broken_for_missing_distro_tree_images(self):
with session.begin():
lc = self.get_lc()
system = data_setup.create_system(arch=u'x86_64', lab_controller=lc,
status=SystemStatus.automated)
distro_tree = data_setup.create_distro_tree(arch=u'x86_64',
lab_controllers=[lc],
urls=['http://localhost:19998/error/404/'])
installation = Installation(distro_tree=distro_tree, system=system,
kernel_options=u'')
system.configure_netboot(installation=installation, service=u'testdata')
wait_for_commands_to_finish(system, timeout=(2 * get_conf().get('SLEEP_TIME')))
self.assertEquals(system.command_queue[0].action, u'configure_netboot')
self.assertEquals(system.command_queue[0].status, CommandStatus.failed)
self.assertEquals(system.status, SystemStatus.automated)
def test_power_quiescent_period(self):
# Test that we do in fact wait for the quiescent period to pass
# before running a command.
# This time is needed to guarantee that we are actually waiting for
# the quiescent period and not waiting for another poll loop:
quiescent_period = get_conf().get('SLEEP_TIME') * 3
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_quiescent_period = quiescent_period
system.power.power_id = u'' # make power script not sleep
system.power.delay_until = None
system.action_power(action=u'off', service=u'testdata')
command = system.command_queue[0]
assert_command_is_delayed(command, quiescent_period - 0.5, 10)
def test_power_passwords_are_not_logged(self):
if daemons_running_externally():
raise SkipTest('cannot examine logs of remote beaker-provision')
provision_process, = [p for p in processes if p.name == 'beaker-provision']
try:
provision_process.start_output_capture()
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = u'' # make power script not sleep
system.power.power_passwd = u'dontleakmebro'
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system, timeout=2 * get_conf().get('SLEEP_TIME'))
finally:
provision_output = provision_process.finish_output_capture()
self.assert_('Handling command' in provision_output, provision_output)
self.assert_('Launching power script' in provision_output, provision_output)
self.assert_(system.power.power_passwd not in provision_output, provision_output)
def test_blank_power_passwords(self):
if daemons_running_externally():
raise SkipTest('cannot examine logs of remote beaker-provision')
provision_process, = [p for p in processes if p.name == 'beaker-provision']
try:
provision_process.start_output_capture()
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.address = None
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = u'' # make power script not sleep
system.power.power_passwd = None
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system, timeout=2 * get_conf().get('SLEEP_TIME'))
finally:
provision_output = provision_process.finish_output_capture()
# The None type is passed in from the db. Later in the code it is converted
# to the empty string, as it should be.
self.assertIn("'passwd': None", provision_output, provision_output)
def test_blank_power_passwords(self):
if daemons_running_externally():
raise SkipTest('cannot examine logs of remote beaker-provision')
provision_process, = [p for p in processes if p.name == 'beaker-provision']
try:
provision_process.start_output_capture()
with session.begin():
system = data_setup.create_system(lab_controller=self.get_lc())
system.power.address = None
system.power.power_type = PowerType.lazy_create(name=u'dummy')
system.power.power_id = u'' # make power script not sleep
system.power.power_passwd = None
system.action_power(action=u'off', service=u'testdata')
wait_for_commands_to_finish(system, timeout=2 * get_conf().get('SLEEP_TIME'))
finally:
provision_output = provision_process.finish_output_capture()
# The None type is passed in from the db. Later in the code it is converted
# to the empty string, as it should be.
self.assertIn("'passwd': None", provision_output, provision_output)
def test_console_log_not_recreated_after_removed(self):
# The scenario for this bug is:
# 1. beaker-watchdog writes the console log
# 2. recipe finishes (but beaker-watchdog hasn't noticed yet)
# 3. beaker-transfer tranfers the logs and removes the local copies
# 4. beaker-watchdog writes more to the end of the console log (in the
# process re-registering the log file, and leaving the start of the
# file filled with zeroes)
# 5. beaker-transfer tranfers the logs again
# This test checks that step 4 is prevented -- the console log updates
# are silently discarded instead.
# Step 1: beaker-watchdog writes the console log
existing_data = 'Existing data\n'
open(self.console_log, 'w').write(existing_data)
wait_for_condition(self.check_console_log_registered)
wait_for_condition(
lambda: self.check_cached_log_contents(existing_data))
# Step 2: the recipe "finishes"
# Don't actually mark it as finished in the database though, to ensure
# the watchdog keeps monitoring the console log.
# Step 3: beaker-transfer tranfers the logs and removes the local copies
with session.begin():
LogRecipe.query.filter_by(
parent=self.recipe,
filename=u'console.log').one().server = u'http://elsewhere'
os.remove(self.cached_console_log)
# Step 4: beaker-watchdog tries to write more to the end of the console log
open(self.console_log,
'a').write('More console output, after the recipe has finished\n')
# Give beaker-watchdog a chance to notice
time.sleep(get_conf().get('SLEEP_TIME') * 2)
self.assert_(not os.path.exists(self.cached_console_log))
with session.begin():
self.assertEquals(
LogRecipe.query.filter_by(
parent=self.recipe, filename=u'console.log').one().server,
u'http://elsewhere')
def fetch_images(distro_tree_id, kernel_url, initrd_url, fqdn):
"""
Creates references to kernel and initrd files at:
<get_tftp_root()>/images/<fqdn>/kernel
<get_tftp_root()>/images/<fqdn>/initrd
"""
images_dir = os.path.join(get_tftp_root(), 'images', fqdn)
makedirs_ignore(images_dir, 0o755)
# Only look for fetched images if distro_tree is registered
if distro_tree_id is not None:
distrotree_dir = os.path.join(get_tftp_root(), 'distrotrees',
str(distro_tree_id))
# beaker-pxemenu might have already fetched the images, so let's try there
# before anywhere else.
try:
atomic_link(os.path.join(distrotree_dir, 'kernel'),
os.path.join(images_dir, 'kernel'))
atomic_link(os.path.join(distrotree_dir, 'initrd'),
os.path.join(images_dir, 'initrd'))
logger.debug('Using images from distro tree %s for %s',
distro_tree_id, fqdn)
return
except OSError as e:
if e.errno != errno.ENOENT:
raise
# No luck there, so try something else...
timeout = get_conf().get('IMAGE_FETCH_TIMEOUT')
logger.debug('Fetching kernel %s for %s', kernel_url, fqdn)
with atomically_replaced_file(os.path.join(images_dir, 'kernel')) as dest:
try:
siphon(urllib2.urlopen(kernel_url, timeout=timeout), dest)
except Exception as e:
raise ImageFetchingError(kernel_url, distro_tree_id, e)
logger.debug('Fetching initrd %s for %s', initrd_url, fqdn)
with atomically_replaced_file(os.path.join(images_dir, 'initrd')) as dest:
try:
siphon(urllib2.urlopen(initrd_url, timeout=timeout), dest)
except Exception as e:
raise ImageFetchingError(initrd_url, distro_tree_id, e)
def test_system_not_marked_broken_for_missing_distro_tree_images(self):
with session.begin():
lc = self.get_lc()
system = data_setup.create_system(arch=u'x86_64',
lab_controller=lc,
status=SystemStatus.automated)
self.addCleanup(self.cleanup_system, system)
distro_tree = data_setup.create_distro_tree(
arch=u'x86_64',
lab_controllers=[lc],
urls=['http://localhost:19998/error/404/'])
installation = Installation(distro_tree=distro_tree,
system=system,
kernel_options=u'')
system.configure_netboot(installation=installation,
service=u'testdata')
wait_for_commands_to_finish(system,
timeout=(2 * get_conf().get('SLEEP_TIME')))
self.assertEquals(system.command_queue[0].action, u'configure_netboot')
self.assertEquals(system.command_queue[0].status, CommandStatus.failed)
self.assertEquals(system.status, SystemStatus.automated)
def test_console_log_not_recreated_after_removed(self):
# The scenario for this bug is:
# 1. beaker-watchdog writes the console log
# 2. recipe finishes (but beaker-watchdog hasn't noticed yet)
# 3. beaker-transfer tranfers the logs and removes the local copies
# 4. beaker-watchdog writes more to the end of the console log (in the
# process re-registering the log file, and leaving the start of the
# file filled with zeroes)
# 5. beaker-transfer tranfers the logs again
# This test checks that step 4 is prevented -- the console log updates
# are silently discarded instead.
# Step 1: beaker-watchdog writes the console log
existing_data = 'Existing data\n'
open(self.console_log, 'w').write(existing_data)
wait_for_condition(self.check_console_log_registered)
wait_for_condition(lambda: self.check_cached_log_contents(existing_data))
# Step 2: the recipe "finishes"
# Don't actually mark it as finished in the database though, to ensure
# the watchdog keeps monitoring the console log.
# Step 3: beaker-transfer tranfers the logs and removes the local copies
with session.begin():
LogRecipe.query.filter_by(parent=self.recipe,
filename=u'console.log').one().server = u'http://elsewhere'
os.remove(self.cached_console_log)
# Step 4: beaker-watchdog tries to write more to the end of the console log
open(self.console_log, 'a').write(
'More console output, after the recipe has finished\n')
# Give beaker-watchdog a chance to notice
time.sleep(get_conf().get('SLEEP_TIME') * 2)
self.assert_(not os.path.exists(self.cached_console_log))
with session.begin():
self.assertEquals(LogRecipe.query.filter_by(parent=self.recipe,
filename=u'console.log').one().server,
u'http://elsewhere')
def main():
parser = OptionParser()
parser.add_option("-c", "--config",
help="Full path to config file to use")
parser.add_option("-f", "--foreground", default=False, action="store_true",
help="run in foreground (do not spawn a daemon)")
parser.add_option("-p", "--pid-file",
help="specify a pid file")
(opts, args) = parser.parse_args()
if opts.config:
load_conf(opts.config)
conf = get_conf()
pid_file = opts.pid_file
if pid_file is None:
pid_file = conf.get("PROVISION_PID_FILE", "/var/run/beaker-lab-controller/beaker-provision.pid")
try:
poller = CommandQueuePoller(conf=conf)
except Exception, ex:
sys.stderr.write('Error initializing CommandQueuePoller: %s\n' % ex)
sys.exit(1)
def __init__(self, conf=None, hub=None, **kwargs):
self.conf = get_conf()
# update data from another config
if conf is not None:
self.conf.load_from_conf(conf)
# update data from config specified in os.environ
conf_environ_key = "BEAKER_PROXY_CONFIG_FILE"
if conf_environ_key in os.environ:
self.conf.load_from_file(os.environ[conf_environ_key])
self.conf.load_from_dict(kwargs)
# self.hub is created here
self.hub = hub
if self.hub is None:
self.hub = HubProxy(logger=logging.getLogger('bkr.common.hub.HubProxy'), conf=self.conf,
**kwargs)
self.log_storage = LogStorage(self.conf.get("CACHEPATH"),
"%s://%s/beaker/logs" % (self.conf.get('URL_SCHEME',
'http'), self.conf.get_url_domain()),
self.hub)
def configure_zpxe(fqdn, kernel_url, initrd_url, kernel_options, basedir):
"""
Creates bootloader files for ZPXE
<get_tftp_root()>/s390x/s_<fqdn>
<get_tftp_root()>/s390x/s_<fqdn>_parm
<get_tftp_root()>/s390x/s_<fqdn>_conf
"""
zpxe_dir = os.path.join(basedir, 's390x')
makedirs_ignore(zpxe_dir, mode=0o755)
kernel_options = "%s netboot_method=zpxe" % kernel_options
# The structure of these files is dictated by zpxe.rexx,
# Cobbler's "pseudo-PXE" for zVM on s390(x).
# XXX I don't think multiple initrds are supported?
logger.debug('Writing zpxe index file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir, 's_%s' % fqdn)) as f:
if get_conf().get('ZPXE_USE_FTP', True):
if not kernel_url.startswith(
'ftp://') or not initrd_url.startswith('ftp://'):
raise ValueError(
'zPXE only supports FTP for downloading images')
f.write('%s\n%s\n\n' % (kernel_url, initrd_url))
else:
f.write('/images/%s/kernel\n/images/%s/initrd\n\n' % (fqdn, fqdn))
logger.debug('Writing zpxe parm file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir,
's_%s_parm' % fqdn)) as f:
# must be wrapped at 80 columns
rest = kernel_options
while rest:
f.write(rest[:80] + '\n')
rest = rest[80:]
logger.debug('Writing zpxe conf file for %s', fqdn)
with atomically_replaced_file(os.path.join(zpxe_dir,
's_%s_conf' % fqdn)) as f:
pass # unused, but zpxe.rexx fetches it anyway
def write_menus(tftp_root, tags, xml_filter):
conf = get_conf()
# The order of steps for cleaning images is important,
# to avoid races and to avoid deleting stuff we shouldn't:
# first read the directory,
# then fetch the list of trees,
# and then remove any which aren't in the list.
try:
existing_tree_ids = os.listdir(os.path.join(tftp_root, 'distrotrees'))
except OSError as e:
if e.errno != errno.ENOENT:
raise
existing_tree_ids = []
proxy = xmlrpclib.ServerProxy('http://localhost:8000', allow_none=True)
distro_trees = proxy.get_distro_trees({
'arch': ['x86_64', 'i386', 'aarch64', 'ppc64', 'ppc64le'],
'tags': tags,
'xml': xml_filter,
})
current_tree_ids = set(str(dt['distro_tree_id'])
for dt in distro_trees)
obsolete_tree_ids = set(existing_tree_ids).difference(current_tree_ids)
print('Removing images for %s obsolete distro trees' % len(obsolete_tree_ids))
for obs in obsolete_tree_ids:
shutil.rmtree(os.path.join(tftp_root, 'distrotrees', obs), ignore_errors=True)
# Fetch images for all the distro trees first.
print('Fetching images for all the distro trees')
distro_trees = _get_all_images(tftp_root, distro_trees)
x86_distrotrees = [distro for distro in distro_trees if distro['arch'] in ['x86_64', 'i386']]
print('Generating PXELINUX menus for %s distro trees' % len(x86_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'pxelinux.cfg'), mode=0o755)
pxe_menu = atomically_replaced_file(os.path.join(tftp_root, 'pxelinux.cfg', 'beaker_menu'))
write_menu(pxe_menu, u'pxelinux-menu', x86_distrotrees)
x86_efi_distrotrees = [distro for distro in distro_trees if distro['arch'] == 'x86_64']
# Regardless of any filtering options selected by the admin, we always
# filter out certain distros which are known not to have EFI support. This
# is a space saving measure for the EFI GRUB menu, which can't be nested so
# we try to keep it as small possible.
x86_efi_distrotrees = [distro for distro in x86_efi_distrotrees
if not re.match(conf['EFI_EXCLUDED_OSMAJORS_REGEX'],
distro['distro_osmajor'])]
print('Generating EFI GRUB menus for %s distro trees' % len(x86_efi_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'grub'), mode=0o755)
atomic_symlink('../distrotrees', os.path.join(tftp_root, 'grub', 'distrotrees'))
efi_grub_menu = atomically_replaced_file(os.path.join(tftp_root, 'grub', 'efidefault'))
write_menu(efi_grub_menu, u'efi-grub-menu', x86_efi_distrotrees)
print('Generating GRUB2 menus for x86 EFI for %s distro trees' % len(x86_efi_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'boot', 'grub2'), mode=0o755)
x86_grub2_menu = atomically_replaced_file(os.path.join(tftp_root, 'boot', 'grub2',
'beaker_menu_x86.cfg'))
write_menu(x86_grub2_menu, u'grub2-menu', x86_efi_distrotrees)
ppc64_distrotrees = [distro for distro in distro_trees if distro['arch'] == 'ppc64']
if ppc64_distrotrees:
print('Generating GRUB2 menus for PPC64 EFI for %s distro trees' % len(ppc64_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'boot', 'grub2'), mode=0o755)
ppc64_grub2_menu = atomically_replaced_file(os.path.join(tftp_root, 'boot', 'grub2',
'beaker_menu_ppc64.cfg'))
write_menu(ppc64_grub2_menu, u'grub2-menu', ppc64_distrotrees)
ppc64le_distrotrees = [distro for distro in distro_trees if distro['arch'] == 'ppc64le']
if ppc64le_distrotrees:
print('Generating GRUB2 menus for PPC64LE EFI for %s distro trees' % len(ppc64_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'boot', 'grub2'), mode=0o755)
ppc64le_grub2_menu = atomically_replaced_file(os.path.join(tftp_root, 'boot', 'grub2',
'beaker_menu_ppc64le.cfg'))
write_menu(ppc64le_grub2_menu, u'grub2-menu', ppc64le_distrotrees)
# XXX: would be nice if we can find a good time to move this into boot/grub2
aarch64_distrotrees = [distro for distro in distro_trees if distro['arch'] == 'aarch64']
if aarch64_distrotrees:
print('Generating GRUB2 menus for aarch64 for %s distro trees' % len(aarch64_distrotrees))
makedirs_ignore(os.path.join(tftp_root, 'aarch64'), mode=0o755)
aarch64_menu = atomically_replaced_file(
os.path.join(tftp_root, 'aarch64', 'beaker_menu.cfg'))
write_menu(aarch64_menu, u'grub2-menu', aarch64_distrotrees)
def get_log_base_url():
protocol = get_conf().get('URL_SCHEME', 'http')
server_name = get_conf().get_url_domain()
return '%s://%s' % (protocol, server_name)
# beaker-pxemenu might have already fetched the images, so let's try there
# before anywhere else.
try:
atomic_link(os.path.join(distrotree_dir, 'kernel'),
os.path.join(images_dir, 'kernel'))
atomic_link(os.path.join(distrotree_dir, 'initrd'),
os.path.join(images_dir, 'initrd'))
logger.debug('Using images from distro tree %s for %s', distro_tree_id,
fqdn)
return
except OSError, e:
if e.errno != errno.ENOENT:
raise
# No luck there, so try something else...
timeout = get_conf().get('IMAGE_FETCH_TIMEOUT')
logger.debug('Fetching kernel %s for %s', kernel_url, fqdn)
with atomically_replaced_file(os.path.join(images_dir, 'kernel')) as dest:
try:
siphon(urllib2.urlopen(kernel_url, timeout=timeout), dest)
except Exception as e:
raise ImageFetchingError(kernel_url, distro_tree_id, e)
logger.debug('Fetching initrd %s for %s', initrd_url, fqdn)
with atomically_replaced_file(os.path.join(images_dir, 'initrd')) as dest:
try:
siphon(urllib2.urlopen(initrd_url, timeout=timeout), dest)
except Exception as e:
raise ImageFetchingError(initrd_url, distro_tree_id, e)
def have_images(fqdn):
def get_tftp_root():
return get_conf().get('TFTP_ROOT', '/var/lib/tftpboot')
def test_general_protection__user_space_is_not_detected_as_panic():
line = "kvm-01-guest19 login: [ 30.165967] traps: bz1172806[2463] general protection fault ip:804b000 sp:5aadc0de error:0 in bz1172806[8048000+5000] \n"
detector = PanicDetector(get_conf().get('PANIC_REGEX'))
failure_found = detector.feed(line)
if failure_found:
raise AssertionError('False panic detection: %s' % failure_found)
def get_log_base_url():
protocol = get_conf().get('URL_SCHEME', 'http')
server_name = get_conf().get_url_domain()
return '%s://%s' % (protocol, server_name)
def get_tftp_root():
return get_conf().get('TFTP_ROOT', '/var/lib/tftpboot')
def setUpClass(cls):
makedirs_ignore(get_conf().get('CONSOLE_LOGS'), 0755)
def setUpClass(cls):
makedirs_ignore(get_conf().get('CONSOLE_LOGS'), 0755)
