def _testcases_for_task(task):
if not task.is_finished():
return
yield E.testcase(
E(u'system-out', _systemout_for_task(task)),
classname=task.name)
for result in task.results:
testcase = E.testcase(
classname=task.name,
name=result.short_path.lstrip('/') or '(none)')
if result.duration:
testcase.set('time', '%.0f' % total_seconds(result.duration))
# For Cancelled and Aborted, the final Warn is the reason message
if (task.status == TaskStatus.cancelled and
result == task.results[-1] and
result.result == TaskResult.warn):
testcase.append(E.skipped(message=result.log or u'', type=u'skipped'))
elif (task.status == TaskStatus.aborted and
result == task.results[-1] and
result.result == TaskResult.warn):
testcase.append(E.error(message=result.log or u'', type=u'error'))
elif result.result in (TaskResult.warn, TaskResult.fail):
testcase.append(E.failure(message=result.log or u'', type=u'failure'))
testcase.append(E(u'system-out', _systemout_for_result(result)))
yield testcase
def handle(self, command, predecessors):
if command.get('delay'):
# Before anything else, we need to wait for our delay period.
# Instead of just doing time.sleep we do a timed wait on
# shutting_down, so that our delay doesn't hold up the shutdown.
logger.debug('Delaying %s seconds for command %s',
command['delay'], command['id'])
if shutting_down.wait(timeout=command['delay']):
return
gevent.joinall(predecessors)
if shutting_down.is_set():
return
quiescent_period = command.get('quiescent_period')
if quiescent_period:
system_fqdn = command.get('fqdn')
last_command_finished_at = self.last_command_datetime.get(
system_fqdn)
if last_command_finished_at:
# Get the difference between the time now and the number of
# seconds until we can run another command
seconds_to_wait = total_seconds(
(last_command_finished_at +
datetime.timedelta(seconds=quiescent_period)) -
datetime.datetime.utcnow())
else:
# Play it safe, wait for the whole period.
seconds_to_wait = quiescent_period
if seconds_to_wait > 0:
logger.debug(
'Entering quiescent period, delaying %s seconds for'
' command %s' % (seconds_to_wait, command['id']))
if shutting_down.wait(timeout=seconds_to_wait):
return
logger.debug('Handling command %r', command)
self.mark_command_running(command['id'])
try:
if command['action'] in (u'on', u'off', 'interrupt'):
handle_power(command)
elif command['action'] == u'reboot':
# For backwards compatibility only. The server now splits
# reboots into 'off' followed by 'on'.
handle_power(dict(command.items() + [('action', u'off')]))
time.sleep(5)
handle_power(dict(command.items() + [('action', u'on')]))
elif command['action'] == u'clear_logs':
handle_clear_logs(self.conf, command)
elif command['action'] == u'configure_netboot':
handle_configure_netboot(command)
elif command['action'] == u'clear_netboot':
handle_clear_netboot(command)
else:
raise ValueError('Unrecognised action %s' % command['action'])
# XXX or should we just ignore it and leave it queued?
except Exception, e:
logger.exception('Error processing command %s', command['id'])
self.last_command_datetime[
command['fqdn']] = datetime.datetime.utcnow()
self.mark_command_failed(command['id'],
'%s: %s' % (e.__class__.__name__, e))
def handle(self, command, predecessors):
if command.get('delay'):
# Before anything else, we need to wait for our delay period.
# Instead of just doing time.sleep we do a timed wait on
# shutting_down, so that our delay doesn't hold up the shutdown.
logger.debug('Delaying %s seconds for command %s',
command['delay'], command['id'])
if shutting_down.wait(timeout=command['delay']):
return
gevent.joinall(predecessors)
if shutting_down.is_set():
return
quiescent_period = command.get('quiescent_period')
if quiescent_period:
system_fqdn = command.get('fqdn')
last_command_finished_at = self.last_command_datetime.get(system_fqdn)
if last_command_finished_at:
# Get the difference between the time now and the number of
# seconds until we can run another command
seconds_to_wait = total_seconds((last_command_finished_at +
datetime.timedelta(seconds=quiescent_period)) -
datetime.datetime.utcnow())
else:
# Play it safe, wait for the whole period.
seconds_to_wait = quiescent_period
if seconds_to_wait > 0:
logger.debug('Entering quiescent period, delaying %s seconds for'
' command %s' % (seconds_to_wait, command['id']))
if shutting_down.wait(timeout=seconds_to_wait):
return
logger.debug('Handling command %r', command)
self.mark_command_running(command['id'])
try:
if command['action'] in (u'on', u'off', 'interrupt'):
handle_power(self.conf, command)
elif command['action'] == u'reboot':
# For backwards compatibility only. The server now splits
# reboots into 'off' followed by 'on'.
handle_power(self.conf, dict(command.items() + [('action', u'off')]))
time.sleep(5)
handle_power(self.conf, dict(command.items() + [('action', u'on')]))
elif command['action'] == u'clear_logs':
handle_clear_logs(self.conf, command)
elif command['action'] == u'configure_netboot':
handle_configure_netboot(command)
elif command['action'] == u'clear_netboot':
handle_clear_netboot(command)
else:
raise ValueError('Unrecognised action %s' % command['action'])
# XXX or should we just ignore it and leave it queued?
except netboot.ImageFetchingError as e:
logger.exception('Error processing command %s', command['id'])
# It's not the system's fault so don't mark it as broken
self.mark_command_failed(command['id'], unicode(e), False)
except Exception, e:
logger.exception('Error processing command %s', command['id'])
self.mark_command_failed(command['id'],
'%s: %s' % (e.__class__.__name__, e), True)
def get_queued_command_details(self):
lab_controller = identity.current.user.lab_controller
max_running_commands = config.get('beaker.max_running_commands')
if max_running_commands:
running_commands = Command.query\
.join(Command.system)\
.filter(System.lab_controller == lab_controller)\
.filter(Command.status == CommandStatus.running)\
.count()
if running_commands >= max_running_commands:
return []
query = Command.query\
.join(Command.system)\
.options(contains_eager(Command.system))\
.filter(System.lab_controller == lab_controller)\
.filter(Command.status == CommandStatus.queued)\
.order_by(Command.id)
if max_running_commands:
query = query.limit(max_running_commands - running_commands)
result = []
for cmd in query:
d = {
'id': cmd.id,
'action': cmd.action,
'fqdn': cmd.system.fqdn,
'delay': 0,
'quiescent_period': cmd.quiescent_period
}
if cmd.delay_until:
d['delay'] = max(
0, total_seconds(cmd.delay_until - datetime.utcnow()))
# Fill in details specific to the type of command
if cmd.action in (u'on', u'off', u'reboot', u'interrupt'):
if not cmd.system.power:
cmd.abort(u'Power control unavailable for %s' % cmd.system)
continue
d['power'] = {
'type': cmd.system.power.power_type.name,
'address': cmd.system.power.power_address,
'id': cmd.system.power.power_id,
'user': cmd.system.power.power_user,
'passwd': cmd.system.power.power_passwd,
}
elif cmd.action == u'configure_netboot':
installation = cmd.installation
distro_tree = cmd.installation.distro_tree
if distro_tree:
schemes = ['http', 'ftp']
if distro_tree.arch.arch == 's390' or distro_tree.arch.arch == 's390x':
# zPXE needs FTP URLs for the images, it has no HTTP client.
# It would be nicer if we could leave this decision up to
# beaker-provision, but the API doesn't work like that...
schemes = ['ftp']
distro_tree_url = distro_tree.url_in_lab(lab_controller,
scheme=schemes)
else:
distro_tree_url = installation.tree_url
if not distro_tree_url:
cmd.abort(
u'No usable URL found for distro tree %s in lab %s' %
(distro_tree.id, lab_controller.fqdn))
continue
d['netboot'] = {
'kernel_url':
urlparse.urljoin(distro_tree_url,
installation.kernel_path),
'initrd_url':
urlparse.urljoin(distro_tree_url,
installation.initrd_path),
'kernel_options':
installation.kernel_options or '',
}
if distro_tree:
d['netboot']['distro_tree_id'] = distro_tree.id
else:
d['netboot']['distro_tree_id'] = None
if installation.arch:
d['netboot']['arch'] = installation.arch.arch
else:
# It must be a queued command left over after migrating from Beaker < 25.
d['netboot']['arch'] = distro_tree.arch.arch
result.append(d)
return result
def get_queued_command_details(self):
lab_controller = identity.current.user.lab_controller
max_running_commands = config.get('beaker.max_running_commands')
if max_running_commands:
running_commands = CommandActivity.query\
.join(CommandActivity.system)\
.filter(System.lab_controller == lab_controller)\
.filter(CommandActivity.status == CommandStatus.running)\
.count()
if running_commands >= max_running_commands:
return []
query = CommandActivity.query\
.join(CommandActivity.system)\
.options(contains_eager(CommandActivity.system))\
.filter(System.lab_controller == lab_controller)\
.filter(CommandActivity.status == CommandStatus.queued)\
.order_by(CommandActivity.id)
if max_running_commands:
query = query.limit(max_running_commands - running_commands)
result = []
for cmd in query:
d = {
'id': cmd.id,
'action': cmd.action,
'fqdn': cmd.system.fqdn,
'delay': 0,
'quiescent_period': cmd.quiescent_period
}
if cmd.delay_until:
d['delay'] = max(
0, total_seconds(cmd.delay_until - datetime.utcnow()))
# Fill in details specific to the type of command
if cmd.action in (u'on', u'off', u'reboot', u'interrupt'):
if not cmd.system.power:
cmd.abort(u'Power control unavailable for %s' % cmd.system)
continue
d['power'] = {
'type': cmd.system.power.power_type.name,
'address': cmd.system.power.power_address,
'id': cmd.system.power.power_id,
'user': cmd.system.power.power_user,
'passwd': cmd.system.power.power_passwd,
}
elif cmd.action == u'configure_netboot':
distro_tree_url = cmd.distro_tree.url_in_lab(
lab_controller, scheme=['http', 'ftp'])
if not distro_tree_url:
cmd.abort(
u'No usable URL found for distro tree %s in lab %s' %
(cmd.distro_tree.id, lab_controller.fqdn))
continue
if cmd.system.kernel_type.uboot:
by_kernel = ImageType.uimage
by_initrd = ImageType.uinitrd
else:
by_kernel = ImageType.kernel
by_initrd = ImageType.initrd
kernel = cmd.distro_tree.image_by_type(by_kernel,
cmd.system.kernel_type)
if not kernel:
cmd.abort(u'Kernel image not found for distro tree %s' %
cmd.distro_tree.id)
continue
initrd = cmd.distro_tree.image_by_type(by_initrd,
cmd.system.kernel_type)
if not initrd:
cmd.abort(u'Initrd image not found for distro tree %s' %
cmd.distro_tree.id)
continue
d['netboot'] = {
'arch': cmd.distro_tree.arch.arch,
'distro_tree_id': cmd.distro_tree.id,
'kernel_url': urlparse.urljoin(distro_tree_url,
kernel.path),
'initrd_url': urlparse.urljoin(distro_tree_url,
initrd.path),
'kernel_options': cmd.kernel_options or '',
}
result.append(d)
return result
def get_queued_command_details(self):
lab_controller = identity.current.user.lab_controller
max_running_commands = config.get('beaker.max_running_commands')
if max_running_commands:
running_commands = CommandActivity.query\
.join(CommandActivity.system)\
.filter(System.lab_controller == lab_controller)\
.filter(CommandActivity.status == CommandStatus.running)\
.count()
if running_commands >= max_running_commands:
return []
query = CommandActivity.query\
.join(CommandActivity.system)\
.options(contains_eager(CommandActivity.system))\
.filter(System.lab_controller == lab_controller)\
.filter(CommandActivity.status == CommandStatus.queued)\
.order_by(CommandActivity.id)
if max_running_commands:
query = query.limit(max_running_commands - running_commands)
result = []
for cmd in query:
d = {
'id': cmd.id,
'action': cmd.action,
'fqdn': cmd.system.fqdn,
'delay': 0,
'quiescent_period': cmd.quiescent_period
}
if cmd.delay_until:
d['delay'] = max(0, total_seconds(cmd.delay_until - datetime.utcnow()))
# Fill in details specific to the type of command
if cmd.action in (u'on', u'off', u'reboot', u'interrupt'):
if not cmd.system.power:
cmd.abort(u'Power control unavailable for %s' % cmd.system)
continue
d['power'] = {
'type': cmd.system.power.power_type.name,
'address': cmd.system.power.power_address,
'id': cmd.system.power.power_id,
'user': cmd.system.power.power_user,
'passwd': cmd.system.power.power_passwd,
}
elif cmd.action == u'configure_netboot':
installation = cmd.installation
distro_tree = cmd.installation.distro_tree
distro_tree_url = distro_tree.url_in_lab(lab_controller,
scheme=['http', 'ftp'])
if not distro_tree_url:
cmd.abort(u'No usable URL found for distro tree %s in lab %s'
% (distro_tree.id, lab_controller.fqdn))
continue
if cmd.system.kernel_type.uboot:
by_kernel = ImageType.uimage
by_initrd = ImageType.uinitrd
else:
by_kernel = ImageType.kernel
by_initrd = ImageType.initrd
kernel = distro_tree.image_by_type(by_kernel, cmd.system.kernel_type)
if not kernel:
cmd.abort(u'Kernel image not found for distro tree %s' % distro_tree.id)
continue
initrd = distro_tree.image_by_type(by_initrd, cmd.system.kernel_type)
if not initrd:
cmd.abort(u'Initrd image not found for distro tree %s' % distro_tree.id)
continue
d['netboot'] = {
'arch': distro_tree.arch.arch,
'distro_tree_id': distro_tree.id,
'kernel_url': urlparse.urljoin(distro_tree_url, kernel.path),
'initrd_url': urlparse.urljoin(distro_tree_url, initrd.path),
'kernel_options': installation.kernel_options or '',
}
result.append(d)
return result