class InstallProgressController(BaseController):
signals = [
('installprogress:filesystem-config-done', 'filesystem_config_done'),
('installprogress:identity-config-done', 'identity_config_done'),
]
def __init__(self, common):
super().__init__(common)
self.answers = self.all_answers.get('InstallProgress', {})
self.answers.setdefault('reboot', False)
self.progress_view = None
self.progress_view_showing = False
self.install_state = InstallState.NOT_STARTED
self.journal_listener_handle = None
self._identity_config_done = False
self._event_indent = ""
self._event_syslog_identifier = 'curtin_event.%s' % (os.getpid(), )
self._log_syslog_identifier = 'curtin_log.%s' % (os.getpid(), )
def filesystem_config_done(self):
self.curtin_start_install()
def identity_config_done(self):
if self.install_state == InstallState.DONE:
self.postinstall_configuration()
else:
self._identity_config_done = True
def curtin_error(self):
log.debug('curtin_error')
self.install_state = InstallState.ERROR
self.progress_view.spinner.stop()
self.progress_view.set_status(
('info_error', _("An error has occurred")))
self.progress_view.show_complete(True)
self.default()
def _bg_run_command_logged(self, cmd, env):
cmd = [
'systemd-cat', '--level-prefix=false',
'--identifier=' + self._log_syslog_identifier
] + cmd
return utils.run_command(cmd, env=env)
def _journal_event(self, event):
if event['SYSLOG_IDENTIFIER'] == self._event_syslog_identifier:
self.curtin_event(event)
elif event['SYSLOG_IDENTIFIER'] == self._log_syslog_identifier:
self.curtin_log(event)
def curtin_event(self, event):
e = {}
for k, v in event.items():
if k.startswith("CURTIN_"):
e[k] = v
log.debug("curtin_event received %r", e)
event_type = event.get("CURTIN_EVENT_TYPE")
if event_type not in ['start', 'finish']:
return
if event_type == 'start':
message = event.get("CURTIN_MESSAGE", "??")
if self.progress_view_showing is not None:
self.footer_description.set_text(message)
self.progress_view.add_event(self._event_indent + message)
self._event_indent += " "
self.footer_spinner.start()
if event_type == 'finish':
self._event_indent = self._event_indent[:-2]
self.footer_spinner.stop()
def curtin_log(self, event):
self.progress_view.add_log_line(event['MESSAGE'])
def start_journald_listener(self, identifiers, callback):
reader = journal.Reader()
args = []
for identifier in identifiers:
args.append("SYSLOG_IDENTIFIER={}".format(identifier))
reader.add_match(*args)
def watch():
if reader.process() != journal.APPEND:
return
for event in reader:
callback(event)
return self.loop.watch_file(reader.fileno(), watch)
def _write_config(self, path, config):
with open(path, 'w') as conf:
datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
str(datetime.datetime.utcnow()))
conf.write(datestr)
conf.write(yaml.dump(config))
def _get_curtin_command(self):
config_file_name = 'subiquity-curtin-install.conf'
if self.opts.dry_run:
log.debug("Installprogress: this is a dry-run")
config_location = os.path.join('.subiquity/', config_file_name)
curtin_cmd = [
"python3", "scripts/replay-curtin-log.py",
"examples/curtin-events.json", self._event_syslog_identifier
]
else:
log.debug("Installprogress: this is the *REAL* thing")
config_location = os.path.join('/var/log/installer',
config_file_name)
curtin_cmd = [
'curtin', '--showtrace', '-c', config_location, 'install'
]
ident = self._event_syslog_identifier
self._write_config(
config_location,
self.base_model.render(target=TARGET, syslog_identifier=ident))
return curtin_cmd
def curtin_start_install(self):
log.debug('Curtin Install: starting curtin')
self.install_state = InstallState.RUNNING
self.footer_description = urwid.Text(_("starting..."))
self.progress_view = ProgressView(self)
self.footer_spinner = self.progress_view.spinner
self.ui.set_footer(
urwid.Columns([('pack', urwid.Text(_("Install in progress:"))),
(self.footer_description),
('pack', self.footer_spinner)],
dividechars=1))
self.journal_listener_handle = self.start_journald_listener(
[self._event_syslog_identifier, self._log_syslog_identifier],
self._journal_event)
curtin_cmd = self._get_curtin_command()
log.debug('Curtin install cmd: {}'.format(curtin_cmd))
env = os.environ.copy()
if 'SNAP' in env:
del env['SNAP']
self.run_in_bg(lambda: self._bg_run_command_logged(curtin_cmd, env),
self.curtin_install_completed)
def curtin_install_completed(self, fut):
cp = fut.result()
log.debug('curtin_install completed: %s', cp.returncode)
if cp.returncode != 0:
self.curtin_error()
return
self.install_state = InstallState.DONE
log.debug('After curtin install OK')
self.ui.progress_current += 1
if not self.progress_view_showing:
self.ui.set_footer(_("Install complete"))
else:
# Re-set footer so progress bar updates.
self.ui.set_footer(_("Thank you for using Ubuntu!"))
if self._identity_config_done:
self.postinstall_configuration()
def cancel(self):
pass
def postinstall_configuration(self):
# If we need to do anything that takes time here (like running
# dpkg-reconfigure maas-rack-controller, for example...) we
# should switch to doing that work in a background thread.
self.configure_cloud_init()
self.copy_logs_to_target()
self.ui.set_header(_("Installation complete!"))
self.progress_view.set_status(_("Finished install!"))
self.progress_view.show_complete()
if self.answers['reboot']:
self.reboot()
def configure_cloud_init(self):
if self.opts.dry_run:
target = '.subiquity'
else:
target = TARGET
self.base_model.configure_cloud_init(target)
def copy_logs_to_target(self):
if self.opts.dry_run:
return
utils.run_command(
['cp', '-aT', '/var/log/installer', '/target/var/log/installer'])
try:
with open('/target/var/log/installer/installer-journal.txt',
'w') as output:
utils.run_command(['journalctl'],
stdout=output,
stderr=subprocess.STDOUT)
except Exception:
log.exception("saving journal failed")
def reboot(self):
if self.opts.dry_run:
log.debug('dry-run enabled, skipping reboot, quiting instead')
self.signal.emit_signal('quit')
else:
# Should probably run curtin -c $CONFIG unmount -t TARGET first.
utils.run_command(["/sbin/reboot"])
def quit(self):
if not self.opts.dry_run:
utils.disable_subiquity()
self.signal.emit_signal('quit')
def default(self):
self.progress_view_showing = True
if self.install_state == InstallState.RUNNING:
self.progress_view.title = _("Installing system")
self.progress_view.footer = _("Thank you for using Ubuntu!")
elif self.install_state == InstallState.DONE:
self.progress_view.title = _("Install complete!")
self.progress_view.footer = _("Thank you for using Ubuntu!")
elif self.install_state == InstallState.ERROR:
self.progress_view.title = (
_('An error occurred during installation'))
self.progress_view.footer = (
_('Please report this error in Launchpad'))
self.ui.set_body(self.progress_view)
class InstallProgressController(BaseController):
signals = [
('installprogress:filesystem-config-done', 'filesystem_config_done'),
('installprogress:identity-config-done', 'identity_config_done'),
('installprogress:ssh-config-done', 'ssh_config_done'),
('installprogress:snap-config-done', 'snap_config_done'),
]
def __init__(self, app):
super().__init__(app)
self.model = app.base_model
self.answers.setdefault('reboot', False)
self.progress_view = None
self.install_state = InstallState.NOT_STARTED
self.journal_listener_handle = None
self._postinstall_prerequisites = {
'install': False,
'ssh': False,
'identity': False,
'snap': False,
}
self._event_indent = ""
self._event_syslog_identifier = 'curtin_event.%s' % (os.getpid(),)
self._log_syslog_identifier = 'curtin_log.%s' % (os.getpid(),)
self.sm = None
def tpath(self, *path):
return os.path.join(self.model.target, *path)
def filesystem_config_done(self):
self.curtin_start_install()
def _step_done(self, step):
self._postinstall_prerequisites[step] = True
log.debug("_step_done %s %s", step, self._postinstall_prerequisites)
if all(self._postinstall_prerequisites.values()):
self.start_postinstall_configuration()
def identity_config_done(self):
self._step_done('identity')
def ssh_config_done(self):
self._step_done('ssh')
def snap_config_done(self):
self._step_done('snap')
def curtin_error(self):
self.install_state = InstallState.ERROR
self.app.make_apport_report(
ErrorReportKind.INSTALL_FAIL, "install failed")
self.progress_view.spinner.stop()
if sys.exc_info()[0] is not None:
self.progress_view.add_log_line(traceback.format_exc())
self.progress_view.set_status(('info_error',
_("An error has occurred")))
self.progress_view.show_complete(True)
self.start_ui()
def _bg_run_command_logged(self, cmd, **kwargs):
cmd = ['systemd-cat', '--level-prefix=false',
'--identifier=' + self._log_syslog_identifier] + cmd
return utils.run_command(cmd, **kwargs)
def _journal_event(self, event):
if event['SYSLOG_IDENTIFIER'] == self._event_syslog_identifier:
self.curtin_event(event)
elif event['SYSLOG_IDENTIFIER'] == self._log_syslog_identifier:
self.curtin_log(event)
def _install_event_start(self, message):
log.debug("_install_event_start %s", message)
self.progress_view.add_event(self._event_indent + message)
self._event_indent += " "
self.progress_view.spinner.start()
def _install_event_finish(self):
self._event_indent = self._event_indent[:-2]
log.debug("_install_event_finish %r", self._event_indent)
self.progress_view.spinner.stop()
def curtin_event(self, event):
e = {}
for k, v in event.items():
if k.startswith("CURTIN_"):
e[k] = v
log.debug("curtin_event received %r", e)
event_type = event.get("CURTIN_EVENT_TYPE")
if event_type not in ['start', 'finish']:
return
if event_type == 'start':
self._install_event_start(event.get("CURTIN_MESSAGE", "??"))
if event_type == 'finish':
self._install_event_finish()
def curtin_log(self, event):
self.progress_view.add_log_line(event['MESSAGE'])
def start_journald_listener(self, identifiers, callback):
reader = journal.Reader()
args = []
for identifier in identifiers:
args.append("SYSLOG_IDENTIFIER={}".format(identifier))
reader.add_match(*args)
def watch():
if reader.process() != journal.APPEND:
return
for event in reader:
callback(event)
return self.loop.watch_file(reader.fileno(), watch)
def _write_config(self, path, config):
with open(path, 'w') as conf:
datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
str(datetime.datetime.utcnow()))
conf.write(datestr)
conf.write(yaml.dump(config))
def _get_curtin_command(self):
config_file_name = 'subiquity-curtin-install.conf'
if self.opts.dry_run:
config_location = os.path.join('.subiquity/', config_file_name)
curtin_cmd = ["python3", "scripts/replay-curtin-log.py",
"examples/curtin-events.json",
self._event_syslog_identifier]
else:
config_location = os.path.join('/var/log/installer',
config_file_name)
curtin_cmd = [sys.executable, '-m', 'curtin', '--showtrace', '-c',
config_location, 'install']
ident = self._event_syslog_identifier
self._write_config(config_location,
self.model.render(syslog_identifier=ident))
self.app.note_file_for_apport("CurtinConfig", config_location)
self.app.note_file_for_apport("CurtinLog", INSTALL_LOG)
self.app.note_file_for_apport("CurtinErrors", ERROR_TARFILE)
return curtin_cmd
def curtin_start_install(self):
log.debug('curtin_start_install')
self.install_state = InstallState.RUNNING
self.progress_view = ProgressView(self)
self.journal_listener_handle = self.start_journald_listener(
[self._event_syslog_identifier, self._log_syslog_identifier],
self._journal_event)
curtin_cmd = self._get_curtin_command()
log.debug('curtin install cmd: {}'.format(curtin_cmd))
self.run_in_bg(
lambda: self._bg_run_command_logged(curtin_cmd),
self.curtin_install_completed)
def curtin_install_completed(self, fut):
cp = fut.result()
log.debug('curtin_install completed: %s', cp.returncode)
if cp.returncode != 0:
self.curtin_error()
return
self.install_state = InstallState.DONE
log.debug('After curtin install OK')
self._step_done('install')
def cancel(self):
pass
def start_postinstall_configuration(self):
has_network = self.model.network.has_network
def filter_task(func):
if func._extra.get('net_only') and not has_network:
return False
if func._name == 'install_openssh' \
and not self.model.ssh.install_server:
return False
return True
log.debug("starting state machine")
self.sm = StateMachine(self, collect_tasks(self, filter_task))
self.sm.run()
@task
def _bg_drain_curtin_events(self):
waited = 0.0
while self._event_indent and waited < 5.0:
time.sleep(0.1)
waited += 0.1
log.debug("waited %s seconds for events to drain", waited)
@task
def start_final_configuration(self):
self._install_event_start("final system configuration")
@task(label="configuring cloud-init")
def _bg_configure_cloud_init(self):
self.model.configure_cloud_init()
@task(label="installing openssh")
def _bg_install_openssh(self):
if self.opts.dry_run:
cmd = ["sleep", str(2/self.app.scale_factor)]
else:
cmd = [
sys.executable, "-m", "curtin", "system-install", "-t",
"/target",
"--", "openssh-server",
]
self._bg_run_command_logged(cmd, check=True)
@task(label="restoring apt configuration")
def _bg_restore_apt_config(self):
if self.opts.dry_run:
cmds = [["sleep", str(1/self.app.scale_factor)]]
else:
cmds = [
["umount", self.tpath('etc/apt')],
]
if self.model.network.has_network:
cmds.append([
sys.executable, "-m", "curtin", "in-target", "-t",
"/target", "--", "apt-get", "update",
])
else:
cmds.append(["umount", self.tpath('var/lib/apt/lists')])
for cmd in cmds:
self._bg_run_command_logged(cmd, check=True)
@task
def postinstall_complete(self):
self._install_event_finish()
self.ui.set_header(_("Installation complete!"))
self.progress_view.set_status(_("Finished install!"))
self.progress_view.show_complete()
self.copy_logs_transition = 'wait'
@task(net_only=True)
def uu_start(self):
self.progress_view.update_running()
@task(label="downloading and installing security updates",
transitions={'reboot': 'abort_uu'},
net_only=True)
def _bg_run_uu(self):
target_tmp = os.path.join(self.model.target, "tmp")
os.makedirs(target_tmp, exist_ok=True)
apt_conf = tempfile.NamedTemporaryFile(
dir=target_tmp, delete=False, mode='w')
apt_conf.write(uu_apt_conf)
apt_conf.close()
env = os.environ.copy()
env["APT_CONFIG"] = apt_conf.name[len(self.model.target):]
if self.opts.dry_run:
self.uu = utils.start_command([
"sleep", str(10/self.app.scale_factor)])
self.uu.wait()
else:
self._bg_run_command_logged([
sys.executable, "-m", "curtin", "in-target", "-t", "/target",
"--", "unattended-upgrades", "-v",
], env=env, check=True)
os.remove(apt_conf.name)
@task(transitions={'success': 'copy_logs_to_target'}, net_only=True)
def uu_done(self):
self.progress_view.update_done()
@task(net_only=True)
def abort_uu(self):
self._install_event_finish()
@task(label="cancelling update", net_only=True)
def _bg_stop_uu(self):
if self.opts.dry_run:
time.sleep(1)
self.uu.terminate()
else:
self._bg_run_command_logged([
'chroot', '/target',
'/usr/share/unattended-upgrades/unattended-upgrade-shutdown',
'--stop-only',
], check=True)
@task(net_only=True, transitions={'success': 'copy_logs_to_target'})
def _bg_wait_for_uu(self):
r, w = os.pipe()
def callback(fut):
os.write(w, b'x')
self.sm.subscribe('run_uu', callback)
os.read(r, 1)
os.close(w)
os.close(r)
self.copy_logs_transition = 'reboot'
@task(label="copying logs to installed system",
transitions={'reboot': 'reboot'})
def _bg_copy_logs_to_target(self):
if self.opts.dry_run:
if 'copy-logs-fail' in self.debug_flags:
raise PermissionError()
return
target_logs = self.tpath('var/log/installer')
utils.run_command(['cp', '-aT', '/var/log/installer', target_logs])
try:
with open(os.path.join(target_logs,
'installer-journal.txt'), 'w') as output:
utils.run_command(
['journalctl'],
stdout=output, stderr=subprocess.STDOUT)
except Exception:
log.exception("saving journal failed")
@task(transitions={'wait': 'wait_for_click', 'reboot': 'reboot'})
def copy_logs_done(self):
self.sm.transition(self.copy_logs_transition)
@task(transitions={'reboot': 'reboot'})
def _bg_wait_for_click(self):
if not self.answers['reboot']:
signal.pause()
@task
def reboot(self):
if self.opts.dry_run:
log.debug('dry-run enabled, skipping reboot, quitting instead')
self.signal.emit_signal('quit')
else:
# TODO Possibly run this earlier, to show a warning; or
# switch to shutdown if chreipl fails
if platform.machine() == 's390x':
utils.run_command(["chreipl", "/target/boot"])
# Should probably run curtin -c $CONFIG unmount -t TARGET first.
utils.run_command(["/sbin/reboot"])
def click_reboot(self):
if self.sm is None:
# If the curtin install itself crashes, the state machine
# that manages post install steps won't be running. Just
# reboot anyway.
self.reboot()
else:
self.sm.transition('reboot')
def quit(self):
if not self.opts.dry_run:
utils.disable_subiquity()
self.signal.emit_signal('quit')
def start_ui(self):
if self.install_state == InstallState.RUNNING:
self.progress_view.title = _("Installing system")
elif self.install_state == InstallState.DONE:
self.progress_view.title = _("Install complete!")
elif self.install_state == InstallState.ERROR:
self.progress_view.title = (
_('An error occurred during installation'))
self.ui.set_body(self.progress_view)
class InstallProgressController(SubiquityController):
def __init__(self, app):
super().__init__(app)
self.model = app.base_model
self.progress_view = ProgressView(self)
app.add_event_listener(self)
self.install_state = InstallState.NOT_STARTED
self.reboot_clicked = asyncio.Event()
if self.answers.get('reboot', False):
self.reboot_clicked.set()
self.unattended_upgrades_proc = None
self.unattended_upgrades_ctx = None
self._event_syslog_identifier = 'curtin_event.%s' % (os.getpid(), )
self._log_syslog_identifier = 'curtin_log.%s' % (os.getpid(), )
self.tb_extractor = TracebackExtractor()
self.curtin_event_contexts = {}
self.confirmation = asyncio.Event()
def interactive(self):
return self.app.interactive()
def start(self):
self.install_task = schedule_task(self.install())
@with_context()
async def apply_autoinstall_config(self, context):
await self.install_task
self.app.reboot_on_exit = True
def _push_to_progress(self, context):
if not self.app.interactive():
return False
if context.get('hidden', False):
return False
controller = context.get('controller')
if controller is None or controller.interactive():
return False
return True
def report_start_event(self, context, description):
if self._push_to_progress(context):
msg = context.full_name()
if description:
msg += ': ' + description
self.progress_view.event_start(context, msg)
if context.get('is-install-context'):
self.progress_view.event_start(context, context.description)
def report_finish_event(self, context, description, status):
if self._push_to_progress(context):
self.progress_view.event_finish(context)
if context.get('is-install-context'):
self.progress_view.event_finish(context)
def tpath(self, *path):
return os.path.join(self.model.target, *path)
def curtin_error(self):
self.install_state = InstallState.ERROR
kw = {}
if sys.exc_info()[0] is not None:
log.exception("curtin_error")
self.progress_view.add_log_line(traceback.format_exc())
if self.tb_extractor.traceback:
kw["Traceback"] = "\n".join(self.tb_extractor.traceback)
crash_report = self.app.make_apport_report(
ErrorReportKind.INSTALL_FAIL,
"install failed",
interrupt=False,
**kw)
self.progress_view.finish_all()
self.progress_view.set_status(
('info_error', _("An error has occurred")))
self.start_ui()
if crash_report is not None:
self.progress_view.show_error(crash_report)
def logged_command(self, cmd):
return [
'systemd-cat', '--level-prefix=false',
'--identifier=' + self._log_syslog_identifier
] + cmd
def _journal_event(self, event):
if event['SYSLOG_IDENTIFIER'] == self._event_syslog_identifier:
self.curtin_event(event)
elif event['SYSLOG_IDENTIFIER'] == self._log_syslog_identifier:
self.curtin_log(event)
def curtin_event(self, event):
e = {
"EVENT_TYPE": "???",
"MESSAGE": "???",
"NAME": "???",
"RESULT": "???",
}
prefix = "CURTIN_"
for k, v in event.items():
if k.startswith(prefix):
e[k[len(prefix):]] = v
event_type = e["EVENT_TYPE"]
if event_type == 'start':
def p(name):
parts = name.split('/')
for i in range(len(parts), -1, -1):
yield '/'.join(parts[:i]), '/'.join(parts[i:])
curtin_ctx = None
for pre, post in p(e["NAME"]):
if pre in self.curtin_event_contexts:
parent = self.curtin_event_contexts[pre]
curtin_ctx = parent.child(post, e["MESSAGE"])
self.curtin_event_contexts[e["NAME"]] = curtin_ctx
break
if curtin_ctx:
curtin_ctx.enter()
if event_type == 'finish':
status = getattr(Status, e["RESULT"], Status.WARN)
curtin_ctx = self.curtin_event_contexts.pop(e["NAME"], None)
if curtin_ctx is not None:
curtin_ctx.exit(status)
def curtin_log(self, event):
log_line = event['MESSAGE']
self.progress_view.add_log_line(log_line)
self.tb_extractor.feed(log_line)
def _write_config(self, path, config):
with open(path, 'w') as conf:
datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
str(datetime.datetime.utcnow()))
conf.write(datestr)
conf.write(yaml.dump(config))
def _get_curtin_command(self):
config_file_name = 'subiquity-curtin-install.conf'
if self.opts.dry_run:
config_location = os.path.join('.subiquity/', config_file_name)
log_location = '.subiquity/install.log'
event_file = "examples/curtin-events.json"
if 'install-fail' in self.app.debug_flags:
event_file = "examples/curtin-events-fail.json"
curtin_cmd = [
"python3",
"scripts/replay-curtin-log.py",
event_file,
self._event_syslog_identifier,
log_location,
]
else:
config_location = os.path.join('/var/log/installer',
config_file_name)
curtin_cmd = [
sys.executable, '-m', 'curtin', '--showtrace', '-c',
config_location, 'install'
]
log_location = INSTALL_LOG
ident = self._event_syslog_identifier
self._write_config(config_location,
self.model.render(syslog_identifier=ident))
self.app.note_file_for_apport("CurtinConfig", config_location)
self.app.note_file_for_apport("CurtinLog", log_location)
self.app.note_file_for_apport("CurtinErrors", ERROR_TARFILE)
return curtin_cmd
@with_context(description="umounting /target dir")
async def unmount_target(self, *, context, target):
cmd = [
sys.executable,
'-m',
'curtin',
'unmount',
'-t',
target,
]
if self.opts.dry_run:
cmd = ['sleep', str(0.2 / self.app.scale_factor)]
await arun_command(cmd)
if not self.opts.dry_run:
shutil.rmtree(target)
@with_context(description="installing system",
level="INFO",
childlevel="DEBUG")
async def curtin_install(self, *, context):
log.debug('curtin_install')
self.install_state = InstallState.RUNNING
self.curtin_event_contexts[''] = context
journal_fd, watcher = journald_listener(
[self._event_syslog_identifier, self._log_syslog_identifier],
self._journal_event)
self.app.aio_loop.add_reader(journal_fd, watcher)
curtin_cmd = self._get_curtin_command()
log.debug('curtin install cmd: {}'.format(curtin_cmd))
async with self.app.install_lock_file.exclusive():
try:
our_tty = os.ttyname(0)
except OSError:
# This is a gross hack for testing in travis.
our_tty = "/dev/not a tty"
self.app.install_lock_file.write_content(our_tty)
journal.send("starting install", SYSLOG_IDENTIFIER="subiquity")
cp = await arun_command(self.logged_command(curtin_cmd),
check=True)
log.debug('curtin_install completed: %s', cp.returncode)
self.install_state = InstallState.DONE
log.debug('After curtin install OK')
def cancel(self):
pass
@with_context()
async def install(self, *, context):
context.set('is-install-context', True)
try:
await asyncio.wait({e.wait() for e in self.model.install_events})
await self.confirmation.wait()
if os.path.exists(self.model.target):
await self.unmount_target(context=context,
target=self.model.target)
await self.curtin_install(context=context)
await asyncio.wait(
{e.wait()
for e in self.model.postinstall_events})
await self.drain_curtin_events(context=context)
await self.postinstall(context=context)
self.ui.set_header(_("Installation complete!"))
self.progress_view.set_status(_("Finished install!"))
self.progress_view.show_complete()
if self.model.network.has_network:
self.progress_view.update_running()
await self.run_unattended_upgrades(context=context)
self.progress_view.update_done()
except Exception:
self.curtin_error()
if not self.interactive():
raise
async def move_on(self):
await self.install_task
self.app.next_screen()
async def drain_curtin_events(self, *, context):
waited = 0.0
while self.progress_view.ongoing and waited < 5.0:
await asyncio.sleep(0.1)
waited += 0.1
log.debug("waited %s seconds for events to drain", waited)
self.curtin_event_contexts.pop('', None)
@with_context(description="final system configuration",
level="INFO",
childlevel="DEBUG")
async def postinstall(self, *, context):
autoinstall_path = os.path.join(
self.app.root, 'var/log/installer/autoinstall-user-data')
autoinstall_config = "#cloud-config\n" + yaml.dump(
{"autoinstall": self.app.make_autoinstall()})
write_file(autoinstall_path, autoinstall_config, mode=0o600)
await self.configure_cloud_init(context=context)
packages = []
if self.model.ssh.install_server:
packages = ['openssh-server']
packages.extend(self.app.base_model.packages)
for package in packages:
await self.install_package(context=context, package=package)
await self.restore_apt_config(context=context)
@with_context(description="configuring cloud-init")
async def configure_cloud_init(self, context):
await run_in_thread(self.model.configure_cloud_init)
@with_context(name="install_{package}", description="installing {package}")
async def install_package(self, *, context, package):
if self.opts.dry_run:
cmd = ["sleep", str(2 / self.app.scale_factor)]
else:
cmd = [
sys.executable,
"-m",
"curtin",
"system-install",
"-t",
"/target",
"--",
package,
]
await arun_command(self.logged_command(cmd), check=True)
@with_context(description="restoring apt configuration")
async def restore_apt_config(self, context):
if self.opts.dry_run:
cmds = [["sleep", str(1 / self.app.scale_factor)]]
else:
cmds = [
["umount", self.tpath('etc/apt')],
]
if self.model.network.has_network:
cmds.append([
sys.executable,
"-m",
"curtin",
"in-target",
"-t",
"/target",
"--",
"apt-get",
"update",
])
else:
cmds.append(["umount", self.tpath('var/lib/apt/lists')])
for cmd in cmds:
await arun_command(self.logged_command(cmd), check=True)
@with_context(description="downloading and installing security updates")
async def run_unattended_upgrades(self, context):
target_tmp = os.path.join(self.model.target, "tmp")
os.makedirs(target_tmp, exist_ok=True)
apt_conf = tempfile.NamedTemporaryFile(dir=target_tmp,
delete=False,
mode='w')
apt_conf.write(uu_apt_conf)
apt_conf.close()
env = os.environ.copy()
env["APT_CONFIG"] = apt_conf.name[len(self.model.target):]
self.unattended_upgrades_ctx = context
if self.opts.dry_run:
self.unattended_upgrades_proc = await astart_command(
self.logged_command(["sleep",
str(5 / self.app.scale_factor)]),
env=env)
else:
self.unattended_upgrades_proc = await astart_command(
self.logged_command([
sys.executable,
"-m",
"curtin",
"in-target",
"-t",
"/target",
"--",
"unattended-upgrades",
"-v",
]),
env=env)
await self.unattended_upgrades_proc.communicate()
self.unattended_upgrades_proc = None
self.unattended_upgrades_ctx = None
os.remove(apt_conf.name)
async def stop_unattended_upgrades(self):
self.progress_view.event_finish(self.unattended_upgrades_ctx)
with self.unattended_upgrades_ctx.parent.child(
"stop_unattended_upgrades", "cancelling update"):
if self.opts.dry_run:
await asyncio.sleep(1)
self.unattended_upgrades_proc.terminate()
else:
await arun_command(self.logged_command([
'chroot',
'/target',
'/usr/share/unattended-upgrades/'
'unattended-upgrade-shutdown',
'--stop-only',
]),
check=True)
async def _click_reboot(self):
if self.unattended_upgrades_ctx is not None:
await self.stop_unattended_upgrades()
self.reboot_clicked.set()
def click_reboot(self):
schedule_task(self._click_reboot())
def start_ui(self):
if self.install_state in [
InstallState.NOT_STARTED,
InstallState.RUNNING,
]:
self.progress_view.title = _("Installing system")
elif self.install_state == InstallState.DONE:
self.progress_view.title = _("Install complete!")
elif self.install_state == InstallState.ERROR:
self.progress_view.title = (
_('An error occurred during installation'))
self.ui.set_body(self.progress_view)
schedule_task(self.move_on())
class InstallProgressController(BaseController):
signals = [
('installprogress:filesystem-config-done', 'filesystem_config_done'),
('installprogress:identity-config-done', 'identity_config_done'),
('installprogress:ssh-config-done', 'ssh_config_done'),
('installprogress:snap-config-done', 'snap_config_done'),
]
def __init__(self, common):
super().__init__(common)
self.answers = self.all_answers.get('InstallProgress', {})
self.answers.setdefault('reboot', False)
self.progress_view = None
self.progress_view_showing = False
self.install_state = InstallState.NOT_STARTED
self.journal_listener_handle = None
self._postinstall_prerequisites = {
'install': False,
'ssh': False,
'identity': False,
'snap': False,
}
self._event_indent = ""
self._event_syslog_identifier = 'curtin_event.%s' % (os.getpid(),)
self._log_syslog_identifier = 'curtin_log.%s' % (os.getpid(),)
def filesystem_config_done(self):
self.curtin_start_install()
def _step_done(self, step):
self._postinstall_prerequisites[step] = True
if all(self._postinstall_prerequisites.values()):
self.start_postinstall_configuration()
def identity_config_done(self):
self._step_done('identity')
def ssh_config_done(self):
self._step_done('ssh')
def snap_config_done(self):
self._step_done('snap')
def curtin_error(self):
log.debug('curtin_error')
self.install_state = InstallState.ERROR
self.progress_view.spinner.stop()
self.progress_view.set_status(('info_error',
_("An error has occurred")))
self.progress_view.show_complete(True)
self.default()
def _bg_run_command_logged(self, cmd):
cmd = ['systemd-cat', '--level-prefix=false',
'--identifier=' + self._log_syslog_identifier] + cmd
return utils.run_command(cmd)
def _journal_event(self, event):
if event['SYSLOG_IDENTIFIER'] == self._event_syslog_identifier:
self.curtin_event(event)
elif event['SYSLOG_IDENTIFIER'] == self._log_syslog_identifier:
self.curtin_log(event)
def _install_event_start(self, message):
log.debug("_install_event_start %s", message)
self.footer_description.set_text(message)
self.progress_view.add_event(self._event_indent + message)
self._event_indent += " "
self.footer_spinner.start()
def _install_event_finish(self):
self._event_indent = self._event_indent[:-2]
log.debug("_install_event_finish %r", self._event_indent)
self.footer_spinner.stop()
def curtin_event(self, event):
e = {}
for k, v in event.items():
if k.startswith("CURTIN_"):
e[k] = v
log.debug("curtin_event received %r", e)
event_type = event.get("CURTIN_EVENT_TYPE")
if event_type not in ['start', 'finish']:
return
if event_type == 'start':
self._install_event_start(event.get("CURTIN_MESSAGE", "??"))
if event_type == 'finish':
self._install_event_finish()
def curtin_log(self, event):
self.progress_view.add_log_line(event['MESSAGE'])
def start_journald_listener(self, identifiers, callback):
reader = journal.Reader()
args = []
for identifier in identifiers:
args.append("SYSLOG_IDENTIFIER={}".format(identifier))
reader.add_match(*args)
def watch():
if reader.process() != journal.APPEND:
return
for event in reader:
callback(event)
return self.loop.watch_file(reader.fileno(), watch)
def _write_config(self, path, config):
with open(path, 'w') as conf:
datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
str(datetime.datetime.utcnow()))
conf.write(datestr)
conf.write(yaml.dump(config))
def _get_curtin_command(self):
config_file_name = 'subiquity-curtin-install.conf'
if self.opts.dry_run:
log.debug("Installprogress: this is a dry-run")
config_location = os.path.join('.subiquity/', config_file_name)
curtin_cmd = ["python3", "scripts/replay-curtin-log.py",
"examples/curtin-events.json",
self._event_syslog_identifier]
else:
log.debug("Installprogress: this is the *REAL* thing")
config_location = os.path.join('/var/log/installer',
config_file_name)
curtin_cmd = [sys.executable, '-m', 'curtin', '--showtrace', '-c',
config_location, 'install']
ident = self._event_syslog_identifier
self._write_config(config_location,
self.base_model.render(syslog_identifier=ident))
return curtin_cmd
def curtin_start_install(self):
log.debug('Curtin Install: starting curtin')
self.install_state = InstallState.RUNNING
self.footer_description = urwid.Text(_("starting..."))
self.progress_view = ProgressView(self)
self.footer_spinner = self.progress_view.spinner
self.ui.auto_footer = False
self.ui.set_footer(urwid.Columns(
[('pack', urwid.Text(_("Install in progress:"))),
(self.footer_description),
('pack', self.footer_spinner)], dividechars=1))
self.journal_listener_handle = self.start_journald_listener(
[self._event_syslog_identifier, self._log_syslog_identifier],
self._journal_event)
curtin_cmd = self._get_curtin_command()
log.debug('Curtin install cmd: {}'.format(curtin_cmd))
self.run_in_bg(
lambda: self._bg_run_command_logged(curtin_cmd),
self.curtin_install_completed)
def curtin_install_completed(self, fut):
cp = fut.result()
log.debug('curtin_install completed: %s', cp.returncode)
if cp.returncode != 0:
self.curtin_error()
return
self.install_state = InstallState.DONE
log.debug('After curtin install OK')
self.ui.progress_current += 1
if not self.progress_view_showing:
self.ui.set_footer(_("Install complete"))
else:
# Re-set footer so progress bar updates.
self.ui.set_footer(_("Thank you for using Ubuntu!"))
self._step_done('install')
def cancel(self):
pass
def _bg_install_openssh_server(self):
if self.opts.dry_run:
cmd = [
"sleep", str(2/self.scale_factor),
]
else:
cmd = [
sys.executable, "-m", "curtin", "system-install", "-t",
"/target",
"--", "openssh-server",
]
self._bg_run_command_logged(cmd)
def _bg_cleanup_apt(self):
if self.opts.dry_run:
cmd = [
"sleep", str(2/self.scale_factor),
]
else:
os.unlink(
os.path.join(
self.base_model.target, "etc/apt/sources.list.d/iso.list"))
cmd = [
sys.executable, "-m", "curtin", "in-target", "-t", "/target",
"--", "apt-get", "update",
]
self._bg_run_command_logged(cmd)
def start_postinstall_configuration(self):
self.copy_logs_to_target()
class w(TaskWatcher):
def __init__(self, controller):
self.controller = controller
def task_complete(self, stage):
pass
def task_error(self, stage, info):
if isinstance(info, tuple):
tb = traceback.format_exception(*info)
self.controller.curtin_error("".join(tb))
else:
self.controller.curtin_error()
def tasks_finished(self):
self.controller.loop.set_alarm_in(
0.0,
lambda loop, ud: self.controller.postinstall_complete())
tasks = [
('drain', WaitForCurtinEventsTask(self)),
('cloud-init', InstallTask(
self, "configuring cloud-init",
self.base_model.configure_cloud_init)),
]
if self.base_model.ssh.install_server:
tasks.extend([
('install-ssh', InstallTask(
self, "installing OpenSSH server",
self._bg_install_openssh_server)),
])
tasks.extend([
('cleanup', InstallTask(
self, "cleaning up apt configuration",
self._bg_cleanup_apt)),
])
ts = TaskSequence(self.run_in_bg, tasks, w(self))
ts.run()
def postinstall_complete(self):
self._install_event_finish()
self.ui.set_header(_("Installation complete!"))
self.progress_view.set_status(_("Finished install!"))
self.progress_view.show_complete()
if self.answers['reboot']:
self.reboot()
def copy_logs_to_target(self):
if self.opts.dry_run:
return
target_logs = os.path.join(self.base_model.target, 'var/log/installer')
utils.run_command(['cp', '-aT', '/var/log/installer', target_logs])
try:
with open(os.path.join(target_logs,
'installer-journal.txt'), 'w') as output:
utils.run_command(
['journalctl'],
stdout=output, stderr=subprocess.STDOUT)
except Exception:
log.exception("saving journal failed")
def reboot(self):
if self.opts.dry_run:
log.debug('dry-run enabled, skipping reboot, quiting instead')
self.signal.emit_signal('quit')
else:
# TODO Possibly run this earlier, to show a warning; or
# switch to shutdown if chreipl fails
if platform.machine() == 's390x':
utils.run_command(["chreipl", "/target/boot"])
# Should probably run curtin -c $CONFIG unmount -t TARGET first.
utils.run_command(["/sbin/reboot"])
def quit(self):
if not self.opts.dry_run:
utils.disable_subiquity()
self.signal.emit_signal('quit')
def default(self):
self.progress_view_showing = True
if self.install_state == InstallState.RUNNING:
self.progress_view.title = _("Installing system")
footer = _("Thank you for using Ubuntu!")
elif self.install_state == InstallState.DONE:
self.progress_view.title = _("Install complete!")
footer = _("Thank you for using Ubuntu!")
elif self.install_state == InstallState.ERROR:
self.progress_view.title = (
_('An error occurred during installation'))
footer = _('Please report this error in Launchpad')
self.ui.set_body(self.progress_view)
self.ui.set_footer(footer)
class InstallProgressController(SubiquityController):
def __init__(self, app):
super().__init__(app)
self.model = app.base_model
self.progress_view = ProgressView(self)
self.install_state = InstallState.NOT_STARTED
self.journal_listener_handle = None
self.reboot_clicked = asyncio.Event()
if self.answers.get('reboot', False):
self.reboot_clicked.set()
self.uu_running = False
self.uu = None
self._event_indent = ""
self._event_syslog_identifier = 'curtin_event.%s' % (os.getpid(), )
self._log_syslog_identifier = 'curtin_log.%s' % (os.getpid(), )
self.tb_extractor = TracebackExtractor()
self.curtin_context = None
def interactive(self):
return self.app.interactive()
def start(self):
self.install_task = schedule_task(self.install(self.context))
async def apply_autoinstall_config(self):
await self.install_task
self.app.reboot_on_exit = True
def tpath(self, *path):
return os.path.join(self.model.target, *path)
def curtin_error(self):
self.install_state = InstallState.ERROR
kw = {}
if sys.exc_info()[0] is not None:
log.exception("curtin_error")
self.progress_view.add_log_line(traceback.format_exc())
if self.tb_extractor.traceback:
kw["Traceback"] = "\n".join(self.tb_extractor.traceback)
crash_report = self.app.make_apport_report(
ErrorReportKind.INSTALL_FAIL,
"install failed",
interrupt=False,
**kw)
self.progress_view.spinner.stop()
self.progress_view.set_status(
('info_error', _("An error has occurred")))
self.start_ui()
self.progress_view.show_error(crash_report)
def logged_command(self, cmd):
return [
'systemd-cat', '--level-prefix=false',
'--identifier=' + self._log_syslog_identifier
] + cmd
def _journal_event(self, event):
if event['SYSLOG_IDENTIFIER'] == self._event_syslog_identifier:
self.curtin_event(event)
elif event['SYSLOG_IDENTIFIER'] == self._log_syslog_identifier:
self.curtin_log(event)
@contextlib.contextmanager
def install_context(self, context, name, description, level, childlevel):
self._install_event_start(description)
try:
subcontext = context.child(name, description, level, childlevel)
with subcontext:
yield subcontext
finally:
self._install_event_finish()
def _install_event_start(self, message):
self.progress_view.add_event(self._event_indent + message)
self._event_indent += " "
self.progress_view.spinner.start()
def _install_event_finish(self):
self._event_indent = self._event_indent[:-2]
self.progress_view.spinner.stop()
def curtin_event(self, event):
e = {
"EVENT_TYPE": "???",
"MESSAGE": "???",
"NAME": "???",
"RESULT": "???",
}
prefix = "CURTIN_"
for k, v in event.items():
if k.startswith(prefix):
e[k[len(prefix):]] = v
event_type = e["EVENT_TYPE"]
if event_type == 'start':
self._install_event_start(e["MESSAGE"])
if self.curtin_context is not None:
self.curtin_context.child(e["NAME"], e["MESSAGE"]).enter()
if event_type == 'finish':
self._install_event_finish()
status = getattr(Status, e["RESULT"], Status.WARN)
if self.curtin_context is not None:
self.curtin_context.child(e["NAME"], e["MESSAGE"]).exit(status)
def curtin_log(self, event):
log_line = event['MESSAGE']
self.progress_view.add_log_line(log_line)
self.tb_extractor.feed(log_line)
def start_journald_listener(self, identifiers, callback):
reader = journal.Reader()
args = []
for identifier in identifiers:
args.append("SYSLOG_IDENTIFIER={}".format(identifier))
reader.add_match(*args)
def watch():
if reader.process() != journal.APPEND:
return
for event in reader:
callback(event)
loop = asyncio.get_event_loop()
return loop.add_reader(reader.fileno(), watch)
def _write_config(self, path, config):
with open(path, 'w') as conf:
datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
str(datetime.datetime.utcnow()))
conf.write(datestr)
conf.write(yaml.dump(config))
def _get_curtin_command(self):
config_file_name = 'subiquity-curtin-install.conf'
if self.opts.dry_run:
config_location = os.path.join('.subiquity/', config_file_name)
log_location = '.subiquity/install.log'
event_file = "examples/curtin-events.json"
if 'install-fail' in self.app.debug_flags:
event_file = "examples/curtin-events-fail.json"
curtin_cmd = [
"python3",
"scripts/replay-curtin-log.py",
event_file,
self._event_syslog_identifier,
log_location,
]
else:
config_location = os.path.join('/var/log/installer',
config_file_name)
curtin_cmd = [
sys.executable, '-m', 'curtin', '--showtrace', '-c',
config_location, 'install'
]
log_location = INSTALL_LOG
ident = self._event_syslog_identifier
self._write_config(config_location,
self.model.render(syslog_identifier=ident))
self.app.note_file_for_apport("CurtinConfig", config_location)
self.app.note_file_for_apport("CurtinLog", log_location)
self.app.note_file_for_apport("CurtinErrors", ERROR_TARFILE)
return curtin_cmd
@install_step("installing system", level="INFO", childlevel="DEBUG")
async def curtin_install(self, context):
log.debug('curtin_install')
self.install_state = InstallState.RUNNING
self.curtin_context = context
self.journal_listener_handle = self.start_journald_listener(
[self._event_syslog_identifier, self._log_syslog_identifier],
self._journal_event)
curtin_cmd = self._get_curtin_command()
log.debug('curtin install cmd: {}'.format(curtin_cmd))
cp = await arun_command(self.logged_command(curtin_cmd), check=True)
log.debug('curtin_install completed: %s', cp.returncode)
self.install_state = InstallState.DONE
log.debug('After curtin install OK')
def cancel(self):
pass
async def install(self, context):
try:
await asyncio.wait({e.wait() for e in self.model.install_events})
await self.curtin_install(context)
await asyncio.wait(
{e.wait()
for e in self.model.postinstall_events})
await self.drain_curtin_events(context)
await self.postinstall(context)
self.ui.set_header(_("Installation complete!"))
self.progress_view.set_status(_("Finished install!"))
self.progress_view.show_complete()
if self.model.network.has_network:
self.progress_view.update_running()
await self.run_unattended_upgrades(context)
self.progress_view.update_done()
await self.copy_logs_to_target(context)
except Exception:
self.curtin_error()
if not self.interactive():
raise
async def move_on(self):
await asyncio.wait({self.reboot_clicked.wait(), self.install_task})
self.app.reboot_on_exit = True
if not self.opts.dry_run and platform.machine() == 's390x':
run_command(["chreipl", "/target/boot"])
self.app.next_screen()
async def drain_curtin_events(self, context):
waited = 0.0
while self._event_indent and waited < 5.0:
await asyncio.sleep(0.1)
waited += 0.1
log.debug("waited %s seconds for events to drain", waited)
self.curtin_context = None
@install_step("final system configuration",
level="INFO",
childlevel="DEBUG")
async def postinstall(self, context):
await self.configure_cloud_init(context)
if self.model.ssh.install_server:
await self.install_openssh(context)
await self.restore_apt_config(context)
@install_step("configuring cloud-init")
async def configure_cloud_init(self, context):
await run_in_thread(self.model.configure_cloud_init)
@install_step("installing openssh")
async def install_openssh(self, context):
if self.opts.dry_run:
cmd = ["sleep", str(2 / self.app.scale_factor)]
else:
cmd = [
sys.executable,
"-m",
"curtin",
"system-install",
"-t",
"/target",
"--",
"openssh-server",
]
await arun_command(self.logged_command(cmd), check=True)
@install_step("restoring apt configuration")
async def restore_apt_config(self, context):
if self.opts.dry_run:
cmds = [["sleep", str(1 / self.app.scale_factor)]]
else:
cmds = [
["umount", self.tpath('etc/apt')],
]
if self.model.network.has_network:
cmds.append([
sys.executable,
"-m",
"curtin",
"in-target",
"-t",
"/target",
"--",
"apt-get",
"update",
])
else:
cmds.append(["umount", self.tpath('var/lib/apt/lists')])
for cmd in cmds:
await arun_command(self.logged_command(cmd), check=True)
@install_step("downloading and installing security updates")
async def run_unattended_upgrades(self, context):
target_tmp = os.path.join(self.model.target, "tmp")
os.makedirs(target_tmp, exist_ok=True)
apt_conf = tempfile.NamedTemporaryFile(dir=target_tmp,
delete=False,
mode='w')
apt_conf.write(uu_apt_conf)
apt_conf.close()
env = os.environ.copy()
env["APT_CONFIG"] = apt_conf.name[len(self.model.target):]
self.uu_running = True
if self.opts.dry_run:
self.uu = await astart_command(self.logged_command(
["sleep", str(5 / self.app.scale_factor)]),
env=env)
else:
self.uu = await astart_command(self.logged_command([
sys.executable,
"-m",
"curtin",
"in-target",
"-t",
"/target",
"--",
"unattended-upgrades",
"-v",
]),
env=env)
await self.uu.communicate()
self.uu_running = False
self.uu = None
os.remove(apt_conf.name)
async def stop_uu(self):
self._install_event_finish()
self._install_event_start("cancelling update")
if self.opts.dry_run:
await asyncio.sleep(1)
self.uu.terminate()
else:
await arun_command(
self.logged_command([
'chroot',
'/target',
'/usr/share/unattended-upgrades/unattended-upgrade-shutdown',
'--stop-only',
],
check=True))
@install_step("copying logs to installed system")
async def copy_logs_to_target(self, context):
if self.opts.dry_run:
if 'copy-logs-fail' in self.app.debug_flags:
raise PermissionError()
return
target_logs = self.tpath('var/log/installer')
await arun_command(['cp', '-aT', '/var/log/installer', target_logs])
try:
with open(os.path.join(target_logs, 'installer-journal.txt'),
'w') as output:
await arun_command(['journalctl'],
stdout=output,
stderr=subprocess.STDOUT)
except Exception:
log.exception("saving journal failed")
async def _click_reboot(self):
if self.uu_running:
await self.stop_uu()
self.reboot_clicked.set()
def click_reboot(self):
schedule_task(self._click_reboot())
def start_ui(self):
if self.install_state in [
InstallState.NOT_STARTED,
InstallState.RUNNING,
]:
self.progress_view.title = _("Installing system")
elif self.install_state == InstallState.DONE:
self.progress_view.title = _("Install complete!")
elif self.install_state == InstallState.ERROR:
self.progress_view.title = (
_('An error occurred during installation'))
self.ui.set_body(self.progress_view)
schedule_task(self.move_on())