def build_widget(self):
self.step_pile = pile = Pile([
Columns([
('fixed', 3, self.icon),
self.description,
],
dividechars=1),
Padding.line_break(""),
Padding.push_4(self.output),
])
if utils.is_linux() and self.model.needs_sudo:
pile.contents.append((Padding.line_break(""), pile.options()))
label = 'This step requires sudo.'
if not self.app.sudo_pass:
label += ' Enter sudo password, if needed:'
self.sudo_input = PasswordEditor()
columns = [
('weight', 0.5, Padding.left(Text(('body', label)), left=5)),
]
if self.sudo_input:
columns.append(
('weight', 1,
Color.string_input(self.sudo_input,
focus_map='string_input focus')))
pile.contents.append((Columns(columns,
dividechars=3), pile.options()))
async def do_steps(self):
step_metas = common.get_step_metadata_filenames()
results = OrderedDict()
for step_meta_path in step_metas:
step_model = common.load_step(step_meta_path)
if utils.is_linux():
if step_model.needs_sudo and not utils.can_sudo():
utils.error("Step requires passwordless sudo: {}".format(
step_model.title))
events.Shutdown.set(1)
return
results[step_model.title] = await common.do_step(step_model,
utils.info)
events.PostDeployComplete.set()
return controllers.use('summary').render(results)
def get_compatible_clouds(cloud_types=None):
""" List cloud types compatible with the current spell and controller.
Arguments:
clouds: optional initial list of clouds to filter
Returns:
List of cloud types
"""
if cloud_types is None:
clouds = get_clouds()
cloud_types = set(c['type'] for c in clouds.values())
# custom providers don't show up in list-clouds but are valid types
cloud_types |= set(consts.CUSTOM_PROVIDERS)
else:
cloud_types = set(cloud_types)
_normalize_cloud_types(cloud_types)
if not is_linux():
# LXD not available on macOS
cloud_types -= {'localhost'}
if app.provider and app.provider.controller:
# if we already have a controller, we should query
# it via the API for what clouds it supports; for now,
# though, just assume it's JAAS and hard-code the options
cloud_types &= consts.JAAS_CLOUDS
whitelist = set(app.metadata.cloud_whitelist)
blacklist = set(app.metadata.cloud_blacklist)
addons_dir = Path(app.config['spell-dir']) / 'addons'
for addon in app.selected_addons:
addon_file = addons_dir / addon / 'metadata.yaml'
addon_meta = yaml.safe_load(addon_file.read_text())
whitelist.update(addon_meta.get('cloud-whitelist', []))
blacklist.update(addon_meta.get('cloud-blacklist', []))
_normalize_cloud_types(whitelist)
_normalize_cloud_types(blacklist)
if len(whitelist) > 0:
return sorted(cloud_types & whitelist)
elif len(blacklist) > 0:
return sorted(cloud_types ^ blacklist)
return sorted(cloud_types)
def generate_additional_input(self):
""" Generates additional input fields, useful for doing it after
a previous step is run
"""
self.set_description(self.model.description, 'body')
self.icon.set_text(('pending_icon', self.icon.get_text()[0]))
if utils.is_linux() and self.model.needs_sudo:
self.step_pile.contents.append(
(Padding.line_break(""), self.step_pile.options()))
can_sudo = utils.can_sudo()
label = 'This step requires sudo.'
if not can_sudo:
label += ' Please enter sudo password:'******'weight', 0.5, Padding.left(Text(('body', label)), left=5)),
]
if not can_sudo:
self.sudo_input = PasswordEditor()
columns.append(
('weight', 1,
Color.string_input(self.sudo_input,
focus_map='string_input focus')))
self.step_pile.contents.append(
(Columns(columns, dividechars=3), self.step_pile.options()))
for i in self.additional_input:
self.app.log.debug(i)
self.step_pile.contents.append(
(Padding.line_break(""), self.step_pile.options()))
column_input = [('weight', 0.5, Padding.left(i['label'], left=5))]
if i['input']:
column_input.append(
('weight', 1,
Color.string_input(i['input'],
focus_map='string_input focus')))
self.step_pile.contents.append(
(Columns(column_input,
dividechars=3), self.step_pile.options()))
self.button = submit_btn(label="Run", on_press=self.submit)
self.step_pile.contents.append(
(Padding.line_break(""), self.step_pile.options()))
self.step_pile.contents.append((Text(""), self.step_pile.options()))
self.step_pile.contents.append((HR(), self.step_pile.options()))
self.show_button()
self.step_pile.focus_position = self.current_button_index
def next_step(self, step_model, step_widget):
""" handles processing step with input data
Arguments:
step_model: step_model returned from widget
done: if True continues on to the summary view
"""
if utils.is_linux() and step_model.needs_sudo:
password = None
if step_widget.sudo_input:
password = step_widget.sudo_input.value
if not utils.can_sudo(password):
step_widget.set_error(
'Sudo failed. Please check your password and ensure that '
'your sudo timeout is not set to zero.')
step_widget.show_button()
return
step_widget.clear_error()
step_widget.clear_button()
step_widget.set_icon_state('waiting')
# Set next button focus here now that the step is complete.
self.view.steps.popleft()
next_step = None
if len(self.view.steps) > 0:
next_step = self.view.steps[0]
next_step.generate_additional_input()
self.view.step_pile.focus_position = self.view.step_pile.focus_position + 1 # noqa
# merge the step_widget input data into our step model
for m in step_model.additional_input:
try:
matching_widget = next(
filter(lambda i: i['key'] == m['key'],
step_widget.additional_input))
m['input'] = matching_widget['input'].value
except StopIteration:
app.log.error("Model field has no input: {}".format(m['key']))
continue
app.loop.create_task(self._put(step_model, step_widget, next_step))
def _build_sudo_field(self):
if not utils.is_linux() or not self.model.needs_sudo:
return []
rows = []
if not self.app.sudo_pass:
self.sudo_input = PasswordEditor()
self.clear_sudo_error()
columns = [
('weight', 0.5, Padding.left(self.sudo_label, left=5)),
]
if self.sudo_input:
columns.append(
('weight', 1,
Filler(Color.string_input(self.sudo_input,
focus_map='string_input focus'),
valign='bottom')))
rows.extend([
Padding.line_break(""),
Columns(columns, dividechars=3, box_columns=[1]),
])
return rows
def render(self):
for step_meta_path in self.step_metas:
try:
model = common.load_step(step_meta_path)
except common.ValidationError as e:
app.log.error(e.msg)
utils.error(e.msg)
sys.exit(1)
if utils.is_linux() and model.needs_sudo and not model.can_sudo():
utils.error("Step requires passwordless sudo: {}".format(
model.title))
sys.exit(1)
app.log.debug("Running step: {}".format(model))
try:
step_model, _ = common.do_step(model,
None,
utils.info)
self.results[step_model.title] = step_model.result
except Exception as e:
utils.error("Failed to run {}: {}".format(model.path, e))
sys.exit(1)
self.finish()
def get_compatible_clouds(cloud_types=None):
""" List cloud types compatible with the current spell and controller.
Arguments:
clouds: optional initial list of clouds to filter
Returns:
List of cloud types
"""
clouds = get_clouds()
cloud_types = set(cloud_types or (c['type'] for c in clouds.values()))
if 'lxd' in cloud_types:
# normalize 'lxd' cloud type to localhost; 'lxd' can happen
# depending on how the controller was bootstrapped
cloud_types -= {'lxd'}
cloud_types |= {'localhost'}
if not is_linux():
# LXD not available on macOS
cloud_types -= {'localhost'}
if app.current_controller:
# if we already have a controller, we should query
# it via the API for what clouds it supports; for now,
# though, just assume it's JAAS and hard-code the options
cloud_types &= consts.JAAS_CLOUDS
whitelist = set(app.config['metadata'].get('cloud-whitelist', []))
blacklist = set(app.config['metadata'].get('cloud-blacklist', []))
if len(whitelist) > 0:
return sorted(cloud_types & whitelist)
elif len(blacklist) > 0:
return sorted(cloud_types ^ blacklist)
return sorted(cloud_types)
async def run(self, msg_cb, event_name=None):
# Define STEP_NAME for use in determining where to store
# our step results,
# state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val"
app.env['CONJURE_UP_STEP'] = self.name
step_path = Path(app.config['spell-dir']) / 'steps' / self.filename
if not step_path.is_file():
return
step_path = str(step_path)
msg = "Running step: {}.".format(self.name)
app.log.info(msg)
msg_cb(msg)
if event_name is not None:
track_event(event_name, "Started", "")
if not os.access(step_path, os.X_OK):
raise Exception("Step {} not executable".format(self.title))
if is_linux() and self.needs_sudo and not await can_sudo():
raise SudoError('Step "{}" requires sudo: {}'.format(
self.title,
'password failed'
if app.sudo_pass else 'passwordless sudo required',
))
cloud_types = juju.get_cloud_types_by_name()
provider_type = cloud_types[app.provider.cloud]
app.env['JUJU_PROVIDERTYPE'] = provider_type
# not all providers have a credential, e.g., localhost
app.env['JUJU_CREDENTIAL'] = app.provider.credential or ''
app.env['JUJU_CONTROLLER'] = app.provider.controller
app.env['JUJU_MODEL'] = app.provider.model
app.env['JUJU_REGION'] = app.provider.region or ''
app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir
if provider_type == "maas":
app.env['MAAS_ENDPOINT'] = app.maas.endpoint
app.env['MAAS_APIKEY'] = app.maas.api_key
for step_name, step_data in app.steps_data.items():
for key, value in step_data.items():
app.env[key.upper()] = step_data[key]
for key, value in app.env.items():
if value is None:
app.log.warning('Env {} is None; '
'replacing with empty string'.format(key))
app.env[key] = ''
app.log.debug("Storing environment")
async with aiofiles.open(step_path + ".env", 'w') as outf:
for k, v in app.env.items():
if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k:
await outf.write("{}=\"{}\" ".format(k.upper(), v))
app.log.debug("Executing script: {}".format(step_path))
async with aiofiles.open(step_path + ".out", 'w') as outf:
async with aiofiles.open(step_path + ".err", 'w') as errf:
proc = await asyncio.create_subprocess_exec(step_path,
env=app.env,
stdout=outf,
stderr=errf)
async with aiofiles.open(step_path + '.out', 'r') as f:
while proc.returncode is None:
async for line in f:
msg_cb(line)
await asyncio.sleep(0.01)
out_log = Path(step_path + '.out').read_text()
err_log = Path(step_path + '.err').read_text()
if proc.returncode != 0:
app.sentry.context.merge({
'extra': {
'out_log_tail': out_log[-400:],
'err_log_tail': err_log[-400:],
}
})
raise Exception("Failure in step {}".format(self.filename))
# special case for 00_deploy-done to report masked
# charm hook failures that were retried automatically
if not app.noreport:
failed_apps = set() # only report each charm once
for line in err_log.splitlines():
if 'hook failure, will retry' in line:
log_leader = line.split()[0]
unit_name = log_leader.split(':')[-1]
app_name = unit_name.split('/')[0]
failed_apps.add(app_name)
for app_name in failed_apps:
# report each individually so that Sentry will give us a
# breakdown of failures per-charm in addition to per-spell
sentry_report('Retried hook failure',
tags={
'app_name': app_name,
})
if event_name is not None:
track_event(event_name, "Done", "")
result_key = "conjure-up.{}.{}.result".format(app.config['spell'],
self.name)
result = app.state.get(result_key)
return (result or '')
async def run(self, phase, msg_cb, event_name=None):
# Define STEP_NAME for use in determining where to store
# our step results,
# state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val"
app.env['CONJURE_UP_STEP'] = self.name
app.env['CONJURE_UP_PHASE'] = phase.value
step_path = self._build_phase_path(phase)
if not step_path.is_file():
return
if not os.access(str(step_path), os.X_OK):
app.log.error(
'Unable to run {} step {} {}, it is not executable'.format(
self.source, step_path.stem, phase.value))
return
step_path = str(step_path)
msg = "Running {} step: {} {}.".format(self.source, self.name,
phase.value)
app.log.info(msg)
msg_cb(msg)
if event_name is not None:
track_event(event_name, "Started", "")
if is_linux() and self.needs_sudo and not await can_sudo():
raise SudoError('The "{}" step "{}" requires sudo: {}'.format(
self.source,
self.title,
'password failed'
if app.sudo_pass else 'passwordless sudo required',
))
app.env['CONJURE_UP_SPELLSDIR'] = app.conjurefile['spells-dir']
app.env['CONJURE_UP_SESSION_ID'] = app.session_id
if app.metadata.spell_type == spell_types.JUJU:
cloud_types = juju.get_cloud_types_by_name()
provider_type = cloud_types[app.provider.cloud]
app.env['JUJU_PROVIDERTYPE'] = provider_type
# not all providers have a credential, e.g., localhost
app.env['JUJU_CREDENTIAL'] = app.provider.credential or ''
app.env['JUJU_CONTROLLER'] = app.provider.controller
app.env['JUJU_MODEL'] = app.provider.model
app.env['JUJU_REGION'] = app.provider.region or ''
if provider_type == "maas":
app.env['MAAS_ENDPOINT'] = app.maas.endpoint
app.env['MAAS_APIKEY'] = app.maas.api_key
for step_name, step_data in app.steps_data.items():
for key, value in step_data.items():
app.env[key.upper()] = str(step_data[key])
for key, value in app.env.items():
if value is None:
app.log.warning('Env {} is None; '
'replacing with empty string'.format(key))
app.env[key] = ''
app.log.debug("Storing environment")
async with aiofiles.open(step_path + ".env", 'w') as outf:
for k, v in app.env.items():
if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k:
await outf.write("{}=\"{}\" ".format(k.upper(), v))
app.log.debug("Executing script: {}".format(step_path))
out_path = step_path + '.out'
err_path = step_path + '.err'
ret, out_log, err_log = await arun([step_path],
stdout=out_path,
stderr=err_path,
cb_stdout=msg_cb)
if ret != 0:
app.sentry.context.merge({
'extra': {
'out_log_tail': out_log[-400:],
'err_log_tail': err_log[-400:],
}
})
raise Exception("Failure in step {} {}".format(
self.name, phase.value))
# special case for 00_deploy-done to report masked
# charm hook failures that were retried automatically
if not app.no_report and app.metadata.spell_type == spell_types.JUJU:
failed_apps = set() # only report each charm once
for line in err_log.splitlines():
if 'hook failure, will retry' in line:
log_leader = line.split()[0]
unit_name = log_leader.split(':')[-1]
app_name = unit_name.split('/')[0]
failed_apps.add(app_name)
for app_name in failed_apps:
# report each individually so that Sentry will give us a
# breakdown of failures per-charm in addition to per-spell
sentry_report('Retried hook failure',
tags={
'app_name': app_name,
})
if event_name is not None:
track_event(event_name, "Done", "")
return self.get_state('result', phase)
