def render(self):
creds = None
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type != 'localhost':
creds = common.try_get_creds(app.current_cloud)
if not creds:
utils.warning("You attempted to do an install against a cloud "
"that requires credentials that could not be "
"found. If you wish to supply those "
"credentials please run "
"`juju add-credential "
"{}`.".format(app.current_cloud))
events.Shutdown.set(1)
return
# LXD is a special case as we want to make sure a bridge
# is configured. If not we'll bring up a new view to allow
# a user to configure a LXD bridge with suggested network
# information.
if cloud_type == 'localhost':
lxd = common.is_lxd_ready()
if not lxd['ready']:
return controllers.use('lxdsetup').render(lxd['msg'])
app.loop.create_task(self.finish(creds))
def render(self, going_back=False):
"Pick or create a cloud to bootstrap a new controller on"
all_clouds = juju.get_clouds()
compatible_clouds = juju.get_compatible_clouds()
cloud_types = juju.get_cloud_types_by_name()
# filter to only public clouds
public_clouds = sorted(name for name, info in all_clouds.items()
if info['defined'] == 'public')
# filter to custom clouds
# exclude localhost because we treat that as "configuring a new cloud"
custom_clouds = sorted(name for name, info in all_clouds.items()
if info['defined'] != 'public'
and cloud_types[name] != 'localhost')
prev_screen = self.prev_screen
if app.alias_given or (app.spell_given and not app.addons):
# we were given an alias (so spell and addons are locked)
# or we were given a spell and there are no addons to change
# so we disable the back button
prev_screen = None
self.view = CloudView(app,
public_clouds,
custom_clouds,
compatible_clouds,
cb=self.finish,
back=prev_screen)
if 'localhost' in compatible_clouds:
app.log.debug(
"Starting watcher for verifying LXD server is available.")
self.cancel_monitor.clear()
app.loop.create_task(self._monitor_localhost(LocalhostProvider()))
self.view.show()
async def ensure_machines(self, application):
"""If 'application' is assigned to any machine that haven't been added yet,
add the machines prior to deployment.
"""
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type == 'maas':
await events.MAASConnected.wait()
app_placements = self.get_all_assignments(application)
juju_machines = app.metadata_controller.bundle.machines
machines = {}
for virt_machine_id, _ in app_placements:
if virt_machine_id in self.deployed_juju_machines:
continue
machine_attrs = {
'series': application.csid.series,
}
if cloud_type == 'maas':
machine_attrs['constraints'] = \
await self.get_maas_constraints(virt_machine_id)
else:
machine_attrs.update(juju_machines[virt_machine_id])
machines[virt_machine_id] = machine_attrs
return await juju.add_machines([application],
machines,
msg_cb=app.ui.set_footer)
def render(self):
"Pick or create a cloud to bootstrap a new controller on"
track_screen("Cloud Select")
all_clouds = juju.get_clouds()
compatible_clouds = juju.get_compatible_clouds()
cloud_types = juju.get_cloud_types_by_name()
# filter to only public clouds
public_clouds = sorted(
name for name, info in all_clouds.items()
if info['defined'] == 'public' and
cloud_types[name] in compatible_clouds)
# filter to custom clouds
# exclude localhost because we treat that as "configuring a new cloud"
custom_clouds = sorted(
name for name, info in all_clouds.items()
if info['defined'] != 'public' and
cloud_types[name] != 'localhost' and
cloud_types[name] in compatible_clouds)
excerpt = app.config.get(
'description',
"Where would you like to deploy?")
view = CloudView(app,
public_clouds,
custom_clouds,
cb=self.finish)
app.ui.set_header(
title="Choose a Cloud",
excerpt=excerpt
)
app.ui.set_body(view)
app.ui.set_footer('Please press [ENTER] on highlighted '
'Cloud to proceed.')
async def do_deploy(msg_cb):
await events.ModelConnected.wait()
cloud_types = juju.get_cloud_types_by_name()
default_series = app.metadata_controller.series
machines = app.metadata_controller.bundle.machines
applications = sorted(app.metadata_controller.bundle.services,
key=attrgetter('service_name'))
await pre_deploy(msg_cb=msg_cb)
machine_map = await juju.add_machines(applications,
machines,
msg_cb=msg_cb)
tasks = []
for service in applications:
if cloud_types[app.provider.cloud] == "localhost":
# ignore placement when deploying to localhost
service.placement_spec = None
elif service.placement_spec:
# remap machine references to actual deployed machine IDs
# (they will only ever not already match if deploying to
# an existing model that has other machines)
new_placements = []
for plabel in service.placement_spec:
if ':' in plabel:
ptype, pid = plabel.split(':')
new_placements.append(':'.join([ptype, machine_map[pid]]))
else:
new_placements.append(machine_map[plabel])
service.placement_spec = new_placements
tasks.append(
juju.deploy_service(service, default_series, msg_cb=msg_cb))
tasks.append(juju.set_relations(service, msg_cb=msg_cb))
await asyncio.gather(*tasks)
events.DeploymentComplete.set()
async def ensure_machines(self, application):
"""If 'application' is assigned to any machine that haven't been added yet,
add the machines prior to deployment.
Note: This no longer actually creates the machine, since the
configuration will be filled out before the controller is bootstrapped.
Instead, it just ensures that the data in app.metadata_controller is
up to date. This needs to be refactored at a later date.
"""
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type == 'maas':
await events.MAASConnected.wait()
app_placements = self.get_all_assignments(application)
juju_machines = app.metadata_controller.bundle.machines
machines = {}
for virt_machine_id, _ in app_placements:
if virt_machine_id in self.deployed_juju_machines:
continue
machine_attrs = {
'series': application.csid.series,
}
if cloud_type == 'maas':
machine_attrs['constraints'] = \
await self.get_maas_constraints(virt_machine_id)
else:
machine_attrs.update(juju_machines[virt_machine_id])
machines[virt_machine_id] = machine_attrs
# store the updated constraints back in the metadata
# the actual machine creation will happen later, during deploy
# we have to reassign it to the bundle because the getter can
# return a new empty dict without storing it back in the bundle
juju_machines.update(machines)
app.metadata_controller.bundle.machines = juju_machines
def build_widgets(self):
cloud_type = get_cloud_types_by_name()[app.current_cloud]
controller_is_maas = cloud_type == 'maas'
if controller_is_maas:
extra = (" Press enter on a machine ID to pin it to "
"a specific MAAS node.")
else:
extra = ""
ws = [
Text("Choose where to place {} unit{} of {}.{}".format(
self.application.num_units,
"" if self.application.num_units == 1 else "s",
self.application.service_name, extra))
]
self.juju_machines_list = JujuMachinesList(
self.application,
self._machines,
self.do_assign,
self.do_unassign,
self.add_machine,
self.remove_machine,
self,
show_filter_box=True,
show_pins=controller_is_maas)
ws.append(self.juju_machines_list)
self.pile = Pile(ws)
return Padding.center_90(Filler(self.pile, valign="top"))
def render(self):
existing_controllers = juju.get_controllers()['controllers']
clouds = juju.get_compatible_clouds()
cloud_types = juju.get_cloud_types_by_name()
app.jaas_ok = set(clouds) & JAAS_CLOUDS
jaas_controller = {
n
for n, c in existing_controllers.items()
if JAAS_ENDPOINT in c['api-endpoints']
}
if jaas_controller:
app.jaas_controller = jaas_controller.pop()
filtered_controllers = {
n: d
for n, d in existing_controllers.items()
if cloud_types.get(d['cloud']) in clouds
}
if not app.jaas_ok and len(filtered_controllers) == 0:
return controllers.use('clouds').render()
track_screen("Controller Picker")
excerpt = app.config.get(
'description', "Please select an existing controller,"
" or choose to bootstrap a new one.")
view = ControllerListView(app, filtered_controllers, self.finish)
app.ui.set_header(title="Choose a Controller or Create new",
excerpt=excerpt)
app.ui.set_body(view)
def finish(self, credentials=None, region=None, back=False):
if region:
app.current_region = region
if back:
return controllers.use('clouds').render()
if credentials is not None:
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type == 'maas':
# Now that we are passed the selection of a cloud we create a
# new cloud name for the remainder of the deployment and make
# sure this cloud is saved for future use.
app.current_cloud = utils.gen_cloud()
# Save credentials for new cloud
common.save_creds(app.current_cloud, credentials)
try:
juju.get_cloud(app.current_cloud)
except LookupError:
juju.add_cloud(app.current_cloud,
credentials.cloud_config())
else:
common.save_creds(app.current_cloud, credentials)
credentials_key = common.try_get_creds(app.current_cloud)
app.loop.create_task(
common.do_bootstrap(credentials_key,
msg_cb=app.ui.set_footer,
fail_msg_cb=lambda e: None))
controllers.use('deploy').render()
def render(self):
"Pick or create a cloud to bootstrap a new controller on"
track_screen("Cloud Select")
all_clouds = juju.get_clouds()
compatible_clouds = juju.get_compatible_clouds()
cloud_types = juju.get_cloud_types_by_name()
# filter to only public clouds
public_clouds = sorted(name for name, info in all_clouds.items()
if info['defined'] == 'public')
# filter to custom clouds
# exclude localhost because we treat that as "configuring a new cloud"
custom_clouds = sorted(name for name, info in all_clouds.items()
if info['defined'] != 'public'
and cloud_types[name] != 'localhost')
excerpt = app.config.get('description',
"Where would you like to deploy?")
self.view = CloudView(app,
public_clouds,
custom_clouds,
compatible_clouds,
cb=self.finish)
if 'localhost' in compatible_clouds:
app.log.debug(
"Starting watcher for verifying LXD server is available.")
app.loop.create_task(
self._monitor_localhost(LocalhostProvider(),
self.view._enable_localhost_widget))
app.ui.set_header(title="Choose a Cloud", excerpt=excerpt)
app.ui.set_body(self.view)
app.ui.set_footer('')
def render(self):
track_screen("Cloud Creation")
if app.current_controller is None:
app.current_controller = "conjure-up-{}-{}".format(
app.current_cloud, utils.gen_hash())
if app.current_model is None:
app.current_model = utils.gen_model()
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
# LXD is a special case as we want to make sure a bridge
# is configured. If not we'll bring up a new view to allow
# a user to configure a LXD bridge with suggested network
# information.
if cloud_type == 'localhost':
lxd = common.is_lxd_ready()
if not lxd['ready']:
return controllers.use('lxdsetup').render(lxd['msg'])
# lxd doesn't require user-provided credentials,
# so never show the editor for localhost
return self.finish()
creds = common.try_get_creds(app.current_cloud)
try:
endpoint = juju.get_cloud(app.current_cloud).get('endpoint', None)
except LookupError:
endpoint = None
if creds and (endpoint or cloud_type != 'maas'):
return self.finish()
# show credentials editor otherwise
try:
creds = load_schema(app.current_cloud)
except Exception as e:
track_exception("Credentials Error: {}".format(e))
app.log.exception("Credentials Error: {}".format(e))
return app.ui.show_exception_message(
Exception("Unable to find credentials for {}, "
"you can double check what credentials you "
"do have available by running "
"`juju credentials`. Please see `juju help "
"add-credential` for more information.".format(
app.current_cloud)))
regions = []
# No regions for these providers
if cloud_type not in ['maas', 'vsphere']:
regions = sorted(juju.get_regions(app.current_cloud).keys())
view = NewCloudView(creds, regions, self.finish)
app.ui.set_header(title="New cloud setup", )
app.ui.set_body(view)
app.ui.set_footer("")
def _build_widget(self):
default_selection = None
cloud_types_by_name = juju.get_cloud_types_by_name()
if len(self.public_clouds) > 0:
self._add_item(Text("Public Clouds"))
self._add_item(HR())
for cloud_name in self.public_clouds:
cloud_type = cloud_types_by_name[cloud_name]
allowed = cloud_type in self.compatible_cloud_types
if allowed and default_selection is None:
default_selection = len(self.items.contents)
self._add_item(
CloudWidget(name=cloud_name,
cb=self.submit,
enabled=allowed))
self._add_item(Padding.line_break(""))
if len(self.custom_clouds) > 0:
self._add_item(Text("Your Clouds"))
self._add_item(HR())
for cloud_name in self.custom_clouds:
cloud_type = cloud_types_by_name[cloud_name]
allowed = cloud_type in self.compatible_cloud_types
if allowed and default_selection is None:
default_selection = len(self.items.contents)
self._add_item(
CloudWidget(name=cloud_name,
cb=self.submit,
enabled=allowed))
self._add_item(Padding.line_break(""))
new_clouds = juju.get_compatible_clouds(CUSTOM_PROVIDERS)
if new_clouds:
lxd_allowed = cloud_types.LOCALHOST in self.compatible_cloud_types
self._add_item(Text("Configure a New Cloud"))
self._add_item(HR())
for cloud_type in sorted(CUSTOM_PROVIDERS):
if cloud_type == cloud_types.LOCALHOST and lxd_allowed:
self._items_localhost_idx = len(self.items.contents)
if default_selection is None:
default_selection = len(self.items.contents)
self._add_item(
CloudWidget(name=cloud_type,
cb=self.submit,
enabled=events.LXDAvailable.is_set(),
disabled_msg=self.lxd_unavailable_msg))
else:
allowed = cloud_type in self.compatible_cloud_types
if allowed and default_selection is None:
default_selection = len(self.items.contents)
self._add_item(
CloudWidget(name=cloud_type,
cb=self.submit,
enabled=allowed))
self.items.focus_position = default_selection or 2
return self.items
def build_widget(self):
widget = MenuSelectButtonList()
cloud_types_by_name = juju.get_cloud_types_by_name()
if len(self.public_clouds) > 0:
widget.append(Text("Public Clouds"))
widget.append(HR())
for cloud_name in self.public_clouds:
cloud_type = cloud_types_by_name[cloud_name]
allowed = cloud_type in self.compatible_cloud_types
widget.append_option(cloud_name, enabled=allowed)
widget.append(Padding.line_break(""))
if len(self.custom_clouds) > 0:
widget.append(Text("Your Clouds"))
widget.append(HR())
for cloud_name in self.custom_clouds:
cloud_type = cloud_types_by_name[cloud_name]
allowed = cloud_type in self.compatible_cloud_types
widget.append_option(cloud_name, enabled=allowed)
widget.append(Padding.line_break(""))
lxd_allowed = cloud_types.LOCALHOST in self.compatible_cloud_types
widget.append(Text("Configure a New Cloud"))
widget.append(HR())
for cloud_type in sorted(CUSTOM_PROVIDERS):
if cloud_type == cloud_types.LOCALHOST and lxd_allowed:
self._items_localhost_idx = len(widget.contents)
widget.append_option(cloud_type,
enabled=events.LXDAvailable.is_set(),
user_data={
'disabled_msg':
self.lxd_unavailable_msg,
})
else:
allowed = cloud_type in self.compatible_cloud_types
widget.append_option(cloud_type, enabled=allowed)
if app.provider and app.provider.cloud:
widget.select_item_by_value(app.provider.cloud)
elif app.metadata.cloud_whitelist:
# whitelist is cloud types, widget is cloud names
# (except for new clouds)
whitelist = app.metadata.cloud_whitelist
values = {opt.value for opt in widget.option_widgets}
whitelisted_values = {
value
for value in values if value in whitelist
or cloud_types_by_name.get(value) in whitelist
}
widget.select_first_of_values(whitelisted_values)
else:
widget.select_first()
return widget
async def pre_bootstrap(msg_cb):
""" runs pre bootstrap script if exists
"""
# Set provider type for post-bootstrap
app.env['JUJU_PROVIDERTYPE'] = juju.get_cloud_types_by_name()[
app.current_cloud]
app.env['JUJU_CONTROLLER'] = app.current_controller
app.env['JUJU_MODEL'] = app.current_model
app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir
await utils.run_step('00_pre-bootstrap', 'pre-bootstrap', msg_cb)
async def do_bootstrap(creds, msg_cb, fail_msg_cb):
if not app.is_jaas:
await pre_bootstrap(msg_cb)
app.log.info('Bootstrapping Juju controller.')
msg_cb('Bootstrapping Juju controller.')
track_event("Juju Bootstrap", "Started", "")
cloud_with_region = app.current_cloud
if app.current_region:
cloud_with_region = '/'.join(
[app.current_cloud, app.current_region])
success = await juju.bootstrap(app.current_controller,
cloud_with_region,
app.current_model,
credential=creds)
if not success:
pathbase = os.path.join(app.config['spell-dir'],
'{}-bootstrap').format(
app.current_controller)
with open(pathbase + ".err") as errf:
err_log = "\n".join(errf.readlines())
msg = "Error bootstrapping controller: {}".format(err_log)
app.log.error(msg)
fail_msg_cb(msg)
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
raise Exception(
'Unable to bootstrap (cloud type: {})'.format(cloud_type))
return
app.log.info('Bootstrap complete.')
msg_cb('Bootstrap complete.')
track_event("Juju Bootstrap", "Done", "")
await juju.login() # login to the newly created (default) model
# Set provider type for post-bootstrap
app.env['JUJU_PROVIDERTYPE'] = app.juju.client.info.provider_type
app.env['JUJU_CONTROLLER'] = app.current_controller
app.env['JUJU_MODEL'] = app.current_model
await utils.run_step('00_post-bootstrap', 'post-bootstrap', msg_cb,
'Juju Post-Bootstrap')
else:
app.log.info('Adding new model in the background.')
msg_cb('Adding new model in the background.')
track_event("Juju Add JaaS Model", "Started", "")
await juju.add_model(app.current_model, app.current_controller,
app.current_cloud)
track_event("Juju Add JaaS Model", "Done", "")
app.log.info('Add model complete.')
msg_cb('Add model complete.')
async def pre_bootstrap(self):
""" runs pre bootstrap script if exists
"""
# Set provider type for post-bootstrap
app.env['JUJU_PROVIDERTYPE'] = juju.get_cloud_types_by_name()[
app.current_cloud]
# Set current credential name (localhost doesn't have one)
app.env['JUJU_CREDENTIAL'] = app.current_credential or ''
app.env['JUJU_CONTROLLER'] = app.current_controller
app.env['JUJU_MODEL'] = app.current_model
app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir
step = StepModel({}, filename='00_pre-bootstrap', name='pre-bootstrap')
await utils.run_step(step, self.msg_cb)
def render(self):
track_screen("Configure Applications")
self.applications = sorted(app.metadata_controller.bundle.services,
key=attrgetter('service_name'))
self.undeployed_applications = self.applications[:]
cloud_type = juju.get_cloud_types_by_name()[app.provider.cloud]
if cloud_type == cloud_types.MAAS:
app.loop.create_task(self.connect_maas())
self.list_view = ApplicationListView(self.applications,
app.metadata_controller, self)
self.list_header = "Review and Configure Applications"
app.ui.set_header(self.list_header)
app.ui.set_body(self.list_view)
def finish(self, cloud):
"""Save the selected cloud and move on to the region selection screen.
"""
if cloud == 'maas':
app.current_cloud_type = 'maas'
app.current_cloud = utils.gen_cloud()
else:
app.current_cloud_type = juju.get_cloud_types_by_name()[cloud]
app.current_cloud = cloud
if app.current_model is None:
app.current_model = utils.gen_model()
track_event("Cloud selection", app.current_cloud, "")
return controllers.use('regions').render()
def __init__(self, application, controller, close_cb):
"""
application: a Service instance representing a juju application
controller: a DeployGUIController instance
"""
self.application = application
self.controller = controller
self.close_cb = close_cb
cloud_type = get_cloud_types_by_name()[app.provider.cloud]
self.controller_is_maas = cloud_type == cloud_types.MAAS
self.title = "Architect {}".format(application.service_name)
if self.controller_is_maas:
extra = (" Press enter on a machine ID to pin it to "
"a specific MAAS node.")
else:
extra = ""
self.subtitle = "Choose where to place {} unit{} of {}.{}".format(
self.application.num_units,
"" if self.application.num_units == 1 else "s",
self.application.service_name, extra)
# Shadow means temporary to the view, they are committed to
# the controller if the user chooses OK
# {machine_id : [(app, assignmenttype) ...]
self.shadow_assignments = defaultdict(list)
for machine_id, al in self.controller.assignments.items():
for (a, at) in al:
if a == self.application:
self.shadow_assignments[machine_id].append((a, at))
# {juju_machine_id : maas machine id}
self.shadow_pins = copy.copy(self.controller.maas_machine_map)
self.machine_pin_view = None
self._machines = copy.deepcopy(app.metadata_controller.bundle.machines)
self.closed = asyncio.Event()
super().__init__()
app.loop.create_task(self.update())
async def do_deploy(self):
cloud_types = juju.get_cloud_types_by_name()
default_series = app.metadata_controller.series
machines = app.metadata_controller.bundle.machines
applications = sorted(app.metadata_controller.bundle.services,
key=attrgetter('service_name'))
await common.pre_deploy(msg_cb=utils.info)
await juju.add_machines(applications, machines, msg_cb=utils.info)
tasks = []
for service in applications:
# ignore placement when deploying to localhost
if cloud_types[app.current_cloud] == "localhost":
service.placement_spec = None
tasks.append(
juju.deploy_service(service, default_series,
msg_cb=utils.info))
tasks.append(juju.set_relations(service, msg_cb=utils.info))
await asyncio.gather(*tasks)
events.DeploymentComplete.set()
controllers.use('deploystatus').render()
def finish(self, cloud):
""" Load the selected cloud provider
"""
self.cancel_monitor.set()
if cloud in CUSTOM_PROVIDERS:
app.provider = load_schema(cloud)
else:
app.provider = load_schema(juju.get_cloud_types_by_name()[cloud])
try:
app.provider.load(cloud)
except SchemaErrorUnknownCloud:
app.provider.cloud = utils.gen_cloud()
if app.provider.model is None:
app.provider.model = utils.gen_model()
track_event("Cloud selection", app.provider.cloud, "")
return controllers.use('credentials').render()
def render(self):
# If bootstrap fails fast, we may be called after the error
# screen was already shown. We should bail to avoid
# overwriting the error screen.
if events.Error.is_set():
return
track_screen("Deploy")
app.loop.create_task(common.pre_deploy(app.ui.set_footer))
self.applications = sorted(app.metadata_controller.bundle.services,
key=attrgetter('service_name'))
self.undeployed_applications = self.applications[:]
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type == 'maas':
app.loop.create_task(self.connect_maas())
self.list_view = ApplicationListView(self.applications,
app.metadata_controller, self)
self.list_header = "Review and Configure Applications"
app.ui.set_header(self.list_header)
app.ui.set_body(self.list_view)
def render(self):
track_screen("Cloud Creation")
self.cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if app.current_controller is None:
app.current_controller = "conjure-up-{}-{}".format(
app.current_cloud,
utils.gen_hash())
if app.current_model is None:
app.current_model = utils.gen_model()
# LXD is a special case as we want to make sure a bridge
# is configured. If not we'll bring up a new view to allow
# a user to configure a LXD bridge with suggested network
# information.
if self.cloud_type == 'localhost':
lxd_setup_path = common.get_lxd_setup_path()
app.log.debug("Determining if embedded LXD is setup and ready.")
if lxd_setup_path.exists():
return self.finish()
else:
self.render_newcloud()
# TODO: Prompt user to select credentials and set a region
creds = common.try_get_creds(app.current_cloud)
try:
endpoint = juju.get_cloud(app.current_cloud).get(
'endpoint', None)
except LookupError:
endpoint = None
if creds and (endpoint or self.cloud_type != 'maas'):
return self.finish()
# No existing credentials found, start credential editor
self.render_newcloud()
def render(self):
creds = None
cloud_type = juju.get_cloud_types_by_name()[app.current_cloud]
if cloud_type != 'localhost':
creds = common.try_get_creds(app.current_cloud)
if not creds:
utils.warning("You attempted to do an install against a cloud "
"that requires credentials that could not be "
"found. If you wish to supply those "
"credentials please run "
"`juju add-credential "
"{}`.".format(app.current_cloud))
events.Shutdown.set(1)
return
if cloud_type == 'localhost':
# Grab list of available physical networks to bind our bridge to
iface = None
try:
ifaces = utils.get_physical_network_interfaces()
# Grab the first physical network device that has an ip address
iface = [
i for i in ifaces if utils.get_physical_network_ipaddr(i)
][0]
except:
utils.warning(
"Could not find a suitable physical network interface "
"to create a LXD bridge on. Please check your network "
"configuration.")
events.Shutdown.set(1)
if not common.get_lxd_setup_path().exists():
common.lxd_init(iface)
common.get_lxd_setup_path().touch()
app.loop.create_task(self.finish(creds))
async def run(self, msg_cb, event_name=None):
# Define STEP_NAME for use in determining where to store
# our step results,
# state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val"
app.env['CONJURE_UP_STEP'] = self.name
step_path = Path(app.config['spell-dir']) / 'steps' / self.filename
if not step_path.is_file():
return
step_path = str(step_path)
msg = "Running step: {}.".format(self.name)
app.log.info(msg)
msg_cb(msg)
if event_name is not None:
track_event(event_name, "Started", "")
if not os.access(step_path, os.X_OK):
raise Exception("Step {} not executable".format(self.title))
if is_linux() and self.needs_sudo and not await can_sudo():
raise SudoError('Step "{}" requires sudo: {}'.format(
self.title,
'password failed'
if app.sudo_pass else 'passwordless sudo required',
))
cloud_types = juju.get_cloud_types_by_name()
provider_type = cloud_types[app.provider.cloud]
app.env['JUJU_PROVIDERTYPE'] = provider_type
# not all providers have a credential, e.g., localhost
app.env['JUJU_CREDENTIAL'] = app.provider.credential or ''
app.env['JUJU_CONTROLLER'] = app.provider.controller
app.env['JUJU_MODEL'] = app.provider.model
app.env['JUJU_REGION'] = app.provider.region or ''
app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir
if provider_type == "maas":
app.env['MAAS_ENDPOINT'] = app.maas.endpoint
app.env['MAAS_APIKEY'] = app.maas.api_key
for step_name, step_data in app.steps_data.items():
for key, value in step_data.items():
app.env[key.upper()] = step_data[key]
for key, value in app.env.items():
if value is None:
app.log.warning('Env {} is None; '
'replacing with empty string'.format(key))
app.env[key] = ''
app.log.debug("Storing environment")
async with aiofiles.open(step_path + ".env", 'w') as outf:
for k, v in app.env.items():
if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k:
await outf.write("{}=\"{}\" ".format(k.upper(), v))
app.log.debug("Executing script: {}".format(step_path))
async with aiofiles.open(step_path + ".out", 'w') as outf:
async with aiofiles.open(step_path + ".err", 'w') as errf:
proc = await asyncio.create_subprocess_exec(step_path,
env=app.env,
stdout=outf,
stderr=errf)
async with aiofiles.open(step_path + '.out', 'r') as f:
while proc.returncode is None:
async for line in f:
msg_cb(line)
await asyncio.sleep(0.01)
out_log = Path(step_path + '.out').read_text()
err_log = Path(step_path + '.err').read_text()
if proc.returncode != 0:
app.sentry.context.merge({
'extra': {
'out_log_tail': out_log[-400:],
'err_log_tail': err_log[-400:],
}
})
raise Exception("Failure in step {}".format(self.filename))
# special case for 00_deploy-done to report masked
# charm hook failures that were retried automatically
if not app.noreport:
failed_apps = set() # only report each charm once
for line in err_log.splitlines():
if 'hook failure, will retry' in line:
log_leader = line.split()[0]
unit_name = log_leader.split(':')[-1]
app_name = unit_name.split('/')[0]
failed_apps.add(app_name)
for app_name in failed_apps:
# report each individually so that Sentry will give us a
# breakdown of failures per-charm in addition to per-spell
sentry_report('Retried hook failure',
tags={
'app_name': app_name,
})
if event_name is not None:
track_event(event_name, "Done", "")
result_key = "conjure-up.{}.{}.result".format(app.config['spell'],
self.name)
result = app.state.get(result_key)
return (result or '')
def main():
if os.geteuid() == 0:
print("")
print(" !! This should _not_ be run as root or with sudo. !!")
print("")
sys.exit(1)
# Verify we can access ~/.local/share/juju if it exists
juju_dir = pathlib.Path('~/.local/share/juju').expanduser()
if juju_dir.exists():
try:
for f in juju_dir.iterdir():
if f.is_file():
f.read_text()
except PermissionError:
print("")
print(" !! Unable to read from ~/.local/share/juju, please "
"double check your permissions on that directory "
"and its files. !!")
print("")
sys.exit(1)
utils.set_terminal_title("conjure-up")
opts = parse_options(sys.argv[1:])
opt_defaults = parse_options([])
# Load conjurefile, merge any overridding options from argv
if not opts.conf_file:
opts.conf_file = []
if pathlib.Path('~/.config/conjure-up.conf').expanduser().exists():
opts.conf_file.insert(
0, pathlib.Path('~/.config/conjure-up.conf').expanduser())
if (pathlib.Path('.') / 'Conjurefile').exists():
opts.conf_file.insert(0, pathlib.Path('.') / 'Conjurefile')
for conf in opts.conf_file:
if not conf.exists():
print("Unable to locate config {} for processing.".format(
str(conf)))
sys.exit(1)
try:
app.conjurefile = Conjurefile.load(opts.conf_file)
except ValueError as e:
print(str(e))
sys.exit(1)
app.conjurefile.merge_argv(opts, opt_defaults)
if app.conjurefile['gen-config']:
Conjurefile.print_tpl()
sys.exit(0)
spell = os.path.basename(os.path.abspath(app.conjurefile['spell']))
if not os.path.isdir(app.conjurefile['cache-dir']):
os.makedirs(app.conjurefile['cache-dir'])
# Application Config
kv_db = os.path.join(app.conjurefile['cache-dir'], '.state.db')
app.state = KV(kv_db)
app.env = os.environ.copy()
app.env['KV_DB'] = kv_db
app.config = {'metadata': None}
app.log = setup_logging(app,
os.path.join(app.conjurefile['cache-dir'],
'conjure-up.log'),
app.conjurefile.get('debug', False))
# Make sure juju paths are setup
juju.set_bin_path()
juju.set_wait_path()
app.no_track = app.conjurefile['no-track']
app.no_report = app.conjurefile['no-report']
# Grab current LXD and Juju versions
app.log.debug("Juju version: {}, "
"conjure-up version: {}".format(
utils.juju_version(),
VERSION))
# Setup proxy
apply_proxy()
app.session_id = os.getenv('CONJURE_TEST_SESSION_ID',
str(uuid.uuid4()))
spells_dir = app.conjurefile['spells-dir']
app.config['spells-dir'] = spells_dir
spells_index_path = os.path.join(app.config['spells-dir'],
'spells-index.yaml')
spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'master')
if not app.conjurefile['no-sync']:
if not os.path.exists(spells_dir):
utils.info("No spells found, syncing from registry, please wait.")
try:
download_or_sync_registry(
app.conjurefile['registry'],
spells_dir, branch=spells_registry_branch)
except subprocess.CalledProcessError as e:
if not os.path.exists(spells_dir):
utils.error("Could not load from registry")
sys.exit(1)
app.log.debug(
'Could not sync spells from github: {}'.format(e))
else:
if not os.path.exists(spells_index_path):
utils.error(
"You opted to not sync from the spells registry, however, "
"we could not find any suitable spells in: "
"{}".format(spells_dir))
sys.exit(1)
with open(spells_index_path) as fp:
app.spells_index = yaml.safe_load(fp.read())
addons_aliases_index_path = os.path.join(app.config['spells-dir'],
'addons-aliases.yaml')
if os.path.exists(addons_aliases_index_path):
with open(addons_aliases_index_path) as fp:
app.addons_aliases = yaml.safe_load(fp.read())
spell_name = spell
app.endpoint_type = detect_endpoint(app.conjurefile['spell'])
if app.conjurefile['spell'] != consts.UNSPECIFIED_SPELL:
app.spell_given = True
# Check if spell is actually an addon
addon = utils.find_addons_matching(app.conjurefile['spell'])
if addon:
app.log.debug("addon found, setting required spell")
utils.set_chosen_spell(addon['spell'],
os.path.join(app.conjurefile['cache-dir'],
addon['spell']))
download_local(os.path.join(app.config['spells-dir'],
addon['spell']),
app.config['spell-dir'])
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
app.selected_addons = addon['addons']
app.alias_given = True
controllers.setup_metadata_controller()
app.endpoint_type = EndpointType.LOCAL_DIR
elif app.endpoint_type == EndpointType.LOCAL_SEARCH:
spells = utils.find_spells_matching(app.conjurefile['spell'])
if len(spells) == 0:
utils.error("Can't find a spell matching '{}'".format(
app.conjurefile['spell']))
sys.exit(1)
# One result means it was a direct match and we can copy it
# now. Changing the endpoint type then stops us from showing
# the picker UI. More than one result means we need to show
# the picker UI and will defer the copy to
# SpellPickerController.finish(), so nothing to do here.
if len(spells) == 1:
app.log.debug("found spell {}".format(spells[0][1]))
spell = spells[0][1]
utils.set_chosen_spell(spell_name,
os.path.join(app.conjurefile['cache-dir'],
spell['key']))
download_local(os.path.join(app.config['spells-dir'],
spell['key']),
app.config['spell-dir'])
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
app.endpoint_type = EndpointType.LOCAL_DIR
# download spell if necessary
elif app.endpoint_type == EndpointType.LOCAL_DIR:
if not os.path.isdir(app.conjurefile['spell']):
utils.warning("Could not find spell {}".format(
app.conjurefile['spell']))
sys.exit(1)
if not os.path.exists(os.path.join(app.conjurefile['spell'],
"metadata.yaml")):
utils.warning("'{}' does not appear to be a spell. "
"{}/metadata.yaml was not found.".format(
app.conjurefile['spell'],
app.conjurefile['spell']))
sys.exit(1)
spell_name = os.path.basename(os.path.abspath(spell))
utils.set_chosen_spell(spell_name,
path.join(app.conjurefile['cache-dir'],
spell_name))
download_local(app.conjurefile['spell'], app.config['spell-dir'])
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]:
utils.set_chosen_spell(spell, path.join(
app.conjurefile['cache-dir'], spell))
remote = get_remote_url(app.conjurefile['spell'])
if remote is None:
utils.warning("Can't guess URL matching '{}'".format(
app.conjurefile['spell']))
sys.exit(1)
download(remote, app.config['spell-dir'], True)
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
app.env['CONJURE_UP_CACHEDIR'] = app.conjurefile['cache-dir']
app.env['PATH'] = "/snap/bin:{}".format(app.env['PATH'])
if app.conjurefile['show-env']:
if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]:
utils.error("Please specify a spell for headless mode.")
sys.exit(1)
show_env()
app.sentry = raven.Client(
dsn=SENTRY_DSN,
release=VERSION,
transport=RequestsHTTPTransport,
processors=(
'conjureup.utils.SanitizeDataProcessor',
)
)
track_screen("Application Start")
track_event("OS", platform.platform(), "")
app.loop = asyncio.get_event_loop()
app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set)
# Enable charmstore querying
app.juju.charmstore = CharmStore(app.loop)
try:
if app.conjurefile.is_valid:
cloud = None
region = None
if '/' in app.conjurefile['cloud']:
parse_cli_cloud = app.conjurefile['cloud'].split('/')
cloud, region = parse_cli_cloud
app.log.debug(
"Region found {} for cloud {}".format(cloud,
region))
else:
cloud = app.conjurefile['cloud']
cloud_types = juju.get_cloud_types_by_name()
if cloud not in cloud_types:
utils.error('Unknown cloud: {}'.format(cloud))
sys.exit(1)
if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]:
utils.error("Please specify a spell for headless mode.")
sys.exit(1)
app.provider = load_schema(cloud_types[cloud])
try:
app.provider.load(cloud)
except errors.SchemaCloudError as e:
utils.error(e)
sys.exit(1)
if region:
app.provider.region = region
app.headless = True
app.ui = None
app.env['CONJURE_UP_HEADLESS'] = "1"
app.loop.create_task(events.shutdown_watcher())
app.loop.create_task(_start())
app.loop.run_forever()
else:
app.ui = ConjureUI()
app.ui.set_footer('Press ? for help')
EventLoop.build_loop(app.ui, STYLES,
unhandled_input=events.unhandled_input,
handle_mouse=False)
app.loop.create_task(events.shutdown_watcher())
app.loop.create_task(_start())
EventLoop.run()
finally:
# explicitly close asyncio event loop to avoid hitting the
# following issue due to signal handlers added by
# asyncio.create_subprocess_exec being cleaned up during final
# garbage collection: https://github.com/python/asyncio/issues/396
app.loop.close()
sys.exit(app.exit_code)
async def run(self, phase, msg_cb, event_name=None):
# Define STEP_NAME for use in determining where to store
# our step results,
# state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val"
app.env['CONJURE_UP_STEP'] = self.name
app.env['CONJURE_UP_PHASE'] = phase.value
step_path = self._build_phase_path(phase)
if not step_path.is_file():
return
if not os.access(str(step_path), os.X_OK):
app.log.error(
'Unable to run {} step {} {}, it is not executable'.format(
self.source, step_path.stem, phase.value))
return
step_path = str(step_path)
msg = "Running {} step: {} {}.".format(self.source, self.name,
phase.value)
app.log.info(msg)
msg_cb(msg)
if event_name is not None:
track_event(event_name, "Started", "")
if is_linux() and self.needs_sudo and not await can_sudo():
raise SudoError('The "{}" step "{}" requires sudo: {}'.format(
self.source,
self.title,
'password failed'
if app.sudo_pass else 'passwordless sudo required',
))
app.env['CONJURE_UP_SPELLSDIR'] = app.conjurefile['spells-dir']
app.env['CONJURE_UP_SESSION_ID'] = app.session_id
if app.metadata.spell_type == spell_types.JUJU:
cloud_types = juju.get_cloud_types_by_name()
provider_type = cloud_types[app.provider.cloud]
app.env['JUJU_PROVIDERTYPE'] = provider_type
# not all providers have a credential, e.g., localhost
app.env['JUJU_CREDENTIAL'] = app.provider.credential or ''
app.env['JUJU_CONTROLLER'] = app.provider.controller
app.env['JUJU_MODEL'] = app.provider.model
app.env['JUJU_REGION'] = app.provider.region or ''
if provider_type == "maas":
app.env['MAAS_ENDPOINT'] = app.maas.endpoint
app.env['MAAS_APIKEY'] = app.maas.api_key
for step_name, step_data in app.steps_data.items():
for key, value in step_data.items():
app.env[key.upper()] = str(step_data[key])
for key, value in app.env.items():
if value is None:
app.log.warning('Env {} is None; '
'replacing with empty string'.format(key))
app.env[key] = ''
app.log.debug("Storing environment")
async with aiofiles.open(step_path + ".env", 'w') as outf:
for k, v in app.env.items():
if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k:
await outf.write("{}=\"{}\" ".format(k.upper(), v))
app.log.debug("Executing script: {}".format(step_path))
out_path = step_path + '.out'
err_path = step_path + '.err'
ret, out_log, err_log = await arun([step_path],
stdout=out_path,
stderr=err_path,
cb_stdout=msg_cb)
if ret != 0:
app.sentry.context.merge({
'extra': {
'out_log_tail': out_log[-400:],
'err_log_tail': err_log[-400:],
}
})
raise Exception("Failure in step {} {}".format(
self.name, phase.value))
# special case for 00_deploy-done to report masked
# charm hook failures that were retried automatically
if not app.no_report and app.metadata.spell_type == spell_types.JUJU:
failed_apps = set() # only report each charm once
for line in err_log.splitlines():
if 'hook failure, will retry' in line:
log_leader = line.split()[0]
unit_name = log_leader.split(':')[-1]
app_name = unit_name.split('/')[0]
failed_apps.add(app_name)
for app_name in failed_apps:
# report each individually so that Sentry will give us a
# breakdown of failures per-charm in addition to per-spell
sentry_report('Retried hook failure',
tags={
'app_name': app_name,
})
if event_name is not None:
track_event(event_name, "Done", "")
return self.get_state('result', phase)
def main():
if os.geteuid() == 0:
print("")
print(" !! This should _not_ be run as root or with sudo. !!")
print("")
sys.exit(1)
# Verify we can access ~/.local/share/juju if it exists
juju_dir = pathlib.Path('~/.local/share/juju').expanduser()
if juju_dir.exists():
try:
for f in juju_dir.iterdir():
if f.is_file():
f.read_text()
except PermissionError:
print("")
print(" !! Unable to read from ~/.local/share/juju, please "
"double check your permissions on that directory "
"and its files. !!")
print("")
sys.exit(1)
utils.set_terminal_title("conjure-up")
opts = parse_options(sys.argv[1:])
spell = os.path.basename(os.path.abspath(opts.spell))
if not os.path.isdir(opts.cache_dir):
os.makedirs(opts.cache_dir)
# Application Config
os.environ['UNIT_STATE_DB'] = os.path.join(opts.cache_dir, '.state.db')
app.state = unitdata.kv()
app.env = os.environ.copy()
app.config = {'metadata': None}
app.argv = opts
app.log = setup_logging(app, os.path.join(opts.cache_dir,
'conjure-up.log'), opts.debug)
if app.argv.conf_file.expanduser().exists():
conf = configparser.ConfigParser()
conf.read_string(app.argv.conf_file.expanduser().read_text())
app.notrack = conf.getboolean('REPORTING', 'notrack', fallback=False)
app.noreport = conf.getboolean('REPORTING', 'noreport', fallback=False)
if app.argv.notrack:
app.notrack = True
if app.argv.noreport:
app.noreport = True
# Grab current LXD and Juju versions
app.log.debug("Juju version: {}, "
"conjure-up version: {}".format(utils.juju_version(),
VERSION))
# Setup proxy
apply_proxy()
app.session_id = os.getenv('CONJURE_TEST_SESSION_ID', str(uuid.uuid4()))
spells_dir = app.argv.spells_dir
app.config['spells-dir'] = spells_dir
spells_index_path = os.path.join(app.config['spells-dir'],
'spells-index.yaml')
spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'stable')
if not app.argv.nosync:
if not os.path.exists(spells_dir):
utils.info("No spells found, syncing from registry, please wait.")
try:
download_or_sync_registry(app.argv.registry,
spells_dir,
branch=spells_registry_branch)
except subprocess.CalledProcessError as e:
if not os.path.exists(spells_dir):
utils.error("Could not load from registry")
sys.exit(1)
app.log.debug('Could not sync spells from github: {}'.format(e))
else:
if not os.path.exists(spells_index_path):
utils.error(
"You opted to not sync from the spells registry, however, "
"we could not find any suitable spells in: "
"{}".format(spells_dir))
sys.exit(1)
with open(spells_index_path) as fp:
app.spells_index = yaml.safe_load(fp.read())
spell_name = spell
app.endpoint_type = detect_endpoint(opts.spell)
if app.endpoint_type == EndpointType.LOCAL_SEARCH:
spells = utils.find_spells_matching(opts.spell)
if len(spells) == 0:
utils.error("Can't find a spell matching '{}'".format(opts.spell))
sys.exit(1)
# One result means it was a direct match and we can copy it
# now. Changing the endpoint type then stops us from showing
# the picker UI. More than one result means we need to show
# the picker UI and will defer the copy to
# SpellPickerController.finish(), so nothing to do here.
if len(spells) == 1:
app.log.debug("found spell {}".format(spells[0][1]))
spell = spells[0][1]
utils.set_chosen_spell(spell_name,
os.path.join(opts.cache_dir, spell['key']))
download_local(
os.path.join(app.config['spells-dir'], spell['key']),
app.config['spell-dir'])
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
app.endpoint_type = EndpointType.LOCAL_DIR
# download spell if necessary
elif app.endpoint_type == EndpointType.LOCAL_DIR:
if not os.path.isdir(opts.spell):
utils.warning("Could not find spell {}".format(opts.spell))
sys.exit(1)
if not os.path.exists(os.path.join(opts.spell, "metadata.yaml")):
utils.warning("'{}' does not appear to be a spell. "
"{}/metadata.yaml was not found.".format(
opts.spell, opts.spell))
sys.exit(1)
spell_name = os.path.basename(os.path.abspath(spell))
utils.set_chosen_spell(spell_name, path.join(opts.cache_dir,
spell_name))
download_local(opts.spell, app.config['spell-dir'])
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]:
utils.set_chosen_spell(spell, path.join(opts.cache_dir, spell))
remote = get_remote_url(opts.spell)
if remote is None:
utils.warning("Can't guess URL matching '{}'".format(opts.spell))
sys.exit(1)
download(remote, app.config['spell-dir'], True)
utils.set_spell_metadata()
StepModel.load_spell_steps()
AddonModel.load_spell_addons()
app.env['CONJURE_UP_CACHEDIR'] = app.argv.cache_dir
if app.argv.show_env:
if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]:
utils.error("Please specify a spell for headless mode.")
sys.exit(1)
show_env()
app.sentry = raven.Client(
dsn=SENTRY_DSN,
release=VERSION,
transport=RequestsHTTPTransport,
processors=('conjureup.utils.SanitizeDataProcessor', ))
track_screen("Application Start")
track_event("OS", platform.platform(), "")
app.loop = asyncio.get_event_loop()
app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set)
try:
if app.argv.cloud:
cloud = None
region = None
if '/' in app.argv.cloud:
parse_cli_cloud = app.argv.cloud.split('/')
cloud, region = parse_cli_cloud
app.log.debug("Region found {} for cloud {}".format(
cloud, region))
else:
cloud = app.argv.cloud
cloud_types = juju.get_cloud_types_by_name()
if cloud not in cloud_types:
utils.error('Unknown cloud: {}'.format(cloud))
sys.exit(1)
if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]:
utils.error("Please specify a spell for headless mode.")
sys.exit(1)
app.provider = load_schema(cloud_types[cloud])
try:
app.provider.load(cloud)
except SchemaErrorUnknownCloud as e:
utils.error(e)
sys.exit(1)
if region:
app.provider.region = region
app.headless = True
app.ui = None
app.env['CONJURE_UP_HEADLESS'] = "1"
app.loop.create_task(events.shutdown_watcher())
app.loop.create_task(_start())
app.loop.run_forever()
else:
app.ui = ConjureUI()
EventLoop.build_loop(app.ui,
STYLES,
unhandled_input=events.unhandled_input)
app.loop.create_task(events.shutdown_watcher())
app.loop.create_task(_start())
EventLoop.run()
finally:
# explicitly close asyncio event loop to avoid hitting the
# following issue due to signal handlers added by
# asyncio.create_subprocess_exec being cleaned up during final
# garbage collection: https://github.com/python/asyncio/issues/396
app.loop.close()
sys.exit(app.exit_code)
