def render(self): creds = None cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type != 'localhost': creds = common.try_get_creds(app.current_cloud) if not creds: utils.warning("You attempted to do an install against a cloud " "that requires credentials that could not be " "found. If you wish to supply those " "credentials please run " "`juju add-credential " "{}`.".format(app.current_cloud)) events.Shutdown.set(1) return # LXD is a special case as we want to make sure a bridge # is configured. If not we'll bring up a new view to allow # a user to configure a LXD bridge with suggested network # information. if cloud_type == 'localhost': lxd = common.is_lxd_ready() if not lxd['ready']: return controllers.use('lxdsetup').render(lxd['msg']) app.loop.create_task(self.finish(creds))
def render(self, going_back=False): "Pick or create a cloud to bootstrap a new controller on" all_clouds = juju.get_clouds() compatible_clouds = juju.get_compatible_clouds() cloud_types = juju.get_cloud_types_by_name() # filter to only public clouds public_clouds = sorted(name for name, info in all_clouds.items() if info['defined'] == 'public') # filter to custom clouds # exclude localhost because we treat that as "configuring a new cloud" custom_clouds = sorted(name for name, info in all_clouds.items() if info['defined'] != 'public' and cloud_types[name] != 'localhost') prev_screen = self.prev_screen if app.alias_given or (app.spell_given and not app.addons): # we were given an alias (so spell and addons are locked) # or we were given a spell and there are no addons to change # so we disable the back button prev_screen = None self.view = CloudView(app, public_clouds, custom_clouds, compatible_clouds, cb=self.finish, back=prev_screen) if 'localhost' in compatible_clouds: app.log.debug( "Starting watcher for verifying LXD server is available.") self.cancel_monitor.clear() app.loop.create_task(self._monitor_localhost(LocalhostProvider())) self.view.show()
async def ensure_machines(self, application): """If 'application' is assigned to any machine that haven't been added yet, add the machines prior to deployment. """ cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type == 'maas': await events.MAASConnected.wait() app_placements = self.get_all_assignments(application) juju_machines = app.metadata_controller.bundle.machines machines = {} for virt_machine_id, _ in app_placements: if virt_machine_id in self.deployed_juju_machines: continue machine_attrs = { 'series': application.csid.series, } if cloud_type == 'maas': machine_attrs['constraints'] = \ await self.get_maas_constraints(virt_machine_id) else: machine_attrs.update(juju_machines[virt_machine_id]) machines[virt_machine_id] = machine_attrs return await juju.add_machines([application], machines, msg_cb=app.ui.set_footer)
def render(self): "Pick or create a cloud to bootstrap a new controller on" track_screen("Cloud Select") all_clouds = juju.get_clouds() compatible_clouds = juju.get_compatible_clouds() cloud_types = juju.get_cloud_types_by_name() # filter to only public clouds public_clouds = sorted( name for name, info in all_clouds.items() if info['defined'] == 'public' and cloud_types[name] in compatible_clouds) # filter to custom clouds # exclude localhost because we treat that as "configuring a new cloud" custom_clouds = sorted( name for name, info in all_clouds.items() if info['defined'] != 'public' and cloud_types[name] != 'localhost' and cloud_types[name] in compatible_clouds) excerpt = app.config.get( 'description', "Where would you like to deploy?") view = CloudView(app, public_clouds, custom_clouds, cb=self.finish) app.ui.set_header( title="Choose a Cloud", excerpt=excerpt ) app.ui.set_body(view) app.ui.set_footer('Please press [ENTER] on highlighted ' 'Cloud to proceed.')
async def do_deploy(msg_cb): await events.ModelConnected.wait() cloud_types = juju.get_cloud_types_by_name() default_series = app.metadata_controller.series machines = app.metadata_controller.bundle.machines applications = sorted(app.metadata_controller.bundle.services, key=attrgetter('service_name')) await pre_deploy(msg_cb=msg_cb) machine_map = await juju.add_machines(applications, machines, msg_cb=msg_cb) tasks = [] for service in applications: if cloud_types[app.provider.cloud] == "localhost": # ignore placement when deploying to localhost service.placement_spec = None elif service.placement_spec: # remap machine references to actual deployed machine IDs # (they will only ever not already match if deploying to # an existing model that has other machines) new_placements = [] for plabel in service.placement_spec: if ':' in plabel: ptype, pid = plabel.split(':') new_placements.append(':'.join([ptype, machine_map[pid]])) else: new_placements.append(machine_map[plabel]) service.placement_spec = new_placements tasks.append( juju.deploy_service(service, default_series, msg_cb=msg_cb)) tasks.append(juju.set_relations(service, msg_cb=msg_cb)) await asyncio.gather(*tasks) events.DeploymentComplete.set()
async def ensure_machines(self, application): """If 'application' is assigned to any machine that haven't been added yet, add the machines prior to deployment. Note: This no longer actually creates the machine, since the configuration will be filled out before the controller is bootstrapped. Instead, it just ensures that the data in app.metadata_controller is up to date. This needs to be refactored at a later date. """ cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type == 'maas': await events.MAASConnected.wait() app_placements = self.get_all_assignments(application) juju_machines = app.metadata_controller.bundle.machines machines = {} for virt_machine_id, _ in app_placements: if virt_machine_id in self.deployed_juju_machines: continue machine_attrs = { 'series': application.csid.series, } if cloud_type == 'maas': machine_attrs['constraints'] = \ await self.get_maas_constraints(virt_machine_id) else: machine_attrs.update(juju_machines[virt_machine_id]) machines[virt_machine_id] = machine_attrs # store the updated constraints back in the metadata # the actual machine creation will happen later, during deploy # we have to reassign it to the bundle because the getter can # return a new empty dict without storing it back in the bundle juju_machines.update(machines) app.metadata_controller.bundle.machines = juju_machines
def build_widgets(self): cloud_type = get_cloud_types_by_name()[app.current_cloud] controller_is_maas = cloud_type == 'maas' if controller_is_maas: extra = (" Press enter on a machine ID to pin it to " "a specific MAAS node.") else: extra = "" ws = [ Text("Choose where to place {} unit{} of {}.{}".format( self.application.num_units, "" if self.application.num_units == 1 else "s", self.application.service_name, extra)) ] self.juju_machines_list = JujuMachinesList( self.application, self._machines, self.do_assign, self.do_unassign, self.add_machine, self.remove_machine, self, show_filter_box=True, show_pins=controller_is_maas) ws.append(self.juju_machines_list) self.pile = Pile(ws) return Padding.center_90(Filler(self.pile, valign="top"))
def render(self): existing_controllers = juju.get_controllers()['controllers'] clouds = juju.get_compatible_clouds() cloud_types = juju.get_cloud_types_by_name() app.jaas_ok = set(clouds) & JAAS_CLOUDS jaas_controller = { n for n, c in existing_controllers.items() if JAAS_ENDPOINT in c['api-endpoints'] } if jaas_controller: app.jaas_controller = jaas_controller.pop() filtered_controllers = { n: d for n, d in existing_controllers.items() if cloud_types.get(d['cloud']) in clouds } if not app.jaas_ok and len(filtered_controllers) == 0: return controllers.use('clouds').render() track_screen("Controller Picker") excerpt = app.config.get( 'description', "Please select an existing controller," " or choose to bootstrap a new one.") view = ControllerListView(app, filtered_controllers, self.finish) app.ui.set_header(title="Choose a Controller or Create new", excerpt=excerpt) app.ui.set_body(view)
def finish(self, credentials=None, region=None, back=False): if region: app.current_region = region if back: return controllers.use('clouds').render() if credentials is not None: cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type == 'maas': # Now that we are passed the selection of a cloud we create a # new cloud name for the remainder of the deployment and make # sure this cloud is saved for future use. app.current_cloud = utils.gen_cloud() # Save credentials for new cloud common.save_creds(app.current_cloud, credentials) try: juju.get_cloud(app.current_cloud) except LookupError: juju.add_cloud(app.current_cloud, credentials.cloud_config()) else: common.save_creds(app.current_cloud, credentials) credentials_key = common.try_get_creds(app.current_cloud) app.loop.create_task( common.do_bootstrap(credentials_key, msg_cb=app.ui.set_footer, fail_msg_cb=lambda e: None)) controllers.use('deploy').render()
def render(self): "Pick or create a cloud to bootstrap a new controller on" track_screen("Cloud Select") all_clouds = juju.get_clouds() compatible_clouds = juju.get_compatible_clouds() cloud_types = juju.get_cloud_types_by_name() # filter to only public clouds public_clouds = sorted(name for name, info in all_clouds.items() if info['defined'] == 'public') # filter to custom clouds # exclude localhost because we treat that as "configuring a new cloud" custom_clouds = sorted(name for name, info in all_clouds.items() if info['defined'] != 'public' and cloud_types[name] != 'localhost') excerpt = app.config.get('description', "Where would you like to deploy?") self.view = CloudView(app, public_clouds, custom_clouds, compatible_clouds, cb=self.finish) if 'localhost' in compatible_clouds: app.log.debug( "Starting watcher for verifying LXD server is available.") app.loop.create_task( self._monitor_localhost(LocalhostProvider(), self.view._enable_localhost_widget)) app.ui.set_header(title="Choose a Cloud", excerpt=excerpt) app.ui.set_body(self.view) app.ui.set_footer('')
def render(self): track_screen("Cloud Creation") if app.current_controller is None: app.current_controller = "conjure-up-{}-{}".format( app.current_cloud, utils.gen_hash()) if app.current_model is None: app.current_model = utils.gen_model() cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] # LXD is a special case as we want to make sure a bridge # is configured. If not we'll bring up a new view to allow # a user to configure a LXD bridge with suggested network # information. if cloud_type == 'localhost': lxd = common.is_lxd_ready() if not lxd['ready']: return controllers.use('lxdsetup').render(lxd['msg']) # lxd doesn't require user-provided credentials, # so never show the editor for localhost return self.finish() creds = common.try_get_creds(app.current_cloud) try: endpoint = juju.get_cloud(app.current_cloud).get('endpoint', None) except LookupError: endpoint = None if creds and (endpoint or cloud_type != 'maas'): return self.finish() # show credentials editor otherwise try: creds = load_schema(app.current_cloud) except Exception as e: track_exception("Credentials Error: {}".format(e)) app.log.exception("Credentials Error: {}".format(e)) return app.ui.show_exception_message( Exception("Unable to find credentials for {}, " "you can double check what credentials you " "do have available by running " "`juju credentials`. Please see `juju help " "add-credential` for more information.".format( app.current_cloud))) regions = [] # No regions for these providers if cloud_type not in ['maas', 'vsphere']: regions = sorted(juju.get_regions(app.current_cloud).keys()) view = NewCloudView(creds, regions, self.finish) app.ui.set_header(title="New cloud setup", ) app.ui.set_body(view) app.ui.set_footer("")
def _build_widget(self): default_selection = None cloud_types_by_name = juju.get_cloud_types_by_name() if len(self.public_clouds) > 0: self._add_item(Text("Public Clouds")) self._add_item(HR()) for cloud_name in self.public_clouds: cloud_type = cloud_types_by_name[cloud_name] allowed = cloud_type in self.compatible_cloud_types if allowed and default_selection is None: default_selection = len(self.items.contents) self._add_item( CloudWidget(name=cloud_name, cb=self.submit, enabled=allowed)) self._add_item(Padding.line_break("")) if len(self.custom_clouds) > 0: self._add_item(Text("Your Clouds")) self._add_item(HR()) for cloud_name in self.custom_clouds: cloud_type = cloud_types_by_name[cloud_name] allowed = cloud_type in self.compatible_cloud_types if allowed and default_selection is None: default_selection = len(self.items.contents) self._add_item( CloudWidget(name=cloud_name, cb=self.submit, enabled=allowed)) self._add_item(Padding.line_break("")) new_clouds = juju.get_compatible_clouds(CUSTOM_PROVIDERS) if new_clouds: lxd_allowed = cloud_types.LOCALHOST in self.compatible_cloud_types self._add_item(Text("Configure a New Cloud")) self._add_item(HR()) for cloud_type in sorted(CUSTOM_PROVIDERS): if cloud_type == cloud_types.LOCALHOST and lxd_allowed: self._items_localhost_idx = len(self.items.contents) if default_selection is None: default_selection = len(self.items.contents) self._add_item( CloudWidget(name=cloud_type, cb=self.submit, enabled=events.LXDAvailable.is_set(), disabled_msg=self.lxd_unavailable_msg)) else: allowed = cloud_type in self.compatible_cloud_types if allowed and default_selection is None: default_selection = len(self.items.contents) self._add_item( CloudWidget(name=cloud_type, cb=self.submit, enabled=allowed)) self.items.focus_position = default_selection or 2 return self.items
def build_widget(self): widget = MenuSelectButtonList() cloud_types_by_name = juju.get_cloud_types_by_name() if len(self.public_clouds) > 0: widget.append(Text("Public Clouds")) widget.append(HR()) for cloud_name in self.public_clouds: cloud_type = cloud_types_by_name[cloud_name] allowed = cloud_type in self.compatible_cloud_types widget.append_option(cloud_name, enabled=allowed) widget.append(Padding.line_break("")) if len(self.custom_clouds) > 0: widget.append(Text("Your Clouds")) widget.append(HR()) for cloud_name in self.custom_clouds: cloud_type = cloud_types_by_name[cloud_name] allowed = cloud_type in self.compatible_cloud_types widget.append_option(cloud_name, enabled=allowed) widget.append(Padding.line_break("")) lxd_allowed = cloud_types.LOCALHOST in self.compatible_cloud_types widget.append(Text("Configure a New Cloud")) widget.append(HR()) for cloud_type in sorted(CUSTOM_PROVIDERS): if cloud_type == cloud_types.LOCALHOST and lxd_allowed: self._items_localhost_idx = len(widget.contents) widget.append_option(cloud_type, enabled=events.LXDAvailable.is_set(), user_data={ 'disabled_msg': self.lxd_unavailable_msg, }) else: allowed = cloud_type in self.compatible_cloud_types widget.append_option(cloud_type, enabled=allowed) if app.provider and app.provider.cloud: widget.select_item_by_value(app.provider.cloud) elif app.metadata.cloud_whitelist: # whitelist is cloud types, widget is cloud names # (except for new clouds) whitelist = app.metadata.cloud_whitelist values = {opt.value for opt in widget.option_widgets} whitelisted_values = { value for value in values if value in whitelist or cloud_types_by_name.get(value) in whitelist } widget.select_first_of_values(whitelisted_values) else: widget.select_first() return widget
async def pre_bootstrap(msg_cb): """ runs pre bootstrap script if exists """ # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = juju.get_cloud_types_by_name()[ app.current_cloud] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir await utils.run_step('00_pre-bootstrap', 'pre-bootstrap', msg_cb)
async def do_bootstrap(creds, msg_cb, fail_msg_cb): if not app.is_jaas: await pre_bootstrap(msg_cb) app.log.info('Bootstrapping Juju controller.') msg_cb('Bootstrapping Juju controller.') track_event("Juju Bootstrap", "Started", "") cloud_with_region = app.current_cloud if app.current_region: cloud_with_region = '/'.join( [app.current_cloud, app.current_region]) success = await juju.bootstrap(app.current_controller, cloud_with_region, app.current_model, credential=creds) if not success: pathbase = os.path.join(app.config['spell-dir'], '{}-bootstrap').format( app.current_controller) with open(pathbase + ".err") as errf: err_log = "\n".join(errf.readlines()) msg = "Error bootstrapping controller: {}".format(err_log) app.log.error(msg) fail_msg_cb(msg) cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] raise Exception( 'Unable to bootstrap (cloud type: {})'.format(cloud_type)) return app.log.info('Bootstrap complete.') msg_cb('Bootstrap complete.') track_event("Juju Bootstrap", "Done", "") await juju.login() # login to the newly created (default) model # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = app.juju.client.info.provider_type app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model await utils.run_step('00_post-bootstrap', 'post-bootstrap', msg_cb, 'Juju Post-Bootstrap') else: app.log.info('Adding new model in the background.') msg_cb('Adding new model in the background.') track_event("Juju Add JaaS Model", "Started", "") await juju.add_model(app.current_model, app.current_controller, app.current_cloud) track_event("Juju Add JaaS Model", "Done", "") app.log.info('Add model complete.') msg_cb('Add model complete.')
async def pre_bootstrap(self): """ runs pre bootstrap script if exists """ # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = juju.get_cloud_types_by_name()[ app.current_cloud] # Set current credential name (localhost doesn't have one) app.env['JUJU_CREDENTIAL'] = app.current_credential or '' app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir step = StepModel({}, filename='00_pre-bootstrap', name='pre-bootstrap') await utils.run_step(step, self.msg_cb)
def render(self): track_screen("Configure Applications") self.applications = sorted(app.metadata_controller.bundle.services, key=attrgetter('service_name')) self.undeployed_applications = self.applications[:] cloud_type = juju.get_cloud_types_by_name()[app.provider.cloud] if cloud_type == cloud_types.MAAS: app.loop.create_task(self.connect_maas()) self.list_view = ApplicationListView(self.applications, app.metadata_controller, self) self.list_header = "Review and Configure Applications" app.ui.set_header(self.list_header) app.ui.set_body(self.list_view)
def finish(self, cloud): """Save the selected cloud and move on to the region selection screen. """ if cloud == 'maas': app.current_cloud_type = 'maas' app.current_cloud = utils.gen_cloud() else: app.current_cloud_type = juju.get_cloud_types_by_name()[cloud] app.current_cloud = cloud if app.current_model is None: app.current_model = utils.gen_model() track_event("Cloud selection", app.current_cloud, "") return controllers.use('regions').render()
def __init__(self, application, controller, close_cb): """ application: a Service instance representing a juju application controller: a DeployGUIController instance """ self.application = application self.controller = controller self.close_cb = close_cb cloud_type = get_cloud_types_by_name()[app.provider.cloud] self.controller_is_maas = cloud_type == cloud_types.MAAS self.title = "Architect {}".format(application.service_name) if self.controller_is_maas: extra = (" Press enter on a machine ID to pin it to " "a specific MAAS node.") else: extra = "" self.subtitle = "Choose where to place {} unit{} of {}.{}".format( self.application.num_units, "" if self.application.num_units == 1 else "s", self.application.service_name, extra) # Shadow means temporary to the view, they are committed to # the controller if the user chooses OK # {machine_id : [(app, assignmenttype) ...] self.shadow_assignments = defaultdict(list) for machine_id, al in self.controller.assignments.items(): for (a, at) in al: if a == self.application: self.shadow_assignments[machine_id].append((a, at)) # {juju_machine_id : maas machine id} self.shadow_pins = copy.copy(self.controller.maas_machine_map) self.machine_pin_view = None self._machines = copy.deepcopy(app.metadata_controller.bundle.machines) self.closed = asyncio.Event() super().__init__() app.loop.create_task(self.update())
async def do_deploy(self): cloud_types = juju.get_cloud_types_by_name() default_series = app.metadata_controller.series machines = app.metadata_controller.bundle.machines applications = sorted(app.metadata_controller.bundle.services, key=attrgetter('service_name')) await common.pre_deploy(msg_cb=utils.info) await juju.add_machines(applications, machines, msg_cb=utils.info) tasks = [] for service in applications: # ignore placement when deploying to localhost if cloud_types[app.current_cloud] == "localhost": service.placement_spec = None tasks.append( juju.deploy_service(service, default_series, msg_cb=utils.info)) tasks.append(juju.set_relations(service, msg_cb=utils.info)) await asyncio.gather(*tasks) events.DeploymentComplete.set() controllers.use('deploystatus').render()
def finish(self, cloud): """ Load the selected cloud provider """ self.cancel_monitor.set() if cloud in CUSTOM_PROVIDERS: app.provider = load_schema(cloud) else: app.provider = load_schema(juju.get_cloud_types_by_name()[cloud]) try: app.provider.load(cloud) except SchemaErrorUnknownCloud: app.provider.cloud = utils.gen_cloud() if app.provider.model is None: app.provider.model = utils.gen_model() track_event("Cloud selection", app.provider.cloud, "") return controllers.use('credentials').render()
def render(self): # If bootstrap fails fast, we may be called after the error # screen was already shown. We should bail to avoid # overwriting the error screen. if events.Error.is_set(): return track_screen("Deploy") app.loop.create_task(common.pre_deploy(app.ui.set_footer)) self.applications = sorted(app.metadata_controller.bundle.services, key=attrgetter('service_name')) self.undeployed_applications = self.applications[:] cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type == 'maas': app.loop.create_task(self.connect_maas()) self.list_view = ApplicationListView(self.applications, app.metadata_controller, self) self.list_header = "Review and Configure Applications" app.ui.set_header(self.list_header) app.ui.set_body(self.list_view)
def render(self): track_screen("Cloud Creation") self.cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if app.current_controller is None: app.current_controller = "conjure-up-{}-{}".format( app.current_cloud, utils.gen_hash()) if app.current_model is None: app.current_model = utils.gen_model() # LXD is a special case as we want to make sure a bridge # is configured. If not we'll bring up a new view to allow # a user to configure a LXD bridge with suggested network # information. if self.cloud_type == 'localhost': lxd_setup_path = common.get_lxd_setup_path() app.log.debug("Determining if embedded LXD is setup and ready.") if lxd_setup_path.exists(): return self.finish() else: self.render_newcloud() # TODO: Prompt user to select credentials and set a region creds = common.try_get_creds(app.current_cloud) try: endpoint = juju.get_cloud(app.current_cloud).get( 'endpoint', None) except LookupError: endpoint = None if creds and (endpoint or self.cloud_type != 'maas'): return self.finish() # No existing credentials found, start credential editor self.render_newcloud()
def render(self): creds = None cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] if cloud_type != 'localhost': creds = common.try_get_creds(app.current_cloud) if not creds: utils.warning("You attempted to do an install against a cloud " "that requires credentials that could not be " "found. If you wish to supply those " "credentials please run " "`juju add-credential " "{}`.".format(app.current_cloud)) events.Shutdown.set(1) return if cloud_type == 'localhost': # Grab list of available physical networks to bind our bridge to iface = None try: ifaces = utils.get_physical_network_interfaces() # Grab the first physical network device that has an ip address iface = [ i for i in ifaces if utils.get_physical_network_ipaddr(i) ][0] except: utils.warning( "Could not find a suitable physical network interface " "to create a LXD bridge on. Please check your network " "configuration.") events.Shutdown.set(1) if not common.get_lxd_setup_path().exists(): common.lxd_init(iface) common.get_lxd_setup_path().touch() app.loop.create_task(self.finish(creds))
async def run(self, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = self.name step_path = Path(app.config['spell-dir']) / 'steps' / self.filename if not step_path.is_file(): return step_path = str(step_path) msg = "Running step: {}.".format(self.name) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if not os.access(step_path, os.X_OK): raise Exception("Step {} not executable".format(self.title)) if is_linux() and self.needs_sudo and not await can_sudo(): raise SudoError('Step "{}" requires sudo: {}'.format( self.title, 'password failed' if app.sudo_pass else 'passwordless sudo required', )) cloud_types = juju.get_cloud_types_by_name() provider_type = cloud_types[app.provider.cloud] app.env['JUJU_PROVIDERTYPE'] = provider_type # not all providers have a credential, e.g., localhost app.env['JUJU_CREDENTIAL'] = app.provider.credential or '' app.env['JUJU_CONTROLLER'] = app.provider.controller app.env['JUJU_MODEL'] = app.provider.model app.env['JUJU_REGION'] = app.provider.region or '' app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir if provider_type == "maas": app.env['MAAS_ENDPOINT'] = app.maas.endpoint app.env['MAAS_APIKEY'] = app.maas.api_key for step_name, step_data in app.steps_data.items(): for key, value in step_data.items(): app.env[key.upper()] = step_data[key] for key, value in app.env.items(): if value is None: app.log.warning('Env {} is None; ' 'replacing with empty string'.format(key)) app.env[key] = '' app.log.debug("Storing environment") async with aiofiles.open(step_path + ".env", 'w') as outf: for k, v in app.env.items(): if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k: await outf.write("{}=\"{}\" ".format(k.upper(), v)) app.log.debug("Executing script: {}".format(step_path)) async with aiofiles.open(step_path + ".out", 'w') as outf: async with aiofiles.open(step_path + ".err", 'w') as errf: proc = await asyncio.create_subprocess_exec(step_path, env=app.env, stdout=outf, stderr=errf) async with aiofiles.open(step_path + '.out', 'r') as f: while proc.returncode is None: async for line in f: msg_cb(line) await asyncio.sleep(0.01) out_log = Path(step_path + '.out').read_text() err_log = Path(step_path + '.err').read_text() if proc.returncode != 0: app.sentry.context.merge({ 'extra': { 'out_log_tail': out_log[-400:], 'err_log_tail': err_log[-400:], } }) raise Exception("Failure in step {}".format(self.filename)) # special case for 00_deploy-done to report masked # charm hook failures that were retried automatically if not app.noreport: failed_apps = set() # only report each charm once for line in err_log.splitlines(): if 'hook failure, will retry' in line: log_leader = line.split()[0] unit_name = log_leader.split(':')[-1] app_name = unit_name.split('/')[0] failed_apps.add(app_name) for app_name in failed_apps: # report each individually so that Sentry will give us a # breakdown of failures per-charm in addition to per-spell sentry_report('Retried hook failure', tags={ 'app_name': app_name, }) if event_name is not None: track_event(event_name, "Done", "") result_key = "conjure-up.{}.{}.result".format(app.config['spell'], self.name) result = app.state.get(result_key) return (result or '')
def main(): if os.geteuid() == 0: print("") print(" !! This should _not_ be run as root or with sudo. !!") print("") sys.exit(1) # Verify we can access ~/.local/share/juju if it exists juju_dir = pathlib.Path('~/.local/share/juju').expanduser() if juju_dir.exists(): try: for f in juju_dir.iterdir(): if f.is_file(): f.read_text() except PermissionError: print("") print(" !! Unable to read from ~/.local/share/juju, please " "double check your permissions on that directory " "and its files. !!") print("") sys.exit(1) utils.set_terminal_title("conjure-up") opts = parse_options(sys.argv[1:]) opt_defaults = parse_options([]) # Load conjurefile, merge any overridding options from argv if not opts.conf_file: opts.conf_file = [] if pathlib.Path('~/.config/conjure-up.conf').expanduser().exists(): opts.conf_file.insert( 0, pathlib.Path('~/.config/conjure-up.conf').expanduser()) if (pathlib.Path('.') / 'Conjurefile').exists(): opts.conf_file.insert(0, pathlib.Path('.') / 'Conjurefile') for conf in opts.conf_file: if not conf.exists(): print("Unable to locate config {} for processing.".format( str(conf))) sys.exit(1) try: app.conjurefile = Conjurefile.load(opts.conf_file) except ValueError as e: print(str(e)) sys.exit(1) app.conjurefile.merge_argv(opts, opt_defaults) if app.conjurefile['gen-config']: Conjurefile.print_tpl() sys.exit(0) spell = os.path.basename(os.path.abspath(app.conjurefile['spell'])) if not os.path.isdir(app.conjurefile['cache-dir']): os.makedirs(app.conjurefile['cache-dir']) # Application Config kv_db = os.path.join(app.conjurefile['cache-dir'], '.state.db') app.state = KV(kv_db) app.env = os.environ.copy() app.env['KV_DB'] = kv_db app.config = {'metadata': None} app.log = setup_logging(app, os.path.join(app.conjurefile['cache-dir'], 'conjure-up.log'), app.conjurefile.get('debug', False)) # Make sure juju paths are setup juju.set_bin_path() juju.set_wait_path() app.no_track = app.conjurefile['no-track'] app.no_report = app.conjurefile['no-report'] # Grab current LXD and Juju versions app.log.debug("Juju version: {}, " "conjure-up version: {}".format( utils.juju_version(), VERSION)) # Setup proxy apply_proxy() app.session_id = os.getenv('CONJURE_TEST_SESSION_ID', str(uuid.uuid4())) spells_dir = app.conjurefile['spells-dir'] app.config['spells-dir'] = spells_dir spells_index_path = os.path.join(app.config['spells-dir'], 'spells-index.yaml') spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'master') if not app.conjurefile['no-sync']: if not os.path.exists(spells_dir): utils.info("No spells found, syncing from registry, please wait.") try: download_or_sync_registry( app.conjurefile['registry'], spells_dir, branch=spells_registry_branch) except subprocess.CalledProcessError as e: if not os.path.exists(spells_dir): utils.error("Could not load from registry") sys.exit(1) app.log.debug( 'Could not sync spells from github: {}'.format(e)) else: if not os.path.exists(spells_index_path): utils.error( "You opted to not sync from the spells registry, however, " "we could not find any suitable spells in: " "{}".format(spells_dir)) sys.exit(1) with open(spells_index_path) as fp: app.spells_index = yaml.safe_load(fp.read()) addons_aliases_index_path = os.path.join(app.config['spells-dir'], 'addons-aliases.yaml') if os.path.exists(addons_aliases_index_path): with open(addons_aliases_index_path) as fp: app.addons_aliases = yaml.safe_load(fp.read()) spell_name = spell app.endpoint_type = detect_endpoint(app.conjurefile['spell']) if app.conjurefile['spell'] != consts.UNSPECIFIED_SPELL: app.spell_given = True # Check if spell is actually an addon addon = utils.find_addons_matching(app.conjurefile['spell']) if addon: app.log.debug("addon found, setting required spell") utils.set_chosen_spell(addon['spell'], os.path.join(app.conjurefile['cache-dir'], addon['spell'])) download_local(os.path.join(app.config['spells-dir'], addon['spell']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.selected_addons = addon['addons'] app.alias_given = True controllers.setup_metadata_controller() app.endpoint_type = EndpointType.LOCAL_DIR elif app.endpoint_type == EndpointType.LOCAL_SEARCH: spells = utils.find_spells_matching(app.conjurefile['spell']) if len(spells) == 0: utils.error("Can't find a spell matching '{}'".format( app.conjurefile['spell'])) sys.exit(1) # One result means it was a direct match and we can copy it # now. Changing the endpoint type then stops us from showing # the picker UI. More than one result means we need to show # the picker UI and will defer the copy to # SpellPickerController.finish(), so nothing to do here. if len(spells) == 1: app.log.debug("found spell {}".format(spells[0][1])) spell = spells[0][1] utils.set_chosen_spell(spell_name, os.path.join(app.conjurefile['cache-dir'], spell['key'])) download_local(os.path.join(app.config['spells-dir'], spell['key']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.endpoint_type = EndpointType.LOCAL_DIR # download spell if necessary elif app.endpoint_type == EndpointType.LOCAL_DIR: if not os.path.isdir(app.conjurefile['spell']): utils.warning("Could not find spell {}".format( app.conjurefile['spell'])) sys.exit(1) if not os.path.exists(os.path.join(app.conjurefile['spell'], "metadata.yaml")): utils.warning("'{}' does not appear to be a spell. " "{}/metadata.yaml was not found.".format( app.conjurefile['spell'], app.conjurefile['spell'])) sys.exit(1) spell_name = os.path.basename(os.path.abspath(spell)) utils.set_chosen_spell(spell_name, path.join(app.conjurefile['cache-dir'], spell_name)) download_local(app.conjurefile['spell'], app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]: utils.set_chosen_spell(spell, path.join( app.conjurefile['cache-dir'], spell)) remote = get_remote_url(app.conjurefile['spell']) if remote is None: utils.warning("Can't guess URL matching '{}'".format( app.conjurefile['spell'])) sys.exit(1) download(remote, app.config['spell-dir'], True) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.env['CONJURE_UP_CACHEDIR'] = app.conjurefile['cache-dir'] app.env['PATH'] = "/snap/bin:{}".format(app.env['PATH']) if app.conjurefile['show-env']: if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) show_env() app.sentry = raven.Client( dsn=SENTRY_DSN, release=VERSION, transport=RequestsHTTPTransport, processors=( 'conjureup.utils.SanitizeDataProcessor', ) ) track_screen("Application Start") track_event("OS", platform.platform(), "") app.loop = asyncio.get_event_loop() app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set) # Enable charmstore querying app.juju.charmstore = CharmStore(app.loop) try: if app.conjurefile.is_valid: cloud = None region = None if '/' in app.conjurefile['cloud']: parse_cli_cloud = app.conjurefile['cloud'].split('/') cloud, region = parse_cli_cloud app.log.debug( "Region found {} for cloud {}".format(cloud, region)) else: cloud = app.conjurefile['cloud'] cloud_types = juju.get_cloud_types_by_name() if cloud not in cloud_types: utils.error('Unknown cloud: {}'.format(cloud)) sys.exit(1) if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) app.provider = load_schema(cloud_types[cloud]) try: app.provider.load(cloud) except errors.SchemaCloudError as e: utils.error(e) sys.exit(1) if region: app.provider.region = region app.headless = True app.ui = None app.env['CONJURE_UP_HEADLESS'] = "1" app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) app.loop.run_forever() else: app.ui = ConjureUI() app.ui.set_footer('Press ? for help') EventLoop.build_loop(app.ui, STYLES, unhandled_input=events.unhandled_input, handle_mouse=False) app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) EventLoop.run() finally: # explicitly close asyncio event loop to avoid hitting the # following issue due to signal handlers added by # asyncio.create_subprocess_exec being cleaned up during final # garbage collection: https://github.com/python/asyncio/issues/396 app.loop.close() sys.exit(app.exit_code)
async def run(self, phase, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = self.name app.env['CONJURE_UP_PHASE'] = phase.value step_path = self._build_phase_path(phase) if not step_path.is_file(): return if not os.access(str(step_path), os.X_OK): app.log.error( 'Unable to run {} step {} {}, it is not executable'.format( self.source, step_path.stem, phase.value)) return step_path = str(step_path) msg = "Running {} step: {} {}.".format(self.source, self.name, phase.value) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if is_linux() and self.needs_sudo and not await can_sudo(): raise SudoError('The "{}" step "{}" requires sudo: {}'.format( self.source, self.title, 'password failed' if app.sudo_pass else 'passwordless sudo required', )) app.env['CONJURE_UP_SPELLSDIR'] = app.conjurefile['spells-dir'] app.env['CONJURE_UP_SESSION_ID'] = app.session_id if app.metadata.spell_type == spell_types.JUJU: cloud_types = juju.get_cloud_types_by_name() provider_type = cloud_types[app.provider.cloud] app.env['JUJU_PROVIDERTYPE'] = provider_type # not all providers have a credential, e.g., localhost app.env['JUJU_CREDENTIAL'] = app.provider.credential or '' app.env['JUJU_CONTROLLER'] = app.provider.controller app.env['JUJU_MODEL'] = app.provider.model app.env['JUJU_REGION'] = app.provider.region or '' if provider_type == "maas": app.env['MAAS_ENDPOINT'] = app.maas.endpoint app.env['MAAS_APIKEY'] = app.maas.api_key for step_name, step_data in app.steps_data.items(): for key, value in step_data.items(): app.env[key.upper()] = str(step_data[key]) for key, value in app.env.items(): if value is None: app.log.warning('Env {} is None; ' 'replacing with empty string'.format(key)) app.env[key] = '' app.log.debug("Storing environment") async with aiofiles.open(step_path + ".env", 'w') as outf: for k, v in app.env.items(): if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k: await outf.write("{}=\"{}\" ".format(k.upper(), v)) app.log.debug("Executing script: {}".format(step_path)) out_path = step_path + '.out' err_path = step_path + '.err' ret, out_log, err_log = await arun([step_path], stdout=out_path, stderr=err_path, cb_stdout=msg_cb) if ret != 0: app.sentry.context.merge({ 'extra': { 'out_log_tail': out_log[-400:], 'err_log_tail': err_log[-400:], } }) raise Exception("Failure in step {} {}".format( self.name, phase.value)) # special case for 00_deploy-done to report masked # charm hook failures that were retried automatically if not app.no_report and app.metadata.spell_type == spell_types.JUJU: failed_apps = set() # only report each charm once for line in err_log.splitlines(): if 'hook failure, will retry' in line: log_leader = line.split()[0] unit_name = log_leader.split(':')[-1] app_name = unit_name.split('/')[0] failed_apps.add(app_name) for app_name in failed_apps: # report each individually so that Sentry will give us a # breakdown of failures per-charm in addition to per-spell sentry_report('Retried hook failure', tags={ 'app_name': app_name, }) if event_name is not None: track_event(event_name, "Done", "") return self.get_state('result', phase)
def main(): if os.geteuid() == 0: print("") print(" !! This should _not_ be run as root or with sudo. !!") print("") sys.exit(1) # Verify we can access ~/.local/share/juju if it exists juju_dir = pathlib.Path('~/.local/share/juju').expanduser() if juju_dir.exists(): try: for f in juju_dir.iterdir(): if f.is_file(): f.read_text() except PermissionError: print("") print(" !! Unable to read from ~/.local/share/juju, please " "double check your permissions on that directory " "and its files. !!") print("") sys.exit(1) utils.set_terminal_title("conjure-up") opts = parse_options(sys.argv[1:]) spell = os.path.basename(os.path.abspath(opts.spell)) if not os.path.isdir(opts.cache_dir): os.makedirs(opts.cache_dir) # Application Config os.environ['UNIT_STATE_DB'] = os.path.join(opts.cache_dir, '.state.db') app.state = unitdata.kv() app.env = os.environ.copy() app.config = {'metadata': None} app.argv = opts app.log = setup_logging(app, os.path.join(opts.cache_dir, 'conjure-up.log'), opts.debug) if app.argv.conf_file.expanduser().exists(): conf = configparser.ConfigParser() conf.read_string(app.argv.conf_file.expanduser().read_text()) app.notrack = conf.getboolean('REPORTING', 'notrack', fallback=False) app.noreport = conf.getboolean('REPORTING', 'noreport', fallback=False) if app.argv.notrack: app.notrack = True if app.argv.noreport: app.noreport = True # Grab current LXD and Juju versions app.log.debug("Juju version: {}, " "conjure-up version: {}".format(utils.juju_version(), VERSION)) # Setup proxy apply_proxy() app.session_id = os.getenv('CONJURE_TEST_SESSION_ID', str(uuid.uuid4())) spells_dir = app.argv.spells_dir app.config['spells-dir'] = spells_dir spells_index_path = os.path.join(app.config['spells-dir'], 'spells-index.yaml') spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'stable') if not app.argv.nosync: if not os.path.exists(spells_dir): utils.info("No spells found, syncing from registry, please wait.") try: download_or_sync_registry(app.argv.registry, spells_dir, branch=spells_registry_branch) except subprocess.CalledProcessError as e: if not os.path.exists(spells_dir): utils.error("Could not load from registry") sys.exit(1) app.log.debug('Could not sync spells from github: {}'.format(e)) else: if not os.path.exists(spells_index_path): utils.error( "You opted to not sync from the spells registry, however, " "we could not find any suitable spells in: " "{}".format(spells_dir)) sys.exit(1) with open(spells_index_path) as fp: app.spells_index = yaml.safe_load(fp.read()) spell_name = spell app.endpoint_type = detect_endpoint(opts.spell) if app.endpoint_type == EndpointType.LOCAL_SEARCH: spells = utils.find_spells_matching(opts.spell) if len(spells) == 0: utils.error("Can't find a spell matching '{}'".format(opts.spell)) sys.exit(1) # One result means it was a direct match and we can copy it # now. Changing the endpoint type then stops us from showing # the picker UI. More than one result means we need to show # the picker UI and will defer the copy to # SpellPickerController.finish(), so nothing to do here. if len(spells) == 1: app.log.debug("found spell {}".format(spells[0][1])) spell = spells[0][1] utils.set_chosen_spell(spell_name, os.path.join(opts.cache_dir, spell['key'])) download_local( os.path.join(app.config['spells-dir'], spell['key']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.endpoint_type = EndpointType.LOCAL_DIR # download spell if necessary elif app.endpoint_type == EndpointType.LOCAL_DIR: if not os.path.isdir(opts.spell): utils.warning("Could not find spell {}".format(opts.spell)) sys.exit(1) if not os.path.exists(os.path.join(opts.spell, "metadata.yaml")): utils.warning("'{}' does not appear to be a spell. " "{}/metadata.yaml was not found.".format( opts.spell, opts.spell)) sys.exit(1) spell_name = os.path.basename(os.path.abspath(spell)) utils.set_chosen_spell(spell_name, path.join(opts.cache_dir, spell_name)) download_local(opts.spell, app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]: utils.set_chosen_spell(spell, path.join(opts.cache_dir, spell)) remote = get_remote_url(opts.spell) if remote is None: utils.warning("Can't guess URL matching '{}'".format(opts.spell)) sys.exit(1) download(remote, app.config['spell-dir'], True) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.env['CONJURE_UP_CACHEDIR'] = app.argv.cache_dir if app.argv.show_env: if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) show_env() app.sentry = raven.Client( dsn=SENTRY_DSN, release=VERSION, transport=RequestsHTTPTransport, processors=('conjureup.utils.SanitizeDataProcessor', )) track_screen("Application Start") track_event("OS", platform.platform(), "") app.loop = asyncio.get_event_loop() app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set) try: if app.argv.cloud: cloud = None region = None if '/' in app.argv.cloud: parse_cli_cloud = app.argv.cloud.split('/') cloud, region = parse_cli_cloud app.log.debug("Region found {} for cloud {}".format( cloud, region)) else: cloud = app.argv.cloud cloud_types = juju.get_cloud_types_by_name() if cloud not in cloud_types: utils.error('Unknown cloud: {}'.format(cloud)) sys.exit(1) if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) app.provider = load_schema(cloud_types[cloud]) try: app.provider.load(cloud) except SchemaErrorUnknownCloud as e: utils.error(e) sys.exit(1) if region: app.provider.region = region app.headless = True app.ui = None app.env['CONJURE_UP_HEADLESS'] = "1" app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) app.loop.run_forever() else: app.ui = ConjureUI() EventLoop.build_loop(app.ui, STYLES, unhandled_input=events.unhandled_input) app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) EventLoop.run() finally: # explicitly close asyncio event loop to avoid hitting the # following issue due to signal handlers added by # asyncio.create_subprocess_exec being cleaned up during final # garbage collection: https://github.com/python/asyncio/issues/396 app.loop.close() sys.exit(app.exit_code)