async def do_destroy(self, model, controller): track_event("Destroying model", "Destroy", "") app.ui.set_footer("Destroying model {} in controller {}".format( model, controller)) await juju.destroy_model(controller, model) app.ui.set_footer("") controllers.use('destroy').render()
async def do_bootstrap(self): self.emit('Bootstrapping Juju controller.') track_event("Juju Bootstrap", "Started", "") cloud_with_region = app.provider.cloud if app.provider.region: cloud_with_region = '/'.join( [app.provider.cloud, app.provider.region]) success = await juju.bootstrap(app.provider.controller, cloud_with_region, app.provider.model, credential=app.provider.credential) if not success: log_file = '{}-bootstrap.err'.format(app.provider.controller) log_file = Path(app.config['spell-dir']) / log_file err_log = log_file.read_text('utf8').splitlines() app.log.error("Error bootstrapping controller: " "{}".format(err_log)) err_tail = err_log[-400:] app.sentry.context.merge({'extra': {'err_tail': err_tail}}) raise errors.BootstrapError( 'Unable to bootstrap (cloud type: {})'.format( app.provider.cloud_type)) self.emit('Bootstrap complete.') track_event("Juju Bootstrap", "Done", "") await juju.login() # login to the newly created (default) model events.Bootstrapped.set()
def __handle_bootstrap_done(self, future): app.log.debug("handle bootstrap") if future.exception(): return result = future.result() if result.returncode < 0: self.__handle_exception( Exception("Bootstrap killed by user: {}".format( result.returncode))) return if result.returncode > 0: pathbase = os.path.join(app.config['spell-dir'], '{}-bootstrap').format( app.current_controller) with open(pathbase + ".err") as errf: err = "\n".join(errf.readlines()) app.log.error(err) e = Exception("Bootstrap error: {}".format(err)) self.__handle_exception(e) return track_event("Juju Bootstrap", "Done", "") app.ui.set_footer('Bootstrap complete...') self.__post_bootstrap_exec()
def _pre_deploy_exec(self): """ runs pre deploy script if exists """ app.env['JUJU_PROVIDERTYPE'] = model_info( app.current_model)['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir pre_deploy_sh = os.path.join(app.config['spell-dir'], 'steps/00_pre-deploy') if os.path.isfile(pre_deploy_sh) \ and os.access(pre_deploy_sh, os.X_OK): track_event("Juju Pre-Deploy", "Started", "") msg = "Running pre-deployment tasks." app.log.debug(msg) app.ui.set_footer(msg) out = utils.run(pre_deploy_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) try: return json.loads(out.stdout.decode()) except json.decoder.JSONDecodeError as e: app.log.exception(out.stdout.decode()) app.log.exception(out.stderr.decode()) raise Exception(out) return { 'message': 'No pre deploy necessary', 'returnCode': 0, 'isComplete': True }
def finish(self, credentials=None, back=False): """ Load the Model controller passing along the selected cloud. Arguments: credentials: credentials to store for provider back: if true loads previous controller """ if back: return controllers.use('clouds').render() if credentials is not None: common.save_creds(self.cloud, credentials) credentials_key = common.try_get_creds(self.cloud) if self.cloud == 'maas': self.cloud = '{}/{}'.format(self.cloud, credentials['@maas-server'].value) # Store the MAAS information for the api client maas_api_key = common.parse_maas_apikey( credentials['maas-oauth'].value) app.maas.client = MaasClient( server_address=credentials['@maas-server'].value, consumer_key=maas_api_key[0], token_key=maas_api_key[1], token_secret=maas_api_key[2]) track_event("Credentials", "Added", "") self.__do_bootstrap(credential=credentials_key)
def __post_bootstrap_exec(self): """ Executes post-bootstrap.sh if exists """ info = model_info(app.current_model) # Set our provider type environment var so that it is # exposed in future processing tasks app.env['JUJU_PROVIDERTYPE'] = info['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model _post_bootstrap_sh = path.join(app.config['spell-dir'], 'steps/00_post-bootstrap') app.log.debug( 'Checking for post bootstrap task: {}'.format(_post_bootstrap_sh)) if path.isfile(_post_bootstrap_sh) \ and os.access(_post_bootstrap_sh, os.X_OK): app.ui.set_footer('Running post-bootstrap tasks...') track_event("Juju Post-Bootstrap", "Started", "") app.log.debug("post_bootstrap running: {}".format( _post_bootstrap_sh )) try: future = async.submit(partial(check_output, _post_bootstrap_sh, shell=True, env=app.env), self.__handle_exception) future.add_done_callback(self.__post_bootstrap_done) except Exception as e: return self.__handle_exception(e)
async def do_destroy(self, model, controller): track_event("Destroying model", "Destroy", "") utils.info("Destroying model {} in controller {}".format( model, controller)) await juju.destroy_model(controller, model) utils.info("Model has been removed") events.Shutdown.set(0)
def __post_bootstrap_exec(self): """ Executes post-bootstrap.sh if exists """ info = model_info(app.current_model) # Set our provider type environment var so that it is # exposed in future processing tasks app.env['JUJU_PROVIDERTYPE'] = info['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model _post_bootstrap_sh = path.join(app.config['spell-dir'], 'steps/00_post-bootstrap') app.log.debug( 'Checking for post bootstrap task: {}'.format(_post_bootstrap_sh)) if path.isfile(_post_bootstrap_sh) \ and os.access(_post_bootstrap_sh, os.X_OK): app.ui.set_footer('Running post-bootstrap tasks...') track_event("Juju Post-Bootstrap", "Started", "") app.log.debug( "post_bootstrap running: {}".format(_post_bootstrap_sh)) try: future = async .submit( partial(check_output, _post_bootstrap_sh, shell=True, env=app.env), self.__handle_exception) if future: future.add_done_callback(self.__post_bootstrap_done) except Exception as e: return self.__handle_exception(e)
def _pre_deploy_exec(self): """ runs pre deploy script if exists """ app.env['JUJU_PROVIDERTYPE'] = model_info( app.current_model)['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model pre_deploy_sh = os.path.join(app.config['spell-dir'], 'steps/00_pre-deploy') if os.path.isfile(pre_deploy_sh) \ and os.access(pre_deploy_sh, os.X_OK): track_event("Juju Pre-Deploy", "Started", "") msg = "Running pre-deployment tasks." app.log.debug(msg) app.ui.set_footer(msg) return utils.run(pre_deploy_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) return json.dumps({ 'message': 'No pre deploy necessary', 'returnCode': 0, 'isComplete': True })
async def do_bootstrap(self): await self.pre_bootstrap() self.emit('Bootstrapping Juju controller.') track_event("Juju Bootstrap", "Started", "") cloud_with_region = app.provider.cloud if app.provider.region: cloud_with_region = '/'.join([app.provider.cloud, app.provider.region]) success = await juju.bootstrap(app.provider.controller, cloud_with_region, app.provider.model, credential=app.provider.credential) if not success: log_file = '{}-bootstrap.err'.format(app.provider.controller) log_file = Path(app.config['spell-dir']) / log_file err_log = log_file.read_text('utf8').splitlines() app.log.error("Error bootstrapping controller: " "{}".format(err_log)) app.sentry.context.merge({'extra': {'err_log': err_log[-400:]}}) raise Exception('Unable to bootstrap (cloud type: {})'.format( app.provider.cloud_type)) self.emit('Bootstrap complete.') track_event("Juju Bootstrap", "Done", "") await juju.login() # login to the newly created (default) model step = StepModel({}, filename='00_post-bootstrap', name='post-bootstrap') await step.run(self.msg_cb, 'Juju Post-Bootstrap') events.Bootstrapped.set()
def finish(self): app.selected_addons = self.view.selected if app.selected_addons: for addon in app.selected_addons: track_event("Addon Selected", addon, "") # reload the bundle data w/ addons merged setup_metadata_controller() self.next_screen()
def finish(self): app.selected_addons = self.view.selected if app.selected_addons: for addon in app.selected_addons: track_event("Addon Selected", addon, "") # reload the bundle data w/ addons merged setup_metadata_controller() controllers.use('clouds').render()
def __do_destroy(self, controller_name, model_name): track_event("Destroying model", "Destroy", "") app.ui.set_footer("Destroying {} deployment, please wait.".format( model_name)) future = juju.destroy_model_async(controller=controller_name, model=model_name, exc_cb=self.__handle_exception) if future: future.add_done_callback(self.__handle_destroy_done)
def finish(self, cloud): """Show the 'newcloud' screen to enter credentials for a new controller on 'cloud'. There will not be an existing controller. Arguments: cloud: Cloud to create the controller/model on. """ app.current_cloud = cloud track_event("Cloud selection", app.current_cloud, "") return controllers.use('newcloud').render()
async def run_step(step, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # redis-cli set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = step.filename step_path = Path(app.config['spell-dir']) / 'steps' / step.filename if not step_path.is_file(): return step_path = str(step_path) msg = "Running step: {}.".format(step.name) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if not os.access(step_path, os.X_OK): raise Exception("Step {} not executable".format(step.title)) app.log.debug("Executing script: {}".format(step_path)) async with aiofiles.open(step_path + ".out", 'w') as outf: async with aiofiles.open(step_path + ".err", 'w') as errf: proc = await asyncio.create_subprocess_exec(step_path, env=app.env, stdout=outf, stderr=errf) async with aiofiles.open(step_path + '.out', 'r') as f: while proc.returncode is None: async for line in f: msg_cb(line) await asyncio.sleep(0.01) if proc.returncode != 0: status = run(['juju', 'status', '--format=yaml']) app.sentry.context.merge({'extra': { 'out_log': Path(step_path + '.out').read_text(), 'err_log': Path(step_path + '.err').read_text(), 'status': (status.stdout or b'').decode('utf8'), }}) raise Exception("Failure in step {}".format(step.filename)) if event_name is not None: track_event(event_name, "Done", "") result = app.state.get( "conjure-up.{}.{}.result".format(app.config['spell'], step.filename)) return (result or b'').decode('utf8')
def __post_bootstrap_done(self, future): try: result = json.loads(future.result().decode('utf8')) except Exception as e: return self.__handle_exception(e) app.log.debug("post_bootstrap_done: {}".format(result)) if result['returnCode'] > 0: track_exception("Error in Post-Bootstrap") return self.__handle_exception( Exception('There was an error during the post ' 'bootstrap processing phase: {}.'.format(result))) track_event("Juju Post-Bootstrap", "Done", "") app.ui.set_footer('')
def __post_bootstrap_done(self, future): try: result = json.loads(future.result().decode('utf8')) except Exception as e: return self.__handle_exception(e) app.log.debug("post_bootstrap_done: {}".format(result)) if result['returnCode'] > 0: track_exception("Error in Post-Bootstrap") return self.__handle_exception(Exception( 'There was an error during the post ' 'bootstrap processing phase: {}.'.format(result))) track_event("Juju Post-Bootstrap", "Done", "") app.ui.set_footer('')
def finish(self, cloud): """Save the selected cloud and move on to the region selection screen. """ if cloud == 'maas': app.current_cloud_type = 'maas' app.current_cloud = utils.gen_cloud() else: app.current_cloud_type = juju.get_cloud_types_by_name()[cloud] app.current_cloud = cloud if app.current_model is None: app.current_model = utils.gen_model() track_event("Cloud selection", app.current_cloud, "") return controllers.use('regions').render()
def __handle_bootstrap_done(self, future): app.log.debug("handle bootstrap") result = future.result() if result.returncode < 0: # bootstrap killed via user signal, we're quitting return if result.returncode > 0: pathbase = os.path.join( app.config['spell-dir'], '{}-bootstrap').format(app.current_controller) with open(pathbase + ".err") as errf: err = "\n".join(errf.readlines()) app.log.error(err) e = Exception("Bootstrap error: {}".format(err)) return self.__handle_exception(e) track_event("Juju Bootstrap", "Done", "") app.ui.set_footer('Bootstrap complete...') self.__post_bootstrap_exec()
async def _start(*args, **kwargs): # NB: we have to set the exception handler here because we need to # override the one set by urwid, which happens in MainLoop.run() app.loop.set_exception_handler(events.handle_exception) track_screen("Application Start") track_event("OS", platform.platform(), "") if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: controllers.use('spellpicker').render() return controllers.setup_metadata_controller() if app.headless: controllers.use('clouds').render() elif app.selected_addons: controllers.use('clouds').render() else: controllers.use('addons').render()
def finish(self, cloud): """ Load the selected cloud provider """ self.cancel_monitor.set() if cloud in CUSTOM_PROVIDERS: app.provider = load_schema(cloud) else: app.provider = load_schema(juju.get_cloud_types_by_name()[cloud]) try: app.provider.load(cloud) except SchemaErrorUnknownCloud: app.provider.cloud = utils.gen_cloud() if app.provider.model is None: app.provider.model = utils.gen_model() track_event("Cloud selection", app.provider.cloud, "") return controllers.use('credentials').render()
def __handle_bootstrap_done(self, future): app.log.debug("handle bootstrap") result = future.result() if result.returncode < 0: # bootstrap killed via user signal, we're quitting return if result.returncode > 0: pathbase = os.path.join(app.config['spell-dir'], '{}-bootstrap').format( app.current_controller) with open(pathbase + ".err") as errf: err = "\n".join(errf.readlines()) app.log.error(err) e = Exception("Bootstrap error: {}".format(err)) return self.__handle_exception(e) track_event("Juju Bootstrap", "Done", "") EventLoop.remove_alarms() app.ui.set_footer('Bootstrap complete...') self.__post_bootstrap_exec()
async def do_bootstrap(self, creds): if app.is_jaas: return await self.pre_bootstrap() self.emit('Bootstrapping Juju controller.') track_event("Juju Bootstrap", "Started", "") cloud_with_region = app.current_cloud if app.current_region: cloud_with_region = '/'.join( [app.current_cloud, app.current_region]) success = await juju.bootstrap(app.current_controller, cloud_with_region, app.current_model, credential=creds) if not success: log_file = '{}-bootstrap.err'.format(app.current_controller) log_file = Path(app.config['spell-dir']) / log_file err_log = log_file.read_text('utf8').splitlines() app.log.error("Error bootstrapping controller: " "{}".format(err_log)) app.sentry.context.merge({'extra': {'err_log': err_log}}) raise Exception('Unable to bootstrap (cloud type: {})'.format( app.current_cloud_type)) self.emit('Bootstrap complete.') track_event("Juju Bootstrap", "Done", "") await juju.login() # login to the newly created (default) model # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = app.juju.client.info.provider_type app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model step = StepModel({}, filename='00_post-bootstrap', name='post-bootstrap') await utils.run_step(step, self.msg_cb, 'Juju Post-Bootstrap') events.Bootstrapped.set()
async def do_bootstrap(creds, msg_cb, fail_msg_cb): if not app.is_jaas: await pre_bootstrap(msg_cb) app.log.info('Bootstrapping Juju controller.') msg_cb('Bootstrapping Juju controller.') track_event("Juju Bootstrap", "Started", "") cloud_with_region = app.current_cloud if app.current_region: cloud_with_region = '/'.join( [app.current_cloud, app.current_region]) success = await juju.bootstrap(app.current_controller, cloud_with_region, app.current_model, credential=creds) if not success: pathbase = os.path.join(app.config['spell-dir'], '{}-bootstrap').format( app.current_controller) with open(pathbase + ".err") as errf: err_log = "\n".join(errf.readlines()) msg = "Error bootstrapping controller: {}".format(err_log) app.log.error(msg) fail_msg_cb(msg) cloud_type = juju.get_cloud_types_by_name()[app.current_cloud] raise Exception( 'Unable to bootstrap (cloud type: {})'.format(cloud_type)) return app.log.info('Bootstrap complete.') msg_cb('Bootstrap complete.') track_event("Juju Bootstrap", "Done", "") await juju.login() # login to the newly created (default) model # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = app.juju.client.info.provider_type app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model await utils.run_step('00_post-bootstrap', 'post-bootstrap', msg_cb, 'Juju Post-Bootstrap') else: app.log.info('Adding new model in the background.') msg_cb('Adding new model in the background.') track_event("Juju Add JaaS Model", "Started", "") await juju.add_model(app.current_model, app.current_controller, app.current_cloud) track_event("Juju Add JaaS Model", "Done", "") app.log.info('Add model complete.') msg_cb('Add model complete.')
async def do_add_model(self): if await juju.model_available(): self.emit('Connecting to Juju model {}...'.format( app.provider.model)) await juju.connect_model() self.emit('Juju model connected.') else: self.emit('Creating Juju model {}...'.format(app.provider.model)) cloud_with_region = app.provider.cloud if app.provider.region: cloud_with_region = '/'.join( [app.provider.cloud, app.provider.region]) track_event( "Juju Add Model", "Started", "{}{}".format(cloud_with_region, ' on JAAS' if app.is_jaas else '')) await juju.create_model() track_event( "Juju Add Model", "Done", "{}{}".format(cloud_with_region, 'on JAAS' if app.is_jaas else '')) self.emit('Juju model created.') events.Bootstrapped.set()
def _pre_deploy_exec(self): """ runs pre deploy script if exists """ app.env['JUJU_PROVIDERTYPE'] = model_info( app.current_model)['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model pre_deploy_sh = os.path.join(app.config['spell-dir'], 'steps/00_pre-deploy') if os.path.isfile(pre_deploy_sh) \ and os.access(pre_deploy_sh, os.X_OK): track_event("Juju Pre-Deploy", "Started", "") msg = "Running pre-deployment tasks." app.log.debug(msg) app.ui.set_footer(msg) return utils.run(pre_deploy_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) return json.dumps({'message': 'No pre deploy necessary', 'returnCode': 0, 'isComplete': True})
def set_chosen_spell(spell_name, spell_dir): track_event("Spell Choice", spell_name, "") app.env['CONJURE_UP_SPELL'] = spell_name app.config.update({'spell-dir': spell_dir, 'spell': spell_name})
async def run_step(step, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # redis-cli set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = step.name step_path = Path(app.config['spell-dir']) / 'steps' / step.filename if not step_path.is_file(): return step_path = str(step_path) msg = "Running step: {}.".format(step.name) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if not os.access(step_path, os.X_OK): raise Exception("Step {} not executable".format(step.title)) if is_linux() and step.needs_sudo and not await can_sudo(): raise SudoError('Step "{}" requires sudo: {}'.format( step.title, 'password failed' if app.sudo_pass else 'passwordless sudo required', )) app.log.debug("Executing script: {}".format(step_path)) async with aiofiles.open(step_path + ".out", 'w') as outf: async with aiofiles.open(step_path + ".err", 'w') as errf: proc = await asyncio.create_subprocess_exec(step_path, env=app.env, stdout=outf, stderr=errf) async with aiofiles.open(step_path + '.out', 'r') as f: while proc.returncode is None: async for line in f: msg_cb(line) await asyncio.sleep(0.01) out_log = Path(step_path + '.out').read_text() err_log = Path(step_path + '.err').read_text() if proc.returncode != 0: app.sentry.context.merge({ 'extra': { 'out_log_tail': out_log[-400:], 'err_log_tail': err_log[-400:], } }) raise Exception("Failure in step {}".format(step.filename)) # special case for 00_deploy-done to report masked # charm hook failures that were retried automatically if not app.noreport: failed_apps = set() # only report each charm once for line in err_log.splitlines(): if 'hook failure, will retry' in line: log_leader = line.split()[0] unit_name = log_leader.split(':')[-1] app_name = unit_name.split('/')[0] failed_apps.add(app_name) for app_name in failed_apps: # report each individually so that Sentry will give us a # breakdown of failures per-charm in addition to per-spell sentry_report('Retried hook failure', tags={ 'app_name': app_name, }) if event_name is not None: track_event(event_name, "Done", "") result_key = "conjure-up.{}.{}.result".format(app.config['spell'], step.name) result = app.state.get(result_key) return (result or b'').decode('utf8')
async def run(self, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = self.name step_path = Path(app.config['spell-dir']) / 'steps' / self.filename if not step_path.is_file(): return step_path = str(step_path) msg = "Running step: {}.".format(self.name) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if not os.access(step_path, os.X_OK): raise Exception("Step {} not executable".format(self.title)) if is_linux() and self.needs_sudo and not await can_sudo(): raise SudoError('Step "{}" requires sudo: {}'.format( self.title, 'password failed' if app.sudo_pass else 'passwordless sudo required', )) cloud_types = juju.get_cloud_types_by_name() provider_type = cloud_types[app.provider.cloud] app.env['JUJU_PROVIDERTYPE'] = provider_type # not all providers have a credential, e.g., localhost app.env['JUJU_CREDENTIAL'] = app.provider.credential or '' app.env['JUJU_CONTROLLER'] = app.provider.controller app.env['JUJU_MODEL'] = app.provider.model app.env['JUJU_REGION'] = app.provider.region or '' app.env['CONJURE_UP_SPELLSDIR'] = app.argv.spells_dir if provider_type == "maas": app.env['MAAS_ENDPOINT'] = app.maas.endpoint app.env['MAAS_APIKEY'] = app.maas.api_key for step_name, step_data in app.steps_data.items(): for key, value in step_data.items(): app.env[key.upper()] = step_data[key] for key, value in app.env.items(): if value is None: app.log.warning('Env {} is None; ' 'replacing with empty string'.format(key)) app.env[key] = '' app.log.debug("Storing environment") async with aiofiles.open(step_path + ".env", 'w') as outf: for k, v in app.env.items(): if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k: await outf.write("{}=\"{}\" ".format(k.upper(), v)) app.log.debug("Executing script: {}".format(step_path)) async with aiofiles.open(step_path + ".out", 'w') as outf: async with aiofiles.open(step_path + ".err", 'w') as errf: proc = await asyncio.create_subprocess_exec(step_path, env=app.env, stdout=outf, stderr=errf) async with aiofiles.open(step_path + '.out', 'r') as f: while proc.returncode is None: async for line in f: msg_cb(line) await asyncio.sleep(0.01) out_log = Path(step_path + '.out').read_text() err_log = Path(step_path + '.err').read_text() if proc.returncode != 0: app.sentry.context.merge({ 'extra': { 'out_log_tail': out_log[-400:], 'err_log_tail': err_log[-400:], } }) raise Exception("Failure in step {}".format(self.filename)) # special case for 00_deploy-done to report masked # charm hook failures that were retried automatically if not app.noreport: failed_apps = set() # only report each charm once for line in err_log.splitlines(): if 'hook failure, will retry' in line: log_leader = line.split()[0] unit_name = log_leader.split(':')[-1] app_name = unit_name.split('/')[0] failed_apps.add(app_name) for app_name in failed_apps: # report each individually so that Sentry will give us a # breakdown of failures per-charm in addition to per-spell sentry_report('Retried hook failure', tags={ 'app_name': app_name, }) if event_name is not None: track_event(event_name, "Done", "") result_key = "conjure-up.{}.{}.result".format(app.config['spell'], self.name) result = app.state.get(result_key) return (result or '')
def main(): if os.geteuid() == 0: print("") print(" !! This should _not_ be run as root or with sudo. !!") print("") sys.exit(1) # Verify we can access ~/.local/share/juju if it exists juju_dir = pathlib.Path('~/.local/share/juju').expanduser() if juju_dir.exists(): try: for f in juju_dir.iterdir(): if f.is_file(): f.read_text() except PermissionError: print("") print(" !! Unable to read from ~/.local/share/juju, please " "double check your permissions on that directory " "and its files. !!") print("") sys.exit(1) utils.set_terminal_title("conjure-up") opts = parse_options(sys.argv[1:]) spell = os.path.basename(os.path.abspath(opts.spell)) if not os.path.isdir(opts.cache_dir): os.makedirs(opts.cache_dir) # Application Config os.environ['UNIT_STATE_DB'] = os.path.join(opts.cache_dir, '.state.db') app.state = unitdata.kv() app.env = os.environ.copy() app.config = {'metadata': None} app.argv = opts app.log = setup_logging(app, os.path.join(opts.cache_dir, 'conjure-up.log'), opts.debug) if app.argv.conf_file.expanduser().exists(): conf = configparser.ConfigParser() conf.read_string(app.argv.conf_file.expanduser().read_text()) app.notrack = conf.getboolean('REPORTING', 'notrack', fallback=False) app.noreport = conf.getboolean('REPORTING', 'noreport', fallback=False) if app.argv.notrack: app.notrack = True if app.argv.noreport: app.noreport = True # Grab current LXD and Juju versions app.log.debug("Juju version: {}, " "conjure-up version: {}".format(utils.juju_version(), VERSION)) # Setup proxy apply_proxy() app.session_id = os.getenv('CONJURE_TEST_SESSION_ID', str(uuid.uuid4())) spells_dir = app.argv.spells_dir app.config['spells-dir'] = spells_dir spells_index_path = os.path.join(app.config['spells-dir'], 'spells-index.yaml') spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'stable') if not app.argv.nosync: if not os.path.exists(spells_dir): utils.info("No spells found, syncing from registry, please wait.") try: download_or_sync_registry(app.argv.registry, spells_dir, branch=spells_registry_branch) except subprocess.CalledProcessError as e: if not os.path.exists(spells_dir): utils.error("Could not load from registry") sys.exit(1) app.log.debug('Could not sync spells from github: {}'.format(e)) else: if not os.path.exists(spells_index_path): utils.error( "You opted to not sync from the spells registry, however, " "we could not find any suitable spells in: " "{}".format(spells_dir)) sys.exit(1) with open(spells_index_path) as fp: app.spells_index = yaml.safe_load(fp.read()) spell_name = spell app.endpoint_type = detect_endpoint(opts.spell) if app.endpoint_type == EndpointType.LOCAL_SEARCH: spells = utils.find_spells_matching(opts.spell) if len(spells) == 0: utils.error("Can't find a spell matching '{}'".format(opts.spell)) sys.exit(1) # One result means it was a direct match and we can copy it # now. Changing the endpoint type then stops us from showing # the picker UI. More than one result means we need to show # the picker UI and will defer the copy to # SpellPickerController.finish(), so nothing to do here. if len(spells) == 1: app.log.debug("found spell {}".format(spells[0][1])) spell = spells[0][1] utils.set_chosen_spell(spell_name, os.path.join(opts.cache_dir, spell['key'])) download_local( os.path.join(app.config['spells-dir'], spell['key']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.endpoint_type = EndpointType.LOCAL_DIR # download spell if necessary elif app.endpoint_type == EndpointType.LOCAL_DIR: if not os.path.isdir(opts.spell): utils.warning("Could not find spell {}".format(opts.spell)) sys.exit(1) if not os.path.exists(os.path.join(opts.spell, "metadata.yaml")): utils.warning("'{}' does not appear to be a spell. " "{}/metadata.yaml was not found.".format( opts.spell, opts.spell)) sys.exit(1) spell_name = os.path.basename(os.path.abspath(spell)) utils.set_chosen_spell(spell_name, path.join(opts.cache_dir, spell_name)) download_local(opts.spell, app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]: utils.set_chosen_spell(spell, path.join(opts.cache_dir, spell)) remote = get_remote_url(opts.spell) if remote is None: utils.warning("Can't guess URL matching '{}'".format(opts.spell)) sys.exit(1) download(remote, app.config['spell-dir'], True) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.env['CONJURE_UP_CACHEDIR'] = app.argv.cache_dir if app.argv.show_env: if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) show_env() app.sentry = raven.Client( dsn=SENTRY_DSN, release=VERSION, transport=RequestsHTTPTransport, processors=('conjureup.utils.SanitizeDataProcessor', )) track_screen("Application Start") track_event("OS", platform.platform(), "") app.loop = asyncio.get_event_loop() app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set) try: if app.argv.cloud: cloud = None region = None if '/' in app.argv.cloud: parse_cli_cloud = app.argv.cloud.split('/') cloud, region = parse_cli_cloud app.log.debug("Region found {} for cloud {}".format( cloud, region)) else: cloud = app.argv.cloud cloud_types = juju.get_cloud_types_by_name() if cloud not in cloud_types: utils.error('Unknown cloud: {}'.format(cloud)) sys.exit(1) if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) app.provider = load_schema(cloud_types[cloud]) try: app.provider.load(cloud) except SchemaErrorUnknownCloud as e: utils.error(e) sys.exit(1) if region: app.provider.region = region app.headless = True app.ui = None app.env['CONJURE_UP_HEADLESS'] = "1" app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) app.loop.run_forever() else: app.ui = ConjureUI() EventLoop.build_loop(app.ui, STYLES, unhandled_input=events.unhandled_input) app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) EventLoop.run() finally: # explicitly close asyncio event loop to avoid hitting the # following issue due to signal handlers added by # asyncio.create_subprocess_exec being cleaned up during final # garbage collection: https://github.com/python/asyncio/issues/396 app.loop.close() sys.exit(app.exit_code)
async def run_step(step_file, step_title, msg_cb, event_name=None): step_path = Path(app.config['spell-dir']) / 'steps' / step_file if not step_path.is_file(): return step_path = str(step_path) msg = "Running {}.".format(step_title) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if not os.access(step_path, os.X_OK): raise Exception("Step {} not executable".format(step_title)) while True: app.log.debug("Executing script: {}".format(step_path)) with open(step_path + ".out", 'w') as outf: with open(step_path + ".err", 'w') as errf: proc = await asyncio.create_subprocess_exec(step_path, env=app.env, stdout=outf, stderr=errf) await proc.wait() try: stderr = Path(step_path + '.err').read_text() except Exception: stderr = None if proc.returncode != 0: raise Exception("Failure in step {}: {}".format(step_file, stderr)) try: result = json.loads(Path(step_path + '.out').read_text()) except OSError as e: raise Exception("Unable to read output from step {}: {}".format( step_file, e)) except json.decoder.JSONDecodeError as e: raise Exception("Unable to parse output from step {}: {}".format( step_file, e)) if 'returnCode' not in result: raise Exception("Invalid message from step {}".format(step_file)) if result['returnCode'] != 0: raise Exception("Failure in step {}: {}".format( step_file, result['message'])) if result.get('isComplete', True): break msg_cb("{}, please wait".format(result['message'])) msg = "Finished {}: {}".format(step_title, result['message']) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Done", "") return result['message']
async def run(self, phase, msg_cb, event_name=None): # Define STEP_NAME for use in determining where to store # our step results, # state set "conjure-up.$SPELL_NAME.$STEP_NAME.result" "val" app.env['CONJURE_UP_STEP'] = self.name app.env['CONJURE_UP_PHASE'] = phase.value step_path = self._build_phase_path(phase) if not step_path.is_file(): return if not os.access(str(step_path), os.X_OK): app.log.error( 'Unable to run {} step {} {}, it is not executable'.format( self.source, step_path.stem, phase.value)) return step_path = str(step_path) msg = "Running {} step: {} {}.".format(self.source, self.name, phase.value) app.log.info(msg) msg_cb(msg) if event_name is not None: track_event(event_name, "Started", "") if is_linux() and self.needs_sudo and not await can_sudo(): raise SudoError('The "{}" step "{}" requires sudo: {}'.format( self.source, self.title, 'password failed' if app.sudo_pass else 'passwordless sudo required', )) app.env['CONJURE_UP_SPELLSDIR'] = app.conjurefile['spells-dir'] app.env['CONJURE_UP_SESSION_ID'] = app.session_id if app.metadata.spell_type == spell_types.JUJU: cloud_types = juju.get_cloud_types_by_name() provider_type = cloud_types[app.provider.cloud] app.env['JUJU_PROVIDERTYPE'] = provider_type # not all providers have a credential, e.g., localhost app.env['JUJU_CREDENTIAL'] = app.provider.credential or '' app.env['JUJU_CONTROLLER'] = app.provider.controller app.env['JUJU_MODEL'] = app.provider.model app.env['JUJU_REGION'] = app.provider.region or '' if provider_type == "maas": app.env['MAAS_ENDPOINT'] = app.maas.endpoint app.env['MAAS_APIKEY'] = app.maas.api_key for step_name, step_data in app.steps_data.items(): for key, value in step_data.items(): app.env[key.upper()] = str(step_data[key]) for key, value in app.env.items(): if value is None: app.log.warning('Env {} is None; ' 'replacing with empty string'.format(key)) app.env[key] = '' app.log.debug("Storing environment") async with aiofiles.open(step_path + ".env", 'w') as outf: for k, v in app.env.items(): if 'JUJU' in k or 'MAAS' in k or 'CONJURE' in k: await outf.write("{}=\"{}\" ".format(k.upper(), v)) app.log.debug("Executing script: {}".format(step_path)) out_path = step_path + '.out' err_path = step_path + '.err' ret, out_log, err_log = await arun([step_path], stdout=out_path, stderr=err_path, cb_stdout=msg_cb) if ret != 0: app.sentry.context.merge({ 'extra': { 'out_log_tail': out_log[-400:], 'err_log_tail': err_log[-400:], } }) raise Exception("Failure in step {} {}".format( self.name, phase.value)) # special case for 00_deploy-done to report masked # charm hook failures that were retried automatically if not app.no_report and app.metadata.spell_type == spell_types.JUJU: failed_apps = set() # only report each charm once for line in err_log.splitlines(): if 'hook failure, will retry' in line: log_leader = line.split()[0] unit_name = log_leader.split(':')[-1] app_name = unit_name.split('/')[0] failed_apps.add(app_name) for app_name in failed_apps: # report each individually so that Sentry will give us a # breakdown of failures per-charm in addition to per-spell sentry_report('Retried hook failure', tags={ 'app_name': app_name, }) if event_name is not None: track_event(event_name, "Done", "") return self.get_state('result', phase)
def __handle_add_model_done(self, future): track_event("Juju Add JaaS Model", "Done", "") app.ui.set_footer('Add model complete.')
def main(): if os.geteuid() == 0: print("") print(" !! This should _not_ be run as root or with sudo. !!") print("") sys.exit(1) # Verify we can access ~/.local/share/juju if it exists juju_dir = pathlib.Path('~/.local/share/juju').expanduser() if juju_dir.exists(): try: for f in juju_dir.iterdir(): if f.is_file(): f.read_text() except PermissionError: print("") print(" !! Unable to read from ~/.local/share/juju, please " "double check your permissions on that directory " "and its files. !!") print("") sys.exit(1) utils.set_terminal_title("conjure-up") opts = parse_options(sys.argv[1:]) opt_defaults = parse_options([]) # Load conjurefile, merge any overridding options from argv if not opts.conf_file: opts.conf_file = [] if pathlib.Path('~/.config/conjure-up.conf').expanduser().exists(): opts.conf_file.insert( 0, pathlib.Path('~/.config/conjure-up.conf').expanduser()) if (pathlib.Path('.') / 'Conjurefile').exists(): opts.conf_file.insert(0, pathlib.Path('.') / 'Conjurefile') for conf in opts.conf_file: if not conf.exists(): print("Unable to locate config {} for processing.".format( str(conf))) sys.exit(1) try: app.conjurefile = Conjurefile.load(opts.conf_file) except ValueError as e: print(str(e)) sys.exit(1) app.conjurefile.merge_argv(opts, opt_defaults) if app.conjurefile['gen-config']: Conjurefile.print_tpl() sys.exit(0) spell = os.path.basename(os.path.abspath(app.conjurefile['spell'])) if not os.path.isdir(app.conjurefile['cache-dir']): os.makedirs(app.conjurefile['cache-dir']) # Application Config kv_db = os.path.join(app.conjurefile['cache-dir'], '.state.db') app.state = KV(kv_db) app.env = os.environ.copy() app.env['KV_DB'] = kv_db app.config = {'metadata': None} app.log = setup_logging(app, os.path.join(app.conjurefile['cache-dir'], 'conjure-up.log'), app.conjurefile.get('debug', False)) # Make sure juju paths are setup juju.set_bin_path() juju.set_wait_path() app.no_track = app.conjurefile['no-track'] app.no_report = app.conjurefile['no-report'] # Grab current LXD and Juju versions app.log.debug("Juju version: {}, " "conjure-up version: {}".format( utils.juju_version(), VERSION)) # Setup proxy apply_proxy() app.session_id = os.getenv('CONJURE_TEST_SESSION_ID', str(uuid.uuid4())) spells_dir = app.conjurefile['spells-dir'] app.config['spells-dir'] = spells_dir spells_index_path = os.path.join(app.config['spells-dir'], 'spells-index.yaml') spells_registry_branch = os.getenv('CONJUREUP_REGISTRY_BRANCH', 'master') if not app.conjurefile['no-sync']: if not os.path.exists(spells_dir): utils.info("No spells found, syncing from registry, please wait.") try: download_or_sync_registry( app.conjurefile['registry'], spells_dir, branch=spells_registry_branch) except subprocess.CalledProcessError as e: if not os.path.exists(spells_dir): utils.error("Could not load from registry") sys.exit(1) app.log.debug( 'Could not sync spells from github: {}'.format(e)) else: if not os.path.exists(spells_index_path): utils.error( "You opted to not sync from the spells registry, however, " "we could not find any suitable spells in: " "{}".format(spells_dir)) sys.exit(1) with open(spells_index_path) as fp: app.spells_index = yaml.safe_load(fp.read()) addons_aliases_index_path = os.path.join(app.config['spells-dir'], 'addons-aliases.yaml') if os.path.exists(addons_aliases_index_path): with open(addons_aliases_index_path) as fp: app.addons_aliases = yaml.safe_load(fp.read()) spell_name = spell app.endpoint_type = detect_endpoint(app.conjurefile['spell']) if app.conjurefile['spell'] != consts.UNSPECIFIED_SPELL: app.spell_given = True # Check if spell is actually an addon addon = utils.find_addons_matching(app.conjurefile['spell']) if addon: app.log.debug("addon found, setting required spell") utils.set_chosen_spell(addon['spell'], os.path.join(app.conjurefile['cache-dir'], addon['spell'])) download_local(os.path.join(app.config['spells-dir'], addon['spell']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.selected_addons = addon['addons'] app.alias_given = True controllers.setup_metadata_controller() app.endpoint_type = EndpointType.LOCAL_DIR elif app.endpoint_type == EndpointType.LOCAL_SEARCH: spells = utils.find_spells_matching(app.conjurefile['spell']) if len(spells) == 0: utils.error("Can't find a spell matching '{}'".format( app.conjurefile['spell'])) sys.exit(1) # One result means it was a direct match and we can copy it # now. Changing the endpoint type then stops us from showing # the picker UI. More than one result means we need to show # the picker UI and will defer the copy to # SpellPickerController.finish(), so nothing to do here. if len(spells) == 1: app.log.debug("found spell {}".format(spells[0][1])) spell = spells[0][1] utils.set_chosen_spell(spell_name, os.path.join(app.conjurefile['cache-dir'], spell['key'])) download_local(os.path.join(app.config['spells-dir'], spell['key']), app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.endpoint_type = EndpointType.LOCAL_DIR # download spell if necessary elif app.endpoint_type == EndpointType.LOCAL_DIR: if not os.path.isdir(app.conjurefile['spell']): utils.warning("Could not find spell {}".format( app.conjurefile['spell'])) sys.exit(1) if not os.path.exists(os.path.join(app.conjurefile['spell'], "metadata.yaml")): utils.warning("'{}' does not appear to be a spell. " "{}/metadata.yaml was not found.".format( app.conjurefile['spell'], app.conjurefile['spell'])) sys.exit(1) spell_name = os.path.basename(os.path.abspath(spell)) utils.set_chosen_spell(spell_name, path.join(app.conjurefile['cache-dir'], spell_name)) download_local(app.conjurefile['spell'], app.config['spell-dir']) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() elif app.endpoint_type in [EndpointType.VCS, EndpointType.HTTP]: utils.set_chosen_spell(spell, path.join( app.conjurefile['cache-dir'], spell)) remote = get_remote_url(app.conjurefile['spell']) if remote is None: utils.warning("Can't guess URL matching '{}'".format( app.conjurefile['spell'])) sys.exit(1) download(remote, app.config['spell-dir'], True) utils.set_spell_metadata() StepModel.load_spell_steps() AddonModel.load_spell_addons() app.env['CONJURE_UP_CACHEDIR'] = app.conjurefile['cache-dir'] app.env['PATH'] = "/snap/bin:{}".format(app.env['PATH']) if app.conjurefile['show-env']: if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) show_env() app.sentry = raven.Client( dsn=SENTRY_DSN, release=VERSION, transport=RequestsHTTPTransport, processors=( 'conjureup.utils.SanitizeDataProcessor', ) ) track_screen("Application Start") track_event("OS", platform.platform(), "") app.loop = asyncio.get_event_loop() app.loop.add_signal_handler(signal.SIGINT, events.Shutdown.set) # Enable charmstore querying app.juju.charmstore = CharmStore(app.loop) try: if app.conjurefile.is_valid: cloud = None region = None if '/' in app.conjurefile['cloud']: parse_cli_cloud = app.conjurefile['cloud'].split('/') cloud, region = parse_cli_cloud app.log.debug( "Region found {} for cloud {}".format(cloud, region)) else: cloud = app.conjurefile['cloud'] cloud_types = juju.get_cloud_types_by_name() if cloud not in cloud_types: utils.error('Unknown cloud: {}'.format(cloud)) sys.exit(1) if app.endpoint_type in [None, EndpointType.LOCAL_SEARCH]: utils.error("Please specify a spell for headless mode.") sys.exit(1) app.provider = load_schema(cloud_types[cloud]) try: app.provider.load(cloud) except errors.SchemaCloudError as e: utils.error(e) sys.exit(1) if region: app.provider.region = region app.headless = True app.ui = None app.env['CONJURE_UP_HEADLESS'] = "1" app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) app.loop.run_forever() else: app.ui = ConjureUI() app.ui.set_footer('Press ? for help') EventLoop.build_loop(app.ui, STYLES, unhandled_input=events.unhandled_input, handle_mouse=False) app.loop.create_task(events.shutdown_watcher()) app.loop.create_task(_start()) EventLoop.run() finally: # explicitly close asyncio event loop to avoid hitting the # following issue due to signal handlers added by # asyncio.create_subprocess_exec being cleaned up during final # garbage collection: https://github.com/python/asyncio/issues/396 app.loop.close() sys.exit(app.exit_code)
def set_chosen_spell(spell_name, spell_dir): track_event("Spell Choice", spell_name, "") app.env = os.environ.copy() app.env['CONJURE_UP_SPELL'] = spell_name app.config.update({'spell-dir': spell_dir, 'spell': spell_name})