def download(src, dst, purge_top_level=True): """ Download and extract archive Arguments: src: path to archive dst: directory to change to before extract, this directory must already exist. purge_top_level: purge the toplevel directory and shift all contents up during unzip. """ try: shutil.rmtree(dst, ignore_errors=True) os.makedirs(dst) request = requests.get(src, stream=True) tmpfile = os.path.join(os.environ.get('TEMPDIR', '/tmp'), 'temp.zip') download_requests_stream(request, tmpfile) bsdtar_cmd = "bsdtar -xf {} ".format(tmpfile) if purge_top_level: bsdtar_cmd += "-s'|[^/]*/||' " bsdtar_cmd += "-C {}".format(dst) app.log.debug("Extracting spell: {}".format(bsdtar_cmd)) run(bsdtar_cmd, shell=True, check=True, env=app.env) except CalledProcessError as e: raise Exception("Unable to download {}: {}".format(src, e))
def setup_lxdbr0_network(): """ This attempts to setup LXD networking if not available """ try: utils.run('lxc network show lxdbr0', shell=True, check=True, stdout=DEVNULL, stderr=DEVNULL) except CalledProcessError: out = utils.run_script('lxc network create lxdbr0 ' 'ipv4.address=10.0.8.1/24 ' 'ipv4.nat=true ' 'ipv6.address=none ' 'ipv6.nat=false') if out.returncode != 0: raise Exception("Failed to create LXD network bridge: {}".format( out.stderr.decode())) out = utils.run_script( "lxc network show lxdbr0 | grep -q 'ipv4\.address:\snone'") if out.returncode == 0: network_set_cmds = [ 'lxc network set lxdbr0 ipv4.address 10.0.8.1/24', 'lxc network set lxdbr0 ipv4.nat true' ] for n in network_set_cmds: out = utils.run_script(n) if out.returncode != 0: raise Exception("Problem with {}: {}".format( n, out.stderr.decode()))
def download(src, dst, purge_top_level=True): """ Download and extract archive Arguments: src: path to archive dst: directory to change to before extract, this directory must already exist. purge_top_level: purge the toplevel directory and shift all contents up during unzip. """ try: shutil.rmtree(dst, ignore_errors=True) os.makedirs(dst) request = requests.get(src, stream=True) tmpfile = os.path.join(os.environ.get('TEMPDIR', '/tmp'), 'temp.zip') download_requests_stream(request, tmpfile) bsdtar_cmd = "bsdtar -xf {} ".format(tmpfile) if purge_top_level: bsdtar_cmd += "-s'|[^/]*/||' " bsdtar_cmd += "-C {}".format(dst) app.log.debug("Extracting spell: {}".format(bsdtar_cmd)) run(bsdtar_cmd, shell=True, check=True) except CalledProcessError as e: raise Exception("Unable to download {}: {}".format(src, e))
def clone(): run("git clone -q --depth 1 --no-single-branch {} {}".format( remote_registry, spells_dir), shell=True, check=True, stdout=DEVNULL, stderr=DEVNULL)
def autoload_credentials(): """ Automatically checks known places for cloud credentials """ try: run('juju autoload-credentials', shell=True, check=True) except CalledProcessError: return False return True
def _do_switch(target): try: app.log.debug('calling juju switch {}'.format(target)) run('juju-2.0 switch {}'.format(target), shell=True, check=True, stdout=DEVNULL, stderr=DEVNULL) except CalledProcessError as e: raise LookupError("Unable to switch: {}".format(e))
def autoload_credentials(): """ Automatically checks known places for cloud credentials """ try: run('{} autoload-credentials'.format(app.juju.bin_path), shell=True, check=True) except CalledProcessError: return False return True
def model_available(): """ Checks if juju is available Returns: True/False if juju status was successful and a working model is found """ try: run('juju status', shell=True, check=True, stderr=DEVNULL, stdout=DEVNULL) except CalledProcessError: return False return True
def run_action(unit, action): """ runs an action on a unit, waits for result """ is_complete = False sh = run( 'juju run-action -m {} {} {}'.format( JUJU_CM_STR, unit, action), shell=True, stdout=PIPE) run_action_output = yaml.load(sh.stdout.decode()) log.debug("{}: {}".format(sh.args, run_action_output)) action_id = run_action_output.get('Action queued with id', None) log.debug("Found action: {}".format(action_id)) if not action_id: fail("Could not determine action id for test") while not is_complete: sh = run( 'juju show-action-output -m {} {}'.format( JUJU_CM_STR, action_id), shell=True, stderr=PIPE, stdout=PIPE) log.debug(sh) try: output = yaml.load(sh.stdout.decode()) log.debug(output) except Exception as e: log.debug(e) if output['status'] == 'running' or output['status'] == 'pending': time.sleep(5) continue if output['status'] == 'failed': fail("The test failed, " "please have a look at `juju show-action-status`") if output['status'] == 'completed': completed_msg = "{} test passed".format(unit) results = output.get('results', None) if not results: is_complete = True success(completed_msg) if results.get('outcome', None): is_complete = True completed_msg = "{}: (result) {}".format( completed_msg, results.get('outcome')) success(completed_msg) fail("There is an unknown issue with running the test, " "please have a look at `juju show-action-status`")
def do_post_bootstrap(self): """ runs post bootstrap script if exists """ # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = model_info('default')['provider-type'] post_bootstrap_sh = os.path.join(app.config['spell-dir'], 'steps/00_post-bootstrap') if os.path.isfile(post_bootstrap_sh) \ and os.access(post_bootstrap_sh, os.X_OK): utils.pollinate(app.session_id, 'J001') utils.info("Running post-bootstrap tasks.") try: sh = utils.run(post_bootstrap_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) result = json.loads(sh.stdout.decode('utf8')) utils.info("Finished post bootstrap task: {}".format( result['message'])) except Exception as e: utils.warning( "Failed to run post bootstrap task: {}".format(e)) sys.exit(1)
def model_available(): """ Checks if juju is available Returns: True/False if juju status was successful and a working model is found """ try: run('juju status -m {}:{}'.format(app.current_controller, app.current_model), shell=True, check=True, stderr=DEVNULL, stdout=DEVNULL) except CalledProcessError: return False return True
def get_credentials(secrets=True): """ List credentials This will fallback to reading the credentials file directly Arguments: secrets: True/False whether to show secrets (ie password) Returns: List of credentials """ cmd = 'juju list-credentials --format yaml' if secrets: cmd += ' --show-secrets' sh = run(cmd, shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: try: env = read_config('credentials') return env['credentials'] except: raise Exception( "Unable to list credentials: {}".format( sh.stderr.decode('utf8'))) env = yaml.safe_load(sh.stdout.decode('utf8')) return env['credentials']
def do_post_bootstrap(self): """ runs post bootstrap script if exists """ # Set provider type for post-bootstrap info = model_info(app.current_model) app.env['JUJU_PROVIDERTYPE'] = info['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model post_bootstrap_sh = os.path.join(app.config['spell-dir'], 'steps/00_post-bootstrap') if os.path.isfile(post_bootstrap_sh) \ and os.access(post_bootstrap_sh, os.X_OK): utils.info("Running post-bootstrap tasks.") try: sh = utils.run(post_bootstrap_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) result = json.loads(sh.stdout.decode('utf8')) utils.info("Finished post bootstrap task: {}".format( result['message'])) except Exception as e: utils.warning( "Failed to run post bootstrap task: {}".format(e)) sys.exit(1)
def get_regions(cloud): """ List available regions for cloud Arguments: cloud: Cloud to list regions for Returns: Dictionary of all known regions for cloud """ sh = run('juju list-regions {} --format yaml'.format(cloud), shell=True, stdout=PIPE, stderr=PIPE) stdout = sh.stdout.decode('utf8') stderr = sh.stderr.decode('utf8') if sh.returncode > 0: raise Exception("Unable to list regions: {}".format(stderr)) if 'no regions' in stdout: return {} result = yaml.safe_load(stdout) if not isinstance(result, dict): msg = 'Unexpected response from regions: {}'.format(result) app.log.error(msg) utils.sentry_report(msg, level=logging.ERROR) result = {} return result
def do_pre_deploy(self): """ runs pre deploy script if exists """ # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = model_info( app.current_model)['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model pre_deploy_sh = os.path.join(app.config['spell-dir'], 'steps/00_pre-deploy') if os.path.isfile(pre_deploy_sh) \ and os.access(pre_deploy_sh, os.X_OK): utils.info("Running pre deployment tasks.") try: sh = utils.run(pre_deploy_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) result = json.loads(sh.stdout.decode('utf8')) if result['returnCode'] > 0: utils.error("Failed to run pre-deploy task: " "{}".format(result['message'])) sys.exit(1) utils.info("Finished pre deploy task: {}".format( result['message'])) except Exception as e: utils.error("Failed to run pre deploy task: {}".format(e)) sys.exit(1)
def do_pre_deploy(self): """ runs pre deploy script if exists """ # Set provider type for post-bootstrap app.env['JUJU_PROVIDERTYPE'] = model_info( app.current_model)['provider-type'] app.env['JUJU_CONTROLLER'] = app.current_controller app.env['JUJU_MODEL'] = app.current_model pre_deploy_sh = os.path.join(app.config['spell-dir'], 'steps/00_pre-deploy') if os.path.isfile(pre_deploy_sh) \ and os.access(pre_deploy_sh, os.X_OK): utils.info("Running pre deployment tasks.") try: sh = utils.run(pre_deploy_sh, shell=True, stdout=PIPE, stderr=PIPE, env=app.env) result = json.loads(sh.stdout.decode('utf8')) if result['returnCode'] > 0: utils.error("Failed to run pre-deploy task: " "{}".format(result['message'])) sys.exit(1) utils.info("Finished pre deploy task: {}".format( result['message'])) except Exception as e: utils.error( "Failed to run pre deploy task: {}".format(e)) sys.exit(1)
def run_action(unit, action): """ runs an action on a unit, waits for result """ is_complete = False sh = run('juju run-action -m {} {} {}'.format(JUJU_CM_STR, unit, action), shell=True, stdout=PIPE) run_action_output = yaml.load(sh.stdout.decode()) log.debug("{}: {}".format(sh.args, run_action_output)) action_id = run_action_output.get('Action queued with id', None) log.debug("Found action: {}".format(action_id)) if not action_id: fail("Could not determine action id for test") while not is_complete: sh = run('juju show-action-output -m {} {}'.format( JUJU_CM_STR, action_id), shell=True, stderr=PIPE, stdout=PIPE) log.debug(sh) try: output = yaml.load(sh.stdout.decode()) log.debug(output) except Exception as e: log.debug(e) if output['status'] == 'running' or output['status'] == 'pending': time.sleep(5) continue if output['status'] == 'failed': fail("The test failed, " "please have a look at `juju show-action-status`") if output['status'] == 'completed': completed_msg = "{} test passed".format(unit) results = output.get('results', None) if not results: is_complete = True success(completed_msg) if results.get('outcome', None): is_complete = True completed_msg = "{}: (result) {}".format( completed_msg, results.get('outcome')) success(completed_msg) fail("There is an unknown issue with running the test, " "please have a look at `juju show-action-status`")
def status(): """ Get juju status """ try: sh = run('juju-2.0 status --format yaml', shell=True, check=True, stdout=PIPE) except CalledProcessError: return None return yaml.load(sh.stdout.decode())
def download_or_sync_registry(remote_registry, spells_dir, update=False): """ If first time run this git clones the spell registry, otherwise will pull the latest spells down. Arguments: remote_registry: git location of spells registry spells_dir: cache location of local spells directory Returns: True if successful, False otherwise """ if not os.path.exists(spells_dir): run("git clone -q --depth 1 {} {}".format(remote_registry, spells_dir), shell=True, check=True) if os.path.exists(spells_dir) and update: run("cd {} && git pull -q".format(spells_dir), shell=True, check=True) return False
def add_model(name): """ Adds a model to current controller """ sh = run('juju-2.0 add-model {}'.format(name), shell=True, stdout=DEVNULL, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to create model: {}".format( sh.stderr.decode('utf8')))
def add_model(name, controller): """ Adds a model to current controller Arguments: controller: controller to add model in """ sh = run('juju add-model {} -c {}'.format(name, controller), shell=True, stdout=DEVNULL, stderr=PIPE) if sh.returncode > 0: raise Exception( "Unable to create model: {}".format(sh.stderr.decode('utf8')))
def finish(self, needs_lxd_setup=False, lxdnetwork=None, back=False): """ Processes the new LXD setup and loads the controller to finish bootstrapping the model. Arguments: back: if true loads previous controller needs_lxd_setup: if true prompt user to run lxd init """ if back: return controllers.use('clouds').render() if needs_lxd_setup: EventLoop.remove_alarms() EventLoop.exit(1) if lxdnetwork is None: return app.ui.show_exception_message( Exception("Unable to configure LXD network bridge.")) formatted_network = self.__format_input(lxdnetwork) app.log.debug("LXD Config {}".format(formatted_network)) out = self.__format_conf(formatted_network) with NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as tempf: app.log.debug("Saving LXD config to {}".format(tempf.name)) utils.spew(tempf.name, out) sh = utils.run('sudo mv {} /etc/default/lxd-bridge'.format( tempf.name), shell=True) if sh.returncode > 0: return app.ui.show_exception_message( Exception("Problem saving config: {}".format( sh.stderr.decode('utf8')))) app.log.debug("Restarting lxd-bridge") utils.run("sudo systemctl restart lxd-bridge.service", shell=True) utils.pollinate(app.session_id, 'L002') controllers.use('newcloud').render( cloud='localhost', bootstrap=True)
def version(): """ Returns version of Juju """ sh = run('juju version', shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to get Juju Version".format( sh.stderr.decode('utf8'))) out = sh.stdout.decode('utf8') if isinstance(out, list): return out.pop() else: return out
def version(): """ Returns version of Juju """ sh = run('juju version', shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception( "Unable to get Juju Version".format(sh.stderr.decode('utf8'))) out = sh.stdout.decode('utf8') if isinstance(out, list): return out.pop() else: return out
def deploy(bundle): """ Juju deploy bundle Arguments: bundle: Name of bundle to deploy, can be a path to local bundle file or charmstore path. """ try: return run('juju deploy {}'.format(bundle), shell=True, stdout=DEVNULL, stderr=PIPE) except CalledProcessError as e: raise e
def download_or_sync_registry(remote_registry, spells_dir, update=False, branch='master'): """ If first time run this git clones the spell registry, otherwise will pull the latest spells down. To specify a different branch to use you must set the environment variable CONJUREUP_REGISTRY_BRANCH=<branchname>. This should be used for testing new spells before they make it into the master branch. Arguments: remote_registry: git location of spells registry spells_dir: cache location of local spells directory update: update the source directory branch: switch to branch Returns: True if successful, False otherwise """ if not os.path.exists(spells_dir): run("git clone -q --depth 1 --no-single-branch {} {}".format( remote_registry, spells_dir), shell=True, check=True) if os.path.exists(spells_dir) and update: run("cd {} && git pull -q".format(spells_dir), shell=True, check=True) run("cd {} && git checkout -q {}".format(spells_dir, branch), shell=True) return False
def get_controllers(): """ List available controllers Returns: List of known controllers """ sh = run('juju list-controllers --format yaml', shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise LookupError( "Unable to list controllers: {}".format(sh.stderr.decode('utf8'))) env = yaml.safe_load(sh.stdout.decode('utf8')) return env
def get_clouds(): """ List available clouds Returns: Dictionary of all known clouds including newly created MAAS/Local """ sh = run('juju list-clouds --format yaml', shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception( "Unable to list clouds: {}".format(sh.stderr.decode('utf8')) ) return yaml.safe_load(sh.stdout.decode('utf8'))
def add_model(name, controller): """ Adds a model to current controller Arguments: controller: controller to add model in """ sh = run('juju add-model {} -c {}'.format(name, controller), shell=True, stdout=DEVNULL, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to create model: {}".format( sh.stderr.decode('utf8')))
def destroy_model(controller, model): """ Destroys a model within a controller Arguments: controller: name of controller model: name of model to destroy """ sh = run('juju destroy-model -y {}:{}'.format(controller, model), shell=True, stdout=DEVNULL, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to destroy model: {}".format( sh.stderr.decode('utf8')))
def get_models(): """ List available models Returns: List of known models """ sh = run('juju-2.0 list-models --format yaml', shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise LookupError("Unable to list models: {}".format( sh.stderr.decode('utf8'))) out = yaml.safe_load(sh.stdout.decode('utf8')) return out
def get_models(controller): """ List available models Arguments: controller: existing controller to get models for Returns: List of known models """ sh = run('juju list-models --format yaml -c {}'.format(controller), shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise LookupError( "Unable to list models: {}".format(sh.stderr.decode('utf8'))) out = yaml.safe_load(sh.stdout.decode('utf8')) return out
def get_regions(cloud): """ List available regions for cloud Arguments: cloud: Cloud to list regions for Returns: Dictionary of all known regions for cloud """ sh = run('juju list-regions {} --format yaml'.format(cloud), shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to list regions: {}".format( sh.stderr.decode('utf8'))) return yaml.safe_load(sh.stdout.decode('utf8'))
def get_controller_info(name=None): """ Returns information on current controller Arguments: name: if set shows info controller, otherwise displays current. """ cmd = 'juju show-controller --format yaml' if name is not None: cmd += ' {}'.format(name) sh = run(cmd, shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to determine controller: {}".format( sh.stderr.decode('utf8'))) out = yaml.safe_load(sh.stdout.decode('utf8')) try: return next(iter(out.values())) except: return out
def add_cloud(name, config): """ Adds a cloud Arguments: name: name of cloud to add config: cloud configuration """ _config = {'clouds': {name: config}} app.log.debug(_config) with NamedTemporaryFile(mode='w', encoding='utf-8', delete=False) as tempf: output = yaml.safe_dump(_config, default_flow_style=False) spew(tempf.name, output) sh = run('juju add-cloud {} {}'.format(name, tempf.name), shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception("Unable to add cloud: {}".format( sh.stderr.decode('utf8')))
def leader(application): """ Grabs the leader of a set of application units Arguments: application: name of application to query. """ try: sh = run( 'juju-2.0 run --application {} is-leader --format yaml'.format( application), shell=True, stdout=PIPE, check=True) except CalledProcessError: return None leader_yaml = yaml.load(sh.stdout.decode()) for leader in leader_yaml: if leader['Stdout'].strip() == 'True': return leader['UnitId']
def get_controller_info(name=None): """ Returns information on current controller Arguments: name: if set shows info controller, otherwise displays current. """ cmd = 'juju show-controller --format yaml' if name is not None: cmd += ' {}'.format(name) sh = run(cmd, shell=True, stdout=PIPE, stderr=PIPE) if sh.returncode > 0: raise Exception( "Unable to determine controller: {}".format( sh.stderr.decode('utf8'))) out = yaml.safe_load(sh.stdout.decode('utf8')) try: return next(iter(out.values())) except: return out
def get_controller_info(name=None): """ Returns information on current controller Arguments: name: if set shows info controller, otherwise displays current. """ cmd = 'juju show-controller --format yaml' if name is not None: cmd += ' {}'.format(name) sh = run(cmd, shell=True, stdout=PIPE, stderr=PIPE) sh_out = sh.stdout.decode('utf8') sh_err = sh.stderr.decode('utf8') try: data = yaml.safe_load(sh_out) except yaml.parser.ParserError: data = None if sh.returncode != 0 or not data: raise Exception("Unable to get info for " "controller {}: {}".format(name, sh_err)) return next(iter(data.values()))