def multimodel_setup(args):
"""
Sets up the juju client and its environment.
Returns return client, charm_prefix and base_env.
"""
base_env = args.env
configure_logging(args.verbose)
# TODO(gz): Logic from deploy_stack, and precise is a bad default series?
series = args.series
if series is None:
series = 'precise'
charm_series = series
client = client_from_config(base_env,
args.juju_bin,
args.debug,
soft_deadline=args.deadline)
if not client.is_jes_enabled():
client.enable_jes()
with boot_context(
args.temp_env_name,
client,
args.bootstrap_host,
args.machine,
args.series,
args.agent_url,
args.agent_stream,
args.logs,
args.keep_env,
upload_tools=False,
region=args.region,
):
yield client, charm_series, base_env
def assess_autoload_credentials(args):
test_scenarios = {
'ec2': [('AWS using environment variables', aws_envvar_test_details),
('AWS using credentials file', aws_directory_test_details)],
'openstack':
[('OS using environment variables', openstack_envvar_test_details),
('OS using credentials file', openstack_directory_test_details)],
'gce': [('GCE using envvar with credentials file',
gce_envvar_with_file_test_details),
('GCE using credentials file', gce_file_test_details)],
}
client = client_from_config(args.env, args.juju_bin, False)
client.env.load_yaml()
provider = client.env.provider
for scenario_name, scenario_setup in test_scenarios[provider]:
log.info('* Starting test scenario: {}'.format(scenario_name))
ensure_autoload_credentials_stores_details(client, scenario_setup)
for scenario_name, scenario_setup in test_scenarios[provider]:
log.info(
'* Starting [overwrite] test, scenario: {}'.format(scenario_name))
ensure_autoload_credentials_overwrite_existing(client, scenario_setup)
real_credential_details = client_credentials_to_details(client)
bs_manager = BootstrapManager.from_args(args)
autoload_and_bootstrap(bs_manager, args.upload_tools,
real_credential_details, scenario_setup)
def get_clients(initial, other, base_env, debug, agent_url):
"""Return the clients to use for testing."""
if initial == 'FAKE':
environment = SimpleEnvironment.from_config(base_env)
client = fake_juju_client(env=environment)
return client, client, client
else:
initial_client = client_from_config(base_env, initial, debug=debug)
environment = initial_client.env
if agent_url is None:
environment.discard_option('tools-metadata-url')
other_client = initial_client.clone_path_cls(other)
# System juju is assumed to be released and the best choice for tearing
# down environments reliably. (For example, 1.18.x cannot tear down
# environments with alpha agent-versions.)
try:
released_client = initial_client.clone_path_cls(None)
except IncompatibleConfigClass:
# If initial_client's config class is incompatible with the system
# juju, use initial client for teardown.
released_client = initial_client
# If released_client is a different major version, it cannot tear down
# initial client, so use initial client for teardown.
if (
isinstance(released_client, EnvJujuClient1X) !=
isinstance(initial_client, EnvJujuClient1X)
):
released_client = initial_client
else:
# If system juju is used, ensure it has identical env to
# initial_client.
released_client.env = initial_client.env
return initial_client, other_client, released_client
def multimodel_setup(args):
"""
Sets up the juju client and its environment.
Returns return client, charm_prefix and base_env.
"""
base_env = args.env
configure_logging(args.verbose)
# TODO(gz): Logic from deploy_stack, and precise is a bad default series?
series = args.series
if series is None:
series = 'precise'
charm_series = series
client = client_from_config(base_env, args.juju_bin, args.debug,
soft_deadline=args.deadline)
if not client.is_jes_enabled():
client.enable_jes()
with boot_context(
args.temp_env_name,
client,
args.bootstrap_host,
args.machine,
args.series,
args.agent_url,
args.agent_stream,
args.logs, args.keep_env,
upload_tools=False,
region=args.region,
):
yield client, charm_series, base_env
def assess_autoload_credentials(args):
test_scenarios = {
'ec2': [('AWS using environment variables', aws_envvar_test_details),
('AWS using credentials file', aws_directory_test_details)],
'openstack':
[('OS using environment variables', openstack_envvar_test_details),
('OS using credentials file', openstack_directory_test_details)],
'gce': [('GCE using envvar with credentials file',
gce_envvar_with_file_test_details),
('GCE using credentials file',
gce_file_test_details)],
}
client = client_from_config(args.env, args.juju_bin, False)
client.env.load_yaml()
provider = client.env.provider
for scenario_name, scenario_setup in test_scenarios[provider]:
log.info('* Starting test scenario: {}'.format(scenario_name))
ensure_autoload_credentials_stores_details(client, scenario_setup)
for scenario_name, scenario_setup in test_scenarios[provider]:
log.info(
'* Starting [overwrite] test, scenario: {}'.format(scenario_name))
ensure_autoload_credentials_overwrite_existing(
client, scenario_setup)
real_credential_details = client_credentials_to_details(client)
bs_manager = BootstrapManager.from_args(args)
autoload_and_bootstrap(bs_manager, args.upload_tools,
real_credential_details, scenario_setup)
def scaleout_setup(args):
"""Setup and bootstrap the Juju environment."""
logger.info("Bootstrapping the scaleout test environment.")
series = args.series
if series is None:
series = 'trusty'
client = client_from_config(args.env,
args.juju_bin,
args.debug,
soft_deadline=args.deadline)
with boot_context(
args.temp_env_name,
client,
args.bootstrap_host,
args.machine,
series,
args.agent_url,
args.agent_stream,
args.logs,
args.keep_env,
args.upload_tools,
region=args.region,
):
if args.machine is not None:
client.add_ssh_machines(args.machine)
yield client
def main():
parser = ArgumentParser()
parser.add_argument('env_name')
parser.add_argument('directory')
args = parser.parse_args()
client = client_from_config(args.env_name, None)
dump_env_logs(client, None, args.directory)
def main():
parser = ArgumentParser()
parser.add_argument('env_name')
parser.add_argument('directory')
args = parser.parse_args()
client = client_from_config(args.env_name, None)
dump_env_logs(client, None, args.directory)
def main(argv=None):
args = parse_args(argv)
configure_logging(args.verbose)
client = client_from_config(args.env, args.juju_bin, False)
agent_stream = args.agent_stream if args.agent_stream else 'devel'
agent_version = get_agent_version(client)
assess_juju_sync_tools(client, agent_stream, agent_version)
return 0
def deploy_stack(environment, debug, machines, deploy_charm):
""""Deploy a test stack in the specified environment.
:param environment: The name of the desired environment.
"""
client = client_from_config(environment, None, debug=debug)
running_domains = dict()
if client.env.provider == 'maas':
# Split the hypervisor_URI and machine name
for machine in machines:
name, URI = machine.split('@')
# Record already running domains, so they can be left running,
# if already running; otherwise start them.
if verify_libvirt_domain(URI, name, LIBVIRT_DOMAIN_RUNNING):
print("%s is already running" % name)
running_domains = {machine: True}
else:
running_domains = {machine: False}
print("Attempting to start %s at %s" % (name, URI))
status_msg = start_libvirt_domain(URI, name)
print("%s" % status_msg)
# Clean up any leftover junk
client.destroy_environment()
client.bootstrap()
try:
# wait for status info....
try:
try:
client.get_status()
except CannotConnectEnv:
print("Status got Unable to connect to env. Retrying...")
client.get_status()
client.wait_for_started()
if deploy_charm:
series = client.env.get_option('default-series', 'trusty')
charm_path = local_charm_path(
'dummy-source', juju_ver=client.version, series=series)
client.deploy(charm_path, series=series)
client.wait_for_started()
except subprocess.CalledProcessError as e:
if getattr(e, 'stderr', None) is not None:
sys.stderr.write(e.stderr)
raise
finally:
client.destroy_environment()
if client.env.provider == 'maas':
sleep(90)
for machine, running in running_domains.items():
name, URI = machine.split('@')
if running:
print("WARNING: %s at %s was running when deploy_job "
"started. Shutting it down to ensure a clean "
"environment."
% (name, URI))
status_msg = stop_libvirt_domain(URI, name)
print("%s" % status_msg)
def deploy_stack(environment, debug, machines, deploy_charm):
""""Deploy a test stack in the specified environment.
:param environment: The name of the desired environment.
"""
client = client_from_config(environment, None, debug=debug)
running_domains = dict()
if client.env.provider == 'maas':
# Split the hypervisor_URI and machine name
for machine in machines:
name, URI = machine.split('@')
# Record already running domains, so they can be left running,
# if already running; otherwise start them.
if verify_libvirt_domain(URI, name, LIBVIRT_DOMAIN_RUNNING):
print("%s is already running" % name)
running_domains = {machine: True}
else:
running_domains = {machine: False}
print("Attempting to start %s at %s" % (name, URI))
status_msg = start_libvirt_domain(URI, name)
print("%s" % status_msg)
# Clean up any leftover junk
client.destroy_environment()
client.bootstrap()
try:
# wait for status info....
try:
try:
client.get_status()
except CannotConnectEnv:
print("Status got Unable to connect to env. Retrying...")
client.get_status()
client.wait_for_started()
if deploy_charm:
series = client.env.get_option('default-series', 'trusty')
charm_path = local_charm_path('dummy-source',
juju_ver=client.version,
series=series)
client.deploy(charm_path, series=series)
client.wait_for_started()
except subprocess.CalledProcessError as e:
if getattr(e, 'stderr', None) is not None:
sys.stderr.write(e.stderr)
raise
finally:
client.destroy_environment()
if client.env.provider == 'maas':
sleep(90)
for machine, running in running_domains.items():
name, URI = machine.split('@')
if running:
print("WARNING: %s at %s was running when deploy_job "
"started. Shutting it down to ensure a clean "
"environment." % (name, URI))
status_msg = stop_libvirt_domain(URI, name)
print("%s" % status_msg)
def from_args(cls, args):
if not args.logs:
args.logs = generate_default_clean_dir(args.temp_env_name)
# GZ 2016-08-11: Move this logic into client_from_config maybe?
if args.juju_bin == 'FAKE':
env = SimpleEnvironment.from_config(args.env)
client = fake_juju_client(env=env)
else:
client = client_from_config(args.env, args.juju_bin,
debug=args.debug,
soft_deadline=args.deadline)
if args.to is not None:
client.env.bootstrap_to = args.to
return cls.from_client(args, client)
def make_client_from_args(args):
client = client_from_config(args.env, args.juju_bin, args.debug)
if args.temp_env_name is not None:
client.env.set_model_name(args.temp_env_name)
update_env(client.env,
args.temp_env_name,
series=args.series,
bootstrap_host=args.bootstrap_host,
agent_url=args.agent_url,
agent_stream=args.agent_stream,
region=args.region)
jes_enabled = client.is_jes_enabled()
if jes_enabled:
client.env.juju_home = jes_home_path(client.env.juju_home,
args.temp_env_name)
client.kill_controller()
return client
def main(argv=None):
args = parse_args(argv)
configure_logging(args.verbose)
start_juju_path = None if args.upgrade else args.juju_bin
client = client_from_config(args.env, start_juju_path, debug=args.debug,
soft_deadline=args.deadline)
with boot_context(args.temp_env_name, client, None, [], args.series,
args.agent_url, args.agent_stream, args.logs,
args.keep_env, upload_tools=args.upload_tools,
region=args.region):
assess_deployer(
args, client, args.agent_timeout, args.workload_timeout)
if args.bundle_verification_script:
client_ser = pickle.dumps(client)
logging.info('Calling bundle verification script {}'.format(
args.bundle_verification_script))
run_command([args.bundle_verification_script, client_ser])
return 0
def _deploy_job(args, charm_series, series):
start_juju_path = None if args.upgrade else args.juju_bin
if sys.platform == 'win32':
# Ensure OpenSSH is never in the path for win tests.
sys.path = [p for p in sys.path if 'OpenSSH' not in p]
# GZ 2016-01-22: When upgrading, could make sure to tear down with the
# newer client instead, this will be required for major version upgrades?
client = client_from_config(args.env, start_juju_path, args.debug,
soft_deadline=args.deadline)
if args.jes and not client.is_jes_enabled():
client.enable_jes()
jes_enabled = client.is_jes_enabled()
controller_strategy = make_controller_strategy(client, client,
args.controller_host)
bs_manager = BootstrapManager(
args.temp_env_name, client, client, args.bootstrap_host, args.machine,
series, args.agent_url, args.agent_stream, args.region, args.logs,
args.keep_env, permanent=jes_enabled, jes_enabled=jes_enabled,
controller_strategy=controller_strategy)
with bs_manager.booted_context(args.upload_tools):
if args.with_chaos > 0:
manager = background_chaos(args.temp_env_name, client,
args.logs, args.with_chaos)
else:
# Create a no-op context manager, to avoid duplicate calls of
# deploy_dummy_stack(), as was the case prior to this revision.
manager = nested()
with manager:
deploy_dummy_stack(client, charm_series, args.use_charmstore)
assess_juju_relations(client)
skip_juju_run = (
(client.version < "2" and sys.platform in ("win32", "darwin")) or
charm_series.startswith(("centos", "win")))
if not skip_juju_run:
assess_juju_run(client)
if args.upgrade:
client.show_status()
assess_upgrade(client, args.juju_bin)
assess_juju_relations(client)
if not skip_juju_run:
assess_juju_run(client)
def bootstrap_cloud_regions(public_clouds, credentials, args):
cloud_regions = args.cloud_regions
if cloud_regions is None:
cloud_regions = list(iter_cloud_regions(public_clouds, credentials))
for num, (cloud, region) in enumerate(cloud_regions):
if num < args.start:
continue
config = CLOUD_CONFIGS[cloud]
logging.info('Bootstrapping {} {} #{}'.format(config, region, num))
try:
client = client_from_config(config, args.juju_bin, args.debug,
args.deadline)
bs_manager = make_bootstrap_manager(config, region, client,
args.logs)
assess_cloud_combined(bs_manager)
except LoggedException as error:
yield config, region, error.exception
except Exception as error:
logging.exception('Assessment of {} {} failed.'.format(
config, region))
yield config, region, error
def bootstrap_cloud_regions(public_clouds, credentials, args):
cloud_regions = args.cloud_regions
if cloud_regions is None:
cloud_regions = list(iter_cloud_regions(public_clouds, credentials))
for num, (cloud, region) in enumerate(cloud_regions):
if num < args.start:
continue
config = CLOUD_CONFIGS[cloud]
logging.info('Bootstrapping {} {} #{}'.format(config, region, num))
try:
client = client_from_config(
config, args.juju_bin, args.debug, args.deadline)
bs_manager = make_bootstrap_manager(config, region, client,
args.logs)
assess_cloud_combined(bs_manager)
except LoggedException as error:
yield config, region, error.exception
except Exception as error:
logging.exception(
'Assessment of {} {} failed.'.format(config, region))
yield config, region, error
def main():
""" Deploy and run chaos monkey, while checking env health.
The Chaos Monkey is deployed into the environment and related to
the specified service. Juju actions are then used to run one chaos
operation at a time. After each operation, the provided health
check script is executed, to ensure the Juju environment or
software stack is still healthy.
"""
configure_logging(logging.INFO)
args = get_args()
client = client_from_config(args.env, None)
monkey_runner = MonkeyRunner(
args.env, client, service=args.service,
health_checker=args.health_checker,
enablement_timeout=args.enablement_timeout,
pause_timeout=args.pause_timeout,
total_timeout=args.total_timeout)
logging.info("Chaos Monkey Start.")
monkey_runner.deploy_chaos_monkey()
run_while_healthy_or_timeout(monkey_runner)
logging.info("Chaos Monkey Complete.")
def main():
parser = ArgumentParser()
parser.add_argument('clouds_file')
add_arg_juju_bin(parser)
args = parser.parse_args()
client = client_from_config(None, args.juju_bin)
with client.env.make_jes_home(client.env.juju_home, 'mytest',
{}) as juju_home:
client.env.juju_home = juju_home
with open(get_home_path(client, 'public-clouds.yaml'), 'w') as f:
f.write('')
with testing('assess_clouds (no_clouds)'):
assess_clouds(client, {})
with open(args.clouds_file) as f:
supplied_clouds = yaml.safe_load(f.read().decode('utf-8'))
client.env.write_clouds(client.env.juju_home, supplied_clouds)
no_region_endpoint = strip_redundant_endpoints(
supplied_clouds['clouds'])
with testing('assess_clouds'):
assess_clouds(client, no_region_endpoint)
with testing('assess_show_cloud'):
assess_show_cloud(client, no_region_endpoint)
def scaleout_setup(args):
"""Setup and bootstrap the Juju environment."""
logger.info("Bootstrapping the scaleout test environment.")
series = args.series
if series is None:
series = 'trusty'
client = client_from_config(args.env, args.juju_bin, args.debug,
soft_deadline=args.deadline)
with boot_context(
args.temp_env_name,
client,
args.bootstrap_host,
args.machine,
series,
args.agent_url,
args.agent_stream,
args.logs, args.keep_env,
args.upload_tools,
region=args.region,
):
if args.machine is not None:
client.add_ssh_machines(args.machine)
yield client