def check_environment():
"""Test uses environment variables to play nicely with TeamCity config templates
Grab all the environment variables here to avoid setting params all over
Returns:
object: generic object used for cleanly passing options through the test
Raises:
AssertionError: if any environment variables or resources are missing
or do not conform
"""
options = type('Options', (object, ), {})()
# Required
options.public_ssh_key = calculate_environment_variable(
'AZURE_PUBLIC_SSH_KEY')
options.subscription_id = calculate_environment_variable(
'AZURE_SUBSCRIPTION_ID')
options.tenant_id = calculate_environment_variable('AZURE_TENANT_ID')
options.client_id = calculate_environment_variable('AZURE_CLIENT_ID')
options.client_secret = calculate_environment_variable(
'AZURE_CLIENT_SECRET')
options.template_url = calculate_environment_variable('AZURE_TEMPLATE_URL')
# Provided if not set
options.name = os.getenv('DCOS_NAME', 'testing-{}'.format(random_id(10)))
options.ssh_key_path = os.getenv('DCOS_SSH_KEY_PATH', 'ssh_key')
options.location = os.getenv('AZURE_LOCATION', 'East US')
options.linux_user = os.getenv('AZURE_LINUX_USER', 'dcos')
# Prefixes must not begin with a number
options.agent_prefix = os.getenv('AZURE_AGENT_PREFIX',
'test' + random_id(10).lower())
options.master_prefix = os.getenv('AZURE_MASTER_PREFIX',
'test' + random_id(10).lower())
options.vm_size = os.getenv('AZURE_VM_SIZE', 'Standard_D2')
options.num_agents = os.getenv('AGENTS', '2')
options.name_suffix = os.getenv('AZURE_DCOS_SUFFIX', '12345')
options.oauth_enabled = os.getenv('AZURE_OAUTH_ENABLED', 'false')
options.vm_diagnostics_enabled = os.getenv('AZURE_VM_DIAGNOSTICS_ENABLED',
'true')
options.azure_cleanup = os.getenv('AZURE_CLEANUP', 'true') == 'true'
options.ci_flags = os.getenv('CI_FLAGS', '')
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
options.test_cmd = os.getenv(
'DCOS_PYTEST_CMD', ' '.join(add_env) +
" py.test -vv -s -rs -m 'not ccm' " + options.ci_flags)
return options
def main():
options = check_environment()
bw = test_util.aws.BotoWrapper(
region=options.aws_region,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
stack_name = 'dcos-ci-test-cf-{}'.format(random_id(10))
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
log.info('Spinning up AWS CloudFormation with ID: {}'.format(stack_name))
if options.advanced:
cf, ssh_info = test_util.aws.DcosZenCfStack.create(
stack_name=stack_name,
boto_wrapper=bw,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
key_pair_name=stack_name,
private_agent_type='m3.xlarge',
public_agent_type='m3.xlarge',
master_type='m3.xlarge',
vpc=options.vpc,
gateway=options.gateway,
private_subnet=options.private_subnet,
public_subnet=options.public_subnet)
else:
cf, ssh_info = test_util.aws.DcosCfStack.create(
stack_name=stack_name,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw)
time.sleep(300) # we know the cluster is not ready yet, don't poll to avoid hitting the rate limit
cf.wait_for_complete()
# Resiliency testing requires knowing the stack name
options.test_cmd = 'AWS_STACK_NAME=' + stack_name + ' ' + options.test_cmd
# hidden hook where user can supply an ssh_key for a preexisting cluster
cluster = test_util.cluster.Cluster.from_cloudformation(cf, ssh_info, ssh_key)
result = test_util.cluster.run_integration_tests(
cluster,
region=options.aws_region,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
test_cmd=options.test_cmd,
)
if result == 0:
log.info('Test successful! Deleting CloudFormation.')
cf.delete()
bw.delete_key_pair(stack_name)
else:
logging.warning('Test exited with an error')
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def main():
options = check_environment()
bw = test_util.aws.BotoWrapper(
region=options.aws_region,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
stack_name = 'dcos-ci-test-cf-{}'.format(random_id(10))
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
log.info('Spinning up AWS CloudFormation with ID: {}'.format(stack_name))
if options.advanced:
cf, ssh_info = test_util.aws.DcosZenCfStack.create(
stack_name=stack_name,
boto_wrapper=bw,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
key_pair_name=stack_name,
private_agent_type='m3.xlarge',
public_agent_type='m3.xlarge',
master_type='m3.xlarge',
vpc=options.vpc,
gateway=options.gateway,
private_subnet=options.private_subnet,
public_subnet=options.public_subnet)
else:
cf, ssh_info = test_util.aws.DcosCfStack.create(
stack_name=stack_name,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw)
cf.wait_for_complete(wait_before_poll_min=5)
# Resiliency testing requires knowing the stack name
options.test_cmd = 'AWS_STACK_NAME=' + stack_name + ' ' + options.test_cmd
# hidden hook where user can supply an ssh_key for a preexisting cluster
cluster = test_util.cluster.Cluster.from_cloudformation(
cf, ssh_info, ssh_key)
result = test_util.cluster.run_integration_tests(
cluster,
region=options.aws_region,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
test_cmd=options.test_cmd,
)
if result == 0:
log.info('Test successful! Deleting CloudFormation.')
cf.delete()
bw.delete_key_pair(stack_name)
else:
logging.warning('Test exited with an error')
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def check_environment():
"""Test uses environment variables to play nicely with TeamCity config templates
Grab all the environment variables here to avoid setting params all over
Returns:
object: generic object used for cleanly passing options through the test
Raises:
AssertionError: if any environment variables or resources are missing
or do not conform
"""
options = type('Options', (object,), {})()
# Required
options.public_ssh_key = calculate_environment_variable('AZURE_PUBLIC_SSH_KEY')
options.subscription_id = calculate_environment_variable('AZURE_SUBSCRIPTION_ID')
options.tenant_id = calculate_environment_variable('AZURE_TENANT_ID')
options.client_id = calculate_environment_variable('AZURE_CLIENT_ID')
options.client_secret = calculate_environment_variable('AZURE_CLIENT_SECRET')
options.template_url = calculate_environment_variable('AZURE_TEMPLATE_URL')
# Provided if not set
options.name = os.getenv('DCOS_NAME', 'testing-{}'.format(random_id(10)))
options.ssh_key_path = os.getenv('DCOS_SSH_KEY_PATH', 'ssh_key')
options.location = os.getenv('AZURE_LOCATION', 'East US')
options.linux_user = os.getenv('AZURE_LINUX_USER', 'dcos')
# Prefixes must not begin with a number
options.agent_prefix = os.getenv('AZURE_AGENT_PREFIX', 'test' + random_id(10).lower())
options.master_prefix = os.getenv('AZURE_MASTER_PREFIX', 'test' + random_id(10).lower())
options.vm_size = os.getenv('AZURE_VM_SIZE', 'Standard_D2')
options.num_agents = os.getenv('AGENTS', '2')
options.name_suffix = os.getenv('AZURE_DCOS_SUFFIX', '12345')
options.oauth_enabled = os.getenv('AZURE_OAUTH_ENABLED', 'false')
options.vm_diagnostics_enabled = os.getenv('AZURE_VM_DIAGNOSTICS_ENABLED', 'true')
options.ci_flags = os.getenv('CI_FLAGS', '')
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
options.test_cmd = os.getenv(
'DCOS_PYTEST_CMD', ' '.join(add_env) + " py.test -vv -s -rs -m 'not ccm' " + options.ci_flags)
return options
def provide_cluster(options):
bw = test_util.aws.BotoWrapper(
region=options.aws_region,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
if not options.stack_name:
stack_name = 'CF-integration-test-{}'.format(random_id(10))
log.info(
'Spinning up AWS CloudFormation with ID: {}'.format(stack_name))
# TODO(mellenburg): use randomly generated keys this key is delivered by CI or user
if options.advanced:
cf, ssh_info = test_util.aws.DcosZenCfStack.create(
stack_name=stack_name,
boto_wrapper=bw,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
key_pair_name='default',
private_agent_type='m3.xlarge',
public_agent_type='m3.xlarge',
master_type='m3.xlarge',
vpc=options.vpc,
gateway=options.gateway,
private_subnet=options.private_subnet,
public_subnet=options.public_subnet)
else:
cf, ssh_info = test_util.aws.DcosCfStack.create(
stack_name=stack_name,
template_url=options.template_url,
private_agents=options.agents,
public_agents=options.public_agents,
admin_location='0.0.0.0/0',
key_pair_name='default',
boto_wrapper=bw)
cf.wait_for_complete(wait_before_poll_min=5)
else:
if options.advanced:
cf = test_util.aws.DcosZenCfStack(options.stack_name, bw)
else:
cf = test_util.aws.DcosCfStack(options.stack_name, bw)
ssh_info = test_util.aws.SSH_INFO[options.host_os]
stack_name = options.stack_name
# Resiliency testing requires knowing the stack name
options.test_cmd = 'AWS_STACK_NAME=' + stack_name + ' ' + options.test_cmd
return cf, ssh_info
def main():
options = check_environment()
unique_cluster_id = "dcos-ci-test-onprem-{}".format(random_id(10))
log.info("Spinning up AWS VPC with ID: {}".format(unique_cluster_id))
if options.test_install_prereqs:
os_name = "cent-os-7"
else:
os_name = "cent-os-7-dcos-prereqs"
bw = test_util.aws.BotoWrapper(
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
ssh_key = bw.create_key_pair(unique_cluster_id)
# Drop the key to disk so CI can cache it as an artifact
write_string('ssh_key', ssh_key)
if options.cluster_ami:
vpc = test_util.aws.VpcCfStack.create_from_ami(
stack_name=unique_cluster_id,
instance_type=options.instance_type,
instance_ami=options.cluster_ami,
# An instance for each cluster node plus the bootstrap.
instance_count=(options.masters + options.agents + options.public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=unique_cluster_id,
boto_wrapper=bw)
ssh_info = SshInfo(user=options.cluster_ami_ssh_user, home_dir=options.cluster_ami_ssh_home_dir)
else:
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=unique_cluster_id,
instance_type=options.instance_type,
instance_os=os_name,
# An instance for each cluster node plus the bootstrap.
instance_count=(options.masters + options.agents + options.public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=unique_cluster_id,
boto_wrapper=bw)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=options.masters,
num_agents=options.agents,
num_public_agents=options.public_agents,
)
test_util.cluster.install_dcos(
cluster,
installer_url=options.installer_url,
setup=options.do_setup,
api=options.use_api,
add_config_path=options.add_config_path,
# If we don't want to test the prereq install, use offline mode to avoid it.
installer_api_offline_mode=(not options.test_install_prereqs),
install_prereqs=options.test_install_prereqs,
install_prereqs_only=options.test_install_prereqs_only,
)
if options.test_install_prereqs and options.test_install_prereqs_only:
# install_dcos() exited after running prereqs, so we're done.
vpc.delete()
bw.delete_key_pair(unique_cluster_id)
sys.exit(0)
result = test_util.cluster.run_integration_tests(
cluster,
# Setting dns_search: mesos not currently supported in API
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
test_cmd=options.test_cmd,
)
if result == 0:
log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(unique_cluster_id)
else:
log.info("Test failed! VPC will remain for debugging 1 hour from instantiation")
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
# Don't install prereqs since stable breaks Docker 1.13. See
# https://jira.mesosphere.com/browse/DCOS_OSS-743.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
viplisten_app = create_marathon_viplisten_app()
viptalk_app = create_marathon_viptalk_app()
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
def main():
options = check_environment()
unique_cluster_id = "dcos-ci-test-onprem-{}".format(random_id(10))
log.info("Spinning up AWS VPC with ID: {}".format(unique_cluster_id))
if options.test_install_prereqs:
os_name = "cent-os-7"
else:
os_name = "cent-os-7-dcos-prereqs"
bw = test_util.aws.BotoWrapper(
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
ssh_key = bw.create_key_pair(unique_cluster_id)
# Drop the key to disk so CI can cache it as an artifact
write_string('ssh_key', ssh_key)
if options.cluster_ami:
vpc = test_util.aws.VpcCfStack.create_from_ami(
stack_name=unique_cluster_id,
instance_type=options.instance_type,
instance_ami=options.cluster_ami,
# An instance for each cluster node plus the bootstrap.
instance_count=(options.masters + options.agents +
options.public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=unique_cluster_id,
boto_wrapper=bw)
ssh_info = SshInfo(user=options.cluster_ami_ssh_user,
home_dir=options.cluster_ami_ssh_home_dir)
else:
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=unique_cluster_id,
instance_type=options.instance_type,
instance_os=os_name,
# An instance for each cluster node plus the bootstrap.
instance_count=(options.masters + options.agents +
options.public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=unique_cluster_id,
boto_wrapper=bw)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=options.masters,
num_agents=options.agents,
num_public_agents=options.public_agents,
)
test_util.cluster.install_dcos(
cluster,
installer_url=options.installer_url,
setup=options.do_setup,
api=options.use_api,
add_config_path=options.add_config_path,
# If we don't want to test the prereq install, use offline mode to avoid it.
installer_api_offline_mode=(not options.test_install_prereqs),
install_prereqs=options.test_install_prereqs,
install_prereqs_only=options.test_install_prereqs_only,
)
if options.test_install_prereqs and options.test_install_prereqs_only:
# install_dcos() exited after running prereqs, so we're done.
vpc.delete()
bw.delete_key_pair(unique_cluster_id)
sys.exit(0)
result = test_util.cluster.run_integration_tests(
cluster,
# Setting dns_search: mesos not currently supported in API
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
test_cmd=options.test_cmd,
)
if result == 0:
log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(unique_cluster_id)
else:
log.info(
"Test failed! VPC will remain for debugging 1 hour from instantiation"
)
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def main():
options = check_environment()
cluster = None
vpc = None
ssh_key = load_string(options.ssh_key_path)
if options.host_list is None:
log.info('VPC_HOSTS not provided, requesting new VPC ...')
unique_cluster_id = "installer-test-{}".format(random_id(10))
log.info("Spinning up AWS VPC with ID: {}".format(unique_cluster_id))
if options.test_install_prereqs:
os_name = "cent-os-7"
else:
os_name = "cent-os-7-dcos-prereqs"
# TODO(mellenburg): Switch to using generated keys
bw = test_util.aws.BotoWrapper(
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=unique_cluster_id,
instance_type=options.instance_type,
instance_os=os_name,
# An instance for each cluster node plus the bootstrap.
instance_count=(options.masters + options.agents +
options.public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name='default',
boto_wrapper=bw)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=options.masters,
num_agents=options.agents,
num_public_agents=options.public_agents,
)
else:
# Assume an existing onprem CentOS cluster.
cluster = test_util.cluster.Cluster.from_hosts(
ssh_info=test_util.aws.SSH_INFO['centos'],
ssh_key=ssh_key,
hosts=options.host_list,
num_masters=options.masters,
num_agents=options.agents,
num_public_agents=options.public_agents,
)
test_util.cluster.install_dcos(
cluster,
installer_url=options.installer_url,
setup=options.do_setup,
api=options.use_api,
add_config_path=options.add_config_path,
# If we don't want to test the prereq install, use offline mode to avoid it.
installer_api_offline_mode=(not options.test_install_prereqs),
install_prereqs=options.test_install_prereqs,
install_prereqs_only=options.test_install_prereqs_only,
)
if options.test_install_prereqs and options.test_install_prereqs_only:
# install_dcos() exited after running prereqs, so we're done.
if vpc:
vpc.delete()
sys.exit(0)
result = test_util.cluster.run_integration_tests(
cluster,
# Setting dns_search: mesos not currently supported in API
region=DEFAULT_AWS_REGION,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
test_cmd=options.test_cmd,
)
if result == 0:
log.info("Test successful! Deleting VPC if provided in this run.")
if vpc:
vpc.delete()
else:
log.info(
"Test failed! VPC will remain for debugging 1 hour from instantiation"
)
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
def main():
num_masters = int(os.getenv('MASTERS', '3'))
num_agents = int(os.getenv('AGENTS', '2'))
num_public_agents = int(os.getenv('PUBLIC_AGENTS', '1'))
stack_name = 'upgrade-test-' + random_id(10)
test_cmd = os.getenv('DCOS_PYTEST_CMD',
'py.test -vv -rs ' + os.getenv('CI_FLAGS', ''))
stable_installer_url = os.environ['STABLE_INSTALLER_URL']
installer_url = os.environ['INSTALLER_URL']
config_yaml_override_install = os.getenv('CONFIG_YAML_OVERRIDE_INSTALL')
config_yaml_override_upgrade = os.getenv('CONFIG_YAML_OVERRIDE_UPGRADE')
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(num_masters + num_agents + num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name='default',
boto_wrapper=test_util.aws.BotoWrapper(
region=os.getenv('DEFAULT_AWS_REGION', 'eu-central-1'),
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
),
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=load_string(os.getenv('DCOS_SSH_KEY_PATH', 'default_ssh_key')),
num_masters=num_masters,
num_agents=num_agents,
num_public_agents=num_public_agents,
)
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
test_util.cluster.install_dcos(
cluster,
stable_installer_url,
api=False,
add_config_path=config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
cluster_api = DcosApiSession(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
"root", # default_os_user
auth_user=DcosUser(CI_CREDENTIALS))
cluster_api.wait_for_dcos()
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(
['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
with cluster_workload(cluster_api):
test_util.cluster.upgrade_dcos(
cluster,
installer_url,
add_config_path=config_yaml_override_upgrade)
result = test_util.cluster.run_integration_tests(cluster,
test_cmd=test_cmd)
if result == 0:
log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
else:
log.info(
"Test failed! VPC cluster will remain available for debugging for 2 hour after instantiation."
)
sys.exit(result)