def check_cache_refresh_fails_409conflict(): try: cmd.run_cli('hello-world state refresh_cache') except Exception as e: if "failed: 409 Conflict" in e.args[0]: return True return False
def _remove_job_by_name(job_name): try: # --stop-current-job-runs ensures that fail-looping jobs (with restart.policy=ON_FAILURE) are consistently removed. sdk_cmd.run_cli('job remove {} --stop-current-job-runs'.format(job_name), print_output=False) except: log.info('Failed to remove any existing job named {} (this is likely as expected): {}'.format( job_name, traceback.format_exc()))
def add_universe_repos(): stub_urls = {} log.info('Adding universe repos') # prepare needed universe repositories stub_universe_urls = os.environ.get('STUB_UNIVERSE_URL') if not stub_universe_urls: return stub_urls log.info('Adding stub URLs: {}'.format(stub_universe_urls)) for url in stub_universe_urls.split(): print('url: {}'.format(url)) package_name = 'testpkg-' package_name += ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8)) stub_urls[package_name] = url # clean up any duplicate repositories current_universes = sdk_cmd.run_cli('package repo list --json') for repo in json.loads(current_universes)['repositories']: if repo['uri'] in stub_urls.values(): log.info('Removing duplicate stub URL: {}'.format(repo['uri'])) sdk_cmd.run_cli('package repo remove {}'.format(repo['name'])) # add the needed universe repositories for name, url in stub_urls.items(): log.info('Adding stub URL: {}'.format(url)) sdk_cmd.run_cli('package repo add --index=0 {} {}'.format(name, url)) log.info('Finished adding universe repos') return stub_urls
def run_job(job_dict, timeout_seconds=600, raise_on_failure=True): job_name = job_dict['id'] sdk_cmd.run_cli('job run {}'.format(job_name)) def wait_for_run_id(): runs = json.loads(sdk_cmd.run_cli('job show runs {} --json'.format(job_name))) if len(runs) > 0: return runs[0]['id'] return '' run_id = shakedown.wait_for(wait_for_run_id, noisy=True, timeout_seconds=timeout_seconds, ignore_exceptions=False) def fun(): # catch errors from CLI: ensure that the only error raised is our own: try: runs = json.loads(sdk_cmd.run_cli( 'job history --show-failures --json {}'.format(job_name), print_output=False)) except: log.info(traceback.format_exc()) return False successful_ids = [r['id'] for r in runs['history']['successfulFinishedRuns']] failed_ids = [r['id'] for r in runs['history']['failedFinishedRuns']] log.info('Job {} run history (waiting for successful {}): successful={} failed={}'.format( job_name, run_id, successful_ids, failed_ids)) # note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails if raise_on_failure and run_id in failed_ids: raise Exception('Job {} with id {} has failed, exiting early'.format(job_name, run_id)) return run_id in successful_ids shakedown.wait_for(fun, noisy=True, timeout_seconds=timeout_seconds, ignore_exceptions=False) return run_id
def _dump_diagnostics_bundle(item: pytest.Item) -> None: """Creates and downloads a DC/OS diagnostics bundle, and saves it to the artifact path for this test.""" rc, _, _ = sdk_cmd.run_cli("node diagnostics create all") if rc: log.error("Diagnostics bundle creation failed.") return @retrying.retry( wait_fixed=5000, stop_max_delay=10 * 60 * 1000, retry_on_result=lambda result: result is None, ) def wait_for_bundle_file() -> Optional[str]: rc, stdout, stderr = sdk_cmd.run_cli("node diagnostics --status --json") if rc: return None # e.g. { "some-ip": { stuff we want } } status = next(iter(json.loads(stdout).values())) if status["job_progress_percentage"] != 100: return None # e.g. "/var/lib/dcos/dcos-diagnostics/diag-bundles/bundle-2018-01-11-1515698691.zip" return str(os.path.basename(status["last_bundle_dir"])) bundle_filename = str(wait_for_bundle_file()) if bundle_filename: sdk_cmd.run_cli( "node diagnostics download {} --location={}".format( bundle_filename, _setup_artifact_path(item, bundle_filename) ) ) else: log.error("Diagnostics bundle didnt finish in time, giving up.")
def add_stub_universe_urls(stub_universe_urls: list) -> dict: stub_urls = {} if not stub_universe_urls: return stub_urls log.info('Adding stub URLs: {}'.format(stub_universe_urls)) for idx, url in enumerate(stub_universe_urls): log.info('URL {}: {}'.format(idx, repr(url))) package_name = 'testpkg-' package_name += ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8)) stub_urls[package_name] = url # clean up any duplicate repositories current_universes = sdk_cmd.run_cli('package repo list --json') for repo in json.loads(current_universes)['repositories']: if repo['uri'] in stub_urls.values(): log.info('Removing duplicate stub URL: {}'.format(repo['uri'])) sdk_cmd.run_cli('package repo remove {}'.format(repo['name'])) # add the needed universe repositories for name, url in stub_urls.items(): log.info('Adding stub repo {} URL: {}'.format(name, url)) rc, stdout, stderr = sdk_cmd.run_raw_cli('package repo add --index=0 {} {}'.format(name, url)) if rc != 0 or stderr: raise Exception( 'Failed to add stub repo {} ({}): stdout=[{}], stderr=[{}]'.format( name, url, stdout, stderr)) log.info('Finished adding universe repos') return stub_urls
def soak_upgrade_downgrade( package_name, service_name, running_task_count, additional_options={}, timeout_seconds=25*60, wait_for_deployment=True): sdk_cmd.run_cli("package install --cli {} --yes".format(package_name)) version = 'stub-universe' print('Upgrading to test version: {} {}'.format(package_name, version)) _upgrade_or_downgrade( package_name, version, service_name, running_task_count, additional_options, timeout_seconds, wait_for_deployment) # Default Universe is at --index=0 version = _get_pkg_version(package_name) print('Downgrading to Universe version: {} {}'.format(package_name, version)) _upgrade_or_downgrade( package_name, version, service_name, running_task_count, additional_options, timeout_seconds, wait_for_deployment)
def get_dcos_credentials() -> Tuple[str, dict]: dcosurl = sdk_cmd.run_cli('config show core.dcos_url', print_output=False) token = sdk_cmd.run_cli('config show core.dcos_acs_token', print_output=False) headers = { 'Content-Type': 'application/json', 'Authorization': 'token={}'.format(token.strip()), } return dcosurl.strip(), headers
def create_secret(secret_value: str, secret_path: str) -> None: install_enterprise_cli() delete_secret(secret=secret_path) sdk_cmd.run_cli( 'security secrets create --value="{account}" "{secret}"'.format( account=secret_value, secret=secret_path ) )
def setup_service_account( service_name: str, service_account_secret: Optional[str] = None, ) -> Dict[str, Any]: """ Setup the service account for TLS. If the account or secret of the specified name already exists, these are deleted. """ if sdk_utils.is_open_dcos(): log.error("The setup of a service account requires DC/OS EE. service_name=%s", service_name) raise Exception("The setup of a service account requires DC/OS EE") secret = service_name if service_account_secret is None else service_account_secret service_account = "{}-service-account".format(service_name.replace("/", "")) service_account_info = sdk_security.setup_security( service_name, linux_user="******", service_account=service_account, service_account_secret=secret, ) log.info("Adding permissions required for TLS.") if sdk_utils.dcos_version_less_than("1.11"): sdk_cmd.run_cli("security org groups add_user superusers {sa}".format(sa=service_account)) else: acls = [ {"rid": "dcos:secrets:default:/{}/*".format(service_name.strip("/")), "action": "full"}, { "rid": "dcos:secrets:list:default:/{}".format(service_name.strip("/")), "action": "read", }, {"rid": "dcos:adminrouter:ops:ca:rw", "action": "full"}, {"rid": "dcos:adminrouter:ops:ca:ro", "action": "full"}, ] for acl in acls: cmd_list = [ "security", "org", "users", "grant", "--description", '"Allow provisioning TLS certificates"', service_account, acl["rid"], acl["action"], ] sdk_cmd.run_cli(" ".join(cmd_list)) return service_account_info
def teardown(args: dict): log.info("Tearing down KDC") sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"])) sdk_security.install_enterprise_cli() if args.binary_secret: sdk_security.delete_secret(args.secret_name) else: sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name)) log.info("KDC cluster successfully torn down")
def service_account(): """ Creates service account and yields the name. """ name = config.SERVICE_NAME sdk_security.create_service_account( service_account_name=name, service_account_secret=name) # TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475 sdk_cmd.run_cli( "security org groups add_user superusers {name}".format(name=name)) yield name sdk_security.delete_service_account( service_account_name=name, service_account_secret=name)
def install_job(job_dict, tmp_dir=None): job_name = job_dict['id'] if not tmp_dir: tmp_dir = tempfile.mkdtemp(prefix='sdk-test') out_filename = os.path.join(tmp_dir, '{}.json'.format(job_name)) job_str = json.dumps(job_dict) log.info('Writing job file for {} to: {}\n{}'.format(job_name, out_filename, job_str)) with open(out_filename, 'w') as f: f.write(job_str) _remove_job_by_name(job_name) sdk_cmd.run_cli('job add {}'.format(out_filename))
def configure_package(configure_security): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) # Create service account sdk_security.create_service_account( service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME ) sdk_cmd.run_cli( "security org groups add_user superusers {name}".format(name=config.SERVICE_NAME) ) sdk_install.install( config.PACKAGE_NAME, config.SERVICE_NAME, 6, additional_options={ "service": { "yaml": "tls", "service_account": config.SERVICE_NAME, "service_account_secret": config.SERVICE_NAME, # Legacy values "principal": config.SERVICE_NAME, "secret_name": config.SERVICE_NAME, }, "tls": {"discovery_task_prefix": DISCOVERY_TASK_PREFIX}, }, ) sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME) yield # let the test session execute finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) sdk_security.delete_service_account( service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME ) # Make sure that all the TLS artifacts were removed from the secrets store. _, output, _ = sdk_cmd.run_cli("security secrets list {name}".format(name=config.SERVICE_NAME)) artifact_suffixes = [ "certificate", "private-key", "root-ca-certificate", "keystore", "truststore", ] for suffix in artifact_suffixes: assert suffix not in output
def test_state_properties_get(): # 'suppressed' could be missing if the scheduler recently started, loop for a bit just in case: def check_for_nonempty_properties(): stdout = cmd.run_cli('hello-world state properties') return len(json.loads(stdout)) > 0 spin.time_wait_noisy(lambda: check_for_nonempty_properties(), timeout_seconds=30.) stdout = cmd.run_cli('hello-world state properties') jsonobj = json.loads(stdout) assert len(jsonobj) == 1 assert jsonobj[0] == "suppressed" stdout = cmd.run_cli('hello-world state property suppressed') assert stdout == "true\n"
def create_service_account(service_account_name: str, service_account_secret: str) -> None: log.info('Creating service account for account={account} secret={secret}'.format( account=service_account_name, secret=service_account_secret)) log.info('Install cli necessary for security') sdk_cmd.run_cli('package install dcos-enterprise-cli --yes') log.info('Remove any existing service account and/or secret') delete_service_account(service_account_name, service_account_secret) log.info('Create keypair') sdk_cmd.run_cli('security org service-accounts keypair private-key.pem public-key.pem') log.info('Create service account') sdk_cmd.run_cli( 'security org service-accounts create -p public-key.pem -d "My service account" "{account}"'.format( account=service_account_name)) log.info('Create secret') sdk_cmd.run_cli( 'security secrets create-sa-secret --strict private-key.pem "{account}" "{secret}"'.format( account=service_account_name, secret=service_account_secret)) log.info('Service account created for account={account} secret={secret}'.format( account=service_account_name, secret=service_account_secret))
def configure_package(configure_security): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) sdk_cmd.run_cli("package install --cli dcos-enterprise-cli --yes") try_delete_secrets("{}/".format(config.SERVICE_NAME)) try_delete_secrets("{}/somePath/".format(config.SERVICE_NAME)) try_delete_secrets() yield # let the test session execute finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) try_delete_secrets("{}/".format(config.SERVICE_NAME)) try_delete_secrets("{}/somePath/".format(config.SERVICE_NAME)) try_delete_secrets()
def delete_service_account(service_account_name: str, service_account_secret: str) -> None: """ Deletes service account with private key that belongs to the service account. """ # ignore any failures: sdk_cmd.run_cli("security org service-accounts delete {name}".format(name=service_account_name)) # Files generated by service-accounts keypair command should get removed for keypair_file in ["private-key.pem", "public-key.pem"]: try: os.unlink(keypair_file) except OSError: pass delete_secret(secret=service_account_secret)
def _bundle_directory_name(self) -> str: _, cluster_name, _ = sdk_cmd.run_cli("config show cluster.name", print_output=False) return "{}_{}_{}".format( cluster_name, sdk_utils.get_deslashed_service_name(self.service_name), directory_date_string(), )
def add_dcos_files_to_registry(tmpdir_factory: TempdirFactory) -> None: # Use DCOS_FILES_PATH if its set to a valid path OR use pytest's tmpdir. dcos_files_path = os.environ.get("DCOS_FILES_PATH", "") valid_path_set = os.path.isdir(dcos_files_path) if valid_path_set and not os.access(dcos_files_path, os.W_OK): log.warning("{} is not writable.".format(dcos_files_path)) valid_path_set = False if not valid_path_set: dcos_files_path = str(tmpdir_factory.mktemp(sdk_utils.random_string())) stub_universe_urls = sdk_repository.get_repos() log.info( "Using {} to build .dcos files (if not exists) from {}".format( dcos_files_path, stub_universe_urls ) ) dcos_files_list = build_dcos_files_from_stubs( stub_universe_urls, dcos_files_path, tmpdir_factory ) log.info("Bundled .dcos files : {}".format(dcos_files_list)) @retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000) def wait_for_added_registry(name: str, version: str) -> None: code, stdout, stderr = sdk_cmd.run_cli( "registry describe --package-name={} --package-version={} --json".format(name, version), print_output=False, ) assert code == 0 and json.loads(stdout).get("status") == "Added" for file_path, name, version in dcos_files_list: rc, out, err = sdk_cmd.run_cli("registry add --dcos-file={} --json".format(file_path)) assert rc == 0 assert len(json.loads(out)["packages"]) > 0, "No packages were added" wait_for_added_registry(name, version)
def grant_perms_for_registry_account(service_uid: str) -> None: # Grant only required permissions to registry perms = "dcos:adminrouter:ops:ca:rw" rc, _, _ = sdk_cmd.run_cli( " ".join(["security", "org", "users", "grant", service_uid, perms, "full"]) ) assert rc == 0, "Required perms [{}] could not be obtained for {}".format(perms, service_uid)
def build_dcos_file_from_universe_definition( package: Dict, dcos_files_path: str, tmpdir_factory: TempdirFactory, ) -> Tuple[str, str, str]: """ Build the .dcos file if its not already present in the given directory. Returns a Tuple containing (path of .dcos file, name, and version) """ # TODO Ideally we should `migrate` and then `build`. name = package["name"] version = package["version"] target = os.path.join(dcos_files_path, "{}-{}.dcos".format(name, version)) if os.path.isfile(target): log.info("Skipping build, using cached file : {}".format(target)) else: del package["releaseVersion"] del package["selected"] package_json_file = tmpdir_factory.mktemp(sdk_utils.random_string()).join( sdk_utils.random_string() ) package_json_file.write(json.dumps(package)) rc, _, _ = sdk_cmd.run_cli( " ".join( [ "registry", "build", "--build-definition-file={}".format(str(package_json_file)), "--output-directory={}".format(dcos_files_path), "--json", ] ) ) assert rc == 0 assert os.path.isfile(target), "No valid .dcos file is built" return target, name, version
def test_pods_restart_graceful_shutdown(): options = { "world": { "kill_grace_period": 30 } } sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) sdk_install.install(config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_TASK_COUNT, additional_options=options) world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world-0') jsonobj = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod restart world-0', json=True) assert len(jsonobj) == 2 assert jsonobj['pod'] == 'world-0' assert len(jsonobj['tasks']) == 1 assert jsonobj['tasks'][0] == 'world-0-server' sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world-0', world_ids) config.check_running() # ensure the SIGTERM was sent via the "all clean" message in the world # service's signal trap/handler, BUT not the shell command, indicated # by "echo". stdout = sdk_cmd.run_cli( "task log --completed --lines=1000 {}".format(world_ids[0])) clean_msg = None for s in stdout.split('\n'): if s.find('echo') < 0 and s.find('all clean') >= 0: clean_msg = s assert clean_msg is not None
def attached_dcos_cluster() -> (int, Any): rc, stdout, stderr = sdk_cmd.run_cli("cluster list --attached --json", print_output=False) try: attached_clusters = json.loads(stdout) except json.JSONDecodeError as e: return (1, "Error decoding JSON while getting attached DC/OS cluster: {}".format(e)) if rc == 0: if len(attached_clusters) == 0: return (1, "No cluster is attached") if len(attached_clusters) > 1: return ( 1, "More than one attached clusters. This is an invalid DC/OS CLI state\n{}".format( stdout ), ) else: if "No cluster is attached" in stderr: return (rc, stderr) else: return (False, "Unexpected error\nstdout: '{}'\nstderr: '{}'".format(stdout, stderr)) # Having more than one attached cluster is an invalid DC/OS CLI state that is handled above. # In normal conditions `attached_clusters` is an unary array. attached_cluster = attached_clusters[0] return (rc, attached_cluster)
def test_pods_restart_graceful_shutdown(): install_options_helper(30) world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "world-0") rc, stdout, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, config.SERVICE_NAME, "pod restart world-0" ) assert rc == 0, "Pod restart failed" jsonobj = json.loads(stdout) assert len(jsonobj) == 2 assert jsonobj["pod"] == "world-0" assert len(jsonobj["tasks"]) == 1 assert jsonobj["tasks"][0] == "world-0-server" sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "world-0", world_ids) check_healthy() # ensure the SIGTERM was sent via the "all clean" message in the world # service's signal trap/handler, BUT not the shell command, indicated # by "echo". _, stdout, _ = sdk_cmd.run_cli("task log --completed --lines=1000 {}".format(world_ids[0])) clean_msg = None for s in stdout.split("\n"): if s.find("echo") < 0 and s.find("all clean") >= 0: clean_msg = s assert clean_msg is not None
def decommission_agent(agent_id: str) -> None: assert sdk_utils.dcos_version_at_least("1.11"),\ "node decommission is supported in DC/OS 1.11 and above only" rc, _, _ = sdk_cmd.run_cli( "node decommission {}".format(agent_id) ) assert rc == 0
def create_tls_artifacts(cn: str, marathon_task: str) -> str: pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) log.info("Generating certificate. cn={}, task={}".format(cn, marathon_task)) output = sdk_cmd.marathon_task_exec( marathon_task, 'openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr ' '-subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format(priv_path, cn)) assert output[0] is 0 rc, raw_csr, _ = sdk_cmd.marathon_task_exec(marathon_task, 'cat request.csr') assert rc is 0 request = { "certificate_request": raw_csr } token = sdk_cmd.run_cli("config show core.dcos_acs_token") output = sdk_cmd.marathon_task_exec( marathon_task, "curl --insecure -L -X POST " "-H 'Authorization: token={}' " "leader.mesos/ca/api/v2/sign " "-d '{}'".format(token, json.dumps(request))) assert output[0] is 0 # Write the public cert to the client certificate = json.loads(output[1])["result"]["certificate"] output = sdk_cmd.marathon_task_exec(marathon_task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path)) assert output[0] is 0 _create_keystore_truststore(cn, marathon_task) return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def get_task_host(task_name): _, out, _ = sdk_cmd.run_cli("task {} --json".format(task_name)) tasks_json = json.loads(out) matching_tasks = list(filter(lambda t: t["name"] == task_name, tasks_json)) assert len(matching_tasks) == 1, "Duplicate tasks found with same name : [{}]".format(tasks_json) task_info = matching_tasks.pop() host = None for label in task_info["labels"]: if label["key"] == "offer_hostname": host = label["value"] break if host is None: raise Exception("offer_hostname label is not present!: {}".format(task_info)) # Validation: Check that label matches summary returned by CLI for task in sdk_tasks.get_summary(): if task.name == task_name: if task.host == host: # OK! return host else: # CLI's hostname doesn't match the TaskInfo labels. Bug! raise Exception( "offer_hostname label [{}] doesn't match CLI output [{}]\nTask:\n{}".format( host, task.host, task_info ) ) # Unable to find desired task in CLI! raise Exception("Unable to find task named {} in CLI".format(task_name))
def _get_universe_url(): repositories = json.loads(sdk_cmd.run_cli('package repo list --json'))['repositories'] for repo in repositories: if repo['name'] == 'Universe': log.info("Found Universe URL: {}".format(repo['uri'])) return repo['uri'] assert False, "Unable to find 'Universe' in list of repos: {}".format(repositories)
def get_task_host(task_name): out = sdk_cmd.run_cli('task {} --json'.format(task_name)) task_info = json.loads(out)[0] host = None for label in task_info['labels']: if label['key'] == 'offer_hostname': host = label['value'] break if host is None: raise Exception("offer_hostname label is not present!: {}".format(task_info)) # Validation: Check that label matches summary returned by CLI for task in sdk_tasks.get_summary(): if task.name == task_name: if task.host == host: # OK! return host else: # CLI's hostname doesn't match the TaskInfo labels. Bug! raise Exception("offer_hostname label {} doesn't match CLI output!\nTask:\n{}".format(task_info)) # Unable to find desired task in CLI! raise Exception("Unable to find task named {} in CLI".format(task_name))
def _retried_uninstall_package_and_wait(package_name, service_name): if sdk_marathon.app_exists(service_name): log.info("Uninstalling package {} with service name {}".format( package_name, service_name)) sdk_cmd.run_cli("package uninstall {} --app-id={} --yes".format( package_name, service_name), check=True) # Wait on the app no longer being listed in Marathon, at which point it is uninstalled. # At the same time, log the deploy plan state as we wait for the app to finish uninstalling. @retrying.retry( stop_max_delay=TIMEOUT_SECONDS * 1000, wait_fixed=5000, retry_on_result=lambda result: not result, ) def wait_for_removal_log_deploy_plan(): if not sdk_marathon.app_exists(service_name): return True # App still exists, print the deploy plan. Best effort: It is expected for the scheduler # to become unavailable once uninstall completes. try: log.info( sdk_plan.plan_string( "deploy", sdk_plan.get_plan_once(service_name, "deploy"))) except Exception: pass # best effort attempt at logging plan content return False log.info("Waiting for {} to be removed".format(service_name)) wait_for_removal_log_deploy_plan() else: log.info( 'Skipping uninstall of package {}/service {}: App named "{}" doesn\'t exist' .format(package_name, service_name, service_name))
def fun(): stdout = cmd.run_cli( 'task exec {} /bin/bash -c "JAVA_HOME=$(ls -d jre*/) apache-cassandra-*/bin/nodetool -p 7199 status"' .format(task_id)) up_ips = [] for line in stdout.split('\n'): words = list(filter(None, line.split())) if len(words) < 2: continue if not 'UN' == words[0]: continue up_ips.append(words[1]) utils.out('UN nodes (want {} entries without {}): {}'.format( DEFAULT_TASK_COUNT, pod_host, up_ips)) return len(up_ips) == DEFAULT_TASK_COUNT and not pod_host in up_ips
def test_node_replace_replaces_node(): tasks = cmd.run_cli('task') node_ip = [t for t in tasks.split('\n') if t.startswith('node-2-server')].pop().split()[1] # Update the placement constraints so the new node doesn't end up on the # same host config = marathon.get_config(PACKAGE_NAME) config['env']['PLACEMENT_CONSTRAINT'] = 'hostname:UNLIKE:{}'.format( node_ip) marathon.update_app(PACKAGE_NAME, config) plan.wait_for_completed_deployment(PACKAGE_NAME) # start replace and wait for it to finish cmd.run_cli('cassandra pods replace node-2') plan.wait_for_completed_recovery(PACKAGE_NAME) # Install replace verification job with correct node IP templated # (the job checks for that IP's absence in the peers list and also verifies # that the expected number of peers is present, meaning that the node was # replaced from Cassandra's perspective) with JobContext([VERIFY_REPLACE_JOB], NODE_IP=node_ip): spin.time_wait_noisy(lambda: try_job(VERIFY_REPLACE_JOB))
def get_marathon_app(service_name: str) -> (int, Any): rc, stdout, stderr = sdk_cmd.run_cli( "marathon app show {}".format(service_name), print_output=False) if rc != 0: if "does not exist" in stderr: return (rc, "Service {} does not exist".format(service_name)) else: return (rc, "Unexpected error\nstdout: '{}'\nstderr: '{}'".format( stdout, stderr)) try: return (rc, json.loads(stdout)) except json.JSONDecodeError as e: return (1, "Error decoding JSON: {}".format(e))
def test_pod_replace(): world_ids = sdk_tasks.get_task_ids(PACKAGE_NAME, 'world-0') # get current agent id (TODO: uncomment if/when agent is guaranteed to change in a replace operation): #stdout = sdk_cmd.run_cli('hello-world pod info world-0') #old_agent = json.loads(stdout)[0]['info']['slaveId']['value'] jsonobj = json.loads(sdk_cmd.run_cli('hello-world pod replace world-0')) assert len(jsonobj) == 2 assert jsonobj['pod'] == 'world-0' assert len(jsonobj['tasks']) == 1 assert jsonobj['tasks'][0] == 'world-0-server' sdk_tasks.check_tasks_updated(PACKAGE_NAME, 'world-0', world_ids) check_running()
def _get_kdc_task(task_name: str) -> dict: """ :return (dict): The task object of the KDC app with desired properties to be retrieved by other methods. """ log.info("Getting KDC task") raw_tasks = sdk_cmd.run_cli("task --json", print_output=False) if raw_tasks: tasks = json.loads(raw_tasks) for task in tasks: if task["name"] == task_name: return task raise RuntimeError( "Expecting marathon KDC task but no such task found. Running tasks: {tasks}" .format(tasks=raw_tasks))
def __create_and_upload_secret(self): """ This method base64 encodes the keytab file and creates a secret with this encoded content so the tasks can fetch it. """ log.info("Creating and uploading the keytab file to the secret store") try: base64_encode_cmd = "base64 -w 0 {source} > {destination}".format( source=os.path.join(self.temp_working_dir.name, self.keytab_file_name), destination=os.path.join(self.temp_working_dir.name, self.base64_encoded_keytab_file_name)) run(base64_encode_cmd, shell=True) except Exception as e: raise Exception( "Failed to base64-encode the keytab file: {}".format(repr(e))) self.keytab_secret_path = "{}_keytab".format(DCOS_BASE64_PREFIX) # TODO: check if a keytab secret of same name already exists create_secret_cmd = "security secrets create {keytab_secret_path} --value-file {encoded_keytab_path}".format( keytab_secret_path=self.keytab_secret_path, encoded_keytab_path=os.path.join( self.temp_working_dir.name, self.base64_encoded_keytab_file_name)) try: sdk_cmd.run_cli(create_secret_cmd) except RuntimeError as e: raise RuntimeError( "Failed to create secret for the base64-encoded keytab file: {}" .format(repr(e))) log.info( "Successfully uploaded a base64-encoded keytab file to the secret store" )
def _get_kdc_task_inner(task_name: str) -> dict: log.info("Getting KDC task") _, raw_tasks, _ = sdk_cmd.run_cli("task --json", print_output=False) if raw_tasks: tasks = json.loads(raw_tasks) for task in tasks: assert isinstance(task, dict) if task["name"] == task_name: return task raise RuntimeError( "Expecting marathon KDC task but no such task found. Running tasks: {tasks}".format( tasks=raw_tasks ) )
def test_secrets_basic(): # 1) create Secrets # 2) install examples/secrets.yml # 3) if secret file is not created, tasks will fail # 4) wait till deployment finishes # 5) do replace operation # 6) ensure all tasks are running # 7) delete Secrets install.uninstall(PACKAGE_NAME) create_secrets("{}/".format(PACKAGE_NAME)) install.install(PACKAGE_NAME, NUM_HELLO + NUM_WORLD, additional_options=secret_options) # default is serial strategy, hello deploys first # launch will fail if secrets are not available or not accessible plan.wait_for_completed_deployment(PACKAGE_NAME) hello_tasks_0 = tasks.get_task_ids(PACKAGE_NAME, "hello-0") world_tasks_0 = tasks.get_task_ids(PACKAGE_NAME, "word-0") # ensure that secrets work after replace cmd.run_cli('hello-world pods replace hello-0') cmd.run_cli('hello-world pods replace world-0') tasks.check_tasks_updated(PACKAGE_NAME, "hello-0", hello_tasks_0) tasks.check_tasks_updated(PACKAGE_NAME, 'world-0', world_tasks_0) # tasks will fail if secret files are not created by mesos module tasks.check_running(PACKAGE_NAME, NUM_HELLO + NUM_WORLD) # clean up and delete secrets delete_secrets("{}/".format(PACKAGE_NAME))
def test_shutdown_host_test(): service_ip = shakedown.get_service_ips(PACKAGE_NAME).pop() sdk_utils.out('marathon ip = {}'.format(service_ip)) node_ip = 0 for pod_id in range(0, DEFAULT_TASK_COUNT): node_ip = get_pod_host(pod_id) if node_ip != service_ip: break if node_ip is None: assert Fail, 'could not find a node to shutdown' old_agent = get_pod_agent(pod_id) sdk_utils.out('pod id = {}, node_ip = {}, agent = {}'.format( pod_id, node_ip, old_agent)) task_ids = tasks.get_task_ids(PACKAGE_NAME, 'node-{}'.format(pod_id)) # instead of partition/reconnect, we shutdown host permanently status, stdout = shakedown.run_command_on_agent(node_ip, 'sudo shutdown -h +1') sdk_utils.out('shutdown agent {}: [{}] {}'.format(node_ip, status, stdout)) assert status is True time.sleep(100) cmd.run_cli('cassandra pods replace node-{}'.format(pod_id)) tasks.check_tasks_updated(PACKAGE_NAME, 'node', task_ids) #double check all tasks are running tasks.check_running(PACKAGE_NAME, DEFAULT_TASK_COUNT) new_agent = get_pod_agent(pod_id) assert old_agent != new_agent
def check_broker(id: int): rc, stdout, _ = sdk_cmd.run_cli( "task log kafka-{}-broker --lines 1000".format(id), print_output=False) if rc or not stdout: raise Exception("No task logs for kafka-{}-broker".format(id)) last_log_index = stdout.rfind(broker_log_line[id]) success_index = stdout.rfind( "zookeeper state changed (SyncConnected) (org.I0Itec.zkclient.ZkClient)" ) assert last_log_index > -1 and last_log_index < success_index, "{}:{} STDOUT: {}".format( last_log_index, success_index, stdout)
def upgrade_downgrade(package_name, running_task_count): install.uninstall(package_name) test_version = get_pkg_version(package_name) print('Found test version: {}'.format(test_version)) repositories = json.loads( cmd.run_cli('package repo list --json'))['repositories'] print("Repositories: " + str(repositories)) universe_url = "fail" for repo in repositories: if repo['name'] == 'Universe': universe_url = repo['uri'] break assert "fail" != universe_url print("Universe URL: " + universe_url) # Move the Universe repo to the top of the repo list shakedown.remove_package_repo('Universe') add_repo('Universe', universe_url, test_version, 0, package_name) universe_version = get_pkg_version(package_name) print('Found Universe version: {}'.format(universe_version)) print('Installing Universe version') install.install(package_name, running_task_count, check_suppression=False) # Move the Universe repo to the bottom of the repo list shakedown.remove_package_repo('Universe') add_last_repo('Universe', universe_url, universe_version, package_name) print('Upgrading to test version') upgrade_or_downgrade(package_name, running_task_count) # Move the Universe repo to the top of the repo list shakedown.remove_package_repo('Universe') add_repo('Universe', universe_url, test_version, 0, package_name) print('Downgrading to master version') upgrade_or_downgrade(package_name, running_task_count) # Move the Universe repo to the bottom of the repo list shakedown.remove_package_repo('Universe') add_last_repo('Universe', universe_url, universe_version, package_name) print('Upgrading to test version') upgrade_or_downgrade(package_name, running_task_count)
def test_pods_list(): stdout = cmd.run_cli('hello-world pods list') jsonobj = json.loads(stdout) assert len(jsonobj) == configured_task_count() # expect: X instances of 'hello-#' followed by Y instances of 'world-#', # in alphanumerical order first_world = -1 for i in range(len(jsonobj)): entry = jsonobj[i] if first_world < 0: if entry.startswith('world-'): first_world = i if first_world == -1: assert jsonobj[i] == 'hello-{}'.format(i) else: assert jsonobj[i] == 'world-{}'.format(i - first_world)
def _get_host_name(host_id: str) -> str: """ Fetches the host name for the host running the KDC app. :param host_id (str): The ID of the host, used to look up the appropriate node. :return (str): Name of the host running the KDC app. """ log.info("Getting hostname") raw_nodes = sdk_cmd.run_cli("node --json") if raw_nodes: nodes = json.loads(raw_nodes) for node in nodes: if "id" in node and node["id"] == host_id: log.info("Host name is %s", node["hostname"]) return node["hostname"] raise RuntimeError("Failed to get name of host running the KDC app: {nodes}")
def test_state_refresh_disable_cache(): '''Disables caching via a scheduler envvar''' check_running(FOLDERED_SERVICE_NAME) task_ids = tasks.get_task_ids(FOLDERED_SERVICE_NAME, '') # caching enabled by default: stdout = cmd.run_cli('hello-world --name={} state refresh_cache'.format( FOLDERED_SERVICE_NAME)) assert "Received cmd: refresh" in stdout config = marathon.get_config(FOLDERED_SERVICE_NAME) config['env']['DISABLE_STATE_CACHE'] = 'any-text-here' marathon.update_app(FOLDERED_SERVICE_NAME, config) tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, '', task_ids) check_running(FOLDERED_SERVICE_NAME) # caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up): def check_cache_refresh_fails_409conflict(): try: cmd.run_cli('hello-world --name={} state refresh_cache'.format( FOLDERED_SERVICE_NAME)) except Exception as e: if "failed: 409 Conflict" in e.args[0]: return True return False shakedown.wait_for(lambda: check_cache_refresh_fails_409conflict(), timeout_seconds=120.) config = marathon.get_config(FOLDERED_SERVICE_NAME) del config['env']['DISABLE_STATE_CACHE'] marathon.update_app(FOLDERED_SERVICE_NAME, config) tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, '', task_ids) check_running(FOLDERED_SERVICE_NAME) shakedown.deployment_wait( ) # ensure marathon thinks the deployment is complete too # caching reenabled, refresh_cache should succeed (eventually, once scheduler is up): def check_cache_refresh(): return cmd.run_cli('hello-world --name={} state refresh_cache'.format( FOLDERED_SERVICE_NAME)) stdout = shakedown.wait_for(lambda: check_cache_refresh(), timeout_seconds=120.) assert "Received cmd: refresh" in stdout
def install_enterprise_cli(force=False): """ Install the enterprise CLI if required """ if not force: _, stdout, _ = sdk_cmd.run_cli("security --version", print_output=False) if stdout: log.info("DC/OS enterprise version %s CLI already installed", stdout.strip()) @retrying.retry( stop_max_attempt_number=3, wait_fixed=2000, retry_on_exception=lambda e: isinstance(e, Exception), ) def _install_impl(): sdk_cmd.run_cli("package install --yes --cli dcos-enterprise-cli", check=True) _install_impl()
def current_cluster_name() -> (bool, str): rc, stdout, stderr = sdk_cmd.run_cli("config show cluster.name", print_output=False) if rc != 0: if "Property 'cluster.name' doesn't exist" in stderr: return ( False, "No cluster is set up. Please run `dcos cluster setup`\nstdout: '{}'\nstderr: '{}'" .format(stdout, stderr), ) else: return (False, "Unexpected error\nstdout: '{}'\nstderr: '{}'".format( stdout, stderr)) return (True, stdout)
def get_task_files_for_id(task_id: str) -> dict: try: ls_lines = sdk_cmd.run_cli('task ls --long --all {}'.format(task_id)).split('\n') ret = {} for line in ls_lines: match = task_ls_pattern.match(line) if not match: log.warning('Unable to parse line: {}'.format(line)) continue # match.group(1): "4096 ", match.group(2): "Jul 21 22:07", match.group(3): "jre1.8.0_144 " filename = match.group(3).strip() # build timestamp for use in output filename: 'Jul 21 22:07' => '0721_2207' timestamp = time.strftime('%m%d_%H%M', time.strptime(match.group(2), '%b %d %H:%M')) ret[filename] = timestamp return ret except: log.exception('Failed to get list of files for task: {}'.format(task_id)) return {}
def fun(): # catch errors from CLI: ensure that the only error raised is our own: try: runs = json.loads(sdk_cmd.run_cli( 'job history --show-failures --json {}'.format(job_name), print_output=False)) except: sdk_utils.out(traceback.format_exc()) return False successful_ids = [r['id'] for r in runs['history']['successfulFinishedRuns']] failed_ids = [r['id'] for r in runs['history']['failedFinishedRuns']] sdk_utils.out('Job {} run history (waiting for successful {}): successful={} failed={}'.format( job_name, run_id, successful_ids, failed_ids)) # note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails if raise_on_failure and run_id in failed_ids: raise Exception('Job {} with id {} has failed, exiting early'.format(job_name, run_id)) return run_id in successful_ids
def get_summary(with_completed=False): '''Returns a summary of task information as returned by the DC/OS CLI. This may be used instead of invoking 'dcos task [--all]' directly. Returns a list of Task objects. ''' # Note: We COULD use --json, but there appears to be some fancy handling done in the CLI for the # non-json version, particularly around the running user. Just grab the non-"--json" version of things. task_lines = sdk_cmd.run_cli('task --all' if with_completed else 'task', print_output=False).split('\n') output = [] for task_line in task_lines[1:]: # First line is the header line task = Task.parse(task_line) if task is not None: output.append(task) log.info('Task summary (with_completed={}):\n- {}'.format( with_completed, '\n- '.join([str(e) for e in output]))) return output
def run_janitor(service_name, role, service_account, znode): if role is None: role = sdk_utils.get_deslashed_service_name(service_name) + '-role' if service_account is None: service_account = service_name + '-principal' if znode is None: znode = sdk_utils.get_zk_path(service_name) auth_token = sdk_cmd.run_cli('config show core.dcos_acs_token', print_output=False).strip() cmd_list = [ "docker", "run", "mesosphere/janitor", "/janitor.py", "-r", role, "-p", service_account, "-z", znode, "--auth_token={}".format(auth_token) ] cmd = " ".join(cmd_list) sdk_cmd.master_ssh(cmd)
def test_state_refresh_disable_cache(): '''Disables caching via a scheduler envvar''' check_running() task_ids = tasks.get_task_ids(PACKAGE_NAME, '') # caching enabled by default: stdout = cmd.run_cli('hello-world state refresh_cache') assert "Received cmd: refresh" in stdout config = marathon.get_config(PACKAGE_NAME) cpus = float(config['env']['HELLO_CPUS']) config['env']['DISABLE_STATE_CACHE'] = 'any-text-here' cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config) tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids) check_running() # caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up): def check_cache_refresh_fails_409conflict(): try: cmd.run_cli('hello-world state refresh_cache') except Exception as e: if "failed: 409 Conflict" in e.args[0]: return True return False spin.time_wait_noisy(lambda: check_cache_refresh_fails_409conflict(), timeout_seconds=120.) config = marathon.get_config(PACKAGE_NAME) cpus = float(config['env']['HELLO_CPUS']) del config['env']['DISABLE_STATE_CACHE'] cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config) tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids) check_running() # caching reenabled, refresh_cache should succeed (eventually, once scheduler is up): def check_cache_refresh(): return cmd.run_cli('hello-world state refresh_cache') stdout = spin.time_wait_return(lambda: check_cache_refresh(), timeout_seconds=120.) assert "Received cmd: refresh" in stdout
def create_tls_artifacts(cn: str, task: str) -> str: pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) log.info("Generating certificate. cn={}, task={}".format(cn, task)) output = sdk_tasks.task_exec( task, 'openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr \ -subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format( priv_path, cn)) log.info(output) assert output[0] is 0 raw_csr = sdk_tasks.task_exec(task, 'cat request.csr') assert raw_csr[0] is 0 request = { "certificate_request": raw_csr[1] # The actual content is second in the array } token = sdk_cmd.run_cli("config show core.dcos_acs_token") cmd = "curl -X POST \ -H 'Authorization: token={}' \ leader.mesos/ca/api/v2/sign \ -d '{}'".format(token, json.dumps(request)) output = sdk_tasks.task_exec( task, "curl -X POST \ -H 'Authorization: token={}' \ leader.mesos/ca/api/v2/sign \ -d '{}'".format(token, json.dumps(request))) log.info(output) assert output[0] is 0 # Write the public cert to the client certificate = json.loads(output[1])["result"]["certificate"] output = sdk_tasks.task_exec( task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path)) log.info(output) assert output[0] is 0 create_keystore_truststore(cn, task) return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def install_app_from_file(app_name: str, app_def_path: str) -> (bool, str): """ Installs a marathon app using the path to an app definition. Args: app_def_path: Path to app definition Returns: (bool, str) tuple: Boolean indicates success of install attempt. String indicates error message if install attempt failed. """ output = sdk_cmd.run_cli("{cmd} {file_path}".format( cmd="marathon app add ", file_path=app_def_path)) if "Created deployment" not in output: return 1, output log.info("Waiting for app to be running...") shakedown.wait_for_task("marathon", app_name) return 0, ""
def _master_zero_http_port(service_name): dns = json.loads( sdk_cmd.run_cli('{} --name={} endpoints master'.format( PACKAGE_NAME, service_name), print_output=False))['dns'] # array will initially look something like this in CCM, with some 9300 ports and some lower ones [ # "master-0-node.elastic.[...]:9300", # "master-0-node.elastic.[...]:1025", # "master-1-node.elastic.[...]:9300", # "master-1-node.elastic.[...]:1025", # "master-2-node.elastic.[...]:9300", # "master-2-node.elastic.[...]:1025" # ] # sort will bubble up "master-0-node.elastic.[...]:1025", the HTTP server host:port dns.sort() port = dns[0].split(':')[-1] sdk_utils.out("Extracted {} as port for {}".format(port, dns[0])) return port
def add_stub_universe_urls(stub_universe_urls: list) -> dict: stub_urls = {} if not stub_universe_urls: return stub_urls # clean up any duplicate repositories _, current_universes, _ = sdk_cmd.run_cli("package repo list --json") for repo in json.loads(current_universes)["repositories"]: if repo["uri"] in stub_universe_urls: log.info("Removing duplicate stub URL: {}".format(repo["uri"])) assert remove_repo(repo["name"]) # add the needed universe repositories log.info("Adding stub URLs: {}".format(stub_universe_urls)) for url in stub_universe_urls: assert add_repo("testpkg-{}".format(sdk_utils.random_string()), url, 0) return stub_urls
def run_job(job_dict, timeout_seconds=600, raise_on_failure=True): job_name = job_dict['id'] # start job run, get run ID to be polled against: run_id = json.loads(sdk_cmd.run_cli( 'job run {} --json'.format(job_name)))['id'] # wait for run to succeed, throw if run fails: @retrying.retry(wait_fixed=1000, stop_max_delay=timeout_seconds * 1000, retry_on_result=lambda res: not res, retry_on_exception=lambda ex: False) def wait(): # catch errors from CLI: ensure that the only error raised is our own: try: successful_runs = json.loads( sdk_cmd.run_cli('job history --json {}'.format(job_name), print_output=False)) failed_runs = json.loads( sdk_cmd.run_cli( 'job history --failures --json {}'.format(job_name), print_output=False)) except: log.info(traceback.format_exc()) return False successful_ids = [r['id'] for r in successful_runs] failed_ids = [r['id'] for r in failed_runs] log.info( 'Job {} run history (waiting for successful {}): successful={} failed={}' .format(job_name, run_id, successful_ids, failed_ids)) # note: if a job has restart.policy=ON_FAILURE, it won't show up in failed_ids if it fails if raise_on_failure and run_id in failed_ids: raise Exception( 'Job {} with id {} has failed, exiting early'.format( job_name, run_id)) return run_id in successful_ids wait() return run_id
def hello_world_service(service_account): sdk_install.install( config.PACKAGE_NAME, 1, service_name=service_account, additional_options={ "service": { "spec_file": "examples/tls.yml", "service_account": service_account, "service_account_secret": service_account, # Legacy values "principal": service_account, "secret_name": service_account, }, "tls": { "discovery_task_prefix": DISCOVERY_TASK_PREFIX, }, } ) sdk_plan.wait_for_completed_deployment(config.PACKAGE_NAME) # Wait for service health check to pass shakedown.service_healthy(config.PACKAGE_NAME) # TODO(mh): Add proper wait for health check time.sleep(15) yield service_account sdk_install.uninstall(config.PACKAGE_NAME) # Make sure that all the TLS artifacts were removed from the secrets store. output = sdk_cmd.run_cli('security secrets list {name}'.format( name=config.PACKAGE_NAME)) artifact_suffixes = [ 'certificate', 'private-key', 'root-ca-certificate', 'keystore', 'truststore' ] for suffix in artifact_suffixes: assert suffix not in output
def test_upgrade_downgrade(): test_repo_name, test_repo_url = get_test_repo_info() test_version = get_pkg_version() print('Found test version: {}'.format(test_version)) repositories = json.loads( cmd.run_cli('package repo list --json'))['repositories'] print("Repositories: " + str(repositories)) universe_url = "fail" for repo in repositories: if repo['name'] == 'Universe': universe_url = repo['uri'] break assert "fail" != universe_url print("Universe URL: " + universe_url) shakedown.remove_package_repo('Universe') add_repo('Universe', universe_url, test_version, 0) universe_version = get_pkg_version() print('Found Universe version: {}'.format(universe_version)) print('Installing Universe version') install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT) shakedown.remove_package_repo('Universe') add_last_repo('Universe', universe_url, universe_version) print('Upgrading to test version') marathon.destroy_app(PACKAGE_NAME) install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT) shakedown.remove_package_repo('Universe') add_repo('Universe', universe_url, test_version, 0) print('Downgrading to master version') marathon.destroy_app(PACKAGE_NAME) install.install(PACKAGE_NAME, DEFAULT_TASK_COUNT) shakedown.remove_package_repo('Universe') add_last_repo('Universe', universe_url, universe_version)
def wait_for_toxic_sidecar(): """ Since the sidecar task fails too quickly, we check for the contents of the file generated in hello-container-path/toxic-output instead Note that we only check the output of hello-0. In DC/OS prior to version 1.10, task exec does not run the command in the MESOS_SANDBOX directory and this causes the check of the file contents to fail. Here we simply rely on the existence of the file. """ if sdk_utils.dcos_version_less_than("1.10"): cmd = "task ls hello-0-server hello-container-path/toxic-output" expected_output = "" else: cmd = "task exec hello-0-server cat hello-container-path/toxic-output" expected_output = "I'm addicted to you / Don't you know that you're toxic?" output = sdk_cmd.run_cli(cmd).strip() logging.info("Checking for toxic output returned: %s", output) return output == expected_output