def setup_cli(self, stub_universes={}): try: self._github_updater.update('pending', 'Setting up CLI') self._configure_cli_sandbox() # adds to os.environ cli_filepath = self._download_cli_to_sandbox() self._configure_cli(self._dcos_url) dcos_login.DCOSLogin(self._dcos_url).login() # check for any preexisting universes and remove them -- the cluster requires no duplicate uris if stub_universes: logger.info('Checking for duplicate stub universes') cur_universes = subprocess.check_output( 'dcos package repo list --json'.split()).decode('utf-8') for repo in json.loads(cur_universes)['repositories']: # {u'name': u'Universe', u'uri': u'https://universe.mesosphere.com/repo'} if repo['uri'] in stub_universes.values(): logger.info( 'Removing duplicate repository: {} {}'.format( repo['name'], repo['uri'])) subprocess.check_call( 'dcos package repo remove {}'.format( repo['name']).split()) for name, url in stub_universes.items(): logger.info('Adding repository: {} {}'.format(name, url)) subprocess.check_call( 'dcos package repo add --index=0 {} {}'.format( name, url).split()) except: self._github_updater.update('error', 'CLI Setup failed') raise
def _initialize_dcos_cli(self): logger.info("Initializing dcos config") subprocess.check_call(['which', 'dcos']) subprocess.check_call(['dcos', 'config', 'set', 'core.dcos_url', self.dcos_url]) subprocess.check_call(['dcos', 'config', 'set', 'core.reporting', 'True']) subprocess.check_call(['dcos', 'config', 'set', 'core.ssl_verify', 'False']) subprocess.check_call(['dcos', 'config', 'set', 'core.timeout', '5']) subprocess.check_call(['dcos', 'config', 'show']) dcos_login.DCOSLogin(self.dcos_url).login()
def _start(self, config): is_17_cluster = config.ccm_channel in self._DCOS_17_CHANNELS if is_17_cluster: hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos' elif config.cf_template.startswith('ee.'): hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos-enterprise' else: hostrepo = 's3-us-west-2.amazonaws.com/downloads.dcos.io/dcos' template_url = 'https://{}/{}/cloudformation/{}'.format( hostrepo, config.ccm_channel, config.cf_template) if config.template_url: template_url = config.template_url cluster_name = config.name_prefix + self._rand_str(8) payload = { 'template_url': template_url, 'name': cluster_name, 'cluster_desc': config.description, 'time': config.duration_mins, 'private_agents': str(config.private_agents), 'public_agents': str(config.public_agents), 'pre_1_8_cluster': is_17_cluster, 'adminlocation': config.admin_location, 'cloud_provider': config.cloud_provider, 'region': config.aws_region } logger.info('''Launching cluster: name={} agents={} private/{} public duration={} minutes mountvols={} permissions={} channel={} template={}'''.format(cluster_name, config.private_agents, config.public_agents, config.duration_mins, config.mount_volumes, config.permissions, config.ccm_channel, config.cf_template)) response = self._query_http('POST', self._CCM_PATH, request_json_payload=payload) if not response: raise Exception('CCM cluster creation request failed') response_content = response.read().decode('utf-8') response_json = json.loads(response_content) logger.info('Launch response:\n{}'.format( pprint.pformat(response_json))) cluster_id = int(response_json.get('id', 0)) if not cluster_id: raise Exception( 'No Cluster ID returned in cluster creation response: {}'. format(response_content)) stack_id = response_json.get('stack_id', '') if not stack_id: raise Exception( 'No Stack ID returned in cluster creation response: {}'.format( response_content)) cluster_info = self.wait_for_status(cluster_id, 'CREATING', 'RUNNING', config.start_timeout_mins) if not cluster_info: raise Exception('CCM cluster creation failed or timed out') dns_address = cluster_info.get('DnsAddress', '') if not dns_address: raise Exception( 'CCM cluster_info is missing DnsAddress: {}'.format( cluster_info)) logger.info('Cluster is now RUNNING: {}'.format(cluster_info)) if config.mount_volumes: logger.info( 'Enabling mount volumes for cluster {} (stack id {})'.format( cluster_id, stack_id)) # fabric spams to stdout, which causes problems with launch_ccm_cluster. # force total redirect to stderr: stdout = sys.stdout sys.stdout = sys.stderr import enable_mount_volumes enable_mount_volumes.main(stack_id) sys.stdout = stdout # we fetch the token once up-front because on Open clusters it must be reused. # given that, we may as well use the same flow across both Open and EE. logger.info('Fetching auth token') dcos_url = 'https://' + dns_address auth_token = dcos_login.DCOSLogin(dcos_url).get_acs_token() if config.permissions: logger.info( 'Setting up permissions for cluster {} (stack id {})'.format( cluster_id, stack_id)) def run_script(scriptname, args=[]): logger.info('Command: {} {}'.format(scriptname, ' '.join(args))) script_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), scriptname) # redirect stdout to stderr: subprocess.check_call(['bash', script_path] + args, stdout=sys.stderr) run_script('create_service_account.sh', [dcos_url, auth_token, '--strict']) # Examples of what individual tests should run. See respective projects' "test.sh": #run_script('setup_permissions.sh', 'nobody cassandra-role'.split()) #run_script('setup_permissions.sh', 'nobody hdfs-role'.split()) #run_script('setup_permissions.sh', 'nobody kafka-role'.split()) #run_script('setup_permissions.sh', 'nobody spark-role'.split()) return {'id': cluster_id, 'url': dcos_url, 'auth_token': auth_token}
def _start(self, config): is_17_cluster = config.ccm_channel in self._DCOS_17_CHANNELS if is_17_cluster: hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos' elif config.cf_template.startswith('ee.'): hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos-enterprise' else: hostrepo = 's3-us-west-2.amazonaws.com/downloads.dcos.io/dcos' template_url = 'https://{}/{}/cloudformation/{}'.format( hostrepo, config.ccm_channel, config.cf_template) if config.template_url: template_url = config.template_url cluster_name = config.name_prefix + self._rand_str(8) payload = { 'template_url': template_url, 'name': cluster_name, 'cluster_desc': config.description, 'time': config.duration_mins, 'private_agents': str(config.private_agents), 'public_agents': str(config.public_agents), 'pre_1_8_cluster': is_17_cluster, 'adminlocation': config.admin_location, 'cloud_provider': config.cloud_provider, 'region': config.aws_region } logger.info('''Launching cluster: name={} agents={} private/{} public duration={} minutes mountvols={} channel={} template={}'''.format(cluster_name, config.private_agents, config.public_agents, config.duration_mins, config.mount_volumes, config.ccm_channel, config.cf_template)) response = self._query_http('POST', self._CCM_PATH, request_json_payload=payload) if not response: raise Exception('CCM cluster creation request failed') response_content = response.read().decode('utf-8') response_json = json.loads(response_content) logger.info('Launch response:\n{}'.format( pprint.pformat(response_json))) cluster_id = int(response_json.get('id', 0)) if not cluster_id: raise Exception( 'No Cluster ID returned in cluster creation response: {}'. format(response_content)) stack_id = response_json.get('stack_id', '') if not stack_id: raise Exception( 'No Stack ID returned in cluster creation response: {}'.format( response_content)) cluster_info = self.wait_for_status(cluster_id, 'CREATING', 'RUNNING', config.start_timeout_mins) if not cluster_info: raise Exception('CCM cluster creation failed or timed out') dns_address = cluster_info.get('DnsAddress', '') if not dns_address: raise Exception( 'CCM cluster_info is missing DnsAddress: {}'.format( cluster_info)) if config.mount_volumes: logger.info( 'Enabling mount volumes for cluster {} (stack id {})'.format( cluster_id, stack_id)) # fabric spams to stdout, which causes problems with launch_ccm_cluster. # force total redirect to stderr: stdout = sys.stdout sys.stdout = sys.stderr import enable_mount_volumes enable_mount_volumes.main(stack_id) sys.stdout = stdout # we fetch the token once up-front because on Open clusters it must be reused. # given that, we may as well use the same flow across both Open and EE. logger.info('Fetching auth token') dcos_url = 'https://' + dns_address auth_token = dcos_login.DCOSLogin(dcos_url).get_acs_token() return {'id': cluster_id, 'url': dcos_url, 'auth_token': auth_token}
def _start(self, config): is_17_cluster = config.ccm_channel in self._DCOS_17_CHANNELS scale_url = None if is_17_cluster: hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos' elif config.cf_scale.startswith('ee.'): hostrepo = 's3.amazonaws.com/downloads.mesosphere.io/dcos-enterprise-aws-advanced' # format is different for enterprise security modes. mode = config.security_mode if not mode: logger.warning("No scales known for enterprise & " "default security (none). Cowardly bringing " "up a permissive cluster") mode = 'permissive' scale_url = 'https://{}/{}/{}/cloudformation/{}'.format( hostrepo, config.ccm_channel, mode, config.cf_scale) else: hostrepo = 's3-us-west-2.amazonaws.com/downloads.dcos.io/dcos' # non-ee mode if not scale_url: scale_url = 'https://{}/{}/cloudformation/{}'.format( hostrepo, config.ccm_channel, config.cf_scale) # external override from DCOS_SCALE_URL if config.scale_url: scale_url = config.scale_url logger.info("Accepting externally provided scale_url from environment.") cluster_name = config.name_prefix + self._rand_str(8) payload = { 'scale_url': scale_url, 'name': cluster_name, 'cluster_desc': config.description, 'time': config.duration_mins, 'private_agents': str(config.private_agents), 'public_agents': str(config.public_agents), 'pre_1_8_cluster': is_17_cluster, 'adminlocation': config.admin_location, 'cloud_provider': config.cloud_provider, 'region': config.aws_region } logger.info('''Launching cluster: name={} agents={} private/{} public duration={} minutes mountvols={} permissions={} channel={} scale={} scale_url={}'''.format( cluster_name, config.private_agents, config.public_agents, config.duration_mins, config.mount_volumes, config.security_mode, config.ccm_channel, config.cf_scale, scale_url)) response = self._query_http('POST', self._CCM_PATH, request_json_payload=payload) if not response: raise ClusterActionException('CCM cluster creation request failed') response_content = response.read().decode('utf-8') response_json = json.loads(response_content) logger.info('Launch response:\n{}'.format(pprint.pformat(response_json))) cluster_id = int(response_json.get('id', 0)) if not cluster_id: raise ClusterActionException('No Cluster ID returned in cluster creation response: {}'.format(response_content)) stack_id = response_json.get('stack_id', '') if not stack_id: raise ClusterActionException('No Stack ID returned in cluster creation response: {}'.format(response_content)) cluster_info = self.wait_for_status( cluster_id, ['CREATING', 'RUNNING_NEEDS_INFO'], # pending states 'RUNNING', # desired state config.start_timeout_mins) if not cluster_info: raise ClusterActionException('CCM cluster creation failed or timed out') dns_address = cluster_info.get('DnsAddress', '') if not dns_address: raise ClusterActionException('CCM cluster_info is missing DnsAddress: {}'.format(cluster_info)) logger.info('Cluster is now RUNNING: {}'.format(cluster_info)) # we fetch the token once up-front because on Open clusters it must be reused. # given that, we may as well use the same flow across both Open and EE. logger.info('Fetching auth token') dcos_url = 'https://' + dns_address auth_token = dcos_login.DCOSLogin(dcos_url).get_acs_token() if config.postlaunch_steps != 'none': is_enterprise = config.cf_scale.startswith('ee.') clustinit = configure_test_cluster.ClusterInitializer( stack_id, auth_token, dns_address, is_enterprise, config.security_mode) initmaster = True if config.postlaunch_steps == 'nomaster': initmaster = False clustinit.apply_default_config(initmaster=initmaster) if config.mount_volumes: clustinit.create_mount_volumes() return { 'id': cluster_id, 'url': dcos_url, 'auth_token': auth_token }