def run_postflight(config,
dcos_diag=None,
block=False,
state_json_dir=None,
async_delegate=None,
retry=False,
options=None):
targets = []
for host in config['master_list']:
s = Node(host, {'role': 'master'})
targets += [s]
for host in config['agent_list']:
s = Node(host, {'role': 'agent'})
targets += [s]
pf = get_async_runner(config, targets, async_delegate=async_delegate)
postflight_chain = ssh.utils.CommandChain('postflight')
add_pre_action(postflight_chain, pf.ssh_user)
if dcos_diag is None:
dcos_diag = """
#!/usr/bin/env bash
# Run the DC/OS diagnostic script for up to 15 minutes (900 seconds) to ensure
# we do not return ERROR on a cluster that hasn't fully achieved quorum.
T=900
until OUT=$(/opt/mesosphere/bin/./3dt -diag) || [[ T -eq 0 ]]; do
sleep 1
let T=T-1
done
RETCODE=$?
for value in $OUT; do
echo $value
done
exit $RETCODE"""
postflight_chain.add_execute(
[dcos_diag],
comment='Executing local post-flight check for DC/OS servces...')
add_post_action(postflight_chain)
# Setup the cleanup chain
cleanup_chain = ssh.utils.CommandChain('postflight_cleanup')
add_post_action(cleanup_chain)
master_agent_count = {
'total_masters': len(config['master_list']),
'total_agents': len(config['agent_list'])
}
result = yield from pf.run_commands_chain_async(
[postflight_chain, cleanup_chain],
block=block,
state_json_dir=state_json_dir,
delegate_extra_params=master_agent_count)
return result
def test_tags_async(sshd_manager, loop):
with sshd_manager.run(1) as sshd_ports:
workspace = str(sshd_manager.tmpdir)
host_ports = ['127.0.0.1:{}'.format(port) for port in sshd_ports]
targets = []
for _port in sshd_ports:
_host = Node('127.0.0.1:{}'.format(_port), {
'tag1': 'test1',
'tag2': 'test2'
})
targets.append(_host)
runner = MultiRunner(targets,
ssh_user=getpass.getuser(),
ssh_key_path=workspace + '/host_key')
chain = CommandChain('test')
chain.add_execute(['sleep', '1'])
try:
loop.run_until_complete(
runner.run_commands_chain_async([chain],
block=True,
state_json_dir=workspace))
finally:
loop.close()
with open(workspace + '/test.json') as fh:
result_json = json.load(fh)
for host_port in host_ports:
assert 'tags' in result_json['hosts'][host_port]
assert len(result_json['hosts'][host_port]['tags']) == 2
assert result_json['hosts'][host_port]['tags'][
'tag1'] == 'test1'
assert result_json['hosts'][host_port]['tags'][
'tag2'] == 'test2'
assert result_json['hosts'][host_port]['commands'][0][
'cmd'] == [
"/usr/bin/ssh", "-oConnectTimeout=10",
"-oStrictHostKeyChecking=no",
"-oUserKnownHostsFile=/dev/null", "-oBatchMode=yes",
"-oPasswordAuthentication=no",
"-p{}".format(sshd_ports[0]), "-i",
"{}/host_key".format(workspace), "-tt",
"{}@127.0.0.1".format(getpass.getuser()), "sleep", "1"
]
def add_nodes(nodes, tag):
return [Node(node, tag) for node in nodes]
def install_dcos(config,
block=False,
state_json_dir=None,
hosts=None,
async_delegate=None,
try_remove_stale_dcos=False,
**kwargs):
if hosts is None:
hosts = []
assert isinstance(hosts, list)
# Role specific parameters
role_params = {
'master': {
'tags': {
'role': 'master',
'dcos_install_param': 'master'
},
'hosts': config['master_list']
},
'agent': {
'tags': {
'role': 'agent',
'dcos_install_param': 'slave'
},
'hosts': config.hacky_default_get('agent_list', [])
},
'public_agent': {
'tags': {
'role': 'public_agent',
'dcos_install_param': 'slave_public'
},
'hosts': config.hacky_default_get('public_agent_list', [])
}
}
bootstrap_tarball = _get_bootstrap_tarball()
log.debug("Local bootstrap found: %s", bootstrap_tarball)
targets = []
if hosts:
targets = hosts
else:
for role, params in role_params.items():
targets += [Node(node, params['tags']) for node in params['hosts']]
runner = get_async_runner(config, targets, async_delegate=async_delegate)
chains = []
if try_remove_stale_dcos:
pkgpanda_uninstall_chain = ssh.utils.CommandChain('remove_stale_dcos')
pkgpanda_uninstall_chain.add_execute(
['sudo', '-i', '/opt/mesosphere/bin/pkgpanda', 'uninstall'],
stage='Trying pkgpanda uninstall')
chains.append(pkgpanda_uninstall_chain)
remove_dcos_chain = ssh.utils.CommandChain('remove_stale_dcos')
remove_dcos_chain.add_execute(
['rm', '-rf', '/opt/mesosphere', '/etc/mesosphere'],
stage="Removing DC/OS files")
chains.append(remove_dcos_chain)
chain = ssh.utils.CommandChain('deploy')
chains.append(chain)
add_pre_action(chain, runner.ssh_user)
_add_copy_dcos_install(chain)
_add_copy_packages(chain)
_add_copy_bootstap(chain, bootstrap_tarball)
chain.add_execute(lambda node: ('sudo bash {}/dcos_install.sh {}'.format(
REMOTE_TEMP_DIR, node.tags['dcos_install_param'])).split(),
stage=lambda node: 'Installing DC/OS')
# UI expects total_masters, total_agents to be top level keys in deploy.json
delegate_extra_params = nodes_count_by_type(config)
if kwargs.get('retry') and state_json_dir:
state_file_path = os.path.join(state_json_dir, 'deploy.json')
log.debug('retry executed for a state file deploy.json')
for _host in hosts:
_remove_host(state_file_path, '{}:{}'.format(_host.ip, _host.port))
# We also need to update total number of hosts
json_state = _read_state_file(state_file_path)
delegate_extra_params['total_hosts'] = json_state['total_hosts']
# Setup the cleanup chain
cleanup_chain = ssh.utils.CommandChain('deploy_cleanup')
add_post_action(cleanup_chain)
chains.append(cleanup_chain)
result = yield from runner.run_commands_chain_async(
chains,
block=block,
state_json_dir=state_json_dir,
delegate_extra_params=delegate_extra_params)
return result
def action_action_name(request):
"""Return /action/<action_name>
:param request: a web requeest object.
:type request: request | None
"""
action_name = request.match_info['action_name']
# Update the global action
json_state = read_json_state(action_name)
app['current_action'] = action_name
if request.method == 'GET':
log.info('GET {}'.format(action_name))
if json_state:
return web.json_response(json_state)
return web.json_response({})
elif request.method == 'POST':
log.info('POST {}'.format(action_name))
action = action_map.get(action_name)
# If the action name is preflight, attempt to run configuration
# generation. If genconf fails, present the UI with a usable error
# for the end-user
if action_name == 'preflight':
try:
print_header("GENERATING CONFIGURATION")
backend.do_configure()
except:
genconf_failure = {
"errors": "Configuration generation failed, please see command line for details"
}
return web.json_response(genconf_failure, status=400)
params = yield from request.post()
if json_state:
if action_name == 'deploy' and 'retry' in params:
if 'hosts' in json_state:
failed_hosts = []
for deploy_host, deploy_params in json_state['hosts'].items():
if deploy_params['host_status'] != 'success':
failed_hosts.append(Node(deploy_host, tags=deploy_params['tags']))
log.debug('failed hosts: {}'.format(failed_hosts))
if failed_hosts:
yield from asyncio.async(
action(
backend.get_config(),
state_json_dir=STATE_DIR,
hosts=failed_hosts,
try_remove_stale_dcos=True,
**params))
return web.json_response({
'status': 'retried',
'details': sorted(['{}:{}'.format(node.ip, node.port) for node in failed_hosts])
})
if action_name not in remove_on_done:
return web.json_response({'status': '{} was already executed, skipping'.format(action_name)})
running = False
for host, attributes in json_state['hosts'].items():
if attributes['host_status'].lower() == 'running':
running = True
log.debug('is action running: {}'.format(running))
if running:
return web.json_response({'status': '{} is running, skipping'.format(action_name)})
else:
unlink_state_file(action_name)
yield from asyncio.async(action(backend.get_config(), state_json_dir=STATE_DIR, options=options, **params))
return web.json_response({'status': '{} started'.format(action_name)})
def run_preflight(config,
pf_script_path='/genconf/serve/dcos_install.sh',
block=False,
state_json_dir=None,
async_delegate=None,
retry=False,
options=None):
'''
Copies preflight.sh to target hosts and executes the script. Gathers
stdout, sterr and return codes and logs them to disk via SSH library.
:param config: Dict, loaded config file from /genconf/config.yaml
:param pf_script_path: preflight.sh script location on a local host
:param preflight_remote_path: destination location
'''
if not os.path.isfile(pf_script_path):
log.error(
"genconf/serve/dcos_install.sh does not exist. Please run --genconf before executing preflight."
)
raise FileNotFoundError('genconf/serve/dcos_install.sh does not exist')
targets = []
for host in config['master_list']:
s = Node(host, {'role': 'master'})
targets += [s]
for host in config['agent_list']:
s = Node(host, {'role': 'agent'})
targets += [s]
pf = get_async_runner(config, targets, async_delegate=async_delegate)
chains = []
preflight_chain = ssh.utils.CommandChain('preflight')
# In web mode run if no --offline flag used.
if options.web:
if options.offline:
log.debug(
'Offline mode used. Do not install prerequisites on CentOS7, RHEL7 in web mode'
)
else:
_add_prereqs_script(preflight_chain)
add_pre_action(preflight_chain, pf.ssh_user)
preflight_chain.add_copy(pf_script_path,
REMOTE_TEMP_DIR,
comment='COPYING PREFLIGHT SCRIPT TO TARGETS')
preflight_chain.add_execute('sudo bash {} --preflight-only master'.format(
os.path.join(REMOTE_TEMP_DIR,
os.path.basename(pf_script_path))).split(),
comment='EXECUTING PREFLIGHT CHECK ON TARGETS')
chains.append(preflight_chain)
# Setup the cleanup chain
cleanup_chain = ssh.utils.CommandChain('preflight_cleanup')
add_post_action(cleanup_chain)
chains.append(cleanup_chain)
master_agent_count = {
'total_masters': len(config['master_list']),
'total_agents': len(config['agent_list'])
}
result = yield from pf.run_commands_chain_async(
chains,
block=block,
state_json_dir=state_json_dir,
delegate_extra_params=master_agent_count)
return result
def install_dcos(config,
block=False,
state_json_dir=None,
hosts=[],
async_delegate=None,
try_remove_stale_dcos=False,
roles=None,
**kwargs):
if roles is None:
roles = ['master', 'agent']
# Role specific parameters
role_params = {
'master': {
'tags': {
'role': 'master',
'dcos_install_param': 'master'
},
'hosts': hosts if hosts else config['master_list']
},
'agent': {
'tags': {
'role': 'agent',
'dcos_install_param': 'slave'
},
'hosts': hosts if hosts else config['agent_list']
}
}
bootstrap_tarball = _get_bootstrap_tarball()
log.debug("Local bootstrap found: %s", bootstrap_tarball)
targets = []
for role in roles:
default_params = role_params[role]
for host in default_params['hosts']:
node = Node(host, default_params['tags'])
targets += [node]
log.debug('Got {} hosts'.format(targets))
runner = get_async_runner(config, targets, async_delegate=async_delegate)
chains = []
if try_remove_stale_dcos:
pkgpanda_uninstall_chain = ssh.utils.CommandChain('remove_stale_dcos')
pkgpanda_uninstall_chain.add_execute(
['sudo', '-i', '/opt/mesosphere/bin/pkgpanda', 'uninstall'],
comment='TRYING pkgpanda uninstall')
chains.append(pkgpanda_uninstall_chain)
remove_dcos_chain = ssh.utils.CommandChain('remove_stale_dcos')
remove_dcos_chain.add_execute(
['rm', '-rf', '/opt/mesosphere', '/etc/mesosphere'])
chains.append(remove_dcos_chain)
chain = ssh.utils.CommandChain('deploy')
chains.append(chain)
add_pre_action(chain, runner.ssh_user)
_add_copy_dcos_install(chain)
_add_copy_packages(chain)
_add_copy_bootstap(chain, bootstrap_tarball)
chain.add_execute(
lambda node: ('sudo bash {}/dcos_install.sh {}'.format(
REMOTE_TEMP_DIR, node.tags['dcos_install_param'])).split(),
comment=lambda node: 'INSTALLING DC/OS ON NODE {}, ROLE {}'.format(
node.ip, node.tags['role']))
# UI expects total_masters, total_agents to be top level keys in deploy.json
delegate_extra_params = {
'total_masters': len(config['master_list']),
'total_agents': len(config['agent_list'])
}
if kwargs.get('retry') and state_json_dir:
state_file_path = os.path.join(state_json_dir, 'deploy.json')
log.debug('retry executed for a state file deploy.json')
for _host in hosts:
_remove_host(state_file_path, _host)
# We also need to update total number of hosts
json_state = _read_state_file(state_file_path)
delegate_extra_params['total_hosts'] = json_state['total_hosts']
# Setup the cleanup chain
cleanup_chain = ssh.utils.CommandChain('deploy_cleanup')
add_post_action(cleanup_chain)
chains.append(cleanup_chain)
result = yield from runner.run_commands_chain_async(
chains,
block=block,
state_json_dir=state_json_dir,
delegate_extra_params=delegate_extra_params)
return result