def fix_ssh(ctx, user):
"""This adds extra users to /etc/ssh/sshd_config.
If the users are not specified, the current user and hpcc are
used alternatively.
"""
user_list = user if (user is not None) and (len(user) > 0) else [
getpass.getuser(), "hpcc"
]
for host in ctx.obj['host_list']:
sshd_config = "/etc/ssh/sshd_config"
RemoteCommand(host,
"sudo cp {} {}.bak".format(sshd_config, sshd_config),
ignore_known_hosts=True).start()
RemoteCommand(
host,
'sudo sed -i "s/^AllowUsers.*/AllowUsers root {}/" {}'.format(
" ".join(user_list), sshd_config),
ignore_known_hosts=True).start()
# workaround to support Ubuntu and
linux_distro = platform.linux_distribution()[0]
if 'centos' in linux_distro.lower():
RemoteCommand(host,
'sudo service sshd reload',
ignore_known_hosts=True).start()
else:
RemoteCommand(host,
'sudo service ssh reload',
ignore_known_hosts=True).start()
def main(argv=None):
global NPS_BASE, REMOTE_MATCH_CMD, REMOTE_MATCH_DIR
if argv is None:
argv = sys.argv[1:]
if (len(argv) < 2):
print("syntax: match <time control> <games>", file=sys.stderr)
exit(1)
try:
with open('machines.json', 'r') as machineFile:
data = machineFile.read()
except:
print("machine file not found or could not be opened", file=sys.stderr)
exit(1)
# machine list
machineList = []
try:
machineList = json.loads(data)
except:
print("failed to parse machine file", file=sys.stderr)
exit(1)
games = argv[0]
tc = argv[1]
# start the matches
for machine in machineList:
try:
host = machine['hostname']
nps = float(machine['nps'])
cores = int(machine['cores'])
except KeyError:
print(
"warning: expected hostname, nps and cores for machine, not found",
file=sys.stderr)
continue
# Parse and scale time control
factor = float(NPS_BASE) / nps
new_tc = scaleTC(tc, factor)
# execute the remote or local match script
try:
cmd = REMOTE_MATCH_CMD + ' ' + str(
games) + ' ' + new_tc + ' ' + str(cores)
print("starting : host=%s tc=%s" % (host, new_tc))
cmd = RemoteCommand(host,
cmd,
capture=True,
directory=REMOTE_MATCH_DIR)
cmd.start()
except:
print("error starting command " + cmd, file=sys.stderr)
traceback.print_tb(tb, limit=None, file=None)
return 2
def fix_firewall(ctx):
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(host,
'sudo systemctl stop firewalld',
ignore_known_hosts=True).start()
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(host,
'sudo systemctl disable firewalld',
ignore_known_hosts=True).start()
def ip(ctx):
"""This return the internal ip address of the hosts.
Here we assume the internal ip is bound to eth0.
"""
for host in ctx.obj['host_list']:
cmd = RemoteCommand(host,
"hostname -I | awk '{print $1}'",
ignore_known_hosts=True,
capture=True)
cmd.start()
click.echo(cmd.output)
def get_metrics(node):
cmd = RemoteCommand(
node.get_ip(),
"sudo /opt/HPCCSystems/bin/testsocket {} '<control:metrics/>'".format(
node.get_ip()),
capture=True,
silent=True)
cmd.start()
output_xml = etree.fromstring(cmd.output)
metrics = {}
for metric in output_xml.xpath("//Control/Endpoint/Metrics/Metric[@name]"):
metrics[metric.get('name')] = metric.get('value')
return metrics
def _run_service(ctx, stack_name, service):
if ctx.obj['test']:
master_ip = '127.0.0.1'
else:
# @TODO: a cache mechanism would be better
try:
master_ip = get_master_ip(stack_name)
except KeyError as e:
print(click.style(str(e), fg='red'))
ctx.abort()
# @TODO: after we finalize the AMI, we don't need to switch to the
# user's directory
cmd = RemoteCommand(
master_ip,
'sudo bash -c "/opt/HPCCSystems/sbin/hpcc-run.sh -a '
'dafilesrv {}"'.format(service),
identity_file=ctx.obj['identity'],
ssh_user=ctx.obj['username'],
)
cmd2 = RemoteCommand(
master_ip,
'sudo bash -c "/opt/HPCCSystems/sbin/hpcc-run.sh -a '
'hpcc-init {}"'.format(service),
identity_file=ctx.obj['identity'],
ssh_user=ctx.obj['username'],
)
if ctx.obj['test']:
print('not executing `{}`'.format(cmd.command_line))
print('not executing `{}`'.format(cmd2.command_line))
else:
cmd.start()
cmd2.start()
def submit_remote_command(self, host, cmd, *args, cid=None, **kwargs):
if 'strict_host_key_checking' not in kwargs:
kwargs['strict_host_key_checking'] = False
if 'ignore_known_hosts' not in kwargs:
kwargs['ignore_known_hosts'] = True
host_used = host if isinstance(
host, str) else host.get_ip() if isinstance(host, Node) else None
if host_used is None:
raise Exception("unknown host type: {}".format(type(host)))
if cid is None:
self.submit(hash(host_used + cmd),
RemoteCommand(host_used, cmd, *args, **kwargs))
else:
self.submit(cid, RemoteCommand(host_used, cmd, *args, **kwargs))
def send_r_cmd(host, settings, r_cmd):
rport = settings['port']
id_file = settings['id_file']
ruser = settings['user']
cmd = RemoteCommand(host,
r_cmd,
capture=True,
ssh_user=ruser,
port=rport,
batch_mode=False,
identity_file=id_file,
async=True,
silent=REMOTE_SILENT)
cmd.start()
return cmd
def _wait_until_complete(master_ip, identity, username):
while True:
cmd = RemoteCommand(master_ip,
"pgrep -f checkpoint.py",
identity_file=identity,
ssh_user=username,
capture=True,
check=False)
cmd.start()
pid_list = cmd.output
# print(pid_list, len(pid_list.splitlines()))
if len(pid_list.splitlines()) > 0:
print("Data service still processing")
time.sleep(5)
else:
break
def submit_remote_command(self, host, cmd, *args, **kwargs):
if 'strict_host_key_checking' not in kwargs:
kwargs['strict_host_key_checking'] = False
if 'ignore_known_hosts' not in kwargs:
kwargs['ignore_known_hosts'] = True
self.submit(hash(host + cmd), RemoteCommand(host, cmd, *args,
**kwargs))
def init(ctx, stack_name):
master_ip = get_master_ip(stack_name)
RemoteCommand(master_ip,
'sudo -u hpcc -i bash /opt/haas/auto_hpcc.sh',
identity_file=ctx.obj['identity'],
ssh_user=ctx.obj['username']).start()
def _wait_until_complete(master_ip, conf):
while True:
cmd = RemoteCommand(
master_ip,
"pgrep -f checkpoint.py",
identity_file=conf.get(HaasConfigurationKey.HAAS_SSH_KEY),
ssh_user=conf.get(HaasConfigurationKey.HAAS_SSH_USER),
capture=True,
check=False)
cmd.start()
pid_list = cmd.output
# print(pid_list, len(pid_list.splitlines()))
if len(pid_list.splitlines()) > 0:
print("Data service still processing")
time.sleep(5)
else:
break
def reset_metrics(node):
RemoteCommand(
node.get_ip(),
"sudo /opt/HPCCSystems/bin/testsocket {} '<control:resetMetrics/>'".
format(node.get_ip()),
capture=True,
silent=True).start()
RemoteCommand(
node.get_ip(),
"sudo /opt/HPCCSystems/bin/testsocket {} '<control:resetindexmetrics/>'"
.format(node.get_ip()),
capture=True,
silent=True).start()
RemoteCommand(
node.get_ip(),
"sudo /opt/HPCCSystems/bin/testsocket {} '<control:resetquerystats/>'".
format(node.get_ip()),
capture=True,
silent=True).start()
def progress(ctx, stack_name):
topology = ClusterTopology.parse(stack_name)
conf = HaasConfigurationManager().get(ctx.obj['config'])
cmd = RemoteCommand(
topology.get_master_ip(),
'source ~/haas/scripts/init.sh; python /home/osr/haas/scripts/checkpoint.py --name {} available; echo $?',
identity_file=conf.get(HaasConfigurationKey.HAAS_SSH_KEY),
ssh_user=conf.get(HaasConfigurationKey.HAAS_SSH_USER),
capture=True)
cmd.start()
if cmd.output == '0':
print("No service is running")
else:
print('Data service is running....')
RemoteCommand(topology.get_master_ip(),
"tail -f /tmp/haas_data.out",
identity_file=conf.get(
HaasConfigurationKey.HAAS_SSH_KEY),
ssh_user=conf.get(HaasConfigurationKey.HAAS_SSH_USER),
check=False).start()
def fix_sudo(ctx, user):
"""This change the permission in sudo vim /etc/sudoers
This makes sure the user has all the permissions, e.g. switch to other user.
"""
for host in ctx.obj['host_list']:
sudo_config = "/etc/sudoers"
RemoteCommand(
host,
'sudo sed -i "s/^{}.*/{} ALL=(ALL) NOPASSWD: ALL/" {}'.format(
user, user, sudo_config),
ignore_known_hosts=True).start()
def progress(ctx, stack_name):
'''Check progress of checkpointing operation'''
master_ip = get_master_ip(stack_name)
cmd = RemoteCommand(master_ip, 'python3 /opt/haas/checkpoint.py '
'--name {} available; echo $?',
identity_file=ctx.obj['identity'],
ssh_user=ctx.obj['username'],
capture=True)
if ctx.obj['test']:
print('not executing `{}`'.format(cmd.command))
else:
cmd.start()
if cmd.output == '0':
print("No service is running")
else:
print('Data service is running....')
RemoteCommand(master_ip,
"tail -f /tmp/haas_data.out",
identity_file=ctx.obj['identity'],
ssh_user=ctx.obj['username'],
check=False).start()
def get_workload_distribution(node):
grep_cmd = "grep '{}' /var/log/HPCCSystems/myroxie/roxie.log | wc -l"
index_cmd = RemoteCommand(node.get_ip(),
grep_cmd.format("23CRoxieIndexReadActivity"),
capture=True,
silent=True)
fetch_cmd = RemoteCommand(node.get_ip(),
grep_cmd.format("19CRoxieFetchActivity"),
capture=True,
silent=True)
index_cmd.start()
fetch_cmd.start()
return {
"IndexReadActivity": index_cmd.output,
"FetchActivity": fetch_cmd.output,
}
def deploy_config(ctx, config):
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(
host,
"cp {}/environment.xml {}/environment.xml.bak".format(
ctx.obj['config_dir'], ctx.obj['config_dir']),
ignore_known_hosts=True,
sudo=True).start()
agent.submit_command(
"scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:/tmp/environment.xml"
.format(config, host),
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
click.echo('{}: deploy configuration'.format(host))
agent.submit_remote_command(
host,
"cp /tmp/environment.xml {}/environment.xml".format(
ctx.obj['config_dir']),
sudo=True,
silent=True)
def check_remote(remote, data, timeout=60, verbose=False):
if verbose:
print("Checking {} ({}s)...".format(remote, timeout),
end=' ',
flush=True)
cmd = RemoteCommand('{}@{}'.format(data['User'], data['Hostname']),
'/bin/true',
connect_timeout=timeout,
port=data['Port'])
try:
cmd.start()
cmd.wait()
except RemoteConnectFailed:
print("Failed")
return False
print("Sucess")
return True
def prepare_command(self, command, options):
"""Refer to :attr:`AbstractContext.prepare_command`."""
return RemoteCommand(self.ssh_alias, *command,
**self.merge_options(options))
def prepare_interactive_shell(self, options):
"""Refer to :attr:`AbstractContext.prepare_interactive_shell`."""
options = self.merge_options(options)
options['tty'] = True
return RemoteCommand(self.ssh_alias, DEFAULT_SHELL, **options)