def secure_sshd(ctx):
rules = '''sshd : localhost : allow
sshd : 192.168.0. : allow
sshd : 10.25. : allow
sshd : 174.99.121. : allow
sshd : 152.14. : allow
sshd : ALL : deny
'''
file_path = '/tmp/hosts.allow'
with open(file_path, 'w') as f:
f.write(rules)
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_command("scp {} {}:{}".format(file_path, host,
file_path))
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_remote_command(
host, "sudo cp {} /etc/hosts.allow".format(file_path))
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_remote_command(host,
'sudo service sshd reload',
ignore_known_hosts=True)
def fix_firewall(ctx):
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(host,
'sudo systemctl stop firewalld',
ignore_known_hosts=True).start()
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(host,
'sudo systemctl disable firewalld',
ignore_known_hosts=True).start()
def cmd(ctx, command):
# @todo: need to make sure the system is turned off.
with parallel.CommandAgent(show_result=False, concurrency=1) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
command,
check=False,
silent=False)
def clear_system(ctx):
# @todo: need to make sure the system is turned off.
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"sudo rm -rf /var/lib/HPCCSystems/*",
check=False,
silent=True)
def truncate_log(ctx):
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo truncate /var/log/HPCCSystems/roxie.log --size 0",
check=False,
silent=True)
def cmd(ctx, cmdline):
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_remote_command(host,
cmdline,
ignore_known_hosts=True,
check=False)
def create_hosts(ctx, prefix, overwrite, os):
ubuntu_preload_config = '''127.0.0.1 localhost
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts'''
centos_preload_config = '''127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6'''
tmp_file = '/tmp/.hosts'
if overwrite:
with open(tmp_file, 'w') as f:
if os == 'cento':
f.write(centos_preload_config)
else:
f.write(ubuntu_preload_config)
f.write('\n\n')
for i in range(len(ctx.obj['host_list'])):
host = ctx.obj['host_list'][i]
f.write("{} {}{}\n".format(host, prefix, i+1))
else:
execute('cp /etc/hosts {}'.format(tmp_file))
execute('echo >> {}'.format(tmp_file))
for i in range(len(ctx.obj['host_list'])):
host = ctx.obj['host_list'][i]
host_mapping = "{} {}{}".format(host, prefix, i + 1)
execute('echo {} >> {}'.format(host_mapping, tmp_file))
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_command("scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}".format(tmp_file, host, tmp_file), check=False)
with parallel.CommandAgent(show_result=False) as agent:
for i in range(len(ctx.obj['host_list'])):
host = ctx.obj['host_list'][i]
agent.submit_remote_command(host, "sudo hostname {}{}".format(prefix, i+1), silent=True, check=True)
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_remote_command(host, "sudo cp {} /etc/hosts".format(tmp_file), silent=True, check=True)
def package(ctx, action, deb):
if action == 'install':
tmp_path = "/tmp/{}".format(os.path.basename(deb))
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_command(
"scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}"
.format(deb, host, tmp_path),
silent=True)
# restrict the number of concurrency to avoid blocked by the APT system durning installation
with parallel.CommandAgent(show_result=False, concurrency=4) as agent:
print(ctx.obj['host_list'])
# workaround
if 'centos' in platform.linux_distribution()[0].lower():
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo yum remove -y hpccsystems-platform; sudo yum install -y {}"
.format(tmp_path),
silent=True)
else:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo dpkg -i {}; sudo apt-get install -f -y".format(
tmp_path),
silent=True)
'''
for host in ctx.obj['host_list']:
click.echo('{}: install package {}'.format(host, tmp_path))
execute("scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}".format(package, host, tmp_path), silent=True)
RemoteCommand(host, "dpkg -i {}; apt-get install -f -y".format(tmp_path), sudo=True, silent=True).start()
'''
elif action == 'uninstall':
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"sudo dpkg -r hpccsystems-platform",
silent=True)
elif action == 'fix':
tmp_path = "/tmp/{}".format(os.path.basename(deb))
with parallel.CommandAgent(show_result=False, concurrency=1) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo dpkg -r hpccsystems-platform; sudo dpkg -i {}; sudo apt-get install -f -y"
.format(tmp_path),
silent=True)
def deploy_key(ctx, user, from_user):
'''This command deploy the current user's key to a remote user.
The current implmentation might be insecure and only works for rsa key. This also assumes the home directory is located at /home.
'''
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_command(
"scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null /home/{}/.ssh/id_rsa* {}@{}:/tmp"
.format(from_user, from_user, host),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo cp /tmp/id_rsa* /home/{}/.ssh; sudo rm -rf /tmp/id_rsa*".
format(user),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo bash -c 'cat /home/{}/.ssh/id_rsa.pub >> /home/{}/.ssh/authorized_keys'"
.format(user, user),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo chmod 600 /home/{}/.ssh/id_rsa*; sudo chmod 644 /home/{}/.ssh/authorized_keys"
.format(user, user),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo chown {} /home/{}/.ssh/id_rsa*; sudo chown {} /home/{}/.ssh/authorized_keys"
.format(user, user, user, user),
check=True,
silent=True)
def deploy_config(ctx, config):
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
RemoteCommand(
host,
"cp {}/environment.xml {}/environment.xml.bak".format(
ctx.obj['config_dir'], ctx.obj['config_dir']),
ignore_known_hosts=True,
sudo=True).start()
agent.submit_command(
"scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:/tmp/environment.xml"
.format(config, host),
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
click.echo('{}: deploy configuration'.format(host))
agent.submit_remote_command(
host,
"cp /tmp/environment.xml {}/environment.xml".format(
ctx.obj['config_dir']),
sudo=True,
silent=True)
def deploy_key(ctx, username):
'''This command deploy the current user's key to a remote user.
The current implmentation might be insecure.
'''
with parallel.CommandAgent(show_result=False) as agent:
for host in ctx.obj['host_list']:
agent.submit_command(
"scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ~/.ssh/id_rsa* {}@{}:/tmp"
.format(username, host),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo cp /tmp/id_rsa* /home/{}/.ssh; sudo rm -rf /tmp/id_rsa*".
format(username),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo cat /home/{}/.ssh/id_rsa.pub >> /home/{}/.ssh/authorized_keys"
.format(username, username),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo chmod 644 /home/{}/.ssh/*".format(username),
check=True,
silent=True)
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"sudo chown {} /home/{}/.ssh/*".format(
username, username),
check=True,
silent=True)
def create_cluster_topology(ctx):
topology = defaultdict(lambda: [])
with CaptureOutput() as output:
with parallel.CommandAgent(concurrency=len(ctx.obj['host_list']),
show_result=True) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"sudo service hpcc-init status",
check=False,
silent=True,
capture=True)
host = None
for line in output:
if '[' in line:
host = line.split('] ')[-1]
elif len(line) > 0:
component = line.split(' ')[0].replace('my', '')
running = 'running' in line
topology[component].append((host, running))
return dict(topology)
def service(ctx, action, component):
if action == 'list_components':
with parallel.CommandAgent() as agent:
cmd = "sudo service hpcc-init --componentlist"
agent.submit_remote_commands(ctx.obj['host_list'],
cmd,
check=False,
silent=False)
elif action == 'list_types':
with parallel.CommandAgent() as agent:
cmd = "sudo service hpcc-init --typelist"
agent.submit_remote_commands(ctx.obj['host_list'],
cmd,
check=False,
silent=False)
else:
if len(component) > 0:
filtered_components = [
n for n in component if n is not "dafilesrv"
]
if len(filtered_components) > 0:
cmd = "sudo service hpcc-init {} {}".format(
" ".join(["-c %s" % n for n in component]), action)
with parallel.CommandAgent() as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
cmd,
check=False,
silent=False)
if 'dafilesrv' in component:
cmd = "sudo service dafilesrv {}".format(action)
with parallel.CommandAgent() as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
cmd,
check=False,
silent=False)
else:
# needs to start the master for avoiding failure when the cluster size is more than 8
#if action == 'start':
# RemoteCommand(ctx.obj['topology']['esp'][0][0], "sudo service hpcc-init {}".format(action), silent=False, check=True).start()
with parallel.CommandAgent(
concurrency=len(ctx.obj['host_list'])) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo service dafilesrv {}".format(action),
check=False,
silent=True,
capture=True)
with parallel.CommandAgent(
concurrency=len(ctx.obj['host_list'])) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo service hpcc-init {}".format(action),
check=False,
silent=True,
capture=True)
if action == 'stop':
with parallel.CommandAgent(
concurrency=len(ctx.obj['host_list'])) as agent:
agent.submit_remote_commands(
ctx.obj['host_list'],
"sudo pkill -9 dafilesrv; sudo pkill -9 roxie",
check=False,
silent=True,
capture=True)
def verify_config(ctx):
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"md5sum /etc/HPCCSystems/environment.xml",
check=False,
silent=False)
def clear_log(ctx):
with parallel.CommandAgent(show_result=False) as agent:
agent.submit_remote_commands(ctx.obj['host_list'],
"sudo rm -rf /var/log/HPCCSystems/*",
check=False,
silent=True)
