def secure_sshd(ctx): rules = '''sshd : localhost : allow sshd : 192.168.0. : allow sshd : 10.25. : allow sshd : 174.99.121. : allow sshd : 152.14. : allow sshd : ALL : deny ''' file_path = '/tmp/hosts.allow' with open(file_path, 'w') as f: f.write(rules) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_command("scp {} {}:{}".format(file_path, host, file_path)) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_remote_command( host, "sudo cp {} /etc/hosts.allow".format(file_path)) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_remote_command(host, 'sudo service sshd reload', ignore_known_hosts=True)
def fix_firewall(ctx): with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: RemoteCommand(host, 'sudo systemctl stop firewalld', ignore_known_hosts=True).start() with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: RemoteCommand(host, 'sudo systemctl disable firewalld', ignore_known_hosts=True).start()
def cmd(ctx, command): # @todo: need to make sure the system is turned off. with parallel.CommandAgent(show_result=False, concurrency=1) as agent: agent.submit_remote_commands(ctx.obj['host_list'], command, check=False, silent=False)
def clear_system(ctx): # @todo: need to make sure the system is turned off. with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "sudo rm -rf /var/lib/HPCCSystems/*", check=False, silent=True)
def truncate_log(ctx): with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo truncate /var/log/HPCCSystems/roxie.log --size 0", check=False, silent=True)
def cmd(ctx, cmdline): with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_remote_command(host, cmdline, ignore_known_hosts=True, check=False)
def create_hosts(ctx, prefix, overwrite, os): ubuntu_preload_config = '''127.0.0.1 localhost # The following lines are desirable for IPv6 capable hosts ::1 ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters ff02::3 ip6-allhosts''' centos_preload_config = '''127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6''' tmp_file = '/tmp/.hosts' if overwrite: with open(tmp_file, 'w') as f: if os == 'cento': f.write(centos_preload_config) else: f.write(ubuntu_preload_config) f.write('\n\n') for i in range(len(ctx.obj['host_list'])): host = ctx.obj['host_list'][i] f.write("{} {}{}\n".format(host, prefix, i+1)) else: execute('cp /etc/hosts {}'.format(tmp_file)) execute('echo >> {}'.format(tmp_file)) for i in range(len(ctx.obj['host_list'])): host = ctx.obj['host_list'][i] host_mapping = "{} {}{}".format(host, prefix, i + 1) execute('echo {} >> {}'.format(host_mapping, tmp_file)) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_command("scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}".format(tmp_file, host, tmp_file), check=False) with parallel.CommandAgent(show_result=False) as agent: for i in range(len(ctx.obj['host_list'])): host = ctx.obj['host_list'][i] agent.submit_remote_command(host, "sudo hostname {}{}".format(prefix, i+1), silent=True, check=True) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_remote_command(host, "sudo cp {} /etc/hosts".format(tmp_file), silent=True, check=True)
def package(ctx, action, deb): if action == 'install': tmp_path = "/tmp/{}".format(os.path.basename(deb)) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_command( "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}" .format(deb, host, tmp_path), silent=True) # restrict the number of concurrency to avoid blocked by the APT system durning installation with parallel.CommandAgent(show_result=False, concurrency=4) as agent: print(ctx.obj['host_list']) # workaround if 'centos' in platform.linux_distribution()[0].lower(): agent.submit_remote_commands( ctx.obj['host_list'], "sudo yum remove -y hpccsystems-platform; sudo yum install -y {}" .format(tmp_path), silent=True) else: agent.submit_remote_commands( ctx.obj['host_list'], "sudo dpkg -i {}; sudo apt-get install -f -y".format( tmp_path), silent=True) ''' for host in ctx.obj['host_list']: click.echo('{}: install package {}'.format(host, tmp_path)) execute("scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:{}".format(package, host, tmp_path), silent=True) RemoteCommand(host, "dpkg -i {}; apt-get install -f -y".format(tmp_path), sudo=True, silent=True).start() ''' elif action == 'uninstall': with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "sudo dpkg -r hpccsystems-platform", silent=True) elif action == 'fix': tmp_path = "/tmp/{}".format(os.path.basename(deb)) with parallel.CommandAgent(show_result=False, concurrency=1) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo dpkg -r hpccsystems-platform; sudo dpkg -i {}; sudo apt-get install -f -y" .format(tmp_path), silent=True)
def deploy_key(ctx, user, from_user): '''This command deploy the current user's key to a remote user. The current implmentation might be insecure and only works for rsa key. This also assumes the home directory is located at /home. ''' with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_command( "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null /home/{}/.ssh/id_rsa* {}@{}:/tmp" .format(from_user, from_user, host), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo cp /tmp/id_rsa* /home/{}/.ssh; sudo rm -rf /tmp/id_rsa*". format(user), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo bash -c 'cat /home/{}/.ssh/id_rsa.pub >> /home/{}/.ssh/authorized_keys'" .format(user, user), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo chmod 600 /home/{}/.ssh/id_rsa*; sudo chmod 644 /home/{}/.ssh/authorized_keys" .format(user, user), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo chown {} /home/{}/.ssh/id_rsa*; sudo chown {} /home/{}/.ssh/authorized_keys" .format(user, user, user, user), check=True, silent=True)
def deploy_config(ctx, config): with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: RemoteCommand( host, "cp {}/environment.xml {}/environment.xml.bak".format( ctx.obj['config_dir'], ctx.obj['config_dir']), ignore_known_hosts=True, sudo=True).start() agent.submit_command( "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {} {}:/tmp/environment.xml" .format(config, host), silent=True) with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: click.echo('{}: deploy configuration'.format(host)) agent.submit_remote_command( host, "cp /tmp/environment.xml {}/environment.xml".format( ctx.obj['config_dir']), sudo=True, silent=True)
def deploy_key(ctx, username): '''This command deploy the current user's key to a remote user. The current implmentation might be insecure. ''' with parallel.CommandAgent(show_result=False) as agent: for host in ctx.obj['host_list']: agent.submit_command( "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ~/.ssh/id_rsa* {}@{}:/tmp" .format(username, host), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo cp /tmp/id_rsa* /home/{}/.ssh; sudo rm -rf /tmp/id_rsa*". format(username), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo cat /home/{}/.ssh/id_rsa.pub >> /home/{}/.ssh/authorized_keys" .format(username, username), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo chmod 644 /home/{}/.ssh/*".format(username), check=True, silent=True) with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "sudo chown {} /home/{}/.ssh/*".format( username, username), check=True, silent=True)
def create_cluster_topology(ctx): topology = defaultdict(lambda: []) with CaptureOutput() as output: with parallel.CommandAgent(concurrency=len(ctx.obj['host_list']), show_result=True) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "sudo service hpcc-init status", check=False, silent=True, capture=True) host = None for line in output: if '[' in line: host = line.split('] ')[-1] elif len(line) > 0: component = line.split(' ')[0].replace('my', '') running = 'running' in line topology[component].append((host, running)) return dict(topology)
def service(ctx, action, component): if action == 'list_components': with parallel.CommandAgent() as agent: cmd = "sudo service hpcc-init --componentlist" agent.submit_remote_commands(ctx.obj['host_list'], cmd, check=False, silent=False) elif action == 'list_types': with parallel.CommandAgent() as agent: cmd = "sudo service hpcc-init --typelist" agent.submit_remote_commands(ctx.obj['host_list'], cmd, check=False, silent=False) else: if len(component) > 0: filtered_components = [ n for n in component if n is not "dafilesrv" ] if len(filtered_components) > 0: cmd = "sudo service hpcc-init {} {}".format( " ".join(["-c %s" % n for n in component]), action) with parallel.CommandAgent() as agent: agent.submit_remote_commands(ctx.obj['host_list'], cmd, check=False, silent=False) if 'dafilesrv' in component: cmd = "sudo service dafilesrv {}".format(action) with parallel.CommandAgent() as agent: agent.submit_remote_commands(ctx.obj['host_list'], cmd, check=False, silent=False) else: # needs to start the master for avoiding failure when the cluster size is more than 8 #if action == 'start': # RemoteCommand(ctx.obj['topology']['esp'][0][0], "sudo service hpcc-init {}".format(action), silent=False, check=True).start() with parallel.CommandAgent( concurrency=len(ctx.obj['host_list'])) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo service dafilesrv {}".format(action), check=False, silent=True, capture=True) with parallel.CommandAgent( concurrency=len(ctx.obj['host_list'])) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo service hpcc-init {}".format(action), check=False, silent=True, capture=True) if action == 'stop': with parallel.CommandAgent( concurrency=len(ctx.obj['host_list'])) as agent: agent.submit_remote_commands( ctx.obj['host_list'], "sudo pkill -9 dafilesrv; sudo pkill -9 roxie", check=False, silent=True, capture=True)
def verify_config(ctx): with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "md5sum /etc/HPCCSystems/environment.xml", check=False, silent=False)
def clear_log(ctx): with parallel.CommandAgent(show_result=False) as agent: agent.submit_remote_commands(ctx.obj['host_list'], "sudo rm -rf /var/log/HPCCSystems/*", check=False, silent=True)