def sshMaster(command): """ Performs a command on master node """ masterIp = ipGetMaster() utils.ssh(masterIp, command)
def install_logrotate_script(himn, username): "Install console logrotate script" utils.scp(himn, username, '/root/', 'rotate_xen_guest_logs.sh') utils.ssh(himn, username, 'mkdir -p /var/log/xen/guest') utils.ssh(himn, username, '''crontab - << CRONTAB * * * * * /root/rotate_xen_guest_logs.sh >/dev/null 2>&1 CRONTAB''')
def install_suppack(himn, username, package): """Install xapi driver supplemental pack. """ tmp = utils.ssh(himn, username, 'mktemp', '-d') utils.scp(himn, username, tmp, package) utils.ssh(himn, username, 'xe-install-supplemental-pack', tmp + '/' + package, prompt='Y\n') utils.ssh(himn, username, 'rm', tmp, '-rf')
def install_suppack(himn, username, package, xcp_version): """Install xapi driver supplemental pack. """ tmp = utils.ssh(himn, username, 'mktemp', '-d') real_pack = "xcp_%s/%s" % (xcp_version, package) if not os.path.exists(real_pack): utils.reportError('Package folder %s not exist' % real_pack) utils.scp(himn, username, tmp, real_pack) if LooseVersion(xcp_version) < LooseVersion('2.2.0'): utils.ssh(himn, username, 'xe-install-supplemental-pack', tmp + '/' + package, prompt='Y\n') else: errcode, uuid, errmsg = \ utils.ssh_detailed(himn, username, 'xe', 'update-upload', 'file-name=' + tmp + '/' + package, allowed_return_codes=[0, 1]) if errcode == 0: utils.ssh(himn, username, 'xe', 'update-apply', 'uuid=' + uuid.strip()) else: LOG.debug("Install supplemental pack failed, err: %s", errmsg) if "The uploaded update already exists" in errmsg: uuid = parse_uuid(errmsg) if uuid is None: raise utils.ExecutionError(errmsg) # Check current update is applied already out = utils.ssh(himn, username, 'xe', 'update-list', 'uuid=' + uuid, '--minimal') # Apply this update if cannot find it with uuid if not out: utils.ssh(himn, username, 'xe', 'update-apply', 'uuid=' + uuid) utils.ssh(himn, username, 'rm', tmp, '-rf')
def check_host_compatibility(himn, username): xcp_version = utils.get_xcp_version(himn, username) if LooseVersion(xcp_version) < LooseVersion(MIN_XCP_VERSION): utils.reportError('Platform version %s should equal or greater than %s' % (xcp_version, MIN_XCP_VERSION)) return version_hotfixes = json.loads(VERSION_HOTFIXES) ver = utils.ssh(himn, username, ('xe host-param-get uuid=$(xe host-list --minimal) ' 'param-name=software-version param-key=product_version')) hotfixes = version_hotfixes.get(ver) if not hotfixes: return for hotfix in hotfixes: if not hotfix: continue installed = utils.ssh(himn, username, 'xe patch-list name-label=%s --minimal' % hotfix) if not installed: utils.reportError('Hotfix %s has not been installed ' % ver)
def configure_dom0_iptables(himn, username): xs_chain = 'XenServer-Neutron-INPUT' # Check XenServer specific chain, create if not exist commands = ('iptables -t filter -L %s' % xs_chain, 'iptables -t filter --new %s' % xs_chain, 'iptables -t filter -I INPUT -j %s' % xs_chain) execute_iptables_commands(himn, username, commands) # Check XenServer rule for ovs native mode, create if not exist commands = ( 'iptables -t filter -C %s -p tcp -m tcp --dport 6640 -j ACCEPT' % xs_chain, 'iptables -t filter -I %s -p tcp --dport 6640 -j ACCEPT' % xs_chain) execute_iptables_commands(himn, username, commands) # Check XenServer rule for vxlan, create if not exist commands = ('iptables -t filter -C %s -p udp -m multiport --dports 4789 ' '-j ACCEPT' % xs_chain, 'iptables -t filter -I %s -p udp -m multiport --dport 4789 -j ' 'ACCEPT' % xs_chain) execute_iptables_commands(himn, username, commands) # Persist iptables rules utils.ssh(himn, username, 'service', 'iptables', 'save')
def runMain(): if utils.inputMatch([]): utils.print_header('deep') utils.print_option("full", "cluster->restore->download deep eval") utils.print_option("restore-eval", "restore deep models evaluations") utils.print_option("upload-eval", "tar and upload deep models evaluations") utils.print_option("restore-model", "restore deep models (download from bucket and untar)") utils.print_option("eval-tomaster", "copy evals from slaves to master") utils.print_option("eval-frommaster", "copy evals from master to slaves") utils.print_option("eval-permanent", "copy evals from slaves to master") utils.print_option("eval-sync", "eval-tomaster->eval-frommaster") utils.print_header('') if utils.inputMatch(['full']): full() if utils.inputMatch(['restore-eval']): restoreDeepEval() if utils.inputMatch(['restore-model']): restoreDeepModels() if utils.inputMatch(['eval-tomaster']): syncEvalToMaster() if utils.inputMatch(['eval-frommaster']): syncEvalFromMaster() if utils.inputMatch(['upload-eval']): uploadDeepEval() if utils.inputMatch(['eval-permanent']): permanentEvalToMaster() if utils.inputMatch(['upload-subdirs'], doPartial=True): host = sys.argv[2] source = sys.argv[3] prefix = sys.argv[4] suffix = sys.argv[5] utils.ssh(cluster.ipGetAny(host),'cd {0};ls -1 -d */ > listdir.txt'.format(source)) utils.exec_command('gcloud compute copy-files '+host+':'+os.path.join(source, 'listdir.txt')+' .') with open ("listdir.txt", "r") as myfile: data=myfile.readlines() utils.exec_command('rm listdir.txt') data = [os.path.dirname(d.rstrip()) for d in data] command = getUploadCommand(source, data, prefix, suffix) print command utils.ssh(cluster.ipGetAny(host), command)
def install_logrotate_script(himn, username): "Install console logrotate script" utils.scp(himn, username, '/root/', 'rotate_xen_guest_logs.sh') utils.ssh(himn, username, 'mkdir -p /var/log/xen/guest') utils.ssh( himn, username, '''crontab - << CRONTAB * * * * * /root/rotate_xen_guest_logs.sh >/dev/null 2>&1 CRONTAB''')
def ssh(self, cmd, host=None): """ this method requires that an ssh daemon is running on the target and that an authorized_keys file is set with a rsa plubilc key, all conditions met by images provided in this project. """ if host: return utils.ssh(self.user, get_container_ip(self.containers[host]), cmd) return {k: utils.ssh(self.user, get_container_ip(v), cmd) for k, v in self.containers.iteritems()}
def run_commands_in_instance(step, vmname, user, key): ip = utils.nova_cli.get_instance_ip(vmname) assert_true(ip != '', vmname) key_path = os.path.join(bunch_working_dir,key) for data in step.hashes: if 'noFail' == data['Expected'].strip(): step_assert(step).assert_true(utils.ssh(ip, command=data['Command'], user=user, key=key_path).successful()) else: step_assert(step).assert_equals(utils.ssh(ip, command=data['Command'], user=user, key=key_path).output_text().strip(),data['Expected'].strip())
def stopDfs(): (listnodes, nodes) = cluster.instanceListAll() mastername = listnodes['master'][0] master = nodes[mastername] masterIp = master['externalIp'] utils.ssh(masterIp, "stop-dfs.sh")
def formatFileSystem(): (listnodes, nodes) = cluster.instanceListAll() mastername = listnodes['master'][0] master = nodes[mastername] masterIp = master['externalIp'] utils.ssh(masterIp, "hdfs namenode -format")
def getStatus(): (listnodes, nodes) = cluster.instanceListAll() mastername = listnodes['master'][0] master = nodes[mastername] masterIp = master['externalIp'] utils.ssh(masterIp, "hdfs dfsadmin -report")
def deployScript(self): # SCP local file to remote host scpTo(self.host, self.local_script_path, self.remote_script_path) # chmod +x remote_script chmod_cmd = "chmod +x %s" % self.remote_script_path ssh(self.host, chmod_cmd) # clean up os.system("rm " + self.local_script_path)
def execute_iptables_commands(himn, username, command_list): # Execute first command and continue based on first command result exitcode, _, _ = utils.ssh_detailed(himn, username, command_list[0], allowed_return_codes=[0, 1]) if exitcode == 1: for command in command_list[1:]: LOG.info('Execute iptables command %s', command) utils.ssh(himn, username, command)
def runOnMaster(command): """ Run a command on all nodes """ (listnodes, nodes) = instanceListAll() mastername = listnodes['master'][0] master = nodes[mastername] externalIp = master['externalIp'] utils.print_header('ON [' + mastername + '] running: ' + command) utils.ssh(externalIp, command)
def run_commands_in_instance(step, vmname, user, key): ip = utils.nova_cli.get_instance_ip(vmname) assert_true(ip != '', vmname) key_path = os.path.join(bunch_working_dir, key) for data in step.hashes: if 'noFail' == data['Expected'].strip(): step_assert(step).assert_true( utils.ssh(ip, command=data['Command'], user=user, key=key_path).successful()) else: step_assert(step).assert_equals( utils.ssh(ip, command=data['Command'], user=user, key=key_path).output_text().strip(), data['Expected'].strip())
def enable_conntrack_service(himn, username): xcp_ver = utils.ssh(himn, username, ('xe host-param-get uuid=$(xe host-list --minimal) ' 'param-name=software-version ' 'param-key=platform_version')) if LooseVersion(xcp_ver) < LooseVersion('2.1.0'): # Only support conntrack-tools since XS7.0(XCP2.1.0) and above LOG.info('No need to enable conntrack-tools with XCP %s' % xcp_ver) return conn_installed = utils.ssh(himn, username, 'find', '/usr/sbin', '-name', 'conntrackd') if not conn_installed: install_suppack(himn, username, CONNTRACK_ISO) # use conntrack statistic mode, so change conntrackd.conf utils.ssh(himn, username, 'mv', '/etc/conntrackd/conntrackd.conf', '/etc/conntrackd/conntrackd.conf.back') utils.ssh(himn, username, 'cp', CONNTRACK_CONF_SAMPLE, '/etc/conntrackd/conntrackd.conf') # Restart conntrackd service utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
def runOnAllNodes(command): """ Run a command on all nodes """ listnodes, nodes = instanceListAll() allnodes = listnodes['all'] for nodename in allnodes: utils.print_header('ON [' + nodename + '] running: ' + command) node = nodes[nodename] externalIp = node['externalIp'] utils.ssh(externalIp, command)
def find_dom0_bridge(himn, username, bridge_name): ethX = get_network_ethX(bridge_name) if not ethX: utils.reportError("Cannot find eth used for private network") ethX = ethX.split('.')[0] # find the ethX mac in /sys/class/net/ethX/address with open('/sys/class/net/%s/address' % ethX, 'r') as fo: mac = fo.readline() network_uuid = utils.ssh(himn, username, ('xe vif-list params=network-uuid ' 'minimal=true MAC=%s') % mac) bridge = utils.ssh(himn, username, ('xe network-param-get param-name=bridge ' 'uuid=%s') % network_uuid) return bridge
def check_local_sr(himn, username): sr_type = utils.ssh(himn, username, ('xe sr-param-get param-name=type ' 'uuid=`xe pool-list params=default-SR --minimal`')) if sr_type != "ext" and sr_type != "nfs": utils.reportError(('Default SR type should be EXT or NFS. If using ' 'local storage, Please make sure thin provisioning ' 'is enabled on your host during installation.'))
def create_dom0_mesh_bridge(himn, username, dom0_bridge, mesh_info): # Create br-mesh and veth pair if not exist in Dom0 exitcode, out, _ = utils.ssh_detailed(himn, username, 'ip', 'addr', 'show', MESH_BRIDGE, allowed_return_codes=[0, 1]) if exitcode == 1: # create mesh bridge if it not exist in Dom0 create_mesh_bridge = True else: # if mesh bridge exist in Dom0, check its ip, re-configure ip if # it's not the same as what we want to set bridge_info = out.split() try: index_inet = bridge_info.index('inet') # get inet info like '192.168.2.2/24' ipaddr = bridge_info[index_inet + 1].split('/')[0] current_ip = ipaddress.ip_address(unicode(ipaddr)) configured_ip = ipaddress.ip_address(unicode(mesh_info['ipaddr'])) if current_ip == configured_ip: LOG.info('Bridge %s already exist in Dom0' % MESH_BRIDGE) return else: create_mesh_bridge = True remove_old_mesh_bridge(himn, username, MESH_BRIDGE) except ValueError: create_mesh_bridge = True remove_old_mesh_bridge(himn, username, MESH_BRIDGE) if create_mesh_bridge: LOG.debug("Create mesh bridge in Dom0") utils.scp(himn, username, '/etc/sysconfig/network-scripts/', AUTO_SCRIPT) utils.ssh(himn, username, 'chmod', '+x', '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT) start_param = '%(bridge)s %(ip)s %(netmask)s %(broadcast)s %(tag)s' \ % {'bridge': dom0_bridge, 'ip': mesh_info['ipaddr'], 'netmask': mesh_info['netmask'], 'broadcast': mesh_info['broadcast'], 'tag': mesh_info['tag']} with open(AUTO_START_SERVICE_TEMPLATE) as f: contents = f.read() contents = contents.replace('@MESH_INFO@', start_param) with open(AUTO_START_SERVICE, 'w') as f: f.write(contents) utils.scp(himn, username, '/etc/systemd/system', AUTO_START_SERVICE) utils.ssh(himn, username, 'systemctl', 'daemon-reload') utils.ssh(himn, username, 'systemctl', 'enable', AUTO_START_SERVICE) utils.ssh(himn, username, 'systemctl', 'start', AUTO_START_SERVICE)
def find_bridge_mappings(astute, himn, username): ethX = get_private_network_ethX() if not ethX: utils.reportError("Cannot find eth used for private network") # find the ethX mac in /sys/class/net/ethX/address with open('/sys/class/net/%s/address' % ethX, 'r') as fo: mac = fo.readline() network_uuid = utils.ssh(himn, username, ('xe vif-list params=network-uuid ' 'minimal=true MAC=%s') % mac) bridge = utils.ssh(himn, username, ('xe network-param-get param-name=bridge ' 'uuid=%s') % network_uuid) # find physical network name phynet_setting = astute['quantum_settings']['L2']['phys_nets'] physnet = phynet_setting.keys()[0] return physnet + ':' + bridge
def check_host_compatibility(himn, username): version_hotfixes = json.loads(VERSION_HOTFIXES) ver = utils.ssh(himn, username, ('xe host-param-get uuid=$(xe host-list --minimal) ' 'param-name=software-version param-key=product_version')) hotfixes = version_hotfixes.get(ver) if not hotfixes: return for hotfix in hotfixes: if not hotfix: continue installed = utils.ssh(himn, username, 'xe patch-list name-label=%s --minimal' % hotfix) if not installed: utils.reportError('Hotfix %s has not been installed ' % ver)
def check_can_log_via_ssh_using_saved_pwd(step, name, user): ip = utils.nova_cli.get_instance_ip(name) assert_true(ip != '', name) assert_true(world.saved_root_password is not None) conf.log(conf.get_bash_log_file(), "load world.saved_root_password=%s" % world.saved_root_password) step_assert(step).assert_true( utils.ssh(ip, command="/bin/ls -l /", user=user, password=world.saved_root_password).successful())
def deployBinary(self): # Kill any running instance of the binary to unlock all files local_kill_file = "kill_" + self.binary_name + ".txt" remote_kill_file = self.remote_binary_path + "kill_" + self.binary_name; script = ("""#!/bin/bash\n""" """pkill -f k3binary\n""") # Dump to a local file with open(local_kill_file,"wb") as script_file: script_file.write(script) # copy local to remote scpTo(self.host, local_kill_file, remote_kill_file) # chmod +x remote_script chmod_cmd = "chmod +x %s" % remote_kill_file ssh(self.host, chmod_cmd) # cleanup the local file cleanup_cmd = "rm %s" % local_kill_file os.system(cleanup_cmd) # call kill command ssh(self.host, remote_kill_file) # give the poor process time to die in peace (and unlock the script file) time.sleep(10) # copy run script over to remote remote_file = self.remote_binary_path + self.binary_name scpTo(self.host, self.local_binary_path, remote_file) # TODO use rsync ssh(self.host, "chmod +x %s" % remote_file)
def remove_old_mesh_bridge(himn, username, bridge): exitcode, _, _ = utils.ssh_detailed(himn, username, 'ip', 'link', 'show', bridge, allowed_return_codes=[0, 1]) if exitcode == 0: # Allow return code 5 to make sure it won't fail when # mos-vxlan.service isn't exist utils.ssh_detailed(himn, username, 'systemctl', 'stop', AUTO_START_SERVICE, allowed_return_codes=[0, 5]) utils.ssh_detailed(himn, username, 'systemctl', 'disable', AUTO_START_SERVICE, allowed_return_codes=[0, 1]) utils.ssh(himn, username, 'rm', '-f', '/etc/systemd/system/%s' % AUTO_START_SERVICE) utils.ssh(himn, username, 'rm', '-f', '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT) utils.ssh(himn, username, 'systemctl', 'daemon-reload')
def enable_conntrack_service(himn, username): # use conntrack statistic mode, so change conntrackd.conf errcode, out, err = utils.ssh_detailed( himn, username, 'ls', '/etc/conntrackd/conntrackd.conf.back', allowed_return_codes=[0, 2]) if errcode == 2: # Only make conntrackd.conf.back if it doesn't exist utils.ssh(himn, username, 'mv', '/etc/conntrackd/conntrackd.conf', '/etc/conntrackd/conntrackd.conf.back') utils.ssh(himn, username, 'cp', CONNTRACK_CONF_SAMPLE, '/etc/conntrackd/conntrackd.conf') # Rotate log file for conntrack utils.scp(himn, username, '/etc/logrotate.d', 'etc/logrotate.d/conntrackd') # Restart conntrackd service utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
def enable_linux_bridge(himn, username): # When using OVS under XS6.5, it will prevent use of Linux bridge in # Dom0, but neutron-openvswitch-agent in compute node will use Linux # bridge, so we remove this restriction here utils.ssh(himn, username, 'rm -f /etc/modprobe.d/blacklist-bridge*')
def check_can_log_via_ssh(step, name, user): ip = utils.nova_cli.get_instance_ip(name) assert_true(ip != '', name) step_assert(step).assert_true(utils.ssh(ip, "exit", user).successful())
def runRemoteScript(self): run_script_cmd = self.remote_script_path ssh(self.host,run_script_cmd)
def check_can_log_via_ssh_with_external_key(step, name, user, key): ip = utils.nova_cli.get_instance_ip(name) assert_true(ip != '', name) key_path = os.path.join(bunch_working_dir, key) step_assert(step).assert_true( utils.ssh(ip, command="exit", user=user, key=key_path).successful())
def route_to_compute(endpoints, himn_xs, himn_local, username): """Route storage/mgmt requests to compute nodes. """ def _net(ip): return '.'.join(ip.split('.')[:-1] + ['0']) def _mask(cidr): return inet_ntoa(pack('>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1)) def _routed(net, mask, gw): return re.search( r'%s\s+%s\s+%s\s+' % (net.replace('.', r'\.'), gw.replace('.', r'\.'), mask), out) out = utils.ssh(himn_xs, username, 'route', '-n') utils.ssh(himn_xs, username, ('printf "#!/bin/bash\nsleep 5\n" >' '/etc/udev/scripts/reroute.sh')) endpoint_names = ['storage', 'mgmt'] for endpoint_name in endpoint_names: endpoint = endpoints.get(endpoint_name) if endpoint: ip, cidr = endpoint.split('/') net, mask = _net(ip), _mask(cidr) if not _routed(net, mask, himn_local): params = [ 'route', 'add', '-net', '"%s"' % net, 'netmask', '"%s"' % mask, 'gw', himn_local ] utils.ssh(himn_xs, username, *params) # Always add the route to the udev, even if it's currently active cmd = ("printf 'if !(/sbin/route -n | /bin/grep -q -F \"{net}\"); " "then\n" "/sbin/route add -net \"{net}\" netmask " "\"{mask}\" gw {himn_local};\n" "fi\n' >> /etc/udev/scripts/reroute.sh") cmd = cmd.format(net=net, mask=mask, himn_local=himn_local) utils.ssh(himn_xs, username, cmd) else: LOG.info('%s network ip is missing' % endpoint_name) utils.ssh(himn_xs, username, 'chmod +x /etc/udev/scripts/reroute.sh') utils.ssh(himn_xs, username, ('echo \'SUBSYSTEM=="net" ACTION=="add" ' 'KERNEL=="xenapi" RUN+="/etc/udev/scripts/reroute.sh"\' ' '> /etc/udev/rules.d/90-reroute.rules'))
def check_can_log_via_ssh_using_saved_pwd(step, name, user): ip = utils.nova_cli.get_instance_ip(name) assert_true(ip != '', name) assert_true(world.saved_root_password is not None) conf.log(conf.get_bash_log_file(),"load world.saved_root_password=%s" % world.saved_root_password) step_assert(step).assert_true(utils.ssh(ip, command="/bin/ls -l /", user=user, password=world.saved_root_password).successful())
def route_to_compute(endpoints, himn_xs, himn_local, username): """Route storage/mgmt requests to compute nodes. """ def _net(ip): return '.'.join(ip.split('.')[:-1] + ['0']) def _mask(cidr): return inet_ntoa(pack('>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1)) def _routed(net, mask, gw): return re.search(r'%s\s+%s\s+%s\s+' % ( net.replace('.', r'\.'), gw.replace('.', r'\.'), mask ), out) out = utils.ssh(himn_xs, username, 'route', '-n') utils.ssh(himn_xs, username, ('printf "#!/bin/bash\nsleep 5\n" >' '/etc/udev/scripts/reroute.sh')) endpoint_names = ['storage', 'mgmt'] for endpoint_name in endpoint_names: endpoint = endpoints.get(endpoint_name) if endpoint: ip, cidr = endpoint.split('/') net, mask = _net(ip), _mask(cidr) if not _routed(net, mask, himn_local): params = ['route', 'add', '-net', '"%s"' % net, 'netmask', '"%s"' % mask, 'gw', himn_local] utils.ssh(himn_xs, username, *params) # Always add the route to the udev, even if it's currently active cmd = ( "printf 'if !(/sbin/route -n | /bin/grep -q -F \"{net}\"); " "then\n" "/sbin/route add -net \"{net}\" netmask " "\"{mask}\" gw {himn_local};\n" "fi\n' >> /etc/udev/scripts/reroute.sh" ) cmd = cmd.format(net=net, mask=mask, himn_local=himn_local) utils.ssh(himn_xs, username, cmd) else: LOG.info('%s network ip is missing' % endpoint_name) utils.ssh(himn_xs, username, 'chmod +x /etc/udev/scripts/reroute.sh') utils.ssh(himn_xs, username, ('echo \'SUBSYSTEM=="net" ACTION=="add" ' 'KERNEL=="xenapi" RUN+="/etc/udev/scripts/reroute.sh"\' ' '> /etc/udev/rules.d/90-reroute.rules'))
def fullCleanup(): (listnodes, nodes) = cluster.instanceListAll() stopDfs(listnodes, nodes) pathnamenode = os.path.join(datadir, 'namenode') pathdatanode = os.path.join(datadir, 'datanode') for namenode in listnodes['all']: node = nodes[namenode] nodeip = node['externalIp'] utils.ssh(nodeip, "rm -rf {0}".format(pathnamenode)) utils.ssh(nodeip, "rm -rf {0}".format(pathdatanode)) utils.ssh(nodeip, "mkdir -p {0}".format(pathnamenode)) utils.ssh(nodeip, "mkdir -p {0}".format(pathdatanode)) utils.ssh(nodeip, "chmod -R a+rwx {0}".format(pathnamenode)) utils.ssh(nodeip, "chmod -R a+rwx {0}".format(pathdatanode)) formatFileSystem(listnodes, nodes) startDfs(listnodes, nodes)
def check_can_log_via_ssh_with_external_key(step, name, user, key): ip = utils.nova_cli.get_instance_ip(name) assert_true(ip != '', name) key_path = os.path.join(bunch_working_dir,key) step_assert(step).assert_true(utils.ssh(ip, command="exit", user=user, key=key_path).successful())