def start_cvm(hypervisor_ip): # Get hostname to be used when killing the CVM later. log.INFO("Starting cvm on host %s" % hypervisor_ip) host_ssh_client = SSHClient(hypervisor_ip, FLAGS.hypervisor_username, password=FLAGS.default_host_password) ret, out, err = host_ssh_client.execute("hostname") if ret != 0: log.ERROR("Error retrieving hyp hostname from ip %s, error %s" "ret %s" % (hypervisor_ip, err, ret)) return 1 hostname = out.strip() # Kill the CVM on each of the removed nodes. define_cmd = "virsh define /root/NTNX-CVM.xml" start_cmd = "virsh start NTNX-%s-CVM" % hostname autostart_cmd = "virsh autostart NTNX-%s-CVM" % hostname ret, out, err = host_ssh_client.execute(define_cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when defining the CVM" " on host %s" % (err, out, ret, hypervisor_ip)) log.INFO("Please start the CVM manually on this host") return 1 log.INFO("Define cmd for cvm %s success") ret, out, err = host_ssh_client.execute(start_cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when starting the CVM" "on host %s" % (ret, out, err, hypervisor_ip)) log.INFO("Please start the CVM manually on this host") return 1 log.INFO("Start cmd for cvm %s success") ret, out, err = host_ssh_client.execute(autostart_cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when autostarting the CVM" "on host %s" % (ret, out, err, hypervisor_ip)) log.INFO("Please autostart the CVM manually on this host") return 1 log.INFO("CVM started on host %s" % hypervisor_ip) # Remove the configured file on the host which is used to indicate it is part # of a cluster. rm_cmd = "rm -f /root/configured" ret, out, err = host_ssh_client.execute(rm_cmd) if ret != 0: log.ERROR( "Cannot remove /root/configured file from the host. Please " "remove the file manually in order to add it back as a CO node") return 1 return 0
def monitor_node_removal(service_vm_id): progress_info_id = ProgressInfoProto.ProgressInfoId() progress_info_id.operation = ProgressInfoProto.kRemove progress_info_id.entity_type = ProgressInfoProto.kNode progress_info_id.entity_id = str(service_vm_id) progress_interface = ProgressMonitorInterface() proto = progress_interface.lookup(progress_info_id) num_tasks = len(proto.progress_task_list) while True: pending_tasks = 0 for task_id in range(num_tasks): task = proto.progress_task_list[task_id] if (task.progress_status == ProgressInfoProto.kQueued) or ( task.progress_status == ProgressInfoProto.kRunning): pending_tasks += 1 elif (task.progress_status == ProgressInfoProto.kFailed) or ( task.progress_status == ProgressInfoProto.kAborted): log.ERROR("Error removing node %s from cluster" % service_vm_id) log.INFO("Node has to be removed manually") return 1 elif task.progress_status == ProgressInfoProto.kSucceeded: log.INFO("Node %s successfully removed" % service_vm_id) return 0 log.INFO("Waiting for remove node to finish on node vm id %s" % service_vm_id) time.sleep(10)
def tokenize_lexical_scanner(source, module): """Tokenizes the python source code using a lexical scanner and looks for gflags used in the file and checks if they are undefined. Arguments: source: file object of the python source code file module: name of the python module in source """ global UNDEFINED_GFLAG_FOUND generator = tokenize.generate_tokens(source.readline) tkn_lst = [] for token_type, token_str, _, _, line in generator: if token_type == tokenize.COMMENT: continue # At the end of one line of code. if token_str == '\n': indices = [idx for idx, tkn in enumerate(tkn_lst) if tkn == "FLAGS"] for idx in indices: try: if tkn_lst[idx + 1] == '.': gflag_name = tkn_lst[idx + 2] log.DEBUG("line: %s" % line) log.DEBUG("token list: %s" % tkn_lst) log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, gflag_name)) if gflag_name not in gflags.FLAGS.RegisteredFlags(): UNDEFINED_GFLAG_FOUND = True log.ERROR("%s used in %s is undefined" % (gflag_name, module)) except IndexError: log.DEBUG("IndexError: %s" % line) continue tkn_lst = [] else: tkn_lst.append(token_str)
def tokenize_regex(source, module): """Tokenizes the python source code file manually and looks for gflags used in the file and checks if they are undefined. Arguments: source: file object of the python source code file module: name of the python module in source """ global UNDEFINED_GFLAG_FOUND for line in source: line = line.strip() # Skip single comment lines. Can't avoid multiline comments though. if line.startswith('#'): continue try: gflag_name = GFLAG_NAME_REGEX.match(line).groups()[2] except AttributeError: # Not a match. pass else: log.DEBUG("line: %s" % line) log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, gflag_name)) if gflag_name not in gflags.FLAGS.RegisteredFlags(): UNDEFINED_GFLAG_FOUND = True log.ERROR("%s used in %s is undefined" % (gflag_name, module))
def tokenize_manually(source, module): """Tokenizes the python source code file manually and looks for gflags used in the file and checks if they are undefined. Arguments: source: file object of the python source code file module: name of the python module in source """ global UNDEFINED_GFLAG_FOUND for line in source: line = line.strip() # Skip single comment lines. Can't avoid multiline comments though. if line.startswith('#'): continue if GFLAGS_PREFIX in line: log.DEBUG("line: %s" % line) for token in line.split(): # Remove any operators and delimiters that could be # surrounding the gflag. token = token.strip(PY_OPERATORS_DELIMITERS) if GFLAGS_PREFIX in token: token = token.split(GFLAGS_PREFIX)[-1] token = token.strip(GFLAGS_PREFIX) elif GFLAGS_PREFIX_FULL in token: token = token.split(GFLAGS_PREFIX_FULL)[-1] token = token.strip(GFLAGS_PREFIX_FULL) else: continue log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, token)) if token not in gflags.FLAGS.RegisteredFlags(): UNDEFINED_GFLAG_FOUND = True log.ERROR("%s used in %s is undefined" % (token, module))
def main(): if FLAGS.help: show_help() # HCI node removal if FLAGS.remove_node: if not FLAGS.node_uuid: log.ERROR("Node uuid missing. Pass node uuid to initiate removal") return 1 else: return remove_node(FLAGS.node_uuid) # Monitor removal. if FLAGS.monitor_removal: if not FLAGS.service_vm_id: log.ERROR("Service vm id missing") return 1 else: return monitor_node_removal(FLAGS.service_vm_id) # Destroy CVM if FLAGS.kill_cvm: if not FLAGS.cvm_ip: log.ERROR("Node cvm ip is missing") return 1 elif not FLAGS.host_ip: log.ERROR("Host ip missing") return 1 else: return kill_cvm(FLAGS.cvm_ip, FLAGS.host_ip) # Add the node. if FLAGS.add_node: if not FLAGS.node_uuid: log.ERROR("Node uuid required to add co node") return 1 if (not FLAGS.cvm_ip) and not (FLAGS.host_ip): log.ERROR("Either cvm ip or host ip is required to add the node") return 1 if FLAGS.cvm_ip: node_ip = FLAGS.cvm_ip compute_only = False else: node_ip = FLAGS.host_ip compute_only = True return add_node(FLAGS.node_uuid, node_ip, compute_only) if FLAGS.remove_co_node: log.INFO("Please edit zeus config manually using edit_zeus " "--editor=/usr/bin/vim. Remove the node info from " "compute_node_list and management_server_list") return 0 if FLAGS.start_cvm: if not FLAGS.host_ip: log.ERROR("Please provide host ip to start cvm on") return 1 return start_cvm(FLAGS.host_ip)
def _change_permission_bits(client, permission_string, filepath): """ Change permission bits on client for filepath. Returns 0 on success, non zero on failure. """ cmd = """chmod %s "%s" """ % (permission_string, filepath) ret, out, error = client.execute(cmd) if ret != 0: log.ERROR("Error %s when changing permission on %s to %s: out %s" % (error, filepath, permission_string, out)) return ret
def remove_node(node_uuid): cmd = "ncli cluster rm-start id=%s" % node_uuid ret, out, err = timed_command(cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when %s was called on %s" % (err, out, ret, cmd, node_uuid)) log.WARNING("Please remove this node manually") return 1 else: log.INFO("NCLI remove returned %s for node %s" % (out, node_uuid)) log.INFO("You can check the progress by calling --monitor_removal " "<service_vm_id>") return 0
def _kill_cvm(cvm_ip, host_ssh_client, hostname): # Kill the CVM on each of the removed nodes. shutdown_cmd = "virsh shutdown NTNX-%s-CVM" % hostname undefine_cmd = "virsh undefine NTNX-%s-CVM" % hostname ret, out, err = host_ssh_client.execute(shutdown_cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when shutting down the CVM" " %s on host" % (err, out, ret, cvm_ip)) log.INFO("Please shutdown the CVM manually on this host") return 1 log.INFO("Shutdown cmd for cvm %s success" % cvm_ip) ret, out, err = host_ssh_client.execute(undefine_cmd) if ret != 0: log.ERROR("Error %s, out %s, ret %s when undefining the CVM" " %s on host" % (ret, out, err, cvm_ip)) log.INFO("Please undefine the CVM manually on this host") return 1 log.INFO("Undefine cmd for cvm %s success" % cvm_ip) return 0
def add_node(node_uuid, node_ip, compute_only): # Add node. from cluster.utils.genesis_client import GenesisApiClient client = GenesisApiClient() ret, error = client.make_rpc("ClusterManager", "add_node", { "node_uuid": node_uuid, "node_svm_ip": node_ip, "compute_only": compute_only }, svm_ips=["localhost"]) if not ret: log.ERROR("Add node for %s failed with error %s" % (node_ip, error)) log.INFO("The node will have to be manually added back") return 1 else: log.INFO("Node %s successfully added" % node_ip) return 0
def read_external_cluster_entry(self, cluster_id): config = ConfigurationProto() cluster_ids = [] if not cluster_id: cluster_ids = self.zks.list(EXTERNAL_CLUSTER_PATH) else: cluster_ids = [cluster_id] config_list = {} for cluster_id in cluster_ids: path = EXTERNAL_CLUSTER_PATH + '/' + cluster_id if not self.zks.stat(path): log.ERROR('Can not find path %s.' % path) config_list.append(None) continue config_str = self.zks.get(EXTERNAL_CLUSTER_PATH + '/' + cluster_id) if not config_str: continue config.ParseFromString(config_str) config_list[cluster_id] = config return config_list
def check_undefined_gflags(module_list): """Checks if any undefined gflags are used in module_list. Arguments: module_list: list of python modules to check for undefined gflags. """ for module in module_list: try: imported_module = importlib.import_module(module) except ImportError as iexc: log.ERROR("Unable to import %s" % module) log.ERROR(traceback.format_exc()) log.ERROR(iexc) sys.exit(1) except AssertionError: log.ERROR("Unable to import %s because of an assert statement in it. " "Remove it temporarily to test it with %s" % (module, sys.argv[0])) sys.exit(1) except Exception as exc: log.ERROR("Unable to import %s" % module) log.ERROR(traceback.format_exc()) log.ERROR(exc) sys.exit(1) # Read from .py fle and not .pyc file. with open(imported_module.__file__.rstrip('c')) as source: #tokenize_manually(source, module) #source.seek(0) #tokenize_regex(source, module) #source.seek(0) tokenize_lexical_scanner(source, module) del imported_module return None
def kill_cvm(cvm_ip, hypervisor_ip): # Get hostname to be used when killing the CVM later. host_ssh_client = SSHClient(hypervisor_ip, FLAGS.hypervisor_username, password=FLAGS.default_host_password) ret, out, err = host_ssh_client.execute("hostname") if ret != 0: log.ERROR("Error retrieving hyp hostname from ip %s, error %s" "ret %s" % (hypervisor_ip, err, ret)) return 1 hostname = out.strip() # Transfer the factory_config.json to the host. cvm_ssh_client = SSHClient(cvm_ip, "nutanix", password=FLAGS.default_cvm_password) with tempfile.NamedTemporaryFile() as tmp: # Transfer from the cvm in question to a local tmp path. ret, out, error = cvm_ssh_client.execute( "cat %s" % FLAGS.factory_config_json_path) if ret != 0: log.ERROR("Unable to read the factory_config.json from node %s" % cvm_ip) return 1 # Modify the contents to add "mode", as foundation would do it. factory_config_dict = json.loads(out.strip()) factory_config_dict["mode"] = "compute only" tmp.write(json.dumps(factory_config_dict)) tmp.flush() # Transfer from the local path to the corresponding host. ret, out, err = host_ssh_client.transfer_to( tmp.name, FLAGS.factory_config_json_path_on_host) if ret != 0: log.ERROR("Unable to transfer %s to the host %s" % (FLAGS.factory_config_json_path, hypervisor_ip)) return 1 # Set the right permission bits. permission_string = "644" filepath = FLAGS.factory_config_json_path_on_host ret = _change_permission_bits(host_ssh_client, permission_string, filepath) if ret != 0: return 1 log.INFO("Transferred factory config json to the host") # Transfer the hardware config json. with tempfile.NamedTemporaryFile() as tmp: # Transfer from the cvm in question to a local tmp path. ret, out, error = cvm_ssh_client.execute( "cat %s" % FLAGS.hardware_config_json_path) if ret != 0: log.ERROR("Unable to read the hardware_config.json from node %s" % cvm_ip) return 1 tmp.write(out.strip()) tmp.flush() # Transfer from the local path to the corresponding host. ret, out, err = host_ssh_client.transfer_to( tmp.name, FLAGS.hardware_config_json_path_on_host) if ret != 0: log.ERROR("Unable to transfer %s to the host %s" % (FLAGS.hardware_config_json_path, hypervisor_ip)) return 1 # Set the right permission bits. permission_string = "644" filepath = FLAGS.hardware_config_json_path_on_host ret = _change_permission_bits(host_ssh_client, permission_string, filepath) if ret != 0: return 1 log.INFO("Transferred hardware config json to the host") return _kill_cvm(hypervisor_ip, host_ssh_client, hostname)
"compute_node_list and management_server_list") return 0 if FLAGS.start_cvm: if not FLAGS.host_ip: log.ERROR("Please provide host ip to start cvm on") return 1 return start_cvm(FLAGS.host_ip) if __name__ == "__main__": try: global __doc__ __doc__ = "Usage: %s [flags]\nTry %s --help" % (sys.argv[0], sys.argv[0]) args = FLAGS(sys.argv) FLAGS.logtostderr = True log.initialize() sys.exit(main()) except gflags.FlagsError, e: log.ERROR("%s\n%s\n" % (str(e), __doc__)) sys.exit(1) except KeyboardInterrupt: log.WARNING("Exiting on Ctrl-C") sys.exit(1) except Exception as ex: log.ERROR(traceback.format_exc()) log.ERROR("Failed to execute action %s error (%s), exiting.." % (args[0], str(ex))) sys.exit(1)
def remove_vgs(): """ List all the VGs on the cluster. """ url = "https://%s%s" % (SERVER, VG_LIST_URL) auth = (FLAGS.username, FLAGS.password) response = requests.get(url, auth=auth, verify=False) if response.status_code == 401: print( "Could not get list of VGs from Prism. Wrong Credentials. " "Aborting!!") log.ERROR("Could not get list of VGs from Prism. Wrong Credentials. " "Aborting!!") sys.exit(1) if not response.ok: print( "Could not get list of VGs from Prism. Error code %s Url: %s" "\nAborting!!\n" % (response.status_code, response.url)) log.ERROR("Could not get list of VGs from Prism. Error code %s Url: %s" "\nAborting!!\n" % (response.status_code, response.url)) sys.exit(1) for entity in response.json()['entities']: url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"])) if not entity["name"].startswith(FLAGS.vg_name_prefix_to_delete): log.INFO("Not deleting VG %s" % entity["name"]) continue if FLAGS.vg_name_prefix_to_skip and entity["name"].startswith( FLAGS.vg_name_prefix_to_skip): log.INFO("Not deleting VG %s" % entity["name"]) continue if FLAGS.ask_for_confirmation: choice = raw_input( "WARNING: Deleting VG %s. \nOK to continue?(y/n):" % entity["name"]) if choice != 'y': log.INFO("User did not respond with \"y\".Skippping VG %s !!" % entity["name"]) continue params = {} params["attached_clients"] = [] params["uuid"] = entity["uuid"] params["name"] = entity["name"] auth = (FLAGS.username, FLAGS.password) print "Processing VG %s" % params["name"] response = requests.put(url, data=json.dumps(params), auth=auth, verify=False) if not response.ok: log.ERROR( "Could not update the VG from Prism. Error code %s Url: %s" "\nAborting!!\n" % (response, url)) sys.exit(1) url = "https://%s%s/%s" % (SERVER, VG_LIST_URL, entity["uuid"]) auth = (FLAGS.username, FLAGS.password) response = requests.get(url, auth=auth, verify=False) if not response.ok: log.ERROR("Could not get the VG from Prism. Error code %s Url: %s" "\nAborting!!\n" % (response, url)) sys.exit(1) for attachment in response.json().get("attachment_list", []): if "vm_uuid" in attachment.keys(): url = "https://%s%s" % (SERVER, VG_DETACH_URL.format(entity["uuid"])) params = {"vm_uuid": attachment["vm_uuid"]} response = requests.post(url, auth=auth, data=json.dumps(params), verify=False) if not response.ok: log.ERROR( "Could not remove Vm attachment from the VG. Error code %s " "Url: %s\nAborting!!\n" % (response, url)) sys.exit(1) url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"])) auth = (FLAGS.username, FLAGS.password) # Sleeping a bit to be a little easy on acropolis. time.sleep(1) log.INFO("Deleting VG %s" % entity["name"]) response = requests.delete(url, auth=auth, verify=False) if not response.ok: log.ERROR( "Could not delete the VG from Prism. Error code %s Url: %s" "\nAborting!!\n" % (response, url)) sys.exit(1) return True