def start_cvm(hypervisor_ip):
# Get hostname to be used when killing the CVM later.
log.INFO("Starting cvm on host %s" % hypervisor_ip)
host_ssh_client = SSHClient(hypervisor_ip,
FLAGS.hypervisor_username,
password=FLAGS.default_host_password)
ret, out, err = host_ssh_client.execute("hostname")
if ret != 0:
log.ERROR("Error retrieving hyp hostname from ip %s, error %s"
"ret %s" % (hypervisor_ip, err, ret))
return 1
hostname = out.strip()
# Kill the CVM on each of the removed nodes.
define_cmd = "virsh define /root/NTNX-CVM.xml"
start_cmd = "virsh start NTNX-%s-CVM" % hostname
autostart_cmd = "virsh autostart NTNX-%s-CVM" % hostname
ret, out, err = host_ssh_client.execute(define_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when defining the CVM"
" on host %s" % (err, out, ret, hypervisor_ip))
log.INFO("Please start the CVM manually on this host")
return 1
log.INFO("Define cmd for cvm %s success")
ret, out, err = host_ssh_client.execute(start_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when starting the CVM"
"on host %s" % (ret, out, err, hypervisor_ip))
log.INFO("Please start the CVM manually on this host")
return 1
log.INFO("Start cmd for cvm %s success")
ret, out, err = host_ssh_client.execute(autostart_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when autostarting the CVM"
"on host %s" % (ret, out, err, hypervisor_ip))
log.INFO("Please autostart the CVM manually on this host")
return 1
log.INFO("CVM started on host %s" % hypervisor_ip)
# Remove the configured file on the host which is used to indicate it is part
# of a cluster.
rm_cmd = "rm -f /root/configured"
ret, out, err = host_ssh_client.execute(rm_cmd)
if ret != 0:
log.ERROR(
"Cannot remove /root/configured file from the host. Please "
"remove the file manually in order to add it back as a CO node")
return 1
return 0
def monitor_node_removal(service_vm_id):
progress_info_id = ProgressInfoProto.ProgressInfoId()
progress_info_id.operation = ProgressInfoProto.kRemove
progress_info_id.entity_type = ProgressInfoProto.kNode
progress_info_id.entity_id = str(service_vm_id)
progress_interface = ProgressMonitorInterface()
proto = progress_interface.lookup(progress_info_id)
num_tasks = len(proto.progress_task_list)
while True:
pending_tasks = 0
for task_id in range(num_tasks):
task = proto.progress_task_list[task_id]
if (task.progress_status == ProgressInfoProto.kQueued) or (
task.progress_status == ProgressInfoProto.kRunning):
pending_tasks += 1
elif (task.progress_status == ProgressInfoProto.kFailed) or (
task.progress_status == ProgressInfoProto.kAborted):
log.ERROR("Error removing node %s from cluster" %
service_vm_id)
log.INFO("Node has to be removed manually")
return 1
elif task.progress_status == ProgressInfoProto.kSucceeded:
log.INFO("Node %s successfully removed" % service_vm_id)
return 0
log.INFO("Waiting for remove node to finish on node vm id %s" %
service_vm_id)
time.sleep(10)
def tokenize_lexical_scanner(source, module):
"""Tokenizes the python source code using a lexical scanner and looks
for gflags used in the file and checks if they are undefined.
Arguments:
source: file object of the python source code file
module: name of the python module in source
"""
global UNDEFINED_GFLAG_FOUND
generator = tokenize.generate_tokens(source.readline)
tkn_lst = []
for token_type, token_str, _, _, line in generator:
if token_type == tokenize.COMMENT:
continue
# At the end of one line of code.
if token_str == '\n':
indices = [idx for idx, tkn in enumerate(tkn_lst) if tkn == "FLAGS"]
for idx in indices:
try:
if tkn_lst[idx + 1] == '.':
gflag_name = tkn_lst[idx + 2]
log.DEBUG("line: %s" % line)
log.DEBUG("token list: %s" % tkn_lst)
log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, gflag_name))
if gflag_name not in gflags.FLAGS.RegisteredFlags():
UNDEFINED_GFLAG_FOUND = True
log.ERROR("%s used in %s is undefined" % (gflag_name, module))
except IndexError:
log.DEBUG("IndexError: %s" % line)
continue
tkn_lst = []
else:
tkn_lst.append(token_str)
def tokenize_regex(source, module):
"""Tokenizes the python source code file manually and looks for
gflags used in the file and checks if they are undefined.
Arguments:
source: file object of the python source code file
module: name of the python module in source
"""
global UNDEFINED_GFLAG_FOUND
for line in source:
line = line.strip()
# Skip single comment lines. Can't avoid multiline comments though.
if line.startswith('#'):
continue
try:
gflag_name = GFLAG_NAME_REGEX.match(line).groups()[2]
except AttributeError: # Not a match.
pass
else:
log.DEBUG("line: %s" % line)
log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, gflag_name))
if gflag_name not in gflags.FLAGS.RegisteredFlags():
UNDEFINED_GFLAG_FOUND = True
log.ERROR("%s used in %s is undefined" % (gflag_name, module))
def tokenize_manually(source, module):
"""Tokenizes the python source code file manually and looks for
gflags used in the file and checks if they are undefined.
Arguments:
source: file object of the python source code file
module: name of the python module in source
"""
global UNDEFINED_GFLAG_FOUND
for line in source:
line = line.strip()
# Skip single comment lines. Can't avoid multiline comments though.
if line.startswith('#'):
continue
if GFLAGS_PREFIX in line:
log.DEBUG("line: %s" % line)
for token in line.split():
# Remove any operators and delimiters that could be
# surrounding the gflag.
token = token.strip(PY_OPERATORS_DELIMITERS)
if GFLAGS_PREFIX in token:
token = token.split(GFLAGS_PREFIX)[-1]
token = token.strip(GFLAGS_PREFIX)
elif GFLAGS_PREFIX_FULL in token:
token = token.split(GFLAGS_PREFIX_FULL)[-1]
token = token.strip(GFLAGS_PREFIX_FULL)
else:
continue
log.DEBUG("%s: %s" % (sys._getframe().f_code.co_name, token))
if token not in gflags.FLAGS.RegisteredFlags():
UNDEFINED_GFLAG_FOUND = True
log.ERROR("%s used in %s is undefined" % (token, module))
def main():
if FLAGS.help:
show_help()
# HCI node removal
if FLAGS.remove_node:
if not FLAGS.node_uuid:
log.ERROR("Node uuid missing. Pass node uuid to initiate removal")
return 1
else:
return remove_node(FLAGS.node_uuid)
# Monitor removal.
if FLAGS.monitor_removal:
if not FLAGS.service_vm_id:
log.ERROR("Service vm id missing")
return 1
else:
return monitor_node_removal(FLAGS.service_vm_id)
# Destroy CVM
if FLAGS.kill_cvm:
if not FLAGS.cvm_ip:
log.ERROR("Node cvm ip is missing")
return 1
elif not FLAGS.host_ip:
log.ERROR("Host ip missing")
return 1
else:
return kill_cvm(FLAGS.cvm_ip, FLAGS.host_ip)
# Add the node.
if FLAGS.add_node:
if not FLAGS.node_uuid:
log.ERROR("Node uuid required to add co node")
return 1
if (not FLAGS.cvm_ip) and not (FLAGS.host_ip):
log.ERROR("Either cvm ip or host ip is required to add the node")
return 1
if FLAGS.cvm_ip:
node_ip = FLAGS.cvm_ip
compute_only = False
else:
node_ip = FLAGS.host_ip
compute_only = True
return add_node(FLAGS.node_uuid, node_ip, compute_only)
if FLAGS.remove_co_node:
log.INFO("Please edit zeus config manually using edit_zeus "
"--editor=/usr/bin/vim. Remove the node info from "
"compute_node_list and management_server_list")
return 0
if FLAGS.start_cvm:
if not FLAGS.host_ip:
log.ERROR("Please provide host ip to start cvm on")
return 1
return start_cvm(FLAGS.host_ip)
def _change_permission_bits(client, permission_string, filepath):
"""
Change permission bits on client for filepath.
Returns 0 on success, non zero on failure.
"""
cmd = """chmod %s "%s" """ % (permission_string, filepath)
ret, out, error = client.execute(cmd)
if ret != 0:
log.ERROR("Error %s when changing permission on %s to %s: out %s" %
(error, filepath, permission_string, out))
return ret
def remove_node(node_uuid):
cmd = "ncli cluster rm-start id=%s" % node_uuid
ret, out, err = timed_command(cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when %s was called on %s" %
(err, out, ret, cmd, node_uuid))
log.WARNING("Please remove this node manually")
return 1
else:
log.INFO("NCLI remove returned %s for node %s" % (out, node_uuid))
log.INFO("You can check the progress by calling --monitor_removal "
"<service_vm_id>")
return 0
def _kill_cvm(cvm_ip, host_ssh_client, hostname):
# Kill the CVM on each of the removed nodes.
shutdown_cmd = "virsh shutdown NTNX-%s-CVM" % hostname
undefine_cmd = "virsh undefine NTNX-%s-CVM" % hostname
ret, out, err = host_ssh_client.execute(shutdown_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when shutting down the CVM"
" %s on host" % (err, out, ret, cvm_ip))
log.INFO("Please shutdown the CVM manually on this host")
return 1
log.INFO("Shutdown cmd for cvm %s success" % cvm_ip)
ret, out, err = host_ssh_client.execute(undefine_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when undefining the CVM"
" %s on host" % (ret, out, err, cvm_ip))
log.INFO("Please undefine the CVM manually on this host")
return 1
log.INFO("Undefine cmd for cvm %s success" % cvm_ip)
return 0
def add_node(node_uuid, node_ip, compute_only):
# Add node.
from cluster.utils.genesis_client import GenesisApiClient
client = GenesisApiClient()
ret, error = client.make_rpc("ClusterManager",
"add_node", {
"node_uuid": node_uuid,
"node_svm_ip": node_ip,
"compute_only": compute_only
},
svm_ips=["localhost"])
if not ret:
log.ERROR("Add node for %s failed with error %s" % (node_ip, error))
log.INFO("The node will have to be manually added back")
return 1
else:
log.INFO("Node %s successfully added" % node_ip)
return 0
def read_external_cluster_entry(self, cluster_id):
config = ConfigurationProto()
cluster_ids = []
if not cluster_id:
cluster_ids = self.zks.list(EXTERNAL_CLUSTER_PATH)
else:
cluster_ids = [cluster_id]
config_list = {}
for cluster_id in cluster_ids:
path = EXTERNAL_CLUSTER_PATH + '/' + cluster_id
if not self.zks.stat(path):
log.ERROR('Can not find path %s.' % path)
config_list.append(None)
continue
config_str = self.zks.get(EXTERNAL_CLUSTER_PATH + '/' + cluster_id)
if not config_str:
continue
config.ParseFromString(config_str)
config_list[cluster_id] = config
return config_list
def check_undefined_gflags(module_list):
"""Checks if any undefined gflags are used in module_list.
Arguments:
module_list: list of python modules to check for undefined gflags.
"""
for module in module_list:
try:
imported_module = importlib.import_module(module)
except ImportError as iexc:
log.ERROR("Unable to import %s" % module)
log.ERROR(traceback.format_exc())
log.ERROR(iexc)
sys.exit(1)
except AssertionError:
log.ERROR("Unable to import %s because of an assert statement in it. "
"Remove it temporarily to test it with %s" % (module,
sys.argv[0]))
sys.exit(1)
except Exception as exc:
log.ERROR("Unable to import %s" % module)
log.ERROR(traceback.format_exc())
log.ERROR(exc)
sys.exit(1)
# Read from .py fle and not .pyc file.
with open(imported_module.__file__.rstrip('c')) as source:
#tokenize_manually(source, module)
#source.seek(0)
#tokenize_regex(source, module)
#source.seek(0)
tokenize_lexical_scanner(source, module)
del imported_module
return None
def kill_cvm(cvm_ip, hypervisor_ip):
# Get hostname to be used when killing the CVM later.
host_ssh_client = SSHClient(hypervisor_ip,
FLAGS.hypervisor_username,
password=FLAGS.default_host_password)
ret, out, err = host_ssh_client.execute("hostname")
if ret != 0:
log.ERROR("Error retrieving hyp hostname from ip %s, error %s"
"ret %s" % (hypervisor_ip, err, ret))
return 1
hostname = out.strip()
# Transfer the factory_config.json to the host.
cvm_ssh_client = SSHClient(cvm_ip,
"nutanix",
password=FLAGS.default_cvm_password)
with tempfile.NamedTemporaryFile() as tmp:
# Transfer from the cvm in question to a local tmp path.
ret, out, error = cvm_ssh_client.execute(
"cat %s" % FLAGS.factory_config_json_path)
if ret != 0:
log.ERROR("Unable to read the factory_config.json from node %s" %
cvm_ip)
return 1
# Modify the contents to add "mode", as foundation would do it.
factory_config_dict = json.loads(out.strip())
factory_config_dict["mode"] = "compute only"
tmp.write(json.dumps(factory_config_dict))
tmp.flush()
# Transfer from the local path to the corresponding host.
ret, out, err = host_ssh_client.transfer_to(
tmp.name, FLAGS.factory_config_json_path_on_host)
if ret != 0:
log.ERROR("Unable to transfer %s to the host %s" %
(FLAGS.factory_config_json_path, hypervisor_ip))
return 1
# Set the right permission bits.
permission_string = "644"
filepath = FLAGS.factory_config_json_path_on_host
ret = _change_permission_bits(host_ssh_client, permission_string,
filepath)
if ret != 0:
return 1
log.INFO("Transferred factory config json to the host")
# Transfer the hardware config json.
with tempfile.NamedTemporaryFile() as tmp:
# Transfer from the cvm in question to a local tmp path.
ret, out, error = cvm_ssh_client.execute(
"cat %s" % FLAGS.hardware_config_json_path)
if ret != 0:
log.ERROR("Unable to read the hardware_config.json from node %s" %
cvm_ip)
return 1
tmp.write(out.strip())
tmp.flush()
# Transfer from the local path to the corresponding host.
ret, out, err = host_ssh_client.transfer_to(
tmp.name, FLAGS.hardware_config_json_path_on_host)
if ret != 0:
log.ERROR("Unable to transfer %s to the host %s" %
(FLAGS.hardware_config_json_path, hypervisor_ip))
return 1
# Set the right permission bits.
permission_string = "644"
filepath = FLAGS.hardware_config_json_path_on_host
ret = _change_permission_bits(host_ssh_client, permission_string,
filepath)
if ret != 0:
return 1
log.INFO("Transferred hardware config json to the host")
return _kill_cvm(hypervisor_ip, host_ssh_client, hostname)
"compute_node_list and management_server_list")
return 0
if FLAGS.start_cvm:
if not FLAGS.host_ip:
log.ERROR("Please provide host ip to start cvm on")
return 1
return start_cvm(FLAGS.host_ip)
if __name__ == "__main__":
try:
global __doc__
__doc__ = "Usage: %s [flags]\nTry %s --help" % (sys.argv[0],
sys.argv[0])
args = FLAGS(sys.argv)
FLAGS.logtostderr = True
log.initialize()
sys.exit(main())
except gflags.FlagsError, e:
log.ERROR("%s\n%s\n" % (str(e), __doc__))
sys.exit(1)
except KeyboardInterrupt:
log.WARNING("Exiting on Ctrl-C")
sys.exit(1)
except Exception as ex:
log.ERROR(traceback.format_exc())
log.ERROR("Failed to execute action %s error (%s), exiting.." %
(args[0], str(ex)))
sys.exit(1)
def remove_vgs():
"""
List all the VGs on the cluster.
"""
url = "https://%s%s" % (SERVER, VG_LIST_URL)
auth = (FLAGS.username, FLAGS.password)
response = requests.get(url, auth=auth, verify=False)
if response.status_code == 401:
print(
"Could not get list of VGs from Prism. Wrong Credentials. "
"Aborting!!")
log.ERROR("Could not get list of VGs from Prism. Wrong Credentials. "
"Aborting!!")
sys.exit(1)
if not response.ok:
print(
"Could not get list of VGs from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response.status_code, response.url))
log.ERROR("Could not get list of VGs from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response.status_code, response.url))
sys.exit(1)
for entity in response.json()['entities']:
url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"]))
if not entity["name"].startswith(FLAGS.vg_name_prefix_to_delete):
log.INFO("Not deleting VG %s" % entity["name"])
continue
if FLAGS.vg_name_prefix_to_skip and entity["name"].startswith(
FLAGS.vg_name_prefix_to_skip):
log.INFO("Not deleting VG %s" % entity["name"])
continue
if FLAGS.ask_for_confirmation:
choice = raw_input(
"WARNING: Deleting VG %s. \nOK to continue?(y/n):" %
entity["name"])
if choice != 'y':
log.INFO("User did not respond with \"y\".Skippping VG %s !!" %
entity["name"])
continue
params = {}
params["attached_clients"] = []
params["uuid"] = entity["uuid"]
params["name"] = entity["name"]
auth = (FLAGS.username, FLAGS.password)
print "Processing VG %s" % params["name"]
response = requests.put(url,
data=json.dumps(params),
auth=auth,
verify=False)
if not response.ok:
log.ERROR(
"Could not update the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
url = "https://%s%s/%s" % (SERVER, VG_LIST_URL, entity["uuid"])
auth = (FLAGS.username, FLAGS.password)
response = requests.get(url, auth=auth, verify=False)
if not response.ok:
log.ERROR("Could not get the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
for attachment in response.json().get("attachment_list", []):
if "vm_uuid" in attachment.keys():
url = "https://%s%s" % (SERVER,
VG_DETACH_URL.format(entity["uuid"]))
params = {"vm_uuid": attachment["vm_uuid"]}
response = requests.post(url,
auth=auth,
data=json.dumps(params),
verify=False)
if not response.ok:
log.ERROR(
"Could not remove Vm attachment from the VG. Error code %s "
"Url: %s\nAborting!!\n" % (response, url))
sys.exit(1)
url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"]))
auth = (FLAGS.username, FLAGS.password)
# Sleeping a bit to be a little easy on acropolis.
time.sleep(1)
log.INFO("Deleting VG %s" % entity["name"])
response = requests.delete(url, auth=auth, verify=False)
if not response.ok:
log.ERROR(
"Could not delete the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
return True
