def monitor_node_removal(service_vm_id):
progress_info_id = ProgressInfoProto.ProgressInfoId()
progress_info_id.operation = ProgressInfoProto.kRemove
progress_info_id.entity_type = ProgressInfoProto.kNode
progress_info_id.entity_id = str(service_vm_id)
progress_interface = ProgressMonitorInterface()
proto = progress_interface.lookup(progress_info_id)
num_tasks = len(proto.progress_task_list)
while True:
pending_tasks = 0
for task_id in range(num_tasks):
task = proto.progress_task_list[task_id]
if (task.progress_status == ProgressInfoProto.kQueued) or (
task.progress_status == ProgressInfoProto.kRunning):
pending_tasks += 1
elif (task.progress_status == ProgressInfoProto.kFailed) or (
task.progress_status == ProgressInfoProto.kAborted):
log.ERROR("Error removing node %s from cluster" %
service_vm_id)
log.INFO("Node has to be removed manually")
return 1
elif task.progress_status == ProgressInfoProto.kSucceeded:
log.INFO("Node %s successfully removed" % service_vm_id)
return 0
log.INFO("Waiting for remove node to finish on node vm id %s" %
service_vm_id)
time.sleep(10)
def remove_node(node_uuid):
cmd = "ncli cluster rm-start id=%s" % node_uuid
ret, out, err = timed_command(cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when %s was called on %s" %
(err, out, ret, cmd, node_uuid))
log.WARNING("Please remove this node manually")
return 1
else:
log.INFO("NCLI remove returned %s for node %s" % (out, node_uuid))
log.INFO("You can check the progress by calling --monitor_removal "
"<service_vm_id>")
return 0
def show_help():
usage = """Usage:
The script takes the following as an argument.
Usage:
./mixed_cluster_setup.py <action> <args>
Action: The action can be one of the following:
1. --remove_node: Removes the node assuming it to be a HCI node.
2. --kill_cvm: Shutsdown and undefines CVM on the node to be added
as CO.
3. --monitor_removal: Returns if node removal was success/failure and
waits till removal is complete. This is only removal of HCI nodes.
4. --add_node: Adds node as HCI or CO.
5. --remove_co_node: Logs how to remove compute only node.
6. --start_cvm: Start cvm on the compute only host.
Arguments for each of the above actions:
1. --remove_node --node_uuid=<node-uuid>
2. --kill_cvm --cvm_ip=<cvm_ip> --host_ip=<host_ip>
3. --monitor_removal --service_vm_id=<service_vm_id>
4. --add_node --node_uuid=<node_uuid> --cvm_ip=<cvm_ip> or
--host_ip = <host_ip> depending on if the node is HCI or CO.
5. --remove_co_node: Logs how to remove node.
6. --start_cvm --host_ip=<host_ip>
Ideally one would start with a jarvis created cluster and use the script to
perform the following actions:
1. Remove a node which has to be added as compute only.
2. Monitor removal.
3. Once removal is a success, kill the cvm on the node.
4. Add the node as compute only.
"""
log.INFO(usage)
sys.exit(0)
def main():
if FLAGS.help:
show_help()
# HCI node removal
if FLAGS.remove_node:
if not FLAGS.node_uuid:
log.ERROR("Node uuid missing. Pass node uuid to initiate removal")
return 1
else:
return remove_node(FLAGS.node_uuid)
# Monitor removal.
if FLAGS.monitor_removal:
if not FLAGS.service_vm_id:
log.ERROR("Service vm id missing")
return 1
else:
return monitor_node_removal(FLAGS.service_vm_id)
# Destroy CVM
if FLAGS.kill_cvm:
if not FLAGS.cvm_ip:
log.ERROR("Node cvm ip is missing")
return 1
elif not FLAGS.host_ip:
log.ERROR("Host ip missing")
return 1
else:
return kill_cvm(FLAGS.cvm_ip, FLAGS.host_ip)
# Add the node.
if FLAGS.add_node:
if not FLAGS.node_uuid:
log.ERROR("Node uuid required to add co node")
return 1
if (not FLAGS.cvm_ip) and not (FLAGS.host_ip):
log.ERROR("Either cvm ip or host ip is required to add the node")
return 1
if FLAGS.cvm_ip:
node_ip = FLAGS.cvm_ip
compute_only = False
else:
node_ip = FLAGS.host_ip
compute_only = True
return add_node(FLAGS.node_uuid, node_ip, compute_only)
if FLAGS.remove_co_node:
log.INFO("Please edit zeus config manually using edit_zeus "
"--editor=/usr/bin/vim. Remove the node info from "
"compute_node_list and management_server_list")
return 0
if FLAGS.start_cvm:
if not FLAGS.host_ip:
log.ERROR("Please provide host ip to start cvm on")
return 1
return start_cvm(FLAGS.host_ip)
def add_node(node_uuid, node_ip, compute_only):
# Add node.
from cluster.utils.genesis_client import GenesisApiClient
client = GenesisApiClient()
ret, error = client.make_rpc("ClusterManager",
"add_node", {
"node_uuid": node_uuid,
"node_svm_ip": node_ip,
"compute_only": compute_only
},
svm_ips=["localhost"])
if not ret:
log.ERROR("Add node for %s failed with error %s" % (node_ip, error))
log.INFO("The node will have to be manually added back")
return 1
else:
log.INFO("Node %s successfully added" % node_ip)
return 0
def start_cvm(hypervisor_ip):
# Get hostname to be used when killing the CVM later.
log.INFO("Starting cvm on host %s" % hypervisor_ip)
host_ssh_client = SSHClient(hypervisor_ip,
FLAGS.hypervisor_username,
password=FLAGS.default_host_password)
ret, out, err = host_ssh_client.execute("hostname")
if ret != 0:
log.ERROR("Error retrieving hyp hostname from ip %s, error %s"
"ret %s" % (hypervisor_ip, err, ret))
return 1
hostname = out.strip()
# Kill the CVM on each of the removed nodes.
define_cmd = "virsh define /root/NTNX-CVM.xml"
start_cmd = "virsh start NTNX-%s-CVM" % hostname
autostart_cmd = "virsh autostart NTNX-%s-CVM" % hostname
ret, out, err = host_ssh_client.execute(define_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when defining the CVM"
" on host %s" % (err, out, ret, hypervisor_ip))
log.INFO("Please start the CVM manually on this host")
return 1
log.INFO("Define cmd for cvm %s success")
ret, out, err = host_ssh_client.execute(start_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when starting the CVM"
"on host %s" % (ret, out, err, hypervisor_ip))
log.INFO("Please start the CVM manually on this host")
return 1
log.INFO("Start cmd for cvm %s success")
ret, out, err = host_ssh_client.execute(autostart_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when autostarting the CVM"
"on host %s" % (ret, out, err, hypervisor_ip))
log.INFO("Please autostart the CVM manually on this host")
return 1
log.INFO("CVM started on host %s" % hypervisor_ip)
# Remove the configured file on the host which is used to indicate it is part
# of a cluster.
rm_cmd = "rm -f /root/configured"
ret, out, err = host_ssh_client.execute(rm_cmd)
if ret != 0:
log.ERROR(
"Cannot remove /root/configured file from the host. Please "
"remove the file manually in order to add it back as a CO node")
return 1
return 0
def bot_respond(self, sender_id, resp):
if (resp.IsTextOnly()):
self._bot.send_text_message(sender_id, resp.text)
log.INFO("Sending Text")
return None
if (resp.image_url):
self._bot.send_image_url(sender_id, resp.image_url)
self._bot.send_text_message(sender_id, resp.text)
log.INFO("Sending Text N Image")
return None
if (resp.video_url):
self._bot.send_video_url(sender_id, resp.video_url)
self._bot.send_text_message(sender_id, resp.text)
log.INFO("Sending Text N Video")
return None
# Must be Audio URL
self._bot.send_audio_url(sender_id, resp.audio_url)
self._bot.send_text_message(sender_id, resp.text)
log.INFO("Sending Text N Audio")
def _webhook_message(self, payload):
"""
Handle messages.
"""
log.INFO("===> Handle message %s" % payload)
# Process webhook messages.
for sender_id, msg_info in self._iter_messaging_events(payload):
if msg_info is None:
return "ok"
try:
handler = message_funcs[msg_info['type']]
except KeyError:
log.DEBUG("Unrecognized message type %s" %
(msg_info['type'], ))
continue
resp = handler(self, sender_id, msg_info['data'])
if resp is None:
continue
log.INFO(
"RSP***> Text:'{%s}', Image:'{%s}', Video:'{%s}', Audio:'{%s}'"
% (resp.text, resp.image_url, resp.video_url, resp.audio_url))
self.bot_respond(sender_id, resp)
def _message_text(self, sender_id, text):
"""
Handle a text message.
Args:
sender_id (int64): Sender ID.
text (string): Text message.
Returns:
response : Response from machine learning.
"""
log.INFO("MSG---> %s" % text)
resp = self._agent_client.process(sender_id, text)
return resp
def _kill_cvm(cvm_ip, host_ssh_client, hostname):
# Kill the CVM on each of the removed nodes.
shutdown_cmd = "virsh shutdown NTNX-%s-CVM" % hostname
undefine_cmd = "virsh undefine NTNX-%s-CVM" % hostname
ret, out, err = host_ssh_client.execute(shutdown_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when shutting down the CVM"
" %s on host" % (err, out, ret, cvm_ip))
log.INFO("Please shutdown the CVM manually on this host")
return 1
log.INFO("Shutdown cmd for cvm %s success" % cvm_ip)
ret, out, err = host_ssh_client.execute(undefine_cmd)
if ret != 0:
log.ERROR("Error %s, out %s, ret %s when undefining the CVM"
" %s on host" % (ret, out, err, cvm_ip))
log.INFO("Please undefine the CVM manually on this host")
return 1
log.INFO("Undefine cmd for cvm %s success" % cvm_ip)
return 0
def bot_pi(self):
if request.method == 'GET':
if (request.args.get("hub.verify_token") !=
FLAGS.bot_verify_token):
return None
return request.args.get("hub.challenge")
if request.method == 'POST':
log.INFO("%s" % request.json)
payload = request.get_data()
webhook_type = self._get_type_from_payload(payload)
# Process webhook events.
try:
handler = webhook_funcs[webhook_type]
except KeyError:
log.DEBUG("Unrecognized webhook type %s" % (webhook_type, ))
return "success"
handler(self, payload)
return "success"
def _message_audio(self, sender_id, audio_url):
"""
Handle an audio message.
Args:
sender_id (int64): Sender ID.
audio_url (string): Audio url.
Returns:
response : Response from machine learning.
"""
# Get text from audio.
try:
text = s2t.transcribe(audio_url)
if text == "" or text == None:
return
except Exception as exc:
return PInfo(text="I'm sorry. I could not follow your question")
text = text.decode('utf-8')
log.INFO("MSG---> %s" % text)
resp = self._agent_client.process(sender_id, text)
return resp
def __get_arithmos_entity_id(self, arithmos_entity_type, primary_entity):
"""
Gets the arithmos specific entity id based on arithmos_entity_type and
primary_entity
Args:
arithmos_entity_type(str): arithmos_entity_type
primary_entity(str): primary_entity
Returns:
entity_id(str)
"""
entity_proto = DataExchangeProto.EntityProto()
entity_proto.type = arithmos_entity_type
identifier = entity_proto.identifiers.add()
identifier.key_name = primary_entity.member_name
if safe_has_field(primary_entity.member_value, "bool_value"):
identifier.bool_value = primary_entity.member_value.bool_value
if safe_has_field(primary_entity.member_value, "int64_value"):
identifier.int_value = primary_entity.member_value.int64_value
if safe_has_field(primary_entity.member_value, "string_value"):
identifier.string_value = primary_entity.member_value.string_value
log.INFO(entity_proto)
entity_id = (HealthServerGlobals().get_arithmos_interface().
get_entity_id(entity_proto))
return entity_id
def kill_cvm(cvm_ip, hypervisor_ip):
# Get hostname to be used when killing the CVM later.
host_ssh_client = SSHClient(hypervisor_ip,
FLAGS.hypervisor_username,
password=FLAGS.default_host_password)
ret, out, err = host_ssh_client.execute("hostname")
if ret != 0:
log.ERROR("Error retrieving hyp hostname from ip %s, error %s"
"ret %s" % (hypervisor_ip, err, ret))
return 1
hostname = out.strip()
# Transfer the factory_config.json to the host.
cvm_ssh_client = SSHClient(cvm_ip,
"nutanix",
password=FLAGS.default_cvm_password)
with tempfile.NamedTemporaryFile() as tmp:
# Transfer from the cvm in question to a local tmp path.
ret, out, error = cvm_ssh_client.execute(
"cat %s" % FLAGS.factory_config_json_path)
if ret != 0:
log.ERROR("Unable to read the factory_config.json from node %s" %
cvm_ip)
return 1
# Modify the contents to add "mode", as foundation would do it.
factory_config_dict = json.loads(out.strip())
factory_config_dict["mode"] = "compute only"
tmp.write(json.dumps(factory_config_dict))
tmp.flush()
# Transfer from the local path to the corresponding host.
ret, out, err = host_ssh_client.transfer_to(
tmp.name, FLAGS.factory_config_json_path_on_host)
if ret != 0:
log.ERROR("Unable to transfer %s to the host %s" %
(FLAGS.factory_config_json_path, hypervisor_ip))
return 1
# Set the right permission bits.
permission_string = "644"
filepath = FLAGS.factory_config_json_path_on_host
ret = _change_permission_bits(host_ssh_client, permission_string,
filepath)
if ret != 0:
return 1
log.INFO("Transferred factory config json to the host")
# Transfer the hardware config json.
with tempfile.NamedTemporaryFile() as tmp:
# Transfer from the cvm in question to a local tmp path.
ret, out, error = cvm_ssh_client.execute(
"cat %s" % FLAGS.hardware_config_json_path)
if ret != 0:
log.ERROR("Unable to read the hardware_config.json from node %s" %
cvm_ip)
return 1
tmp.write(out.strip())
tmp.flush()
# Transfer from the local path to the corresponding host.
ret, out, err = host_ssh_client.transfer_to(
tmp.name, FLAGS.hardware_config_json_path_on_host)
if ret != 0:
log.ERROR("Unable to transfer %s to the host %s" %
(FLAGS.hardware_config_json_path, hypervisor_ip))
return 1
# Set the right permission bits.
permission_string = "644"
filepath = FLAGS.hardware_config_json_path_on_host
ret = _change_permission_bits(host_ssh_client, permission_string,
filepath)
if ret != 0:
return 1
log.INFO("Transferred hardware config json to the host")
return _kill_cvm(hypervisor_ip, host_ssh_client, hostname)
def process(self, client_id, query_text):
request = self._ai.text_request()
request.lang = 'en' # optional, default value equal 'en'
#request.session_id = "<SESSION ID, UNIQUE FOR EACH USER>"
request.query = query_text #"Hello"
response = request.getresponse()
data = json.loads(response.read().decode())
log.INFO("===JSON===> %s" % data)
if client_id not in sessions:
sessions[client_id] = {}
sess = sessions[client_id]
# Response already present in the intent, just return that
if ('fulfillment' in data['result']
and 'speech' in data['result']['fulfillment']
and data['result']['fulfillment']['speech']):
return PInfo(text=data['result']['fulfillment']['speech'])
# Response not present - we need to build one
action = data['result']['action']
pinfo_key = (action, )
if 'parameters' not in data['result']:
return pinfo_lookup(pinfo_key)
# parameters exist
params = data['result']['parameters']
if 'product' in params and params['product'] != '':
prod = params['product']
log.INFO("[session {%s}]: product name is '{%s}'" %
(client_id, prod))
if 'prod' not in sess or sess['prod'] != prod:
sess.clear()
sess['prod'] = prod
# Add the session attribute to pinfo_key
pinfo_key += (sess.get('prod'), )
if 'Model' in params and params['Model'] != '':
model = params['Model']
log.INFO("[session {%s}]: model is '{%s}'" % (client_id, model))
if 'model' not in sess or sess['model'] != model:
if 'feature' in sess:
del sess['feature']
sess['model'] = model
# Add the session attribute to pinfo_key
pinfo_key += (sess.get('model'), )
if 'feature' in params and params['feature'] != '':
feature = params['feature']
log.INFO("[session {%s}]: feature is '{%s}'" %
(client_id, feature))
sess['feature'] = feature
# Add the session attribute to pinfo_key
pinfo_key += (sess.get('feature'), )
log.INFO("===>>>pinfo_key %s" % (pinfo_key, ))
return pinfo_lookup(pinfo_key)
def remove_vgs():
"""
List all the VGs on the cluster.
"""
url = "https://%s%s" % (SERVER, VG_LIST_URL)
auth = (FLAGS.username, FLAGS.password)
response = requests.get(url, auth=auth, verify=False)
if response.status_code == 401:
print(
"Could not get list of VGs from Prism. Wrong Credentials. "
"Aborting!!")
log.ERROR("Could not get list of VGs from Prism. Wrong Credentials. "
"Aborting!!")
sys.exit(1)
if not response.ok:
print(
"Could not get list of VGs from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response.status_code, response.url))
log.ERROR("Could not get list of VGs from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response.status_code, response.url))
sys.exit(1)
for entity in response.json()['entities']:
url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"]))
if not entity["name"].startswith(FLAGS.vg_name_prefix_to_delete):
log.INFO("Not deleting VG %s" % entity["name"])
continue
if FLAGS.vg_name_prefix_to_skip and entity["name"].startswith(
FLAGS.vg_name_prefix_to_skip):
log.INFO("Not deleting VG %s" % entity["name"])
continue
if FLAGS.ask_for_confirmation:
choice = raw_input(
"WARNING: Deleting VG %s. \nOK to continue?(y/n):" %
entity["name"])
if choice != 'y':
log.INFO("User did not respond with \"y\".Skippping VG %s !!" %
entity["name"])
continue
params = {}
params["attached_clients"] = []
params["uuid"] = entity["uuid"]
params["name"] = entity["name"]
auth = (FLAGS.username, FLAGS.password)
print "Processing VG %s" % params["name"]
response = requests.put(url,
data=json.dumps(params),
auth=auth,
verify=False)
if not response.ok:
log.ERROR(
"Could not update the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
url = "https://%s%s/%s" % (SERVER, VG_LIST_URL, entity["uuid"])
auth = (FLAGS.username, FLAGS.password)
response = requests.get(url, auth=auth, verify=False)
if not response.ok:
log.ERROR("Could not get the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
for attachment in response.json().get("attachment_list", []):
if "vm_uuid" in attachment.keys():
url = "https://%s%s" % (SERVER,
VG_DETACH_URL.format(entity["uuid"]))
params = {"vm_uuid": attachment["vm_uuid"]}
response = requests.post(url,
auth=auth,
data=json.dumps(params),
verify=False)
if not response.ok:
log.ERROR(
"Could not remove Vm attachment from the VG. Error code %s "
"Url: %s\nAborting!!\n" % (response, url))
sys.exit(1)
url = "https://%s%s" % (SERVER, VG_UPDATE_URL.format(entity["uuid"]))
auth = (FLAGS.username, FLAGS.password)
# Sleeping a bit to be a little easy on acropolis.
time.sleep(1)
log.INFO("Deleting VG %s" % entity["name"])
response = requests.delete(url, auth=auth, verify=False)
if not response.ok:
log.ERROR(
"Could not delete the VG from Prism. Error code %s Url: %s"
"\nAborting!!\n" % (response, url))
sys.exit(1)
return True
