def node_disruption(self, sync=None, finish_execution=None):
self.sync = sync
self.finish_execution = finish_execution
infra.display_on_terminal(self, "Entering Node Disruption plugin")
table_name = "Node Disruption"
infra.create_report_table(self, table_name)
infra.add_table_headers(self, table_name,
["Node", "IP", "TimeStamp",
"Status of Disruption"])
infra.display_on_terminal(self, "Entering Process Disruption plugin")
input_args_dict = self.get_input_arguments()
node_name = input_args_dict.keys()[0]
input_args = input_args_dict.get(node_name, None)
host_config = infra.get_openstack_config()
if input_args:
print "Inpt " + str(input_args)
role = input_args.get('role', None)
nodes_to_be_disrupted = []
for node in host_config:
if role in host_config[node].get('role', None):
infra.display_on_terminal(self, node, " will be disrupted ")
nodes_to_be_disrupted.append(node)
node_reboot_command = "reboot -f "
if self.sync:
infra.display_on_terminal(self, "Waiting for notification")
infra.wait_for_notification(sync)
infra.display_on_terminal(self, "Received notification, Starting")
ha_interval = self.get_ha_interval()
for i in range(1):
#while infra.is_execution_completed(self.finish_execution) is False:
# for node in nodes_to_be_disrupted:
node = nodes_to_be_disrupted[0]
ip = host_config.get(node, None).get('ip', None)
user = host_config.get(node, None).get('user', None)
password = host_config.get(node, None).get('password', None)
infra.display_on_terminal(self, "IP: ", ip, " User: "******" Pwd: ", password)
infra.display_on_terminal(self, "Executing ",
node_reboot_command)
code, out, error = infra.ssh_and_execute_command(ip, user,
password,
node_reboot_command)
if error:
infra.display_on_terminal(self, "Error ", error,
"color=red")
infra.display_on_terminal(self, "waiting for ", ip, " to "
"come "
"online")
if infra.wait_for_ping(ip, 240, 10):
infra.display_on_terminal(self, "Node ", ip,
" is online", "color=green")
infra.display_on_terminal(self, "Will sleep for interval ",
str(ha_interval))
#time.sleep(ha_interval)
infra.add_table_rows(self, table_name, [[node,
ip,
utils.get_timestamp(),
HAConstants.OKGREEN +
'Rebooted' +
HAConstants.ENDC]])
# bring it back to stable state
infra.display_on_terminal(self, "Waiting for the node to become stable")
if infra.wait_for_ping(ip, 240, 10):
infra.display_on_terminal(self, "Node ", ip, " is in stable state",
"color=green")
infra.display_on_terminal(self, "Finishing Node Disruption")
def vm_disruption(self, sync=None, finish_execution=None):
self.sync = sync
self.finish_execution = finish_execution
infra.display_on_terminal(self, "Entering VM Disruption plugin")
table_name = "VM Disruption"
infra.create_report_table(self, table_name)
infra.add_table_headers(self, table_name,
["VM", "IP", "TimeStamp",
"Status of Disruption"])
infra.display_on_terminal(self, "Entering Process Disruption plugin")
input_args_dict = self.get_input_arguments()
print '========',input_args_dict
node_name = input_args_dict.keys()[0]
input_args = input_args_dict.get(node_name, None)
host_config = infra.get_openstack_config()
if input_args:
print "Inpt " + str(input_args)
role = input_args.get('role', None)
print '*'*20
print 'host_config ==',host_config
print 'input_args ==',input_args
print '*'*20
# nodes_to_be_disrupted = []
for node in host_config:
if role == host_config[node].get('role', None):
jump_host = node
print jump_host
nodes_to_be_disrupted = input_args.get('name')
print nodes_to_be_disrupted
node_reboot_command = "reboot"
if self.sync:
infra.display_on_terminal(self, "Waiting for notification")
infra.wait_for_notification(sync)
infra.display_on_terminal(self, "Received notification, Starting")
openrc = host_config.get(jump_host, None).get('openrc', None)
password = host_config.get(jump_host, None).get('password', None)
print openrc
ha_interval = self.get_ha_interval()
# for i in range(1):
while infra.is_execution_completed(self.finish_execution) is False:
for node in nodes_to_be_disrupted:
# node = nodes_to_be_disrupted[0]
# ip = host_config.get(node, None).get('ip', None)
# user = host_config.get(node, None).get('user', None)
# password = host_config.get(node, None).get('password', None)
ip = node
# openrc = host_config.get(node, None).get('openrc', None)
# password = host_config.get(node, None).get('password', None)
infra.display_on_terminal(self, "IP: ", ip, " openrc: ",
openrc)
infra.display_on_terminal(self, "Executing ",
node_reboot_command)
# Using nova api performing the vm stop operation
cred = credentials.Credentials(openrc, password,'no_env')
try:
nova = nova_api.NovaHealth(cred.get_nova_credentials_v2())
ret = nova.nova_stop_server(ip)
time.sleep(ha_interval)
infra.display_on_terminal(self, "Rebooting ",ip)
nova.nova_start_server(ip)
error = []
except Exception,error:
pass
if error:
infra.display_on_terminal(self, "Error ", error,
"color=red")
infra.display_on_terminal(self, "waiting for ", ip, " to "
"come "
"online")
if infra.wait_for_ping(ip, 240, 10):
infra.display_on_terminal(self, "Node ", ip,
" is online", "color=green")
infra.display_on_terminal(self, "Will sleep for interval ",
str(ha_interval))
#time.sleep(ha_interval)
infra.add_table_rows(self, table_name, [[node,
ip,
utils.get_timestamp(),
HAConstants.OKGREEN +
'Rebooted' +
HAConstants.ENDC]])
def vm_disruption(self, sync=None, finish_execution=None):
self.sync = sync
self.finish_execution = finish_execution
infra.display_on_terminal(self, "Entering VM Disruption plugin")
table_name = "VM Disruption"
infra.create_report_table(self, table_name)
infra.add_table_headers(
self, table_name,
["VM", "IP", "TimeStamp", "Status of Disruption"])
infra.display_on_terminal(self, "Entering Process Disruption plugin")
input_args_dict = self.get_input_arguments()
print '========', input_args_dict
node_name = input_args_dict.keys()[0]
input_args = input_args_dict.get(node_name, None)
host_config = infra.get_openstack_config()
if input_args:
print "Inpt " + str(input_args)
role = input_args.get('role', None)
print '*' * 20
print 'host_config ==', host_config
print 'input_args ==', input_args
print '*' * 20
# nodes_to_be_disrupted = []
for node in host_config:
if role == host_config[node].get('role', None):
jump_host = node
print jump_host
nodes_to_be_disrupted = input_args.get('name')
print nodes_to_be_disrupted
node_reboot_command = "reboot"
if self.sync:
infra.display_on_terminal(self, "Waiting for notification")
infra.wait_for_notification(sync)
infra.display_on_terminal(self, "Received notification, Starting")
openrc = host_config.get(jump_host, None).get('openrc', None)
password = host_config.get(jump_host, None).get('password', None)
print openrc
ha_interval = self.get_ha_interval()
# for i in range(1):
while infra.is_execution_completed(self.finish_execution) is False:
for node in nodes_to_be_disrupted:
# node = nodes_to_be_disrupted[0]
# ip = host_config.get(node, None).get('ip', None)
# user = host_config.get(node, None).get('user', None)
# password = host_config.get(node, None).get('password', None)
ip = node
# openrc = host_config.get(node, None).get('openrc', None)
# password = host_config.get(node, None).get('password', None)
infra.display_on_terminal(self, "IP: ", ip, " openrc: ",
openrc)
infra.display_on_terminal(self, "Executing ",
node_reboot_command)
# Using nova api performing the vm stop operation
cred = credentials.Credentials(openrc, password, 'no_env')
try:
nova = nova_api.NovaHealth(cred.get_nova_credentials_v2())
ret = nova.nova_stop_server(ip)
time.sleep(ha_interval)
infra.display_on_terminal(self, "Rebooting ", ip)
nova.nova_start_server(ip)
error = []
except Exception, error:
pass
if error:
infra.display_on_terminal(self, "Error ", error,
"color=red")
infra.display_on_terminal(self, "waiting for ", ip, " to "
"come "
"online")
if infra.wait_for_ping(ip, 240, 10):
infra.display_on_terminal(self, "Node ", ip, " is online",
"color=green")
infra.display_on_terminal(self, "Will sleep for interval ",
str(ha_interval))
#time.sleep(ha_interval)
infra.add_table_rows(self, table_name, [[
node, ip,
utils.get_timestamp(),
HAConstants.OKGREEN + 'Rebooted' + HAConstants.ENDC
]])
def jump_host_disruption(self, sync=None, finish_execution=None):
self.sync = sync
self.finish_execution = finish_execution
infra.display_on_terminal(self, "Entering Jump Host Disruption plugin")
table_name = "Jump host Disruption"
infra.create_report_table(self, table_name)
infra.add_table_headers(self, table_name,
["VM", "IP", "TimeStamp",
"Status of Disruption"])
infra.display_on_terminal(self, "Entering Process Disruption plugin")
input_args_dict = self.get_input_arguments()
node_name = input_args_dict.keys()[0]
input_args = input_args_dict.get(node_name, None)
host_config = infra.get_openstack_config()
print "*"*20
print input_args_dict
print "input_args ==>",input_args
print "host_config ==>",host_config
nodes_to_be_disrupted = input_args.get('name',[])
if input_args:
print "Inpt " + str(input_args)
role = input_args.get('role', None)
# jump_hosts = []
for node in host_config:
if role in host_config[node].get('role', None):
jump_host = node
# jump_hosts.append(node)
print "###############",jump_host
node_reboot_command = "reboot -f"
if self.sync:
infra.display_on_terminal(self, "Waiting for notification")
infra.wait_for_notification(sync)
infra.display_on_terminal(self, "Received notification, Starting")
ha_interval = self.get_ha_interval()
# jump host details
jump_host_ip = host_config.get(node, None).get('ip', None)
user = host_config.get(node,None).get('user',None)
password = host_config.get(node,None).get('password',None)
#TODO - if its more than one jump host
# Write into txt file to pass via ansible playbook
f = open('/tmp/remote_ips','w+')
for ip in nodes_to_be_disrupted:
f.write(ip+'\n')
f.close()
while infra.is_execution_completed(self.finish_execution) is False:
# for node in nodes_to_be_disrupted:
# node = nodes_to_be_disrupted[0]
# ip = host_config.get(node, None).get('ip', None)
# user = host_config.get(node, None).get('user', None)
# password = host_config.get(node, None).get('password', None)
ip = node
# openrc = host_config.get(node, None).get('openrc', None)
# password = host_config.get(node, None).get('password', None)
infra.display_on_terminal(self, "Nodes to be disrupted: ", str(nodes_to_be_disrupted), " Jump host: ",
jump_host_ip)
infra.display_on_terminal(self, "Executing ",
node_reboot_command)
print "*"*50
print "user :"******"password :"******"jump_host_ip :",jump_host_ip
ret = AnsibleRunner(jump_host_ip,user,password).execute_on_remote()
print ret
# parse the output for report
# node_list = os.walk('/tmp/hainfra').next()[1]
# output_objs = eval(open('/tmp/hainfra/'+node+'/tmp/output','r').read())
output_objs = eval(open('/tmp/hainfra/output','r').read())
print output_objs
for results in output_objs:
error = []
for (hostname, result) in results['contacted'].items():
if 'failed' in result:
print "%s >>> %s" % (hostname, result['msg'])
error = result['msg']
if error:
infra.display_on_terminal(self, "Error ", error,
"color=red")
infra.display_on_terminal(self, "waiting for ", hostname, " to "
"come "
"online")
if infra.wait_for_ping(hostname, 240, 5):
infra.display_on_terminal(self, "Node ", hostname,
" is online", "color=green")
infra.display_on_terminal(self, "Will sleep for interval ",
str(ha_interval))
#time.sleep(ha_interval)
if not error:
infra.add_table_rows(self, table_name, [[jump_host_ip,
hostname,
utils.get_timestamp(),
HAConstants.OKGREEN +
'Rebooted' +
HAConstants.ENDC]])
else:
infra.add_table_rows(self, table_name, [[jump_host_ip,
hostname,
utils.get_timestamp(),
HAConstants.FAIL +
str(error)+
HAConstants.ENDC]])
# bring it back to stable state
'''
infra.display_on_terminal(self, "Waiting for the node to become stable")
if infra.wait_for_ping(hostname, 240, 10):
infra.display_on_terminal(self, "Node ", hostname, " is in stable state",
"color=green")
'''
infra.display_on_terminal(self, "Finishing Node Disruption")
def jump_host_disruption(self, sync=None, finish_execution=None):
self.sync = sync
self.finish_execution = finish_execution
infra.display_on_terminal(self, "Entering Jump Host Disruption plugin")
table_name = "Jump host Disruption"
infra.create_report_table(self, table_name)
infra.add_table_headers(
self, table_name,
["VM", "IP", "TimeStamp", "Status of Disruption"])
infra.display_on_terminal(self, "Entering Process Disruption plugin")
input_args_dict = self.get_input_arguments()
node_name = input_args_dict.keys()[0]
input_args = input_args_dict.get(node_name, None)
host_config = infra.get_openstack_config()
print "*" * 20
print input_args_dict
print "input_args ==>", input_args
print "host_config ==>", host_config
nodes_to_be_disrupted = input_args.get('name', [])
if input_args:
print "Inpt " + str(input_args)
role = input_args.get('role', None)
# jump_hosts = []
for node in host_config:
if role in host_config[node].get('role', None):
jump_host = node
# jump_hosts.append(node)
print "###############", jump_host
node_reboot_command = "reboot -f"
if self.sync:
infra.display_on_terminal(self, "Waiting for notification")
infra.wait_for_notification(sync)
infra.display_on_terminal(self, "Received notification, Starting")
ha_interval = self.get_ha_interval()
# jump host details
jump_host_ip = host_config.get(node, None).get('ip', None)
user = host_config.get(node, None).get('user', None)
password = host_config.get(node, None).get('password', None)
#TODO - if its more than one jump host
# Write into txt file to pass via ansible playbook
f = open('/tmp/remote_ips', 'w+')
for ip in nodes_to_be_disrupted:
f.write(ip + '\n')
f.close()
while infra.is_execution_completed(self.finish_execution) is False:
# for node in nodes_to_be_disrupted:
# node = nodes_to_be_disrupted[0]
# ip = host_config.get(node, None).get('ip', None)
# user = host_config.get(node, None).get('user', None)
# password = host_config.get(node, None).get('password', None)
ip = node
# openrc = host_config.get(node, None).get('openrc', None)
# password = host_config.get(node, None).get('password', None)
infra.display_on_terminal(self, "Nodes to be disrupted: ",
str(nodes_to_be_disrupted),
" Jump host: ", jump_host_ip)
infra.display_on_terminal(self, "Executing ", node_reboot_command)
print "*" * 50
print "user :"******"password :"******"jump_host_ip :", jump_host_ip
ret = AnsibleRunner(jump_host_ip, user,
password).execute_on_remote()
print ret
# parse the output for report
# node_list = os.walk('/tmp/hainfra').next()[1]
# output_objs = eval(open('/tmp/hainfra/'+node+'/tmp/output','r').read())
output_objs = eval(open('/tmp/hainfra/output', 'r').read())
print output_objs
for results in output_objs:
error = []
for (hostname, result) in results['contacted'].items():
if 'failed' in result:
print "%s >>> %s" % (hostname, result['msg'])
error = result['msg']
if error:
infra.display_on_terminal(self, "Error ", error,
"color=red")
infra.display_on_terminal(self, "waiting for ", hostname,
" to "
"come "
"online")
if infra.wait_for_ping(hostname, 240, 5):
infra.display_on_terminal(self, "Node ", hostname,
" is online", "color=green")
infra.display_on_terminal(self, "Will sleep for interval ",
str(ha_interval))
#time.sleep(ha_interval)
if not error:
infra.add_table_rows(self, table_name, [[
jump_host_ip, hostname,
utils.get_timestamp(),
HAConstants.OKGREEN + 'Rebooted' + HAConstants.ENDC
]])
else:
infra.add_table_rows(self, table_name, [[
jump_host_ip, hostname,
utils.get_timestamp(),
HAConstants.FAIL + str(error) + HAConstants.ENDC
]])
# bring it back to stable state
'''
infra.display_on_terminal(self, "Waiting for the node to become stable")
if infra.wait_for_ping(hostname, 240, 10):
infra.display_on_terminal(self, "Node ", hostname, " is in stable state",
"color=green")
'''
infra.display_on_terminal(self, "Finishing Node Disruption")