def mount_fs(self, *args, **kwargs):
banner("Mount FS to Mount Point")
self._load_kwargs(kwargs)
try:
print("Kwargs are: {}".format(kwargs))
#inet_ip = CephCluster().get_ceph_inet_ip(**kwargs)
#print("Inet IP is: {}".format(inet_ip))
#Maps fs
cmd= "sudo mount -t ceph {}:/ /mnt/{} -o name=admin,secret='ceph-authtool -p /etc/ceph/ceph.client.admin.keyring'".format(self.inet_ip,self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.user,linux_password=self.password)
print("cmd1: {} executed successfully and status is: {}".format(cmd, status))
time.sleep(1)
cmd= "sudo mount| grep test_fs_mnt"
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.user,linux_password=self.password)
print("cmd2: {} executed successfully and status is: {}".format(cmd, status))
serialised_status = self._serialize_response(time.time(), status)
cmd_output = str(serialised_status['Result']['stdout']).replace('\n', '').strip()
if '/mnt/{}'.format(self.mount_folder_name) in cmd_output:
print("Found in string")
else:
return "Error: {} file not found".format(self.mount_folder_name)
return "OK"
except Exception as e:
trace("Error in mount_fs: {}".format(e))
def pxe_boot_node(self, *args, **kwargs):
banner("PCC.Pxe-boot node")
self._load_kwargs(kwargs)
logger.console("kwargs are: {}".format(kwargs))
cmd_1 = "ipmitool -I lanplus -H {} -U ADMIN -P ADMIN chassis bootdev pxe".format(
self.bmc_ip)
logger.console("command 1 hitting is: {}".format(cmd_1))
cmd_1_output = cli_run(cmd=cmd_1,
host_ip=self.host_ip,
linux_user=self.username,
linux_password=self.password)
logger.console("cmd_1_output:{}".format(cmd_1_output))
cmd_2 = "ipmitool -I lanplus -H {} -U ADMIN -P ADMIN chassis power cycle".format(
self.bmc_ip)
logger.console("command 2 hitting is: {}".format(cmd_2))
cmd_2_output = cli_run(cmd=cmd_2,
host_ip=self.host_ip,
linux_user=self.username,
linux_password=self.password)
if cmd_1_output and cmd_2_output:
return "OK"
else:
return "Error: Pxe-boot failed over server: {}".format(self.bmc_ip)
def untar_file(self, *arg, **kwargs):
banner("PCC.PhoneHome Untar File")
self._load_kwargs(kwargs)
print("Kwargs:" + str(kwargs))
trace("Kwargs:" + str(kwargs))
try:
#tar_file_name = self.fetch_tar_file_detail(**kwargs)[0]['name']
tar_file_name = self.fetch_tar_file_detail(**kwargs)['name']
cmd = "sudo mkdir /home/pcc/platina-cli-ws/phone-home"
cmd1 = "sudo mkdir /home/pcc/platina-cli-ws/phone-home/without_ssl"
cmd2 = "sudo tar -C /home/pcc/platina-cli-ws/phone-home/without_ssl -xvf /home/pcc/storage-s3/minio/volume/data/phone-home/{}".format(
tar_file_name)
create_phone_home_folder = cli_run(self.host_ip, self.user,
self.password, cmd)
create_folder = cli_run(self.host_ip, self.user, self.password,
cmd1)
untar_file = cli_run(self.host_ip, self.user, self.password, cmd2)
trace("cmd: {} executed and status is :{}".format(
cmd, str(create_phone_home_folder)))
trace("cmd1: {} executed and status is :{}".format(
cmd1, str(create_folder)))
trace("cmd2: {} executed and status is :{}".format(
cmd2, str(untar_file)))
print("cmd: {} executed and status is :{}".format(
cmd, str(create_phone_home_folder)))
print("cmd1: {} executed and status is :{}".format(
cmd1, str(create_folder)))
print("cmd2: {} executed and status is :{}".format(
cmd2, str(untar_file)))
return "OK"
except Exception as e:
return "Exception occured in untar file:{}".format(e)
def map_rbd(self, *args, **kwargs):
banner("Map RBD")
self._load_kwargs(kwargs)
try:
print("Kwargs are: {}".format(kwargs))
#inet_ip = CephCluster().get_ceph_inet_ip(**kwargs)
#print("Inet IP is : {}".format(inet_ip))
#Maps rbd0
cmd= "sudo rbd map {} --pool {} --name client.admin -m {} -k /etc/ceph/ceph.client.admin.keyring".format(self.name, self.pool_name, self.inet_ip)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.username,linux_password=self.password)
trace("cmd1: {} executed successfully and status is:{}".format(cmd,str(status)))
print("cmd1: {} executed successfully and status is:{}".format(cmd,str(status)))
time.sleep(2)
#mkfs.ext4 rbd0 command execution
cmd= "sudo mkfs.ext4 /dev/rbd0"
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.username,linux_password=self.password)
trace("cmd2: {} executed successfully and status is:{}".format(cmd,str(status)))
print("cmd2: {} executed successfully and status is:{}".format(cmd,str(status)))
time.sleep(2)
return "OK"
except Exception as e:
trace("Error in map_rbd: {}".format(e))
def check_fs_mount(self, *args, **kwargs):
banner("PCC.Check FS Mount on other server")
self._load_kwargs(kwargs)
try:
print("Kwargs are: {}".format(kwargs))
print("username is '{}' and password is: '{}'".format(self.user,self.password))
#inet_ip = CephCluster().get_ceph_inet_ip(**kwargs)
cmd= "sudo mkdir /mnt/{}".format(self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.user,linux_password=self.password)
print("cmd1: {} executed successfully and status is: {}".format(cmd, status))
cmd= "sudo mount -t ceph {}:/ /mnt/{} -o name=admin,secret='ceph-authtool -p /etc/ceph/ceph.client.admin.keyring'".format(self.inet_ip,self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.user,linux_password=self.password)
print("cmd2: {} executed successfully and status is: {}".format(cmd, status))
cmd= "sudo ls /mnt/{}".format(self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.user,linux_password=self.password)
print("cmd3: {} executed successfully and status is: {}".format(cmd, status))
serialised_status = self._serialize_response(time.time(), status)
cmd_output = str(serialised_status['Result']['stdout']).replace('\n', '').strip()
if '{}'.format(self.dummy_file_name) in cmd_output:
print("Data Found in output")
else:
return "Error: '{}' file not found".format(self.dummy_file_name)
return "OK"
except Exception as e:
trace("Error in check_fs_mount: {}".format(e))
def unmount_and_unmap_rbd(self, *args, **kwargs):
banner("PCC.Unmount and Unmap RBD")
self._load_kwargs(kwargs)
try:
print("Kwargs are: {}".format(kwargs))
cmd= "sudo umount /mnt/{}".format(self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.username,linux_password=self.password)
print("cmd1: {} executed successfully and status is : {}".format(cmd,status))
trace("cmd1: {} executed successfully and status is : {}".format(cmd,status))
time.sleep(60*1) # Sleep for 2 minutes
cmd= "sudo rbd unmap /dev/rbd0"
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.username,linux_password=self.password)
print("cmd2: {} executed successfully and status is : {}".format(cmd,status))
trace("cmd2: {} executed successfully and status is : {}".format(cmd,status))
time.sleep(60*2) # Sleep for 2 minutes
cmd= "sudo rm -rf /mnt/{}/".format(self.mount_folder_name)
status = cli_run(cmd=cmd, host_ip=self.hostip, linux_user=self.username,linux_password=self.password)
print("cmd3: {} executed successfully and status is : {}".format(cmd,status))
trace("cmd3: {} executed successfully and status is : {}".format(cmd,status))
time.sleep(60*2) # Sleep for 2 minutes
return "OK"
except Exception as e:
trace("Error in unmount_and_unmap_rbd: {}".format(e))
def create_dummy_file_copy_to_mount_path(self, *args, **kwargs):
banner("Create dummy file and copy to mount path")
self._load_kwargs(kwargs)
try:
print("Kwargs are: {}".format(kwargs))
#Creating dummy file
cmd = "sudo dd if=/dev/zero of={} bs={} count=1".format(
self.dummy_file_name, self.dummy_file_size)
status = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.user,
linux_password=self.password)
print("cmd4: {} executed successfully and status is: {}".format(
cmd, status))
time.sleep(2)
#Copying sample 2mb file to mount folder
cmd = "sudo cp /home/pcc/{} /mnt/{}".format(
self.dummy_file_name, self.mount_folder_name)
status = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.user,
linux_password=self.password)
print("cmd5: {} executed successfully and status is:{}".format(
cmd, status))
time.sleep(2)
return "OK"
except Exception as e:
trace(
"Error in create_dummy_file_copy_to_mount_path : {}".format(e))
def force_restart_node(self, *args, **kwargs):
self._load_kwargs(kwargs)
try:
cmd = "sudo reboot -f"
restart_cmd = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
banner("Sleeping")
time.sleep(int(self.time_to_wait))
banner("Done sleeping")
cmd = "ping {} -c 4".format(self.hostip)
restart_up_status = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
serialised_restart_up_status = self._serialize_response(
time.time(), restart_up_status)
print("serialised_restart_up_status is:{}".format(
serialised_restart_up_status))
cmd_output = str(
serialised_restart_up_status['Result']['stdout']).replace(
'\n', '').strip()
if ", 0% packet loss" in cmd_output:
return "OK"
else:
return "Error"
except Exception as e:
logger.console("Error in Force Restart node: " + e)
def set_password_on_server(self, *args, **kwargs):
banner("PCC.Set password on Server")
self._load_kwargs(kwargs)
try:
cmd = """ssh -i id_rsa_khushboo admin@{} -t 'echo -e "cals0ft\ncals0ft" | sudo passwd pcc'""".format(
self.set_password_server)
cmd_output = cli_run(cmd=cmd,
host_ip=self.host_ip,
linux_user=self.username,
linux_password=self.password)
trace("Set_password_on_i28: {}".format(str(cmd_output)))
cmd1 = 'sudo ssh-keygen -f "/root/.ssh/known_hosts" -R "{}"'.format(
self.set_password_server)
cmd_output1 = cli_run(cmd=cmd1,
host_ip="172.17.3.225",
linux_user="******",
linux_password="******")
trace("Output for removing ssh-keygen: {}".format(
str(cmd_output1)))
date_cmd = "date"
cmd_output = cli_run(cmd=date_cmd,
host_ip=self.set_password_server,
linux_user=self.username,
linux_password=self.password)
trace("Date Command output is :{}".format(str(cmd_output)))
return "OK"
except Exception as e:
logger.console("Error in set password on server:" + str(e))
return "OK"
def restart_rsyslog_service(self, *args, **kwargs):
banner("CLI.Restart Rsyslog service")
self._load_kwargs(kwargs)
print("Kwargs are: {}".format(kwargs))
try:
restart_status = []
cmd1 = "sudo systemctl restart rsyslog"
cmd2 = "sudo systemctl status rsyslog"
for hostip in ast.literal_eval(self.host_ips):
cmd1_op=cli_run(hostip,self.linux_user,self.linux_password,cmd1)
print("cmd1: {}\n ====== Output is {} ========".format(cmd1, cmd1_op))
time.sleep(10)
cmd2_op= cli_run(hostip,self.linux_user,self.linux_password,cmd2)
print("cmd2: {}\n ====== Output is {} ========".format(cmd2, cmd2_op))
trace("cmd2: {}\n ====== Output is {} ========".format(cmd2, cmd2_op))
if re.search("active \(running\)", str(cmd2_op)):
print("============================== Inside re search =====================")
restart_status.append("OK")
else:
restart_status.append("Restart validation failed on {}".format(hostip))
result = len(restart_status) > 0 and all(elem == "OK" for elem in restart_status)
if result:
return "OK"
else:
return "Error: while Rsyslog service restart. Restart status is: {}".format(restart_status)
except Exception as e:
return "Exception encountered while restarting Rsyslog client services {}".format(e)
def ceph_cleanup_be(self, **kwargs):
self._load_kwargs(kwargs)
cmd1 = "sudo lsblk | grep 'disk' | awk '{print $1}'"
for ip in self.nodes_ip:
trace("======== cmd: {} is getting executed ========".format(cmd1))
print("======== cmd: {} is getting executed ========".format(cmd1))
drives_op = cli_run(ip, self.user, self.password, cmd1)
trace("Drives_op: {}".format(str(drives_op)))
print("Drives_op: {}".format(str(drives_op)))
serialized_drives = str(
self._serialize_response(
time.time(),
drives_op)['Result']['stdout']).strip().split('\n')[1:]
trace("serialized_drives: {}".format(serialized_drives))
print("serialized_drives: {}".format(serialized_drives))
for drive in serialized_drives:
cmd2 = "sudo wipefs -a /dev/{}".format(drive.strip())
trace("======== cmd: {} is getting executed ========".format(
cmd2))
print("======== cmd: {} is getting executed ========".format(
cmd2))
clean_drives_op = cli_run(ip, self.user, self.password, cmd2)
trace("Clean drives output:{}".format(str(clean_drives_op)))
print("Clean drives output:{}".format(str(clean_drives_op)))
time.sleep(5)
return "OK"
def decrypt_files(self, *arg, **kwargs):
banner("PCC.PhoneHome Decrypt Files")
self._load_kwargs(kwargs)
print("Kwargs:" + str(kwargs))
trace("Kwargs:" + str(kwargs))
try:
#tar_file_name = self.fetch_tar_file_detail(**kwargs)[0]['name']
tar_file_name = self.fetch_tar_file_detail(**kwargs)['name']
cmd = "sudo mkdir /home/pcc/platina-cli-ws/phone-home"
cmd1 = "sudo mkdir /home/pcc/platina-cli-ws/phone-home/with_ssl"
cmd2 = "sudo gpg --output /home/pcc/platina-cli-ws/phone-home/with_ssl/manual_untar_file.tar.xz --decrypt /home/pcc/storage-s3/minio/volume/data/phone-home/{}".format(
tar_file_name)
create_phone_home_folder = cli_run(self.host_ip, self.user,
self.password, cmd)
create_with_ssl_folder = cli_run(self.host_ip, self.user,
self.password, cmd1)
decrypt_manual_tar_file = cli_run(self.host_ip, self.user,
self.password, cmd2)
trace("cmd1 executed: {}".format(cmd1))
print("cmd1 executed: {}".format(cmd1))
trace("cmd2 executed: {}".format(cmd2))
print("cmd2 executed: {}".format(cmd2))
return "OK"
except Exception as e:
return "Exception occured in decrypt file"
def tunnel_interface_switch_invader(self, *args, **kwargs):
banner("Tunnel interface switch from invader")
self._load_kwargs(kwargs)
logger.console("Kwargs are : {}".format(kwargs))
try:
cmd = "sudo ifconfig {} {}".format(self.tun_value, self.tun_switch)
tunnel_down_cmd = cli_run(cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
cmd = "sudo ip addr|grep tun"
tunnel_down_status = cli_run(cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
serialised_tunnel_down_status = self._serialize_response(
time.time(), tunnel_down_status)
print("serialised_tunnel_down_status is:{}".format(
serialised_tunnel_down_status))
self.cmd_output = str(
serialised_tunnel_down_status['Result']['stdout']).replace(
'\n', '').strip()
self.list_tun_interface_state = re.findall("NOARP,(.*),LOWER",
self.cmd_output)
if self.list_tun_interface_state:
self.tun_interface_state = self.list_tun_interface_state[0]
else:
self.tun_interface_state = "DOWN"
if self.tun_switch == "down":
if self.tun_interface_state == "DOWN":
return True
else:
return False
elif self.tun_switch == "up":
if str(self.tun_interface_state) == "UP":
return True
else:
return False
else:
print(
"Didn't get the tunnel switch value - Please provide up or down as args"
)
logger.console(
"Didn't get the tunnel switch value - Please provide up or down as args"
)
return False
except Exception as e:
logger.console(
"Error in tunnel_interface_switch_invader status: {}".format(
e))
def tun_command_execution(self, *args, **kwargs):
banner("Tun command execution")
self._load_kwargs(kwargs)
try:
if self.restart == None:
cmd = "ip addr | grep tun"
tun_cmd_status = cli_run(cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
serialised_tun_cmd_status = self._serialize_response(
time.time(), tun_cmd_status)
print("serialised_tun_cmd_status is:{}".format(
serialised_tun_cmd_status))
self.cmd_output = str(
serialised_tun_cmd_status['Result']['stdout']).replace(
'\n', '').strip()
if self.cmd_output:
self.tun_value = re.findall(": (.*): <POINTOPOINT",
self.cmd_output)[0]
self.peer_ip_address = re.findall("peer (.*)\/",
self.cmd_output)[0]
self.inet_ip_address = re.findall("inet (.*) peer",
self.cmd_output)[0]
return self.tun_value, self.peer_ip_address, self.inet_ip_address
else:
return False
elif self.restart == "Restarted":
cmd = "ip addr | grep tun"
tun_cmd_status = cli_run(cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
serialised_tun_cmd_status = self._serialize_response(
time.time(), tun_cmd_status)
print("serialised_tun_cmd_status is:{}".format(
serialised_tun_cmd_status))
self.cmd_output = str(
serialised_tun_cmd_status['Result']['stdout']).replace(
'\n', '').strip()
if not self.cmd_output:
return True
else:
return False
else:
return False
except Exception as e:
logger.console(
"Error in tun_command_execution status: {}".format(e))
def kafka_memory_validation(self, *args, **kwargs):
banner("Kafka memory validation")
'''args: hostip
'''
self._load_kwargs(kwargs)
logger.console("self.tun_interface_state: {}".format(
self.tun_interface_state))
try:
if self.tun_interface_state == "UP":
cmd = "docker exec kafka kafka-avro-console-consumer --topic memory --bootstrap-server localhost:9092|head -20|grep {}|wc -l".format(
self.host_name)
kafka_memory_validation = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
serialised_kafka_memory_validation = self._serialize_response(
time.time(), kafka_memory_validation)
print("serialised_kafka_memory_validation is:{}".format(
serialised_kafka_memory_validation))
self.cmd_output = str(
serialised_kafka_memory_validation['Result']
['stdout']).replace('\n', '').strip()
if int(self.cmd_output) > 0:
return True
else:
return False
elif self.tun_interface_state == "DOWN":
cmd = "docker exec kafka kafka-avro-console-consumer --topic memory --bootstrap-server localhost:9092|head -20|grep {}|wc -l".format(
self.host_name)
kafka_memory_validation = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
serialised_kafka_memory_validation = self._serialize_response(
time.time(), kafka_memory_validation)
print("serialised_kafka_memory_validation is:{}".format(
serialised_kafka_memory_validation))
self.cmd_output = str(
serialised_kafka_memory_validation['Result']
['stdout']).replace('\n', '').strip()
if int(self.cmd_output) == 0:
return True
else:
return False
else:
return False
except Exception as e:
logger.console(
"Error in kafka_memory_validation status: {}".format(e))
def fetch_tar_file_detail(self, *arg, **kwargs):
banner("PCC.PhoneHome Fetch Tar File Details")
self._load_kwargs(kwargs)
print("Kwargs:" + str(kwargs))
trace("Kwargs:" + str(kwargs))
date_cmd = cli_run(self.host_ip, self.user, self.password,
"date +%Y-%m-%d")
date_cmd_op = self._serialize_response(
time.time(), date_cmd)['Result']['stdout'].strip()
manual_tar_file_command = 'sudo cat /home/pcc/default.log |grep "Successfully uploaded"|grep "manual"|grep "{}"|grep "{}"'.format(
self.get_file_name, date_cmd_op)
manual_tar_file_command_output = cli_run(self.host_ip, self.user,
self.password,
manual_tar_file_command)
manual_tar_file_serialize_output = self._serialize_response(
time.time(),
manual_tar_file_command_output)['Result']['stdout'].strip()
manual_tar_file = midtext('uploaded ', "(",
manual_tar_file_serialize_output)
trace("Manual tar file: {}".format(manual_tar_file))
print("Manual tar file: {}".format(manual_tar_file))
##### Road_Runner/30862a4df11a/2021-02-23-10.09.36.726335-utc-manual.tar.xz". Size: 228.41 MiB
split_manual_tar_file = manual_tar_file.split(" ")
manual_tar = {}
trace("split_manual_tar_file:{}".format(split_manual_tar_file))
print("split_manual_tar_file:{}".format(split_manual_tar_file))
manual_tar['name'] = split_manual_tar_file[0].replace('".', '')
manual_tar['tar_file_without_user'] = split_manual_tar_file[0].replace(
'".', '').replace(self.setup_username, "")
manual_tar['size'] = float(split_manual_tar_file[2])
#daily_tar_file_command = 'sudo cat /home/pcc/default.log |grep "Successfully uploaded"|grep "daily"|grep "{}"'.format(date_cmd_op)
#daily_tar_file_command_output = cli_run(self.host_ip, self.user, self.password, daily_tar_file_command)
#daily_tar_file_serialize_output = self._serialize_response(time.time(), daily_tar_file_command_output)['Result']['stdout'].strip()
#daily_tar_file = midtext('uploaded ', "(", daily_tar_file_serialize_output)
#trace("daily_tar_file: {}".format(daily_tar_file))
#print("daily_tar_file: {}".format(daily_tar_file))
######## Road_Runner/30862a4df11a/2021-02-23-16.30.00.427008-utc-daily.tar.xz". Size: 96.11 MiB
#split_daily_tar_file = daily_tar_file.split(" ")
#daily_tar = {}
#trace("split_daily_tar_file: {}".format(split_daily_tar_file))
#print("split_daily_tar_file : {}".format(split_daily_tar_file))
#daily_tar['name'] = split_daily_tar_file[0].replace('".', '')
#daily_tar['tar_file_without_user'] = split_daily_tar_file[0].replace('".', '').replace(self.setup_username,"")
#daily_tar['size'] = float(split_daily_tar_file[2])
trace("manual_tar:{}".format(manual_tar))
#trace("daily_tar:{}".format(daily_tar))
print("manual_tar:{}".format(manual_tar))
#print("daily_tar:{}".format(daily_tar))
#return manual_tar,daily_tar
return manual_tar
def peer_IP_reachability(self, *args, **kwargs):
banner("Peer IP Address Reachability")
self._load_kwargs(kwargs)
try:
if self.tun_interface_state == "UP":
cmd = "ping {} -c 4".format(self.interface_ip)
peer_ip_reachability_status = cli_run(
cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
serialised_peer_ip_reachability_status = self._serialize_response(
time.time(), peer_ip_reachability_status)
print("serialised_peer_ip_reachability_status is:{}".format(
serialised_peer_ip_reachability_status))
self.cmd_output = str(
serialised_peer_ip_reachability_status['Result']
['stdout']).replace('\n', '').strip()
if ", 0% packet loss" in self.cmd_output:
return True
else:
return False
elif self.tun_interface_state == "DOWN":
cmd = "ping {} -c 4".format(self.interface_ip)
peer_ip_reachability_status = cli_run(
cmd=cmd,
host_ip=self.node_hostip,
linux_user=self.username,
linux_password=self.password)
serialised_peer_ip_reachability_status = self._serialize_response(
time.time(), peer_ip_reachability_status)
print("serialised_peer_ip_reachability_status is:{}".format(
serialised_peer_ip_reachability_status))
self.cmd_output = str(
serialised_peer_ip_reachability_status['Result']
['stdout']).replace('\n', '').strip()
if ", 100% packet loss" in self.cmd_output:
return True
else:
return False
else:
return False
except Exception as e:
logger.console(
"Error in peer_IP_reachability status: {}".format(e))
def get_stored_size(self, *args, **kwargs):
banner("Get Stored Size for Replicated Pool and Erasure Pool")
self._load_kwargs(kwargs)
try:
replicated_pool = str(self.erasure_pool_name) + "-hs"
cmd = "sudo ceph df detail | grep -w {}".format(
self.erasure_pool_name)
erasure_pool_stored_size = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
serialised_erasure_pool_stored_size = self._serialize_response(
time.time(), erasure_pool_stored_size)
cmd_output = str(serialised_erasure_pool_stored_size['Result']
['stdout']).replace('\n', '').strip()
splitting = cmd_output.split()
print("value of erasure_pool: {}".format(splitting[2]))
print("Size of erasure_pool: {}".format(splitting[3]))
size_of_erasure_pool = self.convert(eval(splitting[2]),
splitting[3])
print("Size of erasure pool is: {}".format(size_of_erasure_pool))
cmd = "sudo ceph df detail | grep -w {}".format(replicated_pool)
erasure_pool_stored_size = cli_run(cmd=cmd,
host_ip=self.hostip,
linux_user=self.username,
linux_password=self.password)
serialised_erasure_pool_stored_size = self._serialize_response(
time.time(), erasure_pool_stored_size)
cmd_output = str(serialised_erasure_pool_stored_size['Result']
['stdout']).replace('\n', '').strip()
splitting = cmd_output.split()
print("value of replicated pool: {}".format(splitting[2]))
print("Size of replicated pool: {}".format(splitting[3]))
size_of_replicated_pool = self.convert(eval(splitting[2]),
splitting[3])
print(
"Size of erasure pool is: {}".format(size_of_replicated_pool))
return [size_of_erasure_pool, size_of_replicated_pool]
except Exception as e:
trace("Error in get_stored_size: {}".format(e))
def import_privatekey_for_ssl(self, *arg, **kwargs):
banner("PCC.PhoneHome Import PrivateKey On Platina-cli-ws For SSL")
self._load_kwargs(kwargs)
print("Kwargs:" + str(kwargs))
trace("Kwargs:" + str(kwargs))
try:
cmd1 = "sudo cp /home/pcc/privatekey.asc /home/pcc/platina-cli-ws/"
cmd2 = "sudo cat /home/pcc/passphrase.json | grep 'gpg' | awk '{print $2}'"
copy_privatekey_on_platina_cli = cli_run(self.host_ip, self.user,
self.password, cmd1)
passphrase_gpg_key = cli_run(self.host_ip, self.user,
self.password, cmd2)
trace("copy_privatekey_on_platina_cli: {}".format(
str(copy_privatekey_on_platina_cli)))
print("copy_privatekey_on_platina_cli: {}".format(
str(copy_privatekey_on_platina_cli)))
trace("passphrase_gpg_key: {}".format(passphrase_gpg_key))
print("passphrase_gpg_key: {}".format(passphrase_gpg_key))
passphrase_gpg_key_output = str(
self._serialize_response(
time.time(),
passphrase_gpg_key)['Result']['stdout']).strip().replace(
'"', '')
cmd3 = "sudo gpg --pinentry-mode=loopback --passphrase {} --import /home/pcc/platina-cli-ws/privatekey.asc".format(
passphrase_gpg_key_output)
gpg_import_private_key = cli_run(self.host_ip, self.user,
self.password, cmd3)
trace("cmd3 executed: {}".format(cmd3))
print("cmd3 executed: {}".format(cmd3))
trace("cmd3 status is: {}".format(str(gpg_import_private_key)))
print("cmd3 status is: {}".format(str(gpg_import_private_key)))
cmd4 = "sudo gpg --list-secret-keys | grep Key"
verify_setup_username = cli_run(self.host_ip, self.user,
self.password, cmd4)
trace("cmd4 executed: {}".format(cmd4))
print("cmd4 executed: {}".format(cmd4))
trace("cmd4 status is: {}".format(str(verify_setup_username)))
print("cmd4 status is: {}".format(str(verify_setup_username)))
if re.search(self.setup_username, str(verify_setup_username)):
return "OK"
else:
return "Error : {} not found in GPG secret key list".format(
self.setup_username)
except Exception as e:
return "Exception occured in import privatekey for SSL:{}".format(
e)
def verify_model_and_serial_number(self, *args, **kwargs):
banner("PCC.Node Verify Model And Serial Number")
self._load_kwargs(kwargs)
print("Kwargs:{}".format(kwargs))
conn = BuiltIn().get_variable_value("${PCC_CONN}")
serial_cmd = "sudo dmidecode -s baseboard-serial-number"
model_cmd = "sudo dmidecode -s baseboard-product-name"
failed_host = []
if self.Names:
for name in ast.literal_eval(self.Names):
node_id = self.get_node_id(Name=name)
node_details = get_response_data(
pcc.get_node_summary_by_id(conn, str(node_id)))
print("1. Node Summary:" + str(node_details))
print("++++++++++++++++++++++++++++++++++++++++++++")
print("2. Serial Number" + str(node_details['SN']))
print("++++++++++++++++++++++++++++++++++++++++++++")
print("3. Model Number" + str(node_details['Model']))
print("++++++++++++++++++++++++++++++++++++++++++++")
logger.console("Verifying services for host {} .....".format(
node_details['Host']))
print("Verifying services for host {} .....".format(
node_details['Host']))
serial_number = self._serialize_response(
time.time(),
cli_run(node_details['Host'], self.user, self.password,
serial_cmd))['Result']['stdout']
model_number = self._serialize_response(
time.time(),
cli_run(node_details['Host'], self.user, self.password,
model_cmd))['Result']['stdout']
print("Backend Serial Data:" + str(serial_number))
print("Backend Model Data:" + str(model_number))
if node_details['Model'] == str(
model_number.strip()) and node_details['SN'] == str(
serial_number.strip()):
continue
else:
failed_host.append(name)
continue
else:
print(
"Node names is empty, please provide the node name list for eg. Names=['sv124','sv125']"
)
if failed_host:
print("Couldn't verify Serial and Model Number for {}".format(
failed_host))
return "Error"
else:
return "OK"
def wait_until_phone_home_job_is_finished(self, *args, **kwargs):
self._load_kwargs(kwargs)
trace("Kwargs:" + str(kwargs))
print("Kwargs:" + str(kwargs))
banner("PCC.Wait Until Phone Home Job Is Finished")
PCCSERVER_TIMEOUT = 60 * 60 # 60 minutes timeout
trace("current time is: {}".format(time.time()))
timeout = time.time() + PCCSERVER_TIMEOUT
trace("timeout is: {}".format(timeout))
cmd = "sudo /home/pcc/platina-cli-ws/platina-cli support jobs"
finished = False
cmd_output = str(cli_run(self.host_ip, self.user, self.password, cmd))
file_name = midtext('collecting', '-manual', str(cmd_output))
while not finished:
print(
"=================================================================================================="
)
cmd_output = str(
cli_run(self.host_ip, self.user, self.password, cmd))
trace("cmd_output: {}".format(cmd_output))
print("cmd_output: {}".format(cmd_output))
trace("File name is : {}".format(str(file_name)))
print("File_name is : {}".format(str(file_name)))
if time.time() > timeout:
return "Timeout: Still in collecting state"
if re.search("collecting {}".format(file_name),
cmd_output) or re.search(
"processing {}".format(file_name),
cmd_output) or re.search(
"submitting {}".format(file_name),
cmd_output):
time.sleep(30)
trace(
"Still collecting/processing phone home data... Please wait"
)
print(
"Still collecting/processing phone home data... Please wait"
)
continue
if re.search("end {}".format(file_name), cmd_output):
trace("Finished Phone home job")
print("Finished Phone home job")
return "OK", file_name
if re.search("error {}".format(file_name), cmd_output):
trace("Error: Phone Home Job Failed")
print("Error: Phone Home Job Failed")
return "Error: Phone Home Job Failed", file_name
else:
return ["Error: Phone Home Job Failed", file_name]
def ceph_rgw_verify_be_creation(self, **kwargs):
banner("PCC.Ceph Rgw Verify BE Creation")
self._load_kwargs(kwargs)
try:
conn = BuiltIn().get_variable_value("${PCC_CONN}")
except Exception as e:
raise e
ceph_be_cmd = "sudo ceph -s"
wait_time = 400
for i in range(4):
time.sleep(100)
wait_time -= 100
print("wait time left for RGW backend check {}s".format(wait_time))
trace("wait time left for RGW backend check {}s".format(wait_time))
failed_chk = []
success_chk = []
for ip in eval(str(self.targetNodeIp)):
host_name = easy.get_host_name_by_ip(conn, ip)
cmd_rgw = "sudo systemctl status ceph-radosgw@rgw.{}.rgw0".format(
host_name)
ceph_check = cli_run(ip, self.user, self.password, ceph_be_cmd)
rgw_check = cli_run(ip, self.user, self.password, cmd_rgw)
print("=========== ceph_check output is: {} \n==============".
format(str(ceph_check)))
print("=========== rgw_check output is: {} \n==============".
format(str(rgw_check)))
if re.search("rgw", str(ceph_check)) and re.search(
"running", str(rgw_check)):
success_chk.append(ip)
else:
failed_chk.append(ip)
if len(success_chk) == len(eval(str(self.targetNodeIp))):
print("Backend verification successfuly done for : {}".
format(success_chk))
return "OK"
if wait_time <= 0:
print("Rgw Check: " + str(rgw_check))
print("Ceph Rgw Check: " + str(ceph_check))
if failed_chk:
print("Rgw service are down for {}".format(failed_chk))
return "Error"
else:
return "OK"
def check_ntp_services_backend(self, **kwargs):
banner("PCC.Check NTP services from backend")
self._load_kwargs(kwargs)
conn = BuiltIn().get_variable_value("${PCC_CONN}")
cmd_ntp1 = "sudo systemctl status ntp"
#cmd_ntp2="sudo netstat -anp | grep ntp"
cmd_ntp2 = "sudo ps aux | grep ntp"
cmd_ntp3 = "sudo systemctl status ntpd"
success_chk = []
failed_chk = []
for ip in ast.literal_eval(self.targetNodeIp):
ntp_check1 = cli_run(ip, self.user, self.password, cmd_ntp1)
print("Command_1 is: {}".format(cmd_ntp1))
print("=========== NTP_Check1 output ==========\n{}".format(
ntp_check1))
ntp_check2 = cli_run(ip, self.user, self.password, cmd_ntp2)
print("Command_2 is: {}".format(cmd_ntp2))
print("=========== NTP_Check2 output ==========\n{}".format(
ntp_check2))
ntp_check3 = cli_run(ip, self.user, self.password, cmd_ntp3)
print("Command_3 is: {}".format(cmd_ntp3))
print("=========== NTP_Check3 output ==========\n{}".format(
ntp_check3))
if (re.search("ntpd", str(ntp_check1))
or re.search("ntpd", str(ntp_check3))) and (re.search(
"running", str(ntp_check1)) or re.search(
"running", str(ntp_check3))) and re.search(
"ntpd", str(ntp_check2)):
print("NTP Found")
success_chk.append(ip)
else:
failed_chk.append(ip)
print("Success chk status is : {}".format(success_chk))
print("Failed chk status is : {}".format(failed_chk))
if len(success_chk) == len(ast.literal_eval(self.targetNodeIp)):
print("Backend verification successfuly done for : {}".format(
success_chk))
return "OK"
if failed_chk:
print("NTP service are down for {}".format(failed_chk))
return "Error: NTP service are down for {}".format(failed_chk)
else:
return "OK"
def validate_dns_backend(self, **kwargs):
banner("PCC.Validate DNS From Backend")
self._load_kwargs(kwargs)
conn = BuiltIn().get_variable_value("${PCC_CONN}")
if "search_list" in kwargs:
self.search_list = ast.literal_eval(kwargs["search_list"])
found_result = []
for search_item in self.search_list:
print("============ Search item : {} ===========".format(
search_item))
cmd1 = "sudo cat /etc/resolv.conf"
print(" ========= Command_1 is: {} ==========".format(cmd1))
cmd1_op = cli_run(self.host_ip, self.user, self.password, cmd1)
print("cmd1 op: {}".format(cmd1_op))
if (re.search("{}".format(search_item), str(cmd1_op))):
found_result.append("OK")
else:
found_result.append("Not found: {}".format(search_item))
print("Found result: {}".format(found_result))
result = len(found_result) > 0 and all(elem == "OK"
for elem in found_result)
if result:
return "OK"
else:
return "Error: validation unsuccessful: Check result list: {}".format(
found_result)
else:
return "Error: Please provide valid search list"
def pull_from_registry(self, *args, **kwargs):
self._load_kwargs(kwargs)
print("Kwargs are: {}".format(kwargs))
try:
self.cmd = "docker pull {}:{}/{}".format(self.registry_url,self.registryPort,self.image_name)
time.sleep(5)
print("Command used is: {}".format(self.cmd))
pull_command_execution = cli_run(cmd= self.cmd,host_ip=self.hostip, linux_user=self.username, linux_password = self.password)
serialised_pull_command_execution = self._serialize_response(time.time(), pull_command_execution)
print("serialised_pull_command_execution is:{}".format(serialised_pull_command_execution))
self.cmd_output = str(serialised_pull_command_execution['Result']['stdout']).replace('\n', '').strip()
print("pull_image_cmd with registry url_executed successfully: {}".format(self.cmd_output))
if self.cmd_output:
if ("Downloaded newer image for {}:{}/{}:latest".format(self.registry_url,self.registryPort,self.image_name)) or ("Status: Image is up to date") in self.cmd_output:
banner("Image pulled from Registry successfully")
return "OK"
else:
return str(serialised_pull_command_execution['Result']['stderr']).replace('\n', '').strip()
elif self.cmd_output == "":
raise Exception("No value found in cmd output")
else:
return "Error"
except Exception as e:
logger.console("Error in pull from registry: " + str(e))
def ceph_rgw_verify_service_ip_be(self, **kwargs):
banner("PCC.Ceph Rgw Verify Service IP BE")
self._load_kwargs(kwargs)
try:
conn = BuiltIn().get_variable_value("${PCC_CONN}")
except Exception as e:
raise e
if self.targetNodeIp:
ip = self.targetNodeIp
elif self.pcc:
ip = self.pcc
else:
return "Target node is not provided"
cmd = "sudo netstat -ntlp |grep rados"
ntlp_check = cli_run(ip, self.user, self.password, cmd)
if self.service_ip.lower() == "yes":
if re.search(self.control_cidr[0:8], str(ntlp_check)):
return "OK"
else:
print("Could not verify service ip for ntlp check")
return "Could not verify service ip for ntlp check"
else:
if re.search(self.control_cidr[0:8], str(ntlp_check)):
print("Could not verify service ip for ntlp check")
return "Could not verify service ip for ntlp check"
else:
return "OK"
return "Please provide service_ip keyword with yes/no"
def ceph_rgw_get_file_bucket(self, **kwargs):
banner("PCC.Ceph Rgw Get File To Bucket ")
self._load_kwargs(kwargs)
cmd = 'sudo s3cmd get s3://BUCKET/{} -c /home/pcc/.s3cfg'.format(
self.fileName)
print("Command:" + str(cmd))
data = cli_run(self.pcc, self.user, self.password, cmd)
if re.search("download", str(data)):
print("File is exracted from Bucket")
cmd = 'sudo rm {}'.format(self.fileName)
data = cli_run(self.pcc, self.user, self.password, cmd)
return "OK"
else:
print("File is not extracted")
return "Error"
return
def check_GPG_keys(self, *args, **kwargs):
"""
Check GPG Keys
[Args]
(str) host_ip: Host IP of the node
[Returns]
(dict) Response: Output of check GPG keys command
"""
self._load_kwargs(kwargs)
banner("CLI.Check GPG Keys")
try:
OS_type = Cli().get_OS_version(host_ip=self.host_ip,
linux_user=self.linux_user,
linux_password=self.linux_password)
if re.search("Debian", str(OS_type)) or re.search(
"Ubuntu", str(OS_type)):
cmd = "sudo apt-key list"
if re.search("Red Hat Enterprise", str(OS_type)) or re.search(
"CentOS", str(OS_type)):
cmd = '''sudo rpm -qa gpg-pubkey --qf "%{version}-%{release} %{summary}\n"'''
cmd_op = cli_run(self.host_ip, self.linux_user,
self.linux_password, cmd)
serialised_output = self._serialize_response(time.time(), cmd_op)
output = str(serialised_output['Result']['stdout']).replace(
'\n', '').strip()
return output
except Exception as e:
print("Exception encountered: {}".format(e))
return "Exception encountered: " + str(e)
def show_neighbors_on_server(self, *args, **kwargs):
banner("PCC.Show neighbors on Server")
self._load_kwargs(kwargs)
try:
server_interfaces = []
for hostip in ast.literal_eval(self.servers_hostip):
cmd = "sudo lldpcli show neighbors |grep xeth"
command_execution = cli_run(cmd=cmd,
host_ip=hostip,
linux_user=self.username,
linux_password=self.password)
serialised_command_execution = self._serialize_response(
time.time(), command_execution)
logger.console(
"Serialised: {}".format(serialised_command_execution))
cmd_output = str(
serialised_command_execution['Result']['stdout']).strip()
print("Show neighbor on Server output : {}".format(cmd_output))
interfaces = re.findall("\W(xeth(.*))\n\W", cmd_output)
for interface in interfaces:
server_interfaces.append(interface[0])
banner("Server interfaces : {}".format(set(server_interfaces)))
return server_interfaces
except Exception as e:
logger.console("Error in Show neighbors on Server: " + e)
def cleanup_logs_by_rsyslog(self, *args, **kwargs):
banner("CLI.Cleanup logs created by Rsyslog")
self._load_kwargs(kwargs)
print("Kwargs are: {}".format(kwargs))
try:
cleanup_status = []
cmd = "sudo dd if=/dev/null of=/var/log/messages"
for hostip in ast.literal_eval(self.host_ips):
cmd_op=cli_run(hostip,self.linux_user,self.linux_password,cmd)
print("cmd: {}\n ====== Output is {} ========".format(cmd, cmd_op))
if re.search("0\+0 records out",str(cmd_op)):
cleanup_status.append("OK")
else:
cleanup_status.append("Rsyslog Cleanup failed on {}".format(hostip))
trace("Cleanup status is: {}".format(cleanup_status))
result = len(cleanup_status) > 0 and all(elem == "OK" for elem in cleanup_status)
trace("Result is : {}".format(result))
if result:
return "OK"
else:
return "Error: while Rsyslog cleanup. Cleanup status is: {}".format(cleanup_status)
except Exception as e:
return "Exception encountered while Rsyslog cleanup: {}".format(e)
