def create_delete_lun_overnight(): client = SSHClient(neadm_host, neadm_username, neadm_password) fail = 0 times = 0 print 'Executing this test %s times' % max_run_times while (times < max_run_times) & (fail < max_fail_times): my_time = time.strftime("%Y-%m-%d %H:%M:%S") times += 1 print 'Running time: %s' % times rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{}{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1])) if rc != 0: print cout out.colorPrint('Error: Failed to create LUN {}/{}/{}/{}{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1]), 'r') error = '{} CE-{}: {}'.format(my_time, times, cout) with open(log_file, 'a') as text_file: # Write errors into log file text_file.write('{}'.format(error)) print cout rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times)) if rc != 0: print cout out.colorPrint('Error: Failed to delete LUN {}/{}/{}/{}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times), 'r') error = '{} DE-{}: {}'.format(my_time, times, cout) with open(log_file, 'a') as text_file: # Write errors into log file text_file.write('{}'.format(error)) fail += 1 print 'Failed to delete {}/{} times'.format(fail, max_fail_times) print cout
def cleanup(): # Connecting to MUT client = SSHClient(client_host, client_username, client_password) # Checking content of the destination folder print "Checking if cleanup is needed ..." rc, cout = client.run("ls -1 {}".format(pathto)) if rc == None: out.colorPrint('ERROR: Unknown Error', 'r') exit(1) elif rc == 0: print 'Console output:\n%s' % cout else: out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1) # If file exists we try to delete it if filename in cout: out.colorPrint('Found file you trying to copy, initiating cleanup ...', 'r') rc, cout = client.run('rm {}/{}'.format(pathto, filename)) if rc != 0: out.colorPrint('Cleanup has failed', 'r') out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1) out.colorPrint('Cleanup has finished successfully\n', 'g') else: out.colorPrint('Cleanup is not needed\n', 'g')
def create_ctb(): # Connecting to MUT client = SSHClient(neadm_host, neadm_username, neadm_password) # Show service to get information about preferred node and list of nodes that are in the service rc, cout = client.run('/neadm/neadm cluster create {}'.format(cluster_name)) if rc != 0: print cout out.colorPrint('Failed to create cluster {}'.format(cluster_name), 'r') exit(1) print cout time.sleep(2) rc, cout = client.run('/neadm/neadm tenant create {}/{}'.format(cluster_name, tenant_name)) if rc != 0: print cout out.colorPrint('Failed to create tenant {}/{}'.format(cluster_name, tenant_name), 'r') exit(1) print cout time.sleep(2) rc, cout = client.run('/neadm/neadm bucket create {}/{}/{}'.format(cluster_name, tenant_name, bucket_name)) if rc != 0: print cout out.colorPrint('Failed to create bucket {}/{}/{}'.format(cluster_name, tenant_name, bucket_name), 'r') exit(1) print cout
def run1(): timeout = 300 # 5 minute timeout for copy print( "Executing failover after copying file onto mounted iSCSI share and checking diff: %s times" % runtimes) copyFrom = getSize(pathfrom) copy_file() r = 2 print 'WAITING TO FINISH COPY: Checking if file was fully copyed from %s to %s' % ( pathfrom, pathto) try: with Timeout(timeout): while r == 2: time.sleep(3) copyTo = getSize(pathto) if copyTo == copyFrom: out.colorPrint( 'WAITING TO FINISH COPY: File is fully copyed, %s/%s\n' % (copyTo, copyFrom), 'g') r = 0 else: print 'WAITING TO FINISH COPY: File is still copying ... %s/%s' % ( copyTo, copyFrom) r = 2 except Timeout.Timeout: out.colorPrint('ERROR: Timeout exeeded %s seconds' % timeout, 'r') # Connection to MUT client = SSHClient(client_host, client_username, client_password) # Search for cp process rc, cout = client.run('ps -ef | grep cp {}/{}'.format( pathto, filename)) text = re.split('\n', cout) for line in text: print line if filename in line: out.colorPrint('ERROR: Copying file is taking too long', 'r') print line print 'Killing cp process ...' rc, cout = client.run('pkill cp') cleanup() exit(1) if filename not in line: out.colorPrint( 'ERROR: When timeout exeeded its limit, copy process was not found, it must have been interrupted for some reason', 'r') cleanup() exit(1) else: out.colorPrint('ERROR: Unknown Error', 'r') cleanup() exit(1) time.sleep(3) check_diff() failover() time.sleep(3) check_diff() delete_file()
def create_service(): client = SSHClient(neadm_host, neadm_username, neadm_password) rc, cout = client.run('/neadm/neadm service create iscsi %s' % service_name) print cout rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node1)) print cout rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node2)) print cout rc, cout = client.run('/neadm/neadm service vip add {} {} {}/24 -p {}'.format(service_name, nedge_quorum, nedge_vip, nedge_node1)) print cout rc, cout = client.run('/neadm/neadm service show %s' % service_name) print cout text = re.split('\n', cout) nodeid = [] for line in text: if 'X-Servers' in line: match = re.search('X-Servers\s+(\w+)\D(\w+)', line) if match: nodeid = [match.group(1), match.group(2)] else: out.colorPrint('Couldn\'t match service nodes IDs', 'r') exit(1) if len(nodeid) != 2: out.colorPrint('Not enough nodes in the service, should be 2', 'r') exit(1) print 'Node IDs: {}, {}'.format(nodeid[0], nodeid[1]) rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[0], client_net_name, client_net_ip1)) print cout rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[1], client_net_name, client_net_ip2)) print cout rc, cout = client.run('/neadm/neadm service enable %s' % service_name) if rc != 0: print cout out.colorPrint('Error: unable to enable service: %s' % service_name, 'r') exit(1) print cout rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1])) if rc != 0: print cout out.colorPrint('Error: unable to create LUN {}/{}/{}/{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1]), 'r') exit(1) print cout
def getSize(path): # Connecting to MUT client = SSHClient(client_host, client_username, client_password) rc, cout = client.run('ls -la {}/{}'.format(path, filename)) text = re.split('\n', cout) if rc != 0: out.colorPrint( 'Failed to execute \"ls -la {}/{}\" command'.format( path, filename), 'r') out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1) # Searching for size of file for line in text: match = re.search( '\S+\s+1\s+\S+\s\S+\s(\d+)\s+\S+\s+\S+\s+\S+\s+{}/{}'.format( path, filename), line) if match: return match.group(1) else: print out.colorPrint( 'Couldn\'t match regex to find size of {}/{}'.format( path, filename), 'r') exit(1)
def cleanup_ctb(): client = SSHClient(neadm_host, neadm_username, neadm_password) print 'Checking if cleanup cluster is needed ...' rc, cout = client.run('/neadm/neadm cluster list') if cluster_name in cout: out.colorPrint('Found cluster, preparing to cleanup', 'r') rc, cout = client.run('/neadm/neadm bucket delete {}/{}/{}'.format(cluster_name, tenant_name, bucket_name)) print cout rc, cout = client.run('/neadm/neadm tenant delete {}/{}'.format(cluster_name, tenant_name)) print cout rc, cout = client.run('/neadm/neadm cluster delete {}'.format(cluster_name)) print cout else: out.colorPrint('Cleanup cluster is not needed\n', 'g')
def copy_file(): # Connection to MUT client = SSHClient(client_host, client_username, client_password) # Copy file to directory print 'Copying {} to {} ...'.format(filename, pathto) rc, cout = client.run('cp {}/{} {}/.'.format(pathfrom, filename, pathto)) # Check if the file is in the directory rc, cout = client.run('ls -lah {}'.format(pathto)) if rc == None: out.colorPrint('ERROR: Unknown Error', 'r') exit(1) elif rc == 0: print 'Console output:\n%s' % cout else: out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1)
def delete_file(): # Connection to MUT client = SSHClient(client_host, client_username, client_password) # Delete file from directory print 'Deleting {} from {} ...'.format(filename, pathto) rc, cout = client.run('rm {}/{}'.format(pathto, filename)) # Check if the file was actually removed from the directory rc, cout = client.run('ls -1 {}'.format(pathto)) if rc == None: exit(1) elif rc != 0: out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1) print 'Console output:\n%s' % cout if filename not in cout: out.colorPrint('File was deleted successfully\n', 'g') else: out.colorPrint('Error: File was not deleted\n', 'r') exit(1)
def cleanup_service(): client = SSHClient(neadm_host, neadm_username, neadm_password) print 'Checking if cleanup service is needed ...' rc, cout = client.run('/neadm/neadm service show %s' % service_name) if rc == 0: out.colorPrint('Found service, preparing to cleanup ...\n', 'r') rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name)) print cout rc, cout = client.run('/neadm/neadm service disable %s' % service_name) print cout rc, cout = client.run('/neadm/neadm service vip delete {} {}/24'.format(service_name, nedge_vip)) print cout rc, cout = client.run('/neadm/neadm service delete %s' % service_name) print cout out.colorPrint('Cleanup has finished successfully\n', 'g') else: out.colorPrint('Cleanup service is not needed\n', 'g')
def cleanup_client(): client = SSHClient(client_host, client_username, client_password) cl = 'yes' print 'Checking if cleanup client is needed ...' rc, cout = client.run('mount') text = re.split('\n', cout) for line in text: if pathto in line: print line match = re.search('(\S+)\s+on\s+({})\s+\w+\s+\w+\s+\S+'.format(pathto), line) if match: out.colorPrint('Found LUN \"{}\" mounted onto test filesystem \"{}\", cleanup is needed'.format(match.group(1), match.group(2)), 'r') rc, cout = client.run('umount %s' % pathto) print cout rc, cout = client.run('rm -r %s' % pathto) print cout rc, cout = client.run('iscsiadm -m node') print cout text = re.split('\n', cout) portal = '' targetname = '' for line in text: if nedge_vip in line: match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line) if match: portal = match.group(1) targetname = match.group(2) else: print 'Couldn\'t match' exit(1) print 'Portal: %s' % portal print 'IQN: %s' % targetname rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --logout'.format(targetname, portal)) print cout rc, cout = client.run('iscsiadm -m node -o delete --targetname {} --portal {}'.format(targetname, portal)) print cout cl = 'no' else: out.colorPrint('Couldn\'t match mounted filesystem name', 'r') exit(1) if cl == 'yes': out.colorPrint('Cleanup client is not needed\n', 'g')
def check_diff(): # Connection to MUT client = SSHClient(client_host, client_username, client_password) # Checking diff print 'Checking diff between files {}/{} and {}/{}'.format( pathfrom, filename, pathto, filename) rc, cout = client.run('diff {}/{} {}/{}'.format(pathfrom, filename, pathto, filename)) if rc == None: out.colorPrint('ERROR: Unknown Error', 'r') exit(1) elif rc == 0: out.colorPrint('Diff success\n', 'g') else: out.colorPrint('Failed to match diffs', 'r') out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1)
class CloudDeploy(object): def __init__(self, dha, fuel_ip, fuel_username, fuel_password, dea_file, work_dir): self.dha = dha self.fuel_ip = fuel_ip self.fuel_username = fuel_username self.fuel_password = fuel_password self.dea_file = dea_file self.work_dir = work_dir self.file_dir = os.path.dirname(os.path.realpath(__file__)) self.ssh = SSHClient(self.fuel_ip, self.fuel_username, self.fuel_password) self.macs_file = '%s/macs.yaml' % self.file_dir self.node_ids = self.dha.get_node_ids() def upload_cloud_deployment_files(self): dest ='~/%s/' % self.work_dir with self.ssh as s: s.exec_cmd('rm -rf %s' % self.work_dir, check=False) s.exec_cmd('mkdir ~/%s' % self.work_dir) s.scp_put(self.dea_file, dest) s.scp_put(self.macs_file, dest) s.scp_put('%s/common.py' % self.file_dir, dest) s.scp_put('%s/dea.py' % self.file_dir, dest) for f in glob.glob('%s/cloud/*' % self.file_dir): s.scp_put(f, dest) def power_off_nodes(self): for node_id in self.node_ids: self.dha.node_power_off(node_id) def power_on_nodes(self): for node_id in self.node_ids: self.dha.node_power_on(node_id) def set_boot_order(self, boot_order_list): for node_id in self.node_ids: self.dha.node_set_boot_order(node_id, boot_order_list) def get_mac_addresses(self): macs_per_node = {} for node_id in self.node_ids: macs_per_node[node_id] = self.dha.get_node_pxe_mac(node_id) with io.open(self.macs_file, 'w') as stream: yaml.dump(macs_per_node, stream, default_flow_style=False) def run_cloud_deploy(self, deploy_app): log('START CLOUD DEPLOYMENT') deploy_app = '%s/%s' % (self.work_dir, deploy_app) dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file)) macs_file = '%s/%s' % (self.work_dir, os.path.basename(self.macs_file)) with self.ssh: self.ssh.run('python %s %s %s' % (deploy_app, dea_file, macs_file)) def deploy(self): self.power_off_nodes() self.set_boot_order(['pxe', 'disk']) self.power_on_nodes() self.get_mac_addresses() check_file_exists(self.macs_file) self.upload_cloud_deployment_files() self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
def failover(): # Connecting to MUT client = SSHClient(neadm_host, neadm_username, neadm_password) # Show service to get information about preferred node and list of nodes that are in the service rc, cout = client.run('/neadm/neadm service show %s' % service_name) text = re.split('\n', cout) pref_node = '' nodes = [] for line in text: print line if 'X-VIPS' in line: # Getting preferred node match = re.search('X-VIPS\s+\[\[\"(\S+)\"\W\{\S+', line) if match: pref_node = match.group(1) else: out.colorPrint('Couldn\'t match preferred node', 'r') exit(1) elif 'X-VIP-Nodes' in line: # Getting list of nodes match = re.search('X-VIP-Nodes\s+\[\"(\S+)\".\"(\S+)\"\]', line) if match: nodes = [match.group(1), match.group(2)] else: out.colorPrint('Couldn\'t match service nodes', 'r') exit(1) # If any of there are still 0 something is wrong if len(pref_node) == 0 or len(nodes) == 0: out.colorPrint('Couldn\'t get necessary information from service show', 'r') exit(1) #print len(pref_node), len(nodes) print 'Nodes in service: \"{}\", \"{}\"'.format(nodes[0], nodes[1]) print 'Current preferred node is \"%s\"\n' % pref_node # Check if service has less then 2 nodes then do not perform failover if len(nodes) < 2: out.colorPrint( 'Unable to perform failover, because service has less then 2 nodes', 'r') exit(1) # Figuring out which node to perform failover to failover_node = '' for node in nodes: if node != pref_node: failover_node = node copyFrom = getSize(pathfrom) r = 0 print 'WAITING FOR FAILOVER: Comparing sizes of {}/{} and {}/{}'.format( pathfrom, filename, pathto, filename) while r == 0: time.sleep(3) copyTo = getSize(pathto) if int(copyTo) > int(copyFrom) / 2: out.colorPrint( 'WAITING FOR FAILOVER: At least half the file was copyed successfully, %s/%s\n' % (copyTo, copyFrom), 'g') r = 1 else: print 'WAITING FOR FAILOVER: File is still copying ... %s/%s' % ( copyTo, copyFrom) r = 0 print 'Performing failover from \"{}\" to \"{}\"'.format( pref_node, failover_node) rc, cout = client.run( '/neadm/neadm service vip config {} preferred {}'.format( service_name, failover_node)) if rc == None: out.colorPrint('ERROR: Unknown Error', 'r') exit(1) elif rc == 0: print 'Console output:\n%s' % cout else: out.colorPrint('ReturnCode: %s' % rc, 'r') out.colorPrint('Error: %s' % cout, 'r') exit(1)
def mount_iscsi_lun(): client = SSHClient(client_host, client_username, client_password) rc, cout = client.run('iscsiadm -m discovery -t sendtargets -p %s' % nedge_vip) if rc != 0: print cout out.colorPrint('Error: unable to find iSCSI target via IP: %s' % nedge_vip, 'r') exit(1) out.colorPrint('Successfully discovered target via %s\n' % nedge_vip, 'g') text = re.split('\n', cout) portal = '' targetname = '' for line in text: if nedge_vip in line: match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line) if match: portal = match.group(1) targetname = match.group(2) else: print 'Couldn\'t match' exit(1) print 'Portal: %s' % portal print 'IQN: %s' % targetname rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --login'.format(targetname, portal)) if rc != 0: out.colorPrint(cout, 'r') out.colorPrint('Error: Couldnot login to iSCSI target', 'r') exit(1) out.colorPrint(cout, 'g') rc, cout = client.run('fdisk -l | grep sd') print cout text = re.split('\n', cout) diskname = '' for line in text: if lun_size[0] in line: print line match = re.search('Disk\s(\W\w+\W\w+):\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+', line) if match: diskname = match.group(1) else: print 'Couldn\'t match' exit(1) out.colorPrint('Found iSCSI LUN, disk name is %s\n' % diskname, 'g') rc, cout = client.run('mkfs.ext4 %s' % diskname) if rc != 0: print cout out.colorPrint('Couldn\'t create filesystem on iSCSI LUN %s' % diskname, 'r') exit(1) print cout out.colorPrint('Filesystem was successfully create on top of %s' % diskname,'g') rc, cout = client.run('mkdir %s' % pathto) if rc != 0: out.colorPrint('Error: %s' % cout, 'r') out.colorPrint('Couldn\'t create directory %s' % pathto, 'r') exit(1) print cout out.colorPrint('{} was successfully created\n'.format(pathto), 'g') rc, cout = client.run('mount {} {}'.format(diskname, pathto)) print cout if rc != 0: out.colorPrint('Error: %s' % cout,'r') out.colorPrint('Error: Unable to mount {} to {}'.format(diskname, pathto),'r') exit(1) print cout out.colorPrint('{} was successfully mounted to {}'.format(diskname, pathto),'g')