def create_delete_lun_overnight():
client = SSHClient(neadm_host, neadm_username, neadm_password)
fail = 0
times = 0
print 'Executing this test %s times' % max_run_times
while (times < max_run_times) & (fail < max_fail_times):
my_time = time.strftime("%Y-%m-%d %H:%M:%S")
times += 1
print 'Running time: %s' % times
rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{}{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1]))
if rc != 0:
print cout
out.colorPrint('Error: Failed to create LUN {}/{}/{}/{}{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1]), 'r')
error = '{} CE-{}: {}'.format(my_time, times, cout)
with open(log_file, 'a') as text_file: # Write errors into log file
text_file.write('{}'.format(error))
print cout
rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times))
if rc != 0:
print cout
out.colorPrint('Error: Failed to delete LUN {}/{}/{}/{}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times), 'r')
error = '{} DE-{}: {}'.format(my_time, times, cout)
with open(log_file, 'a') as text_file: # Write errors into log file
text_file.write('{}'.format(error))
fail += 1
print 'Failed to delete {}/{} times'.format(fail, max_fail_times)
print cout
def cleanup():
# Connecting to MUT
client = SSHClient(client_host, client_username, client_password)
# Checking content of the destination folder
print "Checking if cleanup is needed ..."
rc, cout = client.run("ls -1 {}".format(pathto))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
# If file exists we try to delete it
if filename in cout:
out.colorPrint('Found file you trying to copy, initiating cleanup ...',
'r')
rc, cout = client.run('rm {}/{}'.format(pathto, filename))
if rc != 0:
out.colorPrint('Cleanup has failed', 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
out.colorPrint('Cleanup has finished successfully\n', 'g')
else:
out.colorPrint('Cleanup is not needed\n', 'g')
def create_ctb():
# Connecting to MUT
client = SSHClient(neadm_host, neadm_username, neadm_password)
# Show service to get information about preferred node and list of nodes that are in the service
rc, cout = client.run('/neadm/neadm cluster create {}'.format(cluster_name))
if rc != 0:
print cout
out.colorPrint('Failed to create cluster {}'.format(cluster_name), 'r')
exit(1)
print cout
time.sleep(2)
rc, cout = client.run('/neadm/neadm tenant create {}/{}'.format(cluster_name, tenant_name))
if rc != 0:
print cout
out.colorPrint('Failed to create tenant {}/{}'.format(cluster_name, tenant_name), 'r')
exit(1)
print cout
time.sleep(2)
rc, cout = client.run('/neadm/neadm bucket create {}/{}/{}'.format(cluster_name, tenant_name, bucket_name))
if rc != 0:
print cout
out.colorPrint('Failed to create bucket {}/{}/{}'.format(cluster_name, tenant_name, bucket_name), 'r')
exit(1)
print cout
def run1():
timeout = 300 # 5 minute timeout for copy
print(
"Executing failover after copying file onto mounted iSCSI share and checking diff: %s times"
% runtimes)
copyFrom = getSize(pathfrom)
copy_file()
r = 2
print 'WAITING TO FINISH COPY: Checking if file was fully copyed from %s to %s' % (
pathfrom, pathto)
try:
with Timeout(timeout):
while r == 2:
time.sleep(3)
copyTo = getSize(pathto)
if copyTo == copyFrom:
out.colorPrint(
'WAITING TO FINISH COPY: File is fully copyed, %s/%s\n'
% (copyTo, copyFrom), 'g')
r = 0
else:
print 'WAITING TO FINISH COPY: File is still copying ... %s/%s' % (
copyTo, copyFrom)
r = 2
except Timeout.Timeout:
out.colorPrint('ERROR: Timeout exeeded %s seconds' % timeout, 'r')
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Search for cp process
rc, cout = client.run('ps -ef | grep cp {}/{}'.format(
pathto, filename))
text = re.split('\n', cout)
for line in text:
print line
if filename in line:
out.colorPrint('ERROR: Copying file is taking too long', 'r')
print line
print 'Killing cp process ...'
rc, cout = client.run('pkill cp')
cleanup()
exit(1)
if filename not in line:
out.colorPrint(
'ERROR: When timeout exeeded its limit, copy process was not found, it must have been interrupted for some reason',
'r')
cleanup()
exit(1)
else:
out.colorPrint('ERROR: Unknown Error', 'r')
cleanup()
exit(1)
time.sleep(3)
check_diff()
failover()
time.sleep(3)
check_diff()
delete_file()
def create_service():
client = SSHClient(neadm_host, neadm_username, neadm_password)
rc, cout = client.run('/neadm/neadm service create iscsi %s' % service_name)
print cout
rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node1))
print cout
rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node2))
print cout
rc, cout = client.run('/neadm/neadm service vip add {} {} {}/24 -p {}'.format(service_name, nedge_quorum, nedge_vip, nedge_node1))
print cout
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
print cout
text = re.split('\n', cout)
nodeid = []
for line in text:
if 'X-Servers' in line:
match = re.search('X-Servers\s+(\w+)\D(\w+)', line)
if match:
nodeid = [match.group(1), match.group(2)]
else:
out.colorPrint('Couldn\'t match service nodes IDs', 'r')
exit(1)
if len(nodeid) != 2:
out.colorPrint('Not enough nodes in the service, should be 2', 'r')
exit(1)
print 'Node IDs: {}, {}'.format(nodeid[0], nodeid[1])
rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[0], client_net_name, client_net_ip1))
print cout
rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[1], client_net_name, client_net_ip2))
print cout
rc, cout = client.run('/neadm/neadm service enable %s' % service_name)
if rc != 0:
print cout
out.colorPrint('Error: unable to enable service: %s' % service_name, 'r')
exit(1)
print cout
rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1]))
if rc != 0:
print cout
out.colorPrint('Error: unable to create LUN {}/{}/{}/{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1]), 'r')
exit(1)
print cout
def getSize(path):
# Connecting to MUT
client = SSHClient(client_host, client_username, client_password)
rc, cout = client.run('ls -la {}/{}'.format(path, filename))
text = re.split('\n', cout)
if rc != 0:
out.colorPrint(
'Failed to execute \"ls -la {}/{}\" command'.format(
path, filename), 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
# Searching for size of file
for line in text:
match = re.search(
'\S+\s+1\s+\S+\s\S+\s(\d+)\s+\S+\s+\S+\s+\S+\s+{}/{}'.format(
path, filename), line)
if match:
return match.group(1)
else:
print out.colorPrint(
'Couldn\'t match regex to find size of {}/{}'.format(
path, filename), 'r')
exit(1)
def cleanup_ctb():
client = SSHClient(neadm_host, neadm_username, neadm_password)
print 'Checking if cleanup cluster is needed ...'
rc, cout = client.run('/neadm/neadm cluster list')
if cluster_name in cout:
out.colorPrint('Found cluster, preparing to cleanup', 'r')
rc, cout = client.run('/neadm/neadm bucket delete {}/{}/{}'.format(cluster_name, tenant_name, bucket_name))
print cout
rc, cout = client.run('/neadm/neadm tenant delete {}/{}'.format(cluster_name, tenant_name))
print cout
rc, cout = client.run('/neadm/neadm cluster delete {}'.format(cluster_name))
print cout
else:
out.colorPrint('Cleanup cluster is not needed\n', 'g')
def copy_file():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Copy file to directory
print 'Copying {} to {} ...'.format(filename, pathto)
rc, cout = client.run('cp {}/{} {}/.'.format(pathfrom, filename, pathto))
# Check if the file is in the directory
rc, cout = client.run('ls -lah {}'.format(pathto))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
def delete_file():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Delete file from directory
print 'Deleting {} from {} ...'.format(filename, pathto)
rc, cout = client.run('rm {}/{}'.format(pathto, filename))
# Check if the file was actually removed from the directory
rc, cout = client.run('ls -1 {}'.format(pathto))
if rc == None:
exit(1)
elif rc != 0:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
print 'Console output:\n%s' % cout
if filename not in cout:
out.colorPrint('File was deleted successfully\n', 'g')
else:
out.colorPrint('Error: File was not deleted\n', 'r')
exit(1)
def cleanup_service():
client = SSHClient(neadm_host, neadm_username, neadm_password)
print 'Checking if cleanup service is needed ...'
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
if rc == 0:
out.colorPrint('Found service, preparing to cleanup ...\n', 'r')
rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name))
print cout
rc, cout = client.run('/neadm/neadm service disable %s' % service_name)
print cout
rc, cout = client.run('/neadm/neadm service vip delete {} {}/24'.format(service_name, nedge_vip))
print cout
rc, cout = client.run('/neadm/neadm service delete %s' % service_name)
print cout
out.colorPrint('Cleanup has finished successfully\n', 'g')
else:
out.colorPrint('Cleanup service is not needed\n', 'g')
def cleanup_client():
client = SSHClient(client_host, client_username, client_password)
cl = 'yes'
print 'Checking if cleanup client is needed ...'
rc, cout = client.run('mount')
text = re.split('\n', cout)
for line in text:
if pathto in line:
print line
match = re.search('(\S+)\s+on\s+({})\s+\w+\s+\w+\s+\S+'.format(pathto), line)
if match:
out.colorPrint('Found LUN \"{}\" mounted onto test filesystem \"{}\", cleanup is needed'.format(match.group(1), match.group(2)), 'r')
rc, cout = client.run('umount %s' % pathto)
print cout
rc, cout = client.run('rm -r %s' % pathto)
print cout
rc, cout = client.run('iscsiadm -m node')
print cout
text = re.split('\n', cout)
portal = ''
targetname = ''
for line in text:
if nedge_vip in line:
match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line)
if match:
portal = match.group(1)
targetname = match.group(2)
else:
print 'Couldn\'t match'
exit(1)
print 'Portal: %s' % portal
print 'IQN: %s' % targetname
rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --logout'.format(targetname, portal))
print cout
rc, cout = client.run('iscsiadm -m node -o delete --targetname {} --portal {}'.format(targetname, portal))
print cout
cl = 'no'
else:
out.colorPrint('Couldn\'t match mounted filesystem name', 'r')
exit(1)
if cl == 'yes':
out.colorPrint('Cleanup client is not needed\n', 'g')
def check_diff():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Checking diff
print 'Checking diff between files {}/{} and {}/{}'.format(
pathfrom, filename, pathto, filename)
rc, cout = client.run('diff {}/{} {}/{}'.format(pathfrom, filename, pathto,
filename))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
out.colorPrint('Diff success\n', 'g')
else:
out.colorPrint('Failed to match diffs', 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
class CloudDeploy(object):
def __init__(self, dha, fuel_ip, fuel_username, fuel_password, dea_file,
work_dir):
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.work_dir = work_dir
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.macs_file = '%s/macs.yaml' % self.file_dir
self.node_ids = self.dha.get_node_ids()
def upload_cloud_deployment_files(self):
dest ='~/%s/' % self.work_dir
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, check=False)
s.exec_cmd('mkdir ~/%s' % self.work_dir)
s.scp_put(self.dea_file, dest)
s.scp_put(self.macs_file, dest)
s.scp_put('%s/common.py' % self.file_dir, dest)
s.scp_put('%s/dea.py' % self.file_dir, dest)
for f in glob.glob('%s/cloud/*' % self.file_dir):
s.scp_put(f, dest)
def power_off_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_off(node_id)
def power_on_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_on(node_id)
def set_boot_order(self, boot_order_list):
for node_id in self.node_ids:
self.dha.node_set_boot_order(node_id, boot_order_list)
def get_mac_addresses(self):
macs_per_node = {}
for node_id in self.node_ids:
macs_per_node[node_id] = self.dha.get_node_pxe_mac(node_id)
with io.open(self.macs_file, 'w') as stream:
yaml.dump(macs_per_node, stream, default_flow_style=False)
def run_cloud_deploy(self, deploy_app):
log('START CLOUD DEPLOYMENT')
deploy_app = '%s/%s' % (self.work_dir, deploy_app)
dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
macs_file = '%s/%s' % (self.work_dir, os.path.basename(self.macs_file))
with self.ssh:
self.ssh.run('python %s %s %s' % (deploy_app, dea_file, macs_file))
def deploy(self):
self.power_off_nodes()
self.set_boot_order(['pxe', 'disk'])
self.power_on_nodes()
self.get_mac_addresses()
check_file_exists(self.macs_file)
self.upload_cloud_deployment_files()
self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
def failover():
# Connecting to MUT
client = SSHClient(neadm_host, neadm_username, neadm_password)
# Show service to get information about preferred node and list of nodes that are in the service
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
text = re.split('\n', cout)
pref_node = ''
nodes = []
for line in text:
print line
if 'X-VIPS' in line: # Getting preferred node
match = re.search('X-VIPS\s+\[\[\"(\S+)\"\W\{\S+', line)
if match:
pref_node = match.group(1)
else:
out.colorPrint('Couldn\'t match preferred node', 'r')
exit(1)
elif 'X-VIP-Nodes' in line: # Getting list of nodes
match = re.search('X-VIP-Nodes\s+\[\"(\S+)\".\"(\S+)\"\]', line)
if match:
nodes = [match.group(1), match.group(2)]
else:
out.colorPrint('Couldn\'t match service nodes', 'r')
exit(1)
# If any of there are still 0 something is wrong
if len(pref_node) == 0 or len(nodes) == 0:
out.colorPrint('Couldn\'t get necessary information from service show',
'r')
exit(1)
#print len(pref_node), len(nodes)
print 'Nodes in service: \"{}\", \"{}\"'.format(nodes[0], nodes[1])
print 'Current preferred node is \"%s\"\n' % pref_node
# Check if service has less then 2 nodes then do not perform failover
if len(nodes) < 2:
out.colorPrint(
'Unable to perform failover, because service has less then 2 nodes',
'r')
exit(1)
# Figuring out which node to perform failover to
failover_node = ''
for node in nodes:
if node != pref_node:
failover_node = node
copyFrom = getSize(pathfrom)
r = 0
print 'WAITING FOR FAILOVER: Comparing sizes of {}/{} and {}/{}'.format(
pathfrom, filename, pathto, filename)
while r == 0:
time.sleep(3)
copyTo = getSize(pathto)
if int(copyTo) > int(copyFrom) / 2:
out.colorPrint(
'WAITING FOR FAILOVER: At least half the file was copyed successfully, %s/%s\n'
% (copyTo, copyFrom), 'g')
r = 1
else:
print 'WAITING FOR FAILOVER: File is still copying ... %s/%s' % (
copyTo, copyFrom)
r = 0
print 'Performing failover from \"{}\" to \"{}\"'.format(
pref_node, failover_node)
rc, cout = client.run(
'/neadm/neadm service vip config {} preferred {}'.format(
service_name, failover_node))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
def mount_iscsi_lun():
client = SSHClient(client_host, client_username, client_password)
rc, cout = client.run('iscsiadm -m discovery -t sendtargets -p %s' % nedge_vip)
if rc != 0:
print cout
out.colorPrint('Error: unable to find iSCSI target via IP: %s' % nedge_vip, 'r')
exit(1)
out.colorPrint('Successfully discovered target via %s\n' % nedge_vip, 'g')
text = re.split('\n', cout)
portal = ''
targetname = ''
for line in text:
if nedge_vip in line:
match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line)
if match:
portal = match.group(1)
targetname = match.group(2)
else:
print 'Couldn\'t match'
exit(1)
print 'Portal: %s' % portal
print 'IQN: %s' % targetname
rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --login'.format(targetname, portal))
if rc != 0:
out.colorPrint(cout, 'r')
out.colorPrint('Error: Couldnot login to iSCSI target', 'r')
exit(1)
out.colorPrint(cout, 'g')
rc, cout = client.run('fdisk -l | grep sd')
print cout
text = re.split('\n', cout)
diskname = ''
for line in text:
if lun_size[0] in line:
print line
match = re.search('Disk\s(\W\w+\W\w+):\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+', line)
if match:
diskname = match.group(1)
else:
print 'Couldn\'t match'
exit(1)
out.colorPrint('Found iSCSI LUN, disk name is %s\n' % diskname, 'g')
rc, cout = client.run('mkfs.ext4 %s' % diskname)
if rc != 0:
print cout
out.colorPrint('Couldn\'t create filesystem on iSCSI LUN %s' % diskname, 'r')
exit(1)
print cout
out.colorPrint('Filesystem was successfully create on top of %s' % diskname,'g')
rc, cout = client.run('mkdir %s' % pathto)
if rc != 0:
out.colorPrint('Error: %s' % cout, 'r')
out.colorPrint('Couldn\'t create directory %s' % pathto, 'r')
exit(1)
print cout
out.colorPrint('{} was successfully created\n'.format(pathto), 'g')
rc, cout = client.run('mount {} {}'.format(diskname, pathto))
print cout
if rc != 0:
out.colorPrint('Error: %s' % cout,'r')
out.colorPrint('Error: Unable to mount {} to {}'.format(diskname, pathto),'r')
exit(1)
print cout
out.colorPrint('{} was successfully mounted to {}'.format(diskname, pathto),'g')