def create_ctb():
# Connecting to MUT
client = SSHClient(neadm_host, neadm_username, neadm_password)
# Show service to get information about preferred node and list of nodes that are in the service
rc, cout = client.run('/neadm/neadm cluster create {}'.format(cluster_name))
if rc != 0:
print cout
out.colorPrint('Failed to create cluster {}'.format(cluster_name), 'r')
exit(1)
print cout
time.sleep(2)
rc, cout = client.run('/neadm/neadm tenant create {}/{}'.format(cluster_name, tenant_name))
if rc != 0:
print cout
out.colorPrint('Failed to create tenant {}/{}'.format(cluster_name, tenant_name), 'r')
exit(1)
print cout
time.sleep(2)
rc, cout = client.run('/neadm/neadm bucket create {}/{}/{}'.format(cluster_name, tenant_name, bucket_name))
if rc != 0:
print cout
out.colorPrint('Failed to create bucket {}/{}/{}'.format(cluster_name, tenant_name, bucket_name), 'r')
exit(1)
print cout
def cleanup():
# Connecting to MUT
client = SSHClient(client_host, client_username, client_password)
# Checking content of the destination folder
print "Checking if cleanup is needed ..."
rc, cout = client.run("ls -1 {}".format(pathto))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
# If file exists we try to delete it
if filename in cout:
out.colorPrint('Found file you trying to copy, initiating cleanup ...',
'r')
rc, cout = client.run('rm {}/{}'.format(pathto, filename))
if rc != 0:
out.colorPrint('Cleanup has failed', 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
out.colorPrint('Cleanup has finished successfully\n', 'g')
else:
out.colorPrint('Cleanup is not needed\n', 'g')
def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
deploy_timeout, no_deploy_environment, deploy_log):
self.dea = dea
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.updated_dea_file = (
'%s/.%s' %
(os.path.dirname(self.dea_file), os.path.basename(self.dea_file)))
shutil.copy2(self.dea_file, self.updated_dea_file)
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.work_dir = work_dir
self.no_health_check = no_health_check
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.deploy_log = deploy_log
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.node_ids = self.dha.get_node_ids()
self.wanted_release = self.dea.get_property('wanted_release')
self.blade_node_dict = {}
self.macs_per_blade = {}
def create_delete_lun_overnight():
client = SSHClient(neadm_host, neadm_username, neadm_password)
fail = 0
times = 0
print 'Executing this test %s times' % max_run_times
while (times < max_run_times) & (fail < max_fail_times):
my_time = time.strftime("%Y-%m-%d %H:%M:%S")
times += 1
print 'Running time: %s' % times
rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{}{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1]))
if rc != 0:
print cout
out.colorPrint('Error: Failed to create LUN {}/{}/{}/{}{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times, lun_size[0], lun_size[1]), 'r')
error = '{} CE-{}: {}'.format(my_time, times, cout)
with open(log_file, 'a') as text_file: # Write errors into log file
text_file.write('{}'.format(error))
print cout
rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, times))
if rc != 0:
print cout
out.colorPrint('Error: Failed to delete LUN {}/{}/{}/{}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, times), 'r')
error = '{} DE-{}: {}'.format(my_time, times, cout)
with open(log_file, 'a') as text_file: # Write errors into log file
text_file.write('{}'.format(error))
fail += 1
print 'Failed to delete {}/{} times'.format(fail, max_fail_times)
print cout
def getSize(path):
# Connecting to MUT
client = SSHClient(client_host, client_username, client_password)
rc, cout = client.run('ls -la {}/{}'.format(path, filename))
text = re.split('\n', cout)
if rc != 0:
out.colorPrint(
'Failed to execute \"ls -la {}/{}\" command'.format(
path, filename), 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
# Searching for size of file
for line in text:
match = re.search(
'\S+\s+1\s+\S+\s\S+\s(\d+)\s+\S+\s+\S+\s+\S+\s+{}/{}'.format(
path, filename), line)
if match:
return match.group(1)
else:
print out.colorPrint(
'Couldn\'t match regex to find size of {}/{}'.format(
path, filename), 'r')
exit(1)
def run1():
timeout = 300 # 5 minute timeout for copy
print(
"Executing failover after copying file onto mounted iSCSI share and checking diff: %s times"
% runtimes)
copyFrom = getSize(pathfrom)
copy_file()
r = 2
print 'WAITING TO FINISH COPY: Checking if file was fully copyed from %s to %s' % (
pathfrom, pathto)
try:
with Timeout(timeout):
while r == 2:
time.sleep(3)
copyTo = getSize(pathto)
if copyTo == copyFrom:
out.colorPrint(
'WAITING TO FINISH COPY: File is fully copyed, %s/%s\n'
% (copyTo, copyFrom), 'g')
r = 0
else:
print 'WAITING TO FINISH COPY: File is still copying ... %s/%s' % (
copyTo, copyFrom)
r = 2
except Timeout.Timeout:
out.colorPrint('ERROR: Timeout exeeded %s seconds' % timeout, 'r')
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Search for cp process
rc, cout = client.run('ps -ef | grep cp {}/{}'.format(
pathto, filename))
text = re.split('\n', cout)
for line in text:
print line
if filename in line:
out.colorPrint('ERROR: Copying file is taking too long', 'r')
print line
print 'Killing cp process ...'
rc, cout = client.run('pkill cp')
cleanup()
exit(1)
if filename not in line:
out.colorPrint(
'ERROR: When timeout exeeded its limit, copy process was not found, it must have been interrupted for some reason',
'r')
cleanup()
exit(1)
else:
out.colorPrint('ERROR: Unknown Error', 'r')
cleanup()
exit(1)
time.sleep(3)
check_diff()
failover()
time.sleep(3)
check_diff()
delete_file()
def __init__(self, name, address=None, port=None,
user=None, password=None, jump=None, dict=None):
self.name = name
self.address = address
self.jump = jump
self.user = user
self.port = port
self.password = password
if dict:
self.read_from_dic(dict)
self.sshc = SSHClient(self)
self.has_access = False
self.config = dict
def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
deploy_timeout, no_deploy_environment):
self.dea = dea
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.updated_dea_file = (
'%s/.%s' % (os.path.dirname(self.dea_file),
os.path.basename(self.dea_file)))
shutil.copy2(self.dea_file, self.updated_dea_file)
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.work_dir = work_dir
self.no_health_check = no_health_check
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.node_ids = self.dha.get_node_ids()
self.wanted_release = self.dea.get_property('wanted_release')
self.blade_node_dict = {}
self.macs_per_blade = {}
def start_containers():
containers = []
for node_hostname, node_conf in SYS_CONF.items():
ssh_client = SSHClient(
node_hostname, node_conf["ssh"]["port"],
node_conf["ssh"]["username"], node_conf["ssh"]["key_filename"],
os.path.join(DIRNAME, "ssh",
"%s-%s" % (node_hostname, "start_containers")))
for (container_name, container_conf) in \
node_conf.get("containers", {}).items():
containers.append((container_conf["start_order"], node_hostname,
node_conf, container_name, ssh_client))
for current_start_order in sorted(
set([container[0] for container in containers])):
threads = []
for (start_order, node_hostname, node_conf, container_name,
ssh_client) in containers:
if start_order == current_start_order:
threads.append(
threading.Thread(target=start_container,
args=[
node_hostname, node_conf,
container_name, ssh_client
]))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
fuel_password, fuel_node_id, iso_file, work_dir,
fuel_plugins_dir, no_plugins):
self.dea_file = dea_file
self.dha = LibvirtAdapter(dha_file)
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.fuel_node_id = fuel_node_id
self.iso_file = iso_file
self.iso_dir = os.path.dirname(self.iso_file)
self.work_dir = work_dir
self.fuel_plugins_dir = fuel_plugins_dir
self.no_plugins = no_plugins
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
def node_set_boot_order(self, node_id, boot_order_list):
log('Set boot order %s on Node %s' % (boot_order_list, node_id))
ip, username, password = self.get_access_info(node_id)
ssh = SSHClient(ip, username, password)
with ssh as s:
for order, dev in enumerate(boot_order_list):
s.exec_cmd('set %s/%s bootorder=%s'
% (ROOT, DEV[dev], order + 1))
def check_diff():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Checking diff
print 'Checking diff between files {}/{} and {}/{}'.format(
pathfrom, filename, pathto, filename)
rc, cout = client.run('diff {}/{} {}/{}'.format(pathfrom, filename, pathto,
filename))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
out.colorPrint('Diff success\n', 'g')
else:
out.colorPrint('Failed to match diffs', 'r')
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
def copy_file():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Copy file to directory
print 'Copying {} to {} ...'.format(filename, pathto)
rc, cout = client.run('cp {}/{} {}/.'.format(pathfrom, filename, pathto))
# Check if the file is in the directory
rc, cout = client.run('ls -lah {}'.format(pathto))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
def cleanup_client():
client = SSHClient(client_host, client_username, client_password)
cl = 'yes'
print 'Checking if cleanup client is needed ...'
rc, cout = client.run('mount')
text = re.split('\n', cout)
for line in text:
if pathto in line:
print line
match = re.search('(\S+)\s+on\s+({})\s+\w+\s+\w+\s+\S+'.format(pathto), line)
if match:
out.colorPrint('Found LUN \"{}\" mounted onto test filesystem \"{}\", cleanup is needed'.format(match.group(1), match.group(2)), 'r')
rc, cout = client.run('umount %s' % pathto)
print cout
rc, cout = client.run('rm -r %s' % pathto)
print cout
rc, cout = client.run('iscsiadm -m node')
print cout
text = re.split('\n', cout)
portal = ''
targetname = ''
for line in text:
if nedge_vip in line:
match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line)
if match:
portal = match.group(1)
targetname = match.group(2)
else:
print 'Couldn\'t match'
exit(1)
print 'Portal: %s' % portal
print 'IQN: %s' % targetname
rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --logout'.format(targetname, portal))
print cout
rc, cout = client.run('iscsiadm -m node -o delete --targetname {} --portal {}'.format(targetname, portal))
print cout
cl = 'no'
else:
out.colorPrint('Couldn\'t match mounted filesystem name', 'r')
exit(1)
if cl == 'yes':
out.colorPrint('Cleanup client is not needed\n', 'g')
def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, fuel_password,
fuel_node_id, iso_file, work_dir):
self.dea_file = dea_file
self.dha = LibvirtAdapter(dha_file)
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.fuel_node_id = fuel_node_id
self.iso_file = iso_file
self.work_dir = work_dir
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
def __init__(self, dha, fuel_ip, fuel_username, fuel_password, dea_file,
work_dir):
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.work_dir = work_dir
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.macs_file = '%s/macs.yaml' % self.file_dir
self.node_ids = self.dha.get_node_ids()
def delete_file():
# Connection to MUT
client = SSHClient(client_host, client_username, client_password)
# Delete file from directory
print 'Deleting {} from {} ...'.format(filename, pathto)
rc, cout = client.run('rm {}/{}'.format(pathto, filename))
# Check if the file was actually removed from the directory
rc, cout = client.run('ls -1 {}'.format(pathto))
if rc == None:
exit(1)
elif rc != 0:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
print 'Console output:\n%s' % cout
if filename not in cout:
out.colorPrint('File was deleted successfully\n', 'g')
else:
out.colorPrint('Error: File was not deleted\n', 'r')
exit(1)
def func_wrapper(*args, **kwargs):
threads = []
for node_hostname, node_conf in SYS_CONF.items():
ssh_client = SSHClient(
node_hostname, node_conf["ssh"]["port"],
node_conf["ssh"]["username"], node_conf["ssh"]["key_filename"],
os.path.join(DIRNAME, "ssh",
"%s-%s" % (node_hostname, func.__name__)))
threads.append(
threading.Thread(target=func,
args=[node_hostname, node_conf, ssh_client]))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def build(self):
# self.load_kv("test.kv")
layout = BoxLayout(orientation='vertical')
# splitter_layout = BoxLayout(orientation='horizontal')
grid_layout = GridLayout(rows=2)
lab1 = LabelB(text="Local",
bold=True,
background_color=(1, 0, 0, 1),
height=30,
size_hint_y=None)
lab2 = LabelB(text="Remote",
bold=True,
background_color=(0, 1, 0, 1),
height=30,
size_hint_y=None)
# source_layout = BoxLayout(orientation='vertical')
# target_layout = BoxLayout(orientation='vertical')
ssh_client = SSHClient(UNAME, IP, PORT, PWD)
file_system_remote = FileSystemRemote(ssh_client)
self.file_chooser_source = FileChooserListView(rootpath=SOURCE_PATH)
self.file_chooser_target = FileChooserListView(
file_system=file_system_remote, rootpath=TARGET_PATH)
# source_layout.add_widget(lab1)
# source_layout.add_widget(self.file_chooser_source)
# target_layout.add_widget(lab2)
# target_layout.add_widget(self.file_chooser_target)
# splitter = Splitter(sizable_from='right')
# splitter.add_widget(source_layout)
# splitter_layout.add_widget(splitter)
# splitter_layout.add_widget(target_layout)
grid_layout.add_widget(lab1)
grid_layout.add_widget(lab2)
grid_layout.add_widget(self.file_chooser_source)
grid_layout.add_widget(self.file_chooser_target)
btn = Button(text="Sync it !",
font_size="20sp",
background_color=(0, 1, 0, 1),
color=(1, 1, 1, 1),
height=30,
size_hint_y=None)
btn.bind(on_press=self.sync)
self.btn = btn
self.update_view()
# layout.add_widget(splitter_layout)
layout.add_widget(grid_layout)
layout.add_widget(btn)
return layout
def run_job_on_worker(worek_info, call_id, job_payload):
"""
Install all the Lithops dependencies into the worker.
Runs the job
"""
instance_name, ip_address, instance_id = worek_info
logger.info('Going to setup {}, IP address {}'.format(
instance_name, ip_address))
ssh_client = SSHClient(ip_address, INTERNAL_SSH_CREDNTIALS)
wait_instance_ready(ssh_client)
# upload zip lithops package
logger.info('Uploading lithops files to VM instance {}'.format(ip_address))
ssh_client.upload_local_file('/opt/lithops/lithops_standalone.zip',
'/tmp/lithops_standalone.zip')
logger.info(
'Executing lithops installation process on VM instance {}'.format(
ip_address))
cmd = get_setup_cmd(instance_name, ip_address, instance_id)
ssh_client.run_remote_command(cmd, run_async=True)
ssh_client.close()
# Wait until the proxy is ready
wait_proxy_ready(ip_address)
job_payload['job_description']['call_id'] = call_id
executor_id = job_payload['job_description']['executor_id']
job_id = job_payload['job_description']['job_id']
call_key = '-'.join([executor_id, job_id, call_id])
url = "http://{}:{}/run".format(ip_address, PROXY_SERVICE_PORT)
logger.info('Making {} invocation on {}'.format(call_key, ip_address))
r = requests.post(url, data=json.dumps(job_payload))
response = r.json()
if 'activationId' in response:
logger.info('Invocation {} done. Activation ID: {}'.format(
call_key, response['activationId']))
else:
logger.error('Invocation {} failed: {}'.format(call_key,
response['error']))
def func_wrapper_inner(*args, **kwargs):
threads = []
for node_hostname, node_conf in SYS_CONF.items():
for container_name in node_conf.get("containers", {}):
if re.match(container_name_pat, container_name):
break
else:
continue
ssh_client = SSHClient(
node_hostname, node_conf["ssh"]["port"],
node_conf["ssh"]["username"],
node_conf["ssh"]["key_filename"],
os.path.join(DIRNAME, "ssh",
"%s-%s" % (node_hostname, func.__name__)))
threads.append(
threading.Thread(
target=func,
args=[node_hostname, node_conf, ssh_client]))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def cleanup_ctb():
client = SSHClient(neadm_host, neadm_username, neadm_password)
print 'Checking if cleanup cluster is needed ...'
rc, cout = client.run('/neadm/neadm cluster list')
if cluster_name in cout:
out.colorPrint('Found cluster, preparing to cleanup', 'r')
rc, cout = client.run('/neadm/neadm bucket delete {}/{}/{}'.format(cluster_name, tenant_name, bucket_name))
print cout
rc, cout = client.run('/neadm/neadm tenant delete {}/{}'.format(cluster_name, tenant_name))
print cout
rc, cout = client.run('/neadm/neadm cluster delete {}'.format(cluster_name))
print cout
else:
out.colorPrint('Cleanup cluster is not needed\n', 'g')
def cleanup_service():
client = SSHClient(neadm_host, neadm_username, neadm_password)
print 'Checking if cleanup service is needed ...'
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
if rc == 0:
out.colorPrint('Found service, preparing to cleanup ...\n', 'r')
rc, cout = client.run('/neadm/neadm iscsi delete {} {}/{}/{}/{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name))
print cout
rc, cout = client.run('/neadm/neadm service disable %s' % service_name)
print cout
rc, cout = client.run('/neadm/neadm service vip delete {} {}/24'.format(service_name, nedge_vip))
print cout
rc, cout = client.run('/neadm/neadm service delete %s' % service_name)
print cout
out.colorPrint('Cleanup has finished successfully\n', 'g')
else:
out.colorPrint('Cleanup service is not needed\n', 'g')
def connect_to_server():
global client
client = SSHClient(ip_or_url, server_ssh_port, username, password)
print('Successfully connected to server.')
class CloudDeploy(object):
def __init__(self, dha, fuel_ip, fuel_username, fuel_password, dea_file,
work_dir):
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.work_dir = work_dir
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.macs_file = '%s/macs.yaml' % self.file_dir
self.node_ids = self.dha.get_node_ids()
def upload_cloud_deployment_files(self):
dest ='~/%s/' % self.work_dir
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, check=False)
s.exec_cmd('mkdir ~/%s' % self.work_dir)
s.scp_put(self.dea_file, dest)
s.scp_put(self.macs_file, dest)
s.scp_put('%s/common.py' % self.file_dir, dest)
s.scp_put('%s/dea.py' % self.file_dir, dest)
for f in glob.glob('%s/cloud/*' % self.file_dir):
s.scp_put(f, dest)
def power_off_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_off(node_id)
def power_on_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_on(node_id)
def set_boot_order(self, boot_order_list):
for node_id in self.node_ids:
self.dha.node_set_boot_order(node_id, boot_order_list)
def get_mac_addresses(self):
macs_per_node = {}
for node_id in self.node_ids:
macs_per_node[node_id] = self.dha.get_node_pxe_mac(node_id)
with io.open(self.macs_file, 'w') as stream:
yaml.dump(macs_per_node, stream, default_flow_style=False)
def run_cloud_deploy(self, deploy_app):
log('START CLOUD DEPLOYMENT')
deploy_app = '%s/%s' % (self.work_dir, deploy_app)
dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
macs_file = '%s/%s' % (self.work_dir, os.path.basename(self.macs_file))
with self.ssh:
self.ssh.run('python %s %s %s' % (deploy_app, dea_file, macs_file))
def deploy(self):
self.power_off_nodes()
self.set_boot_order(['pxe', 'disk'])
self.power_on_nodes()
self.get_mac_addresses()
check_file_exists(self.macs_file)
self.upload_cloud_deployment_files()
self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
def create_service():
client = SSHClient(neadm_host, neadm_username, neadm_password)
rc, cout = client.run('/neadm/neadm service create iscsi %s' % service_name)
print cout
rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node1))
print cout
rc, cout = client.run('/neadm/neadm service add {} {}'.format(service_name, nedge_node2))
print cout
rc, cout = client.run('/neadm/neadm service vip add {} {} {}/24 -p {}'.format(service_name, nedge_quorum, nedge_vip, nedge_node1))
print cout
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
print cout
text = re.split('\n', cout)
nodeid = []
for line in text:
if 'X-Servers' in line:
match = re.search('X-Servers\s+(\w+)\D(\w+)', line)
if match:
nodeid = [match.group(1), match.group(2)]
else:
out.colorPrint('Couldn\'t match service nodes IDs', 'r')
exit(1)
if len(nodeid) != 2:
out.colorPrint('Not enough nodes in the service, should be 2', 'r')
exit(1)
print 'Node IDs: {}, {}'.format(nodeid[0], nodeid[1])
rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[0], client_net_name, client_net_ip1))
print cout
rc, cout = client.run('/neadm/neadm service configure {} X-Container-Network-{} \"{} --ip {}\"'.format(service_name, nodeid[1], client_net_name, client_net_ip2))
print cout
rc, cout = client.run('/neadm/neadm service enable %s' % service_name)
if rc != 0:
print cout
out.colorPrint('Error: unable to enable service: %s' % service_name, 'r')
exit(1)
print cout
rc, cout = client.run('/neadm/neadm iscsi create {} {}/{}/{}/{} {}{}'.format(service_name, cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1]))
if rc != 0:
print cout
out.colorPrint('Error: unable to create LUN {}/{}/{}/{} with the size {}{}'.format(cluster_name, tenant_name, bucket_name, lun_name, lun_size[0], lun_size[1]), 'r')
exit(1)
print cout
class Node(object):
def __init__(self, name, address=None, port=None,
user=None, password=None, jump=None, dict=None):
self.name = name
self.address = address
self.jump = jump
self.user = user
self.port = port
self.password = password
if dict:
self.read_from_dic(dict)
self.sshc = SSHClient(self)
self.has_access = False
self.config = dict
def read_from_dic(self, dic):
allowed_keys = ['address', 'user', 'jump', 'password', 'port']
for (key, value) in dic.iteritems():
if key in allowed_keys:
setattr(self, key, value)
def ping(self, ip):
self.execute(['ping', '-c', '1', ip])
def execute(self, cmd, **kwargs):
return self.sshc.execute(cmd, **kwargs)
def chown(self, user, path):
self.execute('chown -R %(user)s:%(user)s %(path)s' % {'user': user,
'path': path},
as_root=True)
def is_dir(self, path):
rv, _ = self.execute('test -d %s && echo yes' % path,
check_exit_code=[0, 1])
if rv == 'yes\n':
return True
else:
return False
def is_file(self, path):
rv, _ = self.execute('test -f %s && echo yes' % path,
check_exit_code=[0, 1])
if rv == 'yes\n':
return True
else:
return False
def reboot(self):
self.execute('reboot', as_root=True, check_exit_code=[255])
def create_path_if_not_exsist(self, path, **kwargs):
return self.sshc.execute('mkdir -p %s' % path, **kwargs)
def copy(self, direction, local_path, remote_path, **kwargs):
return self.sshc.copy(direction, local_path, remote_path, **kwargs)
def to_ssh_config(self):
config = ["Host %s" % self.name,
" Hostname %s" %
(self.address if self.address else self.name)]
if self.jump:
config.append(" ProxyCommand ssh -F %(config_path)s "
"-W %%h:%%p %(name)s"
% {'config_path': SshUtil.get_config_file_path(),
'name': self.jump.name})
if self.user:
config.append(" user %s" % self.user)
if self.port:
config.append(" port %s" % self.port)
return '\n'.join(config)
# attaching EBS volume to instance
waiter = client_ec2.get_waiter('volume_available')
waiter.wait(VolumeIds=[volume_id])
try:
client_ec2.attach_volume(VolumeId=volume_id,
InstanceId=instance.instance_id,
Device='/dev/xvdf')
print("EBS attached")
attached = True
except Exception as e:
print("Error while attaching EBS " + e)
attached = False
if attached:
ssh = SSHClient(instance.public_ip_address,
path_to_key_pair + key_pair_name + '.pem')
connected = ssh.open_connection()
if (connected):
print("Conection successful")
stdout, stderr, retcode = ssh.execute_command('ls', output=True)
print("Command ls executed")
print(stdout)
# mounting EBS volume to ebs_path
command = 'mkdir {}'.format(ebs_path)
stdout, stderr, retcode = ssh.execute_command(command, output=True)
print("Command mkdir executed")
print(stdout)
command = 'sudo mount {} {}'.format(
internal_device_name[type_instance], ebs_path)
stdout, stderr, retcode = ssh.execute_command(command, output=True)
print("Command '" + command + "' executed")
def mount_iscsi_lun():
client = SSHClient(client_host, client_username, client_password)
rc, cout = client.run('iscsiadm -m discovery -t sendtargets -p %s' % nedge_vip)
if rc != 0:
print cout
out.colorPrint('Error: unable to find iSCSI target via IP: %s' % nedge_vip, 'r')
exit(1)
out.colorPrint('Successfully discovered target via %s\n' % nedge_vip, 'g')
text = re.split('\n', cout)
portal = ''
targetname = ''
for line in text:
if nedge_vip in line:
match = re.search('({}:\d+),\S+\s+(\S+)'.format(nedge_vip), line)
if match:
portal = match.group(1)
targetname = match.group(2)
else:
print 'Couldn\'t match'
exit(1)
print 'Portal: %s' % portal
print 'IQN: %s' % targetname
rc, cout = client.run('iscsiadm -m node --targetname {} --portal {} --login'.format(targetname, portal))
if rc != 0:
out.colorPrint(cout, 'r')
out.colorPrint('Error: Couldnot login to iSCSI target', 'r')
exit(1)
out.colorPrint(cout, 'g')
rc, cout = client.run('fdisk -l | grep sd')
print cout
text = re.split('\n', cout)
diskname = ''
for line in text:
if lun_size[0] in line:
print line
match = re.search('Disk\s(\W\w+\W\w+):\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+', line)
if match:
diskname = match.group(1)
else:
print 'Couldn\'t match'
exit(1)
out.colorPrint('Found iSCSI LUN, disk name is %s\n' % diskname, 'g')
rc, cout = client.run('mkfs.ext4 %s' % diskname)
if rc != 0:
print cout
out.colorPrint('Couldn\'t create filesystem on iSCSI LUN %s' % diskname, 'r')
exit(1)
print cout
out.colorPrint('Filesystem was successfully create on top of %s' % diskname,'g')
rc, cout = client.run('mkdir %s' % pathto)
if rc != 0:
out.colorPrint('Error: %s' % cout, 'r')
out.colorPrint('Couldn\'t create directory %s' % pathto, 'r')
exit(1)
print cout
out.colorPrint('{} was successfully created\n'.format(pathto), 'g')
rc, cout = client.run('mount {} {}'.format(diskname, pathto))
print cout
if rc != 0:
out.colorPrint('Error: %s' % cout,'r')
out.colorPrint('Error: Unable to mount {} to {}'.format(diskname, pathto),'r')
exit(1)
print cout
out.colorPrint('{} was successfully mounted to {}'.format(diskname, pathto),'g')
class CloudDeploy(object):
def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
deploy_timeout, no_deploy_environment):
self.dea = dea
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.updated_dea_file = (
'%s/.%s' % (os.path.dirname(self.dea_file),
os.path.basename(self.dea_file)))
shutil.copy2(self.dea_file, self.updated_dea_file)
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.work_dir = work_dir
self.no_health_check = no_health_check
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.node_ids = self.dha.get_node_ids()
self.wanted_release = self.dea.get_property('wanted_release')
self.blade_node_dict = {}
self.macs_per_blade = {}
def merge_plugin_config_files_to_dea_file(self):
plugins_conf_dir = (
self.fuel_plugins_conf_dir if self.fuel_plugins_conf_dir
else '%s/plugins_conf' % os.path.dirname(self.dea_file))
if os.path.isdir(plugins_conf_dir):
with io.open(self.updated_dea_file) as stream:
updated_dea = yaml.load(stream)
for plugin_file in glob.glob('%s/*.yaml' % plugins_conf_dir):
with io.open(plugin_file) as stream:
plugin_conf = yaml.load(stream)
updated_dea['settings']['editable'].update(plugin_conf)
with io.open(self.updated_dea_file, 'w') as stream:
yaml.dump(updated_dea, stream, default_flow_style=False)
def upload_cloud_deployment_files(self):
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, False)
s.exec_cmd('mkdir %s' % self.work_dir)
s.scp_put(self.updated_dea_file, '%s/%s' % (
self.work_dir, os.path.basename(self.dea_file)))
s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
for f in glob.glob('%s/cloud/*' % self.file_dir):
s.scp_put(f, self.work_dir)
def power_off_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_off(node_id)
def power_on_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_on(node_id)
def set_boot_order(self, boot_order_list):
for node_id in self.node_ids:
self.dha.node_set_boot_order(node_id, boot_order_list[:])
def get_mac_addresses(self):
self.macs_per_blade = {}
for node_id in self.node_ids:
self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
def run_cloud_deploy(self, deploy_app):
log('START CLOUD DEPLOYMENT')
deploy_app = '%s/%s' % (self.work_dir, deploy_app)
dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
with self.ssh as s:
status = s.run('python %s %s %s %s %s' % (
deploy_app,
('-nh' if self.no_health_check else ''),
('-dt %s' %
self.deploy_timeout if self.deploy_timeout else ''),
('-nde' if self.no_deploy_environment else ''),
dea_file))
return status
def check_supported_release(self):
log('Check supported release: %s' % self.wanted_release)
found = False
release_list = parse(self.ssh.exec_cmd('fuel release -l'))
for release in release_list:
if release[R['name']] == self.wanted_release:
found = True
break
if not found:
err('This Fuel does not contain the following release: %s'
% self.wanted_release)
def check_previous_installation(self):
log('Check previous installation')
env_list = parse(self.ssh.exec_cmd('fuel env list'))
if env_list:
self.cleanup_fuel_environments(env_list)
node_list = parse(self.ssh.exec_cmd('fuel node list'))
if node_list:
self.cleanup_fuel_nodes(node_list)
def cleanup_fuel_environments(self, env_list):
WAIT_LOOP = 60
SLEEP_TIME = 10
for env in env_list:
log('Deleting environment %s' % env[E['id']])
self.ssh.exec_cmd('fuel env --env %s --delete --force'
% env[E['id']])
all_env_erased = False
for i in range(WAIT_LOOP):
env_list = parse(self.ssh.exec_cmd('fuel env list'))
if env_list:
time.sleep(SLEEP_TIME)
else:
all_env_erased = True
break
if not all_env_erased:
err('Could not erase these environments %s'
% [(env[E['id']], env[E['status']]) for env in env_list])
def cleanup_fuel_nodes(self, node_list):
for node in node_list:
if node[N['status']] == 'discover':
log('Deleting node %s' % node[N['id']])
self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
'--force' % node[N['id']])
self.ssh.exec_cmd('cobbler system remove --name node-%s'
% node[N['id']], False)
def check_prerequisites(self):
log('Check prerequisites')
with self.ssh:
self.check_supported_release()
self.check_previous_installation()
def wait_for_discovered_blades(self):
log('Wait for discovered blades')
discovered_macs = []
restart_times = BLADE_RESTART_TIMES
for blade in self.node_ids:
self.blade_node_dict[blade] = None
with self.ssh:
all_discovered = self.discovery_waiting_loop(discovered_macs)
while not all_discovered and restart_times != 0:
restart_times -= 1
for blade in self.get_not_discovered_blades():
self.dha.node_reset(blade)
with self.ssh:
all_discovered = self.discovery_waiting_loop(discovered_macs)
if not all_discovered:
err('Not all blades have been discovered: %s'
% self.not_discovered_blades_summary())
with io.open(self.updated_dea_file) as stream:
updated_dea = yaml.load(stream)
updated_dea.update({'blade_node_map': self.blade_node_dict})
with io.open(self.updated_dea_file, 'w') as stream:
yaml.dump(updated_dea, stream, default_flow_style=False)
def discovery_waiting_loop(self, discovered_macs):
WAIT_LOOP = 360
SLEEP_TIME = 10
all_discovered = False
for i in range(WAIT_LOOP):
node_list = parse(self.ssh.exec_cmd('fuel node list'))
if node_list:
self.node_discovery(node_list, discovered_macs)
if self.all_blades_discovered():
all_discovered = True
break
else:
time.sleep(SLEEP_TIME)
return all_discovered
def node_discovery(self, node_list, discovered_macs):
for node in node_list:
if (node[N['status']] == 'discover' and
node[N['online']] == 'True' and
node[N['mac']] not in discovered_macs):
discovered_macs.append(node[N['mac']])
blade = self.find_mac_in_dict(node[N['mac']])
if blade:
log('Blade %s discovered as Node %s with MAC %s'
% (blade, node[N['id']], node[N['mac']]))
self.blade_node_dict[blade] = int(node[N['id']])
def find_mac_in_dict(self, mac):
for blade, mac_list in self.macs_per_blade.iteritems():
if mac in mac_list:
return blade
def all_blades_discovered(self):
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
return False
return True
def not_discovered_blades_summary(self):
summary = ''
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
summary += '\n[blade %s]' % blade
return summary
def get_not_discovered_blades(self):
not_discovered_blades = []
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
not_discovered_blades.append(blade)
return not_discovered_blades
def set_boot_order_nodes(self):
self.power_off_nodes()
self.set_boot_order(['pxe', 'disk'])
self.power_on_nodes()
def deploy(self):
self.set_boot_order_nodes()
self.check_prerequisites()
self.get_mac_addresses()
self.wait_for_discovered_blades()
self.merge_plugin_config_files_to_dea_file()
self.upload_cloud_deployment_files()
delete(self.updated_dea_file)
return self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
class CloudDeploy(object):
def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
deploy_timeout, no_deploy_environment, deploy_log):
self.dea = dea
self.dha = dha
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.dea_file = dea_file
self.updated_dea_file = (
'%s/.%s' %
(os.path.dirname(self.dea_file), os.path.basename(self.dea_file)))
shutil.copy2(self.dea_file, self.updated_dea_file)
self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
self.work_dir = work_dir
self.no_health_check = no_health_check
self.deploy_timeout = deploy_timeout
self.no_deploy_environment = no_deploy_environment
self.deploy_log = deploy_log
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
self.node_ids = self.dha.get_node_ids()
self.wanted_release = self.dea.get_property('wanted_release')
self.blade_node_dict = {}
self.macs_per_blade = {}
def merge_plugin_config_files_to_dea_file(self):
plugins_conf_dir = (self.fuel_plugins_conf_dir if
self.fuel_plugins_conf_dir else '%s/plugins_conf' %
os.path.dirname(self.dea_file))
if os.path.isdir(plugins_conf_dir):
with io.open(self.updated_dea_file) as stream:
updated_dea = yaml.load(stream)
for plugin_file in glob.glob('%s/*.yaml' % plugins_conf_dir):
with io.open(plugin_file) as stream:
plugin_conf = yaml.load(stream)
updated_dea['settings']['editable'].update(plugin_conf)
with io.open(self.updated_dea_file, 'w') as stream:
yaml.dump(updated_dea, stream, default_flow_style=False)
def upload_cloud_deployment_files(self):
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, False)
s.exec_cmd('mkdir %s' % self.work_dir)
s.scp_put(
self.updated_dea_file,
'%s/%s' % (self.work_dir, os.path.basename(self.dea_file)))
s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
for f in glob.glob('%s/cloud/*' % self.file_dir):
s.scp_put(f, self.work_dir)
def power_off_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_off(node_id)
def power_on_nodes(self):
for node_id in self.node_ids:
self.dha.node_power_on(node_id)
def set_boot_order(self, boot_order_list):
for node_id in self.node_ids:
self.dha.node_set_boot_order(node_id, boot_order_list[:])
def get_mac_addresses(self):
self.macs_per_blade = {}
for node_id in self.node_ids:
self.macs_per_blade[node_id] = self.dha.get_node_pxe_mac(node_id)
def run_cloud_deploy(self, deploy_app):
log('START CLOUD DEPLOYMENT')
deploy_app = '%s/%s' % (self.work_dir, deploy_app)
dea_file = '%s/%s' % (self.work_dir, os.path.basename(self.dea_file))
with self.ssh as s:
status = s.run(
'python %s %s %s %s %s' %
(deploy_app, ('-nh' if self.no_health_check else ''),
('-dt %s' %
self.deploy_timeout if self.deploy_timeout else ''),
('-nde' if self.no_deploy_environment else ''), dea_file))
return status
def check_supported_release(self):
log('Check supported release: %s' % self.wanted_release)
found = False
release_list = parse(self.ssh.exec_cmd('fuel release -l'))
for release in release_list:
if release[R['name']] == self.wanted_release:
found = True
break
if not found:
err('This Fuel does not contain the following release: %s' %
self.wanted_release)
def check_previous_installation(self):
log('Check previous installation')
env_list = parse(self.ssh.exec_cmd('fuel env list'))
if env_list:
self.cleanup_fuel_environments(env_list)
node_list = parse(self.ssh.exec_cmd('fuel node list'))
if node_list:
self.cleanup_fuel_nodes(node_list)
def cleanup_fuel_environments(self, env_list):
WAIT_LOOP = 60
SLEEP_TIME = 10
for env in env_list:
log('Deleting environment %s' % env[E['id']])
self.ssh.exec_cmd('fuel env --env %s --delete --force' %
env[E['id']])
all_env_erased = False
for i in range(WAIT_LOOP):
env_list = parse(self.ssh.exec_cmd('fuel env list'))
if env_list:
time.sleep(SLEEP_TIME)
else:
all_env_erased = True
break
if not all_env_erased:
err('Could not erase these environments %s' %
[(env[E['id']], env[E['status']]) for env in env_list])
def cleanup_fuel_nodes(self, node_list):
for node in node_list:
if node[N['status']] == 'discover':
log('Deleting node %s' % node[N['id']])
self.ssh.exec_cmd('fuel node --node-id %s --delete-from-db '
'--force' % node[N['id']])
self.ssh.exec_cmd(
'cobbler system remove --name node-%s' % node[N['id']],
False)
def check_prerequisites(self):
log('Check prerequisites')
with self.ssh:
self.check_supported_release()
self.check_previous_installation()
def wait_for_discovered_blades(self):
log('Wait for discovered blades')
discovered_macs = []
restart_times = BLADE_RESTART_TIMES
for blade in self.node_ids:
self.blade_node_dict[blade] = None
with self.ssh:
all_discovered = self.discovery_waiting_loop(discovered_macs)
while not all_discovered and restart_times != 0:
restart_times -= 1
for blade in self.get_not_discovered_blades():
self.dha.node_reset(blade)
with self.ssh:
all_discovered = self.discovery_waiting_loop(discovered_macs)
if not all_discovered:
err('Not all blades have been discovered: %s' %
self.not_discovered_blades_summary())
with io.open(self.updated_dea_file) as stream:
updated_dea = yaml.load(stream)
updated_dea.update({'blade_node_map': self.blade_node_dict})
with io.open(self.updated_dea_file, 'w') as stream:
yaml.dump(updated_dea, stream, default_flow_style=False)
def discovery_waiting_loop(self, discovered_macs):
WAIT_LOOP = 360
SLEEP_TIME = 10
all_discovered = False
for i in range(WAIT_LOOP):
node_list = parse(self.ssh.exec_cmd('fuel node list'))
if node_list:
self.node_discovery(node_list, discovered_macs)
if self.all_blades_discovered():
all_discovered = True
break
else:
time.sleep(SLEEP_TIME)
return all_discovered
def node_discovery(self, node_list, discovered_macs):
for node in node_list:
if (node[N['status']] == 'discover' and
(node[N['online']] == 'True' or node[N['online']] == '1')
and node[N['mac']] not in discovered_macs):
discovered_macs.append(node[N['mac']])
blade = self.find_mac_in_dict(node[N['mac']])
if blade:
log('Blade %s discovered as Node %s with MAC %s' %
(blade, node[N['id']], node[N['mac']]))
self.blade_node_dict[blade] = int(node[N['id']])
def find_mac_in_dict(self, mac):
for blade, mac_list in self.macs_per_blade.iteritems():
if mac in mac_list:
return blade
def all_blades_discovered(self):
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
return False
return True
def not_discovered_blades_summary(self):
summary = ''
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
summary += '\n[blade %s]' % blade
return summary
def get_not_discovered_blades(self):
not_discovered_blades = []
for blade, node_id in self.blade_node_dict.iteritems():
if not node_id:
not_discovered_blades.append(blade)
return not_discovered_blades
def set_boot_order_nodes(self):
self.power_off_nodes()
self.set_boot_order(['pxe', 'disk'])
self.power_on_nodes()
def get_put_deploy_log(self):
with self.ssh as s:
s.scp_get("deploy-*", local=self.deploy_log)
def deploy(self):
self.set_boot_order_nodes()
self.check_prerequisites()
self.get_mac_addresses()
self.wait_for_discovered_blades()
self.merge_plugin_config_files_to_dea_file()
self.upload_cloud_deployment_files()
delete(self.updated_dea_file)
rc = self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
self.get_put_deploy_log()
return rc
def failover():
# Connecting to MUT
client = SSHClient(neadm_host, neadm_username, neadm_password)
# Show service to get information about preferred node and list of nodes that are in the service
rc, cout = client.run('/neadm/neadm service show %s' % service_name)
text = re.split('\n', cout)
pref_node = ''
nodes = []
for line in text:
print line
if 'X-VIPS' in line: # Getting preferred node
match = re.search('X-VIPS\s+\[\[\"(\S+)\"\W\{\S+', line)
if match:
pref_node = match.group(1)
else:
out.colorPrint('Couldn\'t match preferred node', 'r')
exit(1)
elif 'X-VIP-Nodes' in line: # Getting list of nodes
match = re.search('X-VIP-Nodes\s+\[\"(\S+)\".\"(\S+)\"\]', line)
if match:
nodes = [match.group(1), match.group(2)]
else:
out.colorPrint('Couldn\'t match service nodes', 'r')
exit(1)
# If any of there are still 0 something is wrong
if len(pref_node) == 0 or len(nodes) == 0:
out.colorPrint('Couldn\'t get necessary information from service show',
'r')
exit(1)
#print len(pref_node), len(nodes)
print 'Nodes in service: \"{}\", \"{}\"'.format(nodes[0], nodes[1])
print 'Current preferred node is \"%s\"\n' % pref_node
# Check if service has less then 2 nodes then do not perform failover
if len(nodes) < 2:
out.colorPrint(
'Unable to perform failover, because service has less then 2 nodes',
'r')
exit(1)
# Figuring out which node to perform failover to
failover_node = ''
for node in nodes:
if node != pref_node:
failover_node = node
copyFrom = getSize(pathfrom)
r = 0
print 'WAITING FOR FAILOVER: Comparing sizes of {}/{} and {}/{}'.format(
pathfrom, filename, pathto, filename)
while r == 0:
time.sleep(3)
copyTo = getSize(pathto)
if int(copyTo) > int(copyFrom) / 2:
out.colorPrint(
'WAITING FOR FAILOVER: At least half the file was copyed successfully, %s/%s\n'
% (copyTo, copyFrom), 'g')
r = 1
else:
print 'WAITING FOR FAILOVER: File is still copying ... %s/%s' % (
copyTo, copyFrom)
r = 0
print 'Performing failover from \"{}\" to \"{}\"'.format(
pref_node, failover_node)
rc, cout = client.run(
'/neadm/neadm service vip config {} preferred {}'.format(
service_name, failover_node))
if rc == None:
out.colorPrint('ERROR: Unknown Error', 'r')
exit(1)
elif rc == 0:
print 'Console output:\n%s' % cout
else:
out.colorPrint('ReturnCode: %s' % rc, 'r')
out.colorPrint('Error: %s' % cout, 'r')
exit(1)
class InstallFuelMaster(object):
def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
fuel_password, fuel_node_id, iso_file, work_dir,
fuel_plugins_dir, no_plugins):
self.dea_file = dea_file
self.dha = LibvirtAdapter(dha_file)
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.fuel_node_id = fuel_node_id
self.iso_file = iso_file
self.iso_dir = os.path.dirname(self.iso_file)
self.work_dir = work_dir
self.fuel_plugins_dir = fuel_plugins_dir
self.no_plugins = no_plugins
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
def install(self):
log('Start Fuel Installation')
self.dha.node_power_off(self.fuel_node_id)
log('Zero the MBR')
self.dha.node_zero_mbr(self.fuel_node_id)
self.dha.node_set_boot_order(self.fuel_node_id, ['disk', 'iso'])
try:
self.proceed_with_installation()
except Exception as e:
self.post_install_cleanup()
err(e)
def proceed_with_installation(self):
log('Eject ISO')
self.dha.node_eject_iso(self.fuel_node_id)
log('Insert ISO %s' % self.iso_file)
self.dha.node_insert_iso(self.fuel_node_id, self.iso_file)
self.dha.node_power_on(self.fuel_node_id)
log('Waiting for Fuel master to accept SSH')
self.wait_for_node_up()
log('Wait until Fuel menu is up')
fuel_menu_pid = self.wait_until_fuel_menu_up()
log('Inject our own astute.yaml settings')
self.inject_own_astute_yaml()
log('Let the Fuel deployment continue')
log('Found FUEL menu as PID %s, now killing it' % fuel_menu_pid)
self.ssh_exec_cmd('kill %s' % fuel_menu_pid, False)
log('Wait until installation is complete')
self.wait_until_installation_completed()
log('Waiting for one minute for Fuel to stabilize')
time.sleep(60)
self.delete_deprecated_fuel_client_config()
if not self.no_plugins:
self.collect_plugin_files()
self.install_plugins()
self.post_install_cleanup()
log('Fuel Master installed successfully !')
def collect_plugin_files(self):
with self.ssh as s:
s.exec_cmd('mkdir %s' % PLUGINS_DIR)
if self.fuel_plugins_dir:
for f in glob.glob('%s/*.rpm' % self.fuel_plugins_dir):
s.scp_put(f, PLUGINS_DIR)
def install_plugins(self):
log('Installing Fuel Plugins')
plugin_files = []
with self.ssh as s:
for plugin_location in [PLUGINS_DIR, LOCAL_PLUGIN_FOLDER]:
s.exec_cmd('mkdir -p %s' % plugin_location)
r = s.exec_cmd('find %s -type f -name \'*.rpm\'' %
plugin_location)
plugin_files.extend(r.splitlines())
for f in plugin_files:
log('Found plugin %s, installing ...' % f)
r, e = s.exec_cmd('fuel plugins --install %s' % f, False)
printout = r + e if e else r
if e and all(
[err not in printout for err in IGNORABLE_FUEL_ERRORS]):
raise Exception('Installation of Fuel Plugin %s '
'failed: %s' % (f, e))
def wait_for_node_up(self):
WAIT_LOOP = 240
SLEEP_TIME = 10
success = False
for i in range(WAIT_LOOP):
try:
self.ssh.open()
success = True
break
except Exception:
log('Trying to SSH into Fuel VM %s ... sleeping %s seconds' %
(self.fuel_ip, SLEEP_TIME))
time.sleep(SLEEP_TIME)
finally:
self.ssh.close()
if not success:
raise Exception('Could not SSH into Fuel VM %s' % self.fuel_ip)
def wait_until_fuel_menu_up(self):
WAIT_LOOP = 60
SLEEP_TIME = 10
CMD = 'ps -ef'
SEARCH = 'fuelmenu'
fuel_menu_pid = None
with self.ssh:
for i in range(WAIT_LOOP):
ret = self.ssh.exec_cmd(CMD)
fuel_menu_pid = self.get_fuel_menu_pid(ret, SEARCH)
if not fuel_menu_pid:
time.sleep(SLEEP_TIME)
else:
break
if not fuel_menu_pid:
raise Exception('Could not find the Fuel Menu Process ID')
return fuel_menu_pid
def get_fuel_menu_pid(self, printout, search):
for line in printout.splitlines():
if line.endswith(search):
return clean(line)[1]
def ssh_exec_cmd(self, cmd, check=True):
with self.ssh:
ret = self.ssh.exec_cmd(cmd, check=check)
return ret
def inject_own_astute_yaml(self):
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, False)
s.exec_cmd('mkdir %s' % self.work_dir)
s.scp_put(self.dea_file, self.work_dir)
s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
s.scp_put('%s/transplant_fuel_settings.py' % self.file_dir,
self.work_dir)
log('Modifying Fuel astute')
s.run('python %s/%s %s/%s' %
(self.work_dir, TRANSPLANT_FUEL_SETTINGS, self.work_dir,
os.path.basename(self.dea_file)))
def wait_until_installation_completed(self):
WAIT_LOOP = 360
SLEEP_TIME = 10
CMD = 'ps -ef | grep %s | grep -v grep' % BOOTSTRAP_ADMIN
install_completed = False
with self.ssh:
for i in range(WAIT_LOOP):
ret = self.ssh.exec_cmd(CMD)
if not ret:
install_completed = True
break
else:
time.sleep(SLEEP_TIME)
if not install_completed:
raise Exception('Fuel installation did not complete')
def post_install_cleanup(self):
log('Eject ISO file %s' % self.iso_file)
self.dha.node_eject_iso(self.fuel_node_id)
delete(self.iso_dir)
def delete_deprecated_fuel_client_config(self):
with self.ssh as s:
response, error = s.exec_cmd('fuel -v', False)
if (error and 'DEPRECATION WARNING' in error
and FUEL_CLIENT_CONFIG in error):
log('Delete deprecated fuel client config %s' % FUEL_CLIENT_CONFIG)
with self.ssh as s:
s.exec_cmd('rm %s' % FUEL_CLIENT_CONFIG, False)
class InstallFuelMaster(object):
def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, fuel_password,
fuel_node_id, iso_file, work_dir):
self.dea_file = dea_file
self.dha = LibvirtAdapter(dha_file)
self.fuel_ip = fuel_ip
self.fuel_username = fuel_username
self.fuel_password = fuel_password
self.fuel_node_id = fuel_node_id
self.iso_file = iso_file
self.work_dir = work_dir
self.file_dir = os.path.dirname(os.path.realpath(__file__))
self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
self.fuel_password)
def install(self):
log('Start Fuel Installation')
self.dha.node_power_off(self.fuel_node_id)
self.zero_mbr_set_boot_order()
self.proceed_with_installation()
def custom_install(self):
log('Start Custom Fuel Installation')
self.dha.node_power_off(self.fuel_node_id)
log('Zero the MBR')
self.dha.node_zero_mbr(self.fuel_node_id)
self.dha.node_set_boot_order(self.fuel_node_id, ['disk', 'iso'])
self.proceed_with_installation()
def proceed_with_installation(self):
log('Eject ISO')
self.dha.node_eject_iso(self.fuel_node_id)
log('Insert ISO %s' % self.iso_file)
self.dha.node_insert_iso(self.fuel_node_id, self.iso_file)
self.dha.node_power_on(self.fuel_node_id)
log('Waiting for Fuel master to accept SSH')
self.wait_for_node_up()
log('Wait until Fuel menu is up')
fuel_menu_pid = self.wait_until_fuel_menu_up()
log('Inject our own astute.yaml settings')
self.inject_own_astute_yaml()
log('Let the Fuel deployment continue')
log('Found FUEL menu as PID %s, now killing it' % fuel_menu_pid)
self.ssh_exec_cmd('kill %s' % fuel_menu_pid)
log('Wait until installation complete')
self.wait_until_installation_completed()
log('Waiting for one minute for Fuel to stabilize')
time.sleep(60)
log('Eject ISO')
self.dha.node_eject_iso(self.fuel_node_id)
log('Fuel Master installed successfully !')
def zero_mbr_set_boot_order(self):
if self.dha.node_can_zero_mbr(self.fuel_node_id):
log('Fuel Node %s capable of zeroing MBR so doing that...'
% self.fuel_node_id)
self.dha.node_zero_mbr(self.fuel_node_id)
self.dha.node_set_boot_order(self.fuel_node_id, ['disk', 'iso'])
elif self.dha.node_can_set_boot_order_live(self.fuel_node_id):
log('Node %s can change ISO boot order live' % self.fuel_node_id)
self.dha.node_set_boot_order(self.fuel_node_id, ['iso', 'disk'])
else:
err('No way to install Fuel node')
def wait_for_node_up(self):
WAIT_LOOP = 60
SLEEP_TIME = 10
success = False
for i in range(WAIT_LOOP):
try:
self.ssh.open()
success = True
break
except Exception as e:
log('EXCEPTION [%s] received when SSH-ing into Fuel VM %s ... '
'sleeping %s seconds' % (e, self.fuel_ip, SLEEP_TIME))
time.sleep(SLEEP_TIME)
finally:
self.ssh.close()
if not success:
err('Could not SSH into Fuel VM %s' % self.fuel_ip)
def wait_until_fuel_menu_up(self):
WAIT_LOOP = 60
SLEEP_TIME = 10
CMD = 'ps -ef'
SEARCH = 'fuelmenu'
fuel_menu_pid = None
with self.ssh:
for i in range(WAIT_LOOP):
ret = self.ssh.exec_cmd(CMD)
fuel_menu_pid = self.get_fuel_menu_pid(ret, SEARCH)
if not fuel_menu_pid:
time.sleep(SLEEP_TIME)
else:
break
if not fuel_menu_pid:
err('Could not find the Fuel Menu Process ID')
return fuel_menu_pid
def get_fuel_menu_pid(self, printout, search):
fuel_menu_pid = None
for line in printout.splitlines():
if search in line:
fuel_menu_pid = clean(line)[1]
break
return fuel_menu_pid
def ssh_exec_cmd(self, cmd):
with self.ssh:
ret = self.ssh.exec_cmd(cmd)
return ret
def inject_own_astute_yaml(self):
dest ='~/%s/' % self.work_dir
with self.ssh as s:
s.exec_cmd('rm -rf %s' % self.work_dir, check=False)
s.exec_cmd('mkdir ~/%s' % self.work_dir)
s.scp_put(self.dea_file, dest)
s.scp_put('%s/common.py' % self.file_dir, dest)
s.scp_put('%s/dea.py' % self.file_dir, dest)
s.scp_put('%s/transplant_fuel_settings.py' % self.file_dir, dest)
log('Modifying Fuel astute')
s.run('python ~/%s/%s ~/%s/%s'
% (self.work_dir, TRANSPLANT_FUEL_SETTINGS,
self.work_dir, os.path.basename(self.dea_file)))
def wait_until_installation_completed(self):
WAIT_LOOP = 180
SLEEP_TIME = 10
CMD = 'ps -ef | grep %s | grep -v grep' % BOOTSTRAP_ADMIN
install_completed = False
with self.ssh:
for i in range(WAIT_LOOP):
ret = self.ssh.exec_cmd(CMD)
if not ret:
install_completed = True
break
else:
time.sleep(SLEEP_TIME)
if not install_completed:
err('Fuel installation did not complete')
'''
Note tests
'''
from behave import given, when, then, model
import subprocess
from ssh_client import SSHClient as shc
@given('I have no output')
def step_impl(context):
pass
@when('I provide a float {input}')
def step_impl(context, input):
p = subprocess.check_call(['chuck', 'test.ck:'+input])
if not p.stderr:
assert True
else:
assert False
@then('I hear a middle C')
step_impl(context):
test_data = shc.get_data("redis-server get note")
ok_(test_data)
assert_true(440 in test_data[1])
def ssh_client():
return SSHClient(UNAME, IP, PORT, PWD)