def create_ovs_bridge(node, bridge):
cmds = []
cmds.append(['ovs-vsctl', 'add-br', bridge])
cmds.append(['ip', 'link', 'set', 'up', 'dev', bridge])
cmds.append(['ip', 'link', 'set', 'mtu', '1450', 'dev', bridge])
for cmd in cmds:
ssh.call(cmd, node=node)
def reboot_nodes(nodes, timeout=600):
old_clients = dict((node, ssh.get_client(node)) for node in nodes)
for node in nodes:
ssh.call(['reboot'], node=node)
node_ids_str = ", ".join(str(node.data['id']) for node in nodes)
LOG.debug("Sent reboot command to nodes: %s", node_ids_str)
wait_list = set(nodes)
start = time.time()
while wait_list:
time.sleep(10)
done = set()
for node in wait_list:
if time.time() - start > timeout:
failed = ", ".join(str(node.data['id']) for node in wait_list)
raise Exception(
"Timeout waiting for nodes {0} to reboot".format(
failed))
try:
new_client = ssh.get_client(node)
except socket.error:
LOG.debug("Failed to connect to node %s", node.data['id'],
exc_info=sys.exc_info())
node.update() # IP could've been changed, use new IP next time
continue
if new_client != old_clients[node]:
done.add(node)
wait_list -= done
def disable_apis(env):
controllers = list(env_util.get_controllers(env))
maintenance_line = 'backend maintenance'
stats_socket_re = re.compile('stats\s+socket\s+/var/lib/haproxy/stats'
'(?!.*level admin)')
mode_tcp_re = re.compile('mode\s+tcp')
use_backend_line = ' use_backend maintenance if TRUE'
for node in controllers:
sftp = ssh.sftp(node)
sftp.chdir('/etc/haproxy')
with ssh.update_file(sftp, 'haproxy.cfg') as (old, new):
found_maint_line = False
for line in old:
if maintenance_line in line:
found_maint_line = True
line = stats_socket_re.sub(r'\g<0> level admin', line)
new.write(line)
if not found_maint_line:
new.write(maintenance_line)
sftp.chdir('/etc/haproxy/conf.d')
for f in sftp.listdir():
with ssh.update_file(sftp, f) as (old, new):
contents = old.read()
if not mode_tcp_re.search(contents):
raise ssh.DontUpdateException
new.write(contents)
if not contents.endswith('\n'):
new.write('\n')
new.write(use_backend_line)
ssh.call(['crm', 'resource', 'restart', 'p_haproxy'], node=node)
def delete_image(node, tenant, user, password, token, container, object_id):
cmd = ". /root/openrc; swift --os-project-name {0}"\
" --os-username {1} --os-password {2} --os-auth-token {3}"\
" delete {4} {5}".format(tenant, user, password, token,
container, object_id)
ssh.call(["sh", "-c", cmd], node=node)
LOG.info("Swift %s image has been deleted" % object_id)
def disable_apis(env):
controllers = list(env_util.get_controllers(env))
maintenance_line = "backend maintenance"
stats_socket_re = re.compile("stats\s+socket\s+/var/lib/haproxy/stats" "(?!.*level admin)")
mode_tcp_re = re.compile("mode\s+tcp")
use_backend_line = " use_backend maintenance if TRUE"
for node in controllers:
sftp = ssh.sftp(node)
sftp.chdir("/etc/haproxy")
with ssh.update_file(sftp, "haproxy.cfg") as (old, new):
found_maint_line = False
for line in old:
if maintenance_line in line:
found_maint_line = True
line = stats_socket_re.sub(r"\g<0> level admin", line)
new.write(line)
if not found_maint_line:
new.write(maintenance_line)
sftp.chdir("/etc/haproxy/conf.d")
for f in sftp.listdir():
with ssh.update_file(sftp, f) as (old, new):
contents = old.read()
if not mode_tcp_re.search(contents):
raise ssh.DontUpdateException
new.write(contents)
if not contents.endswith("\n"):
new.write("\n")
new.write(use_backend_line)
ssh.call(["crm", "resource", "restart", "p_haproxy"], node=node)
def restart_nova_services(node):
nova_services = ssh.call_output(["service", "--status-all"], node=node)
for service_line in nova_services.splitlines():
service_line = service_line.strip()
_, status, _, service = service_line.split()
if status == "+" and service.startswith("nova"):
ssh.call(["service", service, "restart"], node=node)
def delete_patch_ports(node, host_config):
for bridge in magic_consts.BRIDGES:
port, provider = ts.get_patch_port_action(host_config, bridge)
delete_port_cmd = delete_port_providers[provider]
cmds = delete_port_cmd(bridge, port)
for cmd in cmds:
ssh.call(cmd, node=node)
def create_ovs_bridge(node, bridge):
cmds = []
cmds.append(['ovs-vsctl', 'add-br', bridge])
cmds.append(['ip', 'link', 'set', 'up', 'dev', bridge])
cmds.append(['ip', 'link', 'set', 'mtu', '1450', 'dev', bridge])
for cmd in cmds:
ssh.call(cmd, node=node)
def reboot_nodes(nodes, timeout=600):
old_clients = dict((node, ssh.get_client(node)) for node in nodes)
for node in nodes:
ssh.call(['reboot'], node=node)
node_ids_str = ", ".join(str(node.data['id']) for node in nodes)
LOG.debug("Sent reboot command to nodes: %s", node_ids_str)
wait_list = set(nodes)
start = time.time()
while wait_list:
time.sleep(10)
done = set()
for node in wait_list:
if time.time() - start > timeout:
failed = ", ".join(str(node.data['id'] for node in wait_list))
raise Exception(
"Timeout waiting for nodes {0} to reboot".format(
failed))
try:
new_client = ssh.get_client(node)
except socket.error:
LOG.debug("Failed to connect to node %s", node.data['id'],
exc_info=sys.exc_info())
continue
if new_client != old_clients[node]:
done.add(node)
wait_list -= done
def create_tunnel_from_node_lnx(local, remote, bridge, key, admin_iface):
def check_tunnel(node, port):
cmd = ['sh', '-c',
'ip link show dev %s' % (port,)]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
gre_port = 'gre%s-%s' % (remote.id, key)
if check_tunnel(local, gre_port):
return
cmds = []
cmds.append(['ip', 'link', 'add', gre_port,
'type', 'gretap',
'remote', remote.data['ip'],
'local', local.data['ip'],
'key', str(key)])
cmds.append(['ip', 'link', 'set', 'up', 'dev', gre_port])
cmds.append(['ip', 'link', 'set', 'mtu', '1450', 'dev', gre_port])
cmds.append(['ip', 'link', 'set', 'up', 'dev', bridge])
cmds.append(['brctl', 'addif', bridge, gre_port])
for cmd in cmds:
ssh.call(cmd, node=local)
def disable_apis(env):
controllers = list(env_util.get_controllers(env))
maintenance_line = 'backend maintenance'
stats_socket_re = re.compile('stats\s+socket\s+/var/lib/haproxy/stats'
'(?!.*level admin)')
mode_tcp_re = re.compile('mode\s+tcp')
use_backend_line = ' use_backend maintenance if TRUE'
for node in controllers:
sftp = ssh.sftp(node)
sftp.chdir('/etc/haproxy')
with ssh.update_file(sftp, 'haproxy.cfg') as (old, new):
found_maint_line = False
for line in old:
if maintenance_line in line:
found_maint_line = True
line = stats_socket_re.sub(r'\g<0> level admin', line)
new.write(line)
if not found_maint_line:
new.write(maintenance_line)
sftp.chdir('/etc/haproxy/conf.d')
for f in sftp.listdir():
with ssh.update_file(sftp, f) as (old, new):
contents = old.read()
if not mode_tcp_re.search(contents):
raise ssh.DontUpdateException
new.write(contents)
if not contents.endswith('\n'):
new.write('\n')
new.write(use_backend_line)
ssh.call(['crm', 'resource', 'restart', 'p_haproxy'], node=node)
def create_patch_ports(node, host_config):
for bridge in magic_consts.BRIDGES:
port, provider = ts.get_patch_port_action(host_config, bridge)
create_port_cmd = create_port_providers[provider]
cmds = create_port_cmd(bridge, port)
for cmd in cmds:
ssh.call(cmd, node=node)
def create_tunnel_from_node_lnx(local, remote, bridge, key, admin_iface):
def check_tunnel(node, port):
cmd = ['sh', '-c', 'ip link show dev %s' % (port, )]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
gre_port = 'gre%s-%s' % (remote.id, key)
if check_tunnel(local, gre_port):
return
cmds = []
cmds.append([
'ip', 'link', 'add', gre_port, 'type', 'gretap', 'remote',
remote.data['ip'], 'local', local.data['ip'], 'key',
str(key)
])
cmds.append(['ip', 'link', 'set', 'up', 'dev', gre_port])
cmds.append(['ip', 'link', 'set', 'mtu', '1450', 'dev', gre_port])
cmds.append(['ip', 'link', 'set', 'up', 'dev', bridge])
cmds.append(['brctl', 'addif', bridge, gre_port])
for cmd in cmds:
ssh.call(cmd, node=local)
def update_neutron_config(orig_env, seed_env):
controllers = list(env_util.get_controllers(seed_env))
tenant_id = env_util.cache_service_tenant_id(orig_env)
sed_script = 's/^(nova_admin_tenant_id )=.*/\\1 = %s/' % (tenant_id,)
for node in controllers:
ssh.call(['sed', '-re', sed_script, '-i', '/etc/neutron/neutron.conf'],
node=node)
def delete_image(node, tenant, user, password, token, container, object_id):
cmd = (
". /root/openrc; swift --os-project-name {0}"
" --os-username {1} --os-password {2} --os-auth-token {3}"
" delete {4} {5}".format(tenant, user, password, token, container, object_id)
)
ssh.call(["sh", "-c", cmd], node=node)
LOG.info("Swift %s image has been deleted" % object_id)
def postdeploy(self):
controller = env_util.get_one_controller(self.env)
ssh.call(
["sh", "-c", ". /root/openrc; "
"nova service-enable node-{0} nova-compute".format(
self.node.data['id'])],
node=controller,
)
def update_neutron_config(orig_env, seed_env):
controllers = list(env_util.get_controllers(seed_env))
tenant_id = env_util.cache_service_tenant_id(orig_env)
sed_script = 's/^(nova_admin_tenant_id )=.*/\\1 = %s/' % (tenant_id, )
for node in controllers:
ssh.call(['sed', '-re', sed_script, '-i', '/etc/neutron/neutron.conf'],
node=node)
def check_tunnel(node, port):
cmd = ['sh', '-c', 'ip link show dev %s' % (port, )]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
def create_partition(disk_name, size, node):
out = ssh.call_output(
['parted', '/dev/%s' % disk_name, 'unit', 'MB', 'print'], node=node)
start = parse_last_partition_end(out) + 1
end = start + size
ssh.call(['parted', '/dev/%s' % disk_name, 'unit', 'MB', 'mkpart',
'custom', 'ext4', str(start), str(end)],
node=node)
def check_tunnel(node, bridge, port):
cmd = ['sh', '-c',
'ovs-vsctl list-ports %s | grep -q %s' % (bridge, port)]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
def cinder_volume_update_host(orig_env, new_env):
orig_controller = env_util.get_one_controller(orig_env)
new_controller = env_util.get_one_controller(new_env)
current_host = get_current_host(orig_controller)
new_host = get_new_host(new_controller)
ssh.call(["cinder-manage", "volume", "update_host",
"--currenthost", current_host,
"--newhost", new_host],
node=new_controller, parse_levels=True)
def import_bootstrap_osd(node):
ssh.call(
['ceph', 'auth', 'import', '-i', '/root/ceph.bootstrap-osd.keyring'],
node=node)
ssh.call([
'ceph', 'auth', 'caps', 'client.bootstrap-osd', 'mon',
"allow profile bootstrap-osd"
],
node=node)
def check_tunnel(node, bridge, port):
cmd = ['sh', '-c',
'ovs-vsctl list-ports %s | grep -q %s' % (bridge, port)]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
def check_tunnel(node, port):
cmd = ['sh', '-c',
'ip link show dev %s' % (port,)]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
def restore_iscsi_initiator_info(self):
if not plugin.is_enabled(self.env, 'emc_vnx'):
return
bup_file_path = get_iscsi_bup_file_path(self.node)
if not os.path.exists(bup_file_path):
raise Exception("Backup iscsi configuration is not present for "
"compute node %s" % str(self.node.id))
ssh.sftp(self.node).put(bup_file_path, magic_consts.ISCSI_CONFIG_PATH)
for service in ["open-iscsi", "multipath-tools", "nova-compute"]:
ssh.call(['service', service, 'restart'], node=self.node)
def restore_iscsi_initiator_info(self):
if not plugin.is_enabled(self.env, 'emc_vnx'):
return
bup_file_path = get_iscsi_bup_file_path(self.node)
if not os.path.exists(bup_file_path):
raise Exception("Backup iscsi configuration is not present for "
"compute node %s" % str(self.node.id))
ssh.sftp(self.node).put(bup_file_path, magic_consts.ISCSI_CONFIG_PATH)
for service in ["open-iscsi", "multipath-tools", "nova-compute"]:
ssh.call(['service', service, 'restart'], node=self.node)
def create_patch_ports(node, host_config):
for bridge in magic_consts.BRIDGES:
port, provider = ts.get_patch_port_action(host_config, bridge)
create_port_cmd = create_port_providers[provider]
cmds = create_port_cmd(bridge, port)
for cmd in cmds:
ssh.call(cmd, node=node)
save_port_func = save_port_providers.get(provider)
if save_port_func:
save_port_func(node, bridge, port)
def postdeploy(self):
controller = env_util.get_one_controller(self.env)
ssh.call(
[
"sh", "-c", ". /root/openrc; "
"nova service-enable node-{0} nova-compute".format(
self.node.data['id'])
],
node=controller,
)
def transfer_image(node, tenant, user, password, token, container, object_id,
storage_ip, tenant_id):
storage_url = "http://{0}:8080/v1/AUTH_{1}".format(storage_ip, tenant_id)
cmd = [
'swift', '--os-project-name', tenant, '--os-username', user,
'--os-password', password, '--os-auth-token', token,
'--os-storage-url', storage_url, 'upload', container, object_id
]
ssh.call(cmd, node=node)
LOG.info("Swift %s image has been transferred" % object_id)
def delete_fuel_resources(env):
node = get_one_controller(env)
sftp = ssh.sftp(node)
sftp.put(
os.path.join(magic_consts.CWD, "helpers/delete_fuel_resources.py"),
"/tmp/delete_fuel_resources.py",
)
ssh.call(
["sh", "-c", ". /root/openrc; python /tmp/delete_fuel_resources.py"],
node=node,
)
def update_neutron_config(env):
controllers = list(env_util.get_controllers(env))
tenant_file = '%s/env-%s-service-tenant-id' % (magic_consts.FUEL_CACHE,
str(env.id))
with open(tenant_file) as f:
tenant_id = f.read()
sed_script = 's/^(nova_admin_tenant_id )=.*/\1 =%s' % (tenant_id, )
for node in controllers:
ssh.call(['sed', '-re', sed_script, '-i', '/etc/neutron/neutron.conf'],
node=node)
def update_neutron_config(env):
controllers = list(env_util.get_controllers(env))
tenant_file = '%s/env-%s-service-tenant-id' % (magic_consts.FUEL_CACHE,
str(env.id))
with open(tenant_file) as f:
tenant_id = f.read()
sed_script = 's/^(nova_admin_tenant_id )=.*/\1 =%s' % (tenant_id,)
for node in controllers:
ssh.call(['sed', '-re', sed_script, '-i', '/etc/neutron/neutron.conf'],
node=node)
def delete_fuel_resources(env):
node = get_one_controller(env)
sftp = ssh.sftp(node)
sftp.put(
os.path.join(magic_consts.CWD, "helpers/delete_fuel_resources.py"),
"/tmp/delete_fuel_resources.py",
)
ssh.call(
["sh", "-c", ". /root/openrc; python /tmp/delete_fuel_resources.py"],
node=node,
)
def change_repositories(node, repos):
ssh.remove_all_files_from_dirs(['/etc/apt/sources.list.d',
'/etc/apt/preferences.d'], node)
sftp = ssh.sftp(node)
for repo in repos:
filename_source, content_source = apt.create_repo_source(repo)
ssh.write_content_to_file(sftp, filename_source, content_source)
if repo['priority']:
filename_pref, content_pref = apt.create_repo_preferences(repo)
ssh.write_content_to_file(sftp, filename_pref, content_pref)
ssh.call(['apt-get', 'update'], node=node)
def stop_corosync_services(env):
node = env_util.get_one_controller(env)
status_out = ssh.call_output(['crm', 'status'], node=node)
for service in parse_crm_status(status_out):
while True:
try:
ssh.call(['crm', 'resource', 'stop', service], node=node)
except subprocess.CalledProcessError:
pass
else:
break
time.sleep(60)
def stop_corosync_services(env):
node = env_util.get_one_controller(env)
status_out = ssh.call_output(['crm', 'status'], node=node)
for service in parse_crm_status(status_out):
while True:
try:
ssh.call(['crm', 'resource', 'stop', service],
node=node)
except subprocess.CalledProcessError:
pass
else:
break
time.sleep(60)
def evacuate_host(self):
controller = env_util.get_one_controller(self.env)
with ssh.tempdir(controller) as tempdir:
local_path = os.path.join(magic_consts.CWD, 'bin',
'host_evacuation.sh')
remote_path = os.path.join(tempdir, 'host_evacuation.sh')
sftp = ssh.sftp(controller)
sftp.put(local_path, remote_path)
sftp.chmod(remote_path, stat.S_IRWXO)
ssh.call(
[remote_path, 'node-{0}'.format(self.node.data['id'])],
node=controller,
)
def install_openvswitch(node, master_ip):
cmds = []
cmds.append(
['sh', '-c',
'echo'
' "deb http://{0}:8080/2015.1.0-7.0/ubuntu/x86_64 mos7.0'
' main restricted" >> /etc/apt/sources.list'.format(master_ip)])
cmds.append(['apt-get', 'update'])
cmds.append(
['apt-get', 'install', '-y', '--force-yes', 'openvswitch-switch'])
cmds.append(['sh', '-c', 'sed -i 1,2d /etc/apt/sources.list'])
for cmd in cmds:
ssh.call(cmd, node=node)
def install_openvswitch(node, master_ip):
cmds = []
cmds.append([
'sh', '-c', 'echo'
' "deb http://{0}:8080/2015.1.0-7.0/ubuntu/x86_64 mos7.0'
' main restricted" >> /etc/apt/sources.list'.format(master_ip)
])
cmds.append(['apt-get', 'update'])
cmds.append(
['apt-get', 'install', '-y', '--force-yes', 'openvswitch-switch'])
cmds.append(['sh', '-c', 'sed -i 1,2d /etc/apt/sources.list'])
for cmd in cmds:
ssh.call(cmd, node=node)
def start_upstart_services(env):
controllers = list(env_util.get_controllers(env))
for node in controllers:
sftp = ssh.sftp(node)
try:
svc_file = sftp.open('/root/services_list')
except IOError:
raise
else:
with svc_file:
to_start = svc_file.read().splitlines()
for service in to_start:
ssh.call(['start', service], node=node)
def prepare(orig_id, seed_id):
orig_env = environment_obj.Environment(orig_id)
seed_env = environment_obj.Environment(seed_id)
controller = env_util.get_one_controller(seed_env)
with tempfile.NamedTemporaryFile() as temp:
db.mysqldump_from_env(orig_env, ['keystone'], temp.name)
db.mysqldump_restore_to_env(seed_env, temp.name)
ssh.call(['keystone-manage', 'db_sync'],
node=controller, parse_levels=True)
for controller in env_util.get_controllers(seed_env):
ssh.call(['service', 'memcached', 'restart'], node=controller)
def start_upstart_services(env):
controllers = list(env_util.get_controllers(env))
for node in controllers:
sftp = ssh.sftp(node)
try:
svc_file = sftp.open('/root/services_list')
except IOError:
raise
else:
with svc_file:
to_start = svc_file.read().splitlines()
for service in to_start:
ssh.call(['start', service], node=node)
def evacuate_host(self):
controller = env_util.get_one_controller(self.env)
with ssh.tempdir(controller) as tempdir:
local_path = os.path.join(
magic_consts.CWD, 'bin', 'host_evacuation.sh')
remote_path = os.path.join(tempdir, 'host_evacuation.sh')
sftp = ssh.sftp(controller)
sftp.put(local_path, remote_path)
sftp.chmod(remote_path, stat.S_IRWXO)
ssh.call(
[remote_path, 'node-{0}'.format(self.node.data['id'])],
node=controller,
)
def start_corosync_services(env):
node = next(env_util.get_controllers(env))
status_out, _ = ssh.call(['crm', 'resource', 'list'],
stdout=ssh.PIPE,
node=node)
for service in maintenance.parse_crm_status(status_out):
while True:
try:
ssh.call(['crm', 'resource', 'start', service], node=node)
except subprocess.CalledProcessError:
pass
else:
break
def postdeploy(self):
if self.isolated and self.gateway:
# From restore_default_gateway
LOG.info("Deleting default route at node %s",
self.node.id)
try:
ssh.call(['ip', 'route', 'delete', 'default'], node=self.node)
except subprocess.CalledProcessError as exc:
LOG.warn("Cannot delete default route at node %s: %s",
self.node.id, exc.args[0])
LOG.info("Set default route at node %s: %s",
self.node.id, self.gateway)
ssh.call(['ip', 'route', 'add', 'default', 'via', self.gateway],
node=self.node)
def upgrade_osd(orig_env_id, seed_env_id, user, password):
with fuel_client.set_auth_context(
backup_restore.NailgunCredentialsContext(user, password)):
orig_env = env_obj.Environment(orig_env_id)
nodes = list(env.get_nodes(orig_env, ["ceph-osd"]))
seed_env = env_obj.Environment(seed_env_id)
preference_priority = get_repo_highest_priority(orig_env)
seed_repos = get_repos_for_upgrade(orig_env, seed_env)
if not nodes:
LOG.info("Nothing to upgrade")
return
controller = env.get_one_controller(seed_env)
if is_same_versions_on_mon_and_osd(controller):
LOG.warn("MONs and OSDs have the same version, nothing to upgrade.")
return
hostnames = [n.data['hostname'] for n in nodes]
with applied_repos(nodes, preference_priority + 1, seed_repos):
call_node = nodes[0]
ssh.call(["ceph", "osd", "set", "noout"], node=call_node)
ssh.call(['ceph-deploy', 'install', '--release', 'hammer'] + hostnames,
node=call_node)
for node in nodes:
ssh.call(["restart", "ceph-osd-all"], node=node)
ssh.call(["ceph", "osd", "unset", "noout"], node=call_node)
waiting_until_ceph_up(controller)
if not is_same_versions_on_mon_and_osd(controller):
msg = "OSDs not upgraded up to MONs version, please fix the problem"
LOG.error(msg)
raise Exception(msg)
def start_corosync_services(env):
node = next(env_util.get_controllers(env))
status_out, _ = ssh.call(['crm', 'resource', 'list'],
stdout=ssh.PIPE,
node=node)
for service in maintenance.parse_crm_status(status_out):
while True:
try:
ssh.call(['crm', 'resource', 'start', service],
node=node)
except subprocess.CalledProcessError:
pass
else:
break
def create_partition(disk_name, size, node):
out, _ = ssh.call(
['parted', '/dev/%s' % disk_name, 'unit', 'MB', 'print'],
stdout=ssh.PIPE,
node=node)
start = parse_last_partition_end(out) + 1
end = start + size
ssh.call([
'parted',
'/dev/%s' % disk_name, 'unit', 'MB', 'mkpart', 'custom', 'ext4',
str(start),
str(end)
],
node=node)
def get_swift_objects(node, tenant, user, password, token, container):
cmd = (
". /root/openrc; swift --os-project-name {0} --os-username {1}"
" --os-password {2} --os-auth-token {3} list {4}".format(tenant, user, password, token, container)
)
objects_list, _ = ssh.call(["sh", "-c", cmd], stdout=ssh.PIPE, node=node)
return objects_list.split("\n")[:-1]
def restart_mcollective(node):
node_id = node.data["id"]
if not node.data["online"]:
LOG.warning("Not possible to restart mcollective on the offline "
"node %s", node_id)
return None
try:
ssh.call(["service", "mcollective", "restart"], node=node)
except Exception as exc:
LOG.warning("Failed to restart mcollective on the node %s: %s",
node_id, exc)
return False
else:
LOG.info("The mcollective service was successfully restarted on "
"the node %s", node_id)
return True
def list_tunnels_lnx(node, bridge):
tunnels = []
gre_port_re = "gre[0-9]+-[0-9]+".format(node.id)
stdout, _ = ssh.call(['brctl', 'show', bridge], stdout=ssh.PIPE, node=node)
for match in re.finditer(gre_port_re, stdout):
tunnels.append(match.group(0))
return tunnels
def get_service_tenant_id(env, node=None):
env_id = env.data['id']
fname = os.path.join(
magic_consts.FUEL_CACHE,
"env-{0}-service-tenant-id".format(env_id),
)
if os.path.exists(fname):
with open(fname) as f:
return f.readline()
if node is None:
node = get_one_controller(env)
tenant_out, _ = ssh.call(
[
'sh', '-c',
'. /root/openrc; keystone tenant-get services',
],
node=node,
stdout=ssh.PIPE,
)
tenant_id = parse_tenant_get(tenant_out, 'id')
with open(fname, 'w') as f:
f.write(tenant_id)
return tenant_id
def get_auth_token(node, tenant, user, password):
cmd = ". /root/openrc; keystone --os-tenant-name {0}"\
" --os-username {1} --os-password {2} token-get".format(tenant,
user,
password)
token_info, _ = ssh.call(["sh", "-c", cmd], stdout=ssh.PIPE, node=node)
return env_util.parse_tenant_get(token_info, 'id')
def shutoff_vms(self):
password = env_util.get_admin_password(self.env)
cmd = [
'. /root/openrc;', 'nova list --os-password {0} --host {1}'.format(
password, self.node.data['hostname']), '|',
'awk -F\| \'$4~/ACTIVE/{print($2)}', '|', 'xargs -I% nova stop %'
]
out, err = ssh.call(cmd, stdout=ssh.PIPE, node=self.node)
def list_tunnels_ovs(node, bridge):
tunnels = []
stdout, _ = ssh.call(['ovs-vsctl', 'list-ports', bridge],
stdout=ssh.PIPE,
node=node)
for match in re.finditer("[\S]+--gre-[\S]*\n", stdout):
tun = match.group(0)
tunnels.append(tun[:-1])
return tunnels
def get_swift_objects(node, tenant, user, password, token, container):
cmd = ". /root/openrc; swift --os-project-name {0} --os-username {1}"\
" --os-password {2} --os-auth-token {3} list {4}".format(tenant,
user,
password,
token,
container)
objects_list, _ = ssh.call(["sh", "-c", cmd], stdout=ssh.PIPE, node=node)
return objects_list.split('\n')[:-1]
def create_tunnel_from_node_ovs(local, remote, bridge, key, admin_iface):
def check_tunnel(node, bridge, port):
cmd = ['sh', '-c',
'ovs-vsctl list-ports %s | grep -q %s' % (bridge, port)]
try:
ssh.call(cmd, node=node)
except subprocess.CalledProcessError:
return False
else:
return True
gre_port = '%s--gre-%s' % (bridge, remote.data['ip'])
if check_tunnel(local, bridge, gre_port):
return
cmd = ['ovs-vsctl', 'add-port', bridge, gre_port,
'--', 'set', 'Interface', gre_port,
'type=gre',
'options:remote_ip=%s' % (remote.data['ip'],),
'options:key=%d' % (key,)]
ssh.call(cmd, node=local)
def wait_for_mcollective_start(nodes, timeout=600):
start_at = time.time()
wait_list = set(nodes)
node_ids_str = ", ".join(str(node.data['id']) for node in nodes)
LOG.info("Wait for mcollective start on nodes {0}".format(node_ids_str))
while wait_list:
time.sleep(10)
done = set()
for node in wait_list:
try:
ssh.call(['service', 'mcollective', 'status'], node=node)
except Exception as e:
LOG.debug(e)
else:
done.add(node)
if time.time() - start_at > timeout:
failed = ", ".join(str(node.data['id'] for node in wait_list))
raise Exception("Timeout waiting for nodes {0} to start"
" mcollective".format(failed))
wait_list -= done