class EnvironmentModel(object):
hostname = 'nailgun'
domain = 'test.domain.local'
installation_timeout = 1800
deployment_timeout = 1800
puppet_timeout = 2000
nat_interface = '' # INTERFACES.get('admin')
admin_net = 'admin'
admin_net2 = 'admin2'
multiple_cluster_networks = settings.MULTIPLE_NETWORKS
def __init__(self, os_image=None):
self._virtual_environment = None
self._keys = None
self.manager = Manager()
self.os_image = os_image
self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
@property
def nailgun_actions(self):
return FuelActions.Nailgun(self.get_admin_remote())
@property
def postgres_actions(self):
return FuelActions.Postgres(self.get_admin_remote())
def _get_or_create(self):
try:
return self.manager.environment_get(self.env_name)
except Exception:
self._virtual_environment = self.describe_environment()
self._virtual_environment.define()
return self._virtual_environment
def router(self, router_name=None):
router_name = router_name or self.admin_net
if router_name == self.admin_net2:
return str(IPNetwork(self.get_virtual_environment().
network_by_name(router_name).ip_network)[2])
return str(
IPNetwork(
self.get_virtual_environment().network_by_name(router_name).
ip_network)[1])
@property
def fuel_web(self):
"""FuelWebClient
:rtype: FuelWebClient
"""
return self._fuel_web
@property
def admin_node_ip(self):
return self.fuel_web.admin_node_ip
@property
def node_roles(self):
return NodeRoles(
admin_names=['admin'],
other_names=['slave-%02d' % x for x in range(1, int(
settings.NODES_COUNT))]
)
@property
def env_name(self):
return settings.ENV_NAME
def add_empty_volume(self, node, name,
capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024
* 1024, device='disk', bus='virtio', format='qcow2'):
self.manager.node_attach_volume(
node=node,
volume=self.manager.volume_create(
name=name,
capacity=capacity,
environment=self.get_virtual_environment(),
format=format),
device=device,
bus=bus)
def add_node(self, memory, name, vcpu=1, boot=None):
return self.manager.node_create(
name=name,
memory=memory,
vcpu=vcpu,
environment=self.get_virtual_environment(),
boot=boot)
@logwrap
def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server(
cluster_id, self.get_host_node_ip(), port)
def bootstrap_nodes(self, devops_nodes, timeout=600):
"""Lists registered nailgun nodes
Start vms and wait until they are registered on nailgun.
:rtype : List of registered nailgun nodes
"""
# self.dhcrelay_check()
for node in devops_nodes:
node.start()
# TODO(aglarendil): LP#1317213 temporary sleep
# remove after better fix is applied
time.sleep(2)
wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
for node in self.nailgun_nodes(devops_nodes):
self.sync_node_time(self.get_ssh_to_remote(node["ip"]))
return self.nailgun_nodes(devops_nodes)
def create_interfaces(self, networks, node,
model=settings.INTERFACE_MODEL):
if settings.BONDING:
for network in networks:
self.manager.interface_create(
network, node=node, model=model,
interface_map=settings.BONDING_INTERFACES)
else:
for network in networks:
self.manager.interface_create(network, node=node, model=model)
def describe_environment(self):
"""Environment
:rtype : Environment
"""
environment = self.manager.environment_create(self.env_name)
networks = []
interfaces = settings.INTERFACE_ORDER
if self.multiple_cluster_networks:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(self.create_networks(name, environment))
for name in self.node_roles.admin_names:
self.describe_admin_node(name, networks)
for name in self.node_roles.other_names:
if self.multiple_cluster_networks:
networks1 = [net for net in networks if net.name
in settings.NODEGROUPS[0]['pools']]
networks2 = [net for net in networks if net.name
in settings.NODEGROUPS[1]['pools']]
# If slave index is even number, then attach to
# it virtual networks from the second network group.
if int(name[-2:]) % 2 == 1:
self.describe_empty_node(name, networks1)
elif int(name[-2:]) % 2 == 0:
self.describe_empty_node(name, networks2)
else:
self.describe_empty_node(name, networks)
return environment
def create_networks(self, name, environment):
ip_networks = [
IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')]
new_prefix = int(settings.POOLS.get(name)[1])
pool = self.manager.create_network_pool(networks=ip_networks,
prefix=int(new_prefix))
return self.manager.network_create(
name=name,
environment=environment,
pool=pool,
forward=settings.FORWARDING.get(name),
has_dhcp_server=settings.DHCP.get(name))
def devops_nodes_by_names(self, devops_node_names):
return map(
lambda name:
self.get_virtual_environment().node_by_name(name),
devops_node_names)
@logwrap
def describe_admin_node(self, name, networks):
node = self.add_node(
memory=settings.HARDWARE.get("admin_node_memory", 1024),
vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
name=name,
boot=['hd', 'cdrom'])
self.create_interfaces(networks, node)
if self.os_image is None:
self.add_empty_volume(node, name + '-system')
self.add_empty_volume(
node,
name + '-iso',
capacity=_get_file_size(settings.ISO_PATH),
format='raw',
device='cdrom',
bus='ide')
else:
volume = self.manager.volume_get_predefined(self.os_image)
vol_child = self.manager.volume_create_child(
name=name + '-system',
backing_store=volume,
environment=self.get_virtual_environment()
)
self.manager.node_attach_volume(
node=node,
volume=vol_child
)
return node
def describe_empty_node(self, name, networks):
node = self.add_node(
name=name,
memory=settings.HARDWARE.get("slave_node_memory", 1024),
vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
self.create_interfaces(networks, node)
self.add_empty_volume(node, name + '-system')
if settings.USE_ALL_DISKS:
self.add_empty_volume(node, name + '-cinder')
self.add_empty_volume(node, name + '-swift')
return node
@logwrap
def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password']):
"""SSH to admin node
:rtype : SSHClient
"""
return self.nodes().admin.remote(self.admin_net,
login=login,
password=password)
@logwrap
def get_admin_node_ip(self):
return str(
self.nodes().admin.get_ip_address_by_network_name(self.admin_net))
@logwrap
def get_ebtables(self, cluster_id, devops_nodes):
return Ebtables(self.get_target_devs(devops_nodes),
self.fuel_web.client.get_cluster_vlans(cluster_id))
def get_host_node_ip(self):
return self.router()
def get_keys(self, node, custom=None, build_images=None):
params = {
'ip': node.get_ip_address_by_network_name(self.admin_net),
'mask': self.get_net_mask(self.admin_net),
'gw': self.router(),
'hostname': '.'.join((self.hostname, self.domain)),
'nat_interface': self.nat_interface,
'dns1': settings.DNS,
'showmenu': 'yes' if custom else 'no',
'build_images': '1' if build_images else '0'
}
keys = (
"<Wait>\n"
"<Esc><Enter>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n"
) % params
return keys
@logwrap
def get_private_keys(self, force=False):
if force or self._keys is None:
self._keys = []
for key_string in ['/root/.ssh/id_rsa',
'/root/.ssh/bootstrap.rsa']:
with self.get_admin_remote().open(key_string) as f:
self._keys.append(RSAKey.from_private_key(f))
return self._keys
@logwrap
def get_ssh_to_remote(self, ip):
return SSHClient(ip,
username=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'],
private_keys=self.get_private_keys())
@logwrap
def get_ssh_to_remote_by_key(self, ip, keyfile):
try:
with open(keyfile) as f:
keys = [RSAKey.from_private_key(f)]
return SSHClient(ip, private_keys=keys)
except IOError:
logger.warning('Loading of SSH key from file failed. Trying to use'
' SSH agent ...')
keys = Agent().get_keys()
return SSHClient(ip, private_keys=keys)
@logwrap
def get_ssh_to_remote_by_name(self, node_name):
return self.get_ssh_to_remote(
self.fuel_web.get_nailgun_node_by_devops_node(
self.get_virtual_environment().node_by_name(node_name))['ip']
)
def get_target_devs(self, devops_nodes):
return [
interface.target_dev for interface in [
val for var in map(lambda node: node.interfaces, devops_nodes)
for val in var]]
def get_virtual_environment(self):
"""Returns virtual environment
:rtype : devops.models.Environment
"""
if self._virtual_environment is None:
self._virtual_environment = self._get_or_create()
return self._virtual_environment
def get_network(self, net_name):
return str(
IPNetwork(
self.get_virtual_environment().network_by_name(net_name).
ip_network))
def get_net_mask(self, net_name):
return str(
IPNetwork(
self.get_virtual_environment().network_by_name(
net_name).ip_network).netmask)
def make_snapshot(self, snapshot_name, description="", is_make=False):
if settings.MAKE_SNAPSHOT or is_make:
self.get_virtual_environment().suspend(verbose=False)
time.sleep(10)
self.get_virtual_environment().snapshot(snapshot_name, force=True)
revert_info(snapshot_name, description)
if settings.FUEL_STATS_CHECK:
self.get_virtual_environment().resume()
try:
self.nodes().admin.await(self.admin_net, timeout=60)
except Exception:
logger.error('Admin node is unavailable via SSH after '
'environment resume ')
raise
def nailgun_nodes(self, devops_nodes):
return map(
lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
devops_nodes
)
def nodes(self):
return Nodes(self.get_virtual_environment(), self.node_roles)
def revert_snapshot(self, name):
if self.get_virtual_environment().has_snapshot(name):
logger.info('We have snapshot with such name %s' % name)
self.get_virtual_environment().revert(name)
logger.info('Starting snapshot reverting ....')
self.get_virtual_environment().resume()
logger.info('Starting snapshot resuming ...')
admin = self.nodes().admin
try:
admin.await(
self.admin_net, timeout=10 * 60, by_port=8000)
except Exception as e:
logger.warning("From first time admin isn't reverted: "
"{0}".format(e))
admin.destroy()
logger.info('Admin node was destroyed. Wait 10 sec.')
time.sleep(10)
self.get_virtual_environment().start(self.nodes().admins)
logger.info('Admin node started second time.')
self.nodes().admin.await(
self.admin_net, timeout=10 * 60, by_port=8000)
self.set_admin_ssh_password()
try:
_wait(self._fuel_web.client.get_releases,
expected=EnvironmentError, timeout=300)
except exceptions.Unauthorized:
self.set_admin_keystone_password()
self._fuel_web.get_nailgun_version()
self.sync_time_admin_node()
for node in self.nodes().slaves:
if not node.driver.node_active(node):
continue
try:
logger.info("Sync time on revert for node %s" % node.name)
self.sync_node_time(
self.get_ssh_to_remote_by_name(node.name))
except Exception as e:
logger.warning(
'Exception caught while trying to sync time on {0}:'
' {1}'.format(node.name, e))
self.run_nailgun_agent(
self.get_ssh_to_remote_by_name(node.name))
return True
return False
def set_admin_ssh_password(self):
try:
remote = self.get_admin_remote(
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'])
self.execute_remote_cmd(remote, 'date')
logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception:
logger.debug('Accessing admin node using SSH credentials:'
' FAIL, trying to change password from default')
remote = self.get_admin_remote(login='******', password='******')
self.execute_remote_cmd(
remote, 'echo -e "{1}\\n{1}" | passwd {0}'
.format(settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
logger.debug("Admin node password has changed.")
logger.info("Admin node login name: '{0}' , password: '******'".
format(settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
def set_admin_keystone_password(self):
remote = self.get_admin_remote()
try:
self._fuel_web.client.get_releases()
except exceptions.Unauthorized:
self.execute_remote_cmd(
remote, 'fuel user --newpass {0} --change-password'
.format(settings.KEYSTONE_CREDS['password']))
logger.info(
'New Fuel UI (keystone) username: "******", password: "******"'
.format(settings.KEYSTONE_CREDS['username'],
settings.KEYSTONE_CREDS['password']))
def setup_environment(self, custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES):
# start admin node
admin = self.nodes().admin
admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
self.get_virtual_environment().start(self.nodes().admins)
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(self.get_keys(admin, custom=custom,
build_images=build_images))
if custom:
self.setup_customisation()
# wait while installation complete
admin.await(self.admin_net, timeout=10 * 60)
self.set_admin_ssh_password()
self.wait_bootstrap()
time.sleep(10)
self.set_admin_keystone_password()
self.sync_time_admin_node()
if settings.MULTIPLE_NETWORKS:
self.describe_second_admin_interface()
multiple_networks_hacks.configure_second_admin_cobbler(self)
multiple_networks_hacks.configure_second_dhcrelay(self)
self.nailgun_actions.set_collector_address(
settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
if settings.FUEL_STATS_ENABLED:
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT
))
@upload_manifests
def wait_for_provisioning(self):
_wait(lambda: _tcp_ping(
self.nodes().admin.get_ip_address_by_network_name
(self.admin_net), 22), timeout=5 * 60)
def setup_customisation(self):
self.wait_for_provisioning()
try:
remote = self.get_admin_remote()
pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0]
pid.rstrip('\n')
remote.execute("kill -sigusr1 {0}".format(pid))
except Exception:
logger.error("Could not kill pid of fuelmenu")
raise
@retry(count=10, delay=60)
@logwrap
def sync_node_time(self, remote):
self.execute_remote_cmd(remote, 'hwclock -s')
self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \''
'/etc/init.d/\(ntp.?\|ntp-dev\)\');'
'$NTPD stop && ntpd -dqg && $NTPD '
'start')
self.execute_remote_cmd(remote, 'hwclock -w')
remote_date = remote.execute('date')['stdout']
logger.info("Node time: %s" % remote_date)
@retry(count=10, delay=60)
@logwrap
def sync_time_admin_node(self):
logger.info("Sync time on revert for admin")
remote = self.get_admin_remote()
self.execute_remote_cmd(remote, 'hwclock -s')
# Sync time using ntpd
try:
# If public NTP servers aren't accessible ntpdate will fail and
# ntpd daemon shouldn't be restarted to avoid 'Server has gone
# too long without sync' error while syncing time from slaves
self.execute_remote_cmd(remote, "ntpdate -vu $(awk '/^server/ && "
"$2 !~ /127.*/ {print $2}' "
"/etc/ntp.conf)")
except AssertionError as e:
logger.warning('Error occurred while synchronizing time on master'
': {0}'.format(e))
raise
else:
self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -dqg &&'
' service ntpd start')
self.execute_remote_cmd(remote, 'hwclock -w')
remote_date = remote.execute('date')['stdout']
logger.info("Master node time: {0}".format(remote_date))
def verify_network_configuration(self, node_name):
checkers.verify_network_configuration(
node=self.fuel_web.get_nailgun_node_by_name(node_name),
remote=self.get_ssh_to_remote_by_name(node_name)
)
def wait_bootstrap(self):
logger.info("Waiting while bootstrapping is in progress")
log_path = "/var/log/puppet/bootstrap_admin_node.log"
logger.info("Puppet timeout set in {0}".format(
float(settings.PUPPET_TIMEOUT)))
wait(
lambda: not
self.get_admin_remote().execute(
"grep 'Fuel node deployment' '%s'" % log_path
)['exit_code'],
timeout=(float(settings.PUPPET_TIMEOUT))
)
result = self.get_admin_remote().execute("grep 'Fuel node deployment "
"complete' '%s'" % log_path
)['exit_code']
if result != 0:
raise Exception('Fuel node deployment failed.')
def dhcrelay_check(self):
admin_remote = self.get_admin_remote()
out = admin_remote.execute("dhcpcheck discover "
"--ifaces eth0 "
"--repeat 3 "
"--timeout 10")['stdout']
assert_true(self.get_admin_node_ip() in "".join(out),
"dhcpcheck doesn't discover master ip")
def run_nailgun_agent(self, remote):
agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
logger.info("Nailgun agent run with exit_code: %s" % agent)
def get_fuel_settings(self, remote=None):
if not remote:
remote = self.get_admin_remote()
cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
result = remote.execute(cmd)
if result['exit_code'] == 0:
fuel_settings = yaml.load(''.join(result['stdout']))
else:
raise Exception('Can\'t output {cfg_file} file: {error}'.
format(cfg_file=settings.FUEL_SETTINGS_YAML,
error=result['stderr']))
return fuel_settings
def admin_install_pkg(self, pkg_name):
"""Install a package <pkg_name> on the admin node"""
admin_remote = self.get_admin_remote()
remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
if remote_status['exit_code'] == 0:
logger.info("Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = admin_remote.execute("yum -y install {0}"
.format(pkg_name))
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}"
.format(pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def admin_run_service(self, service_name):
"""Start a service <service_name> on the admin node"""
admin_remote = self.get_admin_remote()
admin_remote.execute("service {0} start".format(service_name))
remote_status = admin_remote.execute("service {0} status"
.format(service_name))
if any('running...' in status for status in remote_status['stdout']):
logger.info("Service '{0}' is running".format(service_name))
else:
logger.info("Service '{0}' failed to start"
" with exit code {1} :\n{2}"
.format(service_name,
remote_status['exit_code'],
remote_status['stdout']))
# Modifies a resolv.conf on the Fuel master node and returns
# its original content.
# * adds 'nameservers' at start of resolv.conf if merge=True
# * replaces resolv.conf with 'nameservers' if merge=False
def modify_resolv_conf(self, nameservers=[], merge=True):
remote = self.get_admin_remote()
resolv_conf = remote.execute('cat /etc/resolv.conf')
assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'
.format('cat /etc/resolv.conf', resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = remote.execute(echo_cmd)
assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'
.format(echo_cmd, echo_result['stderr']))
return resolv_conf['stdout']
@logwrap
def execute_remote_cmd(self, remote, cmd, exit_code=0):
result = remote.execute(cmd)
assert_equal(result['exit_code'], exit_code,
'Failed to execute "{0}" on remote host: {1}'.
format(cmd, result))
return result['stdout']
@logwrap
def describe_second_admin_interface(self):
remote = self.get_admin_remote()
second_admin_network = self.get_network(self.admin_net2).split('/')[0]
second_admin_netmask = self.get_net_mask(self.admin_net2)
second_admin_if = settings.INTERFACES.get(self.admin_net2)
second_admin_ip = str(self.nodes().admin.
get_ip_address_by_network_name(self.admin_net2))
logger.info(('Parameters for second admin interface configuration: '
'Network - {0}, Netmask - {1}, Interface - {2}, '
'IP Address - {3}').format(second_admin_network,
second_admin_netmask,
second_admin_if,
second_admin_ip))
add_second_admin_ip = ('DEVICE={0}\\n'
'ONBOOT=yes\\n'
'NM_CONTROLLED=no\\n'
'USERCTL=no\\n'
'PEERDNS=no\\n'
'BOOTPROTO=static\\n'
'IPADDR={1}\\n'
'NETMASK={2}\\n').format(second_admin_if,
second_admin_ip,
second_admin_netmask)
cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
add_second_admin_ip, second_admin_if, second_admin_ip)
logger.debug('Trying to assign {0} IP to the {1} on master node...'.
format(second_admin_ip, second_admin_if))
result = remote.execute(cmd)
assert_equal(result['exit_code'], 0, ('Failed to assign second admin '
'IP address on master node: {0}').format(result))
logger.debug('Done: {0}'.format(result['stdout']))
multiple_networks_hacks.configure_second_admin_firewall(
self,
second_admin_network,
second_admin_netmask)
@logwrap
def get_masternode_uuid(self):
return self.postgres_actions.run_query(
db='nailgun',
query="select master_node_uid from master_node_settings limit 1;")
def __init__(self, os_image=None):
self._virtual_environment = None
self._keys = None
self.manager = Manager()
self.os_image = os_image
self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
class EnvironmentModel(object):
hostname = 'nailgun'
domain = 'test.domain.local'
installation_timeout = 1800
deployment_timeout = 1800
puppet_timeout = 2000
nat_interface = '' # INTERFACES.get('admin')
admin_net = 'admin'
admin_net2 = 'admin2'
multiple_cluster_networks = settings.MULTIPLE_NETWORKS
def __init__(self, os_image=None):
self._virtual_environment = None
self._keys = None
self.manager = Manager()
self.os_image = os_image
self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
@property
def nailgun_actions(self):
return FuelActions.Nailgun(self.get_admin_remote())
@property
def postgres_actions(self):
return FuelActions.Postgres(self.get_admin_remote())
def _get_or_create(self):
try:
return self.manager.environment_get(self.env_name)
except Exception:
self._virtual_environment = self.describe_environment()
self._virtual_environment.define()
return self._virtual_environment
def router(self, router_name=None):
router_name = router_name or self.admin_net
if router_name == self.admin_net2:
return str(
IPNetwork(self.get_virtual_environment().network_by_name(
router_name).ip_network)[2])
return str(
IPNetwork(self.get_virtual_environment().network_by_name(
router_name).ip_network)[1])
@property
def fuel_web(self):
"""FuelWebClient
:rtype: FuelWebClient
"""
return self._fuel_web
@property
def admin_node_ip(self):
return self.fuel_web.admin_node_ip
@property
def node_roles(self):
return NodeRoles(admin_names=['admin'],
other_names=[
'slave-%02d' % x
for x in range(1, int(settings.NODES_COUNT))
])
@property
def env_name(self):
return settings.ENV_NAME
def add_empty_volume(self,
node,
name,
capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 *
1024,
device='disk',
bus='virtio',
format='qcow2'):
self.manager.node_attach_volume(
node=node,
volume=self.manager.volume_create(
name=name,
capacity=capacity,
environment=self.get_virtual_environment(),
format=format),
device=device,
bus=bus)
def add_node(self, memory, name, vcpu=1, boot=None):
return self.manager.node_create(
name=name,
memory=memory,
vcpu=vcpu,
environment=self.get_virtual_environment(),
boot=boot)
@logwrap
def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(),
port)
def bootstrap_nodes(self, devops_nodes, timeout=600):
"""Lists registered nailgun nodes
Start vms and wait until they are registered on nailgun.
:rtype : List of registered nailgun nodes
"""
# self.dhcrelay_check()
for node in devops_nodes:
node.start()
# TODO(aglarendil): LP#1317213 temporary sleep
# remove after better fix is applied
time.sleep(2)
wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
for node in self.nailgun_nodes(devops_nodes):
self.sync_node_time(self.get_ssh_to_remote(node["ip"]))
return self.nailgun_nodes(devops_nodes)
def create_interfaces(self,
networks,
node,
model=settings.INTERFACE_MODEL):
if settings.BONDING:
for network in networks:
self.manager.interface_create(
network,
node=node,
model=model,
interface_map=settings.BONDING_INTERFACES)
else:
for network in networks:
self.manager.interface_create(network, node=node, model=model)
def describe_environment(self):
"""Environment
:rtype : Environment
"""
environment = self.manager.environment_create(self.env_name)
networks = []
interfaces = settings.INTERFACE_ORDER
if self.multiple_cluster_networks:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(self.create_networks(name, environment))
for name in self.node_roles.admin_names:
self.describe_admin_node(name, networks)
for name in self.node_roles.other_names:
if self.multiple_cluster_networks:
networks1 = [
net for net in networks
if net.name in settings.NODEGROUPS[0]['pools']
]
networks2 = [
net for net in networks
if net.name in settings.NODEGROUPS[1]['pools']
]
# If slave index is even number, then attach to
# it virtual networks from the second network group.
if int(name[-2:]) % 2 == 1:
self.describe_empty_node(name, networks1)
elif int(name[-2:]) % 2 == 0:
self.describe_empty_node(name, networks2)
else:
self.describe_empty_node(name, networks)
return environment
def create_networks(self, name, environment):
ip_networks = [
IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')
]
new_prefix = int(settings.POOLS.get(name)[1])
pool = self.manager.create_network_pool(networks=ip_networks,
prefix=int(new_prefix))
return self.manager.network_create(
name=name,
environment=environment,
pool=pool,
forward=settings.FORWARDING.get(name),
has_dhcp_server=settings.DHCP.get(name))
def devops_nodes_by_names(self, devops_node_names):
return map(
lambda name: self.get_virtual_environment().node_by_name(name),
devops_node_names)
@logwrap
def describe_admin_node(self, name, networks):
node = self.add_node(memory=settings.HARDWARE.get(
"admin_node_memory", 1024),
vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
name=name,
boot=['hd', 'cdrom'])
self.create_interfaces(networks, node)
if self.os_image is None:
self.add_empty_volume(node, name + '-system')
self.add_empty_volume(node,
name + '-iso',
capacity=_get_file_size(settings.ISO_PATH),
format='raw',
device='cdrom',
bus='ide')
else:
volume = self.manager.volume_get_predefined(self.os_image)
vol_child = self.manager.volume_create_child(
name=name + '-system',
backing_store=volume,
environment=self.get_virtual_environment())
self.manager.node_attach_volume(node=node, volume=vol_child)
return node
def describe_empty_node(self, name, networks):
node = self.add_node(name=name,
memory=settings.HARDWARE.get(
"slave_node_memory", 1024),
vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
self.create_interfaces(networks, node)
self.add_empty_volume(node, name + '-system')
if settings.USE_ALL_DISKS:
self.add_empty_volume(node, name + '-cinder')
self.add_empty_volume(node, name + '-swift')
return node
@logwrap
def get_admin_remote(self,
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password']):
"""SSH to admin node
:rtype : SSHClient
"""
return self.nodes().admin.remote(self.admin_net,
login=login,
password=password)
@logwrap
def get_admin_node_ip(self):
return str(self.nodes().admin.get_ip_address_by_network_name(
self.admin_net))
@logwrap
def get_ebtables(self, cluster_id, devops_nodes):
return Ebtables(self.get_target_devs(devops_nodes),
self.fuel_web.client.get_cluster_vlans(cluster_id))
def get_host_node_ip(self):
return self.router()
def get_keys(self, node, custom=None, build_images=None):
params = {
'ip': node.get_ip_address_by_network_name(self.admin_net),
'mask': self.get_net_mask(self.admin_net),
'gw': self.router(),
'hostname': '.'.join((self.hostname, self.domain)),
'nat_interface': self.nat_interface,
'dns1': settings.DNS,
'showmenu': 'yes' if custom else 'no',
'build_images': '1' if build_images else '0'
}
keys = ("<Wait>\n"
"<Esc><Enter>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n") % params
return keys
@logwrap
def get_private_keys(self, force=False):
if force or self._keys is None:
self._keys = []
for key_string in [
'/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa'
]:
with self.get_admin_remote().open(key_string) as f:
self._keys.append(RSAKey.from_private_key(f))
return self._keys
@logwrap
def get_ssh_to_remote(self, ip):
return SSHClient(ip,
username=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'],
private_keys=self.get_private_keys())
@logwrap
def get_ssh_to_remote_by_key(self, ip, keyfile):
try:
with open(keyfile) as f:
keys = [RSAKey.from_private_key(f)]
return SSHClient(ip, private_keys=keys)
except IOError:
logger.warning('Loading of SSH key from file failed. Trying to use'
' SSH agent ...')
keys = Agent().get_keys()
return SSHClient(ip, private_keys=keys)
@logwrap
def get_ssh_to_remote_by_name(self, node_name):
return self.get_ssh_to_remote(
self.fuel_web.get_nailgun_node_by_devops_node(
self.get_virtual_environment().node_by_name(node_name))['ip'])
def get_target_devs(self, devops_nodes):
return [
interface.target_dev for interface in [
val for var in map(lambda node: node.interfaces, devops_nodes)
for val in var
]
]
def get_virtual_environment(self):
"""Returns virtual environment
:rtype : devops.models.Environment
"""
if self._virtual_environment is None:
self._virtual_environment = self._get_or_create()
return self._virtual_environment
def get_network(self, net_name):
return str(
IPNetwork(self.get_virtual_environment().network_by_name(
net_name).ip_network))
def get_net_mask(self, net_name):
return str(
IPNetwork(self.get_virtual_environment().network_by_name(
net_name).ip_network).netmask)
def make_snapshot(self, snapshot_name, description="", is_make=False):
if settings.MAKE_SNAPSHOT or is_make:
self.get_virtual_environment().suspend(verbose=False)
time.sleep(10)
self.get_virtual_environment().snapshot(snapshot_name, force=True)
revert_info(snapshot_name, description)
if settings.FUEL_STATS_CHECK:
self.get_virtual_environment().resume()
try:
self.nodes().admin. await (self.admin_net, timeout=60)
except Exception:
logger.error('Admin node is unavailable via SSH after '
'environment resume ')
raise
def nailgun_nodes(self, devops_nodes):
return map(
lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
devops_nodes)
def nodes(self):
return Nodes(self.get_virtual_environment(), self.node_roles)
def revert_snapshot(self, name):
if self.get_virtual_environment().has_snapshot(name):
logger.info('We have snapshot with such name %s' % name)
self.get_virtual_environment().revert(name)
logger.info('Starting snapshot reverting ....')
self.get_virtual_environment().resume()
logger.info('Starting snapshot resuming ...')
admin = self.nodes().admin
try:
admin. await (self.admin_net, timeout=10 * 60, by_port=8000)
except Exception as e:
logger.warning("From first time admin isn't reverted: "
"{0}".format(e))
admin.destroy()
logger.info('Admin node was destroyed. Wait 10 sec.')
time.sleep(10)
self.get_virtual_environment().start(self.nodes().admins)
logger.info('Admin node started second time.')
self.nodes().admin. await (self.admin_net,
timeout=10 * 60,
by_port=8000)
self.set_admin_ssh_password()
try:
_wait(self._fuel_web.client.get_releases,
expected=EnvironmentError,
timeout=300)
except exceptions.Unauthorized:
self.set_admin_keystone_password()
self._fuel_web.get_nailgun_version()
self.sync_time_admin_node()
for node in self.nodes().slaves:
if not node.driver.node_active(node):
continue
try:
logger.info("Sync time on revert for node %s" % node.name)
self.sync_node_time(
self.get_ssh_to_remote_by_name(node.name))
except Exception as e:
logger.warning(
'Exception caught while trying to sync time on {0}:'
' {1}'.format(node.name, e))
self.run_nailgun_agent(
self.get_ssh_to_remote_by_name(node.name))
return True
return False
def set_admin_ssh_password(self):
try:
remote = self.get_admin_remote(
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'])
self.execute_remote_cmd(remote, 'date')
logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception:
logger.debug('Accessing admin node using SSH credentials:'
' FAIL, trying to change password from default')
remote = self.get_admin_remote(login='******', password='******')
self.execute_remote_cmd(
remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format(
settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
logger.debug("Admin node password has changed.")
logger.info("Admin node login name: '{0}' , password: '******'".format(
settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
def set_admin_keystone_password(self):
remote = self.get_admin_remote()
try:
self._fuel_web.client.get_releases()
except exceptions.Unauthorized:
self.execute_remote_cmd(
remote, 'fuel user --newpass {0} --change-password'.format(
settings.KEYSTONE_CREDS['password']))
logger.info(
'New Fuel UI (keystone) username: "******", password: "******"'.
format(settings.KEYSTONE_CREDS['username'],
settings.KEYSTONE_CREDS['password']))
def setup_environment(self,
custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES):
# start admin node
admin = self.nodes().admin
admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
self.get_virtual_environment().start(self.nodes().admins)
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(
self.get_keys(admin, custom=custom, build_images=build_images))
if custom:
self.setup_customisation()
# wait while installation complete
admin. await (self.admin_net, timeout=10 * 60)
self.set_admin_ssh_password()
self.wait_bootstrap()
time.sleep(10)
self.set_admin_keystone_password()
self.sync_time_admin_node()
if settings.MULTIPLE_NETWORKS:
self.describe_second_admin_interface()
multiple_networks_hacks.configure_second_admin_cobbler(self)
multiple_networks_hacks.configure_second_dhcrelay(self)
self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
if settings.FUEL_STATS_ENABLED:
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT))
@upload_manifests
def wait_for_provisioning(self):
_wait(lambda: _tcp_ping(
self.nodes().admin.get_ip_address_by_network_name(self.admin_net),
22),
timeout=5 * 60)
def setup_customisation(self):
self.wait_for_provisioning()
try:
remote = self.get_admin_remote()
pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0]
pid.rstrip('\n')
remote.execute("kill -sigusr1 {0}".format(pid))
except Exception:
logger.error("Could not kill pid of fuelmenu")
raise
@retry(count=10, delay=60)
@logwrap
def sync_node_time(self, remote):
self.execute_remote_cmd(remote, 'hwclock -s')
self.execute_remote_cmd(
remote, 'NTPD=$(find /etc/init.d/ -regex \''
'/etc/init.d/\(ntp.?\|ntp-dev\)\');'
'$NTPD stop && ntpd -dqg && $NTPD '
'start')
self.execute_remote_cmd(remote, 'hwclock -w')
remote_date = remote.execute('date')['stdout']
logger.info("Node time: %s" % remote_date)
@retry(count=10, delay=60)
@logwrap
def sync_time_admin_node(self):
logger.info("Sync time on revert for admin")
remote = self.get_admin_remote()
self.execute_remote_cmd(remote, 'hwclock -s')
# Sync time using ntpd
try:
# If public NTP servers aren't accessible ntpdate will fail and
# ntpd daemon shouldn't be restarted to avoid 'Server has gone
# too long without sync' error while syncing time from slaves
self.execute_remote_cmd(
remote, "ntpdate -vu $(awk '/^server/ && "
"$2 !~ /127.*/ {print $2}' "
"/etc/ntp.conf)")
except AssertionError as e:
logger.warning('Error occurred while synchronizing time on master'
': {0}'.format(e))
raise
else:
self.execute_remote_cmd(
remote, 'service ntpd stop && ntpd -dqg &&'
' service ntpd start')
self.execute_remote_cmd(remote, 'hwclock -w')
remote_date = remote.execute('date')['stdout']
logger.info("Master node time: {0}".format(remote_date))
def verify_network_configuration(self, node_name):
checkers.verify_network_configuration(
node=self.fuel_web.get_nailgun_node_by_name(node_name),
remote=self.get_ssh_to_remote_by_name(node_name))
def wait_bootstrap(self):
logger.info("Waiting while bootstrapping is in progress")
log_path = "/var/log/puppet/bootstrap_admin_node.log"
logger.info("Puppet timeout set in {0}".format(
float(settings.PUPPET_TIMEOUT)))
wait(lambda: not self.get_admin_remote().execute(
"grep 'Fuel node deployment' '%s'" % log_path)['exit_code'],
timeout=(float(settings.PUPPET_TIMEOUT)))
result = self.get_admin_remote().execute("grep 'Fuel node deployment "
"complete' '%s'" %
log_path)['exit_code']
if result != 0:
raise Exception('Fuel node deployment failed.')
def dhcrelay_check(self):
admin_remote = self.get_admin_remote()
out = admin_remote.execute("dhcpcheck discover "
"--ifaces eth0 "
"--repeat 3 "
"--timeout 10")['stdout']
assert_true(self.get_admin_node_ip() in "".join(out),
"dhcpcheck doesn't discover master ip")
def run_nailgun_agent(self, remote):
agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
logger.info("Nailgun agent run with exit_code: %s" % agent)
def get_fuel_settings(self, remote=None):
if not remote:
remote = self.get_admin_remote()
cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
result = remote.execute(cmd)
if result['exit_code'] == 0:
fuel_settings = yaml.load(''.join(result['stdout']))
else:
raise Exception('Can\'t output {cfg_file} file: {error}'.format(
cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr']))
return fuel_settings
def admin_install_pkg(self, pkg_name):
"""Install a package <pkg_name> on the admin node"""
admin_remote = self.get_admin_remote()
remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
if remote_status['exit_code'] == 0:
logger.info("Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = admin_remote.execute(
"yum -y install {0}".format(pkg_name))
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}".format(
pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def admin_run_service(self, service_name):
"""Start a service <service_name> on the admin node"""
admin_remote = self.get_admin_remote()
admin_remote.execute("service {0} start".format(service_name))
remote_status = admin_remote.execute(
"service {0} status".format(service_name))
if any('running...' in status for status in remote_status['stdout']):
logger.info("Service '{0}' is running".format(service_name))
else:
logger.info("Service '{0}' failed to start"
" with exit code {1} :\n{2}".format(
service_name, remote_status['exit_code'],
remote_status['stdout']))
# Modifies a resolv.conf on the Fuel master node and returns
# its original content.
# * adds 'nameservers' at start of resolv.conf if merge=True
# * replaces resolv.conf with 'nameservers' if merge=False
def modify_resolv_conf(self, nameservers=[], merge=True):
remote = self.get_admin_remote()
resolv_conf = remote.execute('cat /etc/resolv.conf')
assert_equal(
0, resolv_conf['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'.format('cat /etc/resolv.conf',
resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = remote.execute(echo_cmd)
assert_equal(
0, echo_result['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'.format(echo_cmd,
echo_result['stderr']))
return resolv_conf['stdout']
@logwrap
def execute_remote_cmd(self, remote, cmd, exit_code=0):
result = remote.execute(cmd)
assert_equal(
result['exit_code'], exit_code,
'Failed to execute "{0}" on remote host: {1}'.format(cmd, result))
return result['stdout']
@logwrap
def describe_second_admin_interface(self):
remote = self.get_admin_remote()
second_admin_network = self.get_network(self.admin_net2).split('/')[0]
second_admin_netmask = self.get_net_mask(self.admin_net2)
second_admin_if = settings.INTERFACES.get(self.admin_net2)
second_admin_ip = str(
self.nodes().admin.get_ip_address_by_network_name(self.admin_net2))
logger.info(
('Parameters for second admin interface configuration: '
'Network - {0}, Netmask - {1}, Interface - {2}, '
'IP Address - {3}').format(second_admin_network,
second_admin_netmask, second_admin_if,
second_admin_ip))
add_second_admin_ip = ('DEVICE={0}\\n'
'ONBOOT=yes\\n'
'NM_CONTROLLED=no\\n'
'USERCTL=no\\n'
'PEERDNS=no\\n'
'BOOTPROTO=static\\n'
'IPADDR={1}\\n'
'NETMASK={2}\\n').format(
second_admin_if, second_admin_ip,
second_admin_netmask)
cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
add_second_admin_ip, second_admin_if, second_admin_ip)
logger.debug(
'Trying to assign {0} IP to the {1} on master node...'.format(
second_admin_ip, second_admin_if))
result = remote.execute(cmd)
assert_equal(result['exit_code'], 0,
('Failed to assign second admin '
'IP address on master node: {0}').format(result))
logger.debug('Done: {0}'.format(result['stdout']))
multiple_networks_hacks.configure_second_admin_firewall(
self, second_admin_network, second_admin_netmask)
@logwrap
def get_masternode_uuid(self):
return self.postgres_actions.run_query(
db='nailgun',
query="select master_node_uid from master_node_settings limit 1;")
def __init__(self, os_image=None):
self._virtual_environment = None
self._keys = None
self.manager = Manager()
self.os_image = os_image
self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
def fuel_web(self):
if self._fuel_web is None:
self._fuel_web = FuelWebClient(self)
return self._fuel_web
def __init__(self):
self._virtual_environment = None
self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
class EnvironmentModel(object):
"""EnvironmentModel.""" # TODO documentation
def __init__(self):
self._virtual_environment = None
self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
@property
def admin_actions(self):
return AdminActions(self.d_env.get_admin_remote())
@property
def nailgun_actions(self):
return NailgunActions(self.d_env.get_admin_remote())
@property
def postgres_actions(self):
return PostgresActions(self.d_env.get_admin_remote())
@property
def cobbler_actions(self):
return CobblerActions(self.d_env.get_admin_remote())
@property
def docker_actions(self):
return DockerActions(self.d_env.get_admin_remote())
@property
def admin_node_ip(self):
return self.fuel_web.admin_node_ip
@property
def collector(self):
return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json')
@logwrap
def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server(
cluster_id, self.d_env.router(), port)
def bootstrap_nodes(self, devops_nodes, timeout=600, skip_timesync=False):
"""Lists registered nailgun nodes
Start vms and wait until they are registered on nailgun.
:rtype : List of registered nailgun nodes
"""
# self.dhcrelay_check()
for node in devops_nodes:
logger.info("Bootstrapping node: {}".format(node.name))
node.start()
# TODO(aglarendil): LP#1317213 temporary sleep
# remove after better fix is applied
time.sleep(2)
with timestat("wait_for_nodes_to_start_and_register_in_nailgun"):
wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
if not skip_timesync:
self.sync_time([node for node in self.nailgun_nodes(devops_nodes)])
return self.nailgun_nodes(devops_nodes)
@logwrap
def get_admin_node_ip(self):
return str(
self.d_env.nodes(
).admin.get_ip_address_by_network_name(
self.d_env.admin_net))
@logwrap
def get_ebtables(self, cluster_id, devops_nodes):
return Ebtables(self.get_target_devs(devops_nodes),
self.fuel_web.client.get_cluster_vlans(cluster_id))
def get_keys(self, node, custom=None, build_images=None,
iso_connect_as='cdrom'):
params = {
'ks': 'hd:LABEL="Mirantis_Fuel":/ks.cfg' if iso_connect_as == 'usb'
else 'cdrom:/ks.cfg',
'repo': 'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot
'ip': node.get_ip_address_by_network_name(
self.d_env.admin_net),
'mask': self.d_env.get_network(
name=self.d_env.admin_net).ip.netmask,
'gw': self.d_env.router(),
'hostname': '.'.join((self.d_env.hostname,
self.d_env.domain)),
'nat_interface': self.d_env.nat_interface,
'dns1': settings.DNS,
'showmenu': 'yes' if custom else 'no',
'build_images': '1' if build_images else '0'
}
keys = ''
if(iso_connect_as == 'usb'):
keys = (
"<Wait>\n" # USB boot uses boot_menu=yes for master node
"<F12>\n"
"2\n"
"<Esc><Enter>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=%(ks)s\n"
" repo=%(repo)s\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n"
) % params
else: # cdrom case is default
keys = (
"<Wait>\n"
"<Esc>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=%(ks)s\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n"
) % params
return keys
def get_target_devs(self, devops_nodes):
return [
interface.target_dev for interface in [
val for var in map(lambda node: node.interfaces, devops_nodes)
for val in var]]
@property
def d_env(self):
if self._virtual_environment is None:
try:
return Environment.get(name=settings.ENV_NAME)
except Exception:
self._virtual_environment = Environment.describe_environment(
boot_from=settings.ADMIN_BOOT_DEVICE)
self._virtual_environment.define()
return self._virtual_environment
def resume_environment(self):
self.d_env.resume()
admin = self.d_env.nodes().admin
try:
admin.await(self.d_env.admin_net, timeout=30, by_port=8000)
except Exception as e:
logger.warning("From first time admin isn't reverted: "
"{0}".format(e))
admin.destroy()
logger.info('Admin node was destroyed. Wait 10 sec.')
time.sleep(10)
admin.start()
logger.info('Admin node started second time.')
self.d_env.nodes().admin.await(self.d_env.admin_net)
self.set_admin_ssh_password()
self.docker_actions.wait_for_ready_containers(timeout=600)
# set collector address in case of admin node destroy
if settings.FUEL_STATS_ENABLED:
self.nailgun_actions.set_collector_address(
settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
# Restart statsenderd in order to apply new collector address
self.nailgun_actions.force_fuel_stats_sending()
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT
))
self.set_admin_ssh_password()
self.docker_actions.wait_for_ready_containers()
def make_snapshot(self, snapshot_name, description="", is_make=False):
if settings.MAKE_SNAPSHOT or is_make:
self.d_env.suspend(verbose=False)
time.sleep(10)
self.d_env.snapshot(snapshot_name, force=True)
revert_info(snapshot_name, self.get_admin_node_ip(), description)
if settings.FUEL_STATS_CHECK:
self.resume_environment()
def nailgun_nodes(self, devops_nodes):
return map(
lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
devops_nodes
)
def check_slaves_are_ready(self):
devops_nodes = [node for node in self.d_env.nodes().slaves
if node.driver.node_active(node)]
# Bug: 1455753
time.sleep(30)
for node in devops_nodes:
try:
wait(lambda:
self.fuel_web.get_nailgun_node_by_devops_node(
node)['online'], timeout=60 * 6)
except TimeoutError:
raise TimeoutError(
"Node {0} does not become online".format(node.name))
return True
def revert_snapshot(self, name, skip_timesync=False):
if not self.d_env.has_snapshot(name):
return False
logger.info('We have snapshot with such name: %s' % name)
logger.info("Reverting the snapshot '{0}' ....".format(name))
self.d_env.revert(name)
logger.info("Resuming the snapshot '{0}' ....".format(name))
self.resume_environment()
if not skip_timesync:
nailgun_nodes = [self.fuel_web.get_nailgun_node_by_name(node.name)
for node in self.d_env.nodes().slaves
if node.driver.node_active(node)]
self.sync_time(nailgun_nodes)
try:
_wait(self.fuel_web.client.get_releases,
expected=EnvironmentError, timeout=300)
except exceptions.Unauthorized:
self.set_admin_keystone_password()
self.fuel_web.get_nailgun_version()
_wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6)
return True
def set_admin_ssh_password(self):
try:
remote = self.d_env.get_admin_remote(
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'])
self.execute_remote_cmd(remote, 'date')
logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception:
logger.debug('Accessing admin node using SSH credentials:'
' FAIL, trying to change password from default')
remote = self.d_env.get_admin_remote(
login='******', password='******')
self.execute_remote_cmd(
remote, 'echo -e "{1}\\n{1}" | passwd {0}'
.format(settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
logger.debug("Admin node password has changed.")
logger.info("Admin node login name: '{0}' , password: '******'".
format(settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
def set_admin_keystone_password(self):
remote = self.d_env.get_admin_remote()
try:
self.fuel_web.client.get_releases()
except exceptions.Unauthorized:
self.execute_remote_cmd(
remote, 'fuel user --newpass {0} --change-password'
.format(settings.KEYSTONE_CREDS['password']))
logger.info(
'New Fuel UI (keystone) username: "******", password: "******"'
.format(settings.KEYSTONE_CREDS['username'],
settings.KEYSTONE_CREDS['password']))
def setup_environment(self, custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES,
iso_connect_as=settings.ADMIN_BOOT_DEVICE):
# start admin node
admin = self.d_env.nodes().admin
if(iso_connect_as == 'usb'):
admin.disk_devices.get(device='disk',
bus='usb').volume.upload(settings.ISO_PATH)
else: # cdrom is default
admin.disk_devices.get(
device='cdrom').volume.upload(settings.ISO_PATH)
self.d_env.start(self.d_env.nodes().admins)
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(self.get_keys(admin, custom=custom,
build_images=build_images,
iso_connect_as=iso_connect_as))
if custom:
self.setup_customisation()
# wait while installation complete
admin.await(self.d_env.admin_net, timeout=10 * 60)
self.set_admin_ssh_password()
self.admin_actions.modify_configs(self.d_env.router())
self.wait_bootstrap()
self.docker_actions.wait_for_ready_containers()
time.sleep(10)
self.set_admin_keystone_password()
self.sync_time()
if settings.MULTIPLE_NETWORKS:
self.describe_second_admin_interface()
multiple_networks_hacks.configure_second_admin_cobbler(self)
self.nailgun_actions.set_collector_address(
settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
# Restart statsenderd in order to apply new settings(Collector address)
self.nailgun_actions.force_fuel_stats_sending()
if settings.FUEL_STATS_ENABLED:
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT
))
@update_packages
@upload_manifests
def wait_for_provisioning(self):
_wait(lambda: _tcp_ping(
self.d_env.nodes(
).admin.get_ip_address_by_network_name
(self.d_env.admin_net), 22), timeout=7 * 60)
def setup_customisation(self):
self.wait_for_provisioning()
try:
remote = self.d_env.get_admin_remote()
cmd = "pkill -sigusr1 -f '^.*/fuelmenu$'"
wait(lambda: remote.execute(cmd)['exit_code'] == 0, timeout=60)
except Exception:
logger.error("Could not kill process of fuelmenu")
raise
@retry(count=3, delay=60)
def sync_time(self, nailgun_nodes=[]):
# with @retry, failure on any step of time synchronization causes
# restart the time synchronization starting from the admin node
admin_ntp = Ntp.get_ntp(self.d_env.get_admin_remote(), 'admin')
controller_nodes = [
n for n in nailgun_nodes if "controller" in n['roles']]
other_nodes = [
n for n in nailgun_nodes if "controller" not in n['roles']]
# 1. The first time source for the environment: admin node
logger.info("Synchronizing time on Fuel admin node")
GroupNtpSync().do_sync_time([admin_ntp])
# 2. Controllers should be synchronized before providing time to others
if controller_nodes:
logger.info("Synchronizing time on all controllers")
GroupNtpSync(self, controller_nodes).do_sync_time()
# 3. Synchronize time on all the rest nodes
if other_nodes:
logger.info("Synchronizing time on other active nodes")
GroupNtpSync(self, other_nodes).do_sync_time()
def verify_network_configuration(self, node_name):
node = self.fuel_web.get_nailgun_node_by_name(node_name)
checkers.verify_network_configuration(
node=node,
remote=self.d_env.get_ssh_to_remote(node['ip'])
)
def wait_bootstrap(self):
logger.info("Waiting while bootstrapping is in progress")
log_path = "/var/log/puppet/bootstrap_admin_node.log"
logger.info("Puppet timeout set in {0}".format(
float(settings.PUPPET_TIMEOUT)))
wait(
lambda: not
self.d_env.get_admin_remote().execute(
"grep 'Fuel node deployment' '%s'" % log_path
)['exit_code'],
timeout=(float(settings.PUPPET_TIMEOUT))
)
result = self.d_env.get_admin_remote().execute(
"grep 'Fuel node deployment "
"complete' '%s'" % log_path)['exit_code']
if result != 0:
raise Exception('Fuel node deployment failed.')
def dhcrelay_check(self):
admin_remote = self.d_env.get_admin_remote()
out = admin_remote.execute("dhcpcheck discover "
"--ifaces eth0 "
"--repeat 3 "
"--timeout 10")['stdout']
assert_true(self.get_admin_node_ip() in "".join(out),
"dhcpcheck doesn't discover master ip")
def run_nailgun_agent(self, remote):
agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
logger.info("Nailgun agent run with exit_code: %s" % agent)
def get_fuel_settings(self, remote=None):
if not remote:
remote = self.d_env.get_admin_remote()
cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
result = remote.execute(cmd)
if result['exit_code'] == 0:
fuel_settings = yaml.load(''.join(result['stdout']))
else:
raise Exception('Can\'t output {cfg_file} file: {error}'.
format(cfg_file=settings.FUEL_SETTINGS_YAML,
error=result['stderr']))
return fuel_settings
def admin_install_pkg(self, pkg_name):
"""Install a package <pkg_name> on the admin node"""
admin_remote = self.d_env.get_admin_remote()
remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
if remote_status['exit_code'] == 0:
logger.info("Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = admin_remote.execute("yum -y install {0}"
.format(pkg_name))
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}"
.format(pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def admin_run_service(self, service_name):
"""Start a service <service_name> on the admin node"""
admin_remote = self.d_env.get_admin_remote()
admin_remote.execute("service {0} start".format(service_name))
remote_status = admin_remote.execute("service {0} status"
.format(service_name))
if any('running...' in status for status in remote_status['stdout']):
logger.info("Service '{0}' is running".format(service_name))
else:
logger.info("Service '{0}' failed to start"
" with exit code {1} :\n{2}"
.format(service_name,
remote_status['exit_code'],
remote_status['stdout']))
# Modifies a resolv.conf on the Fuel master node and returns
# its original content.
# * adds 'nameservers' at start of resolv.conf if merge=True
# * replaces resolv.conf with 'nameservers' if merge=False
def modify_resolv_conf(self, nameservers=[], merge=True):
remote = self.d_env.get_admin_remote()
resolv_conf = remote.execute('cat /etc/resolv.conf')
assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'
.format('cat /etc/resolv.conf', resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = remote.execute(echo_cmd)
assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'
.format(echo_cmd, echo_result['stderr']))
return resolv_conf['stdout']
@logwrap
def execute_remote_cmd(self, remote, cmd, exit_code=0):
result = remote.execute(cmd)
assert_equal(result['exit_code'], exit_code,
'Failed to execute "{0}" on remote host: {1}'.
format(cmd, result))
return result['stdout']
@logwrap
def describe_second_admin_interface(self):
remote = self.d_env.get_admin_remote()
admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2)
second_admin_network = admin_net2_object.ip.network
second_admin_netmask = admin_net2_object.ip.netmask
second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2)
second_admin_ip = str(self.d_env.nodes(
).admin.get_ip_address_by_network_name(self.d_env.admin_net2))
logger.info(('Parameters for second admin interface configuration: '
'Network - {0}, Netmask - {1}, Interface - {2}, '
'IP Address - {3}').format(second_admin_network,
second_admin_netmask,
second_admin_if,
second_admin_ip))
add_second_admin_ip = ('DEVICE={0}\\n'
'ONBOOT=yes\\n'
'NM_CONTROLLED=no\\n'
'USERCTL=no\\n'
'PEERDNS=no\\n'
'BOOTPROTO=static\\n'
'IPADDR={1}\\n'
'NETMASK={2}\\n').format(second_admin_if,
second_admin_ip,
second_admin_netmask)
cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
add_second_admin_ip, second_admin_if, second_admin_ip)
logger.debug('Trying to assign {0} IP to the {1} on master node...'.
format(second_admin_ip, second_admin_if))
result = remote.execute(cmd)
assert_equal(result['exit_code'], 0, ('Failed to assign second admin '
'IP address on master node: {0}').format(result))
logger.debug('Done: {0}'.format(result['stdout']))
multiple_networks_hacks.configure_second_admin_firewall(
self,
second_admin_network,
second_admin_netmask)
@logwrap
def get_masternode_uuid(self):
return self.postgres_actions.run_query(
db='nailgun',
query="select master_node_uid from master_node_settings limit 1;")
def __init__(self):
self._virtual_environment = None
self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
class EnvironmentModel(object):
"""EnvironmentModel.""" # TODO documentation
def __init__(self):
self._virtual_environment = None
self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
if hasattr(self, 'fuel_web'):
ip = self.fuel_web.admin_node_ip
else:
ip = None
return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass,
obj_id=obj_id,
ip=ip)
@property
def admin_actions(self):
return AdminActions(self.d_env.get_admin_remote())
@property
def nailgun_actions(self):
return NailgunActions(self.d_env.get_admin_remote())
@property
def postgres_actions(self):
return PostgresActions(self.d_env.get_admin_remote())
@property
def cobbler_actions(self):
return CobblerActions(self.d_env.get_admin_remote())
@property
def docker_actions(self):
return DockerActions(self.d_env.get_admin_remote())
@property
def admin_node_ip(self):
return self.fuel_web.admin_node_ip
@property
def collector(self):
return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json')
@logwrap
def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port)
def bootstrap_nodes(self, devops_nodes, timeout=900, skip_timesync=False):
"""Lists registered nailgun nodes
Start vms and wait until they are registered on nailgun.
:rtype : List of registered nailgun nodes
"""
# self.dhcrelay_check()
for node in devops_nodes:
logger.info("Bootstrapping node: {}".format(node.name))
node.start()
# TODO(aglarendil): LP#1317213 temporary sleep
# remove after better fix is applied
time.sleep(2)
with timestat("wait_for_nodes_to_start_and_register_in_nailgun"):
wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
if not skip_timesync:
self.sync_time([node for node in self.nailgun_nodes(devops_nodes)])
return self.nailgun_nodes(devops_nodes)
@logwrap
def get_admin_node_ip(self):
return str(self.d_env.nodes().admin.get_ip_address_by_network_name(
self.d_env.admin_net))
@logwrap
def get_ebtables(self, cluster_id, devops_nodes):
return Ebtables(self.get_target_devs(devops_nodes),
self.fuel_web.client.get_cluster_vlans(cluster_id))
def get_keys(self,
node,
custom=None,
build_images=None,
iso_connect_as='cdrom'):
params = {
'ks':
'hd:LABEL="Mirantis_Fuel":/ks.cfg'
if iso_connect_as == 'usb' else 'cdrom:/ks.cfg',
'repo':
'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot
'ip':
node.get_ip_address_by_network_name(self.d_env.admin_net),
'mask':
self.d_env.get_network(name=self.d_env.admin_net).ip.netmask,
'gw':
self.d_env.router(),
'hostname':
''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)),
'nat_interface':
self.d_env.nat_interface,
'dns1':
settings.DNS,
'showmenu':
'yes' if custom else 'no',
'build_images':
'1' if build_images else '0'
}
keys = ''
if (iso_connect_as == 'usb'):
keys = (
"<Wait>\n" # USB boot uses boot_menu=yes for master node
"<F12>\n"
"2\n"
"<Esc><Enter>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=%(ks)s\n"
" repo=%(repo)s\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n") % params
else: # cdrom case is default
keys = ("<Wait>\n"
"<Wait>\n"
"<Wait>\n"
"<Esc>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img ks=%(ks)s\n"
" ip=%(ip)s\n"
" netmask=%(mask)s\n"
" gw=%(gw)s\n"
" dns1=%(dns1)s\n"
" hostname=%(hostname)s\n"
" dhcp_interface=%(nat_interface)s\n"
" showmenu=%(showmenu)s\n"
" build_images=%(build_images)s\n"
" <Enter>\n") % params
return keys
def get_target_devs(self, devops_nodes):
return [
interface.target_dev for interface in [
val for var in map(lambda node: node.interfaces, devops_nodes)
for val in var
]
]
@property
def d_env(self):
if self._virtual_environment is None:
try:
return Environment.get(name=settings.ENV_NAME)
except Exception:
self._virtual_environment = Environment.describe_environment(
boot_from=settings.ADMIN_BOOT_DEVICE)
self._virtual_environment.define()
return self._virtual_environment
def resume_environment(self):
self.d_env.resume()
admin = self.d_env.nodes().admin
try:
admin. await (self.d_env.admin_net, timeout=30, by_port=8000)
except Exception as e:
logger.warning("From first time admin isn't reverted: "
"{0}".format(e))
admin.destroy()
logger.info('Admin node was destroyed. Wait 10 sec.')
time.sleep(10)
admin.start()
logger.info('Admin node started second time.')
self.d_env.nodes().admin. await (self.d_env.admin_net)
self.set_admin_ssh_password()
self.docker_actions.wait_for_ready_containers(timeout=600)
# set collector address in case of admin node destroy
if settings.FUEL_STATS_ENABLED:
self.nailgun_actions.set_collector_address(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
# Restart statsenderd in order to apply new collector address
self.nailgun_actions.force_fuel_stats_sending()
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT))
self.set_admin_ssh_password()
self.docker_actions.wait_for_ready_containers()
def make_snapshot(self, snapshot_name, description="", is_make=False):
if settings.MAKE_SNAPSHOT or is_make:
self.d_env.suspend(verbose=False)
time.sleep(10)
self.d_env.snapshot(snapshot_name, force=True)
revert_info(snapshot_name, self.get_admin_node_ip(), description)
if settings.FUEL_STATS_CHECK:
self.resume_environment()
def nailgun_nodes(self, devops_nodes):
return map(
lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
devops_nodes)
def check_slaves_are_ready(self):
devops_nodes = [
node for node in self.d_env.nodes().slaves
if node.driver.node_active(node)
]
# Bug: 1455753
time.sleep(30)
for node in devops_nodes:
try:
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
node)['online'],
timeout=60 * 6)
except TimeoutError:
raise TimeoutError("Node {0} does not become online".format(
node.name))
return True
def revert_snapshot(self, name, skip_timesync=False):
if not self.d_env.has_snapshot(name):
return False
logger.info('We have snapshot with such name: %s' % name)
logger.info("Reverting the snapshot '{0}' ....".format(name))
self.d_env.revert(name)
logger.info("Resuming the snapshot '{0}' ....".format(name))
self.resume_environment()
if not skip_timesync:
nailgun_nodes = [
self.fuel_web.get_nailgun_node_by_name(node.name)
for node in self.d_env.nodes().slaves
if node.driver.node_active(node)
]
self.sync_time(nailgun_nodes)
try:
_wait(self.fuel_web.client.get_releases,
expected=EnvironmentError,
timeout=300)
except exceptions.Unauthorized:
self.set_admin_keystone_password()
self.fuel_web.get_nailgun_version()
_wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6)
return True
def set_admin_ssh_password(self):
try:
with self.d_env.get_admin_remote(
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password']) as remote:
self.execute_remote_cmd(remote, 'date')
logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception:
logger.debug('Accessing admin node using SSH credentials:'
' FAIL, trying to change password from default')
with self.d_env.get_admin_remote(login='******',
password='******') as remote:
self.execute_remote_cmd(
remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format(
settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
logger.debug("Admin node password has changed.")
logger.info("Admin node login name: '{0}' , password: '******'".format(
settings.SSH_CREDENTIALS['login'],
settings.SSH_CREDENTIALS['password']))
def set_admin_keystone_password(self):
try:
self.fuel_web.client.get_releases()
except exceptions.Unauthorized:
with self.d_env.get_admin_remote() as remote:
self.execute_remote_cmd(
remote, 'fuel user --newpass {0} --change-password'.format(
settings.KEYSTONE_CREDS['password']))
logger.info(
'New Fuel UI (keystone) username: "******", password: "******"'.
format(settings.KEYSTONE_CREDS['username'],
settings.KEYSTONE_CREDS['password']))
def setup_environment(self,
custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES,
iso_connect_as=settings.ADMIN_BOOT_DEVICE,
security=settings.SECURITY_TEST):
# start admin node
admin = self.d_env.nodes().admin
if (iso_connect_as == 'usb'):
admin.disk_devices.get(device='disk',
bus='usb').volume.upload(settings.ISO_PATH)
else: # cdrom is default
admin.disk_devices.get(device='cdrom').volume.upload(
settings.ISO_PATH)
self.d_env.start(self.d_env.nodes().admins)
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(
self.get_keys(admin,
custom=custom,
build_images=build_images,
iso_connect_as=iso_connect_as))
if custom:
self.setup_customisation()
if security:
nessus_node = NessusActions(self.d_env)
nessus_node.add_nessus_node()
# wait while installation complete
admin. await (self.d_env.admin_net, timeout=10 * 60)
self.set_admin_ssh_password()
self.admin_actions.modify_configs(self.d_env.router())
self.wait_bootstrap()
self.docker_actions.wait_for_ready_containers()
time.sleep(10)
self.set_admin_keystone_password()
self.sync_time()
if settings.UPDATE_MASTER:
if settings.UPDATE_FUEL_MIRROR:
for i, url in enumerate(settings.UPDATE_FUEL_MIRROR):
conf_file = '/etc/yum.repos.d/temporary-{}.repo'.format(i)
cmd = ("echo -e"
" '[temporary-{0}]\nname="
"temporary-{0}\nbaseurl={1}/"
"\ngpgcheck=0\npriority="
"1' > {2}").format(i, url, conf_file)
with self.d_env.get_admin_remote() as remote:
remote.execute(cmd)
self.admin_install_updates()
if settings.MULTIPLE_NETWORKS:
self.describe_second_admin_interface()
multiple_networks_hacks.configure_second_admin_cobbler(self)
self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
# Restart statsenderd in order to apply new settings(Collector address)
self.nailgun_actions.force_fuel_stats_sending()
if settings.FUEL_STATS_ENABLED:
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT))
if settings.PATCHING_DISABLE_UPDATES:
with self.d_env.get_admin_remote() as remote:
cmd = "find /etc/yum.repos.d/ -type f -regextype posix-egrep" \
" -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " \
"xargs -n1 -i sed '$aenabled=0' -i {}"
self.execute_remote_cmd(remote, cmd)
@update_packages
@upload_manifests
def wait_for_provisioning(self):
_wait(lambda: _tcp_ping(
self.d_env.nodes().admin.get_ip_address_by_network_name(
self.d_env.admin_net), 22),
timeout=7 * 60)
def setup_customisation(self):
self.wait_for_provisioning()
try:
cmd = "pkill -sigusr1 -f '^.*/fuelmenu$'"
with self.d_env.get_admin_remote() as remote:
wait(lambda: remote.execute(cmd)['exit_code'] == 0, timeout=60)
except Exception:
logger.error("Could not kill process of fuelmenu")
raise
@retry(count=3, delay=60)
def sync_time(self, nailgun_nodes=[]):
# with @retry, failure on any step of time synchronization causes
# restart the time synchronization starting from the admin node
controller_nodes = [
n for n in nailgun_nodes if "controller" in n['roles']
]
other_nodes = [
n for n in nailgun_nodes if "controller" not in n['roles']
]
# 1. The first time source for the environment: admin node
logger.info("Synchronizing time on Fuel admin node")
with GroupNtpSync(self, sync_admin_node=True) as g_ntp:
g_ntp.do_sync_time()
# 2. Controllers should be synchronized before providing time to others
if controller_nodes:
logger.info("Synchronizing time on all controllers")
with GroupNtpSync(self, nailgun_nodes=controller_nodes) as g_ntp:
g_ntp.do_sync_time()
# 3. Synchronize time on all the rest nodes
if other_nodes:
logger.info("Synchronizing time on other active nodes")
with GroupNtpSync(self, nailgun_nodes=other_nodes) as g_ntp:
g_ntp.do_sync_time()
def verify_network_configuration(self, node_name):
node = self.fuel_web.get_nailgun_node_by_name(node_name)
with self.fuel_web.get_ssh_for_node(node_name) as ssh:
checkers.verify_network_configuration(node=node, remote=ssh)
def wait_bootstrap(self):
logger.info("Waiting while bootstrapping is in progress")
log_path = "/var/log/puppet/bootstrap_admin_node.log"
logger.info("Puppet timeout set in {0}".format(
float(settings.PUPPET_TIMEOUT)))
wait(lambda: not self.d_env.get_admin_remote().execute(
"grep 'Fuel node deployment' '%s'" % log_path)['exit_code'],
timeout=(float(settings.PUPPET_TIMEOUT)))
result = self.d_env.get_admin_remote().execute(
"grep 'Fuel node deployment "
"complete' '%s'" % log_path)['exit_code']
if result != 0:
raise Exception('Fuel node deployment failed.')
def dhcrelay_check(self):
with self.d_env.get_admin_remote() as admin_remote:
out = admin_remote.execute("dhcpcheck discover "
"--ifaces eth0 "
"--repeat 3 "
"--timeout 10")['stdout']
assert_true(self.get_admin_node_ip() in "".join(out),
"dhcpcheck doesn't discover master ip")
def get_fuel_settings(self, remote=None):
cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
if not remote:
with self.d_env.get_admin_remote() as remote:
remote.execute(cmd)
else:
result = remote.execute(cmd)
if result['exit_code'] == 0:
fuel_settings = yaml.load(''.join(result['stdout']))
else:
raise Exception('Can\'t output {cfg_file} file: {error}'.format(
cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr']))
return fuel_settings
def admin_install_pkg(self, pkg_name):
"""Install a package <pkg_name> on the admin node"""
with self.d_env.get_admin_remote() as remote:
remote_status = remote.execute("rpm -q {0}'".format(pkg_name))
if remote_status['exit_code'] == 0:
logger.info(
"Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = remote.execute(
"yum -y install {0}".format(pkg_name))
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}".format(
pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def admin_run_service(self, service_name):
"""Start a service <service_name> on the admin node"""
with self.d_env.get_admin_remote() as admin_remote:
admin_remote.execute("service {0} start".format(service_name))
remote_status = admin_remote.execute(
"service {0} status".format(service_name))
if any('running...' in status for status in remote_status['stdout']):
logger.info("Service '{0}' is running".format(service_name))
else:
logger.info("Service '{0}' failed to start"
" with exit code {1} :\n{2}".format(
service_name, remote_status['exit_code'],
remote_status['stdout']))
# Execute yum updates
# If updates installed,
# then `dockerctl destroy all; bootstrap_admin_node.sh;`
def admin_install_updates(self):
logger.info('Searching for updates..')
update_command = 'yum clean expire-cache; yum update -y'
with self.d_env.get_admin_remote() as admin_remote:
update_result = admin_remote.execute(update_command)
logger.info('Result of "{1}" command on master node: '
'{0}'.format(update_result, update_command))
assert_equal(int(update_result['exit_code']), 0,
'Packages update failed, '
'inspect logs for details')
# Check if any packets were updated and update was successful
for str_line in update_result['stdout']:
match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package",
str_line)
if match_updated_count:
updates_count = match_updated_count.group(1)
match_complete_message = re.search("(Complete!)", str_line)
match_no_updates = re.search("No Packages marked for Update",
str_line)
if (not match_updated_count or match_no_updates)\
and not match_complete_message:
logger.warning('No updates were found or update was incomplete.')
return
logger.info('{0} packet(s) were updated'.format(updates_count))
cmd = 'dockerctl destroy all; bootstrap_admin_node.sh;'
with self.d_env.get_admin_remote() as admin_remote:
result = admin_remote.execute(cmd)
logger.info('Result of "{1}" command on master node: '
'{0}'.format(result, cmd))
assert_equal(int(result['exit_code']), 0, 'bootstrap failed, '
'inspect logs for details')
# Modifies a resolv.conf on the Fuel master node and returns
# its original content.
# * adds 'nameservers' at start of resolv.conf if merge=True
# * replaces resolv.conf with 'nameservers' if merge=False
def modify_resolv_conf(self, nameservers=[], merge=True):
with self.d_env.get_admin_remote() as remote:
resolv_conf = remote.execute('cat /etc/resolv.conf')
assert_equal(
0, resolv_conf['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'.format(
'cat /etc/resolv.conf', resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = remote.execute(echo_cmd)
assert_equal(
0, echo_result['exit_code'], 'Executing "{0}" on the '
'admin node has failed with: {1}'.format(
echo_cmd, echo_result['stderr']))
return resolv_conf['stdout']
@logwrap
def execute_remote_cmd(self, remote, cmd, exit_code=0):
result = remote.execute(cmd)
assert_equal(
result['exit_code'], exit_code,
'Failed to execute "{0}" on remote host: {1}'.format(cmd, result))
return result['stdout']
@logwrap
def describe_second_admin_interface(self):
admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2)
second_admin_network = admin_net2_object.ip.network
second_admin_netmask = admin_net2_object.ip.netmask
second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2)
second_admin_ip = str(
self.d_env.nodes().admin.get_ip_address_by_network_name(
self.d_env.admin_net2))
logger.info(
('Parameters for second admin interface configuration: '
'Network - {0}, Netmask - {1}, Interface - {2}, '
'IP Address - {3}').format(second_admin_network,
second_admin_netmask, second_admin_if,
second_admin_ip))
add_second_admin_ip = ('DEVICE={0}\\n'
'ONBOOT=yes\\n'
'NM_CONTROLLED=no\\n'
'USERCTL=no\\n'
'PEERDNS=no\\n'
'BOOTPROTO=static\\n'
'IPADDR={1}\\n'
'NETMASK={2}\\n').format(
second_admin_if, second_admin_ip,
second_admin_netmask)
cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
add_second_admin_ip, second_admin_if, second_admin_ip)
logger.debug(
'Trying to assign {0} IP to the {1} on master node...'.format(
second_admin_ip, second_admin_if))
with self.d_env.get_admin_remote() as remote:
result = remote.execute(cmd)
assert_equal(result['exit_code'], 0,
('Failed to assign second admin '
'IP address on master node: {0}').format(result))
logger.debug('Done: {0}'.format(result['stdout']))
multiple_networks_hacks.configure_second_admin_firewall(
self, second_admin_network, second_admin_netmask)
@logwrap
def get_masternode_uuid(self):
return self.postgres_actions.run_query(
db='nailgun',
query="select master_node_uid from master_node_settings limit 1;")
