def run(self):
configifc = ConfigInterface()
config = configifc.open()
if not config.paths:
LOG.info('Test runner sanity skipped.')
return
assert config.paths.build, 'CM Build path is not set in the config'
assert config.paths.logs, 'Logs path is not set in the config'
sample = os.path.join(config.paths.build, 'bigip')
if not os.path.exists(sample):
raise StageError("%s does not exist" % sample)
sample = config.paths.get('logs')
sample = os.path.expanduser(sample)
sample = os.path.expandvars(sample)
if not os.access(sample, os.W_OK):
raise StageError("Logs dir: %s is not writable" % sample)
stats = os.statvfs(sample)
if not (stats.f_bsize * stats.f_bavail) / 1024 ** 2 > 100:
raise StageError("Logs dir: %s has not enough space left" % sample)
LOG.info('Test runner sanity check passed!')
def run(self):
configifc = ConfigInterface()
config = configifc.open()
if not config.paths:
LOG.info('Test runner sanity skipped.')
return
assert config.paths.build, 'CM Build path is not set in the config'
assert config.paths.logs, 'Logs path is not set in the config'
sample = os.path.join(config.paths.build, 'bigip')
if not os.path.exists(sample):
raise StageError("%s does not exist" % sample)
sample = config.paths.get('logs')
sample = os.path.expanduser(sample)
sample = os.path.expandvars(sample)
if not os.access(sample, os.W_OK):
raise StageError("Logs dir: %s is not writable" % sample)
stats = os.statvfs(sample)
if not (stats.f_bsize * stats.f_bavail) / 1024**2 > 100:
raise StageError("Logs dir: %s has not enough space left" % sample)
LOG.info('Test runner sanity check passed!')
def __init__(self, options, address=None, address_iq=None, *args, **kwargs):
self.context = O()
self.options = O(options)
self.options.setifnone('timeout', DEFAULT_TIMEOUT)
self.options.setifnone('skip_ping', False)
if self.options.device:
self.device = ConfigInterface().get_device(options.device)
self.address = self.device.address
else:
self.device = None
self.address = address
self.options.setifnone('username', DEFAULT_ROOT_USERNAME)
self.options.setifnone('password', DEFAULT_ROOT_PASSWORD)
self.options.setifnone('admin_username', DEFAULT_ADMIN_USERNAME)
self.options.setifnone('admin_password', DEFAULT_ADMIN_PASSWORD)
if self.options.device_biq:
self.device_biq = ConfigInterface().get_device(options.device_biq)
self.address_biq = self.device_biq.address
else:
self.device_biq = None
self.address_biq = address_iq
self.options.setifnone('username_iq', DEFAULT_ROOT_USERNAME)
self.options.setifnone('password_iq', DEFAULT_ROOT_PASSWORD)
super(ScaleCheck, self).__init__(*args, **kwargs)
def set_defaults(self):
if self.options.device:
device = ConfigInterface().get_device(self.options.device)
self.address = device.get_address()
self.options.admin_username = device.get_admin_creds().username
self.options.admin_password = device.get_admin_creds().password
self.options.root_username = device.get_root_creds().username
self.options.root_password = device.get_root_creds().password
self.options.ssl_port = device.ports.get('https', 443)
self.options.ssh_port = device.ports.get('ssh', 22)
self.options.setdefault('admin_username', ADMIN_USERNAME)
self.options.setdefault('admin_password', ADMIN_PASSWORD)
self.options.setdefault('root_username', ROOT_USERNAME)
self.options.setdefault('root_password', ROOT_PASSWORD)
self.options.setdefault('build_path', cm.ROOT_PATH)
def __init__(self, options, address=None):
self.options = Options(options)
self.address = ConfigInterface().get_device_address(options.device) \
if self.options.device else address
if self.options.alias is None:
self.options.alias = []
super(WebCert, self).__init__()
def __init__(self, options, address=None):
self.options = Options(options)
if self.options.device:
self.address = ConfigInterface().get_device_address(options.device)
else:
self.address = address
LOG.info('Doing: %s', self.address)
super(KeySwap, self).__init__()
def __init__(self, options, address=None, *args, **kwargs):
self.context = O()
self.options = O(options)
self.options.setifnone('node_count', DEFAULT_NODES)
self.options.setifnone('pool_count', DEFAULT_POOLS)
self.options.setifnone('pool_members', DEFAULT_MEMBERS)
self.options.setifnone('vip_count', DEFAULT_VIPS)
self.options.setifnone('node_start', DEFAULT_NODE_START)
self.options.setifnone('partitions', DEFAULT_PARTITIONS)
self.options.setifnone('timeout', DEFAULT_TIMEOUT)
if self.options.device:
self.device = ConfigInterface().get_device(options.device)
self.address = self.device.address
else:
self.device = None
self.address = address
self.options.setifnone('username', DEFAULT_ROOT_USERNAME)
self.options.setifnone('password', DEFAULT_ROOT_PASSWORD)
# can.* shortcuts to check for certain features based on the version
self.can = O()
def can_tmsh(v):
return (v.product.is_bigip and v >= 'bigip 11.0.0' or
v.product.is_em and v >= 'em 2.0.0' or
v.product.is_bigiq)
self.can.tmsh = can_tmsh
def can_provision(v):
return (v.product.is_bigip and v >= 'bigip 10.0.1' or
v.product.is_em and v >= 'em 2.0.0' or
v.product.is_bigiq)
self.can.provision = can_provision
def can_lvm(sshifc):
return not sshifc('/usr/lib/install/lvmtest').status
self.can.lvm = can_lvm
self.has = O()
def has_asm(s):
return bool(SCMD.tmsh.get_provision(ifc=s).asm)
self.has.asm = has_asm
def has_lvm(s):
return not s('/usr/lib/install/lvmtest').status
self.has.lvm = has_lvm
super(ConfigPlacer, self).__init__(*args, **kwargs)
def process_stages(stages, section, context, stop_on_error=True):
if not stages:
LOG.debug('No stages found.')
return
# Replicate the "_enabled" flag.
carry_flag(stages)
# Build the stage map with *ALL* defined stage classes in this file.
stages_map = {}
for value in list(globals().values()):
if inspect.isclass(value) and issubclass(value,
Stage) and value != Stage:
stages_map[value.name] = value
# Focus only on our stages section
for key in section.split('.'):
stages = stages.get(key, Options())
# Sort stages by priority attribute and stage name.
stages = sorted(iter(stages.items()),
key=lambda x: (isinstance(x[1], dict) and x[1].get(
PRIORITY_KEY, DEFAULT_PRIORITY), x[0]))
config = ConfigInterface().config
# Group stages of the same type. The we spin up one thread per stage in a
# group and wait for threads within a group to finish.
sg_dict = {}
sg_list = []
for name, specs in stages:
if not specs or name.startswith('_'):
continue
assert TYPE_KEY in specs, "%s stage is invalid. No type specified." % name
specs = Options(specs)
key = specs.get(GROUP_KEY, "{0}-{1}".format(name, specs[TYPE_KEY]))
group = sg_dict.get(key)
if not group:
sg_dict[key] = []
sg_list.append(sg_dict[key])
sg_dict[key].append((name, specs))
LOG.debug("sg_list: %s", sg_list)
for stages in sg_list:
q = Queue()
pool = []
for stage in stages:
description, specs = stage
if not specs or not to_bool(specs.get(ENABLE_KEY)):
continue
LOG.info("Processing stage: %s", description)
# items() reverts <Options> to a simple <dict>
specs = Options(specs)
if not stages_map.get(specs[TYPE_KEY]):
LOG.warning("Stage '%s' (%s) not defined.", description,
specs[TYPE_KEY])
continue
stage_class = stages_map[specs[TYPE_KEY]]
parameters = specs.get(PARAMETERS_KEY) or Options()
parameters._context = context
devices = expand_devices(specs)
if devices is None:
stage_class(parameters).run()
elif devices == []:
LOG.error("Stage %s requires devices but found none" %
description)
else:
if not devices:
LOG.warning('No devices found for stage %s', description)
if specs.get('shuffle', False):
random.shuffle(devices)
for device in devices:
stage = stage_class(device, parameters)
name = '%s :: %s' % (
description, device.alias) if device else description
t = MacroThread(stage, q, name=name)
t.start()
pool.append(t)
if not stage_class.parallelizable or not specs.get(
'parallelizable', True):
t.join()
# Cap the number parallel threads
if len(pool) >= specs.get('threads', MAX_THREADS):
list(map(lambda x: x.join(), pool))
pool[:] = []
LOG.debug('Waiting for threads...')
for t in pool:
t.join()
if not q.empty():
stages = []
while not q.empty():
ret = q.get(block=False)
thread, exc_info = ret.popitem()
stages.append((thread, exc_info))
LOG.error('Exception while "%s"', thread.getName())
for line in traceback.format_exception(*exc_info):
LOG.error(line.strip())
if stop_on_error:
raise StageError(stages)
class ScaleCheck(Macro):
def __init__(self,
options,
address=None,
address_iq=None,
*args,
**kwargs):
self.context = O()
self.options = O(options)
self.options.setifnone('timeout', DEFAULT_TIMEOUT)
self.options.setifnone('skip_ping', False)
if self.options.device:
self.device = ConfigInterface().get_device(options.device)
self.address = self.device.address
else:
self.device = None
self.address = address
self.options.setifnone('username', DEFAULT_ROOT_USERNAME)
self.options.setifnone('password', DEFAULT_ROOT_PASSWORD)
self.options.setifnone('admin_username', DEFAULT_ADMIN_USERNAME)
self.options.setifnone('admin_password', DEFAULT_ADMIN_PASSWORD)
if self.options.device_biq:
self.device_biq = ConfigInterface().get_device(options.device_biq)
self.address_biq = self.device_biq.address
else:
self.device_biq = None
self.address_biq = address_iq
self.options.setifnone('username_iq', DEFAULT_ROOT_USERNAME)
self.options.setifnone('password_iq', DEFAULT_ROOT_PASSWORD)
self.sshifc = None
self.sshifc_biq = None
super(ScaleCheck, self).__init__(*args, **kwargs)
def prep(self):
self.sshifc = SSHInterface(device=self.device,
address=self.address,
username=self.options.username,
password=self.options.password,
timeout=self.options.timeout,
port=self.options.ssh_port)
self.sshifc.open()
self.sshifc_biq = SSHInterface(device=self.options.device_biq,
address=self.address_biq,
username=self.options.username_iq,
password=self.options.password_iq,
timeout=self.options.timeout,
port=self.options.ssh_port)
self.sshifc_biq.open()
def wait_prompt(self):
return SCMD.ssh.GetPrompt(ifc=self.sshifc)\
.run_wait(lambda x: x not in ('INOPERATIVE',),
progress_cb=lambda x: 'Still %s...' % x,
timeout=self.options.timeout, interval=10)
def make_context(self):
ctx = self.context
ctx.version = SCMD.ssh.get_version(ifc=self.sshifc)
ctx.status = self.wait_prompt()
LOG.info('Version: {0.product.to_tmos} {0.version} {0.build}'.format(
ctx.version))
return ctx
def call(self, command):
ret = SCMD.ssh.generic(command=command, ifc=self.sshifc)
if ret and ret.status:
LOG.warn(ret)
else:
LOG.debug(ret)
return ret
def clean_storage(self, ctx):
has_restjavad = None
with EmapiInterface(device=self.device,
username=self.options.admin_username,
password=self.options.admin_password,
port=self.options.ssl_port,
address=self.address) as rstifc:
try:
if ctx.version < "bigip 11.5.0":
has_restjavad = rstifc.api.get(DeviceResolver.URI)
except EmapiResourceError:
LOG.warning("This pre 11.5.0 device hasn't had latest"
" REST Framework upgrades")
pass
self.call("rm -rf /var/log/rest*")
self.call("find /var/log -name '*.gz' -exec rm {} \;")
# # Remove all files that are: blabla.1.blabla or blabla.1
# self.call("find /var/log -regex '.*[.][1-9].*' -exec rm '{}' \\;")
self.call(WIPE_STORAGE)
# Because of a bug where restjavad not knowing about icrd, BZ504333.
if self.sshifc.version.product.is_bigip:
self.call("bigstart restart icrd")
with EmapiInterface(device=self.device,
username=self.options.admin_username,
password=self.options.admin_password,
port=self.options.ssl_port,
address=self.address) as rstifc:
if has_restjavad:
wait_args(
rstifc.api.get,
func_args=[DeviceResolver.URI],
progress_message="Waiting for restjavad...",
timeout=300,
timeout_message="restjavad never came back up after {0}s")
def relicense(self, ctx):
if ctx.status != 'NO LICENSE':
license_date = self.call('grep "License end" %s | cut -d: -f2' %
LICENSE_FILE).stdout.strip()
license_key = self.call(
'grep "Registration Key" %s | cut -d: -f2' %
LICENSE_FILE).stdout.strip()
date_format = "%Y%m%d"
expire_date = datetime.datetime.strptime(license_date, date_format)
delta = expire_date - datetime.datetime.now()
if delta > datetime.timedelta(days=15):
LOG.debug("%s is NOT within 15 days of being expired. "
"Expiration date: %s" % (self.device, license_date))
else:
LOG.info("Re-licensing %s. Expiration date: %s" %
(self.device, license_date))
ret = self.call('SOAPLicenseClient --verbose --basekey %s' %
license_key)
LOG.debug("SOAPLicenseClient returned: %s", ret)
else:
raise MacroError("%s does not have a license. Expect BIG-IP to be "
"functional" % self.device)
def ping_check(self):
bip_selfips = SCMD.tmsh.list("net self", ifc=self.sshifc)
self_ips = [x['address'].split('/')[0] for x in bip_selfips.values()]
for self_ip in self_ips:
self_ip = IPAddress(self_ip)
if self_ip.version == 4:
COMMAND = "ping -c 1 %s" % self_ip.format(ipv6_full)
elif self_ip.version == 6:
COMMAND = "ping6 -c 1 %s" % self_ip.format(ipv6_full)
# TODO: Find out why we can't ping using ipv6 address on Lowell's BIG-IQs
continue
else:
LOG.info(
"You got some weird IP address that isn't ipv4 or ipv6")
LOG.info("Ping %s from %s" %
(self_ip, self.options.device_biq
if self.options.device_biq else self.address_biq))
resp = SCMD.ssh.generic(COMMAND, ifc=self.sshifc_biq)
if '100% packet loss' in resp.stdout:
LOG.info("device: %s not reachable" % self.device)
LOG.debug("device: %s - %s" % (self.device, resp))
raise Exception("device: %s not reachable" % self.device)
if self.device:
self_ip = IPAddress(self.device.get_discover_address())
LOG.info("Verify given %s from yaml matches one on BIG-IP" %
self_ip)
for a in bip_selfips.values():
bip_ip = IPAddress(a['address'].split('/')[0])
if a['vlan'] == 'internal' and \
self_ip.version == bip_ip.version:
internal_ip = bip_ip
break
if self_ip.format(ipv6_full) != internal_ip.format(ipv6_full):
LOG.info("Internal mismatch: %s. %s != %s." %
(self.device, self_ip, internal_ip))
else:
LOG.info(
"This isn't ran as stages so skipping internal selfip check")
def setup(self):
ctx = self.make_context()
LOG.info("Deleting rest logs, *.gz files, and wiping storage")
if ctx.version.product.is_bigip:
self.clean_storage(ctx)
# Check license and re-license if almost expired
self.relicense(ctx)
# Check if BIG-IQ can reach BIG-IP
if not self.options.skip_ping:
self.ping_check()
# bigstart restart if BIG-IP is something else other than 'Active'
LOG.info("State: %s" % ctx.status)
if ctx.status != 'Active':
LOG.info("bigstart restart on {0}..".format(
self.device if self.device else self.address_biq))
self.call("bigstart restart")
self.wait_prompt()
def cleanup(self):
if self.sshifc is not None:
self.sshifc.close()
if self.sshifc_biq is not None:
self.sshifc_biq.close()
