def safe_run(cmd, msg, sleep_time=180):
exit_code = shell.local(cmd)[0]
if exit_code:
LOG.warning('Failed to %s. Retry it after %s seconds' %
(msg, sleep_time))
time.sleep(sleep_time)
shell.local(cmd, raise_error=True)
def deploy_guestbook(template, name, image_url):
obj_config = template.format(suffix=name, image_url=image_url)
file_name = 'guestbook-%s.yaml' % name
with open(file_name, 'w') as fh:
fh.write(obj_config)
LOG.info('Generated guest book configuration file: %s', file_name)
LOG.info('Start to create service frontend-%s', name)
shell.local('kubectl create -f %s' % file_name, raise_error=True)
time.sleep(90)
while True:
out = shell.local('kubectl get svc -o yaml -l app=guestbook-%s' %
name)[1]
svc = yaml.load(out)
lb = svc['items'][0]['status']['loadBalancer']
ext_lb = None
if 'ingress' in lb:
for ingress in lb['ingress']:
if ingress['ip'].startswith('10.111.108'):
ext_lb = ingress['ip']
LOG.info('External lb of frontend-%s: %s', name, ext_lb)
break
if ext_lb:
break
LOG.info('External lb of frontend-%s is not ready', name)
time.sleep(15)
def run_test(component, report_dir, parallel=False, rerun_failed=False):
testr_opts = ''
if parallel:
testr_opts += '--parallel'
if not os.path.isabs(report_dir):
report_dir = os.path.abspath(report_dir)
with shell.cd(TEMPEST_DIR):
LOG.info('Start to run %s tests' % component)
start = time.time()
shell.local("./tools/with_venv.sh testr run %s --subunit --load-list="
"%s.txt | subunit2pyunit" % (testr_opts, component))
end = time.time()
LOG.info('%s tests took %s seconds', component, (end - start))
make_reports(report_dir, component)
failed_tests = shell.local('./tools/with_venv.sh testr failing '
'--subunit | subunit-ls')[1]
if failed_tests.strip():
LOG.info('Failed tests:\n%s', failed_tests)
if rerun_failed:
LOG.info('Rerun above failed tests.')
start = time.time()
shell.local('./tools/with_venv.sh testr run --failing '
'--subunit | subunit2pyunit')
end = time.time()
LOG.info('Rerun %s failed tests took %s seconds', component,
(end - start))
make_reports(report_dir, '%s_rerun' % component)
def safe_run(cmd, msg, sleep_time=180):
exit_code = shell.local(cmd)[0]
if exit_code:
LOG.warning('Failed to %s. Retry it after %s seconds' %
(msg, sleep_time))
time.sleep(sleep_time)
shell.local(cmd, raise_error=True)
def test_cmd_under_dir(self):
shell.local("mkdir -p /tmp/test")
with shell.cd("/tmp"):
with shell.cd("/tmp/test"):
code, output = shell.local("pwd")
self.assertEqual(output, '/tmp/test\n')
code, output = shell.local("pwd")
self.assertEqual(output, '/tmp\n')
def deploy_vapp(vc_host,
vc_user,
vc_password,
dc,
cluster,
ds,
network,
ova_path,
ntp_server=None,
viouser_pwd='vmware',
log_path=None,
ip=None,
netmask=None,
gateway=None,
dns=None,
ovf_tool_path=None):
if not ovf_tool_path:
ovf_tool_path = get_ovf_tool_path()
if not ovf_tool_path:
ovf_tool_path = DEFAULT_LOCAL_OVF_TOOL_PATH
if not os.path.isfile(ovf_tool_path):
LOG.error('ovftool not found.')
raise NotSupportedError('ovftool not found')
if not log_path:
log_path = os.getcwd()
ntp_config = '--prop:ntpServer=%s ' % ntp_server if ntp_server else ''
dns_config = '--prop:vami.DNS.management-server=%s ' % dns if dns else ''
# deploy ova and poweron vm
# TODO: implement deploy with dhcp
cmd = ('"%s" --X:"logFile"="%s/deploy_oms.log" '
'--vService:"installation"='
'"com.vmware.vim.vsm:extension_vservice" '
'--acceptAllEulas --noSSLVerify --powerOn '
'--datastore=%s '
'-dm=thin '
'--net:"VIO Management Server Network"="%s" '
'--prop:vami.ip0.management-server=%s '
'--prop:vami.netmask0.management-server=%s '
'--prop:vami.gateway.management-server=%s '
'%s '
'--prop:viouser_passwd=%s '
'%s %s vi://%s:%s@%s/%s/host/%s'
'' % (ovf_tool_path, log_path, ds, network, ip, netmask, gateway,
dns_config, viouser_pwd, ntp_config, ova_path, vc_user,
vc_password, vc_host, dc, cluster))
LOG.info('Start to deploy management server.')
# LOG.info(cmd)
# exit_code = os.system(cmd)
exit_code = shell.local(cmd)[0]
if exit_code:
LOG.warning('Failed to deploy vApp. Retry deploying after 3 minutes.')
time.sleep(60 * 3)
shell.local(cmd, raise_error=True)
wait_for_mgmt_service(ip, vc_user, vc_password)
LOG.info('Successfully deployed management server.')
def download_file(url, path=None):
file_name = os.path.basename(url)
if path:
abs_path = os.path.join(path, file_name)
else:
abs_path = os.path.join(os.getcwd(), file_name)
shell.local("wget --no-verbose -O %s %s" % (abs_path, url),
capture=False, raise_error=True)
LOG.info('Downloaded %s to %s', url, abs_path)
return abs_path
def make_reports(report_dir, suite_name):
subunit = '/tmp/%s-subunit.txt' % suite_name
junit_xml = os.path.join(report_dir, '%s_results.xml' % suite_name)
shell.local('./tools/with_venv.sh testr last --subunit > %s' % subunit)
shell.local('subunit2junitxml --output-to=%s < %s' % (junit_xml, subunit))
html_report_file = os.path.join(report_dir, '%s_results.html' % suite_name)
try:
generate_html_report(subunit, html_report_file)
LOG.info('Generated report to %s.' % html_report_file)
except Exception:
LOG.exception('Failed to generate report to %s.' % html_report_file)
def run_vmware_test(report_dir):
if not os.path.isabs(report_dir):
report_dir = os.path.abspath(report_dir)
LOG.info('Start to run VMware tempest.')
with shell.cd(VMWARE_TEMPEST_DIR):
shell.local('source .venv/bin/activate && testr run --subunit '
'--load-list run-tests.txt | subunit2pyunit')
failed_tests = shell.local('./tools/with_venv.sh testr failing '
'--subunit | subunit-ls')[1]
make_reports(report_dir, 'vmware')
if failed_tests.strip():
LOG.info('Failed tests:\n%s', failed_tests)
def run_vmware_test(report_dir):
if not os.path.isabs(report_dir):
report_dir = os.path.abspath(report_dir)
LOG.info('Start to run VMware tempest.')
with shell.cd(VMWARE_TEMPEST_DIR):
shell.local('source .venv/bin/activate && testr run --subunit '
'--load-list run-tests.txt | subunit2pyunit')
failed_tests = shell.local('./tools/with_venv.sh testr failing '
'--subunit | subunit-ls')[1]
make_reports(report_dir, 'vmware')
if failed_tests.strip():
LOG.info('Failed tests:\n%s', failed_tests)
def generate_run_list(neutron_backend):
with shell.cd(TEMPEST_DIR):
if not os.path.exists('%s/.testrepository' % TEMPEST_DIR):
shell.local('./tools/with_venv.sh testr init', raise_error=True)
lines = shell.local('./tools/with_venv.sh testr list-tests',
raise_error=True)[1]
# Obtain all tests into a dict {test_name: test_id}
all_tests = dict([split_name_and_id(line) for line in lines.split('\n')
if line.startswith('tempest.') or
line.startswith('vmware_nsx_tempest.')])
# Get excluded tests into a list [test_name]
exclude_file = '%s/%s-excluded-tests.txt' % (get_data_path(),
neutron_backend)
if os.path.exists(exclude_file):
LOG.debug('Found %s, tests in it will be excluded.', exclude_file)
excluded_tests = [strip_id(line) for line in open(exclude_file)
if (line.strip() != '') and
(not line.strip().startswith('#'))]
else:
excluded_tests = []
LOG.debug('Excluded list not found, all tests will be included')
# Get all tests minus excluded tests [test_name + test_id]
exec_tests = [test_name + test_id for (test_name, test_id)
in all_tests.items() if test_name not in excluded_tests]
# Get test case and exclude metrics
num_all_tests = len(all_tests)
num_excluded = len(excluded_tests)
num_tests = len(exec_tests)
LOG.debug('Total number of available tests: %s' % num_all_tests)
LOG.debug('Total number of excluded tests: %s' % num_excluded)
LOG.debug('Total number of tests to run: %s' % num_tests)
outdated_tests = []
if num_tests != num_all_tests - num_excluded:
all_tests_list = all_tests.keys()
outdated_tests = [test_name for test_name in excluded_tests
if test_name not in all_tests_list]
if outdated_tests:
LOG.debug('Below tests in exclude-tests.txt are outdated.')
for test in outdated_tests:
LOG.debug(test)
write_suite_file('included-tests', exec_tests)
test_list = [test_name + test_id for (test_name, test_id)
in all_tests.items()]
write_suite_file('all-tests', test_list)
for key in PACKAGE_MAP:
test_list = [test for test in exec_tests
if test.startswith(PACKAGE_MAP[key])]
write_suite_file(key, test_list)
def install_tempest(repository='github.com/openstack/tempest.git',
branch='11.0.0',
nsx_repo='github.com/openstack/vmware-nsx',
nsx_branch='stable/mitaka',
protocol='http',
conf_template=None):
if os.path.exists(TEMPEST_DIR):
LOG.info('Tempest already exists, skip cloning.')
else:
LOG.info('Clone tempest from repository.')
clone_url = '%s://%s' % (protocol, repository)
shell.local('%s -b %s %s' % (GIT_CLONE, branch, clone_url),
raise_error=True)
# Get vmware_nsx plugin
if os.path.exists(VMWARE_NSX_DIR):
LOG.info('vmware-nsx already exists, skip cloning.')
else:
LOG.info('Clone vmware-nsx from repository.')
clone_url = '%s://%s' % (protocol, nsx_repo)
shell.local('%s -b %s %s' % (GIT_CLONE, nsx_branch, clone_url),
raise_error=True)
with shell.cd(TEMPEST_DIR):
shell.local("sed -i 's/-500/-1500/g' .testr.conf")
LOG.info('Copy template to etc/tempest.conf')
conf_template = conf_template or os.path.join(get_data_path(),
'tempest.conf.template')
shell.local('cp %s etc/tempest.conf' % conf_template, raise_error=True)
LOG.info('Install tempest dependencies.')
cmd = 'python tools/install_venv.py --no-site-packages'
task_utils.safe_run(cmd, 'install tempest dependency')
LOG.info('Install vmware-nsx.')
cmd = './%s/tools/with_venv.sh pip install -e %s' % (TEMPEST_DIR,
VMWARE_NSX_DIR)
task_utils.safe_run(cmd, 'install vmware-nsx')
LOG.info('Tempest has been successfully installed.')
def generate_vmware_run_list():
run_list_file = 'run-tests.txt'
excluded_file = os.path.join(get_data_path(), 'vmware-excluded-tests.txt')
cmd = '''set -ex
cd {path}
source .venv/bin/activate
if [ ! -d '.testrepository' ]; then testr init; fi
testr list-tests | grep '^vmware_tempest' | sort > all-tests.txt
grep '^vmware_tempest' {excluded_file} | sort > excluded-tests.txt
comm -23 all-tests.txt excluded-tests.txt > {run_list_file}
'''.format(path=VMWARE_TEMPEST_DIR, excluded_file=excluded_file,
run_list_file=run_list_file)
LOG.info('Write test list to %s.', run_list_file)
shell.local(cmd, raise_error=True)
def install_vmware_tempest(
repository='http://p3-review.eng.vmware.com/vmware_tempest',
branch='master'):
if os.path.exists(VMWARE_TEMPEST_DIR):
LOG.info('VMware tempest already exists, skip cloning.')
else:
LOG.info('Clone VMware tempest from repository.')
shell.local('git clone -b %s %s' % (branch, repository),
raise_error=True)
# Delete below to use Mitaka tempest after Xiangfei remove dependency to
# p3 tempest
if os.path.exists('p3-tempest'):
LOG.info('P3 tempest already exists, skip cloning.')
else:
LOG.info('Clone P3 tempest from repository.')
shell.local('git clone http://p3-review.eng.vmware.com/tempest '
'p3-tempest', raise_error=True)
# copy tempest.conf
shell.local('cp -f %s/etc/tempest.conf %s/etc/' %
(TEMPEST_DIR, 'p3-tempest'))
# put config in [auth] to [identity]
config_parser = ConfigParser.ConfigParser()
conf_path = '%s/etc/tempest.conf' % 'p3-tempest'
config_parser.read(conf_path)
auth_configs = config_parser.items('auth')
for auth_config in auth_configs:
config_parser.set('identity', auth_config[0], auth_config[1])
config_parser.write(open(conf_path, 'w'))
if not os.path.exists(os.path.join(VMWARE_TEMPEST_DIR, '.venv')):
LOG.info('Create virtual env for VMware tempest.')
shell.local('virtualenv %s/.venv' % VMWARE_TEMPEST_DIR)
tempest_path = os.path.join(os.getcwd(), 'p3-tempest')
install_cmd = '''set -ex
cd {path}
source .venv/bin/activate
pip --no-cache-dir install -e {tempest_path}
pip --no-cache-dir install nose
pip --no-cache-dir install nose-testconfig
pip --no-cache-dir install pyvmomi
cp -f vmware_tempest.cfg.sample vmware_tempest.cfg
'''.format(path=VMWARE_TEMPEST_DIR, tempest_path=tempest_path)
LOG.info('Install VMware tempest dependencies.')
shell.local(install_cmd, raise_error=True)
LOG.info('VMware tempest has been successfully installed.')
def set_up(self):
config = {
"vc_ip": self.oms_spec['vc_host'],
"vc_user": self.oms_spec['vc_user'],
"vc_password": self.oms_spec['vc_password'],
"vapp_name": '',
"oms_ip": self.oms_spec['host_ip'],
"oms_gateway": self.oms_spec['gateway'],
"oms_netmask": self.oms_spec['netmask'],
"oms_dns": self.oms_spec['dns'],
"oms_network": '',
"oms_dc": '',
"oms_cluster": '',
"oms_datastore": '',
"nsxv_manager": self.oms_spec['nsxv_ip'],
"nsxv_username": self.oms_spec['nsxv_user'],
"nsxv_password": self.oms_spec['nsxv_password']
}
branch = self.oms_spec['version'][0:3]
project_path = os.path.join(os.getcwd(), 'vio-api-test')
if not os.path.exists(project_path):
shell.local('git clone -b %s http://p3-review.eng.vmware.com/'
'vio-api-test' % branch)
LOG.info('Install VIO OMS api test project in %s', project_path)
shell.local('sudo pip install -r vio-api-test/requirements.txt')
shell.local('sudo pip install -e vio-api-test/')
config_path = "%s/vio/data/vio.vapp.json" % project_path
LOG.info("Generate VIO OMS API test configuration to %s" % config_path)
LOG.debug("vio.vapp.json: %s" % config)
with open(config_path, 'w+') as fh:
json.dump(config, fh, indent=2, separators=(',', ': '))
self.project_path = project_path
def make_reports(report_dir, suite_name):
subunit = '/tmp/%s-subunit.txt' % suite_name
junit_xml = os.path.join(report_dir, '%s_results.xml' % suite_name)
shell.local('./tools/with_venv.sh testr last --subunit > %s' % subunit)
shell.local('subunit2junitxml --output-to=%s < %s' % (junit_xml, subunit))
html_report_file = os.path.join(report_dir, '%s_results.html' % suite_name)
try:
shell.local('subunit2html %s %s' % (subunit, html_report_file),
raise_error=True)
LOG.info('Generated report to %s.' % html_report_file)
except Exception:
LOG.exception('Failed to generate report to %s.' % html_report_file)
def install_tempest(repository='http://p3-review.eng.vmware.com/tempest',
branch='master',
conf_template=None):
if os.path.exists(TEMPEST_DIR):
LOG.info('Tempest already exists, skip cloning.')
else:
LOG.info('Clone tempest from repository.')
shell.local('git clone -b %s %s' % (branch, repository),
raise_error=True)
with shell.cd(TEMPEST_DIR):
shell.local("sed -i 's/-500/-1500/g' .testr.conf")
LOG.info('Copy template to etc/tempest.conf')
conf_template = conf_template or os.path.join(get_data_path(),
'tempest.conf.template')
shell.local('cp %s etc/tempest.conf' % conf_template, raise_error=True)
LOG.info('Install tempest dependencies.')
cmd = 'python tools/install_venv.py --no-site-packages'
exit_code = shell.local(cmd)[0]
if exit_code:
LOG.warning('Failed to install dependencies. Retry it after 3 '
'minutes.')
time.sleep(60 * 3)
shell.local(cmd, raise_error=True)
LOG.info('Tempest has been successfully installed.')
def install_vmware_tempest(
repository='http://p3-review.eng.vmware.com/vmware_tempest',
branch='master'):
if os.path.exists(VMWARE_TEMPEST_DIR):
LOG.info('VMware tempest already exists, skip cloning.')
else:
LOG.info('Clone VMware tempest from repository.')
shell.local('git clone -b %s %s' % (branch, repository),
raise_error=True)
if not os.path.exists(os.path.join(VMWARE_TEMPEST_DIR, '.venv')):
LOG.info('Copy virtual env packages from tempest to VMware tempest.')
shell.local('cp -rf %s/.venv %s/' % (TEMPEST_DIR, VMWARE_TEMPEST_DIR))
tempest_path = os.path.join(os.getcwd(), TEMPEST_DIR)
install_cmd = '''set -ex
cd {path}
source .venv/bin/activate
pip install -r requirements.txt
pip install -r test-requirements.txt
pip install -e {tempest_path}
cp -f vmware_tempest.cfg.sample vmware_tempest.cfg
'''.format(path=VMWARE_TEMPEST_DIR, tempest_path=tempest_path)
LOG.info('Install VMware tempest dependencies.')
shell.local(install_cmd, raise_error=True)
LOG.info('VMware tempest has been successfully installed.')
def set_up(self):
if not os.path.exists('tempest/included-tests.txt'):
controller = cluster_utils.get_nodegroup_by_role(
self.cluster_spec, 'Controller')
admin_user = controller['attributes']['admin_user']
admin_pwd = controller['attributes']['admin_password']
neutron_backend = controller['attributes']['neutron_backend']
keystone_backend = controller['attributes']['keystone_backend']
admin_tenant = controller['attributes']['admin_tenant_name']
ext_net_cidr = None
ext_net_start_ip = None
ext_net_end_ip = None
ext_net_gateway = None
nsx_manager = None
nsx_user = None
nsx_pwd = None
if LDAP_BACKEND == keystone_backend:
admin_user = self.oms_spec['openstack_admin']
admin_pwd = self.oms_spec['openstack_admin_pwd']
if neutron_backend == NSXV_BACKEND:
nsx_manager = controller['attributes']['nsxv_manager']
nsx_user = controller['attributes']['nsxv_username']
nsx_pwd = controller['attributes']['nsxv_password']
elif neutron_backend == NSXT_BACKEND:
# TODO: get nsxt configure from spec
pass
if neutron_backend in [NSXV_BACKEND, NSXT_BACKEND]:
ext_net_cidr = self.oms_spec['ext_net_cidr']
ext_net_start_ip = self.oms_spec['ext_net_start_ip']
ext_net_end_ip = self.oms_spec['ext_net_end_ip']
ext_net_gateway = self.oms_spec['ext_net_gateway']
creds_provider = self.oms_spec['openstack_creds_provider'].strip()
if creds_provider in [LEGACY_PROVIDER, PRE_PROVISIONED_PROVIDER]:
user1 = self.oms_spec['openstack_user1']
user1_pwd = self.oms_spec['openstack_user1_pwd']
user2 = self.oms_spec['openstack_user2']
user2_pwd = self.oms_spec['openstack_user2_pwd']
else:
user1 = None
user1_pwd = None
user2 = None
user2_pwd = None
if 'compute_clusters' in self.oms_spec:
min_compute_nodes = len(self.oms_spec['compute_clusters'])
else:
min_compute_nodes = 1
tempest_utils.install_tempest()
tempest_log_file = '%s/tempest.log' % self.log_dir
oms_ctl = OmsController(self.oms_spec['host_ip'],
self.oms_spec['vc_user'],
self.oms_spec['vc_password'])
private_vip = cluster_utils.get_private_vip(
oms_ctl, self.cluster_spec['name'])
if not private_vip:
raise NotSupportedError('Can not get private VIP.')
tempest_utils.config_tempest(private_vip=private_vip,
admin_user=admin_user,
admin_pwd=admin_pwd,
neutron_backend=neutron_backend,
creds_provider=creds_provider,
default_user=user1,
default_pwd=user1_pwd,
alter_user=user2,
alter_pwd=user2_pwd,
ext_net_cidr=ext_net_cidr,
ext_net_start_ip=ext_net_start_ip,
ext_net_end_ip=ext_net_end_ip,
ext_net_gateway=ext_net_gateway,
tempest_log_file=tempest_log_file,
admin_tenant=admin_tenant,
min_compute_nodes=min_compute_nodes,
nsx_manager=nsx_manager,
nsx_user=nsx_user,
nsx_pwd=nsx_pwd)
shell.local('cp %s/etc/tempest.conf %s/' %
(tempest_utils.TEMPEST_DIR, self.log_dir))
tempest_utils.generate_run_list(neutron_backend)
else:
LOG.info('Tempest already exists. Skip setting up it.')