def get_mgmt_system(self):
""" Returns the mgmt_system using the :py:func:`utils.providers.provider_factory` method.
"""
if self.provider_data is not None:
return provider_factory(self.provider_data)
elif self.key is not None:
return provider_factory(self.key)
else:
raise ProviderHasNoKey('Provider %s has no key, so cannot get mgmt system')
def get_mgmt_system(self):
""" Returns the mgmt_system using the :py:func:`utils.providers.provider_factory` method.
"""
if not self.key:
raise ProviderHasNoKey('Provider %s has no key, so cannot get mgmt system')
else:
return provider_factory(self.key)
def wait_for_provider_or_timeout(self, provider):
'''Wait for a provider to become available or timeout trying'''
ec, tc = wait_for(self.is_quad_icon_available,
[provider['name']])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider['request'])
host_stats = client.stats('num_datastore',
'num_host',
'num_cluster')
client.disconnect()
wait = partial(wait_for,
detail_pg.do_stats_match,
[host_stats],
message="do_stats_match",
num_sec=180)
try:
ec, tc = wait()
except TimedOutError:
# Help him
assert detail_pg.click_on_refresh_relationships(), "Could not refresh relationships!"
ec, tc = wait()
self.header.site_navigation_menu('Infrastructure'). \
sub_navigation_menu('Providers').click()
return
def get_mgmt_system(self):
""" Returns the mgmt_system using the :py:func:`utils.providers.provider_factory` method.
"""
if not self.key:
raise ProviderHasNoKey("Provider %s has no key, so cannot get mgmt system")
else:
return provider_factory(self.key)
def process_provider_vms(provider_key, matchers, delta, vms_to_delete):
with lock:
print '%s processing' % provider_key
try:
now = datetime.datetime.now()
provider = provider_factory(provider_key)
for vm_name in provider.list_vm():
if not match(matchers, vm_name):
continue
try:
vm_creation_time = provider.vm_creation_time(vm_name)
except:
logger.error('Failed to get creation/boot time for %s on %s' % (
vm_name, provider_key))
continue
if vm_creation_time + delta < now:
vm_delta = now - vm_creation_time
with lock:
vms_to_delete[provider_key].add((vm_name, vm_delta))
with lock:
print '%s finished' % provider_key
except Exception as ex:
with lock:
print '%s failed' % provider_key
logger.error('failed to process vms from provider %s', provider_key)
logger.exception(ex)
def mgmt_sys_api_clients(mozwebqa, cfme_data):
"""Returns a list of management system api clients"""
clients = {}
for sys_name in cfme_data.data["management_systems"]:
if sys_name in clients:
# Overlapping sys_name entry in cfme_data.yaml
logger.warning("Overriding existing entry for %s." % sys_name)
clients[sys_name] = provider_factory(sys_name)
return clients
def __init__(self, provider_name, vm_name):
"""Initializes a deployed appliance VM
"""
self.name = Appliance._default_name
self.db_address = None
self._provider = provider_factory(provider_name)
self._vm_name = vm_name
self._address = None
def provision_appliance(version=None, vm_name_prefix='cfme', template=None, provider_name=None):
"""Provisions fresh, unconfigured appliance of a specific version
Note:
Version must be mapped to template name under ``appliance_provisioning > versions``
in ``cfme_data.yaml``.
Args:
version: version of appliance to provision
vm_name_prefix: name prefix to use when deploying the appliance vm
Returns: Unconfigured appliance; instance of :py:class:`Appliance`
Usage:
my_appliance = provision_appliance('5.2.1.8', 'my_tests')
my_appliance.fix_ntp_clock()
my_appliance.enable_internal_db()
my_appliance.wait_for_web_ui()
or
my_appliance = provision_appliance('5.2.1.8', 'my_tests')
my_appliance.configure(patch_ajax_wait=False)
(identical outcome)
"""
def _generate_vm_name():
if version is not None:
version_digits = ''.join([letter for letter in version if letter.isdigit()])
return '{}_{}_{}'.format(vm_name_prefix, version_digits, generate_random_string())
else:
return '{}_{}'.format(vm_name_prefix, generate_random_string())
if version is not None:
templates_by_version = conf.cfme_data['appliance_provisioning']['versions']
try:
template_name = templates_by_version[version]
except KeyError:
raise ApplianceException('No template found matching version {}'.format(version))
if template is not None:
template_name = template
if provider_name is None:
provider_name = conf.cfme_data['appliance_provisioning']['provider']
prov_data = conf.cfme_data['management_systems'][provider_name]
provider = provider_factory(provider_name)
vm_name = _generate_vm_name()
deploy_args = {}
deploy_args['vm_name'] = vm_name
if prov_data['type'] == 'rhevm':
deploy_args['cluster_name'] = prov_data['default_cluster']
provider.deploy_template(template_name, **deploy_args)
return Appliance(provider_name, vm_name)
def mgmt_sys_api_clients(mozwebqa, cfme_data):
'''Returns a list of management system api clients'''
clients = {}
for sys_name in cfme_data.data['management_systems']:
if sys_name in clients:
# Overlapping sys_name entry in cfme_data.yaml
logger.warning('Overriding existing entry for %s.' % sys_name)
clients[sys_name] = provider_factory(sys_name)
return clients
def power_state_buttons(request, current_provider):
manager = provider_factory(current_provider)
vm_name = request.POST["vm_name"]
power_state = request.POST["power_state"]
can_power_on = power_state in {Appliance.Power.SUSPENDED, Appliance.Power.OFF}
can_power_off = power_state in {Appliance.Power.ON}
can_suspend = power_state in {Appliance.Power.ON} and manager.can_suspend
can_delete = power_state in {Appliance.Power.OFF}
return render(request, 'appliances/vms/_buttons.html', locals())
def call_provider(provider_name, action, *args):
# Given a provider class, find the named method and call it with
# *args. This could possibly be generalized for other CLI tools.
provider = provider_factory(provider_name)
try:
call = getattr(provider, action)
except AttributeError:
raise Exception('Action "%s" not found' % action)
return call(*args)
def test_validate_provider_details(self, infra_providers_pg, provider_data):
prov_pg = infra_providers_pg
prov_pg.select_provider(provider_data['name'])
client = provider_factory(provider_data['request'])
host_stats = client.stats()
detail_pg = prov_pg.quadicon_region.selected[0].click()
Assert.true(detail_pg.do_stats_match(host_stats),
'Host stats should match with mgmt_system stats')
def wait_for_provider_or_timeout(self, provider):
"""Wait for a provider to become available or timeout trying"""
ec, tc = wait_for(self.is_quad_icon_available, [provider["name"]])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider["request"])
host_stats = client.stats("num_template")
client.disconnect()
ec, tc = wait_for(detail_pg.do_stats_match, [host_stats], message="do_stats_match", num_sec=300)
return
def test_host_provisioning(infra_hosts_pg, host_provisioning_data, cfme_data,
server_roles):
prov_request_data = host_provisioning_data['provision_request']
provider_data = cfme_data['management_systems'][
prov_request_data['provider']]
infra_hosts_pg = infra_hosts_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
infra_hosts_pg.select_host(host_provisioning_data['host']['name'])
prov_pg = infra_hosts_pg.click_provision_host()
tab_buttons = prov_pg.tabbutton_region
fill_in_request(prov_request_data, provider_data, host_provisioning_data,
tab_buttons)
requests_pg = prov_pg.click_on_submit()
Assert.equal(
requests_pg.flash.message,
"Host Request was Submitted, you will be notified when your Hosts are ready",
"Flash message should inform of pending notification")
requests_pg.wait_for_request_status('Last 7 Days', 'Ok', 30)
infra_hosts_pg = requests_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
host_pg = infra_hosts_pg.click_host(host_provisioning_data['host']['name'])
Assert.equal(host_pg.provider, provider_data['name'],
"Provider name does not match")
Assert.equal(host_pg.cluster, prov_request_data['environment']['cluster'],
"Cluster does not match")
ds_pg = host_pg.click_on_datastores()
datastores = [ds.title for ds in ds_pg.quadicon_region.quadicons]
Assert.true(
set(prov_request_data['environment']['datastores']).issubset(
set(datastores)), "Datastores are missing some members")
mgmt_system = provider_factory(prov_request_data['provider'])
mgmt_system.remove_host_from_cluster(
host_provisioning_data['host']['ipaddress'])
creds = ds_pg.testsetup.credentials[host_provisioning_data['host']
['ipmi_credentials']]
ipmi = IPMI(host_provisioning_data['host']['ipmi_address'],
creds['username'], creds['password'], 'lanplus')
ipmi.power_off()
infra_hosts_pg = ds_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
infra_hosts_pg.select_host(host_provisioning_data['host']['name'])
infra_hosts_pg.click_remove_host()
wait_for(lambda func, host: not func(host), [
infra_hosts_pg.check_host_and_refresh,
host_provisioning_data['host']['name']
],
message="wait for host delete")
def get_provider_templates(provider_key, templates_providers, thread_lock):
# functionalized to make it easy to farm this out to threads
provider_mgmt = provider_factory(provider_key)
try:
templates = provider_mgmt.list_template()
print provider_key, 'returned %d templates' % len(templates)
with thread_lock:
for template in templates:
templates_providers[template].append(provider_key)
except:
print provider_key, 'failed'
def test_validate_provider_details(self, cloud_providers_pg,
provider_data):
prov_pg = cloud_providers_pg
prov_pg.select_provider(provider_data['name'])
client = provider_factory(provider_data['request'])
host_stats = client.stats()
detail_pg = prov_pg.quadicon_region.selected[0].click()
Assert.true(detail_pg.do_stats_match(host_stats),
'Host stats should match with mgmt_system stats')
def wait_for_provider_or_timeout(self, provider):
"""Wait for a provider to become available or timeout trying"""
ec, tc = wait_for(self.is_quad_icon_available, [provider["name"]])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider["request"])
host_stats = client.stats("num_datastore", "num_host", "num_cluster")
client.disconnect()
ec, tc = wait_for(detail_pg.do_stats_match, [host_stats], message="do_stats_match", num_sec=300)
self.header.site_navigation_menu("Infrastructure").sub_navigation_menu("Providers").click()
return
def wait_for_provider_or_timeout(self, provider):
'''Wait for a provider to become available or timeout trying'''
ec, tc = wait_for(self.is_quad_icon_available, [provider['name']])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider['request'])
host_stats = client.stats('num_template')
client.disconnect()
ec, tc = wait_for(detail_pg.do_stats_match, [host_stats],
message="do_stats_match",
num_sec=300)
return
def vm(request, setup_a_provider, rest_api):
if "refresh" not in rest_api.collections.providers.action.all:
pytest.skip("Refresh action is not implemented in this version")
provider_mgmt = provider_factory(setup_a_provider.key)
provider = rest_api.collections.providers.find_by(name=setup_a_provider.name)[0]
vm_name = deploy_template(
setup_a_provider.key,
"test_rest_vm_{}".format(fauxfactory.gen_alphanumeric(length=4)))
request.addfinalizer(lambda: provider_mgmt.delete_vm(vm_name))
provider.action.refresh()
wait_for(
lambda: len(rest_api.collections.vms.find_by(name=vm_name)) > 0,
num_sec=600, delay=5)
return vm_name
def vm_action(request, current_provider):
if not request.user.is_authenticated():
return HttpResponse("Not authenticated", content_type="text/plain")
try:
provider_factory(current_provider)
except Exception as e:
return HttpResponse(
"Troubles with provider {}: {}".format(current_provider, str(e)),
content_type="text/plain")
vm_name = request.POST["vm_name"]
action = request.POST["action"]
if action == "poweron":
anyvm_power_on.delay(current_provider, vm_name)
elif action == "poweroff":
anyvm_power_off.delay(current_provider, vm_name)
elif action == "suspend":
anyvm_suspend.delay(current_provider, vm_name)
elif action == "delete":
anyvm_delete.delay(current_provider, vm_name)
else:
HttpResponse("No such action {}!".format(action), content_type="text/plain")
logger().info("User {} initiated {} on {}@{}".format(
request.user.username, action, vm_name, current_provider))
return HttpResponse("Action {} was initiated".format(action), content_type="text/plain")
def main(*providers):
for provider_key in list_providers('openstack'):
print 'Checking {}'.format(provider_key)
api = provider_factory(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print 'Unable to get fips for {}:'.format(provider_key)
print format_exc().splitlines()[-1]
continue
for fip in fips:
print 'Deleting {} on {}'.format(fip.ip, provider_key)
fip.delete()
print fip.ip, 'deleted'
def wait_for_provider_or_timeout(self, provider):
'''Wait for a provider to become available or timeout trying'''
ec, tc = wait_for(self.is_quad_icon_available,
[provider['name']])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider['request'])
host_stats = client.stats('num_datastore',
'num_host',
'num_cluster')
client.disconnect()
ec, tc = wait_for(detail_pg.do_stats_match,
[host_stats],
message="do_stats_match",
num_sec=300)
return
def test_host_provisioning(infra_hosts_pg, host_provisioning_data, cfme_data, server_roles):
prov_request_data = host_provisioning_data['provision_request']
provider_data = cfme_data['management_systems'][prov_request_data['provider']]
infra_hosts_pg = infra_hosts_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
infra_hosts_pg.select_host(host_provisioning_data['host']['name'])
prov_pg = infra_hosts_pg.click_provision_host()
tab_buttons = prov_pg.tabbutton_region
fill_in_request(prov_request_data, provider_data, host_provisioning_data, tab_buttons)
requests_pg = prov_pg.click_on_submit()
Assert.equal(requests_pg.flash.message,
"Host Request was Submitted, you will be notified when your Hosts are ready",
"Flash message should inform of pending notification")
requests_pg.wait_for_request_status('Last 7 Days', 'Ok', 30)
infra_hosts_pg = requests_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
host_pg = infra_hosts_pg.click_host(host_provisioning_data['host']['name'])
Assert.equal(host_pg.provider, provider_data['name'],
"Provider name does not match")
Assert.equal(host_pg.cluster, prov_request_data['environment']['cluster'],
"Cluster does not match")
ds_pg = host_pg.click_on_datastores()
datastores = [ds.title for ds in ds_pg.quadicon_region.quadicons]
Assert.true(set(prov_request_data['environment']['datastores']).issubset(set(datastores)),
"Datastores are missing some members")
mgmt_system = provider_factory(prov_request_data['provider'])
mgmt_system.remove_host_from_cluster(host_provisioning_data['host']['ipaddress'])
creds = ds_pg.testsetup.credentials[host_provisioning_data['host']['ipmi_credentials']]
ipmi = IPMI(host_provisioning_data['host']['ipmi_address'],
creds['username'], creds['password'], 'lanplus')
ipmi.power_off()
infra_hosts_pg = ds_pg.header.site_navigation_menu('Infrastructure')\
.sub_navigation_menu('Hosts').click()
infra_hosts_pg.select_host(host_provisioning_data['host']['name'])
infra_hosts_pg.click_remove_host()
wait_for(lambda func, host: not func(host),
[infra_hosts_pg.check_host_and_refresh, host_provisioning_data['host']['name']],
message="wait for host delete")
def main(*providers):
for provider_key in providers:
print "Cleaning up", provider_key
api = provider_factory(provider_key).capi
try:
volumes = api.volumes.findall(attachments=[])
except Exception as e:
print "Connect to provider failed:", provider_key, type(e).__name__, str(e)
continue
for volume in volumes:
if iso8601.parse_date(volume.created_at) < (datetime.now(tz=local_tz) - GRACE_TIME):
print "Deleting", volume.id
try:
volume.delete()
except Exception as e:
print "Delete failed:", type(e).__name__, str(e)
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
providers = providers or list_all_providers()
delta = datetime.timedelta(hours=int(max_hours))
vms_to_delete = defaultdict(set)
thread_queue = []
# precompile regexes
matchers = [re.compile(text) for text in texts]
for provider_key in providers:
thread = Thread(target=process_provider_vms,
args=(provider_key, matchers, delta, vms_to_delete))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
# Join the queued calls
for thread in thread_queue:
thread.join()
for provider_key, vm_set in vms_to_delete.items():
print '%s:' % provider_key
for vm_name, vm_delta in vm_set:
days, hours = vm_delta.days, vm_delta.seconds / 3600
print ' %s is %d days, %s hours old' % (vm_name, days, hours)
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
print 'Exiting.'
return 0
if not vms_to_delete:
print 'No VMs to delete.'
for provider_key, vm_set in vms_to_delete.items():
provider = provider_factory(provider_key)
for vm_name, __ in vm_set:
print 'Deleting %s on %s' % (vm_name, provider_key)
try:
provider.delete_vm(vm_name)
except Exception as ex:
print 'Failed to delete %s on %s' % (vm_name, provider_key)
logger.exception(ex)
def manage_all_queues(target_providers):
'''Scan the templates of given providers for templates named
"autoqueue-nn-*". Create nn vms that are ready to be consumed.
If such a template disappears (or is renamed) its queue will be no
longer be managed. The vms in it will remain but no new ones will
be created.
No distinction is made between vms that are stopped or running.
If someone stops a vm in the queue, it will remain in the queue.
To take a vm from the queue, rename it so that it no longer has a
prefix "appspring-".
The vm becomes your responsibility to manage - destroy it when
you're finished with it.
'''
init()
provs = {(p, providers.provider_factory(p)) for p in target_providers}
global queues
queues = set()
template_re = re.compile("autoqueue-(\d+)")
while True:
for provider_name, provider in provs:
try:
templates = filter(lambda t: template_re.match(t), provider.list_template())
for template_name in templates:
if not _is_in_queue_set(queues, provider_name, template_name):
max_size = int(template_re.match(template_name).group(1))
q = Queue(provider_name, template_name,
max_size=min(max_size, queue_size_guard))
queues.add(q)
q.start()
for q in queues:
if q.template_name not in templates and q.provider_name == provider_name:
q.stop()
queues = set(filter(lambda q: not q.shutdown_flag, queues))
except BaseException as be:
print("Could not process queue for provider {}: {}".format(provider_name, be))
time.sleep(45)
def wait_for_provider_or_timeout(self, provider):
'''Wait for a provider to become available or timeout trying'''
ec, tc = wait_for(self.is_quad_icon_available, [provider['name']])
detail_pg = self.quadicon_region.selected[0].click()
client = provider_factory(provider['request'])
host_stats = client.stats('num_datastore', 'num_host', 'num_cluster')
client.disconnect()
wait = partial(wait_for,
detail_pg.do_stats_match, [host_stats],
message="do_stats_match",
num_sec=180)
try:
ec, tc = wait()
except TimedOutError:
# Help him
assert detail_pg.click_on_refresh_relationships(
), "Could not refresh relationships!"
ec, tc = wait()
self.header.site_navigation_menu('Infrastructure'). \
sub_navigation_menu('Providers').click()
return
def get_provider_templates(provider_key, template_providers, unresponsive_providers, thread_lock):
# functionalized to make it easy to farm this out to threads
provider_mgmt = provider_factory(provider_key)
try:
if cfme_data['management_systems'][provider_key]['type'] == 'ec2':
# dirty hack to filter out ec2 public images, because there are literally hundreds.
templates = provider_mgmt.api.get_all_images(owners=['self'],
filters={'image-type': 'machine'})
templates = map(lambda i: i.name or i.id, templates)
else:
templates = provider_mgmt.list_template()
print provider_key, 'returned %d templates' % len(templates)
with thread_lock:
for template in templates:
# If it ends with 'db', skip it, it's a largedb/nodb variant
if str(template).lower().endswith('db'):
continue
template_providers[template].append(provider_key)
except:
print provider_key, 'failed:', traceback.format_exc().splitlines()[-1]
with thread_lock:
unresponsive_providers.add(provider_key)
def test_provider_refresh(request, setup_a_provider, rest_api):
"""Test checking that refresh invoked from the REST API works.
It provisions a VM when the Provider inventory functionality is disabled, then the functionality
is enabled and we wait for refresh to finish by checking the field in provider and then we check
whether the VM appeared in the provider.
"""
if "refresh" not in rest_api.collections.providers.action.all:
pytest.skip("Refresh action is not implemented in this version")
provider_mgmt = provider_factory(setup_a_provider.key)
provider = rest_api.collections.providers.find_by(name=setup_a_provider.name)[0]
with server_roles_disabled("ems_inventory", "ems_operations"):
vm_name = deploy_template(
setup_a_provider.key,
"test_rest_prov_refresh_{}".format(fauxfactory.gen_alphanumeric(length=4)))
request.addfinalizer(lambda: provider_mgmt.delete_vm(vm_name))
provider.reload()
old_refresh_dt = provider.last_refresh_date
assert provider.action.refresh()["success"], "Refresh was unsuccessful"
wait_for(
lambda: provider.last_refresh_date,
fail_func=provider.reload,
fail_condition=lambda refresh_date: refresh_date == old_refresh_dt,
num_sec=720,
delay=5,
)
# We suppose that thanks to the random string, there will be only one such VM
wait_for(
lambda: len(rest_api.collections.vms.find_by(name=vm_name)),
fail_condition=lambda l: l == 0,
num_sec=180,
delay=10,
)
vms = rest_api.collections.vms.find_by(name=vm_name)
if "delete" in vms[0].action.all:
vms[0].action.delete()
def main():
parser = argparse.ArgumentParser(
epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--provider',
dest='provider_name',
help='provider name in cfme_data')
parser.add_argument('--vm_name', help='the name of the VM on which to act')
parser.add_argument('--remove',
help='remove disk from vm',
action="store_true")
args = parser.parse_args()
provider = provider_factory(args.provider_name)
# check that we are working with a rhev provider
if not isinstance(provider, RHEVMSystem):
sys.exit(args.providername + " is not a RHEVM system, exiting...")
# check that the vm exists on the rhev provider, get the ip address if so
try:
vm = provider.api.vms.get(args.vm_name)
ip_addr = provider.get_ip_address(args.vm_name)
except:
sys.exit(args.vm_name + " vm not found on provider " +
args.providername + ", exiting...")
# check for direct lun definition on provider's cfme_data.yaml
if 'direct_lun_name' not in cfme_data['management_systems'][
args.provider_name]:
sys.exit("direct_lun_name key not in cfme_data.yaml under provider " +
args.providername + ", exiting...")
# does the direct lun exist
dlun_name = cfme_data['management_systems'][
args.provider_name]['direct_lun_name']
dlun = provider.api.disks.get(dlun_name)
if dlun is None:
sys.exit("Direct lun disk named " + dlun_name +
" is not found on provider " + args.provider_name)
# add it
if not args.remove:
# is the disk present and active?
vm_disk_list = vm.get_disks().list()
for vm_disk in vm_disk_list:
if vm_disk.name == dlun_name:
if vm_disk.active:
return
else:
vm_disk.actvate()
return
# if not present, add it and activate
direct_lun = params.Disk(id=dlun.id)
added_lun = vm.disks.add(direct_lun)
added_lun.activate()
# Init SSH client, run pvscan on the appliance
ssh_kwargs = {
'username': credentials['ssh']['username'],
'password': credentials['ssh']['password'],
'hostname': ip_addr
}
client = SSHClient(**ssh_kwargs)
status, out = client.run_command('pvscan')
# remove it
else:
vm_dlun = vm.disks.get(name=dlun_name)
if vm_dlun is None:
return
else:
detach = params.Action(detach=True)
vm_dlun.delete(action=detach)
def _provider(self):
"""
Note:
Cannot be cached because provider object is unpickable.
"""
return provider_factory(self._provider_name)
def provision_appliance(version=None,
vm_name_prefix='cfme',
template=None,
provider_name=None):
"""Provisions fresh, unconfigured appliance of a specific version
Note:
Version must be mapped to template name under ``appliance_provisioning > versions``
in ``cfme_data.yaml``.
Args:
version: version of appliance to provision
vm_name_prefix: name prefix to use when deploying the appliance vm
Returns: Unconfigured appliance; instance of :py:class:`Appliance`
Usage:
my_appliance = provision_appliance('5.2.1.8', 'my_tests')
my_appliance.fix_ntp_clock()
my_appliance.enable_internal_db()
my_appliance.wait_for_web_ui()
or
my_appliance = provision_appliance('5.2.1.8', 'my_tests')
my_appliance.configure(patch_ajax_wait=False)
(identical outcome)
"""
def _generate_vm_name():
if version is not None:
version_digits = ''.join(
[letter for letter in version if letter.isdigit()])
return '{}_{}_{}'.format(vm_name_prefix, version_digits,
generate_random_string())
else:
return '{}_{}'.format(vm_name_prefix, generate_random_string())
if version is not None:
templates_by_version = conf.cfme_data['appliance_provisioning'][
'versions']
try:
template_name = templates_by_version[version]
except KeyError:
raise ApplianceException(
'No template found matching version {}'.format(version))
if template is not None:
template_name = template
if provider_name is None:
provider_name = conf.cfme_data['appliance_provisioning']['provider']
prov_data = conf.cfme_data['management_systems'][provider_name]
provider = provider_factory(provider_name)
vm_name = _generate_vm_name()
deploy_args = {}
deploy_args['vm_name'] = vm_name
if prov_data['type'] == 'rhevm':
deploy_args['cluster_name'] = prov_data['default_cluster']
provider.deploy_template(template_name, **deploy_args)
return Appliance(provider_name, vm_name)
def main():
parser = argparse.ArgumentParser(
epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--provider',
dest='provider_name',
help='provider name in cfme_data')
parser.add_argument('--template', help='the name of the template to clone')
parser.add_argument('--vm_name', help='the name of the VM on which to act')
parser.add_argument('--rhev_cluster',
help='the name of the VM on which to act',
default=None)
parser.add_argument('--rhev_place_policy_host',
help='the host for the vm to start on',
default=None)
parser.add_argument('--rhev_place_policy_aff',
help='the affinity of the vm on a host',
default=None)
parser.add_argument('--ec2_flavor', help='ec2 flavor', default=None)
parser.add_argument('--rhos_flavor', help='rhos flavor', default=None)
parser.add_argument('--rhos_floating_ip_pool',
dest='ip_pool',
default=None,
help='openstack floating ip pool to use')
parser.add_argument('--destroy',
dest='destroy',
help='Destroy the destination VM',
action='store_true')
parser.add_argument('--log',
dest='loglevel',
help='Set the log level',
default='WARNING')
parser.add_argument('--outfile',
dest='outfile',
help='Write provisioning details to the named file',
default='')
args = parser.parse_args()
if not args.provider_name:
parser.error('--provider is required')
logger.info('Connecting to %s', args.provider_name)
provider = provider_factory(args.provider_name)
if args.destroy:
destroy(provider, args)
else:
logger.info('Cloning %s to %s', args.template, args.vm_name)
# passing unused args to ec2 provider would blow up so I
# had to make it a little more specific
deploy_args = {}
if args.vm_name is not None:
deploy_args.update(vm_name=args.vm_name)
if args.rhos_flavor is not None:
deploy_args.update(flavour_name=args.rhos_flavor)
if args.ip_pool is not None:
deploy_args.update(assign_floating_ip=args.ip_pool)
if args.rhev_cluster is not None:
deploy_args.update(cluster_name=args.rhev_cluster)
if args.rhev_place_policy_host is not None:
deploy_args.update(
placement_policy_host=args.rhev_place_policy_host)
if args.rhev_place_policy_aff is not None:
deploy_args.update(
placement_policy_affinity=args.rhev_place_policy_aff)
if args.ec2_flavor is not None:
deploy_args.update(instance_type=args.ec2_flavor)
try:
vm_name = provider.deploy_template(args.template, **deploy_args)
except:
logger.exception(sys.exc_info()[0])
destroy(provider, args)
return 12
if not provider.is_vm_running(vm_name):
logger.error("VM is not running")
return 10
ip, time_taken = wait_for(provider.get_ip_address, [vm_name],
num_sec=600,
fail_condition=None)
logger.info("VM " + vm_name + " is running")
logger.info('IP Address returned is %s', ip)
if args.outfile:
with open(args.outfile, 'w') as outfile:
outfile.write("appliance_ip_address=%s\n" % ip)
# In addition to the outfile, drop the ip address on stdout for easy parsing
print ip
return 0
def provider_by_type(metafunc, provider_types, *fields):
"""Get the values of the named field keys from ``cfme_data['management_systems']``
Args:
provider_types: A list of provider types to include. If None, all providers are considered
*fields: Names of keys in an individual provider dict whose values will be returned when
used as test function arguments
The following test function arguments are special:
``provider_data``
the entire provider data dict from cfme_data.
``provider_key``
the provider's key in ``cfme_data['management_systems']``
``provider_crud``
the provider's CRUD object, either a :py:class:`cfme.cloud.provider.Provider`
or a :py:class:`cfme.infrastructure.provider.Provider`
``provider_mgmt``
the provider's backend manager, from :py:class:`utils.mgmt_system`
Returns:
An tuple of ``(argnames, argvalues, idlist)`` for use in a pytest_generate_tests hook, or
with the :py:func:`parametrize` helper.
Usage:
# In the function itself
def pytest_generate_tests(metafunc):
argnames, argvalues, idlist = testgen.provider_by_type(
['openstack', 'ec2'],
'type', 'name', 'credentials', 'provider_data', 'hosts'
)
metafunc.parametrize(argnames, argvalues, ids=idlist, scope='module')
# Using the parametrize wrapper
pytest_generate_tests = testgen.parametrize(testgen.provider_by_type, ['openstack', 'ec2'],
'type', 'name', 'credentials', 'provider_data', 'hosts', scope='module')
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
"""
argnames = list(fields)
argvalues = []
idlist = []
special_args = ('provider_key', 'provider_data', 'provider_crud',
'provider_mgmt', 'provider_type')
# Hook on special attrs if requested
for argname in special_args:
if argname in metafunc.fixturenames and argname not in argnames:
argnames.append(argname)
for provider, data in cfme_data['management_systems'].iteritems():
prov_type = data['type']
if provider_types is not None and prov_type not in provider_types:
# Skip unwanted types
continue
# Check provider hasn't been filtered out with --use-provider
if provider not in filtered:
continue
# Use the provider name for idlist, helps with readable parametrized test output
idlist.append(provider)
# Get values for the requested fields, filling in with None for undefined fields
data_values = {field: data.get(field, None) for field in fields}
# Go through the values and handle the special 'data' name
# report the undefined fields to the log
for key, value in data_values.iteritems():
if value is None:
logger.warning('Field "%s" not defined for provider "%s", defaulting to None' %
(key, provider)
)
if prov_type in cloud_provider_type_map:
crud = get_cloud_provider(provider)
elif prov_type in infra_provider_type_map:
crud = get_infra_provider(provider)
# else: wat? You deserve the NameError you're about to receive
mgmt = provider_factory(provider)
values = []
special_args_map = dict(zip(special_args, (provider, data, crud, mgmt, prov_type)))
for arg in argnames:
if arg in special_args_map:
values.append(special_args_map[arg])
elif arg in data_values:
values.append(data_values[arg])
argvalues.append(values)
return argnames, argvalues, idlist
def main():
parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("provider_name", help="provider name in cfme_data")
parser.add_argument("vm_name", help="the name of the VM on which to act")
parser.add_argument("uptime", type=int, help="how long do you want the vm to be on (in seconds)")
parser.add_argument("downtime", type=int, help="how long do you want the vm to be off (in seconds)")
args = parser.parse_args()
# Make sure the VM is off to start
provider = provider_factory(args.provider_name)
if provider.is_vm_running(args.vm_name):
provider.stop_vm(args.vm_name)
provider.vm_status(args.vm_name)
# Toggle the VM On and Off based on the Uptime and Downtime input arguments
# The script diconnects from the provider before each sleep and reconnects after
# each sleep to prevent it from timing out
while True:
try:
# Initialize start_success to False so it enters the first while loop
# and the times_failed_counter to 0
start_success = False
times_failed_counter = 0
# Turn the VM on for the specified amount of time
# If it can't find the VM, keep trying for 30 minutes
while not start_success:
try:
provider.start_vm(args.vm_name)
provider.vm_status(args.vm_name)
start_success = True
provider.disconnect()
time.sleep(args.uptime)
provider = provider_factory(args.provider_name)
except Exception:
time.sleep(60)
times_failed_counter += 1
if times_failed_counter == 30:
raise
# Initialize stop_success to False so it enters the first while loop
# and the times_failed_counter to 0
stop_success = False
times_failed_counter = 0
# Turn the VM off for the specified amount of time
# If it can't find the VM, keep trying for 30 minutes
while not stop_success:
try:
provider.stop_vm(args.vm_name)
provider.vm_status(args.vm_name)
stop_success = True
provider.disconnect()
time.sleep(args.downtime)
provider = provider_factory(args.provider_name)
except Exception:
time.sleep(60)
times_failed_counter += 1
if times_failed_counter == 30:
raise
except (KeyboardInterrupt):
return 0
def main():
parser = argparse.ArgumentParser(epilog=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--provider', dest='provider_name',
help='provider name in cfme_data')
parser.add_argument('--template', help='the name of the template to clone')
parser.add_argument('--vm_name', help='the name of the VM on which to act')
parser.add_argument('--rhev_cluster', help='the name of the VM on which to act', default=None)
parser.add_argument('--ec2_flavor', help='ec2 flavor', default=None)
parser.add_argument('--rhos_flavor', help='rhos flavor', default=None)
parser.add_argument('--rhos_floating_ip_pool', dest='ip_pool', default=None,
help='openstack floating ip pool to use')
parser.add_argument('--destroy', dest='destroy',
help='Destroy the destination VM', action='store_true')
parser.add_argument('--log', dest='loglevel',
help='Set the log level', default='WARNING')
parser.add_argument('--outfile', dest='outfile',
help='Write provisioning details to the named file', default='')
args = parser.parse_args()
# Set the logging level from the CLI
try:
loglevel = getattr(logging, args.loglevel)
logging.basicConfig(level=loglevel)
except NameError:
raise ValueError('Invalid log level: %s' % args.loglevel)
logging.info('Connecting to %s', args.provider_name)
provider = provider_factory(args.provider_name)
if args.destroy:
destroy(provider, args)
else:
logging.info('Cloning %s to %s', args.template, args.vm_name)
# passing unused args to ec2 provider would blow up so I
# had to make it a little more specific
deply_args = {}
if args.vm_name is not None:
deply_args.update(vm_name=args.vm_name)
if args.rhos_flavor is not None:
deply_args.update(flavour_name=args.rhos_flavor)
if args.ip_pool is not None:
deply_args.update(assign_floating_ip=args.ip_pool)
if args.rhev_cluster is not None:
deply_args.update(cluster_name=args.rhev_cluster)
if args.ec2_flavor is not None:
deply_args.update(instance_type=args.ec2_flavor)
try:
vm = provider.deploy_template(args.template, **deply_args)
except:
print "Unexpected error:", sys.exc_info()[0]
destroy(provider, args)
return 12
if not provider.is_vm_running(vm):
logging.error("VM is not running")
return 10
ip_found = False
timeout_count = 0
while not ip_found:
try:
ip = provider.get_ip_address(vm)
ip_found = True
except AttributeError as e:
if timeout_count < 20:
time.sleep(15)
timeout_count += 1
else:
raise e
logging.info("VM " + vm + " is running")
logging.info('IP Address returned is %s', ip)
if args.outfile:
with open(args.outfile, 'w') as outfile:
outfile.write("appliance_ip_address=%s\n" % ip)
return 0
def main():
parser = argparse.ArgumentParser(epilog=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--provider', dest='provider_name', help='provider name in cfme_data')
parser.add_argument('--vm_name', help='the name of the VM on which to act')
parser.add_argument('--remove', help='remove disk from vm', action="store_true")
args = parser.parse_args()
provider = provider_factory(args.provider_name)
# check that we are working with a rhev provider
if not isinstance(provider, RHEVMSystem):
sys.exit(args.providername + " is not a RHEVM system, exiting...")
# check that the vm exists on the rhev provider, get the ip address if so
try:
vm = provider.api.vms.get(args.vm_name)
ip_addr = provider.get_ip_address(args.vm_name)
except:
sys.exit(args.vm_name + " vm not found on provider " + args.providername + ", exiting...")
# check for direct lun definition on provider's cfme_data.yaml
if 'direct_lun_name' not in cfme_data['management_systems'][args.provider_name]:
sys.exit("direct_lun_name key not in cfme_data.yaml under provider "
+ args.providername + ", exiting...")
# does the direct lun exist
dlun_name = cfme_data['management_systems'][args.provider_name]['direct_lun_name']
dlun = provider.api.disks.get(dlun_name)
if dlun is None:
sys.exit("Direct lun disk named " + dlun_name + " is not found on provider " +
args.provider_name)
# add it
if not args.remove:
# is the disk present and active?
vm_disk_list = vm.get_disks().list()
for vm_disk in vm_disk_list:
if vm_disk.name == dlun_name:
if vm_disk.active:
return
else:
vm_disk.actvate()
return
# if not present, add it and activate
direct_lun = params.Disk(id=dlun.id)
added_lun = vm.disks.add(direct_lun)
added_lun.activate()
# Init SSH client, run pvscan on the appliance
ssh_kwargs = {
'username': credentials['ssh']['username'],
'password': credentials['ssh']['password'],
'hostname': ip_addr
}
client = SSHClient(**ssh_kwargs)
status, out = client.run_command('pvscan')
# remove it
else:
vm_dlun = vm.disks.get(name=dlun_name)
if vm_dlun is None:
return
else:
detach = params.Action(detach=True)
vm_dlun.delete(action=detach)
def api(self):
return provider_factory(self.id)
def mgmt_sys_api_clients(cfme_data, uses_providers):
"""Returns a list of management system api clients"""
clients = {}
for provider_key in cfme_data['management_systems']:
clients[provider_key] = providers.provider_factory(provider_key)
return clients
def provider_by_type(metafunc, provider_types, *fields):
"""Get the values of the named field keys from ``cfme_data['management_systems']``
Args:
provider_types: A list of provider types to include. If None, all providers are considered
*fields: Names of keys in an individual provider dict whose values will be returned when
used as test function arguments
The following test function arguments are special:
``provider_data``
the entire provider data dict from cfme_data.
``provider_key``
the provider's key in ``cfme_data['management_systems']``
``provider_crud``
the provider's CRUD object, either a :py:class:`cfme.cloud.provider.Provider`
or a :py:class:`cfme.infrastructure.provider.Provider`
``provider_mgmt``
the provider's backend manager, from :py:class:`utils.mgmt_system`
Returns:
An tuple of ``(argnames, argvalues, idlist)`` for use in a pytest_generate_tests hook, or
with the :py:func:`parametrize` helper.
Usage:
# In the function itself
def pytest_generate_tests(metafunc):
argnames, argvalues, idlist = testgen.provider_by_type(
['openstack', 'ec2'],
'type', 'name', 'credentials', 'provider_data', 'hosts'
)
metafunc.parametrize(argnames, argvalues, ids=idlist, scope='module')
# Using the parametrize wrapper
pytest_generate_tests = testgen.parametrize(testgen.provider_by_type, ['openstack', 'ec2'],
'type', 'name', 'credentials', 'provider_data', 'hosts', scope='module')
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
"""
argnames = list(fields)
argvalues = []
idlist = []
special_args = ("provider_key", "provider_data", "provider_crud", "provider_mgmt", "provider_type")
# Hook on special attrs if requested
for argname in special_args:
if argname in metafunc.fixturenames and argname not in argnames:
argnames.append(argname)
for provider, data in cfme_data["management_systems"].iteritems():
prov_type = data["type"]
if provider_types is not None and prov_type not in provider_types:
# Skip unwanted types
continue
# Check provider hasn't been filtered out with --use-provider
if provider not in filtered:
continue
# Use the provider name for idlist, helps with readable parametrized test output
idlist.append(provider)
# Get values for the requested fields, filling in with None for undefined fields
data_values = {field: data.get(field, None) for field in fields}
# Go through the values and handle the special 'data' name
# report the undefined fields to the log
for key, value in data_values.iteritems():
if value is None:
logger.warning('Field "%s" not defined for provider "%s", defaulting to None' % (key, provider))
if prov_type in cloud_provider_type_map:
crud = get_cloud_provider(provider)
elif prov_type in infra_provider_type_map:
crud = get_infra_provider(provider)
# else: wat? You deserve the NameError you're about to receive
mgmt = provider_factory(provider)
values = []
special_args_map = dict(zip(special_args, (provider, data, crud, mgmt, prov_type)))
for arg in argnames:
if arg in special_args_map:
values.append(special_args_map[arg])
elif arg in data_values:
values.append(data_values[arg])
argvalues.append(values)
return argnames, argvalues, idlist
def _provider(self):
"""
Note:
Cannot be cached because provider object is unpickable.
"""
return provider_factory(self._provider_name)
def main():
parser = argparse.ArgumentParser(
epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('provider_name', help='provider name in cfme_data')
parser.add_argument('vm_name', help='the name of the VM on which to act')
parser.add_argument(
'uptime',
type=int,
help='how long do you want the vm to be on (in seconds)')
parser.add_argument(
'downtime',
type=int,
help='how long do you want the vm to be off (in seconds)')
args = parser.parse_args()
# Make sure the VM is off to start
provider = provider_factory(args.provider_name)
if provider.is_vm_running(args.vm_name):
provider.stop_vm(args.vm_name)
provider.vm_status(args.vm_name)
# Toggle the VM On and Off based on the Uptime and Downtime input arguments
# The script diconnects from the provider before each sleep and reconnects after
# each sleep to prevent it from timing out
while True:
try:
# Initialize start_success to False so it enters the first while loop
# and the times_failed_counter to 0
start_success = False
times_failed_counter = 0
# Turn the VM on for the specified amount of time
# If it can't find the VM, keep trying for 30 minutes
while not start_success:
try:
provider.start_vm(args.vm_name)
provider.vm_status(args.vm_name)
start_success = True
provider.disconnect()
time.sleep(args.uptime)
provider = provider_factory(args.provider_name)
except Exception:
time.sleep(60)
times_failed_counter += 1
if (times_failed_counter == 30):
raise
# Initialize stop_success to False so it enters the first while loop
# and the times_failed_counter to 0
stop_success = False
times_failed_counter = 0
# Turn the VM off for the specified amount of time
# If it can't find the VM, keep trying for 30 minutes
while not stop_success:
try:
provider.stop_vm(args.vm_name)
provider.vm_status(args.vm_name)
stop_success = True
provider.disconnect()
time.sleep(args.downtime)
provider = provider_factory(args.provider_name)
except Exception:
time.sleep(60)
times_failed_counter += 1
if (times_failed_counter == 30):
raise
except (KeyboardInterrupt):
return 0
def main():
parser = argparse.ArgumentParser(epilog=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--provider', dest='provider_name',
help='provider name in cfme_data')
parser.add_argument('--template', help='the name of the template to clone')
parser.add_argument('--vm_name', help='the name of the VM on which to act')
parser.add_argument('--rhev_cluster', help='the name of the VM on which to act', default=None)
parser.add_argument('--rhev_place_policy_host', help='the host for the vm to start on',
default=None)
parser.add_argument('--rhev_place_policy_aff', help='the affinity of the vm on a host',
default=None)
parser.add_argument('--ec2_flavor', help='ec2 flavor', default=None)
parser.add_argument('--rhos_flavor', help='rhos flavor', default=None)
parser.add_argument('--rhos_floating_ip_pool', dest='ip_pool', default=None,
help='openstack floating ip pool to use')
parser.add_argument('--destroy', dest='destroy',
help='Destroy the destination VM', action='store_true')
parser.add_argument('--log', dest='loglevel',
help='Set the log level', default='WARNING')
parser.add_argument('--outfile', dest='outfile',
help='Write provisioning details to the named file', default='')
args = parser.parse_args()
if not args.provider_name:
parser.error('--provider is required')
logger.info('Connecting to %s', args.provider_name)
provider = provider_factory(args.provider_name)
if args.destroy:
destroy(provider, args)
else:
logger.info('Cloning %s to %s', args.template, args.vm_name)
# passing unused args to ec2 provider would blow up so I
# had to make it a little more specific
deploy_args = {}
if args.vm_name is not None:
deploy_args.update(vm_name=args.vm_name)
if args.rhos_flavor is not None:
deploy_args.update(flavour_name=args.rhos_flavor)
if args.ip_pool is not None:
deploy_args.update(assign_floating_ip=args.ip_pool)
if args.rhev_cluster is not None:
deploy_args.update(cluster_name=args.rhev_cluster)
if args.rhev_place_policy_host is not None:
deploy_args.update(placement_policy_host=args.rhev_place_policy_host)
if args.rhev_place_policy_aff is not None:
deploy_args.update(placement_policy_affinity=args.rhev_place_policy_aff)
if args.ec2_flavor is not None:
deploy_args.update(instance_type=args.ec2_flavor)
try:
vm_name = provider.deploy_template(args.template, **deploy_args)
except:
logger.exception(sys.exc_info()[0])
destroy(provider, args)
return 12
if not provider.is_vm_running(vm_name):
logger.error("VM is not running")
return 10
ip, time_taken = wait_for(provider.get_ip_address, [vm_name], num_sec=600,
fail_condition=None)
logger.info("VM " + vm_name + " is running")
logger.info('IP Address returned is %s', ip)
if args.outfile:
with open(args.outfile, 'w') as outfile:
outfile.write("appliance_ip_address=%s\n" % ip)
# In addition to the outfile, drop the ip address on stdout for easy parsing
print ip
return 0
