def __init__(self, config, appliances):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = []
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = {}
self.test_groups = self._test_item_generator()
self._pool = []
from utils.conf import cfme_data
self.provs = sorted(set(cfme_data['management_systems'].keys()),
key=len,
reverse=True)
self.used_prov = set()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.appliances = appliances
# set up the ipc socket
zmq_endpoint = 'ipc://{}'.format(
config.cache.makedir('parallelize').join(str(os.getpid())))
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind(zmq_endpoint)
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args': self.config.args,
'options': self.config.option.__dict__,
'zmq_endpoint': zmq_endpoint,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config'][
"appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options'][
'use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for appliance in self.appliances:
slave_data = SlaveDetail(appliance=appliance)
self.slaves[slave_data.id] = slave_data
for slave in sorted(self.slaves):
self.print_message("using appliance {}".format(
self.slaves[slave].appliance.url),
slave,
green=True)
def __init__(self, ipappliance):
self.ipapp = ipappliance
if store.slave_manager:
sublogger_name = '{} coverage'.format(store.slave_manager.slaveid)
else:
sublogger_name = 'coverage'
self.log = create_sublogger(sublogger_name)
def __init__(self, nodemanager, gateway, config, putevent, base_url=None):
self.nodemanager = nodemanager
self.putevent = putevent
self.gateway = gateway
self.config = config
self.slaveinput = {'slaveid': gateway.id, 'base_url': base_url}
self.log = create_sublogger('cmd-%s' % gateway.id)
self._down = False
def __init__(self, numnodes):
self.numnodes = numnodes
self.node2pending = OrderedDict()
self.node2collection = OrderedDict()
self.log = create_sublogger('scheduler')
self.pending = []
self.collection_is_completed = False
self.module_testindex_gen = None
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
self._nodesready = py.std.threading.Event()
self.trace = self.config.trace.get("nodemanager")
self.log = create_sublogger('nodeman')
self.group = execnet.Group()
self.specs = self._getxspecs()
for spec in self.specs:
self.group.allocate_id(spec)
def __init__(self, config, appliances):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = []
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = {}
self.test_groups = self._test_item_generator()
self._pool = []
from utils.conf import cfme_data
self.provs = sorted(set(cfme_data['management_systems'].keys()),
key=len, reverse=True)
self.used_prov = set()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.appliances = appliances
# set up the ipc socket
zmq_endpoint = 'ipc://{}'.format(
config.cache.makedir('parallelize').join(str(os.getpid())))
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind(zmq_endpoint)
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args': self.config.args,
'options': self.config.option.__dict__,
'zmq_endpoint': zmq_endpoint,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for appliance in self.appliances:
slave_data = SlaveDetail(appliance=appliance)
self.slaves[slave_data.id] = slave_data
for slave in sorted(self.slaves):
self.print_message("using appliance {}".format(self.slaves[slave].appliance.url),
slave, green=True)
def __init__(self, selenium, endpoint, extra_objects=None):
extra_objects = extra_objects or {}
extra_objects.update({
'appliance': endpoint.owner,
'endpoint': endpoint,
'store': store,
})
super(MiqBrowser, self).__init__(selenium,
plugin_class=MiqBrowserPlugin,
logger=create_sublogger('MiqBrowser'),
extra_objects=extra_objects)
def __init__(self, config):
self.config = config
self.shuttingdown = False
self.countfailures = 0
self.log = create_sublogger('dsession')
self.maxfail = config.getvalue("maxfail")
self.queue = queue.Queue()
self._failed_collection_errors = {}
self.terminal = reporter()
self.trdist = TerminalDistReporter(config)
config.pluginmanager.register(self.trdist, "terminaldistreporter")
def __init__(self, selenium, endpoint, extra_objects=None):
extra_objects = extra_objects or {}
extra_objects.update({
'appliance': endpoint.owner,
'endpoint': endpoint,
'store': store,
})
super(MiqBrowser, self).__init__(
selenium,
plugin_class=MiqBrowserPlugin,
logger=create_sublogger('MiqBrowser'),
extra_objects=extra_objects)
def __init__(self, config):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = OrderedDict()
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = SlaveDict()
self.slave_urls = SlaveDict()
self.slave_tests = defaultdict(set)
self.test_groups = self._test_item_generator()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.sprout_client = None
self.sprout_timer = None
self.sprout_pool = None
if not self.config.option.use_sprout:
# Without Sprout
self.appliances = self.config.option.appliances
else:
# Using sprout
self.sprout_client = SproutClient.from_config()
self.terminal.write(
"Requesting {} appliances from Sprout at {}\n".format(
self.config.option.sprout_appliances,
self.sprout_client.api_entry))
pool_id = self.sprout_client.request_appliances(
self.config.option.sprout_group,
count=self.config.option.sprout_appliances,
version=self.config.option.sprout_version,
date=self.config.option.sprout_date,
lease_time=self.config.option.sprout_timeout)
self.terminal.write(
"Pool {}. Waiting for fulfillment ...\n".format(pool_id))
self.sprout_pool = pool_id
at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
if self.config.option.sprout_desc is not None:
self.sprout_client.set_pool_description(
pool_id, str(self.config.option.sprout_desc))
try:
result = wait_for(
lambda: self.sprout_client.request_check(self.sprout_pool)[
"fulfilled"],
num_sec=self.config.option.sprout_provision_timeout * 60,
delay=5,
message="requesting appliances was fulfilled")
except:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)),
pool)
self.terminal.write("Destroying the pool on error.\n")
self.sprout_client.destroy_pool(pool_id)
raise
else:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)),
pool)
self.terminal.write("Provisioning took {0:.1f} seconds\n".format(
result.duration))
request = self.sprout_client.request_check(self.sprout_pool)
self.appliances = []
# Push an appliance to the stack to have proper reference for test collection
IPAppliance(address=request["appliances"][0]["ip_address"]).push()
self.terminal.write("Appliances were provided:\n")
for appliance in request["appliances"]:
url = "https://{}/".format(appliance["ip_address"])
self.appliances.append(url)
self.terminal.write("- {} is {}\n".format(
url, appliance['name']))
map(lambda a: "https://{}/".format(a["ip_address"]),
request["appliances"])
self._reset_timer()
# Set the base_url for collection purposes on the first appliance
conf.runtime["env"]["base_url"] = self.appliances[0]
# Retrieve and print the template_name for Jenkins to pick up
template_name = request["appliances"][0]["template_name"]
conf.runtime["cfme_data"]["basic_info"][
"appliance_template"] = template_name
self.terminal.write(
"appliance_template=\"{}\";\n".format(template_name))
with project_path.join('.appliance_template').open(
'w') as template_file:
template_file.write(
'export appliance_template="{}"'.format(template_name))
self.terminal.write("Parallelized Sprout setup finished.\n")
self.slave_appliances_data = {}
for appliance in request["appliances"]:
self.slave_appliances_data[appliance["ip_address"]] = (
appliance["template_name"], appliance["provider"])
# set up the ipc socket
zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind('%s' % zmq_endpoint)
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args':
self.config.args,
'options':
self.config.option.__dict__,
'zmq_endpoint':
zmq_endpoint,
'sprout':
self.sprout_client is not None and self.sprout_pool is not None,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config'][
"appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options'][
'use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for i, base_url in enumerate(self.appliances):
self.slave_urls.add(base_url)
for slave in sorted(self.slave_urls):
self.print_message("using appliance {}".format(
self.slave_urls[slave]),
slave,
green=True)
# Start the recv queue
self._recv_queue = deque()
recv_queuer = Thread(target=_recv_queue, args=(self, ))
recv_queuer.daemon = True
recv_queuer.start()
"""Library for event testing.
"""
from cached_property import cached_property
from contextlib import contextmanager
from collections import Iterable
from datetime import datetime
from numbers import Number
from sqlalchemy.sql.expression import func
from time import sleep
from threading import Thread, Event as ThreadEvent
from utils.log import create_sublogger
logger = create_sublogger('events')
class EventTool(object):
"""EventTool serves as a wrapper to getting the events from the database.
:var OBJECT_TABLE: Mapping of object types to tables and column names.
"""
OBJECT_TABLE = {
# target_type: (table_name, name_column, id_column)
'VmOrTemplate': ('vms', 'name', 'id'),
'Host': ('hosts', 'name', 'id'),
'Service': ('services', 'name', 'id'),
}
def __init__(self, appliance):
def __init__(self, config):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = OrderedDict()
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = SlaveDict()
self.slave_urls = SlaveDict()
self.slave_tests = defaultdict(set)
self.test_groups = self._test_item_generator()
self._pool = []
self.pool_lock = Lock()
from utils.conf import cfme_data
self.provs = sorted(set(cfme_data['management_systems'].keys()),
key=len, reverse=True)
self.slave_allocation = collections.defaultdict(list)
self.used_prov = set()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.sprout_client = None
self.sprout_timer = None
self.sprout_pool = None
if not self.config.option.use_sprout:
# Without Sprout
self.appliances = self.config.option.appliances
else:
# Using sprout
self.sprout_client = SproutClient.from_config()
try:
if self.config.option.sprout_desc is not None:
jenkins_job = re.findall(r"Jenkins.*[^\d+$]", self.config.option.sprout_desc)
if jenkins_job:
self.terminal.write(
"Check if pool already exists for this '{}' Jenkins job\n".format(
jenkins_job[0]))
jenkins_job_pools = self.sprout_client.find_pools_by_description(
jenkins_job[0], partial=True)
for pool in jenkins_job_pools:
self.terminal.write("Destroying the old pool {} for '{}' job.\n".format(
pool, jenkins_job[0]))
self.sprout_client.destroy_pool(pool)
except Exception as e:
self.terminal.write(
"Exception occurred during old pool deletion, this can be ignored"
"proceeding to Request new pool")
self.terminal.write("> The exception was: {}".format(str(e)))
self.terminal.write(
"Requesting {} appliances from Sprout at {}\n".format(
self.config.option.sprout_appliances, self.sprout_client.api_entry))
pool_id = self.sprout_client.request_appliances(
self.config.option.sprout_group,
count=self.config.option.sprout_appliances,
version=self.config.option.sprout_version,
date=self.config.option.sprout_date,
lease_time=self.config.option.sprout_timeout
)
self.println("Pool {}. Waiting for fulfillment ...".format(pool_id))
self.sprout_pool = pool_id
at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
if self.config.option.sprout_desc is not None:
self.sprout_client.set_pool_description(
pool_id, str(self.config.option.sprout_desc))
def detailed_check():
try:
result = self.sprout_client.request_check(self.sprout_pool)
except SproutException as e:
# TODO: ensure we only exit this way on sprout usage
try:
self.sprout_client.destroy_pool(pool_id)
except Exception:
pass
self.println(
"sprout pool could not be fulfilled\n{}".format(e))
pytest.exit(1)
self.println("[{now:%H:%M}] fulfilled at {progress:2}%".format(
now=datetime.now(),
progress=result['progress']
))
return result["fulfilled"]
try:
result = wait_for(
detailed_check,
num_sec=self.config.option.sprout_provision_timeout * 60,
delay=5,
message="requesting appliances was fulfilled"
)
except Exception:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(self.println, pool)
self.println("Destroying the pool on error.")
self.sprout_client.destroy_pool(pool_id)
raise
else:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(self.println, pool)
self.println("Provisioning took {0:.1f} seconds".format(result.duration))
request = self.sprout_client.request_check(self.sprout_pool)
self.appliances = []
# Push an appliance to the stack to have proper reference for test collection
# FIXME: this is a bad hack based on the need for controll of collection partitioning
appliance_stack.push(
IPAppliance(address=request["appliances"][0]["ip_address"]))
self.println("Appliances were provided:")
for appliance in request["appliances"]:
url = "https://{}/".format(appliance["ip_address"])
self.appliances.append(url)
self.println("- {} is {}".format(url, appliance['name']))
map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
self._reset_timer()
# Set the base_url for collection purposes on the first appliance
conf.runtime["env"]["base_url"] = self.appliances[0]
# Retrieve and print the template_name for Jenkins to pick up
template_name = request["appliances"][0]["template_name"]
conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
with project_path.join('.appliance_template').open('w') as template_file:
template_file.write('export appliance_template="{}"'.format(template_name))
self.println("Parallelized Sprout setup finished.")
self.slave_appliances_data = {}
for appliance in request["appliances"]:
self.slave_appliances_data[appliance["ip_address"]] = (
appliance["template_name"], appliance["provider"]
)
# set up the ipc socket
zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind('{}'.format(zmq_endpoint))
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args': self.config.args,
'options': self.config.option.__dict__,
'zmq_endpoint': zmq_endpoint,
'sprout': self.sprout_client is not None and self.sprout_pool is not None,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for i, base_url in enumerate(self.appliances):
self.slave_urls.add(base_url)
for slave in sorted(self.slave_urls):
self.print_message("using appliance {}".format(self.slave_urls[slave]),
slave, green=True)
# Start the recv queue
self._recv_queue = deque()
recv_queuer = Thread(target=_recv_queue, args=(self,))
recv_queuer.daemon = True
recv_queuer.start()
def __init__(self, config):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = OrderedDict()
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = SlaveDict()
self.slave_urls = SlaveDict()
self.slave_tests = defaultdict(set)
self.test_groups = self._test_item_generator()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.sprout_client = None
self.sprout_timer = None
self.sprout_pool = None
if not self.config.option.use_sprout:
# Without Sprout
self.appliances = self.config.option.appliances
else:
# Using sprout
self.sprout_client = SproutClient.from_config()
self.terminal.write(
"Requesting {} appliances from Sprout at {}\n".format(
self.config.option.sprout_appliances, self.sprout_client.api_entry))
pool_id = self.sprout_client.request_appliances(
self.config.option.sprout_group,
count=self.config.option.sprout_appliances,
version=self.config.option.sprout_version,
date=self.config.option.sprout_date,
lease_time=self.config.option.sprout_timeout
)
self.terminal.write("Pool {}. Waiting for fulfillment ...\n".format(pool_id))
self.sprout_pool = pool_id
at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
if self.config.option.sprout_desc is not None:
self.sprout_client.set_pool_description(
pool_id, str(self.config.option.sprout_desc))
try:
result = wait_for(
lambda: self.sprout_client.request_check(self.sprout_pool)["fulfilled"],
num_sec=self.config.option.sprout_provision_timeout * 60,
delay=5,
message="requesting appliances was fulfilled"
)
except:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
self.terminal.write("Destroying the pool on error.\n")
self.sprout_client.destroy_pool(pool_id)
raise
else:
pool = self.sprout_client.request_check(self.sprout_pool)
dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
self.terminal.write("Provisioning took {0:.1f} seconds\n".format(result.duration))
request = self.sprout_client.request_check(self.sprout_pool)
self.appliances = []
# Push an appliance to the stack to have proper reference for test collection
IPAppliance(address=request["appliances"][0]["ip_address"]).push()
self.terminal.write("Appliances were provided:\n")
for appliance in request["appliances"]:
url = "https://{}/".format(appliance["ip_address"])
self.appliances.append(url)
self.terminal.write("- {} is {}\n".format(url, appliance['name']))
map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
self._reset_timer()
# Set the base_url for collection purposes on the first appliance
conf.runtime["env"]["base_url"] = self.appliances[0]
# Retrieve and print the template_name for Jenkins to pick up
template_name = request["appliances"][0]["template_name"]
conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
with project_path.join('.appliance_template').open('w') as template_file:
template_file.write('export appliance_template="{}"'.format(template_name))
self.terminal.write("Parallelized Sprout setup finished.\n")
self.slave_appliances_data = {}
for appliance in request["appliances"]:
self.slave_appliances_data[appliance["ip_address"]] = (
appliance["template_name"], appliance["provider"]
)
# set up the ipc socket
zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind('%s' % zmq_endpoint)
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args': self.config.args,
'options': self.config.option.__dict__,
'zmq_endpoint': zmq_endpoint,
'sprout': self.sprout_client is not None and self.sprout_pool is not None,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for i, base_url in enumerate(self.appliances):
self.slave_urls.add(base_url)
# Fire up the workers
self._slave_audit()
# Start the recv queue
self._recv_queue = deque()
recv_queuer = Thread(target=_recv_queue, args=(self,))
recv_queuer.daemon = True
recv_queuer.start()
def __init__(self, selenium, endpoint, extra_objects=None):
extra_objects = extra_objects or {}
extra_objects.update({"appliance": endpoint.owner, "endpoint": endpoint, "store": store})
super(MiqBrowser, self).__init__(
selenium, plugin_class=MiqBrowserPlugin, logger=create_sublogger("MiqBrowser"), extra_objects=extra_objects
)
def __init__(self, config):
self.config = config
self.session = None
self.session_finished = False
self.countfailures = 0
self.collection = OrderedDict()
self.sent_tests = 0
self.log = create_sublogger('master')
self.maxfail = config.getvalue("maxfail")
self._failed_collection_errors = {}
self.terminal = store.terminalreporter
self.trdist = None
self.slaves = SlaveDict()
self.slave_urls = SlaveDict()
self.slave_tests = defaultdict(set)
self.test_groups = self._test_item_generator()
self._pool = []
self.pool_lock = Lock()
from utils.conf import cfme_data
self.provs = sorted(set(cfme_data['management_systems'].keys()),
key=len, reverse=True)
self.slave_allocation = collections.defaultdict(list)
self.used_prov = set()
self.failed_slave_test_groups = deque()
self.slave_spawn_count = 0
self.appliances = self.config.option.appliances
# set up the ipc socket
zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
ctx = zmq.Context.instance()
self.sock = ctx.socket(zmq.ROUTER)
self.sock.bind('{}'.format(zmq_endpoint))
# clean out old slave config if it exists
slave_config = conf_path.join('slave_config.yaml')
slave_config.check() and slave_config.remove()
# write out the slave config
conf.runtime['slave_config'] = {
'args': self.config.args,
'options': self.config.option.__dict__,
'zmq_endpoint': zmq_endpoint,
}
if hasattr(self, "slave_appliances_data"):
conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout
conf.save('slave_config')
for base_url in self.appliances:
self.slave_urls.add(base_url)
for slave in sorted(self.slave_urls):
self.print_message("using appliance {}".format(self.slave_urls[slave]),
slave, green=True)
# Start the recv queue
self._recv_queue = deque()
recv_queuer = Thread(target=_recv_queue, args=(self,))
recv_queuer.daemon = True
recv_queuer.start()
