def __init__(self, config, appliances): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = [] self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = {} self.test_groups = self._test_item_generator() self._pool = [] from utils.conf import cfme_data self.provs = sorted(set(cfme_data['management_systems'].keys()), key=len, reverse=True) self.used_prov = set() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.appliances = appliances # set up the ipc socket zmq_endpoint = 'ipc://{}'.format( config.cache.makedir('parallelize').join(str(os.getpid()))) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind(zmq_endpoint) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config'][ "appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options'][ 'use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for appliance in self.appliances: slave_data = SlaveDetail(appliance=appliance) self.slaves[slave_data.id] = slave_data for slave in sorted(self.slaves): self.print_message("using appliance {}".format( self.slaves[slave].appliance.url), slave, green=True)
def __init__(self, ipappliance): self.ipapp = ipappliance if store.slave_manager: sublogger_name = '{} coverage'.format(store.slave_manager.slaveid) else: sublogger_name = 'coverage' self.log = create_sublogger(sublogger_name)
def __init__(self, nodemanager, gateway, config, putevent, base_url=None): self.nodemanager = nodemanager self.putevent = putevent self.gateway = gateway self.config = config self.slaveinput = {'slaveid': gateway.id, 'base_url': base_url} self.log = create_sublogger('cmd-%s' % gateway.id) self._down = False
def __init__(self, numnodes): self.numnodes = numnodes self.node2pending = OrderedDict() self.node2collection = OrderedDict() self.log = create_sublogger('scheduler') self.pending = [] self.collection_is_completed = False self.module_testindex_gen = None
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() self.trace = self.config.trace.get("nodemanager") self.log = create_sublogger('nodeman') self.group = execnet.Group() self.specs = self._getxspecs() for spec in self.specs: self.group.allocate_id(spec)
def __init__(self, config, appliances): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = [] self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = {} self.test_groups = self._test_item_generator() self._pool = [] from utils.conf import cfme_data self.provs = sorted(set(cfme_data['management_systems'].keys()), key=len, reverse=True) self.used_prov = set() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.appliances = appliances # set up the ipc socket zmq_endpoint = 'ipc://{}'.format( config.cache.makedir('parallelize').join(str(os.getpid()))) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind(zmq_endpoint) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for appliance in self.appliances: slave_data = SlaveDetail(appliance=appliance) self.slaves[slave_data.id] = slave_data for slave in sorted(self.slaves): self.print_message("using appliance {}".format(self.slaves[slave].appliance.url), slave, green=True)
def __init__(self, selenium, endpoint, extra_objects=None): extra_objects = extra_objects or {} extra_objects.update({ 'appliance': endpoint.owner, 'endpoint': endpoint, 'store': store, }) super(MiqBrowser, self).__init__(selenium, plugin_class=MiqBrowserPlugin, logger=create_sublogger('MiqBrowser'), extra_objects=extra_objects)
def __init__(self, config): self.config = config self.shuttingdown = False self.countfailures = 0 self.log = create_sublogger('dsession') self.maxfail = config.getvalue("maxfail") self.queue = queue.Queue() self._failed_collection_errors = {} self.terminal = reporter() self.trdist = TerminalDistReporter(config) config.pluginmanager.register(self.trdist, "terminaldistreporter")
def __init__(self, selenium, endpoint, extra_objects=None): extra_objects = extra_objects or {} extra_objects.update({ 'appliance': endpoint.owner, 'endpoint': endpoint, 'store': store, }) super(MiqBrowser, self).__init__( selenium, plugin_class=MiqBrowserPlugin, logger=create_sublogger('MiqBrowser'), extra_objects=extra_objects)
def __init__(self, config): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = OrderedDict() self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = SlaveDict() self.slave_urls = SlaveDict() self.slave_tests = defaultdict(set) self.test_groups = self._test_item_generator() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.sprout_client = None self.sprout_timer = None self.sprout_pool = None if not self.config.option.use_sprout: # Without Sprout self.appliances = self.config.option.appliances else: # Using sprout self.sprout_client = SproutClient.from_config() self.terminal.write( "Requesting {} appliances from Sprout at {}\n".format( self.config.option.sprout_appliances, self.sprout_client.api_entry)) pool_id = self.sprout_client.request_appliances( self.config.option.sprout_group, count=self.config.option.sprout_appliances, version=self.config.option.sprout_version, date=self.config.option.sprout_date, lease_time=self.config.option.sprout_timeout) self.terminal.write( "Pool {}. Waiting for fulfillment ...\n".format(pool_id)) self.sprout_pool = pool_id at_exit(self.sprout_client.destroy_pool, self.sprout_pool) if self.config.option.sprout_desc is not None: self.sprout_client.set_pool_description( pool_id, str(self.config.option.sprout_desc)) try: result = wait_for( lambda: self.sprout_client.request_check(self.sprout_pool)[ "fulfilled"], num_sec=self.config.option.sprout_provision_timeout * 60, delay=5, message="requesting appliances was fulfilled") except: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool) self.terminal.write("Destroying the pool on error.\n") self.sprout_client.destroy_pool(pool_id) raise else: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool) self.terminal.write("Provisioning took {0:.1f} seconds\n".format( result.duration)) request = self.sprout_client.request_check(self.sprout_pool) self.appliances = [] # Push an appliance to the stack to have proper reference for test collection IPAppliance(address=request["appliances"][0]["ip_address"]).push() self.terminal.write("Appliances were provided:\n") for appliance in request["appliances"]: url = "https://{}/".format(appliance["ip_address"]) self.appliances.append(url) self.terminal.write("- {} is {}\n".format( url, appliance['name'])) map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"]) self._reset_timer() # Set the base_url for collection purposes on the first appliance conf.runtime["env"]["base_url"] = self.appliances[0] # Retrieve and print the template_name for Jenkins to pick up template_name = request["appliances"][0]["template_name"] conf.runtime["cfme_data"]["basic_info"][ "appliance_template"] = template_name self.terminal.write( "appliance_template=\"{}\";\n".format(template_name)) with project_path.join('.appliance_template').open( 'w') as template_file: template_file.write( 'export appliance_template="{}"'.format(template_name)) self.terminal.write("Parallelized Sprout setup finished.\n") self.slave_appliances_data = {} for appliance in request["appliances"]: self.slave_appliances_data[appliance["ip_address"]] = ( appliance["template_name"], appliance["provider"]) # set up the ipc socket zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port()) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind('%s' % zmq_endpoint) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, 'sprout': self.sprout_client is not None and self.sprout_pool is not None, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config'][ "appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options'][ 'use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for i, base_url in enumerate(self.appliances): self.slave_urls.add(base_url) for slave in sorted(self.slave_urls): self.print_message("using appliance {}".format( self.slave_urls[slave]), slave, green=True) # Start the recv queue self._recv_queue = deque() recv_queuer = Thread(target=_recv_queue, args=(self, )) recv_queuer.daemon = True recv_queuer.start()
"""Library for event testing. """ from cached_property import cached_property from contextlib import contextmanager from collections import Iterable from datetime import datetime from numbers import Number from sqlalchemy.sql.expression import func from time import sleep from threading import Thread, Event as ThreadEvent from utils.log import create_sublogger logger = create_sublogger('events') class EventTool(object): """EventTool serves as a wrapper to getting the events from the database. :var OBJECT_TABLE: Mapping of object types to tables and column names. """ OBJECT_TABLE = { # target_type: (table_name, name_column, id_column) 'VmOrTemplate': ('vms', 'name', 'id'), 'Host': ('hosts', 'name', 'id'), 'Service': ('services', 'name', 'id'), } def __init__(self, appliance):
def __init__(self, config): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = OrderedDict() self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = SlaveDict() self.slave_urls = SlaveDict() self.slave_tests = defaultdict(set) self.test_groups = self._test_item_generator() self._pool = [] self.pool_lock = Lock() from utils.conf import cfme_data self.provs = sorted(set(cfme_data['management_systems'].keys()), key=len, reverse=True) self.slave_allocation = collections.defaultdict(list) self.used_prov = set() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.sprout_client = None self.sprout_timer = None self.sprout_pool = None if not self.config.option.use_sprout: # Without Sprout self.appliances = self.config.option.appliances else: # Using sprout self.sprout_client = SproutClient.from_config() try: if self.config.option.sprout_desc is not None: jenkins_job = re.findall(r"Jenkins.*[^\d+$]", self.config.option.sprout_desc) if jenkins_job: self.terminal.write( "Check if pool already exists for this '{}' Jenkins job\n".format( jenkins_job[0])) jenkins_job_pools = self.sprout_client.find_pools_by_description( jenkins_job[0], partial=True) for pool in jenkins_job_pools: self.terminal.write("Destroying the old pool {} for '{}' job.\n".format( pool, jenkins_job[0])) self.sprout_client.destroy_pool(pool) except Exception as e: self.terminal.write( "Exception occurred during old pool deletion, this can be ignored" "proceeding to Request new pool") self.terminal.write("> The exception was: {}".format(str(e))) self.terminal.write( "Requesting {} appliances from Sprout at {}\n".format( self.config.option.sprout_appliances, self.sprout_client.api_entry)) pool_id = self.sprout_client.request_appliances( self.config.option.sprout_group, count=self.config.option.sprout_appliances, version=self.config.option.sprout_version, date=self.config.option.sprout_date, lease_time=self.config.option.sprout_timeout ) self.println("Pool {}. Waiting for fulfillment ...".format(pool_id)) self.sprout_pool = pool_id at_exit(self.sprout_client.destroy_pool, self.sprout_pool) if self.config.option.sprout_desc is not None: self.sprout_client.set_pool_description( pool_id, str(self.config.option.sprout_desc)) def detailed_check(): try: result = self.sprout_client.request_check(self.sprout_pool) except SproutException as e: # TODO: ensure we only exit this way on sprout usage try: self.sprout_client.destroy_pool(pool_id) except Exception: pass self.println( "sprout pool could not be fulfilled\n{}".format(e)) pytest.exit(1) self.println("[{now:%H:%M}] fulfilled at {progress:2}%".format( now=datetime.now(), progress=result['progress'] )) return result["fulfilled"] try: result = wait_for( detailed_check, num_sec=self.config.option.sprout_provision_timeout * 60, delay=5, message="requesting appliances was fulfilled" ) except Exception: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(self.println, pool) self.println("Destroying the pool on error.") self.sprout_client.destroy_pool(pool_id) raise else: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(self.println, pool) self.println("Provisioning took {0:.1f} seconds".format(result.duration)) request = self.sprout_client.request_check(self.sprout_pool) self.appliances = [] # Push an appliance to the stack to have proper reference for test collection # FIXME: this is a bad hack based on the need for controll of collection partitioning appliance_stack.push( IPAppliance(address=request["appliances"][0]["ip_address"])) self.println("Appliances were provided:") for appliance in request["appliances"]: url = "https://{}/".format(appliance["ip_address"]) self.appliances.append(url) self.println("- {} is {}".format(url, appliance['name'])) map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"]) self._reset_timer() # Set the base_url for collection purposes on the first appliance conf.runtime["env"]["base_url"] = self.appliances[0] # Retrieve and print the template_name for Jenkins to pick up template_name = request["appliances"][0]["template_name"] conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name self.terminal.write("appliance_template=\"{}\";\n".format(template_name)) with project_path.join('.appliance_template').open('w') as template_file: template_file.write('export appliance_template="{}"'.format(template_name)) self.println("Parallelized Sprout setup finished.") self.slave_appliances_data = {} for appliance in request["appliances"]: self.slave_appliances_data[appliance["ip_address"]] = ( appliance["template_name"], appliance["provider"] ) # set up the ipc socket zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port()) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind('{}'.format(zmq_endpoint)) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, 'sprout': self.sprout_client is not None and self.sprout_pool is not None, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for i, base_url in enumerate(self.appliances): self.slave_urls.add(base_url) for slave in sorted(self.slave_urls): self.print_message("using appliance {}".format(self.slave_urls[slave]), slave, green=True) # Start the recv queue self._recv_queue = deque() recv_queuer = Thread(target=_recv_queue, args=(self,)) recv_queuer.daemon = True recv_queuer.start()
def __init__(self, config): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = OrderedDict() self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = SlaveDict() self.slave_urls = SlaveDict() self.slave_tests = defaultdict(set) self.test_groups = self._test_item_generator() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.sprout_client = None self.sprout_timer = None self.sprout_pool = None if not self.config.option.use_sprout: # Without Sprout self.appliances = self.config.option.appliances else: # Using sprout self.sprout_client = SproutClient.from_config() self.terminal.write( "Requesting {} appliances from Sprout at {}\n".format( self.config.option.sprout_appliances, self.sprout_client.api_entry)) pool_id = self.sprout_client.request_appliances( self.config.option.sprout_group, count=self.config.option.sprout_appliances, version=self.config.option.sprout_version, date=self.config.option.sprout_date, lease_time=self.config.option.sprout_timeout ) self.terminal.write("Pool {}. Waiting for fulfillment ...\n".format(pool_id)) self.sprout_pool = pool_id at_exit(self.sprout_client.destroy_pool, self.sprout_pool) if self.config.option.sprout_desc is not None: self.sprout_client.set_pool_description( pool_id, str(self.config.option.sprout_desc)) try: result = wait_for( lambda: self.sprout_client.request_check(self.sprout_pool)["fulfilled"], num_sec=self.config.option.sprout_provision_timeout * 60, delay=5, message="requesting appliances was fulfilled" ) except: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool) self.terminal.write("Destroying the pool on error.\n") self.sprout_client.destroy_pool(pool_id) raise else: pool = self.sprout_client.request_check(self.sprout_pool) dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool) self.terminal.write("Provisioning took {0:.1f} seconds\n".format(result.duration)) request = self.sprout_client.request_check(self.sprout_pool) self.appliances = [] # Push an appliance to the stack to have proper reference for test collection IPAppliance(address=request["appliances"][0]["ip_address"]).push() self.terminal.write("Appliances were provided:\n") for appliance in request["appliances"]: url = "https://{}/".format(appliance["ip_address"]) self.appliances.append(url) self.terminal.write("- {} is {}\n".format(url, appliance['name'])) map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"]) self._reset_timer() # Set the base_url for collection purposes on the first appliance conf.runtime["env"]["base_url"] = self.appliances[0] # Retrieve and print the template_name for Jenkins to pick up template_name = request["appliances"][0]["template_name"] conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name self.terminal.write("appliance_template=\"{}\";\n".format(template_name)) with project_path.join('.appliance_template').open('w') as template_file: template_file.write('export appliance_template="{}"'.format(template_name)) self.terminal.write("Parallelized Sprout setup finished.\n") self.slave_appliances_data = {} for appliance in request["appliances"]: self.slave_appliances_data[appliance["ip_address"]] = ( appliance["template_name"], appliance["provider"] ) # set up the ipc socket zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port()) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind('%s' % zmq_endpoint) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, 'sprout': self.sprout_client is not None and self.sprout_pool is not None, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for i, base_url in enumerate(self.appliances): self.slave_urls.add(base_url) # Fire up the workers self._slave_audit() # Start the recv queue self._recv_queue = deque() recv_queuer = Thread(target=_recv_queue, args=(self,)) recv_queuer.daemon = True recv_queuer.start()
def __init__(self, selenium, endpoint, extra_objects=None): extra_objects = extra_objects or {} extra_objects.update({"appliance": endpoint.owner, "endpoint": endpoint, "store": store}) super(MiqBrowser, self).__init__( selenium, plugin_class=MiqBrowserPlugin, logger=create_sublogger("MiqBrowser"), extra_objects=extra_objects )
def __init__(self, config): self.config = config self.session = None self.session_finished = False self.countfailures = 0 self.collection = OrderedDict() self.sent_tests = 0 self.log = create_sublogger('master') self.maxfail = config.getvalue("maxfail") self._failed_collection_errors = {} self.terminal = store.terminalreporter self.trdist = None self.slaves = SlaveDict() self.slave_urls = SlaveDict() self.slave_tests = defaultdict(set) self.test_groups = self._test_item_generator() self._pool = [] self.pool_lock = Lock() from utils.conf import cfme_data self.provs = sorted(set(cfme_data['management_systems'].keys()), key=len, reverse=True) self.slave_allocation = collections.defaultdict(list) self.used_prov = set() self.failed_slave_test_groups = deque() self.slave_spawn_count = 0 self.appliances = self.config.option.appliances # set up the ipc socket zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port()) ctx = zmq.Context.instance() self.sock = ctx.socket(zmq.ROUTER) self.sock.bind('{}'.format(zmq_endpoint)) # clean out old slave config if it exists slave_config = conf_path.join('slave_config.yaml') slave_config.check() and slave_config.remove() # write out the slave config conf.runtime['slave_config'] = { 'args': self.config.args, 'options': self.config.option.__dict__, 'zmq_endpoint': zmq_endpoint, } if hasattr(self, "slave_appliances_data"): conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data conf.runtime['slave_config']['options']['use_sprout'] = False # Slaves don't use sprout conf.save('slave_config') for base_url in self.appliances: self.slave_urls.add(base_url) for slave in sorted(self.slave_urls): self.print_message("using appliance {}".format(self.slave_urls[slave]), slave, green=True) # Start the recv queue self._recv_queue = deque() recv_queuer = Thread(target=_recv_queue, args=(self,)) recv_queuer.daemon = True recv_queuer.start()