Esempio n. 1
0
    def __init__(self, config, appliances):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = []
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = {}
        self.test_groups = self._test_item_generator()

        self._pool = []
        from utils.conf import cfme_data
        self.provs = sorted(set(cfme_data['management_systems'].keys()),
                            key=len,
                            reverse=True)
        self.used_prov = set()

        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.appliances = appliances

        # set up the ipc socket

        zmq_endpoint = 'ipc://{}'.format(
            config.cache.makedir('parallelize').join(str(os.getpid())))
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind(zmq_endpoint)

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config'][
                "appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options'][
            'use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for appliance in self.appliances:
            slave_data = SlaveDetail(appliance=appliance)
            self.slaves[slave_data.id] = slave_data

        for slave in sorted(self.slaves):
            self.print_message("using appliance {}".format(
                self.slaves[slave].appliance.url),
                               slave,
                               green=True)
Esempio n. 2
0
    def __init__(self, config, appliances):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = []
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = {}
        self.test_groups = self._test_item_generator()

        self._pool = []
        from utils.conf import cfme_data
        self.provs = sorted(set(cfme_data['management_systems'].keys()),
                            key=len, reverse=True)
        self.used_prov = set()

        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.appliances = appliances

        # set up the ipc socket

        zmq_endpoint = 'ipc://{}'.format(
            config.cache.makedir('parallelize').join(str(os.getpid())))
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind(zmq_endpoint)

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options']['use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for appliance in self.appliances:
            slave_data = SlaveDetail(appliance=appliance)
            self.slaves[slave_data.id] = slave_data

        for slave in sorted(self.slaves):
            self.print_message("using appliance {}".format(self.slaves[slave].appliance.url),
                slave, green=True)
Esempio n. 3
0
def load_yaml(filename=None):
    # Find the requested yaml in the config dir, relative to this file's location
    # (aiming for cfme_tests/config)
    path = conf_path.join('%s.yaml' % filename)

    if path.check():
        with path.open() as config_fh:
            return yaml.load(config_fh, Loader=YamlConfigLoader)
    else:
        msg = 'Unable to load configuration file at %s' % path
        raise ConfigNotFoundException(msg)
Esempio n. 4
0
def load_yaml(filename=None, warn_on_fail=True):
    # Find the requested yaml in the config dir, relative to this file's location
    # (aiming for cfme_tests/config)
    path = conf_path.join('%s.yaml' % filename)

    if path.check():
        with path.open() as config_fh:
            return yaml.load(config_fh, Loader=YamlConfigLoader)

    if warn_on_fail:
        msg = 'Unable to load configuration file at %s' % path
        warn(msg, ConfigNotFound)

    return {}
Esempio n. 5
0
def main():
    """Use quickstart to ensure we have correct env, then execute imports in ipython and done."""
    quickstart.main(quickstart.parser.parse_args(['--mk-virtualenv', sys.prefix]))
    print('Welcome to IPython designed for running CFME QE code.')
    ipython = TerminalInteractiveShell.instance()
    for code_import in IMPORTS:
        print('> {}'.format(code_import))
        ipython.run_cell(code_import)
    from utils.path import conf_path
    custom_import_path = conf_path.join('miq_python_startup.py')
    if custom_import_path.exists():
        with open(custom_import_path.strpath, 'r') as custom_import_file:
            custom_import_code = custom_import_file.read()
        print('Importing custom code:\n{}'.format(custom_import_code.strip()))
        ipython.run_cell(custom_import_code)
    else:
        print(
            'You can create your own python file with imports you use frequently. '
            'Just create a conf/miq_python_startup.py file in your repo. '
            'This file can contain arbitrary python code that is executed in this context.')
    ipython.interact()
Esempio n. 6
0
    def __init__(self, config):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = OrderedDict()
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = SlaveDict()
        self.slave_urls = SlaveDict()
        self.slave_tests = defaultdict(set)
        self.test_groups = self._test_item_generator()
        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.sprout_client = None
        self.sprout_timer = None
        self.sprout_pool = None
        if not self.config.option.use_sprout:
            # Without Sprout
            self.appliances = self.config.option.appliances
        else:
            # Using sprout
            self.sprout_client = SproutClient.from_config()
            self.terminal.write(
                "Requesting {} appliances from Sprout at {}\n".format(
                    self.config.option.sprout_appliances,
                    self.sprout_client.api_entry))
            pool_id = self.sprout_client.request_appliances(
                self.config.option.sprout_group,
                count=self.config.option.sprout_appliances,
                version=self.config.option.sprout_version,
                date=self.config.option.sprout_date,
                lease_time=self.config.option.sprout_timeout)
            self.terminal.write(
                "Pool {}. Waiting for fulfillment ...\n".format(pool_id))
            self.sprout_pool = pool_id
            at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
            if self.config.option.sprout_desc is not None:
                self.sprout_client.set_pool_description(
                    pool_id, str(self.config.option.sprout_desc))
            try:
                result = wait_for(
                    lambda: self.sprout_client.request_check(self.sprout_pool)[
                        "fulfilled"],
                    num_sec=self.config.option.sprout_provision_timeout * 60,
                    delay=5,
                    message="requesting appliances was fulfilled")
            except:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)),
                               pool)
                self.terminal.write("Destroying the pool on error.\n")
                self.sprout_client.destroy_pool(pool_id)
                raise
            else:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)),
                               pool)
            self.terminal.write("Provisioning took {0:.1f} seconds\n".format(
                result.duration))
            request = self.sprout_client.request_check(self.sprout_pool)
            self.appliances = []
            # Push an appliance to the stack to have proper reference for test collection
            IPAppliance(address=request["appliances"][0]["ip_address"]).push()
            self.terminal.write("Appliances were provided:\n")
            for appliance in request["appliances"]:
                url = "https://{}/".format(appliance["ip_address"])
                self.appliances.append(url)
                self.terminal.write("- {} is {}\n".format(
                    url, appliance['name']))
            map(lambda a: "https://{}/".format(a["ip_address"]),
                request["appliances"])
            self._reset_timer()
            # Set the base_url for collection purposes on the first appliance
            conf.runtime["env"]["base_url"] = self.appliances[0]
            # Retrieve and print the template_name for Jenkins to pick up
            template_name = request["appliances"][0]["template_name"]
            conf.runtime["cfme_data"]["basic_info"][
                "appliance_template"] = template_name
            self.terminal.write(
                "appliance_template=\"{}\";\n".format(template_name))
            with project_path.join('.appliance_template').open(
                    'w') as template_file:
                template_file.write(
                    'export appliance_template="{}"'.format(template_name))
            self.terminal.write("Parallelized Sprout setup finished.\n")
            self.slave_appliances_data = {}
            for appliance in request["appliances"]:
                self.slave_appliances_data[appliance["ip_address"]] = (
                    appliance["template_name"], appliance["provider"])

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('%s' % zmq_endpoint)

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args':
            self.config.args,
            'options':
            self.config.option.__dict__,
            'zmq_endpoint':
            zmq_endpoint,
            'sprout':
            self.sprout_client is not None and self.sprout_pool is not None,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config'][
                "appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options'][
            'use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for i, base_url in enumerate(self.appliances):
            self.slave_urls.add(base_url)

        for slave in sorted(self.slave_urls):
            self.print_message("using appliance {}".format(
                self.slave_urls[slave]),
                               slave,
                               green=True)

        # Start the recv queue
        self._recv_queue = deque()
        recv_queuer = Thread(target=_recv_queue, args=(self, ))
        recv_queuer.daemon = True
        recv_queuer.start()
Esempio n. 7
0
#: Corresponds to Rails.root in the rails env
rails_root = local('/var/www/miq/vmdb')
#: coverage root, should match what's in the coverage hook and merger scripts
appliance_coverage_root = rails_root.join('coverage')

# local
coverage_data = scripts_data_path.join('coverage')
gemfile = coverage_data.join('coverage_gem.rb')
bundler_d = rails_root.join('bundler.d')
coverage_hook_file_name = 'coverage_hook.rb'
coverage_hook = coverage_data.join(coverage_hook_file_name)
coverage_merger = coverage_data.join('coverage_merger.rb')
thing_toucher = coverage_data.join('thing_toucher.rb')
coverage_output_dir = log_path.join('coverage')
coverage_results_archive = coverage_output_dir.join('coverage-results.tgz')
coverage_appliance_conf = conf_path.join('.ui-coverage')

# This is set in sessionfinish, and should be reliably readable
# in post-yield sessionfinish hook wrappers and all hooks thereafter
ui_coverage_percent = None


def _thing_toucher_async(ssh_client):
    # for use in a subprocess to kick off the thing toucher
    result = ssh_client.run_rails_command('thing_toucher.rb', timeout=0)
    return result.rc == 0


def clean_coverage_dir():
    try:
        coverage_output_dir.remove(ignore_errors=True)
Esempio n. 8
0
    def __init__(self, config):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = OrderedDict()
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = SlaveDict()
        self.slave_urls = SlaveDict()
        self.slave_tests = defaultdict(set)
        self.test_groups = self._test_item_generator()

        self._pool = []
        self.pool_lock = Lock()
        from utils.conf import cfme_data
        self.provs = sorted(set(cfme_data['management_systems'].keys()),
                            key=len, reverse=True)
        self.slave_allocation = collections.defaultdict(list)
        self.used_prov = set()

        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.sprout_client = None
        self.sprout_timer = None
        self.sprout_pool = None
        if not self.config.option.use_sprout:
            # Without Sprout
            self.appliances = self.config.option.appliances
        else:
            # Using sprout
            self.sprout_client = SproutClient.from_config()
            try:
                if self.config.option.sprout_desc is not None:
                    jenkins_job = re.findall(r"Jenkins.*[^\d+$]", self.config.option.sprout_desc)
                    if jenkins_job:
                        self.terminal.write(
                            "Check if pool already exists for this '{}' Jenkins job\n".format(
                                jenkins_job[0]))
                        jenkins_job_pools = self.sprout_client.find_pools_by_description(
                            jenkins_job[0], partial=True)
                        for pool in jenkins_job_pools:
                            self.terminal.write("Destroying the old pool {} for '{}' job.\n".format(
                                pool, jenkins_job[0]))
                            self.sprout_client.destroy_pool(pool)
            except Exception as e:
                self.terminal.write(
                    "Exception occurred during old pool deletion, this can be ignored"
                    "proceeding to Request new pool")
                self.terminal.write("> The exception was: {}".format(str(e)))

            self.terminal.write(
                "Requesting {} appliances from Sprout at {}\n".format(
                    self.config.option.sprout_appliances, self.sprout_client.api_entry))
            pool_id = self.sprout_client.request_appliances(
                self.config.option.sprout_group,
                count=self.config.option.sprout_appliances,
                version=self.config.option.sprout_version,
                date=self.config.option.sprout_date,
                lease_time=self.config.option.sprout_timeout
            )
            self.println("Pool {}. Waiting for fulfillment ...".format(pool_id))
            self.sprout_pool = pool_id
            at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
            if self.config.option.sprout_desc is not None:
                self.sprout_client.set_pool_description(
                    pool_id, str(self.config.option.sprout_desc))

            def detailed_check():
                try:
                    result = self.sprout_client.request_check(self.sprout_pool)
                except SproutException as e:
                    # TODO: ensure we only exit this way on sprout usage
                    try:
                        self.sprout_client.destroy_pool(pool_id)
                    except Exception:
                        pass
                    self.println(
                        "sprout pool could not be fulfilled\n{}".format(e))
                    pytest.exit(1)

                self.println("[{now:%H:%M}] fulfilled at {progress:2}%".format(
                    now=datetime.now(),
                    progress=result['progress']
                ))
                return result["fulfilled"]
            try:
                result = wait_for(
                    detailed_check,
                    num_sec=self.config.option.sprout_provision_timeout * 60,
                    delay=5,
                    message="requesting appliances was fulfilled"
                )
            except Exception:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(self.println, pool)
                self.println("Destroying the pool on error.")
                self.sprout_client.destroy_pool(pool_id)
                raise
            else:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(self.println, pool)
            self.println("Provisioning took {0:.1f} seconds".format(result.duration))
            request = self.sprout_client.request_check(self.sprout_pool)
            self.appliances = []
            # Push an appliance to the stack to have proper reference for test collection
            # FIXME: this is a bad hack based on the need for controll of collection partitioning
            appliance_stack.push(
                IPAppliance(address=request["appliances"][0]["ip_address"]))
            self.println("Appliances were provided:")
            for appliance in request["appliances"]:
                url = "https://{}/".format(appliance["ip_address"])
                self.appliances.append(url)
                self.println("- {} is {}".format(url, appliance['name']))
            map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
            self._reset_timer()
            # Set the base_url for collection purposes on the first appliance
            conf.runtime["env"]["base_url"] = self.appliances[0]
            # Retrieve and print the template_name for Jenkins to pick up
            template_name = request["appliances"][0]["template_name"]
            conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
            self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
            with project_path.join('.appliance_template').open('w') as template_file:
                template_file.write('export appliance_template="{}"'.format(template_name))
            self.println("Parallelized Sprout setup finished.")
            self.slave_appliances_data = {}
            for appliance in request["appliances"]:
                self.slave_appliances_data[appliance["ip_address"]] = (
                    appliance["template_name"], appliance["provider"]
                )

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('{}'.format(zmq_endpoint))

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
            'sprout': self.sprout_client is not None and self.sprout_pool is not None,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options']['use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for i, base_url in enumerate(self.appliances):
            self.slave_urls.add(base_url)

        for slave in sorted(self.slave_urls):
            self.print_message("using appliance {}".format(self.slave_urls[slave]),
                slave, green=True)

        # Start the recv queue
        self._recv_queue = deque()
        recv_queuer = Thread(target=_recv_queue, args=(self,))
        recv_queuer.daemon = True
        recv_queuer.start()
Esempio n. 9
0
def checkout(appliances, timeout, provision_timeout, group, version, date,
             desc, override_ram, override_cpu, populate_yaml, provider):
    """Function to show the given credentials, takes either a provider key or a credential key"""
    override_cpu = override_cpu or None
    override_ram = override_ram or None
    sr = SproutProvisioningRequest(group=group,
                                   count=appliances,
                                   version=version,
                                   date=date,
                                   lease_time=timeout,
                                   provision_timeout=provision_timeout,
                                   desc=desc,
                                   cpu=override_cpu,
                                   ram=override_ram,
                                   provider=provider)
    sm = SproutManager()

    def exit_gracefully(signum, frame):
        sm.destroy_pool()
        sys.exit(0)

    signal.signal(signal.SIGINT, exit_gracefully)
    signal.signal(signal.SIGTERM, exit_gracefully)
    try:
        appliance_data = sm.request_appliances(sr)
        while not sm.check_fullfilled():
            print("waiting...")
            time.sleep(10)
        sm.reset_timer()
        for app in appliance_data:
            print "{}: {}".format(app['name'], app['ip_address'])
        if populate_yaml:
            file_name = conf_path.join('env.local.yaml').strpath
            if os.path.exists(file_name):
                with open(file_name) as f:
                    y_data = yaml.load(f)
                if not y_data:
                    y_data = {}
            else:
                y_data = {}
            if y_data:
                with open(conf_path.join('env.local.backup').strpath,
                          'w') as f:
                    yaml.dump(y_data, f, default_flow_style=False)

            y_data['appliances'] = []
            for app in appliance_data:
                y_data['appliances'].append(
                    {'base_url': 'https://{}/'.format(app['ip_address'])})
            with open(file_name, 'w') as f:
                try:
                    del y_data['base_url']
                except KeyError:
                    pass
                yaml.dump(y_data, f, default_flow_style=False)

        print("Appliance checked out, hit ctrl+c to checkin")

        while True:
            time.sleep(10)

    except KeyboardInterrupt:
        try:
            sm.destroy_pool()
        except:
            print("Error in pool destroy")
Esempio n. 10
0
    def __init__(self, config):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = OrderedDict()
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = SlaveDict()
        self.slave_urls = SlaveDict()
        self.slave_tests = defaultdict(set)
        self.test_groups = self._test_item_generator()
        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.sprout_client = None
        self.sprout_timer = None
        self.sprout_pool = None
        if not self.config.option.use_sprout:
            # Without Sprout
            self.appliances = self.config.option.appliances
        else:
            # Using sprout
            self.sprout_client = SproutClient.from_config()
            self.terminal.write(
                "Requesting {} appliances from Sprout at {}\n".format(
                    self.config.option.sprout_appliances, self.sprout_client.api_entry))
            pool_id = self.sprout_client.request_appliances(
                self.config.option.sprout_group,
                count=self.config.option.sprout_appliances,
                version=self.config.option.sprout_version,
                date=self.config.option.sprout_date,
                lease_time=self.config.option.sprout_timeout
            )
            self.terminal.write("Pool {}. Waiting for fulfillment ...\n".format(pool_id))
            self.sprout_pool = pool_id
            at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
            if self.config.option.sprout_desc is not None:
                self.sprout_client.set_pool_description(
                    pool_id, str(self.config.option.sprout_desc))
            try:
                result = wait_for(
                    lambda: self.sprout_client.request_check(self.sprout_pool)["fulfilled"],
                    num_sec=self.config.option.sprout_provision_timeout * 60,
                    delay=5,
                    message="requesting appliances was fulfilled"
                )
            except:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
                self.terminal.write("Destroying the pool on error.\n")
                self.sprout_client.destroy_pool(pool_id)
                raise
            else:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
            self.terminal.write("Provisioning took {0:.1f} seconds\n".format(result.duration))
            request = self.sprout_client.request_check(self.sprout_pool)
            self.appliances = []
            # Push an appliance to the stack to have proper reference for test collection
            IPAppliance(address=request["appliances"][0]["ip_address"]).push()
            self.terminal.write("Appliances were provided:\n")
            for appliance in request["appliances"]:
                url = "https://{}/".format(appliance["ip_address"])
                self.appliances.append(url)
                self.terminal.write("- {} is {}\n".format(url, appliance['name']))
            map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
            self._reset_timer()
            # Set the base_url for collection purposes on the first appliance
            conf.runtime["env"]["base_url"] = self.appliances[0]
            # Retrieve and print the template_name for Jenkins to pick up
            template_name = request["appliances"][0]["template_name"]
            conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
            self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
            with project_path.join('.appliance_template').open('w') as template_file:
                template_file.write('export appliance_template="{}"'.format(template_name))
            self.terminal.write("Parallelized Sprout setup finished.\n")
            self.slave_appliances_data = {}
            for appliance in request["appliances"]:
                self.slave_appliances_data[appliance["ip_address"]] = (
                    appliance["template_name"], appliance["provider"]
                )

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('%s' % zmq_endpoint)

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
            'sprout': self.sprout_client is not None and self.sprout_pool is not None,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options']['use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for i, base_url in enumerate(self.appliances):
            self.slave_urls.add(base_url)
        # Fire up the workers
        self._slave_audit()

        # Start the recv queue
        self._recv_queue = deque()
        recv_queuer = Thread(target=_recv_queue, args=(self,))
        recv_queuer.daemon = True
        recv_queuer.start()
Esempio n. 11
0
    def __init__(self, config):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = OrderedDict()
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = SlaveDict()
        self.slave_urls = SlaveDict()
        self.slave_tests = defaultdict(set)
        self.test_groups = self._test_item_generator()

        self._pool = []
        self.pool_lock = Lock()
        from utils.conf import cfme_data
        self.provs = sorted(set(cfme_data['management_systems'].keys()),
                            key=len, reverse=True)
        self.slave_allocation = collections.defaultdict(list)
        self.used_prov = set()

        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.appliances = self.config.option.appliances

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('{}'.format(zmq_endpoint))

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options']['use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for base_url in self.appliances:
            self.slave_urls.add(base_url)

        for slave in sorted(self.slave_urls):
            self.print_message("using appliance {}".format(self.slave_urls[slave]),
                slave, green=True)

        # Start the recv queue
        self._recv_queue = deque()
        recv_queuer = Thread(target=_recv_queue, args=(self,))
        recv_queuer.daemon = True
        recv_queuer.start()