Example #1
0
 def __init__(self, test_module, test_class, test_name, options,
              cycle, cvus, thread_id, thread_signaller, sleep_time,
              debug=False):
     meta_method_name = mmn_encode(test_name, cycle, cvus, thread_id)
     threading.Thread.__init__(self, target=self.run, name=meta_method_name,
                               args=())
     self.test = load_unittest(test_module, test_class, meta_method_name,
                               options)
     self.color = not options.no_color
     self.sleep_time = sleep_time
     self.debug = debug
     self.thread_signaller = thread_signaller
     # this makes threads endings if main stop with a KeyboardInterupt
     self.setDaemon(1)
Example #2
0
    def __init__(self, module_name, class_name, method_name, options):
        self.module_name = module_name
        self.class_name = class_name
        self.method_name = method_name
        self.options = options
        self.color = not options.no_color
        # create a unittest to get the configuration file
        test = load_unittest(self.module_name, class_name,
                             mmn_encode(method_name, 0, 0, 0), options)
        self.config_path = test._config_path
        self.result_path = test.result_path
        self.class_title = test.conf_get('main', 'title')
        self.class_description = test.conf_get('main', 'description')
        self.test_id = self.method_name
        self.test_description = test.conf_get(self.method_name, 'description',
                                              'No test description')
        self.test_url = test.conf_get('main', 'url')
        self.cycles = map(int, test.conf_getList('bench', 'cycles'))
        self.duration = test.conf_getInt('bench', 'duration')
        self.startup_delay = test.conf_getFloat('bench', 'startup_delay')
        self.cycle_time = test.conf_getFloat('bench', 'cycle_time')
        self.sleep_time = test.conf_getFloat('bench', 'sleep_time')
        self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min')
        self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max')
        self.threads = []  # Contains list of ThreadData objects
        self.last_thread_id = -1
        self.thread_creation_lock = threading.Lock()

        # setup monitoring
        monitor_hosts = []                  # list of (host, port, descr)
        if not options.is_distributed:
            hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split()
            for host in hosts:
                name = host
                host = test.conf_get(host,'host',host.strip())
                monitor_hosts.append((name, host, test.conf_getInt(name, 'port'),
                                      test.conf_get(name, 'description', '')))
        self.monitor_hosts = monitor_hosts
        # keep the test to use the result logger for monitoring
        # and call setUp/tearDown Cycle
        self.test = test

        # set up the feedback sender
        if LIVE_FEEDBACK and options.is_distributed and options.feedback:
            trace("* Creating Feedback sender")
            self.feedback = FeedbackSender(endpoint=options.feedback_endpoint or
                                           DEFAULT_ENDPOINT)
        else:
            self.feedback = None
Example #3
0
    def __init__(self, module_file, class_name, method_name, options):
        self.module_name = os.path.basename(os.path.splitext(module_file)[0])
        self.class_name = class_name
        self.method_name = method_name
        self.options = options
        self.color = not options.no_color
        # create a unittest to get the configuration file
        test = load_unittest(self.module_name, class_name,
                             mmn_encode(method_name, 0, 0, 0), options)
        self.config_path = test._config_path
        self.result_path = test.result_path
        self.class_title = test.conf_get('main', 'title')
        self.class_description = test.conf_get('main', 'description')
        self.test_id = self.method_name
        self.test_description = test.conf_get(self.method_name, 'description',
                                              'No test description')
        self.test_url = test.conf_get('main', 'url')
        self.cycles = map(int, test.conf_getList('bench', 'cycles'))
        self.duration = test.conf_getInt('bench', 'duration')
        self.startup_delay = test.conf_getFloat('bench', 'startup_delay')
        self.cycle_time = test.conf_getFloat('bench', 'cycle_time')
        self.sleep_time = test.conf_getFloat('bench', 'sleep_time')
        self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min')
        self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max')
        self.threads = []  # Contains list of ThreadData objects
        self.last_thread_id = -1
        self.thread_creation_lock = threading.Lock()

        # setup monitoring
        monitor_hosts = []                  # list of (host, port, descr)
        if not options.is_distributed:
            hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split()
            for host in hosts:
                host = host.strip()
                monitor_hosts.append((host, test.conf_getInt(host, 'port'),
                                      test.conf_get(host, 'description', '')))
        self.monitor_hosts = monitor_hosts
        # keep the test to use the result logger for monitoring
        # and call setUp/tearDown Cycle
        self.test = test
Example #4
0
    def __init__(self, module_name, class_name, method_name, options,
                                                                cmd_args):
        """
        mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner`
        """
        # store the args. these can be passed to BenchRunner later.
        self.module_name = module_name
        self.class_name = class_name
        self.method_name = method_name
        self.options = options
        self.cmd_args = cmd_args

        self.cmd_args += " --is-distributed"

        if options.feedback:
            self.cmd_args += " --feedback"

        module = load_module(module_name)
        module_file = module.__file__
        self.tarred_tests, self.tarred_testsdir = package_tests(module_file)

        self.remote_res_dir = "/tmp/funkload-bench-sandbox/"

        test = load_unittest(self.module_name, class_name,
                             mmn_encode(method_name, 0, 0, 0), options)

        self.config_path = test._config_path
        self.result_path = test.result_path
        self.class_title = test.conf_get('main', 'title')
        self.class_description = test.conf_get('main', 'description')
        self.test_id = self.method_name
        self.test_url = test.conf_get('main', 'url')
        self.cycles = map(int, test.conf_getList('bench', 'cycles'))
        self.duration = test.conf_getInt('bench', 'duration')
        self.startup_delay = test.conf_getFloat('bench', 'startup_delay')
        self.cycle_time = test.conf_getFloat('bench', 'cycle_time')
        self.sleep_time = test.conf_getFloat('bench', 'sleep_time')
        self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min')
        self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max')
        if test.conf_get('distribute', 'channel_timeout', '', quiet=True):
            self.channel_timeout = test.conf_getFloat(
                                        'distribute', 'channel_timeout')
        else:
            self.channel_timeout = None
        self.threads = []  # Contains list of ThreadData objects
        self.last_thread_id = -1
        self.thread_creation_lock = threading.Lock()

        if options.python_bin:
            self.python_bin = options.python_bin
        else:
            self.python_bin = test.conf_get(
                'distribute', 'python_bin', 'python')

        if options.distributed_packages:
            self.distributed_packages = options.distributed_packages
        else:
            self.distributed_packages = test.conf_get(
                                                'distribute', 'packages', '')

        try:
            desc = getattr(test, self.method_name).__doc__.strip()
        except:
            desc = ""
        self.test_description = test.conf_get(self.method_name, 'description',
                                              desc)
        # make a collection output location
        if options.distributed_log_path:
            self.distribution_output = options.distributed_log_path
        elif test.conf_get('distribute', 'log_path', '', quiet=True):
            self.distribution_output = test.conf_get('distribute', 'log_path')
        else:
            raise UserWarning("log_path isn't defined in section [distribute]")

        # check if user has overridden the default funkload distro download
        # location this will be used to download funkload on the worker nodes.
        self.funkload_location = test.conf_get(
            'distribute', 'funkload_location', 'funkload')

        if not os.path.isdir(self.distribution_output):
            os.makedirs(self.distribution_output)

        # check if hosts are in options
        workers = []                  # list of (host, port, descr)
        if options.workerlist:
            for h in  options.workerlist.split(","):
                cred_host = h.split("@")
                if len(cred_host) == 1:
                    uname, pwd, host = None, None, cred_host[0]
                else:
                    cred = cred_host[0]
                    host = cred_host[1]
                    uname_pwd = cred.split(":")
                    if len(uname_pwd) == 1:
                        uname, pwd = uname_pwd[0], None
                    else:
                        uname, pwd = uname_pwd

                workers.append({
                    "name": host,
                    "host": host,
                    "password": pwd,
                    "username": uname,
                    "channel_timeout": self.channel_timeout})
        else:
            hosts = test.conf_get('workers', 'hosts', '', quiet=True).split()
            for host in hosts:
                host = host.strip()
                workers.append({
                    "name": host,
                    "host": test.conf_get(host, "host", host),
                    "password": test.conf_get(host, 'password', ''),
                    "username": test.conf_get(host, 'username', ''),
                    "key_filename": test.conf_get(host, 'ssh_key', ''),
                    "channel_timeout": self.channel_timeout})

        self._workers = []
        [self._workers.append(SSHDistributor(**w)) for w in workers]
        self._worker_results = {}
        trace(str(self))

        # setup monitoring
        monitor_hosts = []                  # list of (host, port, descr)
        if not options.is_distributed:
            hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split()
            for host in sorted(hosts):
                name = host
                host = test.conf_get(host,'host',host.strip())
                monitor_hosts.append((name, host, test.conf_getInt(name, 'port'),
                                      test.conf_get(name, 'description', '')))
        self.monitor_hosts = monitor_hosts
        # keep the test to use the result logger for monitoring
        # and call setUp/tearDown Cycle
        self.test = test

        # start the feedback receiver
        if LIVE_FEEDBACK and options.feedback:
            trace("* Starting the Feedback Publisher\n")
            self.feedback = FeedbackPublisher(
                    endpoint=options.feedback_endpoint or DEFAULT_ENDPOINT,
                    pubsub_endpoint=options.feedback_pubsub_endpoint or
                    DEFAULT_PUBSUB
                    )
            self.feedback.start()
        else:
            self.feedback = None
Example #5
0
    def __init__(self, module_file, class_name, method_name, options, cmd_args):
        """
        mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner`
        """
        # store the args. these can be passed to BenchRunner later.
        self.module_file, self.class_name, self.method_name, self.options, self.cmd_args = (
            module_file,
            class_name,
            method_name,
            options,
            cmd_args,
        )
        self.cmd_args += " --is-distributed"
        self.module_name = os.path.basename(os.path.splitext(module_file)[0])
        self.tarred_tests, self.tarred_testsdir = package_tests(module_file)
        self.remote_res_dir = "/tmp/funkload-bench-sandbox/"

        test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options)

        self.config_path = test._config_path
        self.result_path = test.result_path
        self.class_title = test.conf_get("main", "title")
        self.class_description = test.conf_get("main", "description")
        self.test_id = self.method_name
        self.test_url = test.conf_get("main", "url")
        self.cycles = map(int, test.conf_getList("bench", "cycles"))
        self.duration = test.conf_getInt("bench", "duration")
        self.startup_delay = test.conf_getFloat("bench", "startup_delay")
        self.cycle_time = test.conf_getFloat("bench", "cycle_time")
        self.sleep_time = test.conf_getFloat("bench", "sleep_time")
        self.sleep_time_min = test.conf_getFloat("bench", "sleep_time_min")
        self.sleep_time_max = test.conf_getFloat("bench", "sleep_time_max")
        self.threads = []  # Contains list of ThreadData objects
        self.last_thread_id = -1
        self.thread_creation_lock = threading.Lock()
        try:
            desc = getattr(test, self.method_name).__doc__.strip()
        except:
            desc = ""
        self.test_description = test.conf_get(self.method_name, "description", desc)
        # make a collection output location
        if test.conf_get("distribute", "log_path", "", quiet=True):
            self.distribution_output = test.conf_get("distribute", "log_path")
        else:
            raise UserWarning("log_path isn't defined in section [distribute]")

        # check if user has overridden the default funkload distro download location
        # this will be used to download funkload on the worker nodes.
        self.funkload_location = test.conf_get("distribute", "funkload_location", "funkload")

        if not os.path.isdir(self.distribution_output):
            os.makedirs(self.distribution_output)

        # check if hosts are in options
        expr = re.compile("((\w+)(:[.*]*)@)*([\w\.]+)")
        workers = []  # list of (host, port, descr)
        if options.workerlist:
            for h in options.workerlist.split(","):
                cred_host = h.split("@")
                if len(cred_host) == 1:
                    uname, pwd, host = None, None, cred_host[0]
                else:
                    cred = cred_host[0]
                    host = cred_host[1]
                    uname_pwd = cred.split(":")
                    if len(uname_pwd) == 1:
                        uname, pwd = uname_pwd[0], None
                    else:
                        uname, pwd = uname_pwd

                workers.append({"host": host, "password": pwd, "username": uname})
        else:
            for host in test.conf_get("workers", "hosts", "", quiet=True).split():
                host = host.strip()
                workers.append(
                    {
                        "host": host,
                        "password": test.conf_get(host, "password", ""),
                        "username": test.conf_get(host, "username", ""),
                    }
                )

        self._workers = []
        [self._workers.append(SSHDistributor(**w)) for w in workers]
        self._worker_results = {}
        trace(str(self))