def __init__(self, module_name, class_name, method_name, options, cmd_args): """ mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner` """ # store the args. these can be passed to BenchRunner later. self.module_name = module_name self.class_name = class_name self.method_name = method_name self.options = options self.cmd_args = cmd_args self.cmd_args += " --is-distributed" if options.feedback: self.cmd_args += " --feedback" module = load_module(module_name) module_file = module.__file__ self.tarred_tests, self.tarred_testsdir = package_tests(module_file) self.remote_res_dir = "/tmp/funkload-bench-sandbox/" test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options) self.config_path = test._config_path self.result_path = test.result_path self.class_title = test.conf_get('main', 'title') self.class_description = test.conf_get('main', 'description') self.test_id = self.method_name self.test_url = test.conf_get('main', 'url') self.cycles = map(int, test.conf_getList('bench', 'cycles')) self.duration = test.conf_getInt('bench', 'duration') self.startup_delay = test.conf_getFloat('bench', 'startup_delay') self.cycle_time = test.conf_getFloat('bench', 'cycle_time') self.sleep_time = test.conf_getFloat('bench', 'sleep_time') self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min') self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max') if test.conf_get('distribute', 'channel_timeout', '', quiet=True): self.channel_timeout = test.conf_getFloat( 'distribute', 'channel_timeout') else: self.channel_timeout = None self.threads = [] # Contains list of ThreadData objects self.last_thread_id = -1 self.thread_creation_lock = threading.Lock() if options.python_bin: self.python_bin = options.python_bin else: self.python_bin = test.conf_get( 'distribute', 'python_bin', 'python') if options.distributed_packages: self.distributed_packages = options.distributed_packages else: self.distributed_packages = test.conf_get( 'distribute', 'packages', '') try: desc = getattr(test, self.method_name).__doc__.strip() except: desc = "" self.test_description = test.conf_get(self.method_name, 'description', desc) # make a collection output location if options.distributed_log_path: self.distribution_output = options.distributed_log_path elif test.conf_get('distribute', 'log_path', '', quiet=True): self.distribution_output = test.conf_get('distribute', 'log_path') else: raise UserWarning("log_path isn't defined in section [distribute]") # check if user has overridden the default funkload distro download # location this will be used to download funkload on the worker nodes. self.funkload_location = test.conf_get( 'distribute', 'funkload_location', 'funkload') if not os.path.isdir(self.distribution_output): os.makedirs(self.distribution_output) # check if hosts are in options workers = [] # list of (host, port, descr) if options.workerlist: for h in options.workerlist.split(","): cred_host = h.split("@") if len(cred_host) == 1: uname, pwd, host = None, None, cred_host[0] else: cred = cred_host[0] host = cred_host[1] uname_pwd = cred.split(":") if len(uname_pwd) == 1: uname, pwd = uname_pwd[0], None else: uname, pwd = uname_pwd workers.append({ "name": host, "host": host, "password": pwd, "username": uname, "channel_timeout": self.channel_timeout}) else: hosts = test.conf_get('workers', 'hosts', '', quiet=True).split() for host in hosts: host = host.strip() workers.append({ "name": host, "host": test.conf_get(host, "host", host), "password": test.conf_get(host, 'password', ''), "username": test.conf_get(host, 'username', ''), "key_filename": test.conf_get(host, 'ssh_key', ''), "channel_timeout": self.channel_timeout}) self._workers = [] [self._workers.append(SSHDistributor(**w)) for w in workers] self._worker_results = {} trace(str(self)) # setup monitoring monitor_hosts = [] # list of (host, port, descr) if not options.is_distributed: hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split() for host in sorted(hosts): name = host host = test.conf_get(host,'host',host.strip()) monitor_hosts.append((name, host, test.conf_getInt(name, 'port'), test.conf_get(name, 'description', ''))) self.monitor_hosts = monitor_hosts # keep the test to use the result logger for monitoring # and call setUp/tearDown Cycle self.test = test # start the feedback receiver if LIVE_FEEDBACK and options.feedback: trace("* Starting the Feedback Publisher\n") self.feedback = FeedbackPublisher( endpoint=options.feedback_endpoint or DEFAULT_ENDPOINT, pubsub_endpoint=options.feedback_pubsub_endpoint or DEFAULT_PUBSUB ) self.feedback.start() else: self.feedback = None
def __init__(self, module_name, class_name, method_name, options, cmd_args): """ mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner` """ # store the args. these can be passed to BenchRunner later. self.module_name = module_name self.class_name = class_name self.method_name = method_name self.options = options self.cmd_args = cmd_args wanted = lambda x: ('--distribute' not in x) and ('discover' != x) self.cmd_args = filter(wanted, self.cmd_args) self.cmd_args.append("--is-distributed") # ? Won't this double the --feedback option? if options.feedback: self.cmd_args.append("--feedback") module = load_module(module_name) module_file = module.__file__ self.tarred_tests, self.tarred_testsdir = package_tests(module_file) self.remote_res_dir = "/tmp/funkload-bench-sandbox/" test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options) self.config_path = test._config_path self.result_path = test.result_path self.class_title = test.conf_get('main', 'title') self.class_description = test.conf_get('main', 'description') self.test_id = self.method_name self.test_url = test.conf_get('main', 'url') self.cycles = map(int, test.conf_getList('bench', 'cycles')) self.duration = test.conf_getInt('bench', 'duration') self.startup_delay = test.conf_getFloat('bench', 'startup_delay') self.cycle_time = test.conf_getFloat('bench', 'cycle_time') self.sleep_time = test.conf_getFloat('bench', 'sleep_time') self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min') self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max') if test.conf_get('distribute', 'channel_timeout', '', quiet=True): self.channel_timeout = test.conf_getFloat('distribute', 'channel_timeout') else: self.channel_timeout = None self.threads = [] # Contains list of ThreadData objects self.last_thread_id = -1 self.thread_creation_lock = threading.Lock() if options.python_bin: self.python_bin = options.python_bin else: self.python_bin = test.conf_get('distribute', 'python_bin', 'python') if options.distributed_packages: self.distributed_packages = options.distributed_packages else: self.distributed_packages = test.conf_get('distribute', 'packages', '') try: desc = getattr(test, self.method_name).__doc__.strip() except: desc = "" self.test_description = test.conf_get(self.method_name, 'description', desc) # make a collection output location if options.distributed_log_path: self.distribution_output = options.distributed_log_path elif test.conf_get('distribute', 'log_path', '', quiet=True): self.distribution_output = test.conf_get('distribute', 'log_path') else: raise UserWarning("log_path isn't defined in section [distribute]") # check if user has overridden the default funkload distro download # location this will be used to download funkload on the worker nodes. self.funkload_location = test.conf_get('distribute', 'funkload_location', 'funkload') if not os.path.isdir(self.distribution_output): os.makedirs(self.distribution_output) # check if hosts are in options workers = [] # list of (host, port, descr) if options.workerlist: for h in options.workerlist.split(","): cred_host = h.split("@") if len(cred_host) == 1: uname, pwd, host = None, None, cred_host[0] else: cred = cred_host[0] host = cred_host[1] uname_pwd = cred.split(":") if len(uname_pwd) == 1: uname, pwd = uname_pwd[0], None else: uname, pwd = uname_pwd worker = { "name": host.replace(":", "_"), "host": host, "password": pwd, "username": uname, "channel_timeout": self.channel_timeout } if options.distributed_key_filename: worker['key_filename'] = options.distributed_key_filename workers.append(worker) else: hosts = test.conf_get('workers', 'hosts', '', quiet=True).split() for host in hosts: host = host.strip() if options.distributed_key_filename: key_filename = options.distributed_key_filename else: key_filename = test.conf_get(host, 'ssh_key', '') workers.append({ "name": host.replace(":", "_"), "host": test.conf_get(host, "host", host), "password": test.conf_get(host, 'password', ''), "username": test.conf_get(host, 'username', ''), "key_filename": key_filename, "channel_timeout": self.channel_timeout }) self._workers = [] [self._workers.append(SSHDistributor(**w)) for w in workers] self._worker_results = {} trace(str(self)) # setup monitoring monitor_hosts = [] # list of (host, port, descr) if not options.is_distributed: hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split() for host in sorted(hosts): name = host host = test.conf_get(host, 'host', host.strip()) monitor_hosts.append( (name, host, test.conf_getInt(name, 'port'), test.conf_get(name, 'description', ''))) self.monitor_hosts = monitor_hosts # keep the test to use the result logger for monitoring # and call setUp/tearDown Cycle self.test = test # start the feedback receiver if LIVE_FEEDBACK and options.feedback: trace("* Starting the Feedback Publisher\n") self.feedback = FeedbackPublisher( endpoint=options.feedback_endpoint or DEFAULT_ENDPOINT, pubsub_endpoint=options.feedback_pubsub_endpoint or DEFAULT_PUBSUB, handler=_print_rt) self.feedback.start() else: self.feedback = None
def __init__(self, module_file, class_name, method_name, options, cmd_args): """ mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner` """ # store the args. these can be passed to BenchRunner later. self.module_file, self.class_name, self.method_name, self.options, self.cmd_args = ( module_file, class_name, method_name, options, cmd_args, ) self.cmd_args += " --is-distributed" self.module_name = os.path.basename(os.path.splitext(module_file)[0]) self.tarred_tests, self.tarred_testsdir = package_tests(module_file) self.remote_res_dir = "/tmp/funkload-bench-sandbox/" test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options) self.config_path = test._config_path self.result_path = test.result_path self.class_title = test.conf_get("main", "title") self.class_description = test.conf_get("main", "description") self.test_id = self.method_name self.test_url = test.conf_get("main", "url") self.cycles = map(int, test.conf_getList("bench", "cycles")) self.duration = test.conf_getInt("bench", "duration") self.startup_delay = test.conf_getFloat("bench", "startup_delay") self.cycle_time = test.conf_getFloat("bench", "cycle_time") self.sleep_time = test.conf_getFloat("bench", "sleep_time") self.sleep_time_min = test.conf_getFloat("bench", "sleep_time_min") self.sleep_time_max = test.conf_getFloat("bench", "sleep_time_max") self.threads = [] # Contains list of ThreadData objects self.last_thread_id = -1 self.thread_creation_lock = threading.Lock() try: desc = getattr(test, self.method_name).__doc__.strip() except: desc = "" self.test_description = test.conf_get(self.method_name, "description", desc) # make a collection output location if test.conf_get("distribute", "log_path", "", quiet=True): self.distribution_output = test.conf_get("distribute", "log_path") else: raise UserWarning("log_path isn't defined in section [distribute]") # check if user has overridden the default funkload distro download location # this will be used to download funkload on the worker nodes. self.funkload_location = test.conf_get("distribute", "funkload_location", "funkload") if not os.path.isdir(self.distribution_output): os.makedirs(self.distribution_output) # check if hosts are in options expr = re.compile("((\w+)(:[.*]*)@)*([\w\.]+)") workers = [] # list of (host, port, descr) if options.workerlist: for h in options.workerlist.split(","): cred_host = h.split("@") if len(cred_host) == 1: uname, pwd, host = None, None, cred_host[0] else: cred = cred_host[0] host = cred_host[1] uname_pwd = cred.split(":") if len(uname_pwd) == 1: uname, pwd = uname_pwd[0], None else: uname, pwd = uname_pwd workers.append({"host": host, "password": pwd, "username": uname}) else: for host in test.conf_get("workers", "hosts", "", quiet=True).split(): host = host.strip() workers.append( { "host": host, "password": test.conf_get(host, "password", ""), "username": test.conf_get(host, "username", ""), } ) self._workers = [] [self._workers.append(SSHDistributor(**w)) for w in workers] self._worker_results = {} trace(str(self))
def __init__(self, module_file, class_name, method_name, options, cmd_args): """ mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner` """ # store the args. these can be passed to BenchRunner later. self.module_file = module_file self.class_name = class_name self.method_name = method_name self.options = options self.cmd_args = cmd_args self.cmd_args += " --is-distributed" self.module_name = os.path.basename(os.path.splitext(module_file)[0]) self.tarred_tests, self.tarred_testsdir = package_tests(module_file) self.remote_res_dir = "/tmp/funkload-bench-sandbox/" test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options) self.config_path = test._config_path self.result_path = test.result_path self.class_title = test.conf_get('main', 'title') self.class_description = test.conf_get('main', 'description') self.test_id = self.method_name self.test_url = test.conf_get('main', 'url') self.cycles = map(int, test.conf_getList('bench', 'cycles')) self.duration = test.conf_getInt('bench', 'duration') self.startup_delay = test.conf_getFloat('bench', 'startup_delay') self.cycle_time = test.conf_getFloat('bench', 'cycle_time') self.sleep_time = test.conf_getFloat('bench', 'sleep_time') self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min') self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max') self.threads = [] # Contains list of ThreadData objects self.last_thread_id = -1 self.thread_creation_lock = threading.Lock() if options.python_bin: self.python_bin = options.python_bin else: self.python_bin = test.conf_get( 'distribute', 'python_bin', 'python') if options.distributed_packages: self.distributed_packages = options.distributed_packages else: self.distributed_packages = test.conf_get( 'distribute', 'packages', '') try: desc = getattr(test, self.method_name).__doc__.strip() except: desc = "" self.test_description = test.conf_get(self.method_name, 'description', desc) # make a collection output location if test.conf_get('distribute', 'log_path', '', quiet=True): self.distribution_output = test.conf_get('distribute', 'log_path') else: raise UserWarning("log_path isn't defined in section [distribute]") # check if user has overridden the default funkload distro download # location this will be used to download funkload on the worker nodes. self.funkload_location = test.conf_get( 'distribute', 'funkload_location', 'funkload') if not os.path.isdir(self.distribution_output): os.makedirs(self.distribution_output) # check if hosts are in options workers = [] # list of (host, port, descr) if options.workerlist: for h in options.workerlist.split(","): cred_host = h.split("@") if len(cred_host) == 1: uname, pwd, host = None, None, cred_host[0] else: cred = cred_host[0] host = cred_host[1] uname_pwd = cred.split(":") if len(uname_pwd) == 1: uname, pwd = uname_pwd[0], None else: uname, pwd = uname_pwd workers.append({ "host": host, "password": pwd, "username": uname}) else: hosts = test.conf_get('workers', 'hosts', '', quiet=True).split() for host in hosts: host = host.strip() workers.append({ "host": host, "password": test.conf_get(host, 'password', ''), "username": test.conf_get(host, 'username', '')}) self._workers = [] [self._workers.append(SSHDistributor(**w)) for w in workers] self._worker_results = {} trace(str(self)) # setup monitoring monitor_hosts = [] # list of (host, port, descr) if not options.is_distributed: hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split() for host in sorted(hosts): host = host.strip() monitor_hosts.append((host, test.conf_getInt(host, 'port'), test.conf_get(host, 'description', ''))) self.monitor_hosts = monitor_hosts # keep the test to use the result logger for monitoring # and call setUp/tearDown Cycle self.test = test