def takeEnvDown(self, fullShutDown=False): if not self.currEnv: return needShutdown = True if self.args.env_reuse and not fullShutDown: try: self.currEnv.flush() needShutdown = False except Exception as e: self.currEnv.stop() self.handleFailure(exception=e, testname='[env dtor]', env=self.currEnv) if needShutdown: if self.currEnv.isUp(): self.currEnv.flush() self.currEnv.stop() if self.require_clean_exit and self.currEnv and not self.currEnv.checkExitCode( ): print(Colors.Bred('\tRedis did not exit cleanly')) self.addFailure(self.currEnv.testName, ['redis process failure']) if self.args.check_exitcode: raise Exception('Process exited dirty') self.currEnv = None
def takeEnvDown(self, fullShutDown=False): if not self.currEnv: return needShutdown = True if self.args.env_reuse and not fullShutDown: try: self.currEnv.flush() needShutdown = False except Exception as e: self.handleFailure(exception=e, testname='[env dtor]', env=self.currEnv) if needShutdown: self.currEnv.stop() if self.args.use_valgrind and self.currEnv and not self.currEnv.checkExitCode( ): print Colors.Bred('\tvalgrind check failure') self.addFailure(self.currEnv.testName, ['<Valgrind Failure>']) self.currEnv = None
def execute(self): Env.RTestInstance = self if self.args.env_only: Env.defaultVerbose = 2 env = Env(testName='manual test env') if self.args.interactive_debugger: while env.isUp(): time.sleep(1) else: cmd = MyCmd(env) cmd.cmdloop() env.stop() return done = 0 startTime = time.time() if self.args.interactive_debugger and len(self.loader.tests) != 1: print(self.tests) print( Colors.Bred( 'only one test can be run on interactive-debugger use -t')) sys.exit(1) for test in self.loader: with self.envScopeGuard(): if test.is_class: try: obj = test.create_instance() except unittest.SkipTest: self.printSkip() continue except Exception as e: self.printException(e) self.addFailure(test.name + " [__init__]") continue print(Colors.Cyan(test.name)) failures = 0 before = getattr(obj, 'setUp', None) after = getattr(obj, 'tearDown', None) for subtest in test.get_functions(obj): failures += self._runTest( subtest, prefix='\t', numberOfAssertionFailed=failures, before=before, after=after) done += 1 else: self._runTest(test) done += 1 self.takeEnvDown(fullShutDown=True) endTime = time.time() print(Colors.Bold('Test Took: %d sec' % (endTime - startTime))) print( Colors.Bold( 'Total Tests Run: %d, Total Tests Failed: %d, Total Tests Passed: %d' % (done, self.getTotalFailureCount(), done - self.getTotalFailureCount()))) if self.testsFailed: print(Colors.Bold('Failed Tests Summary:')) for group, failures in self.testsFailed: print('\t' + Colors.Bold(group)) if not failures: print('\t\t' + Colors.Bred( 'Exception raised during test execution. See logs')) for failure in failures: print('\t\t' + failure) sys.exit(1)
def printFail(self): print('\t' + Colors.Bred('[FAIL]'))
def printException(self, err): msg = 'Unhandled exception: {}'.format(err) print('\t' + Colors.Bred(msg)) traceback.print_exc(file=sys.stdout)
def __init__(self): # adding the current path to sys.path for test import puspused sys.path.append(os.getcwd()) configFilePath = './%s' % RLTest_CONFIG_FILE_NAME if os.path.exists(configFilePath): args = [ '%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME) ] + sys.argv[1:] else: args = sys.argv[1:] self.args = parser.parse_args(args=args) if self.args.interactive_debugger: if self.args.env != 'oss' and self.args.env != 'enterprise': print( Colors.Bred( 'interactive debugger can only be used on non cluster env' )) sys.exit(1) if self.args.use_valgrind: print( Colors.Bred( 'can not use valgrind with interactive debugger')) sys.exit(1) if self.args.use_slaves: print( Colors.Bred( 'can not use slaves with interactive debugger')) sys.exit(1) self.args.no_output_catch = True self.args.stop_on_failure = True if self.args.download_enterprise_binaries: br = binaryrepo.BinaryRepository() br.download_binaries() if self.args.clear_logs: try: shutil.rmtree(self.args.log_dir) except Exception as e: print(e) debugger = None if self.args.use_valgrind: vg_debugger = debuggers.Valgrind( suppressions=self.args.vg_suppressions) if self.args.vg_no_leakcheck: vg_debugger.leakcheck = False if self.args.no_output_catch or self.args.vg_verbose: vg_debugger.verbose = True debugger = vg_debugger elif self.args.interactive_debugger: debugger = debuggers.DefaultInteractiveDebugger() elif self.args.debugger: debugger = debuggers.GenericInteractiveDebugger(self.args.debugger) Env.defaultModule = self.args.module Env.defaultModuleArgs = self.args.module_args Env.defaultEnv = self.args.env Env.defaultOssRedisBinary = self.args.oss_redis_path Env.defaultVerbose = self.args.verbose Env.defaultLogDir = self.args.log_dir Env.defaultUseSlaves = self.args.use_slaves Env.defaultShardsCount = self.args.shards_count Env.defaultProxyBinaryPath = self.args.proxy_binary_path Env.defaultEnterpriseRedisBinaryPath = self.args.enterprise_redis_path Env.defaultEnterpriseLibsPath = self.args.enterprise_lib_path Env.defaultUseAof = self.args.use_aof Env.defaultDebug = self.args.debug Env.defaultDebugPrints = self.args.debug_print Env.defaultNoCatch = self.args.no_output_catch Env.defaultDebugger = debugger Env.defaultExitOnFailure = self.args.exit_on_failure self.tests = [] self.testsFailed = [] self.currEnv = None self.loader = TestLoader() if self.args.test: self.loader.load_spec(self.args.test) else: self.loader.scan_dir(os.getcwd()) if self.args.collect_only: self.loader.print_tests() sys.exit(0)
def __init__(self): # adding the current path to sys.path for test import puspused sys.path.append(os.getcwd()) configFilePath = './%s' % RLTest_CONFIG_FILE_NAME if os.path.exists(configFilePath): args = ['%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME)] + sys.argv[1:] else: args = sys.argv[1:] self.args = parser.parse_args(args=args) if self.args.interactive_debugger: if self.args.env != 'oss' and self.args.env != 'enterprise': print(Colors.Bred('interactive debugger can only be used on non cluster env')) sys.exit(1) if self.args.use_valgrind: print(Colors.Bred('can not use valgrind with interactive debugger')) sys.exit(1) if self.args.use_slaves: print(Colors.Bred('can not use slaves with interactive debugger')) sys.exit(1) self.args.no_output_catch = True self.args.stop_on_failure = True if self.args.download_enterprise_binaries: br = binaryrepo.BinaryRepository() br.download_binaries() if self.args.clear_logs: try: shutil.rmtree(self.args.log_dir) except Exception as e: print(e) debugger = None if self.args.debugger: if self.args.env.endswith('existing-env'): print(Colors.Bred('can not use debug with existing-env')) sys.exit(1) debuggers.set_interactive_debugger(self.args.debugger) self.args.interactive_debugger = True if self.args.use_valgrind: if self.args.env.endswith('existing-env'): print(Colors.Bred('can not use valgrind with existing-env')) sys.exit(1) vg_debugger = debuggers.Valgrind(suppressions=self.args.vg_suppressions) if self.args.vg_no_leakcheck: vg_debugger.leakcheck = False if self.args.no_output_catch or self.args.vg_verbose: vg_debugger.verbose = True debugger = vg_debugger elif self.args.interactive_debugger: debugger = debuggers.default_interactive_debugger if self.args.env.endswith('existing-env'): # when running on existing env we always reuse it self.args.env_reuse = True Defaults.module = self.args.module Defaults.module_args = ' '.join(self.args.module_args) Defaults.env = self.args.env Defaults.binary = self.args.oss_redis_path Defaults.verbose = self.args.verbose Defaults.logdir = self.args.log_dir Defaults.use_slaves = self.args.use_slaves Defaults.num_shards = self.args.shards_count Defaults.shards_ports = self.args.shards_ports.split(',') if self.args.shards_ports is not None else None Defaults.cluster_address = self.args.cluster_address Defaults.cluster_credentials = self.args.cluster_credentials Defaults.internal_password = self.args.internal_password Defaults.proxy_binary = self.args.proxy_binary_path Defaults.re_binary = self.args.enterprise_redis_path Defaults.re_libdir = self.args.enterprise_lib_path Defaults.use_aof = self.args.use_aof Defaults.debug_pause = self.args.debug Defaults.debug_print = self.args.debug_print Defaults.no_capture_output = self.args.no_output_catch Defaults.debugger = debugger Defaults.exit_on_failure = self.args.exit_on_failure Defaults.external_addr = self.args.existing_env_addr Defaults.use_unix = self.args.unix Defaults.randomize_ports = self.args.randomize_ports if Defaults.use_unix and Defaults.use_slaves: raise Exception('Cannot use unix sockets with slaves') self.tests = [] self.testsFailed = [] self.currEnv = None self.loader = TestLoader() if self.args.test: self.loader.load_spec(self.args.test) else: self.loader.scan_dir(os.getcwd()) if self.args.collect_only: self.loader.print_tests() sys.exit(0) if self.args.use_valgrind or self.args.check_exitcode: self.require_clean_exit = True else: self.require_clean_exit = False
def execute(self): Env.RTestInstance = self if self.args.env_only: Defaults.verbose = 2 env = Env(testName='manual test env') if self.args.interactive_debugger: while env.isUp(): time.sleep(1) else: cmd = MyCmd(env) cmd.cmdloop() env.stop() return done = 0 startTime = time.time() if self.args.interactive_debugger and len(self.loader.tests) != 1: print(self.tests) print( Colors.Bred( 'only one test can be run on interactive-debugger use -t')) sys.exit(1) jobs = Queue() for test in self.loader: jobs.put(test, block=False) def run_jobs(jobs, results, port): Defaults.port = port done = 0 while True: try: test = jobs.get(timeout=0.1) except Exception as e: break with self.envScopeGuard(): if test.is_class: test.initialize() Defaults.curr_test_name = test.name try: obj = test.create_instance() except unittest.SkipTest: self.printSkip(test.name) continue except Exception as e: self.printException(e) self.addFailure(test.name + " [__init__]") continue failures = 0 before = getattr(obj, 'setUp', None) after = getattr(obj, 'tearDown', None) for subtest in test.get_functions(obj): failures += self._runTest( subtest, prefix='\t', numberOfAssertionFailed=failures, before=before, after=after) done += 1 else: self._runTest(test) done += 1 self.takeEnvDown(fullShutDown=True) # serialized the results back results.put({ 'done': done, 'failures': self.testsFailed }, block=False) results = Queue() if self.parallelism == 1: run_jobs(jobs, results, Defaults.port) else: processes = [] currPort = Defaults.port for i in range(self.parallelism): p = Process(target=run_jobs, args=(jobs, results, currPort)) currPort += 30 # safe distance for cluster and replicas processes.append(p) p.start() for p in processes: p.join() # join results while True: try: res = results.get(timeout=0.1) except Exception as e: break done += res['done'] self.testsFailed.extend(res['failures']) endTime = time.time() print(Colors.Bold('Test Took: %d sec' % (endTime - startTime))) print( Colors.Bold( 'Total Tests Run: %d, Total Tests Failed: %d, Total Tests Passed: %d' % (done, self.getTotalFailureCount(), done - self.getTotalFailureCount()))) if self.testsFailed: print(Colors.Bold('Failed Tests Summary:')) for group, failures in self.testsFailed: print('\t' + Colors.Bold(group)) if not failures: print('\t\t' + Colors.Bred( 'Exception raised during test execution. See logs')) for failure in failures: print('\t\t' + failure) sys.exit(1)
def printError(self, name): print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Bred('[ERROR]')))
def printFail(self, name): print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Bred('[FAIL]')))
def __init__(self): # adding the current path to sys.path for test import puspused sys.path.append(os.getcwd()) configFilePath = './%s' % RLTest_CONFIG_FILE_NAME if os.path.exists(configFilePath): args = [ '%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME) ] + sys.argv[1:] else: args = sys.argv[1:] self.args = parser.parse_args(args=args) if self.args.version: print(Colors.Green('RLTest version {}'.format(__version__))) sys.exit(0) if self.args.interactive_debugger: if self.args.env != 'oss' and not ( self.args.env == 'oss-cluster' and Defaults.num_shards == 1) and self.args.env != 'enterprise': print( Colors.Bred( 'interactive debugger can only be used on non cluster env' )) sys.exit(1) if self.args.use_valgrind: print( Colors.Bred( 'can not use valgrind with interactive debugger')) sys.exit(1) if self.args.use_slaves: print( Colors.Bred( 'can not use slaves with interactive debugger')) sys.exit(1) self.args.no_output_catch = True self.args.stop_on_failure = True if self.args.download_enterprise_binaries: br = binaryrepo.BinaryRepository() br.download_binaries() if self.args.clear_logs: if os.path.exists(self.args.log_dir): try: shutil.rmtree(self.args.log_dir) except Exception as e: print(e, file=sys.stderr) debugger = None if self.args.debugger: if self.args.env.endswith('existing-env'): print(Colors.Bred('can not use debug with existing-env')) sys.exit(1) debuggers.set_interactive_debugger(self.args.debugger) self.args.interactive_debugger = True if self.args.use_valgrind: if self.args.env.endswith('existing-env'): print(Colors.Bred('can not use valgrind with existing-env')) sys.exit(1) if self.args.vg_options is None: self.args.vg_options = os.getenv( 'VG_OPTIONS', '--leak-check=full --errors-for-leak-kinds=definite') vg_debugger = debuggers.Valgrind( options=self.args.vg_options, suppressions=self.args.vg_suppressions, fail_on_errors=not (self.args.vg_no_fail_on_errors), leakcheck=not (self.args.vg_no_leakcheck)) if self.args.vg_no_leakcheck: vg_debugger.leakcheck = False if self.args.no_output_catch or self.args.vg_verbose: vg_debugger.verbose = True debugger = vg_debugger elif self.args.interactive_debugger: debugger = debuggers.default_interactive_debugger if self.args.env.endswith('existing-env'): # when running on existing env we always reuse it self.args.env_reuse = True # unless None, they must match in length if self.args.module_args: len_module_args = len(self.args.module_args) modules = self.args.module if type(modules) == list: if (len(modules) != len_module_args): print( Colors.Bred( 'Using `--module` multiple time implies that you specify the `--module-args` in the the same number' )) sys.exit(1) Defaults.module = fix_modules(self.args.module) Defaults.module_args = fix_modulesArgs(Defaults.module, self.args.module_args) Defaults.env = self.args.env Defaults.binary = self.args.oss_redis_path Defaults.verbose = self.args.verbose Defaults.logdir = self.args.log_dir Defaults.use_slaves = self.args.use_slaves Defaults.num_shards = self.args.shards_count Defaults.shards_ports = self.args.shards_ports.split( ',') if self.args.shards_ports is not None else None Defaults.cluster_address = self.args.cluster_address Defaults.cluster_credentials = self.args.cluster_credentials Defaults.internal_password = self.args.internal_password Defaults.proxy_binary = self.args.proxy_binary_path Defaults.re_binary = self.args.enterprise_redis_path Defaults.re_libdir = self.args.enterprise_lib_path Defaults.use_aof = self.args.use_aof Defaults.debug_pause = self.args.debug Defaults.debug_print = self.args.debug_print Defaults.no_capture_output = self.args.no_output_catch Defaults.debugger = debugger Defaults.exit_on_failure = self.args.exit_on_failure Defaults.external_addr = self.args.existing_env_addr Defaults.use_unix = self.args.unix Defaults.randomize_ports = self.args.randomize_ports Defaults.use_TLS = self.args.tls Defaults.tls_cert_file = self.args.tls_cert_file Defaults.tls_key_file = self.args.tls_key_file Defaults.tls_ca_cert_file = self.args.tls_ca_cert_file Defaults.tls_passphrase = self.args.tls_passphrase Defaults.oss_password = self.args.oss_password Defaults.cluster_node_timeout = self.args.cluster_node_timeout if Defaults.use_unix and Defaults.use_slaves: raise Exception('Cannot use unix sockets with slaves') self.tests = [] self.testsFailed = [] self.currEnv = None self.loader = TestLoader() if self.args.test: self.loader.load_spec(self.args.test) else: self.loader.scan_dir(os.getcwd()) if self.args.collect_only: self.loader.print_tests() sys.exit(0) if self.args.use_valgrind or self.args.check_exitcode: self.require_clean_exit = True else: self.require_clean_exit = False self.parallelism = self.args.parallelism