def PrintEnvData(self, prefix=''):
print Colors.Yellow(prefix + 'dmc binary path: %s' % self.dmcBinaryPath)
print Colors.Yellow(prefix + 'dmc env: %s' % str(self.env))
print Colors.Yellow(prefix + 'dir path: %s' % str(self.directory))
print Colors.Yellow(prefix + 'log file name: %s' % str(self.DMC_LOG_FILE_NAME))
if self.libPath:
print Colors.Yellow(prefix + 'lib path: %s' % str(self.libPath))
def printEnvData(self, prefix=''):
print(Colors.Yellow(prefix + 'info:'))
print(Colors.Yellow(prefix + '\tshards count:%d' % len(self.shards)))
if self.modulePath:
print(Colors.Yellow(prefix + '\tzip module path:%s' % self.modulePath))
if self.moduleArgs:
print(Colors.Yellow(prefix + '\tmodule args:%s' % self.moduleArgs))
for i, shard in enumerate(self.shards):
print(Colors.Yellow(prefix + 'shard: %d' % (i + 1)))
shard.printEnvData(prefix + '\t')
def takeEnvDown(self, fullShutDown=False):
if not self.currEnv:
return
needShutdown = True
if self.args.env_reuse and not fullShutDown:
try:
self.currEnv.flush()
needShutdown = False
except Exception as e:
self.currEnv.stop()
self.handleFailure(exception=e,
testname='[env dtor]',
env=self.currEnv)
if needShutdown:
if self.currEnv.isUp():
self.currEnv.flush()
self.currEnv.stop()
if self.require_clean_exit and self.currEnv and not self.currEnv.checkExitCode(
):
print(Colors.Bred('\tRedis did not exit cleanly'))
self.addFailure(self.currEnv.testName,
['redis process failure'])
if self.args.check_exitcode:
raise Exception('Process exited dirty')
self.currEnv = None
def download_binaries(self, binariesName='binaries.tar'):
print(Colors.Yellow('installing enterprise binaries'))
print(Colors.Yellow('creating RLTest working dir: %s' % self.root))
try:
shutil.rmtree(self.root)
os.makedirs(self.root)
except Exception:
pass
print(Colors.Yellow('download binaries'))
args = ['wget', self.url, '-O', os.path.join(self.root, binariesName)]
process = subprocess.Popen(args=args,
stdout=sys.stdout,
stderr=sys.stdout)
process.wait()
if process.poll() != 0:
raise Exception('failed to download enterprise binaries from s3')
print(Colors.Yellow('extracting binaries'))
args = [
'tar', '-xvf',
os.path.join(self.root, binariesName), '--directory', self.root,
self.debname
]
process = subprocess.Popen(args=args,
stdout=sys.stdout,
stderr=sys.stdout)
process.wait()
if process.poll() != 0:
raise Exception('failed to extract binaries to %s' %
self.self.root)
# TODO: Support centos that does not have dpkg command
args = ['dpkg', '-x', os.path.join(self.root, self.debname), self.root]
process = subprocess.Popen(args=args,
stdout=sys.stdout,
stderr=sys.stdout)
process.wait()
if process.poll() != 0:
raise Exception('failed to extract binaries to %s' %
self.self.root)
print(Colors.Yellow('finished installing enterprise binaries'))
def printEnvData(self, prefix=''):
print Colors.Yellow(prefix + 'bdb info:')
print Colors.Yellow(prefix + '\tlistening port:%d' % self.DMC_PORT)
print Colors.Yellow(prefix + '\tshards count:%d' % len(self.shards))
if self.modulePath:
print Colors.Yellow(prefix +
'\tzip module path:%s' % self.modulePath)
if self.moduleSoFilePath:
print Colors.Yellow(prefix +
'\tso module path:%s' % self.moduleSoFilePath)
if self.moduleArgs:
print Colors.Yellow(prefix + '\tmodule args:%s' % self.moduleArgs)
for i, shard in enumerate(self.shards):
print Colors.Yellow(prefix + 'shard: %d' % (i + 1))
shard.printEnvData(prefix + '\t')
print Colors.Yellow(prefix + 'ccs:')
self.ccs.PrintEnvData(prefix + '\t')
print Colors.Yellow(prefix + 'dmc:')
self.dmc.PrintEnvData(prefix + '\t')
def PrintEnvData(self, prefix=''):
print Colors.Yellow(prefix + 'pid: %d' % self.process.pid)
print Colors.Yellow(prefix + 'unix socket: %s' %
self.CCS_UNIX_SOCKET_DEFAULT_PATH)
print Colors.Yellow(prefix + 'binary path: %s' % self.redisBinaryPath)
if self.directory:
print Colors.Yellow(prefix + 'db dir path: %s' % (self.directory))
print Colors.Yellow(prefix + 'rdb file name: %s' %
(self.CCS_DB_RDB_FILE_NAME))
print Colors.Yellow(prefix + 'log file name: %s' %
(self.CCS_LOG_FILE_NAME))
if self.libPath:
print Colors.Yellow(prefix + 'lib path: %s' % (self.libPath))
def takeEnvDown(self, fullShutDown=False):
if not self.currEnv:
return
needShutdown = True
if self.args.env_reuse and not fullShutDown:
try:
self.currEnv.flush()
needShutdown = False
except Exception as e:
self.handleFailure(exception=e,
testname='[env dtor]',
env=self.currEnv)
if needShutdown:
self.currEnv.stop()
if self.args.use_valgrind and self.currEnv and not self.currEnv.checkExitCode(
):
print Colors.Bred('\tvalgrind check failure')
self.addFailure(self.currEnv.testName, ['<Valgrind Failure>'])
self.currEnv = None
def printException(self, err):
msg = 'Unhandled exception: {}'.format(err)
print('\t' + Colors.Bred(msg))
traceback.print_exc(file=sys.stdout)
def __init__(self):
# adding the current path to sys.path for test import puspused
sys.path.append(os.getcwd())
configFilePath = './%s' % RLTest_CONFIG_FILE_NAME
if os.path.exists(configFilePath):
args = [
'%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME)
] + sys.argv[1:]
else:
args = sys.argv[1:]
self.args = parser.parse_args(args=args)
if self.args.interactive_debugger:
if self.args.env != 'oss' and self.args.env != 'enterprise':
print(
Colors.Bred(
'interactive debugger can only be used on non cluster env'
))
sys.exit(1)
if self.args.use_valgrind:
print(
Colors.Bred(
'can not use valgrind with interactive debugger'))
sys.exit(1)
if self.args.use_slaves:
print(
Colors.Bred(
'can not use slaves with interactive debugger'))
sys.exit(1)
self.args.no_output_catch = True
self.args.stop_on_failure = True
if self.args.download_enterprise_binaries:
br = binaryrepo.BinaryRepository()
br.download_binaries()
if self.args.clear_logs:
try:
shutil.rmtree(self.args.log_dir)
except Exception as e:
print(e)
debugger = None
if self.args.use_valgrind:
vg_debugger = debuggers.Valgrind(
suppressions=self.args.vg_suppressions)
if self.args.vg_no_leakcheck:
vg_debugger.leakcheck = False
if self.args.no_output_catch or self.args.vg_verbose:
vg_debugger.verbose = True
debugger = vg_debugger
elif self.args.interactive_debugger:
debugger = debuggers.DefaultInteractiveDebugger()
elif self.args.debugger:
debugger = debuggers.GenericInteractiveDebugger(self.args.debugger)
Env.defaultModule = self.args.module
Env.defaultModuleArgs = self.args.module_args
Env.defaultEnv = self.args.env
Env.defaultOssRedisBinary = self.args.oss_redis_path
Env.defaultVerbose = self.args.verbose
Env.defaultLogDir = self.args.log_dir
Env.defaultUseSlaves = self.args.use_slaves
Env.defaultShardsCount = self.args.shards_count
Env.defaultProxyBinaryPath = self.args.proxy_binary_path
Env.defaultEnterpriseRedisBinaryPath = self.args.enterprise_redis_path
Env.defaultEnterpriseLibsPath = self.args.enterprise_lib_path
Env.defaultUseAof = self.args.use_aof
Env.defaultDebug = self.args.debug
Env.defaultDebugPrints = self.args.debug_print
Env.defaultNoCatch = self.args.no_output_catch
Env.defaultDebugger = debugger
Env.defaultExitOnFailure = self.args.exit_on_failure
self.tests = []
self.testsFailed = []
self.currEnv = None
self.loader = TestLoader()
if self.args.test:
self.loader.load_spec(self.args.test)
else:
self.loader.scan_dir(os.getcwd())
if self.args.collect_only:
self.loader.print_tests()
sys.exit(0)
def __init__(self):
# adding the current path to sys.path for test import puspused
sys.path.append(os.getcwd())
configFilePath = './%s' % RLTest_CONFIG_FILE_NAME
if os.path.exists(configFilePath):
args = ['%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME)] + sys.argv[1:]
else:
args = sys.argv[1:]
self.args = parser.parse_args(args=args)
if self.args.interactive_debugger:
if self.args.env != 'oss' and self.args.env != 'enterprise':
print(Colors.Bred('interactive debugger can only be used on non cluster env'))
sys.exit(1)
if self.args.use_valgrind:
print(Colors.Bred('can not use valgrind with interactive debugger'))
sys.exit(1)
if self.args.use_slaves:
print(Colors.Bred('can not use slaves with interactive debugger'))
sys.exit(1)
self.args.no_output_catch = True
self.args.stop_on_failure = True
if self.args.download_enterprise_binaries:
br = binaryrepo.BinaryRepository()
br.download_binaries()
if self.args.clear_logs:
try:
shutil.rmtree(self.args.log_dir)
except Exception as e:
print(e)
debugger = None
if self.args.debugger:
if self.args.env.endswith('existing-env'):
print(Colors.Bred('can not use debug with existing-env'))
sys.exit(1)
debuggers.set_interactive_debugger(self.args.debugger)
self.args.interactive_debugger = True
if self.args.use_valgrind:
if self.args.env.endswith('existing-env'):
print(Colors.Bred('can not use valgrind with existing-env'))
sys.exit(1)
vg_debugger = debuggers.Valgrind(suppressions=self.args.vg_suppressions)
if self.args.vg_no_leakcheck:
vg_debugger.leakcheck = False
if self.args.no_output_catch or self.args.vg_verbose:
vg_debugger.verbose = True
debugger = vg_debugger
elif self.args.interactive_debugger:
debugger = debuggers.default_interactive_debugger
if self.args.env.endswith('existing-env'):
# when running on existing env we always reuse it
self.args.env_reuse = True
Defaults.module = self.args.module
Defaults.module_args = ' '.join(self.args.module_args)
Defaults.env = self.args.env
Defaults.binary = self.args.oss_redis_path
Defaults.verbose = self.args.verbose
Defaults.logdir = self.args.log_dir
Defaults.use_slaves = self.args.use_slaves
Defaults.num_shards = self.args.shards_count
Defaults.shards_ports = self.args.shards_ports.split(',') if self.args.shards_ports is not None else None
Defaults.cluster_address = self.args.cluster_address
Defaults.cluster_credentials = self.args.cluster_credentials
Defaults.internal_password = self.args.internal_password
Defaults.proxy_binary = self.args.proxy_binary_path
Defaults.re_binary = self.args.enterprise_redis_path
Defaults.re_libdir = self.args.enterprise_lib_path
Defaults.use_aof = self.args.use_aof
Defaults.debug_pause = self.args.debug
Defaults.debug_print = self.args.debug_print
Defaults.no_capture_output = self.args.no_output_catch
Defaults.debugger = debugger
Defaults.exit_on_failure = self.args.exit_on_failure
Defaults.external_addr = self.args.existing_env_addr
Defaults.use_unix = self.args.unix
Defaults.randomize_ports = self.args.randomize_ports
if Defaults.use_unix and Defaults.use_slaves:
raise Exception('Cannot use unix sockets with slaves')
self.tests = []
self.testsFailed = []
self.currEnv = None
self.loader = TestLoader()
if self.args.test:
self.loader.load_spec(self.args.test)
else:
self.loader.scan_dir(os.getcwd())
if self.args.collect_only:
self.loader.print_tests()
sys.exit(0)
if self.args.use_valgrind or self.args.check_exitcode:
self.require_clean_exit = True
else:
self.require_clean_exit = False
def printPass(self):
print('\t' + Colors.Green('[PASS]'))
def printFail(self):
print('\t' + Colors.Bred('[FAIL]'))
def printFail(self, name):
print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Bred('[FAIL]')))
def printSkip(self, name):
print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Green('[SKIP]')))
def __init__(self):
# adding the current path to sys.path for test import puspused
sys.path.append(os.getcwd())
configFilePath = './%s' % RLTest_CONFIG_FILE_NAME
if os.path.exists(configFilePath):
args = [
'%s%s' % (RLTest_CONFIG_FILE_PREFIX, RLTest_CONFIG_FILE_NAME)
] + sys.argv[1:]
else:
args = sys.argv[1:]
self.args = parser.parse_args(args=args)
if self.args.version:
print(Colors.Green('RLTest version {}'.format(__version__)))
sys.exit(0)
if self.args.interactive_debugger:
if self.args.env != 'oss' and not (
self.args.env == 'oss-cluster' and Defaults.num_shards
== 1) and self.args.env != 'enterprise':
print(
Colors.Bred(
'interactive debugger can only be used on non cluster env'
))
sys.exit(1)
if self.args.use_valgrind:
print(
Colors.Bred(
'can not use valgrind with interactive debugger'))
sys.exit(1)
if self.args.use_slaves:
print(
Colors.Bred(
'can not use slaves with interactive debugger'))
sys.exit(1)
self.args.no_output_catch = True
self.args.stop_on_failure = True
if self.args.download_enterprise_binaries:
br = binaryrepo.BinaryRepository()
br.download_binaries()
if self.args.clear_logs:
if os.path.exists(self.args.log_dir):
try:
shutil.rmtree(self.args.log_dir)
except Exception as e:
print(e, file=sys.stderr)
debugger = None
if self.args.debugger:
if self.args.env.endswith('existing-env'):
print(Colors.Bred('can not use debug with existing-env'))
sys.exit(1)
debuggers.set_interactive_debugger(self.args.debugger)
self.args.interactive_debugger = True
if self.args.use_valgrind:
if self.args.env.endswith('existing-env'):
print(Colors.Bred('can not use valgrind with existing-env'))
sys.exit(1)
if self.args.vg_options is None:
self.args.vg_options = os.getenv(
'VG_OPTIONS',
'--leak-check=full --errors-for-leak-kinds=definite')
vg_debugger = debuggers.Valgrind(
options=self.args.vg_options,
suppressions=self.args.vg_suppressions,
fail_on_errors=not (self.args.vg_no_fail_on_errors),
leakcheck=not (self.args.vg_no_leakcheck))
if self.args.vg_no_leakcheck:
vg_debugger.leakcheck = False
if self.args.no_output_catch or self.args.vg_verbose:
vg_debugger.verbose = True
debugger = vg_debugger
elif self.args.interactive_debugger:
debugger = debuggers.default_interactive_debugger
if self.args.env.endswith('existing-env'):
# when running on existing env we always reuse it
self.args.env_reuse = True
# unless None, they must match in length
if self.args.module_args:
len_module_args = len(self.args.module_args)
modules = self.args.module
if type(modules) == list:
if (len(modules) != len_module_args):
print(
Colors.Bred(
'Using `--module` multiple time implies that you specify the `--module-args` in the the same number'
))
sys.exit(1)
Defaults.module = fix_modules(self.args.module)
Defaults.module_args = fix_modulesArgs(Defaults.module,
self.args.module_args)
Defaults.env = self.args.env
Defaults.binary = self.args.oss_redis_path
Defaults.verbose = self.args.verbose
Defaults.logdir = self.args.log_dir
Defaults.use_slaves = self.args.use_slaves
Defaults.num_shards = self.args.shards_count
Defaults.shards_ports = self.args.shards_ports.split(
',') if self.args.shards_ports is not None else None
Defaults.cluster_address = self.args.cluster_address
Defaults.cluster_credentials = self.args.cluster_credentials
Defaults.internal_password = self.args.internal_password
Defaults.proxy_binary = self.args.proxy_binary_path
Defaults.re_binary = self.args.enterprise_redis_path
Defaults.re_libdir = self.args.enterprise_lib_path
Defaults.use_aof = self.args.use_aof
Defaults.debug_pause = self.args.debug
Defaults.debug_print = self.args.debug_print
Defaults.no_capture_output = self.args.no_output_catch
Defaults.debugger = debugger
Defaults.exit_on_failure = self.args.exit_on_failure
Defaults.external_addr = self.args.existing_env_addr
Defaults.use_unix = self.args.unix
Defaults.randomize_ports = self.args.randomize_ports
Defaults.use_TLS = self.args.tls
Defaults.tls_cert_file = self.args.tls_cert_file
Defaults.tls_key_file = self.args.tls_key_file
Defaults.tls_ca_cert_file = self.args.tls_ca_cert_file
Defaults.tls_passphrase = self.args.tls_passphrase
Defaults.oss_password = self.args.oss_password
Defaults.cluster_node_timeout = self.args.cluster_node_timeout
if Defaults.use_unix and Defaults.use_slaves:
raise Exception('Cannot use unix sockets with slaves')
self.tests = []
self.testsFailed = []
self.currEnv = None
self.loader = TestLoader()
if self.args.test:
self.loader.load_spec(self.args.test)
else:
self.loader.scan_dir(os.getcwd())
if self.args.collect_only:
self.loader.print_tests()
sys.exit(0)
if self.args.use_valgrind or self.args.check_exitcode:
self.require_clean_exit = True
else:
self.require_clean_exit = False
self.parallelism = self.args.parallelism
def _runTest(self,
test,
numberOfAssertionFailed=0,
prefix='',
before=None,
after=None):
msgPrefix = test.name
print(Colors.Cyan(prefix + test.name))
if len(inspect.getargspec(
test.target).args) > 0 and not test.is_method:
try:
env = Env(testName=test.name)
except Exception as e:
self.handleFailure(exception=e,
prefix=msgPrefix,
testname=test.name)
return 0
fn = lambda: test.target(env)
before_func = (lambda: before(env)) if before is not None else None
after_func = (lambda: after(env)) if after is not None else None
else:
fn = test.target
before_func = before
after_func = after
hasException = False
try:
if before_func:
before_func()
fn()
passed = True
except unittest.SkipTest:
self.printSkip()
return 0
except TestAssertionFailure:
if self.args.exit_on_failure:
self.takeEnvDown(fullShutDown=True)
# Don't fall-through
raise
except Exception as err:
if self.args.exit_on_failure:
self.takeEnvDown(fullShutDown=True)
after = None
raise
self.handleFailure(exception=err,
prefix=msgPrefix,
testname=test.name,
env=self.currEnv)
hasException = True
passed = False
finally:
if after_func:
after_func()
numFailed = 0
if self.currEnv:
numFailed = self.currEnv.getNumberOfFailedAssertion()
if numFailed > numberOfAssertionFailed:
self.handleFailure(prefix=msgPrefix,
testname=test.name,
env=self.currEnv)
passed = False
elif not hasException:
self.addFailure(test.name, '<Environment destroyed>')
passed = False
# Handle debugger, if needed
if self.args.stop_on_failure and not passed:
if self.args.interactive_debugger:
while self.currEnv.isUp():
time.sleep(1)
raw_input('press any button to move to the next test')
if passed:
self.printPass()
return numFailed
def printSkip(self):
print('\t' + Colors.Green('[SKIP]'))
def printError(self, name):
print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Bred('[ERROR]')))
def printError(self):
print('\t' + Colors.Yellow('[ERROR]'))
def printPass(self, name):
print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Green('[PASS]')))
def execute(self):
Env.RTestInstance = self
if self.args.env_only:
Env.defaultVerbose = 2
env = Env(testName='manual test env')
if self.args.interactive_debugger:
while env.isUp():
time.sleep(1)
else:
cmd = MyCmd(env)
cmd.cmdloop()
env.stop()
return
done = 0
startTime = time.time()
if self.args.interactive_debugger and len(self.loader.tests) != 1:
print(self.tests)
print(
Colors.Bred(
'only one test can be run on interactive-debugger use -t'))
sys.exit(1)
for test in self.loader:
with self.envScopeGuard():
if test.is_class:
try:
obj = test.create_instance()
except unittest.SkipTest:
self.printSkip()
continue
except Exception as e:
self.printException(e)
self.addFailure(test.name + " [__init__]")
continue
print(Colors.Cyan(test.name))
failures = 0
before = getattr(obj, 'setUp', None)
after = getattr(obj, 'tearDown', None)
for subtest in test.get_functions(obj):
failures += self._runTest(
subtest,
prefix='\t',
numberOfAssertionFailed=failures,
before=before,
after=after)
done += 1
else:
self._runTest(test)
done += 1
self.takeEnvDown(fullShutDown=True)
endTime = time.time()
print(Colors.Bold('Test Took: %d sec' % (endTime - startTime)))
print(
Colors.Bold(
'Total Tests Run: %d, Total Tests Failed: %d, Total Tests Passed: %d'
% (done, self.getTotalFailureCount(),
done - self.getTotalFailureCount())))
if self.testsFailed:
print(Colors.Bold('Failed Tests Summary:'))
for group, failures in self.testsFailed:
print('\t' + Colors.Bold(group))
if not failures:
print('\t\t' + Colors.Bred(
'Exception raised during test execution. See logs'))
for failure in failures:
print('\t\t' + failure)
sys.exit(1)
def execute(self):
Env.RTestInstance = self
if self.args.env_only:
Defaults.verbose = 2
env = Env(testName='manual test env')
if self.args.interactive_debugger:
while env.isUp():
time.sleep(1)
else:
cmd = MyCmd(env)
cmd.cmdloop()
env.stop()
return
done = 0
startTime = time.time()
if self.args.interactive_debugger and len(self.loader.tests) != 1:
print(self.tests)
print(
Colors.Bred(
'only one test can be run on interactive-debugger use -t'))
sys.exit(1)
jobs = Queue()
for test in self.loader:
jobs.put(test, block=False)
def run_jobs(jobs, results, port):
Defaults.port = port
done = 0
while True:
try:
test = jobs.get(timeout=0.1)
except Exception as e:
break
with self.envScopeGuard():
if test.is_class:
test.initialize()
Defaults.curr_test_name = test.name
try:
obj = test.create_instance()
except unittest.SkipTest:
self.printSkip(test.name)
continue
except Exception as e:
self.printException(e)
self.addFailure(test.name + " [__init__]")
continue
failures = 0
before = getattr(obj, 'setUp', None)
after = getattr(obj, 'tearDown', None)
for subtest in test.get_functions(obj):
failures += self._runTest(
subtest,
prefix='\t',
numberOfAssertionFailed=failures,
before=before,
after=after)
done += 1
else:
self._runTest(test)
done += 1
self.takeEnvDown(fullShutDown=True)
# serialized the results back
results.put({
'done': done,
'failures': self.testsFailed
},
block=False)
results = Queue()
if self.parallelism == 1:
run_jobs(jobs, results, Defaults.port)
else:
processes = []
currPort = Defaults.port
for i in range(self.parallelism):
p = Process(target=run_jobs, args=(jobs, results, currPort))
currPort += 30 # safe distance for cluster and replicas
processes.append(p)
p.start()
for p in processes:
p.join()
# join results
while True:
try:
res = results.get(timeout=0.1)
except Exception as e:
break
done += res['done']
self.testsFailed.extend(res['failures'])
endTime = time.time()
print(Colors.Bold('Test Took: %d sec' % (endTime - startTime)))
print(
Colors.Bold(
'Total Tests Run: %d, Total Tests Failed: %d, Total Tests Passed: %d'
% (done, self.getTotalFailureCount(),
done - self.getTotalFailureCount())))
if self.testsFailed:
print(Colors.Bold('Failed Tests Summary:'))
for group, failures in self.testsFailed:
print('\t' + Colors.Bold(group))
if not failures:
print('\t\t' + Colors.Bred(
'Exception raised during test execution. See logs'))
for failure in failures:
print('\t\t' + failure)
sys.exit(1)