def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if 'slave' in self.kwargs:
dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if 'slave' in self.kwargs:
argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
else:
argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
call(argv)
utils.ensureDir(dir_name)
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
# This should always be set for tests
argv += ['--setParameter', 'enableTestCommands=1']
if self.kwargs.get('small_oplog'):
argv += ["--master", "--oplogSize", "511"]
if self.kwargs.get('small_oplog_rs'):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ['--slave', '--source', 'localhost:' + str(srcport)]
if self.kwargs.get('no_journal'):
argv += ['--nojournal']
if self.kwargs.get('no_preallocj'):
argv += ['--nopreallocj']
if self.kwargs.get('auth'):
argv += ['--auth']
self.auth = True
if self.kwargs.get('use_ssl'):
argv += ['--sslOnNormalPorts',
'--sslPEMKeyFile', 'jstests/libs/server.pem',
'--sslCAFile', 'jstests/libs/ca.pem']
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def buildlogger(cmd, is_global=False):
# if the environment variable MONGO_USE_BUILDLOGGER
# is set to 'true', then wrap the command with a call
# to buildlogger.py, which sends output to the buidlogger
# machine; otherwise, return as usual.
if os.environ.get("MONGO_USE_BUILDLOGGER", "").lower().strip() == "true":
if is_global:
return [utils.find_python(), "buildscripts/buildlogger.py", "-g"] + cmd
else:
return [utils.find_python(), "buildscripts/buildlogger.py"] + cmd
return cmd
def buildlogger(cmd, is_global=False):
# if the environment variable MONGO_USE_BUILDLOGGER
# is set to 'true', then wrap the command with a call
# to buildlogger.py, which sends output to the buidlogger
# machine; otherwise, return as usual.
if os.environ.get('MONGO_USE_BUILDLOGGER', '').lower().strip() == 'true':
if is_global:
return [utils.find_python(), 'buildscripts/buildlogger.py', '-g'] + cmd
else:
return [utils.find_python(), 'buildscripts/buildlogger.py'] + cmd
return cmd
def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if "slave" in self.kwargs:
dir_name = smoke_db_prefix + "/data/db/sconsTestsSlave/"
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if "slave" in self.kwargs:
argv = [utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dir_name]
else:
argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
call(argv)
utils.ensureDir(dir_name)
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
if self.kwargs.get("small_oplog"):
argv += ["--master", "--oplogSize", "511"]
if self.kwargs.get("small_oplog_rs"):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ["--slave", "--source", "localhost:" + str(srcport)]
if self.kwargs.get("no_journal"):
argv += ["--nojournal"]
if self.kwargs.get("no_preallocj"):
argv += ["--nopreallocj"]
if self.kwargs.get("auth"):
argv += ["--auth"]
self.auth = True
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if 'slave' in self.kwargs:
dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if 'slave' in self.kwargs:
argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
else:
argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
call(argv)
utils.ensureDir(dir_name)
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
if self.kwargs.get('small_oplog'):
argv += ["--master", "--oplogSize", "511"]
if self.kwargs.get('small_oplog_rs'):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ['--slave', '--source', 'localhost:' + str(srcport)]
if self.kwargs.get('no_journal'):
argv += ['--nojournal']
if self.kwargs.get('no_preallocj'):
argv += ['--nopreallocj']
if self.kwargs.get('auth'):
argv += ['--auth']
self.auth = True
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def main():
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, set_parameters, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option(
'--mode',
dest='mode',
default='suite',
help=
'If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)'
)
# Some of our tests hard-code pathnames e.g., to execute, so until
# that changes we don't have the freedom to run from anyplace.
# parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
parser.add_option('--test-path',
dest='test_path',
default=None,
help="Path to the test executables to run, "
"currently only used for 'client' (%default)")
parser.add_option('--mongod',
dest='mongod_executable',
default=os.path.join(mongo_repo, 'mongod'),
help='Path to mongod to run (%default)')
parser.add_option('--port',
dest='mongod_port',
default="27999",
help='Port the mongod will bind to (%default)')
parser.add_option('--mongo',
dest='shell_executable',
default=os.path.join(mongo_repo, 'mongo'),
help='Path to mongo, for .js test files (%default)')
parser.add_option(
'--continue-on-failure',
dest='continue_on_failure',
action="store_true",
default=False,
help='If supplied, continue testing even after a test fails')
parser.add_option(
'--from-file',
dest='File',
help=
"Run tests/suites named in FILE, one test per line, '-' means stdin")
parser.add_option(
'--smoke-db-prefix',
dest='smoke_db_prefix',
default=smoke_db_prefix,
help="Prefix to use for the mongods' dbpaths ('%default')")
parser.add_option(
'--small-oplog',
dest='small_oplog',
default=False,
action="store_true",
help='Run tests with master/slave replication & use a small oplog')
parser.add_option(
'--small-oplog-rs',
dest='small_oplog_rs',
default=False,
action="store_true",
help='Run tests with replica set replication & use a small oplog')
parser.add_option('--nojournal',
dest='no_journal',
default=False,
action="store_true",
help='Do not turn on journaling in tests')
parser.add_option('--nopreallocj',
dest='no_preallocj',
default=False,
action="store_true",
help='Do not preallocate journal files in tests')
parser.add_option(
'--auth',
dest='auth',
default=False,
action="store_true",
help='Run standalone mongods in tests with authentication enabled')
parser.add_option(
'--authMechanism',
dest='authMechanism',
default='MONGODB-CR',
help='Use the given authentication mechanism, when --auth is used.')
parser.add_option(
'--keyFile',
dest='keyFile',
default=None,
help=
'Path to keyFile to use to run replSet and sharding tests with authentication enabled'
)
parser.add_option('--ignore',
dest='ignore_files',
default=None,
help='Pattern of files to ignore in tests')
parser.add_option(
'--only-old-fails',
dest='only_old_fails',
default=False,
action="store_true",
help='Check the failfile and only run all tests that failed last time')
parser.add_option('--reset-old-fails',
dest='reset_old_fails',
default=False,
action="store_true",
help='Clear the failfile. Do this if all tests pass')
parser.add_option('--with-cleanbb',
dest='with_cleanbb',
default=False,
action="store_true",
help='Clear database files from previous smoke.py runs')
parser.add_option(
'--dont-start-mongod',
dest='start_mongod',
default=True,
action='store_false',
help='Do not start mongod before commencing test running')
parser.add_option(
'--use-ssl',
dest='use_ssl',
default=False,
action='store_true',
help='Run mongo shell and mongod instances with SSL encryption')
parser.add_option(
'--set-parameters',
dest='set_parameters',
default="",
help=
'Adds --setParameter for each passed in items in the csv list - ex. "param1=1,param2=foo" '
)
# Buildlogger invocation from command line
parser.add_option('--buildlogger-builder',
dest='buildlogger_builder',
default=None,
action="store",
help='Set the "builder name" for buildlogger')
parser.add_option('--buildlogger-buildnum',
dest='buildlogger_buildnum',
default=None,
action="store",
help='Set the "build number" for buildlogger')
parser.add_option(
'--buildlogger-credentials',
dest='buildlogger_credentials',
default=None,
action="store",
help='Path to Python file containing buildlogger credentials')
parser.add_option(
'--buildlogger-phase',
dest='buildlogger_phase',
default=None,
action="store",
help=
'Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)'
)
global tests
(options, tests) = parser.parse_args()
set_globals(options, tests)
buildlogger_opts = (options.buildlogger_builder,
options.buildlogger_buildnum,
options.buildlogger_credentials)
if all(buildlogger_opts):
os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
if options.buildlogger_phase:
os.environ['MONGO_PHASE'] = options.buildlogger_phase
elif any(buildlogger_opts):
# some but not all of the required options were sete
raise Exception(
"you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials"
)
if options.File:
if options.File == '-':
tests = sys.stdin.readlines()
else:
f = open(options.File)
tests = f.readlines()
tests = [t.rstrip('\n') for t in tests]
if options.only_old_fails:
run_old_fails()
return
elif options.reset_old_fails:
clear_failfile()
return
# If we're in suite mode, tests is a list of names of sets of tests.
if options.mode == 'suite':
tests = expand_suites(tests)
elif options.mode == 'files':
tests = [(os.path.abspath(test), start_mongod) for test in tests]
if options.ignore_files != None:
ignore_patt = re.compile(options.ignore_files)
print "Ignoring files with pattern: ", ignore_patt
def ignore_test(test):
if ignore_patt.search(test[0]) != None:
print "Ignoring test ", test[0]
return False
else:
return True
tests = filter(ignore_test, tests)
if not tests:
print "warning: no tests specified"
return
if options.with_cleanbb:
dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
call([
utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot
])
try:
run_tests(tests)
finally:
add_to_failfile(fails, options)
f = open("smoke-last.json", "wb")
f.write(json.dumps({"results": all_test_results}))
f.close()
report()
def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if 'slave' in self.kwargs:
dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if 'slave' in self.kwargs:
argv = [
utils.find_python(), "buildscripts/cleanbb.py", '--nokill',
dir_name
]
else:
argv = [
utils.find_python(), "buildscripts/cleanbb.py", dir_name
]
call(argv)
utils.ensureDir(dir_name)
argv = [
mongod_executable, "--port",
str(self.port), "--dbpath", dir_name
]
# These parameters are alwas set for tests
# SERVER-9137 Added httpinterface parameter to keep previous behavior
argv += ['--setParameter', 'enableTestCommands=1', '--httpinterface']
if self.kwargs.get('small_oplog'):
argv += ["--master", "--oplogSize", "511"]
params = self.kwargs.get('set_parameters', None)
if params:
for p in params.split(','):
argv += ['--setParameter', p]
if self.kwargs.get('small_oplog_rs'):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ['--slave', '--source', 'localhost:' + str(srcport)]
if self.kwargs.get('no_journal'):
argv += ['--nojournal']
if self.kwargs.get('no_preallocj'):
argv += ['--nopreallocj']
if self.kwargs.get('auth'):
argv += ['--auth']
authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
if authMechanism != 'MONGODB-CR':
argv += [
'--setParameter',
'authenticationMechanisms=' + authMechanism
]
self.auth = True
if self.kwargs.get('use_ssl'):
argv += [
'--sslOnNormalPorts', '--sslPEMKeyFile',
'jstests/libs/server.pem', '--sslCAFile',
'jstests/libs/ca.pem', '--sslWeakCertificateValidation'
]
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source[
"syncedTo"]
def main():
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option(
"--mode",
dest="mode",
default="suite",
help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)',
)
# Some of our tests hard-code pathnames e.g., to execute, so until
# that changes we don't have the freedom to run from anyplace.
# parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
parser.add_option(
"--test-path",
dest="test_path",
default=None,
help="Path to the test executables to run, " "currently only used for 'client' (%default)",
)
parser.add_option(
"--mongod",
dest="mongod_executable",
default=os.path.join(mongo_repo, "mongod"),
help="Path to mongod to run (%default)",
)
parser.add_option("--port", dest="mongod_port", default="27999", help="Port the mongod will bind to (%default)")
parser.add_option(
"--mongo",
dest="shell_executable",
default=os.path.join(mongo_repo, "mongo"),
help="Path to mongo, for .js test files (%default)",
)
parser.add_option(
"--continue-on-failure",
dest="continue_on_failure",
action="store_true",
default=False,
help="If supplied, continue testing even after a test fails",
)
parser.add_option(
"--from-file", dest="File", help="Run tests/suites named in FILE, one test per line, '-' means stdin"
)
parser.add_option(
"--smoke-db-prefix",
dest="smoke_db_prefix",
default=smoke_db_prefix,
help="Prefix to use for the mongods' dbpaths ('%default')",
)
parser.add_option(
"--small-oplog",
dest="small_oplog",
default=False,
action="store_true",
help="Run tests with master/slave replication & use a small oplog",
)
parser.add_option(
"--small-oplog-rs",
dest="small_oplog_rs",
default=False,
action="store_true",
help="Run tests with replica set replication & use a small oplog",
)
parser.add_option(
"--nojournal", dest="no_journal", default=False, action="store_true", help="Do not turn on journaling in tests"
)
parser.add_option(
"--nopreallocj",
dest="no_preallocj",
default=False,
action="store_true",
help="Do not preallocate journal files in tests",
)
parser.add_option(
"--auth",
dest="auth",
default=False,
action="store_true",
help="Run standalone mongods in tests with authentication enabled",
)
parser.add_option(
"--authMechanism",
dest="authMechanism",
default="MONGO-CR",
help="Use the given authentication mechanism, when --auth is used.",
)
parser.add_option(
"--keyFile",
dest="keyFile",
default=None,
help="Path to keyFile to use to run replSet and sharding tests with authentication enabled",
)
parser.add_option("--ignore", dest="ignore_files", default=None, help="Pattern of files to ignore in tests")
parser.add_option(
"--only-old-fails",
dest="only_old_fails",
default=False,
action="store_true",
help="Check the failfile and only run all tests that failed last time",
)
parser.add_option(
"--reset-old-fails",
dest="reset_old_fails",
default=False,
action="store_true",
help="Clear the failfile. Do this if all tests pass",
)
parser.add_option(
"--with-cleanbb",
dest="with_cleanbb",
default=False,
action="store_true",
help="Clear database files from previous smoke.py runs",
)
parser.add_option(
"--dont-start-mongod",
dest="start_mongod",
default=True,
action="store_false",
help="Do not start mongod before commencing test running",
)
parser.add_option(
"--use-ssl",
dest="use_ssl",
default=False,
action="store_true",
help="Run mongo shell and mongod instances with SSL encryption",
)
# Buildlogger invocation from command line
parser.add_option(
"--buildlogger-builder",
dest="buildlogger_builder",
default=None,
action="store",
help='Set the "builder name" for buildlogger',
)
parser.add_option(
"--buildlogger-buildnum",
dest="buildlogger_buildnum",
default=None,
action="store",
help='Set the "build number" for buildlogger',
)
parser.add_option(
"--buildlogger-credentials",
dest="buildlogger_credentials",
default=None,
action="store",
help="Path to Python file containing buildlogger credentials",
)
parser.add_option(
"--buildlogger-phase",
dest="buildlogger_phase",
default=None,
action="store",
help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)',
)
global tests
(options, tests) = parser.parse_args()
set_globals(options, tests)
buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
if all(buildlogger_opts):
os.environ["MONGO_USE_BUILDLOGGER"] = "true"
os.environ["MONGO_BUILDER_NAME"] = options.buildlogger_builder
os.environ["MONGO_BUILD_NUMBER"] = options.buildlogger_buildnum
os.environ["BUILDLOGGER_CREDENTIALS"] = options.buildlogger_credentials
if options.buildlogger_phase:
os.environ["MONGO_PHASE"] = options.buildlogger_phase
elif any(buildlogger_opts):
# some but not all of the required options were sete
raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")
if options.File:
if options.File == "-":
tests = sys.stdin.readlines()
else:
f = open(options.File)
tests = f.readlines()
tests = [t.rstrip("\n") for t in tests]
if options.only_old_fails:
run_old_fails()
return
elif options.reset_old_fails:
clear_failfile()
return
# If we're in suite mode, tests is a list of names of sets of tests.
if options.mode == "suite":
tests = expand_suites(tests)
elif options.mode == "files":
tests = [(os.path.abspath(test), start_mongod) for test in tests]
if options.ignore_files != None:
ignore_patt = re.compile(options.ignore_files)
print "Ignoring files with pattern: ", ignore_patt
def ignore_test(test):
if ignore_patt.search(test[0]) != None:
print "Ignoring test ", test[0]
return False
else:
return True
tests = filter(ignore_test, tests)
if not tests:
print "warning: no tests specified"
return
if options.with_cleanbb:
dbroot = os.path.join(options.smoke_db_prefix, "data", "db")
call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])
try:
run_tests(tests)
finally:
add_to_failfile(fails, options)
f = open("smoke-last.json", "wb")
f.write(json.dumps({"results": all_test_results}))
f.close()
report()
def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if "slave" in self.kwargs:
dir_name = smoke_db_prefix + "/data/db/sconsTestsSlave/"
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if "slave" in self.kwargs:
argv = [utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dir_name]
else:
argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
call(argv)
utils.ensureDir(dir_name)
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
# This should always be set for tests
argv += ["--setParameter", "enableTestCommands=1"]
if self.kwargs.get("small_oplog"):
argv += ["--master", "--oplogSize", "511"]
if self.kwargs.get("small_oplog_rs"):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ["--slave", "--source", "localhost:" + str(srcport)]
if self.kwargs.get("no_journal"):
argv += ["--nojournal"]
if self.kwargs.get("no_preallocj"):
argv += ["--nopreallocj"]
if self.kwargs.get("auth"):
argv += ["--auth"]
authMechanism = self.kwargs.get("authMechanism", "MONGO-CR")
if authMechanism != "MONGO-CR":
argv.append("--setParameter=authenticationMechanisms=" + authMechanism)
self.auth = True
if self.kwargs.get("use_ssl"):
argv += [
"--sslOnNormalPorts",
"--sslPEMKeyFile",
"jstests/libs/server.pem",
"--sslCAFile",
"jstests/libs/ca.pem",
"--sslWeakCertificateValidation",
]
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def main():
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, set_parameters, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option('--mode', dest='mode', default='suite',
help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
# Some of our tests hard-code pathnames e.g., to execute, so until
# that changes we don't have the freedom to run from anyplace.
# parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
parser.add_option('--test-path', dest='test_path', default=None,
help="Path to the test executables to run, "
"currently only used for 'client' (%default)")
parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
help='Path to mongod to run (%default)')
parser.add_option('--port', dest='mongod_port', default="27999",
help='Port the mongod will bind to (%default)')
parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
help='Path to mongo, for .js test files (%default)')
parser.add_option('--continue-on-failure', dest='continue_on_failure',
action="store_true", default=False,
help='If supplied, continue testing even after a test fails')
parser.add_option('--from-file', dest='File',
help="Run tests/suites named in FILE, one test per line, '-' means stdin")
parser.add_option('--smoke-db-prefix', dest='smoke_db_prefix', default=smoke_db_prefix,
help="Prefix to use for the mongods' dbpaths ('%default')")
parser.add_option('--small-oplog', dest='small_oplog', default=False,
action="store_true",
help='Run tests with master/slave replication & use a small oplog')
parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False,
action="store_true",
help='Run tests with replica set replication & use a small oplog')
parser.add_option('--nojournal', dest='no_journal', default=False,
action="store_true",
help='Do not turn on journaling in tests')
parser.add_option('--nopreallocj', dest='no_preallocj', default=False,
action="store_true",
help='Do not preallocate journal files in tests')
parser.add_option('--auth', dest='auth', default=False,
action="store_true",
help='Run standalone mongods in tests with authentication enabled')
parser.add_option('--authMechanism', dest='authMechanism', default='MONGODB-CR',
help='Use the given authentication mechanism, when --auth is used.')
parser.add_option('--keyFile', dest='keyFile', default=None,
help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
parser.add_option('--ignore', dest='ignore_files', default=None,
help='Pattern of files to ignore in tests')
parser.add_option('--only-old-fails', dest='only_old_fails', default=False,
action="store_true",
help='Check the failfile and only run all tests that failed last time')
parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
action="store_true",
help='Clear the failfile. Do this if all tests pass')
parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
action="store_true",
help='Clear database files from previous smoke.py runs')
parser.add_option('--dont-start-mongod', dest='start_mongod', default=True,
action='store_false',
help='Do not start mongod before commencing test running')
parser.add_option('--use-ssl', dest='use_ssl', default=False,
action='store_true',
help='Run mongo shell and mongod instances with SSL encryption')
parser.add_option('--set-parameters', dest='set_parameters', default="",
help='Adds --setParameter for each passed in items in the csv list - ex. "param1=1,param2=foo" ')
# Buildlogger invocation from command line
parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
action="store", help='Set the "builder name" for buildlogger')
parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
action="store", help='Set the "build number" for buildlogger')
parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
action="store", help='Path to Python file containing buildlogger credentials')
parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')
parser.add_option('--report-file', dest='report_file', default=None,
action='store',
help='Path to generate detailed json report containing all test details')
global tests
(options, tests) = parser.parse_args()
set_globals(options, tests)
buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
if all(buildlogger_opts):
os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
if options.buildlogger_phase:
os.environ['MONGO_PHASE'] = options.buildlogger_phase
elif any(buildlogger_opts):
# some but not all of the required options were sete
raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")
if options.File:
if options.File == '-':
tests = sys.stdin.readlines()
else:
f = open(options.File)
tests = f.readlines()
tests = [t.rstrip('\n') for t in tests]
if options.only_old_fails:
run_old_fails()
return
elif options.reset_old_fails:
clear_failfile()
return
# If we're in suite mode, tests is a list of names of sets of tests.
if options.mode == 'suite':
tests = expand_suites(tests)
elif options.mode == 'files':
tests = [(os.path.abspath(test), start_mongod) for test in tests]
if options.ignore_files != None :
ignore_patt = re.compile( options.ignore_files )
print "Ignoring files with pattern: ", ignore_patt
def ignore_test( test ):
if ignore_patt.search( test[0] ) != None:
print "Ignoring test ", test[0]
return False
else:
return True
tests = filter( ignore_test, tests )
if not tests:
print "warning: no tests specified"
return
if options.with_cleanbb:
dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])
test_report["start"] = time.time()
try:
run_tests(tests)
finally:
add_to_failfile(fails, options)
test_report["end"] = time.time()
test_report["elapsed"] = test_report["end"] - test_report["start"]
test_report["failures"] = len(losers.keys())
if report_file:
f = open( report_file, "wb" )
f.write( json.dumps( test_report ) )
f.close()
report()
def start(self):
global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
self.ensure_test_dirs()
dir_name = smoke_db_prefix + "/data/db/sconsTests/"
self.port = int(mongod_port)
self.slave = False
if 'slave' in self.kwargs:
dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
srcport = mongod_port
self.port += 1
self.slave = True
if os.path.exists(dir_name):
if 'slave' in self.kwargs:
argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
else:
argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
call(argv)
utils.ensureDir(dir_name)
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
# These parameters are alwas set for tests
# SERVER-9137 Added httpinterface parameter to keep previous behavior
argv += ['--setParameter', 'enableTestCommands=1', '--httpinterface']
if self.kwargs.get('small_oplog'):
argv += ["--master", "--oplogSize", "511"]
params = self.kwargs.get('set_parameters', None)
if params:
for p in params.split(','): argv += ['--setParameter', p]
if self.kwargs.get('small_oplog_rs'):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
argv += ['--slave', '--source', 'localhost:' + str(srcport)]
if self.kwargs.get('no_journal'):
argv += ['--nojournal']
if self.kwargs.get('no_preallocj'):
argv += ['--nopreallocj']
if self.kwargs.get('auth'):
argv += ['--auth']
authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
if authMechanism != 'MONGODB-CR':
argv += ['--setParameter', 'authenticationMechanisms=' + authMechanism]
self.auth = True
if self.kwargs.get('use_ssl') or self.kwargs.get('use_x509'):
argv += ['--sslMode', "sslOnly",
'--sslPEMKeyFile', 'jstests/libs/server.pem',
'--sslCAFile', 'jstests/libs/ca.pem',
'--sslWeakCertificateValidation']
if self.kwargs.get('use_x509'):
argv += ['--clusterAuthMode','x509'];
self.auth = True
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
if self.auth:
self.setup_admin_user(self.port)
if self.slave:
local = Connection(port=self.port, slave_okay=True).local
synced = False
while not synced:
synced = True
for source in local.sources.find(fields=["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def main():
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, smoke_server_opts, test_path
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option('--mode', dest='mode', default='suite',
help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
# Some of our tests hard-code pathnames e.g., to execute, so until
# that changes we don't have the freedom to run from anyplace.
# parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
parser.add_option('--test-path', dest='test_path', default=None,
help="Path to the test executables to run, "
"currently only used for 'client' (%default)")
parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
help='Path to mongod to run (%default)')
parser.add_option('--port', dest='mongod_port', default="27999",
help='Port the mongod will bind to (%default)')
parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
help='Path to mongo, for .js test files (%default)')
parser.add_option('--continue-on-failure', dest='continue_on_failure',
action="store_true", default=False,
help='If supplied, continue testing even after a test fails')
parser.add_option('--from-file', dest='File',
help="Run tests/suites named in FILE, one test per line, '-' means stdin")
parser.add_option('--smoke-db-prefix', dest='smoke_db_prefix', default=smoke_db_prefix,
help="Prefix to use for the mongods' dbpaths ('%default')")
parser.add_option('--smoke-server-opts', dest='smoke_server_opts', default=smoke_server_opts,
help="Additional options for the mongod server ('%default')")
parser.add_option('--small-oplog', dest='small_oplog', default=False,
action="store_true",
help='Run tests with master/slave replication & use a small oplog')
parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False,
action="store_true",
help='Run tests with replica set replication & use a small oplog')
parser.add_option('--nojournal', dest='no_journal', default=False,
action="store_true",
help='Do not turn on journaling in tests')
parser.add_option('--nopreallocj', dest='no_preallocj', default=False,
action="store_true",
help='Do not preallocate journal files in tests')
parser.add_option('--auth', dest='auth', default=False,
action="store_true",
help='Run standalone mongods in tests with authentication enabled')
parser.add_option('--keyFile', dest='keyFile', default=None,
help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
parser.add_option('--ignore', dest='ignore_files', default=None,
help='Pattern of files to ignore in tests')
parser.add_option('--only-old-fails', dest='only_old_fails', default=False,
action="store_true",
help='Check the failfile and only run all tests that failed last time')
parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
action="store_true",
help='Clear the failfile. Do this if all tests pass')
parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
action="store_true",
help='Clear database files from previous smoke.py runs')
parser.add_option('--server-log', dest='server_log_file', default=server_log_file,
help="Log file for the mongod server ('%default')")
parser.add_option('--tests-log', dest='tests_log_file', default='',
help="Log file for the tests ('%default')")
parser.add_option('--quiet', dest='quiet', default=False,
action="store_true",
help='Generate a quieter report (use with --tests-log)')
parser.add_option('--shuffle', dest='shuffle', default=False,
action="store_true",
help='Shuffle tests instead of running them in alphabetical order')
# Buildlogger invocation from command line
parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
action="store", help='Set the "builder name" for buildlogger')
parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
action="store", help='Set the "build number" for buildlogger')
parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
action="store", help='Path to Python file containing buildlogger credentials')
parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')
global tests
(options, tests) = parser.parse_args()
set_globals(options, tests)
buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
if all(buildlogger_opts):
os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
if options.buildlogger_phase:
os.environ['MONGO_PHASE'] = options.buildlogger_phase
elif any(buildlogger_opts):
# some but not all of the required options were sete
raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")
if options.File:
if options.File == '-':
tests = sys.stdin.readlines()
else:
f = open(options.File)
tests = f.readlines()
tests = [t.rstrip('\n') for t in tests]
if options.only_old_fails:
run_old_fails()
return
elif options.reset_old_fails:
clear_failfile()
return
# If we're in suite mode, tests is a list of names of sets of tests.
if options.mode == 'suite':
tests = expand_suites(tests)
elif options.mode == 'files':
tests = [(os.path.abspath(test), True) for test in tests]
if options.ignore_files != None :
ignore_patt = re.compile( options.ignore_files )
tests = filter( lambda x : ignore_patt.search( x[0] ) == None, tests )
if not tests:
print "warning: no tests specified"
return
if options.with_cleanbb:
dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
if options.quiet:
f = open(server_log_file, "a")
try:
call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot], stdout=f)
finally:
f.close()
else:
call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])
if options.shuffle:
import random
random.shuffle(tests)
try:
run_tests(tests)
finally:
add_to_failfile(fails, options)
report()
