Exemple #1
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if 'slave' in self.kwargs:
            dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if 'slave' in self.kwargs:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
            else:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
        # This should always be set for tests
        argv += ['--setParameter', 'enableTestCommands=1']
        if self.kwargs.get('small_oplog'):
            argv += ["--master", "--oplogSize", "511"]
        if self.kwargs.get('small_oplog_rs'):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ['--slave', '--source', 'localhost:' + str(srcport)]
        if self.kwargs.get('no_journal'):
            argv += ['--nojournal']
        if self.kwargs.get('no_preallocj'):
            argv += ['--nopreallocj']
        if self.kwargs.get('auth'):
            argv += ['--auth']
            self.auth = True
        if self.kwargs.get('use_ssl'):
            argv += ['--sslOnNormalPorts',
                     '--sslPEMKeyFile', 'jstests/libs/server.pem',
                     '--sslCAFile', 'jstests/libs/ca.pem']
        
        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source["syncedTo"]
Exemple #2
0
def buildlogger(cmd, is_global=False):
    # if the environment variable MONGO_USE_BUILDLOGGER
    # is set to 'true', then wrap the command with a call
    # to buildlogger.py, which sends output to the buidlogger
    # machine; otherwise, return as usual.
    if os.environ.get("MONGO_USE_BUILDLOGGER", "").lower().strip() == "true":
        if is_global:
            return [utils.find_python(), "buildscripts/buildlogger.py", "-g"] + cmd
        else:
            return [utils.find_python(), "buildscripts/buildlogger.py"] + cmd
    return cmd
Exemple #3
0
def buildlogger(cmd, is_global=False):
    # if the environment variable MONGO_USE_BUILDLOGGER
    # is set to 'true', then wrap the command with a call
    # to buildlogger.py, which sends output to the buidlogger
    # machine; otherwise, return as usual.
    if os.environ.get('MONGO_USE_BUILDLOGGER', '').lower().strip() == 'true':
        if is_global:
            return [utils.find_python(), 'buildscripts/buildlogger.py', '-g'] + cmd
        else:
            return [utils.find_python(), 'buildscripts/buildlogger.py'] + cmd
    return cmd
Exemple #4
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if "slave" in self.kwargs:
            dir_name = smoke_db_prefix + "/data/db/sconsTestsSlave/"
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if "slave" in self.kwargs:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dir_name]
            else:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
        if self.kwargs.get("small_oplog"):
            argv += ["--master", "--oplogSize", "511"]
        if self.kwargs.get("small_oplog_rs"):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ["--slave", "--source", "localhost:" + str(srcport)]
        if self.kwargs.get("no_journal"):
            argv += ["--nojournal"]
        if self.kwargs.get("no_preallocj"):
            argv += ["--nopreallocj"]
        if self.kwargs.get("auth"):
            argv += ["--auth"]
            self.auth = True
        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source["syncedTo"]
Exemple #5
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if 'slave' in self.kwargs:
            dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if 'slave' in self.kwargs:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
            else:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
        if self.kwargs.get('small_oplog'):
            argv += ["--master", "--oplogSize", "511"]
        if self.kwargs.get('small_oplog_rs'):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ['--slave', '--source', 'localhost:' + str(srcport)]
        if self.kwargs.get('no_journal'):
            argv += ['--nojournal']
        if self.kwargs.get('no_preallocj'):
            argv += ['--nopreallocj']
        if self.kwargs.get('auth'):
            argv += ['--auth']
            self.auth = True
        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source["syncedTo"]
Exemple #6
0
def main():
    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, set_parameters, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
    parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
    parser.add_option(
        '--mode',
        dest='mode',
        default='suite',
        help=
        'If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)'
    )
    # Some of our tests hard-code pathnames e.g., to execute, so until
    # that changes we don't have the freedom to run from anyplace.
    # parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
    parser.add_option('--test-path',
                      dest='test_path',
                      default=None,
                      help="Path to the test executables to run, "
                      "currently only used for 'client' (%default)")
    parser.add_option('--mongod',
                      dest='mongod_executable',
                      default=os.path.join(mongo_repo, 'mongod'),
                      help='Path to mongod to run (%default)')
    parser.add_option('--port',
                      dest='mongod_port',
                      default="27999",
                      help='Port the mongod will bind to (%default)')
    parser.add_option('--mongo',
                      dest='shell_executable',
                      default=os.path.join(mongo_repo, 'mongo'),
                      help='Path to mongo, for .js test files (%default)')
    parser.add_option(
        '--continue-on-failure',
        dest='continue_on_failure',
        action="store_true",
        default=False,
        help='If supplied, continue testing even after a test fails')
    parser.add_option(
        '--from-file',
        dest='File',
        help=
        "Run tests/suites named in FILE, one test per line, '-' means stdin")
    parser.add_option(
        '--smoke-db-prefix',
        dest='smoke_db_prefix',
        default=smoke_db_prefix,
        help="Prefix to use for the mongods' dbpaths ('%default')")
    parser.add_option(
        '--small-oplog',
        dest='small_oplog',
        default=False,
        action="store_true",
        help='Run tests with master/slave replication & use a small oplog')
    parser.add_option(
        '--small-oplog-rs',
        dest='small_oplog_rs',
        default=False,
        action="store_true",
        help='Run tests with replica set replication & use a small oplog')
    parser.add_option('--nojournal',
                      dest='no_journal',
                      default=False,
                      action="store_true",
                      help='Do not turn on journaling in tests')
    parser.add_option('--nopreallocj',
                      dest='no_preallocj',
                      default=False,
                      action="store_true",
                      help='Do not preallocate journal files in tests')
    parser.add_option(
        '--auth',
        dest='auth',
        default=False,
        action="store_true",
        help='Run standalone mongods in tests with authentication enabled')
    parser.add_option(
        '--authMechanism',
        dest='authMechanism',
        default='MONGODB-CR',
        help='Use the given authentication mechanism, when --auth is used.')
    parser.add_option(
        '--keyFile',
        dest='keyFile',
        default=None,
        help=
        'Path to keyFile to use to run replSet and sharding tests with authentication enabled'
    )
    parser.add_option('--ignore',
                      dest='ignore_files',
                      default=None,
                      help='Pattern of files to ignore in tests')
    parser.add_option(
        '--only-old-fails',
        dest='only_old_fails',
        default=False,
        action="store_true",
        help='Check the failfile and only run all tests that failed last time')
    parser.add_option('--reset-old-fails',
                      dest='reset_old_fails',
                      default=False,
                      action="store_true",
                      help='Clear the failfile. Do this if all tests pass')
    parser.add_option('--with-cleanbb',
                      dest='with_cleanbb',
                      default=False,
                      action="store_true",
                      help='Clear database files from previous smoke.py runs')
    parser.add_option(
        '--dont-start-mongod',
        dest='start_mongod',
        default=True,
        action='store_false',
        help='Do not start mongod before commencing test running')
    parser.add_option(
        '--use-ssl',
        dest='use_ssl',
        default=False,
        action='store_true',
        help='Run mongo shell and mongod instances with SSL encryption')
    parser.add_option(
        '--set-parameters',
        dest='set_parameters',
        default="",
        help=
        'Adds --setParameter for each passed in items in the csv list - ex. "param1=1,param2=foo" '
    )
    # Buildlogger invocation from command line
    parser.add_option('--buildlogger-builder',
                      dest='buildlogger_builder',
                      default=None,
                      action="store",
                      help='Set the "builder name" for buildlogger')
    parser.add_option('--buildlogger-buildnum',
                      dest='buildlogger_buildnum',
                      default=None,
                      action="store",
                      help='Set the "build number" for buildlogger')
    parser.add_option(
        '--buildlogger-credentials',
        dest='buildlogger_credentials',
        default=None,
        action="store",
        help='Path to Python file containing buildlogger credentials')
    parser.add_option(
        '--buildlogger-phase',
        dest='buildlogger_phase',
        default=None,
        action="store",
        help=
        'Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)'
    )

    global tests
    (options, tests) = parser.parse_args()

    set_globals(options, tests)

    buildlogger_opts = (options.buildlogger_builder,
                        options.buildlogger_buildnum,
                        options.buildlogger_credentials)
    if all(buildlogger_opts):
        os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
        os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
        os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
        os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
        if options.buildlogger_phase:
            os.environ['MONGO_PHASE'] = options.buildlogger_phase
    elif any(buildlogger_opts):
        # some but not all of the required options were sete
        raise Exception(
            "you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials"
        )

    if options.File:
        if options.File == '-':
            tests = sys.stdin.readlines()
        else:
            f = open(options.File)
            tests = f.readlines()
    tests = [t.rstrip('\n') for t in tests]

    if options.only_old_fails:
        run_old_fails()
        return
    elif options.reset_old_fails:
        clear_failfile()
        return

    # If we're in suite mode, tests is a list of names of sets of tests.
    if options.mode == 'suite':
        tests = expand_suites(tests)
    elif options.mode == 'files':
        tests = [(os.path.abspath(test), start_mongod) for test in tests]

    if options.ignore_files != None:
        ignore_patt = re.compile(options.ignore_files)
        print "Ignoring files with pattern: ", ignore_patt

        def ignore_test(test):
            if ignore_patt.search(test[0]) != None:
                print "Ignoring test ", test[0]
                return False
            else:
                return True

        tests = filter(ignore_test, tests)

    if not tests:
        print "warning: no tests specified"
        return

    if options.with_cleanbb:
        dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
        call([
            utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot
        ])

    try:
        run_tests(tests)
    finally:
        add_to_failfile(fails, options)

        f = open("smoke-last.json", "wb")
        f.write(json.dumps({"results": all_test_results}))
        f.close()

        report()
Exemple #7
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if 'slave' in self.kwargs:
            dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if 'slave' in self.kwargs:
                argv = [
                    utils.find_python(), "buildscripts/cleanbb.py", '--nokill',
                    dir_name
                ]
            else:
                argv = [
                    utils.find_python(), "buildscripts/cleanbb.py", dir_name
                ]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [
            mongod_executable, "--port",
            str(self.port), "--dbpath", dir_name
        ]
        # These parameters are alwas set for tests
        # SERVER-9137 Added httpinterface parameter to keep previous behavior
        argv += ['--setParameter', 'enableTestCommands=1', '--httpinterface']
        if self.kwargs.get('small_oplog'):
            argv += ["--master", "--oplogSize", "511"]
        params = self.kwargs.get('set_parameters', None)
        if params:
            for p in params.split(','):
                argv += ['--setParameter', p]
        if self.kwargs.get('small_oplog_rs'):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ['--slave', '--source', 'localhost:' + str(srcport)]
        if self.kwargs.get('no_journal'):
            argv += ['--nojournal']
        if self.kwargs.get('no_preallocj'):
            argv += ['--nopreallocj']
        if self.kwargs.get('auth'):
            argv += ['--auth']
            authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
            if authMechanism != 'MONGODB-CR':
                argv += [
                    '--setParameter',
                    'authenticationMechanisms=' + authMechanism
                ]
            self.auth = True
        if self.kwargs.get('use_ssl'):
            argv += [
                '--sslOnNormalPorts', '--sslPEMKeyFile',
                'jstests/libs/server.pem', '--sslCAFile',
                'jstests/libs/ca.pem', '--sslWeakCertificateValidation'
            ]

        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source[
                        "syncedTo"]
Exemple #8
0
def main():
    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
    parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
    parser.add_option(
        "--mode",
        dest="mode",
        default="suite",
        help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)',
    )
    # Some of our tests hard-code pathnames e.g., to execute, so until
    # that changes we don't have the freedom to run from anyplace.
    # parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
    parser.add_option(
        "--test-path",
        dest="test_path",
        default=None,
        help="Path to the test executables to run, " "currently only used for 'client' (%default)",
    )
    parser.add_option(
        "--mongod",
        dest="mongod_executable",
        default=os.path.join(mongo_repo, "mongod"),
        help="Path to mongod to run (%default)",
    )
    parser.add_option("--port", dest="mongod_port", default="27999", help="Port the mongod will bind to (%default)")
    parser.add_option(
        "--mongo",
        dest="shell_executable",
        default=os.path.join(mongo_repo, "mongo"),
        help="Path to mongo, for .js test files (%default)",
    )
    parser.add_option(
        "--continue-on-failure",
        dest="continue_on_failure",
        action="store_true",
        default=False,
        help="If supplied, continue testing even after a test fails",
    )
    parser.add_option(
        "--from-file", dest="File", help="Run tests/suites named in FILE, one test per line, '-' means stdin"
    )
    parser.add_option(
        "--smoke-db-prefix",
        dest="smoke_db_prefix",
        default=smoke_db_prefix,
        help="Prefix to use for the mongods' dbpaths ('%default')",
    )
    parser.add_option(
        "--small-oplog",
        dest="small_oplog",
        default=False,
        action="store_true",
        help="Run tests with master/slave replication & use a small oplog",
    )
    parser.add_option(
        "--small-oplog-rs",
        dest="small_oplog_rs",
        default=False,
        action="store_true",
        help="Run tests with replica set replication & use a small oplog",
    )
    parser.add_option(
        "--nojournal", dest="no_journal", default=False, action="store_true", help="Do not turn on journaling in tests"
    )
    parser.add_option(
        "--nopreallocj",
        dest="no_preallocj",
        default=False,
        action="store_true",
        help="Do not preallocate journal files in tests",
    )
    parser.add_option(
        "--auth",
        dest="auth",
        default=False,
        action="store_true",
        help="Run standalone mongods in tests with authentication enabled",
    )
    parser.add_option(
        "--authMechanism",
        dest="authMechanism",
        default="MONGO-CR",
        help="Use the given authentication mechanism, when --auth is used.",
    )
    parser.add_option(
        "--keyFile",
        dest="keyFile",
        default=None,
        help="Path to keyFile to use to run replSet and sharding tests with authentication enabled",
    )
    parser.add_option("--ignore", dest="ignore_files", default=None, help="Pattern of files to ignore in tests")
    parser.add_option(
        "--only-old-fails",
        dest="only_old_fails",
        default=False,
        action="store_true",
        help="Check the failfile and only run all tests that failed last time",
    )
    parser.add_option(
        "--reset-old-fails",
        dest="reset_old_fails",
        default=False,
        action="store_true",
        help="Clear the failfile. Do this if all tests pass",
    )
    parser.add_option(
        "--with-cleanbb",
        dest="with_cleanbb",
        default=False,
        action="store_true",
        help="Clear database files from previous smoke.py runs",
    )
    parser.add_option(
        "--dont-start-mongod",
        dest="start_mongod",
        default=True,
        action="store_false",
        help="Do not start mongod before commencing test running",
    )
    parser.add_option(
        "--use-ssl",
        dest="use_ssl",
        default=False,
        action="store_true",
        help="Run mongo shell and mongod instances with SSL encryption",
    )

    # Buildlogger invocation from command line
    parser.add_option(
        "--buildlogger-builder",
        dest="buildlogger_builder",
        default=None,
        action="store",
        help='Set the "builder name" for buildlogger',
    )
    parser.add_option(
        "--buildlogger-buildnum",
        dest="buildlogger_buildnum",
        default=None,
        action="store",
        help='Set the "build number" for buildlogger',
    )
    parser.add_option(
        "--buildlogger-credentials",
        dest="buildlogger_credentials",
        default=None,
        action="store",
        help="Path to Python file containing buildlogger credentials",
    )
    parser.add_option(
        "--buildlogger-phase",
        dest="buildlogger_phase",
        default=None,
        action="store",
        help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)',
    )

    global tests
    (options, tests) = parser.parse_args()

    set_globals(options, tests)

    buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
    if all(buildlogger_opts):
        os.environ["MONGO_USE_BUILDLOGGER"] = "true"
        os.environ["MONGO_BUILDER_NAME"] = options.buildlogger_builder
        os.environ["MONGO_BUILD_NUMBER"] = options.buildlogger_buildnum
        os.environ["BUILDLOGGER_CREDENTIALS"] = options.buildlogger_credentials
        if options.buildlogger_phase:
            os.environ["MONGO_PHASE"] = options.buildlogger_phase
    elif any(buildlogger_opts):
        # some but not all of the required options were sete
        raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")

    if options.File:
        if options.File == "-":
            tests = sys.stdin.readlines()
        else:
            f = open(options.File)
            tests = f.readlines()
    tests = [t.rstrip("\n") for t in tests]

    if options.only_old_fails:
        run_old_fails()
        return
    elif options.reset_old_fails:
        clear_failfile()
        return

    # If we're in suite mode, tests is a list of names of sets of tests.
    if options.mode == "suite":
        tests = expand_suites(tests)
    elif options.mode == "files":
        tests = [(os.path.abspath(test), start_mongod) for test in tests]

    if options.ignore_files != None:
        ignore_patt = re.compile(options.ignore_files)
        print "Ignoring files with pattern: ", ignore_patt

        def ignore_test(test):
            if ignore_patt.search(test[0]) != None:
                print "Ignoring test ", test[0]
                return False
            else:
                return True

        tests = filter(ignore_test, tests)

    if not tests:
        print "warning: no tests specified"
        return

    if options.with_cleanbb:
        dbroot = os.path.join(options.smoke_db_prefix, "data", "db")
        call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])

    try:
        run_tests(tests)
    finally:
        add_to_failfile(fails, options)

        f = open("smoke-last.json", "wb")
        f.write(json.dumps({"results": all_test_results}))
        f.close()

        report()
Exemple #9
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if "slave" in self.kwargs:
            dir_name = smoke_db_prefix + "/data/db/sconsTestsSlave/"
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if "slave" in self.kwargs:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dir_name]
            else:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
        # This should always be set for tests
        argv += ["--setParameter", "enableTestCommands=1"]
        if self.kwargs.get("small_oplog"):
            argv += ["--master", "--oplogSize", "511"]
        if self.kwargs.get("small_oplog_rs"):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ["--slave", "--source", "localhost:" + str(srcport)]
        if self.kwargs.get("no_journal"):
            argv += ["--nojournal"]
        if self.kwargs.get("no_preallocj"):
            argv += ["--nopreallocj"]
        if self.kwargs.get("auth"):
            argv += ["--auth"]
            authMechanism = self.kwargs.get("authMechanism", "MONGO-CR")
            if authMechanism != "MONGO-CR":
                argv.append("--setParameter=authenticationMechanisms=" + authMechanism)
            self.auth = True
        if self.kwargs.get("use_ssl"):
            argv += [
                "--sslOnNormalPorts",
                "--sslPEMKeyFile",
                "jstests/libs/server.pem",
                "--sslCAFile",
                "jstests/libs/ca.pem",
                "--sslWeakCertificateValidation",
            ]

        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source["syncedTo"]
Exemple #10
0
def main():
    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, set_parameters, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
    parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
    parser.add_option('--mode', dest='mode', default='suite',
                      help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
    # Some of our tests hard-code pathnames e.g., to execute, so until
    # that changes we don't have the freedom to run from anyplace.
    # parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
    parser.add_option('--test-path', dest='test_path', default=None,
                      help="Path to the test executables to run, "
                      "currently only used for 'client' (%default)")
    parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
                      help='Path to mongod to run (%default)')
    parser.add_option('--port', dest='mongod_port', default="27999",
                      help='Port the mongod will bind to (%default)')
    parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
                      help='Path to mongo, for .js test files (%default)')
    parser.add_option('--continue-on-failure', dest='continue_on_failure',
                      action="store_true", default=False,
                      help='If supplied, continue testing even after a test fails')
    parser.add_option('--from-file', dest='File',
                      help="Run tests/suites named in FILE, one test per line, '-' means stdin")
    parser.add_option('--smoke-db-prefix', dest='smoke_db_prefix', default=smoke_db_prefix,
                      help="Prefix to use for the mongods' dbpaths ('%default')")
    parser.add_option('--small-oplog', dest='small_oplog', default=False,
                      action="store_true",
                      help='Run tests with master/slave replication & use a small oplog')
    parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False,
                      action="store_true",
                      help='Run tests with replica set replication & use a small oplog')
    parser.add_option('--nojournal', dest='no_journal', default=False,
                      action="store_true",
                      help='Do not turn on journaling in tests')
    parser.add_option('--nopreallocj', dest='no_preallocj', default=False,
                      action="store_true",
                      help='Do not preallocate journal files in tests')
    parser.add_option('--auth', dest='auth', default=False,
                      action="store_true",
                      help='Run standalone mongods in tests with authentication enabled')
    parser.add_option('--authMechanism', dest='authMechanism', default='MONGODB-CR',
                      help='Use the given authentication mechanism, when --auth is used.')
    parser.add_option('--keyFile', dest='keyFile', default=None,
                      help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
    parser.add_option('--ignore', dest='ignore_files', default=None,
                      help='Pattern of files to ignore in tests')
    parser.add_option('--only-old-fails', dest='only_old_fails', default=False,
                      action="store_true",
                      help='Check the failfile and only run all tests that failed last time')
    parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
                      action="store_true",
                      help='Clear the failfile. Do this if all tests pass')
    parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
                      action="store_true",
                      help='Clear database files from previous smoke.py runs')
    parser.add_option('--dont-start-mongod', dest='start_mongod', default=True,
                      action='store_false',
                      help='Do not start mongod before commencing test running')
    parser.add_option('--use-ssl', dest='use_ssl', default=False,
                      action='store_true',
                      help='Run mongo shell and mongod instances with SSL encryption')
    parser.add_option('--set-parameters', dest='set_parameters', default="",
                      help='Adds --setParameter for each passed in items in the csv list - ex. "param1=1,param2=foo" ')
    # Buildlogger invocation from command line
    parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
                      action="store", help='Set the "builder name" for buildlogger')
    parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
                      action="store", help='Set the "build number" for buildlogger')
    parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
                      action="store", help='Path to Python file containing buildlogger credentials')
    parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
                      action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')
    parser.add_option('--report-file', dest='report_file', default=None,
                      action='store',
                      help='Path to generate detailed json report containing all test details')


    global tests
    (options, tests) = parser.parse_args()

    set_globals(options, tests)

    buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
    if all(buildlogger_opts):
        os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
        os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
        os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
        os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
        if options.buildlogger_phase:
            os.environ['MONGO_PHASE'] = options.buildlogger_phase
    elif any(buildlogger_opts):
        # some but not all of the required options were sete
        raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")

    if options.File:
        if options.File == '-':
            tests = sys.stdin.readlines()
        else:
            f = open(options.File)
            tests = f.readlines()
    tests = [t.rstrip('\n') for t in tests]

    if options.only_old_fails:
        run_old_fails()
        return
    elif options.reset_old_fails:
        clear_failfile()
        return

    # If we're in suite mode, tests is a list of names of sets of tests.
    if options.mode == 'suite':
        tests = expand_suites(tests)
    elif options.mode == 'files':
        tests = [(os.path.abspath(test), start_mongod) for test in tests]

    if options.ignore_files != None :
        ignore_patt = re.compile( options.ignore_files )
        print "Ignoring files with pattern: ", ignore_patt

        def ignore_test( test ):
            if ignore_patt.search( test[0] ) != None:
                print "Ignoring test ", test[0]
                return False
            else:
                return True

        tests = filter( ignore_test, tests )

    if not tests:
        print "warning: no tests specified"
        return

    if options.with_cleanbb:
        dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
        call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])

    test_report["start"] = time.time()
    try:
        run_tests(tests)
    finally:
        add_to_failfile(fails, options)

        test_report["end"] = time.time()
        test_report["elapsed"] = test_report["end"] - test_report["start"]
        test_report["failures"] = len(losers.keys())
        if report_file:
            f = open( report_file, "wb" )
            f.write( json.dumps( test_report ) )
            f.close()

        report()
Exemple #11
0
    def start(self):
        global mongod_port
        global mongod
        if self.proc:
            print >> sys.stderr, "probable bug: self.proc already set in start()"
            return
        self.ensure_test_dirs()
        dir_name = smoke_db_prefix + "/data/db/sconsTests/"
        self.port = int(mongod_port)
        self.slave = False
        if 'slave' in self.kwargs:
            dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
            srcport = mongod_port
            self.port += 1
            self.slave = True
        if os.path.exists(dir_name):
            if 'slave' in self.kwargs:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
            else:
                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
            call(argv)
        utils.ensureDir(dir_name)
        argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
        # These parameters are alwas set for tests
        # SERVER-9137 Added httpinterface parameter to keep previous behavior
        argv += ['--setParameter', 'enableTestCommands=1', '--httpinterface']
        if self.kwargs.get('small_oplog'):
            argv += ["--master", "--oplogSize", "511"]
        params = self.kwargs.get('set_parameters', None)
        if params:
            for p in params.split(','): argv += ['--setParameter', p]
        if self.kwargs.get('small_oplog_rs'):
            argv += ["--replSet", "foo", "--oplogSize", "511"]
        if self.slave:
            argv += ['--slave', '--source', 'localhost:' + str(srcport)]
        if self.kwargs.get('no_journal'):
            argv += ['--nojournal']
        if self.kwargs.get('no_preallocj'):
            argv += ['--nopreallocj']
        if self.kwargs.get('auth'):
            argv += ['--auth']
            authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
            if authMechanism != 'MONGODB-CR':
                argv += ['--setParameter', 'authenticationMechanisms=' + authMechanism]
            self.auth = True
        if self.kwargs.get('use_ssl') or self.kwargs.get('use_x509'):
            argv += ['--sslMode', "sslOnly",
                     '--sslPEMKeyFile', 'jstests/libs/server.pem',
                     '--sslCAFile', 'jstests/libs/ca.pem',
                     '--sslWeakCertificateValidation']
        if self.kwargs.get('use_x509'):
            argv += ['--clusterAuthMode','x509'];
            self.auth = True
        print "running " + " ".join(argv)
        self.proc = self._start(buildlogger(argv, is_global=True))

        if not self.did_mongod_start(self.port):
            raise Exception("Failed to start mongod")

        if self.auth:
            self.setup_admin_user(self.port)

        if self.slave:
            local = Connection(port=self.port, slave_okay=True).local
            synced = False
            while not synced:
                synced = True
                for source in local.sources.find(fields=["syncedTo"]):
                    synced = synced and "syncedTo" in source and source["syncedTo"]
Exemple #12
0
def main():
    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, smoke_server_opts, test_path
    parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
    parser.add_option('--mode', dest='mode', default='suite',
                      help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
    # Some of our tests hard-code pathnames e.g., to execute, so until
    # that changes we don't have the freedom to run from anyplace.
    # parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
    parser.add_option('--test-path', dest='test_path', default=None,
                      help="Path to the test executables to run, "
                      "currently only used for 'client' (%default)")
    parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
                      help='Path to mongod to run (%default)')
    parser.add_option('--port', dest='mongod_port', default="27999",
                      help='Port the mongod will bind to (%default)')
    parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
                      help='Path to mongo, for .js test files (%default)')
    parser.add_option('--continue-on-failure', dest='continue_on_failure',
                      action="store_true", default=False,
                      help='If supplied, continue testing even after a test fails')
    parser.add_option('--from-file', dest='File',
                      help="Run tests/suites named in FILE, one test per line, '-' means stdin")
    parser.add_option('--smoke-db-prefix', dest='smoke_db_prefix', default=smoke_db_prefix,
                      help="Prefix to use for the mongods' dbpaths ('%default')")
    parser.add_option('--smoke-server-opts', dest='smoke_server_opts', default=smoke_server_opts,
                      help="Additional options for the mongod server ('%default')")
    parser.add_option('--small-oplog', dest='small_oplog', default=False,
                      action="store_true",
                      help='Run tests with master/slave replication & use a small oplog')
    parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False,
                      action="store_true",
                      help='Run tests with replica set replication & use a small oplog')
    parser.add_option('--nojournal', dest='no_journal', default=False,
                      action="store_true",
                      help='Do not turn on journaling in tests')
    parser.add_option('--nopreallocj', dest='no_preallocj', default=False,
                      action="store_true",
                      help='Do not preallocate journal files in tests')
    parser.add_option('--auth', dest='auth', default=False,
                      action="store_true",
                      help='Run standalone mongods in tests with authentication enabled')
    parser.add_option('--keyFile', dest='keyFile', default=None,
                      help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
    parser.add_option('--ignore', dest='ignore_files', default=None,
                      help='Pattern of files to ignore in tests')
    parser.add_option('--only-old-fails', dest='only_old_fails', default=False,
                      action="store_true",
                      help='Check the failfile and only run all tests that failed last time')
    parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
                      action="store_true",
                      help='Clear the failfile. Do this if all tests pass')
    parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
                      action="store_true",
                      help='Clear database files from previous smoke.py runs')
    parser.add_option('--server-log', dest='server_log_file', default=server_log_file,
                      help="Log file for the mongod server ('%default')")
    parser.add_option('--tests-log', dest='tests_log_file', default='',
                      help="Log file for the tests ('%default')")
    parser.add_option('--quiet', dest='quiet', default=False,
                      action="store_true",
                      help='Generate a quieter report (use with --tests-log)')
    parser.add_option('--shuffle', dest='shuffle', default=False,
                      action="store_true",
                      help='Shuffle tests instead of running them in alphabetical order')

    # Buildlogger invocation from command line
    parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
                      action="store", help='Set the "builder name" for buildlogger')
    parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
                      action="store", help='Set the "build number" for buildlogger')
    parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
                      action="store", help='Path to Python file containing buildlogger credentials')
    parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
                      action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')

    global tests
    (options, tests) = parser.parse_args()

    set_globals(options, tests)

    buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
    if all(buildlogger_opts):
        os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
        os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
        os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
        os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
        if options.buildlogger_phase:
            os.environ['MONGO_PHASE'] = options.buildlogger_phase
    elif any(buildlogger_opts):
        # some but not all of the required options were sete
        raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")

    if options.File:
        if options.File == '-':
            tests = sys.stdin.readlines()
        else:
            f = open(options.File)
            tests = f.readlines()
    tests = [t.rstrip('\n') for t in tests]

    if options.only_old_fails:
        run_old_fails()
        return
    elif options.reset_old_fails:
        clear_failfile()
        return

    # If we're in suite mode, tests is a list of names of sets of tests.
    if options.mode == 'suite':
        tests = expand_suites(tests)
    elif options.mode == 'files':
        tests = [(os.path.abspath(test), True) for test in tests]

    if options.ignore_files != None :
        ignore_patt = re.compile( options.ignore_files )
        tests = filter( lambda x : ignore_patt.search( x[0] ) == None, tests )

    if not tests:
        print "warning: no tests specified"
        return

    if options.with_cleanbb:
        dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
        if options.quiet:
            f = open(server_log_file, "a")
            try:
                call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot], stdout=f)
            finally:
                f.close()
        else:
            call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])

    if options.shuffle:
        import random
        random.shuffle(tests)

    try:
        run_tests(tests)
    finally:
        add_to_failfile(fails, options)
        report()