def setup(self):
"""Set up the mongod."""
if not self.preserve_dbpath and os.path.lexists(self._dbpath):
utils.rmtree(self._dbpath, ignore_errors=False)
try:
os.makedirs(self._dbpath)
except os.error:
# Directory already exists.
pass
if "port" not in self.mongod_options:
self.mongod_options[
"port"] = core.network.PortAllocator.next_fixture_port(
self.job_num)
self.port = self.mongod_options["port"]
mongod = core.programs.mongod_program(
self.logger,
self.job_num,
executable=self.mongod_executable,
mongod_options=self.mongod_options)
try:
self.logger.info("Starting mongod on port %d...\n%s", self.port,
mongod.as_command())
mongod.start()
self.logger.info("mongod started on port %d with pid %d.",
self.port, mongod.pid)
except Exception as err:
msg = "Failed to start mongod on port {:d}: {}".format(
self.port, err)
self.logger.exception(msg)
raise errors.ServerFailure(msg)
self.mongod = mongod
def configure_shell(self):
"""Set up the global variables for the shell, and data/ directory for the mongod.
configure_shell() only needs to be called once per test. Therefore if creating multiple
_SingleJSTestCase instances to be run in parallel, only call configure_shell() on one of
them.
"""
global_vars = self.shell_options.get("global_vars", {}).copy()
data_dir = self._get_data_dir(global_vars)
# Set MongoRunner.dataPath if overridden at command line or not specified in YAML.
if config.DBPATH_PREFIX is not None or "MongoRunner.dataPath" not in global_vars:
# dataPath property is the dataDir property with a trailing slash.
data_path = os.path.join(data_dir, "")
else:
data_path = global_vars["MongoRunner.dataPath"]
global_vars["MongoRunner.dataDir"] = data_dir
global_vars["MongoRunner.dataPath"] = data_path
test_data = global_vars.get("TestData", {}).copy()
test_data["minPort"] = core.network.PortAllocator.min_test_port(
self.fixture.job_num)
test_data["maxPort"] = core.network.PortAllocator.max_test_port(
self.fixture.job_num)
test_data["peerPids"] = self.fixture.pids()
test_data["alwaysUseLogFiles"] = config.ALWAYS_USE_LOG_FILES
test_data["failIfUnterminatedProcesses"] = True
global_vars["TestData"] = test_data
self.shell_options["global_vars"] = global_vars
utils.rmtree(data_dir, ignore_errors=True)
try:
os.makedirs(data_dir)
except os.error:
# Directory already exists.
pass
process_kwargs = copy.deepcopy(
self.shell_options.get("process_kwargs", {}))
if process_kwargs \
and "env_vars" in process_kwargs \
and "KRB5_CONFIG" in process_kwargs["env_vars"] \
and "KRB5CCNAME" not in process_kwargs["env_vars"]:
# Use a job-specific credential cache for JavaScript tests involving Kerberos.
krb5_dir = os.path.join(data_dir, "krb5")
try:
os.makedirs(krb5_dir)
except os.error:
pass
process_kwargs["env_vars"]["KRB5CCNAME"] = "DIR:" + krb5_dir
self.shell_options["process_kwargs"] = process_kwargs
def setUp(self):
super(TestTimeout, self).setUp()
self.logger.info("Cleaning temp directory %s", self.test_dir_inner)
rmtree(self.test_dir_inner, ignore_errors=True)
self.logger.info("Cleaning hang analyzer files %s",
str(self.analysis_files))
for filename in self.analysis_files:
if os.path.exists(filename):
os.remove(filename)
def setUp(self):
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(fmt="%(message)s"))
self.logger.addHandler(handler)
self.logger.info("Cleaning temp directory %s", self.test_dir)
rmtree(self.test_dir, ignore_errors=True)
os.makedirs(self.test_dir, mode=0o755, exist_ok=True)
def configure(self, fixture, *args, **kwargs):
"""Configure BenchrunEmbeddedTestCase."""
interface.ProcessTestCase.configure(self, fixture, *args, **kwargs)
# 1. Set the default benchmark options.
benchrun_options = {
"time": _config.DEFAULT_BENCHMARK_MIN_TIME.total_seconds()
}
# 2. Override Benchmark options with options set through `program_options` in the suite
# configuration.
suite_benchrun_options = utils.default_if_none(
self.suite_benchrun_options, {})
benchrun_options.update(suite_benchrun_options)
# 3. Override Benchmark options with options set through resmoke's command line.
resmoke_benchrun_options = {
"dbpath": self.dbpath,
"time": _config.BENCHMARK_MIN_TIME
}
for key, value in resmoke_benchrun_options.items():
if value is not None:
# 4. sanitize options before passing them to Benchmark's command line.
if key == "time":
value = value.total_seconds()
benchrun_options[key] = value
self.benchrun_options = benchrun_options
# Create the test report directory.
utils.rmtree(self._report_dir(), ignore_errors=True)
try:
os.makedirs(self._report_dir())
except os.error:
# Directory already exists.
pass
# Create the dbpath.
if self.android_device:
self.adb.shell("rm -fr {}".format(self.dbpath))
self.adb.shell("mkdir {}".format(self.dbpath))
else:
utils.rmtree(self.dbpath, ignore_errors=True)
try:
os.makedirs(self.dbpath)
except os.error:
# Directory already exists.
pass
def setUp(self):
#self.test_dir = os.path.normpath("/data/db/selftest")
self.end2end_root = "buildscripts/tests/resmoke_end2end"
self.suites_root = f"{self.end2end_root}/suites"
self.testfiles_root = f"{self.end2end_root}/testfiles"
self.report_file = os.path.join(self.test_dir, "reports.json")
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(fmt="%(message)s"))
self.logger.addHandler(handler)
self.logger.info("Cleaning temp directory %s", self.test_dir)
rmtree(self.test_dir, ignore_errors=True)
os.makedirs(self.test_dir, mode=0o755, exist_ok=True)
def configure(self, fixture, *args, **kwargs):
"""Configure BenchrunEmbeddedTestCase."""
interface.ProcessTestCase.configure(self, fixture, *args, **kwargs)
# 1. Set the default benchmark options.
benchrun_options = {"time": _config.DEFAULT_BENCHMARK_MIN_TIME.total_seconds()}
# 2. Override Benchmark options with options set through `program_options` in the suite
# configuration.
suite_benchrun_options = utils.default_if_none(self.suite_benchrun_options, {})
benchrun_options.update(suite_benchrun_options)
# 3. Override Benchmark options with options set through resmoke's command line.
resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME}
for key, value in resmoke_benchrun_options.items():
if value is not None:
# 4. sanitize options before passing them to Benchmark's command line.
if key == "time":
value = value.total_seconds()
benchrun_options[key] = value
self.benchrun_options = benchrun_options
# Create the test report directory.
utils.rmtree(self._report_dir(), ignore_errors=True)
try:
os.makedirs(self._report_dir())
except os.error:
# Directory already exists.
pass
# Create the dbpath.
if self.android_device:
self.adb.shell("rm -fr {}".format(self.dbpath))
self.adb.shell("mkdir {}".format(self.dbpath))
else:
utils.rmtree(self.dbpath, ignore_errors=True)
try:
os.makedirs(self.dbpath)
except os.error:
# Directory already exists.
pass
def _setup_jasper(self):
"""Start up the jasper process manager."""
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
proto_file = os.path.join(root_dir, "buildscripts", "resmokelib",
"core", "jasper.proto")
try:
well_known_protos_include = pkg_resources.resource_filename(
"grpc_tools", "_proto")
except ImportError:
raise ImportError(
"You must run: sys.executable + '-m pip install grpcio grpcio-tools "
"googleapis-common-protos' to use --spawnUsing=jasper.")
# We use the build/ directory as the output directory because the generated files aren't
# meant to because tracked by git or linted.
proto_out = os.path.join(root_dir, "build", "jasper")
utils.rmtree(proto_out, ignore_errors=True)
os.makedirs(proto_out)
# We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the
# *pb2*.py modules from it.
with open(os.path.join(proto_out, "__init__.py"), "w"):
pass
ret = grpc_tools.protoc.main([
grpc_tools.protoc.__file__,
"--grpc_python_out",
proto_out,
"--python_out",
proto_out,
"--proto_path",
os.path.dirname(proto_file),
"--proto_path",
well_known_protos_include,
os.path.basename(proto_file),
])
if ret != 0:
raise RuntimeError(
"Failed to generated gRPC files from the jasper.proto file")
sys.path.append(os.path.dirname(proto_out))
from jasper import jasper_pb2
from jasper import jasper_pb2_grpc
jasper_process.Process.jasper_pb2 = jasper_pb2
jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc
curator_path = "build/curator"
if sys.platform == "win32":
curator_path += ".exe"
git_hash = "d846f0c875716e9377044ab2a50542724369662a"
curator_exists = os.path.isfile(curator_path)
curator_same_version = False
if curator_exists:
curator_version = subprocess.check_output(
[curator_path, "--version"]).decode('utf-8').split()
curator_same_version = git_hash in curator_version
if curator_exists and not curator_same_version:
os.remove(curator_path)
self._resmoke_logger.info(
"Found a different version of curator. Downloading version %s of curator to enable"
"process management using jasper.", git_hash)
if not curator_exists or not curator_same_version:
if sys.platform == "darwin":
os_platform = "macos"
elif sys.platform == "win32":
os_platform = "windows-64"
elif sys.platform.startswith("linux"):
os_platform = "ubuntu1604"
else:
raise OSError(
"Unrecognized platform. "
"This program is meant to be run on MacOS, Windows, or Linux."
)
url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/"
"curator-dist-%s-%s.tar.gz") % (os_platform, git_hash)
response = requests.get(url, stream=True)
with tarfile.open(mode="r|gz", fileobj=response.raw) as tf:
tf.extractall(path="./build/")
jasper_port = config.BASE_PORT - 1
jasper_conn_str = "localhost:%d" % jasper_port
jasper_process.Process.connection_str = jasper_conn_str
jasper_command = [
curator_path, "jasper", "grpc", "--port",
str(jasper_port)
]
self._jasper_server = process.Process(self._resmoke_logger,
jasper_command)
self._jasper_server.start()
channel = grpc.insecure_channel(jasper_conn_str)
grpc.channel_ready_future(channel).result()
def setUp(self):
super(TestTimeout, self).setUp()
self.logger.info("Cleaning temp directory %s", self.test_dir_inner)
rmtree(self.test_dir_inner, ignore_errors=True)
def _setup_jasper(self):
"""Start up the jasper process manager."""
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto")
try:
well_known_protos_include = pkg_resources.resource_filename("grpc_tools", "_proto")
except ImportError:
raise ImportError("You must run: sys.executable + '-m pip install grpcio grpcio-tools "
"googleapis-common-protos' to use --spawnUsing=jasper.")
# We use the build/ directory as the output directory because the generated files aren't
# meant to because tracked by git or linted.
proto_out = os.path.join(root_dir, "build", "jasper")
utils.rmtree(proto_out, ignore_errors=True)
os.makedirs(proto_out)
# We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the
# *pb2*.py modules from it.
with open(os.path.join(proto_out, "__init__.py"), "w"):
pass
ret = grpc_tools.protoc.main([
grpc_tools.protoc.__file__,
"--grpc_python_out",
proto_out,
"--python_out",
proto_out,
"--proto_path",
os.path.dirname(proto_file),
"--proto_path",
well_known_protos_include,
os.path.basename(proto_file),
])
if ret != 0:
raise RuntimeError("Failed to generated gRPC files from the jasper.proto file")
sys.path.append(os.path.dirname(proto_out))
from jasper import jasper_pb2
from jasper import jasper_pb2_grpc
jasper_process.Process.jasper_pb2 = jasper_pb2
jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc
curator_path = "build/curator"
if sys.platform == "win32":
curator_path += ".exe"
git_hash = "d846f0c875716e9377044ab2a50542724369662a"
curator_exists = os.path.isfile(curator_path)
curator_same_version = False
if curator_exists:
curator_version = subprocess.check_output([curator_path,
"--version"]).decode('utf-8').split()
curator_same_version = git_hash in curator_version
if curator_exists and not curator_same_version:
os.remove(curator_path)
self._resmoke_logger.info(
"Found a different version of curator. Downloading version %s of curator to enable"
"process management using jasper.", git_hash)
if not curator_exists or not curator_same_version:
if sys.platform == "darwin":
os_platform = "macos"
elif sys.platform == "win32":
os_platform = "windows-64"
elif sys.platform.startswith("linux"):
os_platform = "ubuntu1604"
else:
raise OSError("Unrecognized platform. "
"This program is meant to be run on MacOS, Windows, or Linux.")
url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/"
"curator-dist-%s-%s.tar.gz") % (os_platform, git_hash)
response = requests.get(url, stream=True)
with tarfile.open(mode="r|gz", fileobj=response.raw) as tf:
tf.extractall(path="./build/")
jasper_port = config.BASE_PORT - 1
jasper_conn_str = "localhost:%d" % jasper_port
jasper_process.Process.connection_str = jasper_conn_str
jasper_command = [curator_path, "jasper", "grpc", "--port", str(jasper_port)]
self._jasper_server = process.Process(self._resmoke_logger, jasper_command)
self._jasper_server.start()
channel = grpc.insecure_channel(jasper_conn_str)
grpc.channel_ready_future(channel).result()
def rmtree(dir_root):
"""Invoke utils.rmtree(dir_root) and return True if removed."""
utils.rmtree(dir_root)
return not os.path.exists(dir_root)
def setUp(self):
self.logger.info("Cleaning temp directory %s", self.test_dir)
rmtree(self.test_dir, ignore_errors=True)
def _clear_dbpath(self):
utils.rmtree(self.dbtest_options["dbpath"], ignore_errors=True)
def rmtree(dir_root):
"""Invoke utils.rmtree(dir_root) and return True if removed."""
utils.rmtree(dir_root)
return not os.path.exists(dir_root)
def _clear_dbpath(self):
utils.rmtree(self.dbpath, ignore_errors=True)
