def validate_all(self):
"""Start a standalone node to validate all collections on the copied data files."""
for node in self.fixture.nodes:
path = node.get_dbpath_prefix() + "/simulateCrashes/{}".format(
self.backup_num)
self.logger.info(
"Starting to validate. DBPath: {} Port: {}".format(
path, self.validate_port))
mdb = process.Process(self.logger, [
node.mongod_executable, "--dbpath", path, "--port",
str(self.validate_port), "--logpath",
node.get_dbpath_prefix() + "/simulateCrashes/validate.log",
"--setParameter", "enableTestCommands=1"
])
mdb.start()
client = pymongo.MongoClient(host="localhost",
port=self.validate_port,
connect=True,
connectTimeoutMS=120000,
serverSelectionTimeoutMS=120000,
directConnection=True)
is_valid = validate(client, self.logger, self.acceptable_err_codes)
mdb.stop()
mdb.wait()
if not is_valid:
return False
shutil.rmtree(path, ignore_errors=True)
return True
def validate_all(self, backup_num):
"""Start a standalone node to validate all collections on the copied data files."""
for node in self.fixture.nodes:
if self.last_validate_port >= 20000:
self.last_validate_port = 19000
validate_port = self.last_validate_port
self.last_validate_port += 1
path = node.get_dbpath_prefix() + "/simulateCrashes/{}".format(
backup_num)
self.logger.info(
"Starting to validate. DBPath: {} Port: {}".format(
path, validate_port))
mdb = process.Process(self.logger, [
node.mongod_executable, "--dbpath", path, "--port",
str(validate_port), "--logpath",
node.get_dbpath_prefix() + "/simulateCrashes/validate.log"
])
mdb.start()
client = pymongo.MongoClient("localhost:{}".format(validate_port))
is_valid = validate(client, self.logger)
mdb.stop()
mdb.wait()
if not is_valid:
return False
shutil.rmtree(path, ignore_errors=True)
return True
def test_off(self):
buildscripts.resmokelib.config.AUTO_KILL = 'off'
buildscripts.resmokelib.config.SHELL_CONN_STRING = None
test_runner = MockTestRunner("test")
test_runner._setup_logging()
test_runner._check_for_mongo_processes()
proc = process.Process(logging.getLogger(), self.command)
proc.start()
test_runner._check_for_mongo_processes()
proc.stop(mode=fixture_interface.TeardownMode.ABORT)
if proc._process.returncode == self.sigkill_return:
self.fail("Process was killed when it should not have been.")
proc.wait()
def test_warn(self):
buildscripts.resmokelib.config.AUTO_KILL = 'warn'
buildscripts.resmokelib.config.SHELL_CONN_STRING = None
test_runner = MockTestRunner("test")
test_runner._setup_logging()
try:
test_runner._check_for_mongo_processes()
except errors.ResmokeError:
self.fail("Detected processes when there should be none.")
tmp_ctime = os.environ['RESMOKE_PARENT_CTIME']
os.environ['RESMOKE_PARENT_CTIME'] = str("rogue_process")
proc = process.Process(logging.getLogger(), self.command)
proc.start()
os.environ['RESMOKE_PARENT_CTIME'] = tmp_ctime
with self.assertRaises(errors.ResmokeError):
test_runner._check_for_mongo_processes()
proc.stop(mode=fixture_interface.TeardownMode.KILL)
proc.wait()
def test_on(self):
buildscripts.resmokelib.config.AUTO_KILL = 'on'
buildscripts.resmokelib.config.SHELL_CONN_STRING = None
test_runner = MockTestRunner("test")
test_runner._setup_logging()
test_runner._check_for_mongo_processes()
tmp_ctime = os.environ['RESMOKE_PARENT_CTIME']
os.environ['RESMOKE_PARENT_CTIME'] = str("rogue_process")
proc = process.Process(logging.getLogger(), self.command)
proc.start()
os.environ['RESMOKE_PARENT_CTIME'] = tmp_ctime
test_runner._check_for_mongo_processes()
proc.wait()
if proc._process.returncode != self.sigkill_return:
self.fail(
f"Detected processes was not killed by resmoke, exit code was {proc._process.returncode}, expected {self.sigkill_return}"
)
def _setup_jasper(self):
"""Start up the jasper process manager."""
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
proto_file = os.path.join(root_dir, "buildscripts", "resmokelib",
"core", "jasper.proto")
try:
well_known_protos_include = pkg_resources.resource_filename(
"grpc_tools", "_proto")
except ImportError:
raise ImportError(
"You must run: sys.executable + '-m pip install grpcio grpcio-tools "
"googleapis-common-protos' to use --spawnUsing=jasper.")
# We use the build/ directory as the output directory because the generated files aren't
# meant to because tracked by git or linted.
proto_out = os.path.join(root_dir, "build", "jasper")
utils.rmtree(proto_out, ignore_errors=True)
os.makedirs(proto_out)
# We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the
# *pb2*.py modules from it.
with open(os.path.join(proto_out, "__init__.py"), "w"):
pass
ret = grpc_tools.protoc.main([
grpc_tools.protoc.__file__,
"--grpc_python_out",
proto_out,
"--python_out",
proto_out,
"--proto_path",
os.path.dirname(proto_file),
"--proto_path",
well_known_protos_include,
os.path.basename(proto_file),
])
if ret != 0:
raise RuntimeError(
"Failed to generated gRPC files from the jasper.proto file")
sys.path.append(os.path.dirname(proto_out))
from jasper import jasper_pb2
from jasper import jasper_pb2_grpc
jasper_process.Process.jasper_pb2 = jasper_pb2
jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc
curator_path = "build/curator"
if sys.platform == "win32":
curator_path += ".exe"
git_hash = "d846f0c875716e9377044ab2a50542724369662a"
curator_exists = os.path.isfile(curator_path)
curator_same_version = False
if curator_exists:
curator_version = subprocess.check_output(
[curator_path, "--version"]).decode('utf-8').split()
curator_same_version = git_hash in curator_version
if curator_exists and not curator_same_version:
os.remove(curator_path)
self._resmoke_logger.info(
"Found a different version of curator. Downloading version %s of curator to enable"
"process management using jasper.", git_hash)
if not curator_exists or not curator_same_version:
if sys.platform == "darwin":
os_platform = "macos"
elif sys.platform == "win32":
os_platform = "windows-64"
elif sys.platform.startswith("linux"):
os_platform = "ubuntu1604"
else:
raise OSError(
"Unrecognized platform. "
"This program is meant to be run on MacOS, Windows, or Linux."
)
url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/"
"curator-dist-%s-%s.tar.gz") % (os_platform, git_hash)
response = requests.get(url, stream=True)
with tarfile.open(mode="r|gz", fileobj=response.raw) as tf:
tf.extractall(path="./build/")
jasper_port = config.BASE_PORT - 1
jasper_conn_str = "localhost:%d" % jasper_port
jasper_process.Process.connection_str = jasper_conn_str
jasper_command = [
curator_path, "jasper", "grpc", "--port",
str(jasper_port)
]
self._jasper_server = process.Process(self._resmoke_logger,
jasper_command)
self._jasper_server.start()
channel = grpc.insecure_channel(jasper_conn_str)
grpc.channel_ready_future(channel).result()