def validate_all(self): """Start a standalone node to validate all collections on the copied data files.""" for node in self.fixture.nodes: path = node.get_dbpath_prefix() + "/simulateCrashes/{}".format( self.backup_num) self.logger.info( "Starting to validate. DBPath: {} Port: {}".format( path, self.validate_port)) mdb = process.Process(self.logger, [ node.mongod_executable, "--dbpath", path, "--port", str(self.validate_port), "--logpath", node.get_dbpath_prefix() + "/simulateCrashes/validate.log", "--setParameter", "enableTestCommands=1" ]) mdb.start() client = pymongo.MongoClient(host="localhost", port=self.validate_port, connect=True, connectTimeoutMS=120000, serverSelectionTimeoutMS=120000, directConnection=True) is_valid = validate(client, self.logger, self.acceptable_err_codes) mdb.stop() mdb.wait() if not is_valid: return False shutil.rmtree(path, ignore_errors=True) return True
def validate_all(self, backup_num): """Start a standalone node to validate all collections on the copied data files.""" for node in self.fixture.nodes: if self.last_validate_port >= 20000: self.last_validate_port = 19000 validate_port = self.last_validate_port self.last_validate_port += 1 path = node.get_dbpath_prefix() + "/simulateCrashes/{}".format( backup_num) self.logger.info( "Starting to validate. DBPath: {} Port: {}".format( path, validate_port)) mdb = process.Process(self.logger, [ node.mongod_executable, "--dbpath", path, "--port", str(validate_port), "--logpath", node.get_dbpath_prefix() + "/simulateCrashes/validate.log" ]) mdb.start() client = pymongo.MongoClient("localhost:{}".format(validate_port)) is_valid = validate(client, self.logger) mdb.stop() mdb.wait() if not is_valid: return False shutil.rmtree(path, ignore_errors=True) return True
def test_off(self): buildscripts.resmokelib.config.AUTO_KILL = 'off' buildscripts.resmokelib.config.SHELL_CONN_STRING = None test_runner = MockTestRunner("test") test_runner._setup_logging() test_runner._check_for_mongo_processes() proc = process.Process(logging.getLogger(), self.command) proc.start() test_runner._check_for_mongo_processes() proc.stop(mode=fixture_interface.TeardownMode.ABORT) if proc._process.returncode == self.sigkill_return: self.fail("Process was killed when it should not have been.") proc.wait()
def test_warn(self): buildscripts.resmokelib.config.AUTO_KILL = 'warn' buildscripts.resmokelib.config.SHELL_CONN_STRING = None test_runner = MockTestRunner("test") test_runner._setup_logging() try: test_runner._check_for_mongo_processes() except errors.ResmokeError: self.fail("Detected processes when there should be none.") tmp_ctime = os.environ['RESMOKE_PARENT_CTIME'] os.environ['RESMOKE_PARENT_CTIME'] = str("rogue_process") proc = process.Process(logging.getLogger(), self.command) proc.start() os.environ['RESMOKE_PARENT_CTIME'] = tmp_ctime with self.assertRaises(errors.ResmokeError): test_runner._check_for_mongo_processes() proc.stop(mode=fixture_interface.TeardownMode.KILL) proc.wait()
def test_on(self): buildscripts.resmokelib.config.AUTO_KILL = 'on' buildscripts.resmokelib.config.SHELL_CONN_STRING = None test_runner = MockTestRunner("test") test_runner._setup_logging() test_runner._check_for_mongo_processes() tmp_ctime = os.environ['RESMOKE_PARENT_CTIME'] os.environ['RESMOKE_PARENT_CTIME'] = str("rogue_process") proc = process.Process(logging.getLogger(), self.command) proc.start() os.environ['RESMOKE_PARENT_CTIME'] = tmp_ctime test_runner._check_for_mongo_processes() proc.wait() if proc._process.returncode != self.sigkill_return: self.fail( f"Detected processes was not killed by resmoke, exit code was {proc._process.returncode}, expected {self.sigkill_return}" )
def _setup_jasper(self): """Start up the jasper process manager.""" root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto") try: well_known_protos_include = pkg_resources.resource_filename( "grpc_tools", "_proto") except ImportError: raise ImportError( "You must run: sys.executable + '-m pip install grpcio grpcio-tools " "googleapis-common-protos' to use --spawnUsing=jasper.") # We use the build/ directory as the output directory because the generated files aren't # meant to because tracked by git or linted. proto_out = os.path.join(root_dir, "build", "jasper") utils.rmtree(proto_out, ignore_errors=True) os.makedirs(proto_out) # We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the # *pb2*.py modules from it. with open(os.path.join(proto_out, "__init__.py"), "w"): pass ret = grpc_tools.protoc.main([ grpc_tools.protoc.__file__, "--grpc_python_out", proto_out, "--python_out", proto_out, "--proto_path", os.path.dirname(proto_file), "--proto_path", well_known_protos_include, os.path.basename(proto_file), ]) if ret != 0: raise RuntimeError( "Failed to generated gRPC files from the jasper.proto file") sys.path.append(os.path.dirname(proto_out)) from jasper import jasper_pb2 from jasper import jasper_pb2_grpc jasper_process.Process.jasper_pb2 = jasper_pb2 jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc curator_path = "build/curator" if sys.platform == "win32": curator_path += ".exe" git_hash = "d846f0c875716e9377044ab2a50542724369662a" curator_exists = os.path.isfile(curator_path) curator_same_version = False if curator_exists: curator_version = subprocess.check_output( [curator_path, "--version"]).decode('utf-8').split() curator_same_version = git_hash in curator_version if curator_exists and not curator_same_version: os.remove(curator_path) self._resmoke_logger.info( "Found a different version of curator. Downloading version %s of curator to enable" "process management using jasper.", git_hash) if not curator_exists or not curator_same_version: if sys.platform == "darwin": os_platform = "macos" elif sys.platform == "win32": os_platform = "windows-64" elif sys.platform.startswith("linux"): os_platform = "ubuntu1604" else: raise OSError( "Unrecognized platform. " "This program is meant to be run on MacOS, Windows, or Linux." ) url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/" "curator-dist-%s-%s.tar.gz") % (os_platform, git_hash) response = requests.get(url, stream=True) with tarfile.open(mode="r|gz", fileobj=response.raw) as tf: tf.extractall(path="./build/") jasper_port = config.BASE_PORT - 1 jasper_conn_str = "localhost:%d" % jasper_port jasper_process.Process.connection_str = jasper_conn_str jasper_command = [ curator_path, "jasper", "grpc", "--port", str(jasper_port) ] self._jasper_server = process.Process(self._resmoke_logger, jasper_command) self._jasper_server.start() channel = grpc.insecure_channel(jasper_conn_str) grpc.channel_ready_future(channel).result()