Example #1
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments, get_unknowns=True)

        # load or store parameters
        if self.args["restart"]:
            self.load_parameters()
        elif self.args["stop"]:
            if self.args["verbose"]:
                self.verbose = True
            self.load_parameters()
            self.args["stop"] = True
        else:
            self.store_parameters()

        # check if authentication is enabled
        if self.args["authentication"]:
            if not os.path.exists(self.args["dir"]):
                os.makedirs(self.args["dir"])
            os.system("openssl rand -base64 753 > %s/keyfile" % self.args["dir"])
            os.system("chmod 600 %s/keyfile" % self.args["dir"])

        if self.args["stop"]:
            self._stopAll(self.args["dir"])
        else:
            if self.args["sharded"]:
                self._launchSharded()
            elif self.args["single"]:
                self._launchSingle(self.args["dir"], self.args["port"])
            elif self.args["replicaset"]:
                self._launchReplSet(self.args["dir"], self.args["port"], self.args["name"])
Example #2
0
    def __init__(self):
        BaseCmdLineTool.__init__(self)

        self.argparser.description = "Script to generate pseudo-random data based on template documents."

        self.argparser.add_argument("template", action="store", help="template for data generation, JSON or file")
        self.argparser.add_argument(
            "--number", "-n", action="store", type=int, metavar="NUM", default=1, help="number of documents to insert."
        )
        self.argparser.add_argument(
            "--host", action="store", default="localhost", help="mongod/s host to import data, default=localhost"
        )
        self.argparser.add_argument(
            "--port", action="store", default=27017, type=int, help="mongod/s port to import data, default=27017"
        )
        self.argparser.add_argument(
            "--database",
            "-d",
            action="store",
            metavar="D",
            default="test",
            help="database D to insert data, default=test",
        )
        self.argparser.add_argument(
            "--collection",
            "-c",
            action="store",
            metavar="C",
            default="mgendata",
            help="collection C to import data, default=mgendata",
        )
        self.argparser.add_argument(
            "--drop", action="store_true", default=False, help="drop collection before inserting data"
        )
        self.argparser.add_argument(
            "--stdout",
            action="store_true",
            default=False,
            help="prints data to stdout instead of inserting to mongod/s instance.",
        )
        self.argparser.add_argument(
            "--pretty",
            action="store_true",
            default=False,
            help="if set, prettyfies the output to stdout (indented), requires --stdout",
        )
        self.argparser.add_argument(
            "--write-concern", "-w", action="store", metavar="W", default=1, help="write concern for inserts, default=1"
        )
        self.argparser.add_argument(
            "--processes",
            "-p",
            action="store",
            type=int,
            default=0,
            help="specify number of processes (# cpus by default)",
        )
Example #3
0
    def __init__(self):
        """ Constructor: add description to argparser. """
        BaseCmdLineTool.__init__(self)

        self.argparser.description = "Performs a health check on config servers and compares them for inconsistencies."
        self.argparser.add_argument(
            "config",
            action="store",
            nargs="*",
            metavar="URI",
            default=["mongodb://localhost:27017/config"],
            help="provide uri(s) to config server(s), default is mongodb://localhost:27017/config",
        )
Example #4
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments)

        # number of configs
        self.num_configs = len(self.args["config"])

        # parse config server URIs
        regex = re.compile(
            r"(?P<mongodb>mongodb://)?((?P<user>\w+):(?P<password>\w+)@)?(?P<host>\w+)(:(?P<port>\d+))?/(?P<database>\w+)"
        )
        matches = [regex.match(uri) for uri in self.args["config"]]

        # verify that all config URIs are parsed correctly and contain a database
        # TODO check for empty config dbs
        if not all(matches) or not all(m.groupdict()["database"] for m in matches):
            raise SystemExit(
                "Unable to parse config server URIs, please check syntax: mongodb://[username:password@]host[:port]/database"
            )

        # for convenience mongodb:// can be omitted for this script, but MongoClient expects it
        for i, uri in enumerate(self.args["config"]):
            if not uri.startswith("mongodb://"):
                self.args["config"][i] = "mongodb://" + uri

        # add uri and short_uri to self.parsed_uri dicts
        self.parsed_uris = [m.groupdict() for m in matches]
        for i, (uri, puri) in enumerate(zip(self.args["config"], self.parsed_uris)):
            self.parsed_uris[i]["uri"] = uri
            if puri["port"] == None:
                self.parsed_uris[i]["port"] = "27017"
            self.parsed_uris[i]["short_uri"] = "%s:%s/%s" % (puri["host"], puri["port"], puri["database"])

        # connect to databases
        self.config_dbs = [MongoClient(puri["uri"])[puri["database"]] for puri in self.parsed_uris]

        print "\n>> individual health checks on all config servers"
        print "   (verifies that for each namespace, the chunk ranges reach from MinKey to MaxKey without gaps or overlaps)\n"
        for puri, database in zip(self.parsed_uris, self.config_dbs):
            print puri["short_uri"]
            self._health_check(database)
            print

        if len(self.config_dbs) > 1:
            print "\n>> comparing config.collections collection for each config server"
            print "   (verifies that they agree on the state of each collection)\n"
            self._compare_collections()

            print "\n>> comparing config.chunks collection for each config server"
            print "   (verifies that they agree on chunk ranges for each namespace and finds deviation point)\n"
            self._compare_chunks_and_reconstruct()
Example #5
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments)

        if self.args["template"].startswith("{"):
            # not a file
            try:
                template = json.loads(self.args["template"])
            except ValueError as e:
                raise SystemExit("can't parse template: %s" % e)
        else:
            try:
                f = open(self.args["template"])
            except IOError as e:
                raise SystemExit("can't open file %s: %s" % (self.args["template"], e))

            try:
                template = json.load(f)
            except ValueError as e:
                raise SystemExit("can't parse template in %s: %s" % (self.args["template"], e))

        if not self.args["stdout"]:
            mc = Connection(host=self.args["host"], port=self.args["port"], w=self.args["write_concern"], connect=False)
            col = mc[self.args["database"]][self.args["collection"]]
            if self.args["drop"]:
                col.drop()
        else:
            col = None

        # divide work over number of cores
        if self.args["stdout"]:
            num_cores = 1
        elif self.args["processes"] > 0:
            num_cores = self.args["processes"]
        else:
            num_cores = cpu_count()

        num_list = [self.args["number"] // num_cores] * num_cores
        num_list[0] += self.args["number"] % num_cores

        processes = []

        for n in num_list:
            p = InsertProcess(n, template, col, self.args)
            p.start()
            processes.append(p)

        for p in processes:
            p.join()
Example #6
0
    def __init__(self):
        BaseCmdLineTool.__init__(self)

        self.hostname = socket.gethostname()

        # arguments
        self.args = None

        # startup parameters for each port
        self.startup_info = {}

        # data structures for the discovery feature
        self.cluster_tree = {}
        self.cluster_tags = defaultdict(list)
        self.cluster_running = {}

        # config docs for replica sets (key is replica set name)
        self.config_docs = {}

        # shard connection strings
        self.shard_connection_str = []
Example #7
0
    def __init__(self):
        BaseCmdLineTool.__init__(self)
        self.argparser.description = "script to launch MongoDB stand-alone servers, replica sets and shards. You must specify either --single or --replicaset. \
            In addition to the optional arguments below, you can specify any mongos and mongod argument, which will be passed on, if the process accepts it."

        # either single or replica set
        me_group = self.argparser.add_mutually_exclusive_group(required=True)
        me_group.add_argument("--single", action="store_true", help="creates a single stand-alone mongod instance")
        me_group.add_argument(
            "--replicaset", action="store_true", help="creates replica set with several mongod instances"
        )
        me_group.add_argument(
            "--restart",
            action="store_true",
            help="restarts a previously launched existing configuration from the data directory.",
        )

        # replica set arguments
        self.argparser.add_argument(
            "--nodes",
            action="store",
            metavar="NUM",
            type=int,
            default=3,
            help="adds NUM data nodes to replica set (requires --replicaset, default=3)",
        )
        self.argparser.add_argument(
            "--arbiter", action="store_true", default=False, help="adds arbiter to replica set (requires --replicaset)"
        )
        self.argparser.add_argument(
            "--name", action="store", metavar="NAME", default="replset", help="name for replica set (default=replset)"
        )

        # sharded clusters
        self.argparser.add_argument(
            "--sharded",
            action="store",
            nargs="*",
            metavar="N",
            help="creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards (default=1)",
        )
        self.argparser.add_argument(
            "--config",
            action="store",
            default=1,
            type=int,
            metavar="NUM",
            choices=[1, 3],
            help="adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)",
        )
        self.argparser.add_argument(
            "--mongos",
            action="store",
            default=1,
            type=int,
            metavar="NUM",
            help="starts NUM mongos processes (requires --sharded, default=1)",
        )

        # dir, verbose, port, auth
        self.argparser.add_argument(
            "--dir",
            action="store",
            default="./data",
            help="base directory to create db and log paths (default=./data/)",
        )
        self.argparser.add_argument(
            "--verbose", action="store_true", default=False, help="outputs information about the launch"
        )
        self.argparser.add_argument(
            "--port",
            action="store",
            type=int,
            default=27017,
            help="port for mongod, start of port range in case of replica set or shards (default=27017)",
        )
        self.argparser.add_argument(
            "--authentication",
            action="store_true",
            default=False,
            help="enable authentication and create a key file and admin user (admin/mypassword)",
        )
        self.argparser.add_argument(
            "--binarypath",
            action="store",
            default=None,
            metavar="PATH",
            help="search for mongod/s binaries in the specified PATH.",
        )

        self.hostname = socket.gethostname()
Example #8
0
    def run(self, arguments=None):
        """ This is the main run method, called for all sub-commands and parameters.
            It sets up argument parsing, then calls the sub-command method with the same name.
        """

        # set up argument parsing in run, so that subsequent calls to run can call different sub-commands
        self.argparser = argparse.ArgumentParser()
        self.argparser.add_argument("--version", action="version", version="mtools version %s" % __version__)

        self.argparser.description = "script to launch MongoDB stand-alone servers, replica sets and shards."

        # make sure init is default command even when specifying arguments directly
        if arguments and arguments.startswith("-"):
            arguments = "init " + arguments

        # default sub-command is `init` if none provided
        elif len(sys.argv) > 1 and sys.argv[1].startswith("-") and sys.argv[1] not in ["-h", "--help"]:
            sys.argv = sys.argv[0:1] + ["init"] + sys.argv[1:]

        # create command sub-parsers
        subparsers = self.argparser.add_subparsers(dest="command")
        self.argparser._action_groups[0].title = "commands"
        self.argparser._action_groups[
            0
        ].description = "init is the default command and can be omitted. To get help on individual commands, run mlaunch [command] --help"

        # init command
        init_parser = subparsers.add_parser(
            "init",
            help="initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters.",
            description="initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters",
        )

        # either single or replica set
        me_group = init_parser.add_mutually_exclusive_group(required=True)
        me_group.add_argument("--single", action="store_true", help="creates a single stand-alone mongod instance")
        me_group.add_argument(
            "--replicaset", action="store_true", help="creates replica set with several mongod instances"
        )

        # replica set arguments
        init_parser.add_argument(
            "--nodes",
            action="store",
            metavar="NUM",
            type=int,
            default=3,
            help="adds NUM data nodes to replica set (requires --replicaset, default=3)",
        )
        init_parser.add_argument(
            "--arbiter", action="store_true", default=False, help="adds arbiter to replica set (requires --replicaset)"
        )
        init_parser.add_argument(
            "--name", action="store", metavar="NAME", default="replset", help="name for replica set (default=replset)"
        )

        # sharded clusters
        init_parser.add_argument(
            "--sharded",
            action="store",
            nargs="*",
            metavar="N",
            help="creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards (default=1)",
        )
        init_parser.add_argument(
            "--config",
            action="store",
            default=1,
            type=int,
            metavar="NUM",
            choices=[1, 3],
            help="adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)",
        )
        init_parser.add_argument(
            "--mongos",
            action="store",
            default=1,
            type=int,
            metavar="NUM",
            help="starts NUM mongos processes (requires --sharded, default=1)",
        )

        # verbose, port, auth, binary path
        init_parser.add_argument(
            "--verbose", action="store_true", default=False, help="outputs more verbose information."
        )
        init_parser.add_argument(
            "--port",
            action="store",
            type=int,
            default=27017,
            help="port for mongod, start of port range in case of replica set or shards (default=27017)",
        )
        init_parser.add_argument(
            "--authentication",
            action="store_true",
            default=False,
            help="enable authentication and create a key file and admin user (admin/mypassword)",
        )
        init_parser.add_argument(
            "--binarypath",
            action="store",
            default=None,
            metavar="PATH",
            help="search for mongod/s binaries in the specified PATH.",
        )
        init_parser.add_argument(
            "--dir",
            action="store",
            default="./data",
            help="base directory to create db and log paths (default=./data/)",
        )

        # start command
        start_parser = subparsers.add_parser(
            "start",
            help='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.',
            description='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.',
        )
        start_parser.add_argument(
            "tags",
            metavar="TAG",
            action="store",
            nargs="*",
            default=[],
            help="without tags, all non-running nodes will be restarted. Provide additional tags to narrow down the set of nodes to start.",
        )
        start_parser.add_argument(
            "--verbose", action="store_true", default=False, help="outputs more verbose information."
        )
        start_parser.add_argument(
            "--dir", action="store", default="./data", help="base directory to start nodes (default=./data/)"
        )

        # stop command
        stop_parser = subparsers.add_parser(
            "stop",
            help='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.',
            description='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.',
        )
        stop_parser.add_argument(
            "tags",
            metavar="TAG",
            action="store",
            nargs="*",
            default=[],
            help="without tags, all running nodes will be stopped. Provide additional tags to narrow down the set of nodes to stop.",
        )
        stop_parser.add_argument(
            "--verbose", action="store_true", default=False, help="outputs more verbose information."
        )
        stop_parser.add_argument(
            "--dir", action="store", default="./data", help="base directory to stop nodes (default=./data/)"
        )

        # list command
        list_parser = subparsers.add_parser(
            "list",
            help="list MongoDB instances for this configuration",
            description="list MongoDB instances for this configuration",
        )
        list_parser.add_argument(
            "--dir", action="store", default="./data", help="base directory to list nodes (default=./data/)"
        )
        list_parser.add_argument(
            "--verbose", action="store_true", default=False, help="outputs more verbose information."
        )

        # argparser is set up, now call base class run()
        BaseCmdLineTool.run(self, arguments, get_unknowns=True)

        # replace path with absolute path
        self.dir = os.path.abspath(self.args["dir"])

        # branch out in sub-commands
        getattr(self, self.args["command"])()