Example #1
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments, get_unknowns=True)

        # load or store parameters
        if self.args['restart']:
            self.load_parameters()
        elif self.args['stop']:
            if self.args['verbose']:
                self.verbose = True
            self.load_parameters()
            self.args['stop'] = True
        else:
            self.store_parameters()

        
        # check if authentication is enabled        
        if self.args['authentication']:
            if not os.path.exists(self.args['dir']):
                os.makedirs(self.args['dir'])
            os.system('openssl rand -base64 753 > %s/keyfile'%self.args['dir'])
            os.system('chmod 600 %s/keyfile'%self.args['dir'])

        if self.args['stop']:
            self._stopAll(self.args['dir'])
        else:
            if self.args['sharded']:
                self._launchSharded()
            elif self.args['single']:
                self._launchSingle(self.args['dir'], self.args['port'])
            elif self.args['replicaset']:
                self._launchReplSet(self.args['dir'], self.args['port'], self.args['name'])
Example #2
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments)

        if self.args['template'].startswith('{'):
            # not a file
            try:
                template = json.loads(self.args['template'])
            except ValueError as e:
                raise SystemExit("can't parse template: %s" % e)
        else:
            try:
                f = open(self.args['template'])
            except IOError as e:
                raise SystemExit("can't open file %s: %s" % (self.args['template'], e))

            try:
                template = json.load(f)
            except ValueError as e:
                raise SystemExit("can't parse template in %s: %s" % (self.args['template'], e))


        if not self.args['stdout']:        
            mc = Connection(host=self.args['host'], port=self.args['port'], w=self.args['write_concern'])        
            col = mc[self.args['database']][self.args['collection']]
            if self.args['drop']:
                col.drop()
        else:
            col = None

        # divide work over number of cores
        if self.args['stdout']:
            num_cores = 1
        elif self.args['processes'] > 0:
            num_cores = self.args['processes']
        else:
            num_cores = cpu_count()

        num_list = [self.args['number'] // num_cores] * num_cores
        num_list[0] += self.args['number'] % num_cores

        processes = []

        for n in num_list:
            p = InsertProcess(n, template, col, self.args)
            p.start()
            processes.append(p)

        for p in processes:
            p.join()
Example #3
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments)

        # number of configs
        self.num_configs = len(self.args['config'])

        # parse config server URIs
        regex = re.compile(r'(?P<mongodb>mongodb://)?((?P<user>\w+):(?P<password>\w+)@)?(?P<host>\w+)(:(?P<port>\d+))?/(?P<database>\w+)')
        matches = [ regex.match(uri) for uri in self.args['config'] ]

        # verify that all config URIs are parsed correctly and contain a database
        # TODO check for empty config dbs
        if not all(matches) or not all(m.groupdict()['database'] for m in matches):
            raise SystemExit('Unable to parse config server URIs, please check syntax: mongodb://[username:password@]host[:port]/database')

        # for convenience mongodb:// can be omitted for this script, but MongoClient expects it
        for i, uri in enumerate(self.args['config']):
            if not uri.startswith('mongodb://'):
                self.args['config'][i] = 'mongodb://' + uri

        # add uri and short_uri to self.parsed_uri dicts
        self.parsed_uris = [m.groupdict() for m in matches]
        for i, (uri, puri) in enumerate(zip(self.args['config'], self.parsed_uris)):
            self.parsed_uris[i]['uri'] = uri
            if puri['port'] == None:
                self.parsed_uris[i]['port'] = '27017'
            self.parsed_uris[i]['short_uri'] = '%s:%s/%s' % (puri['host'], puri['port'], puri['database'])

        # connect to databases
        self.config_dbs = [ MongoClient(puri['uri'])[puri['database']] for puri in self.parsed_uris ]

        print "\n>> individual health checks on all config servers"
        print   "   (verifies that for each namespace, the chunk ranges reach from MinKey to MaxKey without gaps or overlaps)\n"
        for puri, database in zip(self.parsed_uris, self.config_dbs):
            print puri['short_uri']
            self._health_check(database)
            print 

        if len(self.config_dbs) > 1:
            print "\n>> comparing config.collections collection for each config server"
            print   "   (verifies that they agree on the state of each collection)\n"
            self._compare_collections()


            print "\n>> comparing config.chunks collection for each config server"
            print   "   (verifies that they agree on chunk ranges for each namespace and finds deviation point)\n"
            self._compare_chunks_and_reconstruct()
Example #4
0
    def run(self, arguments=None):
        BaseCmdLineTool.run(self, arguments, get_unknowns=True)

        # load or store parameters
        if self.args["restart"]:
            self.load_parameters()
        else:
            self.store_parameters()

        # check if authentication is enabled
        if self.args["authentication"]:
            if not os.path.exists(self.args["dir"]):
                os.makedirs(self.args["dir"])
            os.system("openssl rand -base64 753 > %s/keyfile" % self.args["dir"])
            os.system("chmod 600 %s/keyfile" % self.args["dir"])

        if self.args["sharded"]:
            self._launchSharded()
        elif self.args["single"]:
            self._launchSingle(self.args["dir"], self.args["port"])
        elif self.args["replicaset"]:
            self._launchReplSet(self.args["dir"], self.args["port"], self.args["name"])
Example #5
0
    def run(self, arguments=None):
        """ This is the main run method, called for all sub-commands and parameters.
            It sets up argument parsing, then calls the sub-command method with the same name.
        """

        # set up argument parsing in run, so that subsequent calls to run can call different sub-commands
        self.argparser = argparse.ArgumentParser()
        self.argparser.add_argument('--version', action='version', version="mtools version %s" % __version__)

        self.argparser.description = 'script to launch MongoDB stand-alone servers, replica sets and shards.'

        # make sure init is default command even when specifying arguments directly
        if arguments and arguments.startswith('-'):
            arguments = 'init ' + arguments
        
        # default sub-command is `init` if none provided
        elif len(sys.argv) > 1 and sys.argv[1].startswith('-') and sys.argv[1] not in ['-h', '--help']:
            sys.argv = sys.argv[0:1] + ['init'] + sys.argv[1:]

        # create command sub-parsers
        subparsers = self.argparser.add_subparsers(dest='command')
        self.argparser._action_groups[0].title = 'commands'
        self.argparser._action_groups[0].description = 'init is the default command and can be omitted. To get help on individual commands, run mlaunch [command] --help'
        
        # init command 
        init_parser = subparsers.add_parser('init', help='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters.',
            description='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters')

        # either single or replica set
        me_group = init_parser.add_mutually_exclusive_group(required=True)
        me_group.add_argument('--single', action='store_true', help='creates a single stand-alone mongod instance')
        me_group.add_argument('--replicaset', action='store_true', help='creates replica set with several mongod instances')

        # replica set arguments
        init_parser.add_argument('--nodes', action='store', metavar='NUM', type=int, default=3, help='adds NUM data nodes to replica set (requires --replicaset, default=3)')
        init_parser.add_argument('--arbiter', action='store_true', default=False, help='adds arbiter to replica set (requires --replicaset)')
        init_parser.add_argument('--name', action='store', metavar='NAME', default='replset', help='name for replica set (default=replset)')
        
        # sharded clusters
        init_parser.add_argument('--sharded', action='store', nargs='*', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards (default=1)')
        init_parser.add_argument('--config', action='store', default=1, type=int, metavar='NUM', choices=[1, 3], help='adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)')
        init_parser.add_argument('--mongos', action='store', default=1, type=int, metavar='NUM', help='starts NUM mongos processes (requires --sharded, default=1)')

        # verbose, port, auth, binary path
        init_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
        init_parser.add_argument('--port', action='store', type=int, default=27017, help='port for mongod, start of port range in case of replica set or shards (default=27017)')
        init_parser.add_argument('--authentication', action='store_true', default=False, help='enable authentication and create a key file and admin user (admin/mypassword)')
        init_parser.add_argument('--binarypath', action='store', default=None, metavar='PATH', help='search for mongod/s binaries in the specified PATH.')
        init_parser.add_argument('--dir', action='store', default='./data', help='base directory to create db and log paths (default=./data/)')

        # start command
        start_parser = subparsers.add_parser('start', help='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.', 
            description='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.')
        start_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all non-running nodes will be restarted. Provide additional tags to narrow down the set of nodes to start.')
        start_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
        start_parser.add_argument('--dir', action='store', default='./data', help='base directory to start nodes (default=./data/)')

        # stop command
        stop_parser = subparsers.add_parser('stop', help='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.',
            description='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.')
        stop_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all running nodes will be stopped. Provide additional tags to narrow down the set of nodes to stop.')
        stop_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
        stop_parser.add_argument('--dir', action='store', default='./data', help='base directory to stop nodes (default=./data/)')
        
        # list command
        list_parser = subparsers.add_parser('list', help='list MongoDB instances for this configuration',
            description='list MongoDB instances for this configuration')
        list_parser.add_argument('--dir', action='store', default='./data', help='base directory to list nodes (default=./data/)')
        list_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')

        # argparser is set up, now call base class run()
        BaseCmdLineTool.run(self, arguments, get_unknowns=True)

        # replace path with absolute path
        self.dir = os.path.abspath(self.args['dir'])

        # branch out in sub-commands
        getattr(self, self.args['command'])()