def run(self, arguments=None): BaseCmdLineTool.run(self, arguments, get_unknowns=True) # load or store parameters if self.args['restart']: self.load_parameters() elif self.args['stop']: if self.args['verbose']: self.verbose = True self.load_parameters() self.args['stop'] = True else: self.store_parameters() # check if authentication is enabled if self.args['authentication']: if not os.path.exists(self.args['dir']): os.makedirs(self.args['dir']) os.system('openssl rand -base64 753 > %s/keyfile'%self.args['dir']) os.system('chmod 600 %s/keyfile'%self.args['dir']) if self.args['stop']: self._stopAll(self.args['dir']) else: if self.args['sharded']: self._launchSharded() elif self.args['single']: self._launchSingle(self.args['dir'], self.args['port']) elif self.args['replicaset']: self._launchReplSet(self.args['dir'], self.args['port'], self.args['name'])
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments) if self.args['template'].startswith('{'): # not a file try: template = json.loads(self.args['template']) except ValueError as e: raise SystemExit("can't parse template: %s" % e) else: try: f = open(self.args['template']) except IOError as e: raise SystemExit("can't open file %s: %s" % (self.args['template'], e)) try: template = json.load(f) except ValueError as e: raise SystemExit("can't parse template in %s: %s" % (self.args['template'], e)) if not self.args['stdout']: mc = Connection(host=self.args['host'], port=self.args['port'], w=self.args['write_concern'], connect=False) col = mc[self.args['database']][self.args['collection']] if self.args['drop']: col.drop() else: col = None # divide work over number of cores if self.args['stdout']: num_cores = 1 elif self.args['processes'] > 0: num_cores = self.args['processes'] else: num_cores = cpu_count() num_list = [self.args['number'] // num_cores] * num_cores num_list[0] += self.args['number'] % num_cores processes = [] for n in num_list: p = InsertProcess(n, template, col, self.args) p.start() processes.append(p) for p in processes: p.join()
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments) self.dbpath = self.args['dbpath'][0] self.force = self.args['force'] self.verbose = self.args['verbose'] # Read storage.bson, sanity check. try: storage_raw = open(os.path.join(self.dbpath, 'storage.bson'), 'rb').read() except Exception as e: sys.stderr.write('Failed to open storage.bson in "{0}": {1}\n'.format(self.dbpath, e)) return settings = bson.decode(storage_raw)["storage"]["options"] if not settings["directoryPerDB"]: sys.stderr.write('Requires a database created with --directoryperdb\n') return if settings["directoryForIndexes"] or settings.get("groupCollections", False): sys.stderr.write('Incompatible storage settings detected: ' 'directoryForIndexes or groupCollections\n') if not self.force: return self.database = self.args['database'][0] self.nsprefix = self.database + '.' mtransfer_dir = os.path.join(self.dbpath, self.database) mtransfer_file = os.path.join(mtransfer_dir, 'mtransfer.bson') if self.args['command'] == 'export': if not os.path.exists(mtransfer_dir): sys.stderr.write('Expected source directory "{0}" does not exist. ' 'Check the database name is correct.\n'.format(mtransfer_dir)) return if not self.force and os.path.exists(mtransfer_file): sys.stderr.write('Output file "{0}" already exists\n'.format(mtransfer_file)) return with open(mtransfer_file, 'wb') as outf: self.doExport(outf) elif self.args['command'] == 'import': if not os.path.exists(mtransfer_dir): sys.stderr.write('Expected target directory "{0}" does not exist. ' 'Check the database name is correct.\n'.format(mtransfer_dir)) return if not os.path.exists(mtransfer_file): sys.stderr.write('Cannot import: mtransfer file "{0}" does not exist.\n'. format(mtransfer_file)) return with open(mtransfer_file, 'rb') as inf: self.doImport(inf)
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments) if self.args['template'].startswith('{'): # not a file try: template = json.loads(self.args['template']) except ValueError as e: raise SystemExit("can't parse template: %s" % e) else: try: f = open(self.args['template']) except IOError as e: raise SystemExit("can't open file %s: %s" % (self.args['template'], e)) try: template = json.load(f) except ValueError as e: raise SystemExit("can't parse template in %s: %s" % (self.args['template'], e)) if not self.args['stdout']: mc = Connection(host=self.args['host'], port=self.args['port'], w=self.args['write_concern']) col = mc[self.args['database']][self.args['collection']] if self.args['drop']: col.drop() else: col = None # divide work over number of cores if self.args['stdout']: num_cores = 1 elif self.args['processes'] > 0: num_cores = self.args['processes'] else: num_cores = cpu_count() num_list = [self.args['number'] // num_cores] * num_cores num_list[0] += self.args['number'] % num_cores processes = [] for n in num_list: p = InsertProcess(n, template, col, self.args) p.start() processes.append(p) for p in processes: p.join()
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments) if self.args['template'].startswith('{'): # not a file try: template = json.loads(self.args['template']) except ValueError as e: raise SystemExit("can't parse template: %s" % e) else: try: f = open(self.args['template']) except IOError as e: raise SystemExit("can't open file %s: %s" % (self.args['template'], e)) try: template = json.load(f) except ValueError as e: raise SystemExit("can't parse template in %s: %s" % (self.args['template'], e)) if not self.args['stdout']: mc = Connection(host=self.args['host'], port=self.args['port'], w=self.args['write_concern']) col = mc[self.args['database']][self.args['collection']] if self.args['drop']: col.drop() batch = [] for n in xrange(self.args['number']): # decode the template doc = self._decode(template) if self.args['stdout']: print doc else: batch.append(doc) if n % 100 == 0: col.insert(batch) batch = [] self.update_progress(float(n) / self.args['number'], prefix='inserting data') if not self.args['stdout']: self.update_progress(1.0, prefix='inserting data') if batch: col.insert(batch)
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments) # number of configs self.num_configs = len(self.args['config']) # parse config server URIs regex = re.compile(r'(?P<mongodb>mongodb://)?((?P<user>\w+):(?P<password>\w+)@)?(?P<host>\w+)(:(?P<port>\d+))?/(?P<database>\w+)') matches = [ regex.match(uri) for uri in self.args['config'] ] # verify that all config URIs are parsed correctly and contain a database # TODO check for empty config dbs if not all(matches) or not all(m.groupdict()['database'] for m in matches): raise SystemExit('Unable to parse config server URIs, please check syntax: mongodb://[username:password@]host[:port]/database') # for convenience mongodb:// can be omitted for this script, but MongoClient expects it for i, uri in enumerate(self.args['config']): if not uri.startswith('mongodb://'): self.args['config'][i] = 'mongodb://' + uri # add uri and short_uri to self.parsed_uri dicts self.parsed_uris = [m.groupdict() for m in matches] for i, (uri, puri) in enumerate(zip(self.args['config'], self.parsed_uris)): self.parsed_uris[i]['uri'] = uri if puri['port'] == None: self.parsed_uris[i]['port'] = '27017' self.parsed_uris[i]['short_uri'] = '%s:%s/%s' % (puri['host'], puri['port'], puri['database']) # connect to databases self.config_dbs = [ MongoClient(puri['uri'])[puri['database']] for puri in self.parsed_uris ] print "\n>> individual health checks on all config servers" print " (verifies that for each namespace, the chunk ranges reach from MinKey to MaxKey without gaps or overlaps)\n" for puri, database in zip(self.parsed_uris, self.config_dbs): print puri['short_uri'] self._health_check(database) print if len(self.config_dbs) > 1: print "\n>> comparing config.collections collection for each config server" print " (verifies that they agree on the state of each collection)\n" self._compare_collections() print "\n>> comparing config.chunks collection for each config server" print " (verifies that they agree on chunk ranges for each namespace and finds deviation point)\n" self._compare_chunks_and_reconstruct()
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments, get_unknowns=True) # load or store parameters if self.args["restart"]: self.load_parameters() else: self.store_parameters() # check if authentication is enabled if self.args["authentication"]: if not os.path.exists(self.args["dir"]): os.makedirs(self.args["dir"]) os.system("openssl rand -base64 753 > %s/keyfile" % self.args["dir"]) os.system("chmod 600 %s/keyfile" % self.args["dir"]) if self.args["sharded"]: self._launchSharded() elif self.args["single"]: self._launchSingle(self.args["dir"], self.args["port"]) elif self.args["replicaset"]: self._launchReplSet(self.args["dir"], self.args["port"], self.args["name"])
def run(self, arguments=None): BaseCmdLineTool.run(self, arguments, get_unknowns=True) # load or store parameters if self.args['restart']: self.load_parameters() else: self.store_parameters() # check if authentication is enabled if self.args['authentication']: if not os.path.exists(self.args['dir']): os.makedirs(self.args['dir']) os.system('openssl rand -base64 753 > %s/keyfile' % self.args['dir']) os.system('chmod 600 %s/keyfile' % self.args['dir']) if self.args['sharded']: self._launchSharded() elif self.args['single']: self._launchSingle(self.args['dir'], self.args['port']) elif self.args['replicaset']: self._launchReplSet(self.args['dir'], self.args['port'], self.args['name'])
def run(self, arguments=None): """ This is the main run method, called for all sub-commands and parameters. It sets up argument parsing, then calls the sub-command method with the same name. """ # set up argument parsing in run, so that subsequent calls to run can call different sub-commands self.argparser = argparse.ArgumentParser() self.argparser.add_argument('--version', action='version', version="mtools version %s" % __version__) self.argparser.description = 'script to launch MongoDB stand-alone servers, replica sets and shards.' # make sure init is default command even when specifying arguments directly if arguments and arguments.startswith('-'): arguments = 'init ' + arguments # default sub-command is `init` if none provided elif len(sys.argv) > 1 and sys.argv[1].startswith('-') and sys.argv[1] not in ['-h', '--help']: sys.argv = sys.argv[0:1] + ['init'] + sys.argv[1:] # create command sub-parsers subparsers = self.argparser.add_subparsers(dest='command') self.argparser._action_groups[0].title = 'commands' self.argparser._action_groups[0].description = 'init is the default command and can be omitted. To get help on individual commands, run mlaunch [command] --help' # init command init_parser = subparsers.add_parser('init', help='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters.', description='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters') # either single or replica set me_group = init_parser.add_mutually_exclusive_group(required=True) me_group.add_argument('--single', action='store_true', help='creates a single stand-alone mongod instance') me_group.add_argument('--replicaset', action='store_true', help='creates replica set with several mongod instances') # replica set arguments init_parser.add_argument('--nodes', action='store', metavar='NUM', type=int, default=3, help='adds NUM data nodes to replica set (requires --replicaset, default=3)') init_parser.add_argument('--arbiter', action='store_true', default=False, help='adds arbiter to replica set (requires --replicaset)') init_parser.add_argument('--name', action='store', metavar='NAME', default='replset', help='name for replica set (default=replset)') # sharded clusters init_parser.add_argument('--sharded', action='store', nargs='*', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards (default=1)') init_parser.add_argument('--config', action='store', default=1, type=int, metavar='NUM', choices=[1, 3], help='adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)') init_parser.add_argument('--mongos', action='store', default=1, type=int, metavar='NUM', help='starts NUM mongos processes (requires --sharded, default=1)') # verbose, port, auth, binary path init_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.') init_parser.add_argument('--port', action='store', type=int, default=27017, help='port for mongod, start of port range in case of replica set or shards (default=27017)') init_parser.add_argument('--authentication', action='store_true', default=False, help='enable authentication and create a key file and admin user (admin/mypassword)') init_parser.add_argument('--binarypath', action='store', default=None, metavar='PATH', help='search for mongod/s binaries in the specified PATH.') init_parser.add_argument('--dir', action='store', default='./data', help='base directory to create db and log paths (default=./data/)') # start command start_parser = subparsers.add_parser('start', help='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.', description='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.') start_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all non-running nodes will be restarted. Provide additional tags to narrow down the set of nodes to start.') start_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.') start_parser.add_argument('--dir', action='store', default='./data', help='base directory to start nodes (default=./data/)') # stop command stop_parser = subparsers.add_parser('stop', help='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.', description='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.') stop_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all running nodes will be stopped. Provide additional tags to narrow down the set of nodes to stop.') stop_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.') stop_parser.add_argument('--dir', action='store', default='./data', help='base directory to stop nodes (default=./data/)') # list command list_parser = subparsers.add_parser('list', help='list MongoDB instances for this configuration', description='list MongoDB instances for this configuration') list_parser.add_argument('--dir', action='store', default='./data', help='base directory to list nodes (default=./data/)') list_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.') # argparser is set up, now call base class run() BaseCmdLineTool.run(self, arguments, get_unknowns=True) # replace path with absolute path self.dir = os.path.abspath(self.args['dir']) # branch out in sub-commands getattr(self, self.args['command'])()