def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments, get_unknowns=True)
# load or store parameters
if self.args['restart']:
self.load_parameters()
elif self.args['stop']:
if self.args['verbose']:
self.verbose = True
self.load_parameters()
self.args['stop'] = True
else:
self.store_parameters()
# check if authentication is enabled
if self.args['authentication']:
if not os.path.exists(self.args['dir']):
os.makedirs(self.args['dir'])
os.system('openssl rand -base64 753 > %s/keyfile'%self.args['dir'])
os.system('chmod 600 %s/keyfile'%self.args['dir'])
if self.args['stop']:
self._stopAll(self.args['dir'])
else:
if self.args['sharded']:
self._launchSharded()
elif self.args['single']:
self._launchSingle(self.args['dir'], self.args['port'])
elif self.args['replicaset']:
self._launchReplSet(self.args['dir'], self.args['port'], self.args['name'])
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments)
if self.args['template'].startswith('{'):
# not a file
try:
template = json.loads(self.args['template'])
except ValueError as e:
raise SystemExit("can't parse template: %s" % e)
else:
try:
f = open(self.args['template'])
except IOError as e:
raise SystemExit("can't open file %s: %s" %
(self.args['template'], e))
try:
template = json.load(f)
except ValueError as e:
raise SystemExit("can't parse template in %s: %s" %
(self.args['template'], e))
if not self.args['stdout']:
mc = Connection(host=self.args['host'],
port=self.args['port'],
w=self.args['write_concern'],
connect=False)
col = mc[self.args['database']][self.args['collection']]
if self.args['drop']:
col.drop()
else:
col = None
# divide work over number of cores
if self.args['stdout']:
num_cores = 1
elif self.args['processes'] > 0:
num_cores = self.args['processes']
else:
num_cores = cpu_count()
num_list = [self.args['number'] // num_cores] * num_cores
num_list[0] += self.args['number'] % num_cores
processes = []
for n in num_list:
p = InsertProcess(n, template, col, self.args)
p.start()
processes.append(p)
for p in processes:
p.join()
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments)
self.dbpath = self.args['dbpath'][0]
self.force = self.args['force']
self.verbose = self.args['verbose']
# Read storage.bson, sanity check.
try:
storage_raw = open(os.path.join(self.dbpath, 'storage.bson'), 'rb').read()
except Exception as e:
sys.stderr.write('Failed to open storage.bson in "{0}": {1}\n'.format(self.dbpath, e))
return
settings = bson.decode(storage_raw)["storage"]["options"]
if not settings["directoryPerDB"]:
sys.stderr.write('Requires a database created with --directoryperdb\n')
return
if settings["directoryForIndexes"] or settings.get("groupCollections", False):
sys.stderr.write('Incompatible storage settings detected: '
'directoryForIndexes or groupCollections\n')
if not self.force:
return
self.database = self.args['database'][0]
self.nsprefix = self.database + '.'
mtransfer_dir = os.path.join(self.dbpath, self.database)
mtransfer_file = os.path.join(mtransfer_dir, 'mtransfer.bson')
if self.args['command'] == 'export':
if not os.path.exists(mtransfer_dir):
sys.stderr.write('Expected source directory "{0}" does not exist. '
'Check the database name is correct.\n'.format(mtransfer_dir))
return
if not self.force and os.path.exists(mtransfer_file):
sys.stderr.write('Output file "{0}" already exists\n'.format(mtransfer_file))
return
with open(mtransfer_file, 'wb') as outf:
self.doExport(outf)
elif self.args['command'] == 'import':
if not os.path.exists(mtransfer_dir):
sys.stderr.write('Expected target directory "{0}" does not exist. '
'Check the database name is correct.\n'.format(mtransfer_dir))
return
if not os.path.exists(mtransfer_file):
sys.stderr.write('Cannot import: mtransfer file "{0}" does not exist.\n'.
format(mtransfer_file))
return
with open(mtransfer_file, 'rb') as inf:
self.doImport(inf)
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments)
if self.args['template'].startswith('{'):
# not a file
try:
template = json.loads(self.args['template'])
except ValueError as e:
raise SystemExit("can't parse template: %s" % e)
else:
try:
f = open(self.args['template'])
except IOError as e:
raise SystemExit("can't open file %s: %s" % (self.args['template'], e))
try:
template = json.load(f)
except ValueError as e:
raise SystemExit("can't parse template in %s: %s" % (self.args['template'], e))
if not self.args['stdout']:
mc = Connection(host=self.args['host'], port=self.args['port'], w=self.args['write_concern'])
col = mc[self.args['database']][self.args['collection']]
if self.args['drop']:
col.drop()
else:
col = None
# divide work over number of cores
if self.args['stdout']:
num_cores = 1
elif self.args['processes'] > 0:
num_cores = self.args['processes']
else:
num_cores = cpu_count()
num_list = [self.args['number'] // num_cores] * num_cores
num_list[0] += self.args['number'] % num_cores
processes = []
for n in num_list:
p = InsertProcess(n, template, col, self.args)
p.start()
processes.append(p)
for p in processes:
p.join()
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments)
if self.args['template'].startswith('{'):
# not a file
try:
template = json.loads(self.args['template'])
except ValueError as e:
raise SystemExit("can't parse template: %s" % e)
else:
try:
f = open(self.args['template'])
except IOError as e:
raise SystemExit("can't open file %s: %s" %
(self.args['template'], e))
try:
template = json.load(f)
except ValueError as e:
raise SystemExit("can't parse template in %s: %s" %
(self.args['template'], e))
if not self.args['stdout']:
mc = Connection(host=self.args['host'],
port=self.args['port'],
w=self.args['write_concern'])
col = mc[self.args['database']][self.args['collection']]
if self.args['drop']:
col.drop()
batch = []
for n in xrange(self.args['number']):
# decode the template
doc = self._decode(template)
if self.args['stdout']:
print doc
else:
batch.append(doc)
if n % 100 == 0:
col.insert(batch)
batch = []
self.update_progress(float(n) / self.args['number'],
prefix='inserting data')
if not self.args['stdout']:
self.update_progress(1.0, prefix='inserting data')
if batch:
col.insert(batch)
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments)
# number of configs
self.num_configs = len(self.args['config'])
# parse config server URIs
regex = re.compile(r'(?P<mongodb>mongodb://)?((?P<user>\w+):(?P<password>\w+)@)?(?P<host>\w+)(:(?P<port>\d+))?/(?P<database>\w+)')
matches = [ regex.match(uri) for uri in self.args['config'] ]
# verify that all config URIs are parsed correctly and contain a database
# TODO check for empty config dbs
if not all(matches) or not all(m.groupdict()['database'] for m in matches):
raise SystemExit('Unable to parse config server URIs, please check syntax: mongodb://[username:password@]host[:port]/database')
# for convenience mongodb:// can be omitted for this script, but MongoClient expects it
for i, uri in enumerate(self.args['config']):
if not uri.startswith('mongodb://'):
self.args['config'][i] = 'mongodb://' + uri
# add uri and short_uri to self.parsed_uri dicts
self.parsed_uris = [m.groupdict() for m in matches]
for i, (uri, puri) in enumerate(zip(self.args['config'], self.parsed_uris)):
self.parsed_uris[i]['uri'] = uri
if puri['port'] == None:
self.parsed_uris[i]['port'] = '27017'
self.parsed_uris[i]['short_uri'] = '%s:%s/%s' % (puri['host'], puri['port'], puri['database'])
# connect to databases
self.config_dbs = [ MongoClient(puri['uri'])[puri['database']] for puri in self.parsed_uris ]
print "\n>> individual health checks on all config servers"
print " (verifies that for each namespace, the chunk ranges reach from MinKey to MaxKey without gaps or overlaps)\n"
for puri, database in zip(self.parsed_uris, self.config_dbs):
print puri['short_uri']
self._health_check(database)
print
if len(self.config_dbs) > 1:
print "\n>> comparing config.collections collection for each config server"
print " (verifies that they agree on the state of each collection)\n"
self._compare_collections()
print "\n>> comparing config.chunks collection for each config server"
print " (verifies that they agree on chunk ranges for each namespace and finds deviation point)\n"
self._compare_chunks_and_reconstruct()
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments, get_unknowns=True)
# load or store parameters
if self.args["restart"]:
self.load_parameters()
else:
self.store_parameters()
# check if authentication is enabled
if self.args["authentication"]:
if not os.path.exists(self.args["dir"]):
os.makedirs(self.args["dir"])
os.system("openssl rand -base64 753 > %s/keyfile" % self.args["dir"])
os.system("chmod 600 %s/keyfile" % self.args["dir"])
if self.args["sharded"]:
self._launchSharded()
elif self.args["single"]:
self._launchSingle(self.args["dir"], self.args["port"])
elif self.args["replicaset"]:
self._launchReplSet(self.args["dir"], self.args["port"], self.args["name"])
def run(self, arguments=None):
BaseCmdLineTool.run(self, arguments, get_unknowns=True)
# load or store parameters
if self.args['restart']:
self.load_parameters()
else:
self.store_parameters()
# check if authentication is enabled
if self.args['authentication']:
if not os.path.exists(self.args['dir']):
os.makedirs(self.args['dir'])
os.system('openssl rand -base64 753 > %s/keyfile' %
self.args['dir'])
os.system('chmod 600 %s/keyfile' % self.args['dir'])
if self.args['sharded']:
self._launchSharded()
elif self.args['single']:
self._launchSingle(self.args['dir'], self.args['port'])
elif self.args['replicaset']:
self._launchReplSet(self.args['dir'], self.args['port'],
self.args['name'])
def run(self, arguments=None):
""" This is the main run method, called for all sub-commands and parameters.
It sets up argument parsing, then calls the sub-command method with the same name.
"""
# set up argument parsing in run, so that subsequent calls to run can call different sub-commands
self.argparser = argparse.ArgumentParser()
self.argparser.add_argument('--version', action='version', version="mtools version %s" % __version__)
self.argparser.description = 'script to launch MongoDB stand-alone servers, replica sets and shards.'
# make sure init is default command even when specifying arguments directly
if arguments and arguments.startswith('-'):
arguments = 'init ' + arguments
# default sub-command is `init` if none provided
elif len(sys.argv) > 1 and sys.argv[1].startswith('-') and sys.argv[1] not in ['-h', '--help']:
sys.argv = sys.argv[0:1] + ['init'] + sys.argv[1:]
# create command sub-parsers
subparsers = self.argparser.add_subparsers(dest='command')
self.argparser._action_groups[0].title = 'commands'
self.argparser._action_groups[0].description = 'init is the default command and can be omitted. To get help on individual commands, run mlaunch [command] --help'
# init command
init_parser = subparsers.add_parser('init', help='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters.',
description='initialize and start MongoDB stand-alone instances, replica sets, or sharded clusters')
# either single or replica set
me_group = init_parser.add_mutually_exclusive_group(required=True)
me_group.add_argument('--single', action='store_true', help='creates a single stand-alone mongod instance')
me_group.add_argument('--replicaset', action='store_true', help='creates replica set with several mongod instances')
# replica set arguments
init_parser.add_argument('--nodes', action='store', metavar='NUM', type=int, default=3, help='adds NUM data nodes to replica set (requires --replicaset, default=3)')
init_parser.add_argument('--arbiter', action='store_true', default=False, help='adds arbiter to replica set (requires --replicaset)')
init_parser.add_argument('--name', action='store', metavar='NAME', default='replset', help='name for replica set (default=replset)')
# sharded clusters
init_parser.add_argument('--sharded', action='store', nargs='*', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards (default=1)')
init_parser.add_argument('--config', action='store', default=1, type=int, metavar='NUM', choices=[1, 3], help='adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)')
init_parser.add_argument('--mongos', action='store', default=1, type=int, metavar='NUM', help='starts NUM mongos processes (requires --sharded, default=1)')
# verbose, port, auth, binary path
init_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
init_parser.add_argument('--port', action='store', type=int, default=27017, help='port for mongod, start of port range in case of replica set or shards (default=27017)')
init_parser.add_argument('--authentication', action='store_true', default=False, help='enable authentication and create a key file and admin user (admin/mypassword)')
init_parser.add_argument('--binarypath', action='store', default=None, metavar='PATH', help='search for mongod/s binaries in the specified PATH.')
init_parser.add_argument('--dir', action='store', default='./data', help='base directory to create db and log paths (default=./data/)')
# start command
start_parser = subparsers.add_parser('start', help='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.',
description='starts existing MongoDB instances. Example: "mlaunch start config" will start all config servers.')
start_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all non-running nodes will be restarted. Provide additional tags to narrow down the set of nodes to start.')
start_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
start_parser.add_argument('--dir', action='store', default='./data', help='base directory to start nodes (default=./data/)')
# stop command
stop_parser = subparsers.add_parser('stop', help='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.',
description='stops running MongoDB instances. Example: "mlaunch stop shard 2 secondary" will stop all secondary nodes of shard 2.')
stop_parser.add_argument('tags', metavar='TAG', action='store', nargs='*', default=[], help='without tags, all running nodes will be stopped. Provide additional tags to narrow down the set of nodes to stop.')
stop_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
stop_parser.add_argument('--dir', action='store', default='./data', help='base directory to stop nodes (default=./data/)')
# list command
list_parser = subparsers.add_parser('list', help='list MongoDB instances for this configuration',
description='list MongoDB instances for this configuration')
list_parser.add_argument('--dir', action='store', default='./data', help='base directory to list nodes (default=./data/)')
list_parser.add_argument('--verbose', action='store_true', default=False, help='outputs more verbose information.')
# argparser is set up, now call base class run()
BaseCmdLineTool.run(self, arguments, get_unknowns=True)
# replace path with absolute path
self.dir = os.path.abspath(self.args['dir'])
# branch out in sub-commands
getattr(self, self.args['command'])()
