dest = utils.parse_mongo_url(args.dest)
if os.path.exists(args.source):
sources = utils.parse_source_file(args.source)
else:
sources = [utils.parse_mongo_url(args.source)]
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
if not args.state_db:
args.state_db = "%s.%s.db" % (sources[0]["db"], sources[0]["collection"])
if args.state_db.startswith("/"):
state_db_path = args.state_db
else:
state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), args.state_db)
log.info("using state db %s" % state_db_path)
state_db_exists = os.path.exists(state_db_path)
state_db = CopyStateDB(state_db_path)
if not state_db_exists:
state_db.drop_and_create()
if args.restart:
state_db.drop_and_create()
# do the real work
copy_collection_parent(sources, dest, state_db, args)
log.error("shouldn't reach this point")
sys.exit(1)
if os.path.exists(args.source):
sources = utils.parse_source_file(args.source)
else:
sources = [utils.parse_mongo_url(args.source)]
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
if not args.state_db:
args.state_db = '%s.%s.db' % (sources[0]['db'], sources[0]['collection'])
if args.state_db.startswith('/'):
state_db_path = args.state_db
else:
state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
args.state_db)
log.info('using state db %s' % state_db_path)
state_db_exists = os.path.exists(state_db_path)
state_db = CopyStateDB(state_db_path)
if not state_db_exists:
state_db.drop_and_create()
if args.restart:
state_db.drop_and_create()
# do the real work
copy_collection_parent(sources, dest, state_db, args)
log.error("shouldn't reach this point")
sys.exit(1)
def main():
# NOTE: we are not gevent monkey-patched here; only child processes are monkey-patched,
# so all ops below are synchronous
# parse command-line options
import argparse
parser = argparse.ArgumentParser(
description='Copy collections from one mongod to another. demo:\n'
'mongo_copier --srchost 192.168.37.12 --srcport 1234 '
'--desthost 192.168.37.14 --destport 5678 --destuser write --destpwd write '
'--manifests /tmp/1.txt --restart --drop',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--srchost',
type=str,
required=True,
metavar='HOST',
help='hostname or IP of source mongod')
parser.add_argument('--srcport',
type=int,
required=True,
metavar='PORT',
help='port of source mongod')
parser.add_argument('--desthost',
type=str,
required=True,
metavar='HOST',
help='hostname or IP of destination mongod')
parser.add_argument('--destport',
type=int,
required=True,
metavar='PORT',
help='port of destination mongod')
parser.add_argument(
'--destuser',
type=str,
required=True,
metavar='USERNAME',
help='username of destination mongod, must be writable')
parser.add_argument('--destpwd',
type=str,
required=True,
metavar='PASSWORD',
help='password of destination mongod')
parser.add_argument('--percent',
type=int,
metavar='PCT',
default=None,
help='copy only PCT%% of data')
parser.add_argument('--drop',
action='store_true',
help='delete destination collection data before copy')
parser.add_argument(
'--restart',
action='store_true',
help='restart from the beginning, ignoring any prior progress')
parser.add_argument(
'--state-db',
type=str,
metavar='PATH',
default=None,
help=
'path to state file (defaults to\n/tmp/mongo_copier_states/<srchost>_<desthost>.db)'
)
parser.add_argument(
'--manifests',
type=str,
required=True,
metavar='FILE',
help='a file containing collections to copy, one strategy per line.\n'
'e.g.\n'
'copy a collection\n'
' dbname.colname\n'
'copy a collection with change destination database or collection name\n'
' dbname.colname newdbname.newcolname\n'
'copy a collection with query\n'
' dbname.colname {"id":{"$in":[123,456]}}\n'
'copy all collections of a database\n'
' dbname.*\n'
'copy all collections of a database with change destination database name\n'
' dbname.* newdbname.*\n'
'copy all collections of a database with query\n'
' dbname.* {"id":{"$in":[123,456]}}\n'
'copy the collection of all databases\n'
' *.dbname\n'
'copy the collection of all databases with change destination collection name\n'
' *.dbname *.newdbname\n'
'copy the collection of all databases with query\n'
' *.dbname {"id":{"$in":[123,456]}}\n'
'copy all collections of all databases\n'
' *.*\n'
'copy all collections of all databases with query\n'
' *.* {"id":{"$in":[123,456]}}\n')
args = parser.parse_args()
log.debug(args)
# parse source and destination
if os.path.exists(args.manifests):
manifests = utils.parse_manifests_file(args)
else:
die(log, "manifests not exist: %s" % manifests)
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
if not args.state_db:
args.state_db = '/tmp/mongo_copier_states/%s_%s.db' % (args.srchost,
args.desthost)
if not os.path.isdir("/tmp/mongo_copier_states/"):
os.mkdir("/tmp/mongo_copier_states/")
if args.state_db.startswith('/'):
state_db_path = args.state_db
else:
state_db_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), args.state_db)
log.info('using state db %s' % state_db_path)
state_db_exists = os.path.exists(state_db_path)
state_db = CopyStateDB(state_db_path)
if not state_db_exists or args.restart:
state_db.drop_and_create()
# do the real work
copy_collection_parent(manifests, state_db, args)
