def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(
description='Check data consistency including data and indexes.')
parser.add_argument(
'--from',
nargs='?',
required=True,
help='the source must be a mongod instance of replica-set')
parser.add_argument('--src-authdb',
nargs='?',
required=False,
default='admin',
help="authentication database, default is 'admin'")
parser.add_argument('--src-username',
nargs='?',
required=False,
help='src username')
parser.add_argument('--src-password',
nargs='?',
required=False,
help='src password')
parser.add_argument(
'--to',
nargs='?',
required=True,
help='the destionation should be a mongos or mongod instance')
parser.add_argument('--dst-authdb',
nargs='?',
required=False,
default='admin',
help="authentication database, default is 'admin'")
parser.add_argument('--dst-username',
nargs='?',
required=False,
help='dst username')
parser.add_argument('--dst-password',
nargs='?',
required=False,
help='dst password')
parser.add_argument('--dbs',
nargs='+',
required=False,
help='databases to check')
parser.add_argument(
'--src-db',
nargs='?',
required=False,
help=
"src database to check, work with '--dst-db', conflict with '--dbs'"
)
parser.add_argument(
'--dst-db',
nargs='?',
required=False,
help=
"dst database to check, work with '--src-db', conflict with '--dbs'"
)
args = vars(parser.parse_args())
if args['from'] is not None:
conf.src_hostportstr = args['from']
conf.src_host, conf.src_port = parse_hostportstr(
conf.src_hostportstr)
if args['src_authdb'] is not None:
conf.src_authdb = args['src_authdb']
if args['src_username'] is not None:
conf.src_username = args['src_username']
if args['src_password'] is not None:
conf.src_password = args['src_password']
if args['to'] is not None:
conf.dst_hostportstr = args['to']
conf.dst_host, conf.dst_port = parse_hostportstr(
conf.dst_hostportstr)
if args['dst_authdb'] is not None:
conf.dst_authdb = args['dst_authdb']
if args['dst_username'] is not None:
conf.dst_username = args['dst_username']
if args['dst_password'] is not None:
conf.dst_password = args['dst_password']
if args['dbs'] is not None:
conf.dbs = args['dbs']
if args['src_db'] is not None:
conf.src_db = args['src_db']
if args['dst_db'] is not None:
conf.dst_db = args['dst_db']
if conf.dbs and (conf.src_db or conf.dst_db):
print "Terminated, conflict command options found"
sys.exit(1)
if conf.src_db and not conf.dst_db:
print "Terminated, require command option '--dst-db'"
sys.exit(1)
if conf.dst_db and not conf.src_db:
print "Terminated, require command option '--src-db'"
sys.exit(1)
if conf.src_db and conf.dst_db and conf.src_db == conf.dst_db:
print 'Terminated, src_db is same as dst_db'
sys.exit(1)
return conf
def load(filepath):
""" Load config file and generate conf.
"""
conf = Config()
tml = toml.load(filepath)
conf.src_conf = MongoConfig(tml['src']['hosts'],
tml['src'].get('authdb', 'admin'),
tml['src'].get('username', ''),
tml['src'].get('password', ''))
if type not in tml['dst'] or tml['dst']['type'] == 'mongo':
conf.dst_conf = MongoConfig(tml['dst']['hosts'],
tml['dst'].get('authdb', 'admin'),
tml['dst'].get('username', ''),
tml['dst'].get('password', ''))
elif tml['dst']['type'] == 'es':
conf.dst_conf = EsConfig(tml['dst']['hosts'])
else:
raise Exception('invalid dst.type')
if 'sync' in tml and 'dbs' in tml['sync']:
for dbentry in tml['sync']['dbs']:
if 'db' not in dbentry:
raise Exception("'db' is missing in sync.dbs")
if not dbentry['db']:
raise Exception("'db' is empty in sync.dbs")
dbname = dbentry['db'].strip()
rename_db = dbentry['rename_db'].strip(
) if 'rename_db' in dbentry else ""
# update db map
if dbname and rename_db:
if dbname in conf.dbmap:
raise Exception('duplicate dbname in sync.dbs: %s' %
dbname)
conf.dbmap[dbname] = rename_db
if 'colls' in dbentry and dbentry['colls']:
for collentry in dbentry['colls']:
if isinstance(collentry, str):
collname = collentry.strip()
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
elif isinstance(collentry, dict):
if 'coll' not in collentry:
raise Exception(
"'coll' is missing in sync.dbs.colls")
if not collentry['coll']:
raise Exception(
"'coll' is empty in sync.dbs.colls")
collname = collentry['coll'].strip()
fields = frozenset(
[f.strip()
for f in collentry['fields']] if 'fields' in
collentry else [])
# update coll filter
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
# update fields
if fields:
if ns in conf.fieldmap:
raise Exception(
"duplicate collname in sync.dbs.colls: %s"
% ns)
conf.fieldmap[ns] = fields
else:
raise Exception(
'invalid entry in sync.dbs.colls: %s' %
collentry)
else:
# update coll filter
conf.data_filter.add_include_coll(
gen_namespace(dbname, '*'))
if 'sync' in tml and 'start_optime' in tml['sync']:
conf.start_optime = Timestamp(tml['sync']['start_optime'], 0)
if 'log' in tml and 'filepath' in tml['log']:
conf.logfilepath = tml['log']['filepath']
if 'log' in tml and 'op_time_path' in tml['log']:
conf.optime_logfilepath = tml['log']['op_time_path']
optime_logger = OptimeLogger(conf.optime_logfilepath)
if optime_logger.read():
conf.start_optime = optime_logger.read()
return conf
def load(filepath):
""" Load config file and generate conf.
"""
conf = Config()
tml = toml.load(filepath)
conf.src_conf = MongoConfig(tml['src']['hosts'],
tml['src'].get('authdb', 'admin'),
tml['src'].get('username', ''),
tml['src'].get('password', ''))
if type not in tml['dst'] or tml['dst']['type'] == 'mongo':
conf.dst_conf = MongoConfig(tml['dst']['hosts'],
tml['dst'].get('authdb', 'admin'),
tml['dst'].get('username', ''),
tml['dst'].get('password', ''))
elif tml['dst']['type'] == 'es':
conf.dst_conf = EsConfig(tml['dst']['hosts'])
else:
raise Exception('invalid dst.type')
if 'sync' in tml and 'dbs' in tml['sync']:
for dbentry in tml['sync']['dbs']:
if 'db' not in dbentry:
raise Exception("'db' is missing in sync.dbs")
if not dbentry['db']:
raise Exception("'db' is empty in sync.dbs")
dbname = dbentry['db'].strip()
rename_db = dbentry['rename_db'].strip() if 'rename_db' in dbentry else ""
# update db map
if dbname and rename_db:
if dbname in conf.dbmap:
raise Exception('duplicate dbname in sync.dbs: %s' % dbname)
conf.dbmap[dbname] = rename_db
if 'colls' in dbentry and dbentry['colls']:
for collentry in dbentry['colls']:
if isinstance(collentry, str) or isinstance(collentry, unicode):
collname = collentry.strip()
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
elif isinstance(collentry, dict):
if 'coll' not in collentry:
raise Exception("'coll' is missing in sync.dbs.colls")
if not collentry['coll']:
raise Exception("'coll' is empty in sync.dbs.colls")
collname = collentry['coll'].strip()
fields = frozenset([f.strip() for f in collentry['fields']] if 'fields' in collentry else [])
# update coll filter
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
# update fields
if fields:
if ns in conf.fieldmap:
raise Exception("duplicate collname in sync.dbs.colls: %s" % ns)
conf.fieldmap[ns] = fields
else:
raise Exception('invalid entry in sync.dbs.colls: %s' % collentry)
else:
# update coll filter
conf.data_filter.add_include_coll(gen_namespace(dbname, '*'))
if 'sync' in tml and 'start_optime' in tml['sync']:
conf.start_optime = Timestamp(tml['sync']['start_optime'], 0)
if 'log' in tml and 'filepath' in tml['log']:
conf.logfilepath = tml['log']['filepath']
return conf
def load(filepath):
""" Load config file and generate conf.
"""
conf = Config()
tml = toml.load(filepath)
conf.src_conf = MongoConfig(
tml['src']['hosts'],
authdb=tml['src'].get('authdb',
'admin'), # default authdb is 'admin'
username=tml['src'].get('username', ''),
password=tml['src'].get('password', ''))
if tml['dst']['type'] == 'mongo':
conf.dst_conf = MongoConfig(
tml['dst']['mongo']['hosts'],
authdb=tml['dst']['mongo'].get(
'authdb', 'admin'), # default authdb is 'admin'
username=tml['dst']['mongo'].get('username', ''),
password=tml['dst']['mongo'].get('password', ''))
elif tml['dst']['type'] == 'es':
conf.dst_conf = EsConfig(tml['dst']['es']['hosts'])
if 'sync' in tml and 'dbs' in tml['sync']:
for dbentry in tml['sync']['dbs']:
if 'db' not in dbentry:
raise Exception("required option 'db' is missing")
if not dbentry['db']:
raise Exception("required option 'db' is empty")
dbname = dbentry['db'].strip()
rename_db = dbentry['rename_db'].strip(
) if 'rename_db' in dbentry else ""
# update db map
if dbname and rename_db:
if dbname in conf.dbmap:
raise Exception("conflict dbname in 'sync.dbs': %s" %
dbname)
conf.dbmap[dbname] = rename_db
if 'colls' in dbentry and dbentry['colls']:
for collentry in dbentry['colls']:
if 'coll' not in collentry:
raise Exception(
"required option 'coll' is missing")
if not collentry['coll']:
raise Exception("required option 'coll' is empty")
collname = collentry['coll'].strip()
fields = frozenset(
[f.strip()
for f in collentry['fields']] if 'fields' in
collentry else [])
if collname == '*' and fields:
raise Exception(
"'fileds' should be empty if 'coll' is '*'")
if collname == '*':
# update coll filter
conf.data_filter.add_include_coll(
gen_namespace(dbname, '*'))
else:
# update coll filter
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
# update fields
if fields:
if ns in conf.fieldmap:
raise Exception(
"conflict namespace in 'sync.colls': %s"
% ns)
conf.fieldmap[ns] = fields
else:
# update coll filter
conf.data_filter.add_include_coll(
gen_namespace(dbname, '*'))
if 'sync' in tml and 'start_optime' in tml['sync']:
conf.start_optime = Timestamp(tml['sync']['start_optime'], 0)
if 'log' in tml and 'filepath' in tml['log']:
conf.logfilepath = tml['log']['filepath']
return conf
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Check data consistency including data and indexes.')
parser.add_argument('--from', nargs='?', required=True, help='the source must be a mongod instance of replica-set')
parser.add_argument('--src-authdb', nargs='?', required=False, default='admin', help="authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--to', nargs='?', required=True, help='the destionation should be a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, default='admin', help="authentication database, default is 'admin'")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password')
parser.add_argument('--dbs', nargs='+', required=False, help='databases to check')
parser.add_argument('--src-db', nargs='?', required=False, help="src database to check, work with '--dst-db', conflict with '--dbs'")
parser.add_argument('--dst-db', nargs='?', required=False, help="dst database to check, work with '--src-db', conflict with '--dbs'")
args = vars(parser.parse_args())
if args['from'] is not None:
conf.src_hostportstr = args['from']
conf.src_host, conf.src_port = parse_hostportstr(conf.src_hostportstr)
if args['src_authdb'] is not None:
conf.src_authdb = args['src_authdb']
if args['src_username'] is not None:
conf.src_username = args['src_username']
if args['src_password'] is not None:
conf.src_password = args['src_password']
if args['to'] is not None:
conf.dst_hostportstr = args['to']
conf.dst_host, conf.dst_port = parse_hostportstr(conf.dst_hostportstr)
if args['dst_authdb'] is not None:
conf.dst_authdb = args['dst_authdb']
if args['dst_username'] is not None:
conf.dst_username = args['dst_username']
if args['dst_password'] is not None:
conf.dst_password = args['dst_password']
if args['dbs'] is not None:
conf.dbs = args['dbs']
if args['src_db'] is not None:
conf.src_db = args['src_db']
if args['dst_db'] is not None:
conf.dst_db = args['dst_db']
if conf.dbs and (conf.src_db or conf.dst_db):
print "Terminated, conflict command options found"
sys.exit(1)
if conf.src_db and not conf.dst_db:
print "Terminated, require command option '--dst-db'"
sys.exit(1)
if conf.dst_db and not conf.src_db:
print "Terminated, require command option '--src-db'"
sys.exit(1)
if conf.src_db and conf.dst_db and conf.src_db == conf.dst_db:
print 'Terminated, src_db is same as dst_db'
sys.exit(1)
return conf