def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def load(filepath):
""" Load config file and generate conf.
"""
conf = Config()
tml = toml.load(filepath)
conf.src_conf = MongoConfig(tml['src']['hosts'],
tml['src'].get('authdb', 'admin'),
tml['src'].get('username', ''),
tml['src'].get('password', ''))
if type not in tml['dst'] or tml['dst']['type'] == 'mongo':
conf.dst_conf = MongoConfig(tml['dst']['hosts'],
tml['dst'].get('authdb', 'admin'),
tml['dst'].get('username', ''),
tml['dst'].get('password', ''))
elif tml['dst']['type'] == 'es':
conf.dst_conf = EsConfig(tml['dst']['hosts'])
else:
raise Exception('invalid dst.type')
if 'sync' in tml and 'dbs' in tml['sync']:
for dbentry in tml['sync']['dbs']:
if 'db' not in dbentry:
raise Exception("'db' is missing in sync.dbs")
if not dbentry['db']:
raise Exception("'db' is empty in sync.dbs")
dbname = dbentry['db'].strip()
rename_db = dbentry['rename_db'].strip(
) if 'rename_db' in dbentry else ""
# update db map
if dbname and rename_db:
if dbname in conf.dbmap:
raise Exception('duplicate dbname in sync.dbs: %s' %
dbname)
conf.dbmap[dbname] = rename_db
if 'colls' in dbentry and dbentry['colls']:
for collentry in dbentry['colls']:
if isinstance(collentry, str):
collname = collentry.strip()
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
elif isinstance(collentry, dict):
if 'coll' not in collentry:
raise Exception(
"'coll' is missing in sync.dbs.colls")
if not collentry['coll']:
raise Exception(
"'coll' is empty in sync.dbs.colls")
collname = collentry['coll'].strip()
fields = frozenset(
[f.strip()
for f in collentry['fields']] if 'fields' in
collentry else [])
# update coll filter
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
# update fields
if fields:
if ns in conf.fieldmap:
raise Exception(
"duplicate collname in sync.dbs.colls: %s"
% ns)
conf.fieldmap[ns] = fields
else:
raise Exception(
'invalid entry in sync.dbs.colls: %s' %
collentry)
else:
# update coll filter
conf.data_filter.add_include_coll(
gen_namespace(dbname, '*'))
if 'sync' in tml and 'start_optime' in tml['sync']:
conf.start_optime = Timestamp(tml['sync']['start_optime'], 0)
if 'log' in tml and 'filepath' in tml['log']:
conf.logfilepath = tml['log']['filepath']
if 'log' in tml and 'op_time_path' in tml['log']:
conf.optime_logfilepath = tml['log']['op_time_path']
optime_logger = OptimeLogger(conf.optime_logfilepath)
if optime_logger.read():
conf.start_optime = optime_logger.read()
return conf