parser.add_argument(
'--dest', type=str, required=True, metavar='URL',
help='source to read from; see --source for format')
parser.add_argument(
'--percent', type=int, metavar='PCT', default=None,
help='copy only PCT%% of data')
parser.add_argument(
'--restart', action='store_true',
help='restart from the beginning, ignoring any prior progress')
parser.add_argument(
'--state-db', type=str, metavar='PATH', default=None,
help='path to state file (defaults to ./<source_database>.<source_collection>.db)')
args = parser.parse_args()
# parse source and destination
dest = utils.parse_mongo_url(args.dest)
if os.path.exists(args.source):
sources = utils.parse_source_file(args.source)
else:
sources = [utils.parse_mongo_url(args.source)]
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
if not args.state_db:
args.state_db = '%s.%s.db' % (sources[0]['db'], sources[0]['collection'])
if args.state_db.startswith('/'):
state_db_path = args.state_db
else:
state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
args.state_db)
)
parser.add_argument("--percent", type=int, metavar="PCT", default=None, help="copy only PCT%% of data")
parser.add_argument(
"--restart", action="store_true", help="restart from the beginning, ignoring any prior progress"
)
parser.add_argument(
"--state-db",
type=str,
metavar="PATH",
default=None,
help="path to state file (defaults to ./<source_database>.<source_collection>.db)",
)
args = parser.parse_args()
# parse source and destination
dest = utils.parse_mongo_url(args.dest)
if os.path.exists(args.source):
sources = utils.parse_source_file(args.source)
else:
sources = [utils.parse_mongo_url(args.source)]
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
if not args.state_db:
args.state_db = "%s.%s.db" % (sources[0]["db"], sources[0]["collection"])
if args.state_db.startswith("/"):
state_db_path = args.state_db
else:
state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), args.state_db)
help=
'source to read from; can be a file containing sources or a url like: host[:port]/db/collection; '
'e.g. localhost:27017/prod_maestro.emails')
parser.add_argument(
'--seconds',
type=int,
required=True,
metavar="Minutes",
help='Adjust the TTL of the messages database by N minutes')
args = parser.parse_args()
# parse source
if os.path.exists(args.source):
sources = utils.parse_source_file(args.source)
else:
sources = [utils.parse_mongo_url(args.source)]
# initialize sqlite database that holds our state (this may seem like overkill,
# but it's actually needed to ensure proper synchronization of subprocesses)
args.state_db = '%s.%s.db' % (sources[0]['db'], sources[0]['collection'])
state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
args.state_db)
log.info('using state db %s' % state_db_path)
state_db_exists = os.path.exists(state_db_path)
state_db = CopyStateDB(state_db_path)
if not state_db_exists:
state_db.drop_and_create()
# do the real work
gevent.monkey.patch_socket()
parser = argparse.ArgumentParser(description='Through stdin, reads JSON documents containing _ids and shark keys for mismatching documents and re-copies those documents.')
parser.add_argument(
'--source', type=str, required=True, metavar='URL',
help='source to read from; e.g. localhost:27017/prod_maestro.emails')
parser.add_argument(
'--dest', type=str, required=True, metavar='URL',
help='destination to copy to; e.g. localhost:27017/destination_db.emails')
parser.add_argument(
'--mismatches-file', type=str, default=None, required=True, metavar='FILENAME',
help='read ids to copy from this file, which is generated by compare_collections.py')
args = parser.parse_args()
# connect to source and destination
source = utils.parse_mongo_url(args.source)
source_client = utils.mongo_connect(source,
ensure_direct=True,
max_pool_size=POOL_SIZE,
read_preference=ReadPreference.SECONDARY_PREFERRED,
document_class=FasterOrderedDict)
source_collection = source_client[source['db']][source['collection']]
if not source_client.is_mongos or source_client.is_primary:
raise Exception("source must be a mongos instance or a primary")
dest = utils.parse_mongo_url(args.dest)
dest_client = utils.mongo_connect(dest,
max_pool_size=POOL_SIZE,
document_class=FasterOrderedDict)
dest_collection = dest_client[dest['db']][dest['collection']]
gevent.monkey.patch_socket()
parser = argparse.ArgumentParser(description='Through stdin, reads JSON documents containing _ids and shark keys for mismatching documents and re-copies those documents.')
parser.add_argument(
'--source', type=str, required=True, metavar='URL',
help='source to read from; e.g. localhost:27017/prod_maestro.emails')
parser.add_argument(
'--dest', type=str, required=True, metavar='URL',
help='destination to copy to; e.g. localhost:27017/destination_db.emails')
parser.add_argument(
'--mismatches-file', type=str, default=None, required=True, metavar='FILENAME',
help='read ids to copy from this file, which is generated by compare_collections.py')
args = parser.parse_args()
# connect to source and destination
source = utils.parse_mongo_url(args.source)
source_client = utils.mongo_connect(source['host'], source['port'],
ensure_direct=True,
max_pool_size=POOL_SIZE,
read_preference=ReadPreference.SECONDARY_PREFERRED,
document_class=FasterOrderedDict)
source_collection = source_client[source['db']][source['collection']]
if not source_client.is_mongos or source_client.is_primary:
raise Exception("source must be a mongos instance or a primary")
dest = utils.parse_mongo_url(args.dest)
dest_client = utils.mongo_connect(dest['host'], dest['port'],
max_pool_size=POOL_SIZE,
document_class=FasterOrderedDict)
dest_collection = dest_client[dest['db']][dest['collection']]