parser.add_argument( '--dest', type=str, required=True, metavar='URL', help='source to read from; see --source for format') parser.add_argument( '--percent', type=int, metavar='PCT', default=None, help='copy only PCT%% of data') parser.add_argument( '--restart', action='store_true', help='restart from the beginning, ignoring any prior progress') parser.add_argument( '--state-db', type=str, metavar='PATH', default=None, help='path to state file (defaults to ./<source_database>.<source_collection>.db)') args = parser.parse_args() # parse source and destination dest = utils.parse_mongo_url(args.dest) if os.path.exists(args.source): sources = utils.parse_source_file(args.source) else: sources = [utils.parse_mongo_url(args.source)] # initialize sqlite database that holds our state (this may seem like overkill, # but it's actually needed to ensure proper synchronization of subprocesses) if not args.state_db: args.state_db = '%s.%s.db' % (sources[0]['db'], sources[0]['collection']) if args.state_db.startswith('/'): state_db_path = args.state_db else: state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), args.state_db)
) parser.add_argument("--percent", type=int, metavar="PCT", default=None, help="copy only PCT%% of data") parser.add_argument( "--restart", action="store_true", help="restart from the beginning, ignoring any prior progress" ) parser.add_argument( "--state-db", type=str, metavar="PATH", default=None, help="path to state file (defaults to ./<source_database>.<source_collection>.db)", ) args = parser.parse_args() # parse source and destination dest = utils.parse_mongo_url(args.dest) if os.path.exists(args.source): sources = utils.parse_source_file(args.source) else: sources = [utils.parse_mongo_url(args.source)] # initialize sqlite database that holds our state (this may seem like overkill, # but it's actually needed to ensure proper synchronization of subprocesses) if not args.state_db: args.state_db = "%s.%s.db" % (sources[0]["db"], sources[0]["collection"]) if args.state_db.startswith("/"): state_db_path = args.state_db else: state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), args.state_db)
help= 'source to read from; can be a file containing sources or a url like: host[:port]/db/collection; ' 'e.g. localhost:27017/prod_maestro.emails') parser.add_argument( '--seconds', type=int, required=True, metavar="Minutes", help='Adjust the TTL of the messages database by N minutes') args = parser.parse_args() # parse source if os.path.exists(args.source): sources = utils.parse_source_file(args.source) else: sources = [utils.parse_mongo_url(args.source)] # initialize sqlite database that holds our state (this may seem like overkill, # but it's actually needed to ensure proper synchronization of subprocesses) args.state_db = '%s.%s.db' % (sources[0]['db'], sources[0]['collection']) state_db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), args.state_db) log.info('using state db %s' % state_db_path) state_db_exists = os.path.exists(state_db_path) state_db = CopyStateDB(state_db_path) if not state_db_exists: state_db.drop_and_create() # do the real work
gevent.monkey.patch_socket() parser = argparse.ArgumentParser(description='Through stdin, reads JSON documents containing _ids and shark keys for mismatching documents and re-copies those documents.') parser.add_argument( '--source', type=str, required=True, metavar='URL', help='source to read from; e.g. localhost:27017/prod_maestro.emails') parser.add_argument( '--dest', type=str, required=True, metavar='URL', help='destination to copy to; e.g. localhost:27017/destination_db.emails') parser.add_argument( '--mismatches-file', type=str, default=None, required=True, metavar='FILENAME', help='read ids to copy from this file, which is generated by compare_collections.py') args = parser.parse_args() # connect to source and destination source = utils.parse_mongo_url(args.source) source_client = utils.mongo_connect(source, ensure_direct=True, max_pool_size=POOL_SIZE, read_preference=ReadPreference.SECONDARY_PREFERRED, document_class=FasterOrderedDict) source_collection = source_client[source['db']][source['collection']] if not source_client.is_mongos or source_client.is_primary: raise Exception("source must be a mongos instance or a primary") dest = utils.parse_mongo_url(args.dest) dest_client = utils.mongo_connect(dest, max_pool_size=POOL_SIZE, document_class=FasterOrderedDict) dest_collection = dest_client[dest['db']][dest['collection']]
gevent.monkey.patch_socket() parser = argparse.ArgumentParser(description='Through stdin, reads JSON documents containing _ids and shark keys for mismatching documents and re-copies those documents.') parser.add_argument( '--source', type=str, required=True, metavar='URL', help='source to read from; e.g. localhost:27017/prod_maestro.emails') parser.add_argument( '--dest', type=str, required=True, metavar='URL', help='destination to copy to; e.g. localhost:27017/destination_db.emails') parser.add_argument( '--mismatches-file', type=str, default=None, required=True, metavar='FILENAME', help='read ids to copy from this file, which is generated by compare_collections.py') args = parser.parse_args() # connect to source and destination source = utils.parse_mongo_url(args.source) source_client = utils.mongo_connect(source['host'], source['port'], ensure_direct=True, max_pool_size=POOL_SIZE, read_preference=ReadPreference.SECONDARY_PREFERRED, document_class=FasterOrderedDict) source_collection = source_client[source['db']][source['collection']] if not source_client.is_mongos or source_client.is_primary: raise Exception("source must be a mongos instance or a primary") dest = utils.parse_mongo_url(args.dest) dest_client = utils.mongo_connect(dest['host'], dest['port'], max_pool_size=POOL_SIZE, document_class=FasterOrderedDict) dest_collection = dest_client[dest['db']][dest['collection']]