def test_move(self): dir_path = os.path.dirname(os.path.realpath(__file__)) config_file_content = f"""migration-folder: {dir_path}/../resources python3-path: python3 nextflow-binary-path: nextflow nextflow-config-path: {dir_path}/workflow.config script-path: {dir_path}/../../ mongo-source-uri: mongodb://localhost:27017/admin mongo-source-secrets-file: {dir_path}/empty_secret_file mongo-dest-uri: mongodb://localhost:27018/admin mongo-dest-secrets-file: {dir_path}/empty_secret_file """ open(f"{dir_path}/migration_config.yml", "w").write(config_file_content) mover = MoveMongoDBs(migration_config_file=f"{dir_path}/migration_config.yml", dbs_to_migrate_list=f"{dir_path}/dbs_to_migrate.txt", batch_number="1", resume_flag=False) # Load data to source for db_name in mover.dbs_to_migrate: source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name) source_db.drop() source_db.restore_data(dump_dir=f"{dir_path}/../resources/{db_name}") mover.move() # Check if source data made it to the destination for db_name in mover.dbs_to_migrate: source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name) dest_db = MongoDatabase(mover.migration_config["mongo-dest-uri"], db_name=db_name) for collection_name in source_db.get_collection_names(): self.assertEqual(source_db.mongo_handle[db_name][collection_name].count_documents(filter={}), dest_db.mongo_handle[db_name][collection_name].count_documents(filter={}))
def _restore_data_to_another_db(self): with tempfile.TemporaryDirectory() as tempdir: self.test_mongo_db.dump_data(tempdir) test_restore_db = MongoDatabase(uri=self.uri, db_name=self.test_mongo_db.db_name + "_restore") test_restore_db.drop() test_restore_db.restore_data(dump_dir=tempdir, mongorestore_args={ "nsFrom": f'"{self.test_mongo_db.db_name}.*"', "nsTo": f'"{test_restore_db.db_name}.*"'}) return test_restore_db
def restore_data_to_dest(mongo_dest: MongoDatabase, top_level_dump_dir): try: dump_dir = os.path.join(top_level_dump_dir, mongo_dest.db_name) logger.info(f"Loading data in target database from source dump {dump_dir}...") # noIndexRestore - Do not restore indexes because MongoDB 3.2 does not have index compatibility with MongoDB 4.0 mongo_dest.restore_data(dump_dir=dump_dir, mongorestore_args={"noIndexRestore": "", "numParallelCollections": 4, "numInsertionWorkersPerCollection": 4}) except Exception as ex: logger.error(f"Error while restoring data to the destination database!\n{ex.__str__()}") sys.exit(1)