def test_move(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
config_file_content = f"""migration-folder: {dir_path}/../resources
python3-path: python3
nextflow-binary-path: nextflow
nextflow-config-path: {dir_path}/workflow.config
script-path: {dir_path}/../../
mongo-source-uri: mongodb://localhost:27017/admin
mongo-source-secrets-file: {dir_path}/empty_secret_file
mongo-dest-uri: mongodb://localhost:27018/admin
mongo-dest-secrets-file: {dir_path}/empty_secret_file
"""
open(f"{dir_path}/migration_config.yml", "w").write(config_file_content)
mover = MoveMongoDBs(migration_config_file=f"{dir_path}/migration_config.yml",
dbs_to_migrate_list=f"{dir_path}/dbs_to_migrate.txt",
batch_number="1", resume_flag=False)
# Load data to source
for db_name in mover.dbs_to_migrate:
source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name)
source_db.drop()
source_db.restore_data(dump_dir=f"{dir_path}/../resources/{db_name}")
mover.move()
# Check if source data made it to the destination
for db_name in mover.dbs_to_migrate:
source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name)
dest_db = MongoDatabase(mover.migration_config["mongo-dest-uri"], db_name=db_name)
for collection_name in source_db.get_collection_names():
self.assertEqual(source_db.mongo_handle[db_name][collection_name].count_documents(filter={}),
dest_db.mongo_handle[db_name][collection_name].count_documents(filter={}))
def _restore_data_to_another_db(self):
with tempfile.TemporaryDirectory() as tempdir:
self.test_mongo_db.dump_data(tempdir)
test_restore_db = MongoDatabase(uri=self.uri, db_name=self.test_mongo_db.db_name + "_restore")
test_restore_db.drop()
test_restore_db.restore_data(dump_dir=tempdir,
mongorestore_args={
"nsFrom": f'"{self.test_mongo_db.db_name}.*"',
"nsTo": f'"{test_restore_db.db_name}.*"'})
return test_restore_db
def restore_data_to_dest(mongo_dest: MongoDatabase, top_level_dump_dir):
try:
dump_dir = os.path.join(top_level_dump_dir, mongo_dest.db_name)
logger.info(f"Loading data in target database from source dump {dump_dir}...")
# noIndexRestore - Do not restore indexes because MongoDB 3.2 does not have index compatibility with MongoDB 4.0
mongo_dest.restore_data(dump_dir=dump_dir,
mongorestore_args={"noIndexRestore": "",
"numParallelCollections": 4,
"numInsertionWorkersPerCollection": 4})
except Exception as ex:
logger.error(f"Error while restoring data to the destination database!\n{ex.__str__()}")
sys.exit(1)