def test_move(self): dir_path = os.path.dirname(os.path.realpath(__file__)) config_file_content = f"""migration-folder: {dir_path}/../resources python3-path: python3 nextflow-binary-path: nextflow nextflow-config-path: {dir_path}/workflow.config script-path: {dir_path}/../../ mongo-source-uri: mongodb://localhost:27017/admin mongo-source-secrets-file: {dir_path}/empty_secret_file mongo-dest-uri: mongodb://localhost:27018/admin mongo-dest-secrets-file: {dir_path}/empty_secret_file """ open(f"{dir_path}/migration_config.yml", "w").write(config_file_content) mover = MoveMongoDBs(migration_config_file=f"{dir_path}/migration_config.yml", dbs_to_migrate_list=f"{dir_path}/dbs_to_migrate.txt", batch_number="1", resume_flag=False) # Load data to source for db_name in mover.dbs_to_migrate: source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name) source_db.drop() source_db.restore_data(dump_dir=f"{dir_path}/../resources/{db_name}") mover.move() # Check if source data made it to the destination for db_name in mover.dbs_to_migrate: source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name) dest_db = MongoDatabase(mover.migration_config["mongo-dest-uri"], db_name=db_name) for collection_name in source_db.get_collection_names(): self.assertEqual(source_db.mongo_handle[db_name][collection_name].count_documents(filter={}), dest_db.mongo_handle[db_name][collection_name].count_documents(filter={}))
def create_collection_count_validation_report(mongo_source: MongoDatabase, database_list, private_config_xml_file): report_timestamp = datetime.now() mongo_host = mongo_source.mongo_handle.address[0] for db in database_list: mongo_source.db_name = db source_collections = mongo_source.get_collection_names() if not source_collections: logger.warning( f"database {db} does not exist in mongo instances {mongo_host}" ) continue for coll in sorted(source_collections): logger.info( f"fetching count for database ({db}) - collection ({coll})") no_of_documents = get_documents_count_for_collection( mongo_source, db, coll) logger.info( f"Found {no_of_documents} documents in database ({db}) - collection ({coll})" ) insert_count_validation_result_to_db( private_config_xml_file, (mongo_host, db, coll, no_of_documents, report_timestamp))
def provision_new_database_for_variant_warehouse(db_name): """Create a variant warehouse database of the specified name and shared the collections""" # Passing the secrets_file override the password already in the uri db_handle = MongoDatabase( uri=cfg['mongodb']['mongo_admin_uri'], secrets_file=cfg['mongodb']['mongo_admin_secrets_file'], db_name=db_name) if len(db_handle.get_collection_names()) > 0: logger.info(f'Found existing database named {db_name}.') else: db_handle.enable_sharding() db_handle.shard_collections( collections_shard_key_map, collections_to_shard=collections_shard_key_map.keys()) logger.info(f'Created new database named {db_name}.')
def prepare_dest_db(mongo_source_db: MongoDatabase, mongo_dest_db: MongoDatabase): try: logger.info("Dropping target database if it already exists...") mongo_dest_db.drop() logger.info("Enabling sharding in the target database...") mongo_dest_db.enable_sharding() logger.info("Sharding collections in the target database...") mongo_dest_db.shard_collections( collections_shard_key_map, collections_to_shard=mongo_source_db.get_collection_names()) except Exception as ex: logger.error( f"Error while preparing destination database!\n{ex.__str__()}") sys.exit(1)