def test_move(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
config_file_content = f"""migration-folder: {dir_path}/../resources
python3-path: python3
nextflow-binary-path: nextflow
nextflow-config-path: {dir_path}/workflow.config
script-path: {dir_path}/../../
mongo-source-uri: mongodb://localhost:27017/admin
mongo-source-secrets-file: {dir_path}/empty_secret_file
mongo-dest-uri: mongodb://localhost:27018/admin
mongo-dest-secrets-file: {dir_path}/empty_secret_file
"""
open(f"{dir_path}/migration_config.yml", "w").write(config_file_content)
mover = MoveMongoDBs(migration_config_file=f"{dir_path}/migration_config.yml",
dbs_to_migrate_list=f"{dir_path}/dbs_to_migrate.txt",
batch_number="1", resume_flag=False)
# Load data to source
for db_name in mover.dbs_to_migrate:
source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name)
source_db.drop()
source_db.restore_data(dump_dir=f"{dir_path}/../resources/{db_name}")
mover.move()
# Check if source data made it to the destination
for db_name in mover.dbs_to_migrate:
source_db = MongoDatabase(mover.migration_config["mongo-source-uri"], db_name=db_name)
dest_db = MongoDatabase(mover.migration_config["mongo-dest-uri"], db_name=db_name)
for collection_name in source_db.get_collection_names():
self.assertEqual(source_db.mongo_handle[db_name][collection_name].count_documents(filter={}),
dest_db.mongo_handle[db_name][collection_name].count_documents(filter={}))
def create_collection_count_validation_report(mongo_source: MongoDatabase,
database_list,
private_config_xml_file):
report_timestamp = datetime.now()
mongo_host = mongo_source.mongo_handle.address[0]
for db in database_list:
mongo_source.db_name = db
source_collections = mongo_source.get_collection_names()
if not source_collections:
logger.warning(
f"database {db} does not exist in mongo instances {mongo_host}"
)
continue
for coll in sorted(source_collections):
logger.info(
f"fetching count for database ({db}) - collection ({coll})")
no_of_documents = get_documents_count_for_collection(
mongo_source, db, coll)
logger.info(
f"Found {no_of_documents} documents in database ({db}) - collection ({coll})"
)
insert_count_validation_result_to_db(
private_config_xml_file,
(mongo_host, db, coll, no_of_documents, report_timestamp))
def provision_new_database_for_variant_warehouse(db_name):
"""Create a variant warehouse database of the specified name and shared the collections"""
# Passing the secrets_file override the password already in the uri
db_handle = MongoDatabase(
uri=cfg['mongodb']['mongo_admin_uri'],
secrets_file=cfg['mongodb']['mongo_admin_secrets_file'],
db_name=db_name)
if len(db_handle.get_collection_names()) > 0:
logger.info(f'Found existing database named {db_name}.')
else:
db_handle.enable_sharding()
db_handle.shard_collections(
collections_shard_key_map,
collections_to_shard=collections_shard_key_map.keys())
logger.info(f'Created new database named {db_name}.')
def prepare_dest_db(mongo_source_db: MongoDatabase,
mongo_dest_db: MongoDatabase):
try:
logger.info("Dropping target database if it already exists...")
mongo_dest_db.drop()
logger.info("Enabling sharding in the target database...")
mongo_dest_db.enable_sharding()
logger.info("Sharding collections in the target database...")
mongo_dest_db.shard_collections(
collections_shard_key_map,
collections_to_shard=mongo_source_db.get_collection_names())
except Exception as ex:
logger.error(
f"Error while preparing destination database!\n{ex.__str__()}")
sys.exit(1)