def validate_rs_release_files(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version, species_release_folder): port_forwarding_process_id, mongo_port, exit_code = None, None, None try: port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version) db_name_in_tempmongo_instance = get_release_db_name_in_tempmongo_instance(taxonomy_id) with MongoClient(port=mongo_port) as client: mongo_unique_rs_ids_file = os.path.join(species_release_folder, assembly_accession, "{0}_mongo_unique_rs_ids.txt".format(assembly_accession)) export_unique_rs_ids_from_mongo(mongo_port, db_name_in_tempmongo_instance, assembly_accession, mongo_unique_rs_ids_file) unique_release_rs_ids_file = get_unique_release_rs_ids(species_release_folder, assembly_accession) missing_rs_ids_file = os.path.join(os.path.dirname(unique_release_rs_ids_file), assembly_accession + "_missing_ids.txt") file_diff(mongo_unique_rs_ids_file, unique_release_rs_ids_file, FileDiffOption.NOT_IN, missing_rs_ids_file) get_missing_ids_attributions(assembly_accession, missing_rs_ids_file, client[db_name_in_tempmongo_instance]) exit_code = 0 except Exception as ex: logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n" + traceback.format_exc()) exit_code = -1 finally: close_mongo_port_to_tempmongo(port_forwarding_process_id) logger.info("Validate RS release file process completed with exit_code: " + str(exit_code)) sys.exit(exit_code)
def run_release_for_assembly(private_config_xml_file, taxonomy_id, assembly_accession, release_species_inventory_table, release_version, species_release_folder, release_jar_path, job_repo_url, memory): exit_code = 0 try: port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo( private_config_xml_file, taxonomy_id, release_species_inventory_table, release_version) release_properties_file = create_release_properties_file_for_assembly( private_config_xml_file, taxonomy_id, assembly_accession, release_species_inventory_table, release_version, species_release_folder, job_repo_url) release_command = 'java -Xmx{0}g -jar {1} --spring.config.location="{2}" -Dspring.data.mongodb.port={3}'\ .format(memory, release_jar_path, release_properties_file, mongo_port) run_command_with_output( "Running release pipeline for assembly: " + assembly_accession, release_command) except Exception as ex: logger.error( "Encountered an error while running release for assembly: " + assembly_accession + "\n" + traceback.format_exc()) exit_code = -1 finally: close_mongo_port_to_tempmongo(port_forwarding_process_id) logger.info("Java release pipeline run completed with exit_code: " + str(exit_code)) sys.exit(exit_code)
def copy_accessioning_collections_to_embassy(private_config_xml_file, profile, taxonomy_id, assembly_accession, collections_to_copy, release_species_inventory_table, release_version, dump_dir): port_forwarding_process_id, mongo_port, exit_code = None, None, None try: port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version) with psycopg2.connect(get_pg_metadata_uri_for_eva_profile(profile, private_config_xml_file), user="******") as \ metadata_connection_handle: # To be idempotent, clear destination tempmongo database destination_db_name = get_release_db_name_in_tempmongo_instance(taxonomy_id) MongoClient(port=mongo_port).drop_database(destination_db_name) release_info = get_release_inventory_info_for_assembly(taxonomy_id, assembly_accession, release_species_inventory_table, release_version, metadata_connection_handle) logger.info("Beginning data copy to remote MongoDB host {0} on port {1}..." .format(release_info["tempmongo_instance"], mongo_port)) collections_to_copy_map = get_collections_to_copy(collections_to_copy, sources=release_info["sources"]) mongo_data_copy_to_remote_host(mongo_port, private_config_xml_file, profile, assembly_accession, collections_to_copy_map, dump_dir, destination_db_name) exit_code = 0 except Exception as ex: logger.error("Encountered an error while copying species data to Embassy for assemblies in " + release_info["tempmongo_instance"] + "\n" + traceback.format_exc()) exit_code = -1 finally: close_mongo_port_to_tempmongo(port_forwarding_process_id) logger.info("Copy process completed with exit_code: " + str(exit_code)) sys.exit(exit_code)