def validate_rs_release_files(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table,
release_version, species_release_folder):
port_forwarding_process_id, mongo_port, exit_code = None, None, None
try:
port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, taxonomy_id,
assembly_accession, release_species_inventory_table,
release_version)
db_name_in_tempmongo_instance = get_release_db_name_in_tempmongo_instance(taxonomy_id)
with MongoClient(port=mongo_port) as client:
mongo_unique_rs_ids_file = os.path.join(species_release_folder, assembly_accession,
"{0}_mongo_unique_rs_ids.txt".format(assembly_accession))
export_unique_rs_ids_from_mongo(mongo_port, db_name_in_tempmongo_instance, assembly_accession,
mongo_unique_rs_ids_file)
unique_release_rs_ids_file = get_unique_release_rs_ids(species_release_folder, assembly_accession)
missing_rs_ids_file = os.path.join(os.path.dirname(unique_release_rs_ids_file),
assembly_accession + "_missing_ids.txt")
file_diff(mongo_unique_rs_ids_file, unique_release_rs_ids_file, FileDiffOption.NOT_IN, missing_rs_ids_file)
get_missing_ids_attributions(assembly_accession, missing_rs_ids_file, client[db_name_in_tempmongo_instance])
exit_code = 0
except Exception as ex:
logger.error("Encountered an error while running release for assembly: " + assembly_accession + "\n"
+ traceback.format_exc())
exit_code = -1
finally:
close_mongo_port_to_tempmongo(port_forwarding_process_id)
logger.info("Validate RS release file process completed with exit_code: " + str(exit_code))
sys.exit(exit_code)
def run_release_for_assembly(private_config_xml_file, taxonomy_id,
assembly_accession,
release_species_inventory_table, release_version,
species_release_folder, release_jar_path,
job_repo_url, memory):
exit_code = 0
try:
port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(
private_config_xml_file, taxonomy_id,
release_species_inventory_table, release_version)
release_properties_file = create_release_properties_file_for_assembly(
private_config_xml_file, taxonomy_id, assembly_accession,
release_species_inventory_table, release_version,
species_release_folder, job_repo_url)
release_command = 'java -Xmx{0}g -jar {1} --spring.config.location="{2}" -Dspring.data.mongodb.port={3}'\
.format(memory, release_jar_path, release_properties_file, mongo_port)
run_command_with_output(
"Running release pipeline for assembly: " + assembly_accession,
release_command)
except Exception as ex:
logger.error(
"Encountered an error while running release for assembly: " +
assembly_accession + "\n" + traceback.format_exc())
exit_code = -1
finally:
close_mongo_port_to_tempmongo(port_forwarding_process_id)
logger.info("Java release pipeline run completed with exit_code: " +
str(exit_code))
sys.exit(exit_code)
def copy_accessioning_collections_to_embassy(private_config_xml_file, profile, taxonomy_id, assembly_accession,
collections_to_copy, release_species_inventory_table, release_version,
dump_dir):
port_forwarding_process_id, mongo_port, exit_code = None, None, None
try:
port_forwarding_process_id, mongo_port = open_mongo_port_to_tempmongo(private_config_xml_file, profile, taxonomy_id,
assembly_accession, release_species_inventory_table,
release_version)
with psycopg2.connect(get_pg_metadata_uri_for_eva_profile(profile, private_config_xml_file),
user="******") as \
metadata_connection_handle:
# To be idempotent, clear destination tempmongo database
destination_db_name = get_release_db_name_in_tempmongo_instance(taxonomy_id)
MongoClient(port=mongo_port).drop_database(destination_db_name)
release_info = get_release_inventory_info_for_assembly(taxonomy_id, assembly_accession,
release_species_inventory_table,
release_version, metadata_connection_handle)
logger.info("Beginning data copy to remote MongoDB host {0} on port {1}..."
.format(release_info["tempmongo_instance"], mongo_port))
collections_to_copy_map = get_collections_to_copy(collections_to_copy, sources=release_info["sources"])
mongo_data_copy_to_remote_host(mongo_port, private_config_xml_file, profile, assembly_accession,
collections_to_copy_map, dump_dir, destination_db_name)
exit_code = 0
except Exception as ex:
logger.error("Encountered an error while copying species data to Embassy for assemblies in "
+ release_info["tempmongo_instance"] + "\n" + traceback.format_exc())
exit_code = -1
finally:
close_mongo_port_to_tempmongo(port_forwarding_process_id)
logger.info("Copy process completed with exit_code: " + str(exit_code))
sys.exit(exit_code)