def prepare_release_workflow_file_for_species(common_release_properties,
taxonomy_id, assemblies, memory):
process_index = 1
release_properties = merge_two_dicts(
common_release_properties,
get_release_properties_for_current_species(common_release_properties,
taxonomy_id, memory))
# This hack is needed to kick-off the initial process in Nextflow
release_properties["previous-process-output-flag"] = "true"
release_properties["current-process-output-flag"] = "flag" + str(
process_index)
# Ensure that PYTHONPATH is properly set so that scripts can be run
# as "python3 -m run_release_in_embassy.<script_name>"
release_properties["script-path"] = os.environ["PYTHONPATH"]
workflow_file_name = os.path.join(
release_properties["species-release-folder"],
"{taxonomy-id}_release_workflow_{timestamp}.nf".format(
**release_properties))
with open(workflow_file_name, "w") as workflow_file_handle:
header = "#!/usr/bin/env nextflow"
workflow_file_handle.write(header + "\n")
for assembly_accession in assemblies:
release_properties = merge_two_dicts(
release_properties,
get_release_properties_for_current_assembly(
release_properties, assembly_accession))
for workflow_process_name, workflow_process_args in workflow_process_arguments_map.items(
):
release_properties[
"current-process-output-flag"] = "flag" + str(
process_index)
if workflow_process_name == copy_process:
# Copy process to Embassy must be carried out only once per species
if process_index == 1:
workflow_file_handle.write(
get_nextflow_process_definition(
release_properties,
workflow_process_name,
workflow_process_args,
process_name_suffix=str(taxonomy_id)))
else:
continue
else:
workflow_file_handle.write(
get_nextflow_process_definition(
release_properties,
workflow_process_name,
workflow_process_args,
process_name_suffix=assembly_accession.replace(
".", "_")))
workflow_file_handle.write("\n")
process_index += 1
# Set the flag that will capture the output status of the current process
# This variable will be used to decide whether the next process should be started
# See http://nextflow-io.github.io/patterns/index.html#_mock_dependency
release_properties[
"previous-process-output-flag"] = "flag" + str(
process_index - 1)
return workflow_file_name, release_properties["release-log-file"]
def copy_db_with_config(mongo_source_config: MongoConfig,
mongo_destination_config: MongoConfig,
mongodump_args: dict, mongorestore_args: dict):
copy_db(
merge_two_dicts(mongo_source_config.parameters, mongodump_args),
merge_two_dicts(mongo_destination_config.parameters,
mongorestore_args))
def get_release_properties_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession,
release_species_inventory_table, release_version, species_release_folder):
with psycopg2.connect(get_pg_metadata_uri_for_eva_profile(profile, private_config_xml_file),
user="******") as \
metadata_connection_handle:
release_inventory_info_for_assembly = get_release_inventory_info_for_assembly(taxonomy_id, assembly_accession,
release_species_inventory_table,
release_version,
metadata_connection_handle)
if not release_inventory_info_for_assembly["report_path"].startswith("file:"):
release_inventory_info_for_assembly["report_path"] = "file:" + \
release_inventory_info_for_assembly["report_path"]
release_inventory_info_for_assembly["output_folder"] = os.path.join(species_release_folder, assembly_accession)
release_inventory_info_for_assembly["mongo_accessioning_db"] = \
get_release_db_name_in_tempmongo_instance(taxonomy_id)
return merge_two_dicts(release_inventory_info_for_assembly,
get_release_job_repo_properties(private_config_xml_file, profile))