def sort_bgzip_tabix_release_files(bgzip_path, tabix_path, vcf_sort_script_path, assembly_accession,
species_release_folder):
commands = []
# These files are left behind by the sort_vcf_sorted_chromosomes.sh script
# To be idempotent, remove such files
commands.append("rm -f {0}/{1}/*.chromosomes".format(species_release_folder, assembly_accession))
for vcf_file_category in release_vcf_file_categories:
unsorted_release_file_name = get_unsorted_release_vcf_file_name(species_release_folder, assembly_accession,
vcf_file_category)
sorted_release_file_name = get_release_vcf_file_name(species_release_folder, assembly_accession,
vcf_file_category)
commands.append("rm -f {2} && {0} -f {1} {2}".format(vcf_sort_script_path,
unsorted_release_file_name,
sorted_release_file_name))
commands.extend(get_bgzip_tabix_commands_for_file(bgzip_path, tabix_path, sorted_release_file_name))
for text_release_file_category in release_text_file_categories:
unsorted_release_file_name = get_unsorted_release_text_file_name(species_release_folder, assembly_accession,
text_release_file_category)
sorted_release_file_name = get_release_text_file_name(species_release_folder, assembly_accession,
text_release_file_category)
commands.append("(sort -V {1} | uniq > {2})".format(vcf_sort_script_path,
unsorted_release_file_name,
sorted_release_file_name))
commands.append("(gzip < {0} > {0}.gz)".format(sorted_release_file_name))
command = " && ".join(commands)
run_command_with_output("Sort, bgzip and tabix release files for assembly: " + assembly_accession,
command)
def count_rs_ids_in_release_files(count_ids_script_path, assembly_accession,
species_release_folder):
release_count_filename = os.path.join(species_release_folder,
assembly_accession,
"README_rs_ids_counts.txt")
with open(release_count_filename, "w") as release_count_file_handle:
release_count_file_handle.write("# Unique RS ID counts\n")
for vcf_file_category in release_vcf_file_categories:
release_vcf_file_name = get_release_vcf_file_name(
species_release_folder, assembly_accession, vcf_file_category)
num_ids_in_file = run_command_with_output(
"Counting RS IDs in file: " + release_vcf_file_name,
"{0} {1}.gz".format(count_ids_script_path,
release_vcf_file_name),
return_process_output=True)
release_count_file_handle.write(num_ids_in_file)
for text_release_file_category in release_text_file_categories:
text_release_file_name = get_release_text_file_name(
species_release_folder, assembly_accession,
text_release_file_category)
num_ids_in_file = run_command_with_output(
"Counting RS IDs in file: " + text_release_file_name,
"zcat {0}.gz | cut -f1 | uniq | wc -l".format(
text_release_file_name),
return_process_output=True)
release_count_file_handle.write("{0}.gz\t{1}".format(
os.path.basename(text_release_file_name),
str(num_ids_in_file)))
def validate_release_vcf_files(private_config_xml_file, profile, taxonomy_id,
assembly_accession,
release_species_inventory_table,
release_version, species_release_folder,
vcf_validator_path, assembly_checker_path):
run_command_with_output(
"Remove existing VCF validation and assembly report outputs...",
"rm -f {0}/{1}/{2} {0}/{1}/{3}".format(
species_release_folder, assembly_accession,
vcf_validation_output_file_pattern,
asm_report_output_file_pattern))
validate_release_vcf_files_commands = []
with psycopg2.connect(get_pg_metadata_uri_for_eva_profile(profile, private_config_xml_file),
user="******") as \
metadata_connection_handle:
release_inventory_info_for_assembly = get_release_inventory_info_for_assembly(
taxonomy_id, assembly_accession, release_species_inventory_table,
release_version, metadata_connection_handle)
fasta_path = release_inventory_info_for_assembly["fasta_path"]
assembly_report_path = release_inventory_info_for_assembly[
"report_path"]
remove_index_if_outdated(fasta_path)
if assembly_report_path.startswith("file:/"):
assembly_report_path = assembly_report_path.replace("file:/", "/")
for vcf_file_category in release_vcf_file_categories:
release_vcf_file_name = get_release_vcf_file_name(
species_release_folder, assembly_accession, vcf_file_category)
release_vcf_dir = os.path.dirname(release_vcf_file_name)
if "multimap" not in vcf_file_category:
validate_release_vcf_files_commands.append(
"({0} -i {1} -o {2}) || true".format(
vcf_validator_path, release_vcf_file_name,
release_vcf_dir))
validate_release_vcf_files_commands.append(
"({0} -i {1} -f {2} -a {3} -o {4} -r text,summary) || true"
.format(assembly_checker_path, release_vcf_file_name,
fasta_path, assembly_report_path, release_vcf_dir))
# We don't expect the validation commands to all pass, hence use semi-colon to run them back to back
final_validate_command = " ; ".join(
validate_release_vcf_files_commands)
run_command_with_output(
"Validating release files for assembly: " + assembly_accession,
final_validate_command)
def move_release_files_to_unsorted_category(assembly_accession,
species_release_folder,
vcf_file_category,
unsorted_release_file_path):
unsorted_release_file_name = os.path.basename(unsorted_release_file_path)
release_file_path = get_release_vcf_file_name(species_release_folder,
assembly_accession,
vcf_file_category)
release_file_name = os.path.basename(release_file_path)
for variant_source in ["eva", "dbsnp"]:
vcf_file_name = release_file_path.replace(
release_file_name, "{0}_{1}".format(variant_source,
release_file_name))
unsorted_file_name = unsorted_release_file_path.replace(
unsorted_release_file_name,
"{0}_{1}".format(variant_source, unsorted_release_file_name))
if os.path.exists(
vcf_file_name) and not os.path.exists(unsorted_file_name):
os.rename(vcf_file_name, unsorted_file_name)