def add_length_to_accordance_file(accordance_file, length_file,
output_prefix):
accordance_dict = SynDict(filename=accordance_file,
allow_repeats_of_key=True)
length_dict = SynDict(filename=length_file, expression=int)
print(length_dict)
longest_list = IdList()
all_output_file = "%s.all.correspondence" % output_prefix
longest_output_file = "%s.longest.correspondence" % output_prefix
longest_id_file = "%s.longest.ids" % output_prefix
with open(all_output_file, "w") as all_out_fd:
with open(longest_output_file, "w") as longest_out_fd:
for gene in accordance_dict:
current_transcript = None
current_length = 0
for transcript in accordance_dict[gene]:
if length_dict[transcript] > current_length:
current_transcript = transcript
current_length = length_dict[transcript]
all_out_fd.write(
"%s\t%s\t%i\n" %
(gene, transcript, length_dict[transcript]))
longest_out_fd.write(
"%s\t%s\t%i\n" %
(gene, current_transcript, current_length))
longest_list.append(current_transcript)
longest_list.write(longest_id_file)
def get_longest_pep_per_gene_from_ensembl_pep_dict(protein_dict,
output_prefix=None):
length_file = "%s.protein_length.tsv" % output_prefix
if output_prefix:
longest_protein_id_file = "%s.longest_pep.ids" % output_prefix
len_fd = open(length_file, 'w')
len_fd.write("#gene_id\tprotein_id\tprotein_length\n")
data_dict = OrderedDict()
for protein_id in protein_dict:
length = len(protein_dict[protein_id].seq)
description_list = protein_dict[protein_id].description.split()
#print protein_dict[protein_id]
#print ''
#print description_list
for entry in description_list:
if "gene:" in entry:
gene_id = entry.split(":")[1]
if output_prefix:
len_fd.write("%s\t%s\t%i\n" % (gene_id, protein_id, length))
if gene_id not in data_dict:
data_dict[gene_id] = protein_id
else:
if length > len(protein_dict[data_dict[gene_id]].seq):
data_dict[gene_id] = protein_id
longest_pep_ids = IdList(data_dict.values())
if output_prefix:
longest_pep_ids.write(longest_protein_id_file)
len_fd.close()
return longest_pep_ids
def create_per_cluster_element_id_files(self, cluster_dict,
output_directory):
self.safe_mkdir(output_directory)
for cluster_id in cluster_dict:
cluster_element_id_list = IdList(cluster_dict[cluster_id])
cluster_element_id_list.write("%s/%s.ids" %
(output_directory, cluster_id))
def prepare_annotation_file_from_transcript_and_cds(
self,
transcript_file,
cds_file,
correspondence_file,
output_prefix,
format="fasta",
correspondence_key_column=0,
correspondence_value_column=1,
verbose=False):
transcript_dict = self.parse_seq_file(transcript_file,
"parse",
format=format)
cds_dict = self.parse_seq_file(cds_file, "parse", format=format)
correspondence_dict = SynDict(filename=correspondence_file,
comments_prefix="#",
key_index=correspondence_key_column,
value_index=correspondence_value_column)
no_corresponding_cds_transcript_list = IdList()
cds_not_found_transcript_list = IdList()
annotation_file = "%s.annotation" % output_prefix
no_corresponding_cds_transcript_file = "%s.no_cds.id" % output_prefix
cds_not_found_transcript_file = "%s.not_found_cds.id" % output_prefix
with open(annotation_file, "w") as annotation_fd:
for transcript_id in transcript_dict:
if transcript_id not in correspondence_dict:
no_corresponding_cds_transcript_list.append(transcript_id)
if verbose:
print(
"No cds in correspondence file for transcript %s" %
transcript_id)
continue
cds_id = correspondence_dict[transcript_id]
length = len(cds_dict[cds_id].seq)
start = transcript_dict[transcript_id].seq.upper().find(
cds_dict[cds_id].seq.upper())
if start == -1:
cds_not_found_transcript_list.append(transcript_id)
if verbose:
print("CDS was not found for transcript %s" %
transcript_id)
continue
annotation_string = "%s\t+\t%i\t%i\n" % (transcript_id,
start + 1, length)
annotation_fd.write(annotation_string)
no_corresponding_cds_transcript_list.write(
no_corresponding_cds_transcript_file)
cds_not_found_transcript_list.write(cds_not_found_transcript_file)
def extract_ids_from_file(input_file,
output_file=None,
header=False,
column_separator="\t",
comments_prefix="#",
column_number=None):
id_list = IdList()
id_list.read(input_file,
column_separator=column_separator,
comments_prefix=comments_prefix,
column_number=column_number,
header=header)
if output_file:
id_list.write(output_file, header=header)
return id_list
def extract_eggnog_fam_by_protein_syn_dict(self, eggnog_fam_dict, protein_syn_dict, output_prefix=None, species_id=None):
extracted_families = SynDict()
common_protein_names_to_families_dict = SynDict()
common_names_to_eggnog_proteins_syn_dict = SynDict()
not_found_proteins_common_names = IdList()
transposed_eggnog_fam_dict = eggnog_fam_dict.exchange_key_and_value()
for common_protein_name in protein_syn_dict:
not_found = True
for protein_id in protein_syn_dict[common_protein_name]:
extended_protein_id = protein_id if species_id is None else species_id + "." + protein_id
if extended_protein_id in transposed_eggnog_fam_dict:
not_found = False
if common_protein_name not in common_protein_names_to_families_dict:
common_protein_names_to_families_dict[common_protein_name] = [transposed_eggnog_fam_dict[extended_protein_id][0]]
common_names_to_eggnog_proteins_syn_dict[common_protein_name] = [extended_protein_id]
else:
common_protein_names_to_families_dict[common_protein_name].append(transposed_eggnog_fam_dict[extended_protein_id][0])
common_names_to_eggnog_proteins_syn_dict[common_protein_name].append(extended_protein_id)
if transposed_eggnog_fam_dict[extended_protein_id][0] not in extracted_families:
extracted_families[transposed_eggnog_fam_dict[extended_protein_id][0]] = eggnog_fam_dict[transposed_eggnog_fam_dict[extended_protein_id][0]]
if not_found:
not_found_proteins_common_names.append(common_protein_name)
if output_prefix:
extracted_families.write(filename="%s.extracted_families.fam" % output_prefix, splited_values=True)
common_protein_names_to_families_dict.write(filename="%s.common_protein_names_to_families.correspondence" % output_prefix, splited_values=True)
common_names_to_eggnog_proteins_syn_dict.write(filename="%s.common_protein_names_to_eggnog_proteins.correspondence" % output_prefix, splited_values=True)
not_found_proteins_common_names.write(filename="%s.not_found.common_names" % output_prefix)
#print common_names_to_eggnog_proteins_syn_dict
#print common_protein_names_to_families_dict
return extracted_families, common_protein_names_to_families_dict, \
common_names_to_eggnog_proteins_syn_dict, not_found_proteins_common_names
def cluster_sequence_names_by_id_fragment(self,
seq_id_list,
id_element_index,
id_separator="_",
output_prefix=None):
cluster_dict = SynDict()
skipped_id_list = IdList()
for seq_id in seq_id_list:
seq_id_splited = seq_id.split(id_separator)
if id_element_index < len(seq_id_splited):
if seq_id_list[id_element_index] in cluster_dict:
cluster_dict[seq_id_list[id_element_index]].append(seq_id)
else:
cluster_dict[seq_id_list[id_element_index]] = [seq_id]
else:
skipped_id_list.append(seq_id)
if output_prefix:
cluster_dict.write("%s.seqid.clusters" % output_prefix,
splited_values=True)
skipped_id_list.write("%s.seqid.skipped.ids" % output_prefix)
return cluster_dict
__author__ = 'Sergei F. Kliver'
import sys
import argparse
from RouToolPa.Collections.General import IdList
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--fam_file",
action="store",
dest="fam_file",
required=True,
help="File with families")
parser.add_argument("-o",
"--output",
action="store",
dest="output",
default="stdout",
help="File to write ids")
args = parser.parse_args()
out_fd = sys.stdout if args.output == "stdout" else open(args.output, "w")
id_list = IdList()
id_list.read(args.fam_file,
close_after_if_file_object=True,
column_number=1,
id_in_column_separator=",")
id_list.write(args.output, close_after_if_file_object=True)
def handle_sanger_data(self,
input_dir,
output_prefix,
outdir=None,
read_subfolders=False,
min_mean_qual=0,
min_median_qual=0,
min_len=50):
if outdir:
self.workdir = outdir
self.init_dirs()
sanger_filelist = self.make_list_of_path_to_files(
input_dir,
expression=self.is_sanger_file,
recursive=read_subfolders,
return_absolute_paths=True)
stat_dict = TwoLvlDict()
record_dict = OrderedDict()
trimmed_record_dict = OrderedDict()
excluded_list = IdList()
excluded_counter = 0
low_quality_counter = 0
too_short_counter = 0
merged_raw_fastq = "%s/%s.raw.fastq" % (self.workdir, output_prefix)
merged_raw_fasta = "%s/%s.raw.fasta" % (self.workdir, output_prefix)
merged_trimmed_fastq = "%s/%s.trimmed.fastq" % (self.workdir,
output_prefix)
merged_trimmed_fasta = "%s/%s.trimmed.fasta" % (self.workdir,
output_prefix)
for filename in sanger_filelist:
filename_list = self.split_filename(filename)
record_raw_fastq = "%s/fastq/raw/%s.raw.fastq" % (self.workdir,
filename_list[1])
record_raw_fasta = "%s/fasta/raw/%s.raw.fasta" % (self.workdir,
filename_list[1])
record_raw_qual_plot_prefix = "%s/qual_plot/raw/%s.raw.qual" % (
self.workdir, filename_list[1])
record_trimmed_fastq = "%s/fastq/trimmed/%s.trimmed.fastq" % (
self.workdir, filename_list[1])
record_trimmed_fasta = "%s/fasta/trimmed/%s.trimmed.fasta" % (
self.workdir, filename_list[1])
record_trimmed_qual_plot_prefix = "%s/qual_plot/trimmed/%s.trimmed.qual" % (
self.workdir, filename_list[1])
record = SeqIO.read(self.metaopen(filename, "rb"), format="abi")
record_dict[record.id] = record
SeqIO.write(record, record_raw_fastq, format="fastq")
SeqIO.write(record, record_raw_fasta, format="fasta")
trimmed_record = SeqIO.AbiIO._abi_trim(record)
stat_dict[record.id] = OrderedDict({
"raw_len":
len(record),
"raw_mean_qual":
np.mean(record.letter_annotations["phred_quality"]),
"raw_median_qual":
np.median(record.letter_annotations["phred_quality"]),
"trimmed_len":
len(trimmed_record),
"trimmed_mean_qual":
np.mean(trimmed_record.letter_annotations["phred_quality"]),
"trimmed_median_qual":
np.median(trimmed_record.letter_annotations["phred_quality"]),
"retained":
"-",
})
MatplotlibRoutines.draw_bar_plot(
record.letter_annotations["phred_quality"],
record_raw_qual_plot_prefix,
extentions=["png"],
xlabel="Position",
ylabel="Phred quality",
title="Per base quality",
min_value=None,
max_value=None,
new_figure=True,
figsize=(3 * (int(len(record) / 100) + 1), 3),
close_figure=True)
if stat_dict[record.id]["trimmed_len"] >= min_len:
if min_median_qual:
if (stat_dict[record.id]["trimmed_median_qual"] >=
min_median_qual) and (
stat_dict[record.id]["trimmed_mean_qual"] >=
min_mean_qual):
stat_dict[record.id]["retained"] = "+"
else:
low_quality_counter += 1
else:
stat_dict[record.id]["retained"] = "+"
else:
too_short_counter += 1
if stat_dict[record.id]["retained"] == "-":
excluded_list.append(record.id)
continue
SeqIO.write(trimmed_record, record_trimmed_fastq, format="fastq")
SeqIO.write(trimmed_record, record_trimmed_fasta, format="fasta")
MatplotlibRoutines.draw_bar_plot(
trimmed_record.letter_annotations["phred_quality"],
record_trimmed_qual_plot_prefix,
extentions=["png"],
xlabel="Position",
ylabel="Phred quality",
title="Per base quality",
min_value=None,
max_value=None,
new_figure=True,
figsize=(3 * (int(len(record) / 100) + 1), 3),
close_figure=True)
trimmed_record_dict[record.id] = trimmed_record
SeqIO.write(self.record_from_dict_generator(record_dict),
merged_raw_fastq,
format="fastq")
SeqIO.write(self.record_from_dict_generator(record_dict),
merged_raw_fasta,
format="fasta")
SeqIO.write(self.record_from_dict_generator(trimmed_record_dict),
merged_trimmed_fastq,
format="fastq")
SeqIO.write(self.record_from_dict_generator(trimmed_record_dict),
merged_trimmed_fasta,
format="fasta")
excluded_list.write("%s.excluded.ids" % output_prefix)
stat_dict.write(out_filename="%s.stats" % output_prefix)
print("Excluded: %i" % excluded_counter)
print("\tToo short( < %i ): %i" % (min_len, too_short_counter))
print("\tLow quality( median < %i or mean < %i ): %i" %
(min_median_qual, min_mean_qual, low_quality_counter))
for entry in complicated_families_dict.all_values():
tmp = entry.split(";")
for i in range(0, len(tmp)):
if "_" in tmp[i]:
tmp[i] = tmp[i][2]
tmp[i] = tmp[i].split(",")
for syn_id in tmp[i]:
complicated_families_syn_ids.add(syn_id)
complicated_families_syn_ids.write("complicated_families_check.ids")
nonassembled.write("splited_to_several_families.t", absent_symbol=".")
assemled_to_different_families = species_syn_dict.filter_by_line(
filter_different_assembly)
species_syn_dict.write("correctly_assembled_families_in_all_species.t",
absent_symbol=".")
assemled_to_different_families.write(
"assembled_to_different_families_in_all_species.t", absent_symbol=".")
correctly_assembled_families_synonym = IdList(
set(species_syn_dict.all_values()))
assemled_to_different_families_synonym = IdList(
set(assemled_to_different_families.all_values()))
correctly_assembled_families_synonym.write(
"correctly_assembled_families_syn_in_all_species.ids")
assemled_to_different_families_synonym.write(
"assembled_to_different_families_syn_in_all_species.ids")
if args.output != "output":
out_fd.close()
count_dict[args.name_b][scaffold][i], ratio))
elif ratio < (1.0 / float(args.minimal_ratio)):
vcf_b_more_variants_file_fd.write(
"%s\t%i\t%i\t%i\t%i\t%i\t%.3f\n" %
(scaffold, start, stop, i,
count_dict[args.name_a][scaffold][i],
count_dict[args.name_b][scaffold][i], ratio))
elif count_dict[args.name_a][scaffold] == 0:
vcf_a_no_variants_file_fd.write("%s\t%i\t%i\t%i" %
(scaffold, start, stop, i))
elif count_dict[args.name_b][scaffold] == 0:
vcf_b_no_variants_file_fd.write("%s\t%i\t%i\t%i" %
(scaffold, start, stop, i))
else:
if scaffold not in count_dict[args.name_a]:
vcf_a_absent_scaffolds_id_list.append(scaffold)
if scaffold not in count_dict[args.name_b]:
vcf_b_absent_scaffolds_id_list.append(scaffold)
vcf_a_more_variants_file_fd.close()
vcf_b_more_variants_file_fd.close()
vcf_a_no_variants_file_fd.close()
vcf_b_no_variants_file_fd.close()
vcf_density_ratio_fd.close()
vcf_a_absent_scaffolds_id_list.write(vcf_a_absent_scaffolds_id_file)
vcf_b_absent_scaffolds_id_list.write(vcf_b_absent_scaffolds_id_file)
out_fd = sys.stdout if args.output == "stdout" else open(args.output, "w")
annotations_dict = SeqIO.to_dict(GFF.parse(open(args.input)))
single_gene_id_list = IdList()
for record in annotations_dict:
for feature in annotations_dict[record].features:
#print feature.id
if feature.type != "gene":
continue
for subfeature in feature.sub_features:
if subfeature.type != "mRNA":
continue
exon_number = 0
for mRNA_subfeature in subfeature.sub_features:
if mRNA_subfeature.type == "exon":
exon_number += 1
if exon_number == 1:
single_gene_id_list.append(feature.id)
single_gene_id_list.write(out_fd, close_after_if_file_object=True)
"""
sequence_groups_id = SynDict()
sequence_groups_id.read(args.id_file, split_values=True)
#print("Parsing %s..." % args.input_file)
sequence_dict = SeqIO.index_db(tmp_index_file, args.input, format=args.format)
for group in sequence_groups_id:
SeqIO.write(record_by_id_generator(sequence_dict, sequence_groups_id[group]),
"%s%s.%s" % (args.output, group, args.extension), format=args.format)
"""
def split_hmm(self,
hmmfile,
output_dir,
num_of_recs_per_file,
num_of_files=None,
output_prefix=None,
threads=4):
try:
os.mkdir(output_dir)
except OSError:
pass
id_fd = CGAS.cgas(hmmfile,
grep_pattern="NAME",
whole_word_match=True,
awk_code="{print $2}",
capture_output=True)
split_index = 1
ids_written = 0
ids_list = IdList()
#ids_list = read_ids(id_fd, close_after_if_file_object=False)
ids_list.read(id_fd, close_after_if_file_object=True)
number_of_ids = len(ids_list)
out_prefix = self.split_filename(
hmmfile)[1] if output_prefix is None else output_prefix
num_of_ids = int(
number_of_ids /
num_of_files) + 1 if num_of_files else num_of_recs_per_file
common_options = " -f"
common_options += " %s" % hmmfile
options_list = []
while (ids_written + num_of_ids) <= number_of_ids:
tmp_id_list = IdList(ids_list[ids_written:ids_written +
num_of_ids])
tmp_id_list.write("%s/%s_%i.ids" %
(output_dir, out_prefix, split_index))
options = common_options
options += " %s/%s_%i.ids" % (output_dir, out_prefix, split_index)
options += " > %s" % ("%s/%s_%i.hmm" %
(output_dir, out_prefix, split_index))
options_list.append(options)
split_index += 1
ids_written += num_of_ids
if ids_written != number_of_ids:
tmp_id_list = IdList(ids_list[ids_written:])
tmp_id_list.write("%s/%s_%i.ids" %
(output_dir, out_prefix, split_index))
options = common_options
options += " %s/%s_%i.ids" % (output_dir, out_prefix, split_index)
options += " > %s" % ("%s/%s_%i.hmm" %
(output_dir, out_prefix, split_index))
options_list.append(options)
split_index += 1
#print options_list
self.parallel_execute(options_list, cmd="hmmfetch", threads=threads)
def parallel_run(
self,
input_dir,
output_dir,
output_prefix,
input_type="codon",
min_seq_number_for_conserved_position=None,
min_seq_number_for_flank_position=None,
max_pos_number_for_noncons_contig_pos=None,
min_block_len=None,
allow_gaps="half",
save_postscript=True,
output_type="htm",
threads=None,
):
if threads:
self.threads = threads
data_dir = "%s/data/" % output_dir
postscript_dir = "%s/ps/" % output_dir
results_dir = "%s/results/" % output_dir
htm_dir = "%s/htm/" % output_dir
for directory in output_dir, data_dir, postscript_dir, results_dir, htm_dir:
self.safe_mkdir(directory)
#input_files_list = map(os.path.abspath, self.make_list_of_path_to_files(input_directory))
input_files_list = self.make_list_of_path_to_files(
input_dir, return_absolute_paths=True)
for entry in input_files_list:
directory, prefix, extension = self.split_filename(entry)
os.system("ln -s %s %s/%s%s" %
(entry, data_dir, prefix, extension))
data_files_list = self.make_list_of_path_to_files(
data_dir, return_absolute_paths=True)
common_options = self.parse_options(
input_type=input_type,
min_seq_number_for_conserved_position=
min_seq_number_for_conserved_position,
min_seq_number_for_flank_position=min_seq_number_for_flank_position,
max_pos_number_for_noncons_contig_pos=
max_pos_number_for_noncons_contig_pos,
min_block_len=min_block_len,
allow_gaps=allow_gaps,
save_postscript=save_postscript,
output_type=output_type,
concatenate_blocks_from_aignments=None)
options_list = []
for data_file in data_files_list:
options = " %s" % data_file
options += " %s" % common_options
options_list.append(options)
self.parallel_execute(options_list=options_list)
block_coordinates = OrderedDict()
skipped_ids_file = "%s/%s.skipped.ids" % (output_dir, output_prefix)
skipped_ids = IdList()
for filename in data_files_list:
data_dir, prefix, extension = self.split_filename(filename)
blocks_file = "%s-gb" % filename
htm_file = "%s-gb.htm" % filename
postscript_file = "%s-gbPS" % filename
if (not os.path.exists(blocks_file)) or (
not os.path.exists(htm_file)):
skipped_ids.append(prefix)
print("Warning!!! %s skipped..." % prefix)
continue
block_coordinates[prefix] = self.extract_block_coordinates(
htm_file)
os.system("mv %s %s/%s.ps" %
(postscript_file, postscript_dir, prefix))
os.system("mv %s %s/%s.htm" % (htm_file, htm_dir, prefix))
self.convert_output_to_fasta(
blocks_file, "%s/%s%s" % (results_dir, prefix, extension))
os.remove(blocks_file)
block_coordinates_file = "%s/%s.block.coordinates" % (output_dir,
output_prefix)
skipped_ids.write(skipped_ids_file)
with open(block_coordinates_file, "w") as block_fd:
for entry in block_coordinates:
coordinates_string = ";".join(
map(lambda s: "%i,%i" % (s[0], s[1]),
block_coordinates[entry]))
block_fd.write("%s\t%s\n" % (entry, coordinates_string))
if args.all or args.tree:
os.system("wget %s" % tree_options)
if args.all or args.hmm:
os.system("wget %s" % hmm_options)
pool = Pool(args.threads)
pool.map(download_data, family_ids)
pool.close()
for fam_id in family_ids:
if args.all or args.alignment:
if os.path.getsize("%s%s.fasta" % (args.output_dir, fam_id)) == 0:
absent_alignment_list.append(fam_id)
if args.all or args.tree:
if os.path.getsize("%s%s.nwk" % (args.output_dir, fam_id)) == 0:
absent_tree_list.append(fam_id)
if args.all or args.hmm:
if os.path.getsize("%s%s.hmm" % (args.output_dir, fam_id)) == 0:
absent_hmm_list.append(fam_id)
if absent_alignment_list:
absent_alignment_list.write("absent_alignments.ids")
print("%i alignments were not downloaded" % len(absent_alignment_list))
if absent_tree_list:
absent_tree_list.write("absent_trees.ids")
print("%i trees were not downloaded" % len(absent_tree_list))
if absent_hmm_list:
absent_hmm_list.write("absent_hmms.ids")
print("%i hmms were not downloaded" % len(absent_hmm_list))