def sampling_by_sample( input_biom, output_biom, nb_sampled=None, sampled_ratio=None ): """ @summary: Writes a BIOM after a random sampling in each sample. @param input_biom: [str] Path to the processed BIOM. @param output_biom: [str] Path to outputed BIOM. @param nb_sampled: [int] Number of sampled sequences by sample. @param sampled_ratio: [float] Ratio of sampled sequences by sample. @note: nb_sampled and sampled_ratio are mutually exclusive. """ initial_biom = BiomIO.from_json( input_biom ) new_biom = Biom( matrix_type="sparse", generated_by="Sampling " + (str(nb_sampled) if nb_sampled is not None else str(sampled_ratio) + "%" ) + " elements by sample from " + input_biom ) observations_already_added = dict() for sample_name in initial_biom.get_samples_names(): new_biom.add_sample( sample_name, initial_biom.get_sample_metadata(sample_name) ) sample_seq = initial_biom.get_sample_count(sample_name) sample_nb_sampled = nb_sampled if nb_sampled is None: sample_nb_sampled = int(sample_seq * sampled_ratio) if sample_seq < nb_sampled: raise_exception( Exception( "\n\n#ERROR : " + str(sample_nb_sampled) + " sequences cannot be sampled in sample '" + str(sample_name) + "'. It only contains " + str(sample_seq) + " sequences.\n\n" )) else: for current_nb_iter in range(sample_nb_sampled): # Take an observation in initial BIOM selected_observation = initial_biom.random_obs_by_sample(sample_name) selected_observation_id = selected_observation['id'] initial_biom.subtract_count( selected_observation_id, sample_name, 1 ) # Put in new BIOM if selected_observation_id not in observations_already_added: new_biom.add_observation( selected_observation_id, initial_biom.get_observation_metadata(selected_observation_id) ) observations_already_added[selected_observation_id] = True new_biom.add_count( selected_observation_id, sample_name, 1 ) BiomIO.write( output_biom, new_biom )
def mask_observation(rdp_clusters_discards, blast_clusters_discards, input_biom, output_biom): """ @summary : mask either rdp affiliations and/or blast affiliations @param rdp_clusters_discards : [list] of clusters whith rdp affiliations to mask @param blast_clusters_discards : [list] of clusters whith blast consensus affiliations to mask @param input_biom : [str] Path to input biom file @param input_biom : [str] Path to output biom file with affiliations masked """ biom = BiomIO.from_json(input_biom) for observation in biom.get_observations(): # remove rdp taxonomic metadata if rdp_clusters_discards is not None and observation[ 'id'] in rdp_clusters_discards: if issubclass(observation['metadata']["rdp_taxonomy"].__class__, str): observation['metadata']["rdp_taxonomy"] = "" observation['metadata']["rdp_bootstrap"] = "" elif issubclass(observation['metadata']["rdp_taxonomy"].__class__, str): observation['metadata']["rdp_taxonomy"] = list() observation['metadata']["rdp_bootstrap"] = list() # remove blast metadata if observation['id'] in blast_clusters_discards: observation['metadata']["blast_affiliations"] = list() observation['metadata']["blast_taxonomy"] = list() BiomIO.write(output_biom, biom)
def aff_to_metadata(reference_file, biom_in, biom_out, blast_files=None, rdp_files=None): """ @summary: Add taxonomy metadata on biom file from a blast result. @param reference_file: [str] The path to the reference file. @param biom_in: [str] The path to the Biom file to process. @param biom_out: [str] The path to the biom output file. @param blast_files: [list] the list of the path to the blast results in tabular format (outfmt 6 with NCBI Blast+). @param rdp_files: [list] the list of path to the RDPClassifier results. """ # Build an hash with the taxonomy for each gene (key=gene_id ; value=gene_taxonomy) taxonomy_by_reference = get_tax_from_fasta(reference_file) # Retrieve blast clusters annotations cluster_blast_annot = dict() if blast_files is not None: cluster_blast_annot = get_bests_blast_affi(blast_files, taxonomy_by_reference) del taxonomy_by_reference # Retrieve rdp clusters annotations cluster_rdp_annot = dict() if rdp_files is not None: cluster_rdp_annot = get_rdp_affi(rdp_files) # Add metadata to biom biom = BiomIO.from_json(biom_in) for cluster in biom.get_observations(): cluster_id = cluster["id"] # Blast if blast_files is not None: blast_taxonomy = list() blast_affiliations = list() if cluster_id in cluster_blast_annot: # Current observation has a match blast_taxonomy = get_tax_consensus([ taxonomy.split(';') for taxonomy in cluster_blast_annot[cluster_id]['alignments'] ]) for taxonomy in cluster_blast_annot[cluster_id]['alignments']: blast_affiliations.extend(cluster_blast_annot[cluster_id] ['alignments'][taxonomy]) biom.add_metadata(cluster_id, "blast_affiliations", blast_affiliations, "observation") biom.add_metadata(cluster_id, "blast_taxonomy", blast_taxonomy, "observation") # RDP if rdp_files is not None: rdp_taxonomy = list() rdp_bootstrap = list() if cluster_id in cluster_rdp_annot: rdp_taxonomy = cluster_rdp_annot[cluster_id]['taxonomy'] rdp_bootstrap = cluster_rdp_annot[cluster_id]['bootstrap'] biom.add_metadata(cluster_id, "rdp_taxonomy", rdp_taxonomy, "observation") biom.add_metadata(cluster_id, "rdp_bootstrap", rdp_bootstrap, "observation") BiomIO.write(biom_out, biom)
def sampling_by_sample( input_biom, output_biom, nb_sampled=None, sampled_ratio=None ): """ @summary: Writes a BIOM after a random sampling in each sample. @param input_biom: [str] Path to the processed BIOM. @param output_biom: [str] Path to outputed BIOM. @param nb_sampled: [int] Number of sampled sequences by sample. @param sampled_ratio: [float] Ratio of sampled sequences by sample. @note: nb_sampled and sampled_ratio are mutually exclusive. """ initial_biom = BiomIO.from_json( input_biom ) new_biom = Biom( matrix_type="sparse", generated_by="Sampling " + (str(nb_sampled) if nb_sampled is not None else str(sampled_ratio) + "%" ) + " elements by sample from " + input_biom ) observations_already_added = dict() for sample_name in initial_biom.get_samples_names(): new_biom.add_sample( sample_name, initial_biom.get_sample_metadata(sample_name) ) sample_seq = initial_biom.get_sample_count(sample_name) sample_nb_sampled = nb_sampled if nb_sampled is None: sample_nb_sampled = int(sample_seq * sampled_ratio) if sample_seq < nb_sampled: raise Exception( str(sample_nb_sampled) + " sequences cannot be sampled in sample '" + str(sample_name) + "'. It only contains " + str(sample_seq) + " sequences." ) else: for current_nb_iter in range(sample_nb_sampled): # Take an observation in initial BIOM selected_observation = initial_biom.random_obs_by_sample(sample_name) selected_observation_id = selected_observation['id'] initial_biom.subtract_count( selected_observation_id, sample_name, 1 ) # Put in new BIOM if not observations_already_added.has_key(selected_observation_id): new_biom.add_observation( selected_observation_id, initial_biom.get_observation_metadata(selected_observation_id) ) observations_already_added[selected_observation_id] = True new_biom.add_count( selected_observation_id, sample_name, 1 ) BiomIO.write( output_biom, new_biom )
def process( in_biom, out_biom, out_metadata ): ordered_blast_keys = ["taxonomy", "subject", "evalue", "perc_identity", "perc_query_coverage", "aln_length"] # Keys in blast_affiliations metadata taxonomy_depth = 0 unclassified_observations = list() FH_metadata = open( out_metadata, "w" ) FH_metadata.write( "#OTUID\t" + "\t".join([item for item in ordered_blast_keys]) + "\n" ) biom = BiomIO.from_json( in_biom ) for observation in biom.get_observations(): for metadata_key in observation["metadata"].keys(): if metadata_key == "blast_affiliations": # Extract blast_affiliations metadata in metadata_file if observation["metadata"][metadata_key] is not None: for current_affi in observation["metadata"][metadata_key]: if isinstance(current_affi["taxonomy"], list) or isinstance(current_affi["taxonomy"], tuple): current_affi["taxonomy"] = ";".join( current_affi["taxonomy"] ) FH_metadata.write( observation["id"] + "\t" + "\t".join([str(current_affi[item]) for item in ordered_blast_keys]) + "\n" ) del observation["metadata"][metadata_key] elif observation["metadata"][metadata_key] is not None: # All list are transformed in string if isinstance(observation["metadata"][metadata_key], list) or isinstance(observation["metadata"][metadata_key], tuple): observation["metadata"][metadata_key] = ";".join( map(str, observation["metadata"][metadata_key]) ) if observation["metadata"].has_key( "blast_taxonomy" ): if observation["metadata"]["blast_taxonomy"] is None: unclassified_observations.append( observation["id"] ) observation["metadata"]["taxonomy"] = list() else: taxonomy_depth = len(observation["metadata"]["blast_taxonomy"].split(";")) observation["metadata"]["taxonomy"] = observation["metadata"]["blast_taxonomy"].split(";") # Add "Unclassified" ranks in unclassified observations if taxonomy_depth > 0: for observation_id in unclassified_observations: observation_metadata = biom.get_observation_metadata(observation_id) observation_metadata["taxonomy"] = ["Unclassified"] * taxonomy_depth BiomIO.write( out_biom, biom )
def process( in_biom, out_biom, out_metadata ): ordered_blast_keys = ["taxonomy", "subject", "evalue", "perc_identity", "perc_query_coverage", "aln_length"] # Keys in blast_affiliations metadata taxonomy_depth = 0 unclassified_observations = list() FH_metadata = open( out_metadata, "w" ) FH_metadata.write( "#OTUID\t" + "\t".join([item for item in ordered_blast_keys]) + "\n" ) biom = BiomIO.from_json( in_biom ) for observation in biom.get_observations(): for metadata_key in observation["metadata"].keys(): if metadata_key == "blast_affiliations": # Extract blast_affiliations metadata in metadata_file if observation["metadata"][metadata_key] is not None: for current_affi in observation["metadata"][metadata_key]: if isinstance(current_affi["taxonomy"], list) or isinstance(current_affi["taxonomy"], tuple): current_affi["taxonomy"] = ";".join( current_affi["taxonomy"] ) FH_metadata.write( observation["id"] + "\t" + "\t".join([str(current_affi[item]) for item in ordered_blast_keys]) + "\n" ) del observation["metadata"][metadata_key] elif observation["metadata"][metadata_key] is not None: # All list are transformed in string if isinstance(observation["metadata"][metadata_key], list) or isinstance(observation["metadata"][metadata_key], tuple): observation["metadata"][metadata_key] = ";".join( map(str, observation["metadata"][metadata_key]) ) if observation["metadata"].has_key( "blast_taxonomy" ): if observation["metadata"]["blast_taxonomy"] is None: unclassified_observations.append( observation["id"] ) observation["metadata"]["taxonomy"] = list() else: taxonomy_depth = len(observation["metadata"]["blast_taxonomy"].split(";")) observation["metadata"]["taxonomy"] = observation["metadata"]["blast_taxonomy"].split(";") # Add "Unclassified" ranks in unclassified observations if taxonomy_depth > 0: for observation_id in unclassified_observations: observation_metadata = biom.get_observation_metadata(observation_id) observation_metadata["taxonomy"] = ["Unclassified"] * taxonomy_depth BiomIO.write( out_biom, biom )
def to_biom( clusters_file, count_file, output_biom, size_separator ): """ @summary : Write a biom file from swarm results. @param clusters_file : [str] path to the '.clstr' file. @param count_file : [str] path to the count file. It contains the count of sequences by sample of each preclusters. Line format : "Precluster_id nb_in_sampleA nb_in_sampleB" @param output_biom : [str] path to the output file. @param size_separator : [str] the pre-cluster abundance separator. """ biom = Biom( generated_by='swarm', matrix_type="sparse" ) # Preclusters count by sample preclusters_count = dict() count_fh = open( count_file ) samples = count_fh.readline().strip().split()[1:] for line in count_fh: precluster_id, count_str = line.strip().split(None, 1) preclusters_count[precluster_id] = count_str # For large dataset store count into a string consumes minus RAM than a sparse count count_fh.close() # Add samples for sample_name in samples: biom.add_sample( sample_name ) # Process count cluster_idx = 1 clusters_fh = open( clusters_file ) for line in clusters_fh: seed_id = line.strip().split()[0] if "FROGS_combined" in seed_id: cluster_name = "Cluster_" + str(cluster_idx) + "_FROGS_combined" comment = "WARNING" else: cluster_name = "Cluster_" + str(cluster_idx) comment = "na" cluster_count = {key:0 for key in samples} line_fields = line.strip().split() # Retrieve count by sample for seq_id in line_fields: real_seq_id = seq_id.rsplit(size_separator, 1)[0] sample_counts = preclusters_count[real_seq_id].split() for sample_idx, sample_name in enumerate(samples): cluster_count[sample_name] += int(sample_counts[sample_idx]) preclusters_count[real_seq_id] = None # Add cluster on biom biom.add_observation( cluster_name, {'comment': comment, 'seed_id':line_fields[0].rsplit(size_separator, 1)[0]} ) observation_idx = biom.find_idx("observation", cluster_name) for sample_idx, sample_name in enumerate(samples): if cluster_count[sample_name] > 0: biom.data.change( observation_idx, sample_idx, cluster_count[sample_name] ) # Next cluster cluster_idx += 1 # Write BiomIO.write( output_biom, biom )
def to_biom(clusters_file, count_file, output_biom, size_separator): """ @summary : Write a biom file from swarm results. @param clusters_file : [str] path to the '.clstr' file. @param count_file : [str] path to the count file. It contains the count of sequences by sample of each preclusters. Line format : "Precluster_id nb_in_sampleA nb_in_sampleB" @param output_biom : [str] path to the output file. @param size_separator : [str] the pre-cluster abundance separator. """ biom = Biom(generated_by='swarm', matrix_type="sparse") # Preclusters count by sample preclusters_count = dict() count_fh = open(count_file) samples = count_fh.readline().strip().split()[1:] for line in count_fh: line_fields = line.strip().split() count_by_sample = {} for idx, val in enumerate(line_fields[1:]): if val > 0: count_by_sample[samples[idx]] = int(val) preclusters_count[line_fields[0]] = count_by_sample count_fh.close() # Add samples for sample_name in samples: biom.add_sample(sample_name) # Process count cluster_idx = 1 clusters_fh = open(clusters_file) for line in clusters_fh: cluster_name = "Cluster_" + str(cluster_idx) cluster_count = {key: 0 for key in samples} line_fields = line.strip().split() # Retrieve count by sample for seq_id in line_fields: real_seq_id = seq_id.rsplit(size_separator, 1)[0] for preclust_sample in preclusters_count[real_seq_id]: cluster_count[preclust_sample] += preclusters_count[ real_seq_id][preclust_sample] preclusters_count[real_seq_id] = None # Add cluster on biom biom.add_observation( cluster_name, {'seed_id': line_fields[0].rsplit(size_separator, 1)[0]}) for sample_name in samples: if cluster_count[sample_name] > 0: biom.add_count(cluster_name, sample_name, cluster_count[sample_name]) # Next cluster cluster_idx += 1 # Write BiomIO.write(output_biom, biom)
def filter_biom( removed_observations, in_biom, out_biom ): """ @summary: Removed the specified observations from BIOM. @param removed_observations: [dict] Each key is an observation name. @param in_biom: [str]: Path to the processed BIOM file. @param out_biom: [str]: Path to the cleaned BIOM file. """ biom = BiomIO.from_json(in_biom) biom.remove_observations(removed_observations) BiomIO.write(out_biom, biom)
def filter_biom( removed_observations, in_biom, out_biom ): """ @summary: Removed the specified observations from BIOM. @param removed_observations: [dict] Each key is an observation name. @param in_biom: [str]: Path to the processed BIOM file. @param out_biom: [str]: Path to the cleaned BIOM file. """ biom = BiomIO.from_json(in_biom) biom.remove_observations(removed_observations) BiomIO.write(out_biom, biom)
def remove_observations( removed_observations, input_biom, output_biom ): """ @summary: Removes the specified list of observations. @param removed_observations: [list] The names of the observations to remove. @param input_biom: [str] The path to the input BIOM. @param output_biom: [str] The path to the output BIOM. """ biom = BiomIO.from_json( input_biom ) biom.remove_observations( removed_observations ) BiomIO.write( output_biom, biom )
def remove_observations(removed_observations, input_biom, output_biom): """ @summary: Removes the specified list of observations. @param removed_observations: [list] The names of the observations to remove. @param input_biom: [str] The path to the input BIOM. @param output_biom: [str] The path to the output BIOM. """ biom = BiomIO.from_json(input_biom) biom.remove_observations(removed_observations) BiomIO.write(output_biom, biom)
def to_biom( clusters_file, count_file, output_biom, size_separator ): """ @summary : Write a biom file from swarm results. @param clusters_file : [str] path to the '.clstr' file. @param count_file : [str] path to the count file. It contains the count of sequences by sample of each preclusters. Line format : "Precluster_id nb_in_sampleA nb_in_sampleB" @param output_biom : [str] path to the output file. @param size_separator : [str] the pre-cluster abundance separator. """ biom = Biom( generated_by='swarm', matrix_type="sparse" ) # Preclusters count by sample preclusters_count = dict() count_fh = open( count_file ) samples = count_fh.readline().strip().split()[1:] for line in count_fh: precluster_id, count_str = line.strip().split(None, 1) preclusters_count[precluster_id] = count_str # For large dataset store count into a string consumes minus RAM than a sparse count count_fh.close() # Add samples for sample_name in samples: biom.add_sample( sample_name ) # Process count cluster_idx = 1 clusters_fh = open( clusters_file ) for line in clusters_fh: cluster_name = "Cluster_" + str(cluster_idx) cluster_count = {key:0 for key in samples} line_fields = line.strip().split() # Retrieve count by sample for seq_id in line_fields: real_seq_id = seq_id.rsplit(size_separator, 1)[0] sample_counts = preclusters_count[real_seq_id].split() for sample_idx, sample_name in enumerate(samples): cluster_count[sample_name] += int(sample_counts[sample_idx]) preclusters_count[real_seq_id] = None # Add cluster on biom biom.add_observation( cluster_name, {'seed_id':line_fields[0].rsplit(size_separator, 1)[0]} ) observation_idx = biom.find_idx("observation", cluster_name) for sample_idx, sample_name in enumerate(samples): if cluster_count[sample_name] > 0: biom.data.change( observation_idx, sample_idx, cluster_count[sample_name] ) # Next cluster cluster_idx += 1 # Write BiomIO.write( output_biom, biom )
def aff_to_metadata(reference_file, biom_in, biom_out, blast_files=None, rdp_files=None): """ @summary: Add taxonomy metadata on biom file from a blast result. @param reference_file: [str] The path to the reference file. @param biom_in: [str] The path to the Biom file to process. @param biom_out: [str] The path to the biom output file. @param blast_files: [list] the list of the path to the blast results in tabular format (outfmt 6 with NCBI Blast+). @param rdp_files: [list] the list of path to the RDPClassifier results. """ # Build an hash with the taxonomy for each gene (key=gene_id ; value=gene_taxonomy) taxonomy_by_reference = get_tax_from_fasta( reference_file ) # Retrieve blast clusters annotations cluster_blast_annot = dict() if blast_files is not None: cluster_blast_annot = get_bests_blast_affi( blast_files, taxonomy_by_reference ) del taxonomy_by_reference # Retrieve rdp clusters annotations cluster_rdp_annot = dict() if rdp_files is not None: cluster_rdp_annot = get_rdp_affi( rdp_files ) # Add metadata to biom biom = BiomIO.from_json(biom_in) for cluster in biom.get_observations(): cluster_id = cluster["id"] # Blast if blast_files is not None: blast_taxonomy = None blast_affiliations = list() if cluster_blast_annot.has_key(cluster_id): # Current observation has a match blast_taxonomy = get_tax_consensus( [alignment['taxonomy'] for alignment in cluster_blast_annot[cluster_id]['alignments']] ) blast_affiliations = cluster_blast_annot[cluster_id]['alignments'] biom.add_metadata( cluster_id, "blast_affiliations", blast_affiliations, "observation" ) biom.add_metadata( cluster_id, "blast_taxonomy", blast_taxonomy, "observation" ) # RDP if rdp_files is not None: rdp_taxonomy = None rdp_bootstrap = None if cluster_rdp_annot.has_key(cluster_id): rdp_taxonomy = cluster_rdp_annot[cluster_id]['taxonomy'] rdp_bootstrap = cluster_rdp_annot[cluster_id]['bootstrap'] biom.add_metadata(cluster_id, "rdp_taxonomy", rdp_taxonomy, "observation") biom.add_metadata(cluster_id, "rdp_bootstrap", rdp_bootstrap, "observation") BiomIO.write(biom_out, biom)
def process(args): tmp_files = TmpFiles(os.path.split(args.output_file)[0]) try: # Add temp taxonomy if multiple and without consensus tmp_biom = args.input_biom used_taxonomy_tag = args.taxonomy_tag if args.multiple_tag is not None: used_taxonomy_tag = args.tax_consensus_tag if args.tax_consensus_tag is None: used_taxonomy_tag = "Used_taxonomy_FROGS-affi" tmp_biom = tmp_files.add("tax.biom") biom = BiomIO.from_json(args.input_biom) for observation in biom.get_observations(): metadata = observation["metadata"] if metadata[args.multiple_tag] is not None and len( metadata[args.multiple_tag]) > 0: metadata[used_taxonomy_tag] = metadata[ args.multiple_tag][0][args.taxonomy_tag] BiomIO.write(tmp_biom, biom) del biom # Rarefaction tax_depth = [ args.taxonomic_ranks.index(rank) for rank in args.rarefaction_ranks ] rarefaction_cmd = Rarefaction(tmp_biom, tmp_files, used_taxonomy_tag, tax_depth) rarefaction_cmd.submit(args.log_file) rarefaction_files = rarefaction_cmd.output_files # Taxonomy tree tree_count_file = tmp_files.add("taxCount.enewick") tree_ids_file = tmp_files.add("taxCount_ids.tsv") TaxonomyTree(tmp_biom, used_taxonomy_tag, tree_count_file, tree_ids_file).submit(args.log_file) # Writes summary write_summary(args.output_file, args.input_biom, tree_count_file, tree_ids_file, rarefaction_files, args) finally: if not args.debug: tmp_files.deleteAll()
def impacted_obs_by_undesired_taxon(input_biom, undesired_taxon_list, in_all_or_in_consensus, biom_out, impacted_file): """ @summary : write the list of observation with affiliations including undesired taxon. @param input_biom: [str] The path to the BIOM file to check. @param undesired_taxon_list: [list] list of string to look for @param in_all_or_in_consensus: [bool] if True, one taxon_ignored must be in the consensus or all affiliation must one of the taxon ignored @param biom_out: [str] path to biom with removed undesired taxonomy @param impacted_file: [str] The path to the output file. """ biom = BiomIO.from_json(input_biom) FH_impacted_file = open(impacted_file, "w") for observation in biom.get_observations(): # update blast_affiliations without ignored taxon and recompute de blast_taxonomy new_blast_affi = list() for affiliation in observation['metadata']['blast_affiliations']: if not any(t in ";".join(affiliation["taxonomy"]) for t in undesired_taxon_list): new_blast_affi.append(affiliation) # if some affi are masked, update blast_affiliations and blast_taxonomy if len(new_blast_affi) != len( observation['metadata']['blast_affiliations']): observation['metadata']['blast_affiliations'] = new_blast_affi new_consensus = get_tax_consensus( [affi['taxonomy'] for affi in new_blast_affi]) # delete mode if all affiliations belons to one of undesired taxon if in_all_or_in_consensus and len(new_blast_affi) == 0: FH_impacted_file.write(str(observation["id"]) + "\n") # masking mode if the new consensus is changed because of ignoring undesired taxon elif not in_all_or_in_consensus and new_consensus != observation[ 'metadata']['blast_taxonomy']: FH_impacted_file.write(str(observation["id"]) + "\n") observation['metadata']['blast_taxonomy'] = new_consensus BiomIO.write(biom_out, biom)
def process( args ): tmp_files = TmpFiles( os.path.split(args.output_file)[0] ) try: # Add temp taxonomy if multiple and without consensus tmp_biom = args.input_biom used_taxonomy_tag = args.taxonomy_tag if args.multiple_tag is not None: used_taxonomy_tag = args.tax_consensus_tag if args.tax_consensus_tag is None: used_taxonomy_tag = "Used_taxonomy_FROGS-affi" tmp_biom = tmp_files.add( "tax.biom" ) biom = BiomIO.from_json( args.input_biom ) for observation in biom.get_observations(): metadata = observation["metadata"] if len(metadata[args.multiple_tag]) > 0: metadata[used_taxonomy_tag] = metadata[args.multiple_tag][0][args.taxonomy_tag] BiomIO.write( tmp_biom, biom ) del biom # Rarefaction tax_depth = [args.taxonomic_ranks.index(rank) for rank in args.rarefaction_ranks] rarefaction_cmd = Rarefaction(tmp_biom, tmp_files, used_taxonomy_tag, tax_depth) rarefaction_cmd.submit( args.log_file ) rarefaction_files = rarefaction_cmd.output_files # Taxonomy tree tree_count_file = tmp_files.add( "taxCount.enewick" ) tree_ids_file = tmp_files.add( "taxCount_ids.tsv" ) TaxonomyTree(tmp_biom, used_taxonomy_tag, tree_count_file, tree_ids_file).submit( args.log_file ) # Writes summary write_summary( args.output_file, args.input_biom, tree_count_file, tree_ids_file, rarefaction_files, args ) finally: if not args.debug: tmp_files.deleteAll()
cmd_grinder2biom = os.path.join(os.path.dirname(os.path.abspath(__file__)), "grinder2biom.py") + \ " --affiliation " + os.path.abspath(args.databank) + \ " --output " + real_biom + \ " --samples" for current_sample in samples: cmd_grinder2biom += " '" + current_sample['name'] + ":" + current_sample['path'] + "'" subprocess.check_call( cmd_grinder2biom, shell=True ) # Add reference id in checked BIOM biom = BiomIO.from_json( args.checked_biom ) fasta = FastaIO( args.checked_fasta ) for record in fasta: reference = re.search("reference=([^\s]+)", record.description).group(1) biom.add_metadata( record.id, "grinder_source", reference, "observation" ) fasta.close() BiomIO.write( checked_biom, biom ) del(biom) # Compare expected to obtained for current_sample in samples: print current_sample['name'] cmd_compareSample = os.path.join(os.path.dirname(os.path.abspath(__file__)), "biomCmpTax.py") \ + " --real-biom " + os.path.abspath(real_biom) \ + " --real-tax-key 'real_taxonomy'" \ + " --checked-biom " + os.path.abspath(checked_biom) \ + " --checked-tax-key '" + args.taxonomy_key + "'" \ + (" --multi-affiliations" if args.multi_affiliations else "") \ + (" --uniq-groups " + args.uniq_groups if args.uniq_groups is not None else "") \ + " --sample " + current_sample['name'] print subprocess.check_output( cmd_compareSample, shell=True ) print ""
# ################################################################################################################################################## if __name__ == "__main__": # Manage parameters parser = argparse.ArgumentParser(description="Add taxonomy from UTAX result in BIOM file.") parser.add_argument( '-t', '--taxonomy-tag', default="taxonomy", help="The taxonomy tag in BIOM file. [Default: taxonomy]") parser.add_argument( '-v', '--version', action='version', version=__version__) # Inputs group_input = parser.add_argument_group('Inputs') group_input.add_argument('-f', '--input-fasta', required=True, help='Path to the sequence file outputed by UTAX (format: fasta).') group_input.add_argument('-b', '--input-biom', required=True, help='Path to the abundance file (format: BIOM).') # Outputs group_output = parser.add_argument_group('Outputs') group_output.add_argument('-o', '--output-biom', required=True, help='Path to the abundance file with taxonomy (format: BIOM).') args = parser.parse_args() # Process biom = BiomIO.from_json( args.input_biom ) fasta = FastaIO( args.input_fasta ) for record in fasta: # record.id example: Cluster_1;size=19714;tax=d:Bacteria(1.0000),p:"Proteobacteria"(0.9997),c:Alphaproteobacteria(0.9903),o:Rhodospirillales(0.9940),f:Acetobacteraceae(0.9887),g:Humitalea(0.9724); match = re.search("^([^\;]+)\;size\=\d+\;tax=(.+)$", record.id) if match is None: fasta.close() raise Exception("ID and taxonomy cannot be retrieved from '" + record.id + "'") record.id = match.group(1) record.description = match.group(2) biom.add_metadata( record.id, args.taxonomy_tag, record.description, "observation" ) fasta.close() BiomIO.write( args.output_biom, biom )
def remove_chimera_biom(samples, chimera_files, in_biom_file, out_biom_file, lenient_filter, global_report, bySample_report, log_file): """ @summary: Removes the chimera observation from BIOM. @param samples: [list] samples name list @param chimera_files : [list] samples chimera files @param in_biom_file: [str] The path to the BIOM file to filter. @param out_biom_file: [str] The path to the BIOM after filter. @param lenient_filter: [bool] True: removes one sequence in all samples only if it is detected as chimera in all samples where it is present. With False removes one sequence in all samples if it is detected as chimera in at least one sample. @param global_report: [dict] This dictionary is update with the global number of removed observations, the global removed abundance, ... @param bySample_report: [dict] This dictionary is update for add by sample the number of removed observations, the removed abundance, ... @param log_file : [path] Path to general log output file """ FH_log = Logger(log_file) FH_log.write("## Removes the chimera observation from BIOM.\n") nb_sample_by_chimera = dict() # Init bySample_report for sample_name in samples: bySample_report[sample_name] = { 'nb_kept': 0, 'kept_abundance': 0, 'nb_removed': 0, 'removed_abundance': 0, 'removed_max_abundance': 0 } # Retrieve chimera for chimera_file in chimera_files: chimera_fh = open(chimera_file) for line in chimera_fh: observation_name = line.strip() if not nb_sample_by_chimera.has_key(observation_name): nb_sample_by_chimera[observation_name] = 0 nb_sample_by_chimera[observation_name] += 1 chimera_fh.close() # Remove chimera removed_chimera = list() biom = BiomIO.from_json(in_biom_file) for chimera_name in nb_sample_by_chimera.keys(): is_always_chimera = True nb_sample_with_obs = sum( 1 for sample in biom.get_samples_by_observation(chimera_name)) observation_abundance = biom.get_observation_count(chimera_name) if nb_sample_with_obs != nb_sample_by_chimera[chimera_name]: is_always_chimera = False global_report['nb_ambiguous'] += 1 global_report['abundance_ambiguous'] += observation_abundance FH_log.write( "'" + chimera_name + "' is not interpreted as chimera in all samples where it is present.\n" ) if not lenient_filter or is_always_chimera: removed_chimera.append(chimera_name) # Global metrics global_report['nb_removed'] += 1 global_report['abundance_removed'] += observation_abundance # By sample metrics for sample in biom.get_samples_by_observation(chimera_name): sample_count = biom.get_count(chimera_name, sample['id']) bySample_report[sample['id']]['nb_removed'] += 1 bySample_report[ sample['id']]['removed_abundance'] += sample_count bySample_report[sample['id']]['removed_max_abundance'] = max( bySample_report[sample['id']]['removed_max_abundance'], sample_count) biom.remove_observations(removed_chimera) # Nb non-chimera for observation_name in biom.get_observations_names(): global_report['nb_kept'] += 1 global_report['abundance_kept'] += biom.get_observation_count( observation_name) # By sample metrics for sample in biom.get_samples_by_observation(observation_name): sample_count = biom.get_count(observation_name, sample['id']) bySample_report[sample['id']]['nb_kept'] += 1 bySample_report[sample['id']]['kept_abundance'] += sample_count BiomIO.write(out_biom_file, biom) FH_log.close()
help='Path to the sequence file outputed by UTAX (format: fasta).') group_input.add_argument('-b', '--input-biom', required=True, help='Path to the abundance file (format: BIOM).') # Outputs group_output = parser.add_argument_group('Outputs') group_output.add_argument( '-o', '--output-biom', required=True, help='Path to the abundance file with taxonomy (format: BIOM).') args = parser.parse_args() # Process biom = BiomIO.from_json(args.input_biom) fasta = FastaIO(args.input_fasta) for record in fasta: # record.id example: Cluster_1;size=19714;tax=d:Bacteria(1.0000),p:"Proteobacteria"(0.9997),c:Alphaproteobacteria(0.9903),o:Rhodospirillales(0.9940),f:Acetobacteraceae(0.9887),g:Humitalea(0.9724); match = re.search("^([^\;]+)\;size\=\d+\;tax=(.+)$", record.id) if match is None: fasta.close() raise Exception("ID and taxonomy cannot be retrieved from '" + record.id + "'") record.id = match.group(1) record.description = match.group(2) biom.add_metadata(record.id, args.taxonomy_tag, record.description, "observation") fasta.close() BiomIO.write(args.output_biom, biom)
biom.add_sample(sample_name) fh_abund = open(args.samples[sample_name]) for line in fh_abund: # Content format: "# rank<TAB>seq_id<TAB>rel_abund_perc" if not line.startswith('#'): fields = line.strip().split() try: biom.add_observation(fields[1]) except: # already exist pass biom.change_count( fields[1], sample_name, int(float(fields[2]) * 100000000000000 )) ################## depend de la precision grinder fh_abund.close() # Set taxonomy metadata fh_classif = FastaIO(args.affiliation) for record in fh_classif: try: metadata = biom.get_observation_metadata(record.id) if metadata is None or not metadata.has_key(taxonomy_key): taxonomy = getCleanedTaxonomy(record.description) biom.add_metadata(record.id, taxonomy_key, taxonomy, "observation") except ValueError: # is not in BIOM pass fh_classif.close() # Write BIOM BiomIO.write(args.output, biom)
def remove_chimera_biom( samples, in_biom_file, out_biom_file, lenient_filter, global_report, bySample_report ): """ @summary: Removes the chimera observation from BIOM. @param samples: [dict] The chimera observations by sample. Example for sample splA: sample['splA']['chimera_path'] where the value is the path to the file containing the list of the chimera observations names. @param in_biom_file: [str] The path to the BIOM file to filter. @param out_biom_file: [str] The path to the BIOM after filter. @param lenient_filter: [bool] True: removes one sequence in all samples only if it is detected as chimera in all samples where it is present. With False removes one sequence in all samples if it is detected as chimera in at least one sample. @param global_report: [dict] This dictionary is update with the global number of removed observations, the global removed abundance, ... @param bySample_report: [dict] This dictionary is update for add by sample the number of removed observations, the removed abundance, ... """ nb_sample_by_chimera = dict() # Init bySample_report for sample_name in samples.keys(): bySample_report[sample_name] = { 'nb_kept': 0, 'kept_abundance': 0, 'nb_removed': 0, 'removed_abundance': 0, 'removed_max_abundance': 0 } # Retrieve chimera for sample_name in samples.keys(): chimera_fh = open( samples[sample_name]['chimera_path'] ) for line in chimera_fh: observation_name = line.strip() if not nb_sample_by_chimera.has_key(observation_name): nb_sample_by_chimera[observation_name] = 0 nb_sample_by_chimera[observation_name] += 1 chimera_fh.close() # Remove chimera removed_chimera = list() biom = BiomIO.from_json(in_biom_file) for chimera_name in nb_sample_by_chimera.keys(): is_always_chimera = True nb_sample_with_obs = sum( 1 for sample in biom.get_samples_by_observation(chimera_name) ) observation_abundance = biom.get_observation_count(chimera_name) if nb_sample_with_obs != nb_sample_by_chimera[chimera_name]: is_always_chimera = False global_report['nb_ambiguous'] += 1 global_report['abundance_ambiguous'] += observation_abundance print "'" + chimera_name + "' is not interpreted as chimera in all samples where it is present." if not lenient_filter or is_always_chimera: removed_chimera.append(chimera_name) # Global metrics global_report['nb_removed'] += 1 global_report['abundance_removed'] += observation_abundance # By sample metrics for sample in biom.get_samples_by_observation(chimera_name): sample_count = biom.get_count(chimera_name, sample['id']) bySample_report[sample['id']]['nb_removed'] += 1 bySample_report[sample['id']]['removed_abundance'] += sample_count bySample_report[sample['id']]['removed_max_abundance'] = max(bySample_report[sample['id']]['removed_max_abundance'], sample_count) biom.remove_observations(removed_chimera) # Nb non-chimera for observation_name in biom.get_observations_names(): global_report['nb_kept'] += 1 global_report['abundance_kept'] += biom.get_observation_count(observation_name) # By sample metrics for sample in biom.get_samples_by_observation(observation_name): sample_count = biom.get_count(observation_name, sample['id']) bySample_report[sample['id']]['nb_kept'] += 1 bySample_report[sample['id']]['kept_abundance'] += sample_count BiomIO.write(out_biom_file, biom)
biom = Biom( generated_by="grinder", matrix_type="sparse" ) # Set observations count for sample_name in args.samples: biom.add_sample( sample_name ) fh_abund = open( args.samples[sample_name] ) for line in fh_abund: # Content format: "# rank<TAB>seq_id<TAB>rel_abund_perc" if not line.startswith('#'): fields = line.strip().split() try: biom.add_observation( fields[1] ) except: # already exist pass biom.change_count( fields[1], sample_name, int(float(fields[2])*100000000000000) )################## depend de la precision grinder fh_abund.close() # Set taxonomy metadata fh_classif = FastaIO( args.affiliation ) for record in fh_classif: try: metadata = biom.get_observation_metadata( record.id ) if metadata is None or not metadata.has_key( taxonomy_key ): taxonomy = getCleanedTaxonomy(record.description) biom.add_metadata( record.id, taxonomy_key, taxonomy, "observation" ) except ValueError: # is not in BIOM pass fh_classif.close() # Write BIOM BiomIO.write( args.output, biom )
" --samples" for current_sample in samples: cmd_grinder2biom += " '" + current_sample[ 'name'] + ":" + current_sample['path'] + "'" subprocess.check_call(cmd_grinder2biom, shell=True) # Add reference id in checked BIOM biom = BiomIO.from_json(args.checked_biom) fasta = FastaIO(args.checked_fasta) for record in fasta: reference = re.search("reference=([^\s]+)", record.description).group(1) biom.add_metadata(record.id, "grinder_source", reference, "observation") fasta.close() BiomIO.write(checked_biom, biom) del (biom) # Compare expected to obtained for current_sample in samples: print current_sample['name'] cmd_compareSample = os.path.join(os.path.dirname(os.path.abspath(__file__)), "biomCmpTax.py") \ + " --real-biom " + os.path.abspath(real_biom) \ + " --real-tax-key 'real_taxonomy'" \ + " --checked-biom " + os.path.abspath(checked_biom) \ + " --checked-tax-key '" + args.taxonomy_key + "'" \ + (" --multi-affiliations" if args.multi_affiliations else "") \ + (" --uniq-groups " + args.uniq_groups if args.uniq_groups is not None else "") \ + " --sample " + current_sample['name'] print subprocess.check_output(cmd_compareSample, shell=True) print ""
def tsv_to_biom( input_tsv, multi_hit_dict, fields, samples_names, output_biom, output_fasta ): """ @summary: Convert TSV file to Biom file. @param input_tsv: [str] Path to the TSV file. @param multi_hit_dict: [dict] Dictionnary describing equivalent multi blast hit : dict[observation_name]=[ {"blast_taxonomy":taxonomy, "blast_subject":subject, "blast_perc_identity": per_id, "blast_perc_query_coverage":per_cov, "blast_evalue":eval, "blast_aln_length":aln}] @param fields: [list] column name to include as metadata (must at least contain observation_name): observation_sum and seed_sequence will be excluded, rdp_tax_and_bootstrap will be split in two metadata @param samples_names: [list] list of sample names. @param output_biom: [str] Path to the output file (format : BIOM). @param output_fasta: [str] Path to the output file (format : fasta). """ # biom = Biom( generated_by='frogs', matrix_type="sparse" ) biom = Biom( matrix_type="sparse" ) seed_seq_idx = -1 metadata_index = dict() sample_index = dict() clusters_count = dict() clusters_metadata = dict() in_fh = open( input_tsv ) if not output_fasta is None: Fasta_fh=FastaIO(output_fasta , "w" ) # parse header and store column index header=in_fh.readline() if header.startswith("#"): header=header[1:] header = header.strip() seed_seq_idx, metadata_index, sample_index = header_line_dict(fields,header,samples_names) if not output_fasta is None and seed_seq_idx == -1: raise Exception("\nYou want to extract seed fasta sequence but there is no seed_sequence column in your TSV file\n\n") # count by sample, and metadata for line in in_fh: cluster_name="" line_list=line.strip().split("\t") count_by_sample = {} metadata_dict = {} # parse columns for idx,val in enumerate(line_list): # recover metadata if idx in metadata_index: if metadata_index[idx]=="observation_name" : cluster_name = val else: metadata_dict[metadata_index[idx]] = val # recover samples count elif idx in sample_index and val > 0: count_by_sample[sample_index[idx]] = int(val) # recover seed sequence elif idx == seed_seq_idx: seed_seq = val # if fasta output file => store de seed sequence if not output_fasta is None: seq = Sequence( cluster_name, seed_seq) Fasta_fh.write(seq) if "taxonomy" in metadata_dict: metadata_dict["taxonomy"] = metadata_dict["taxonomy"].split(";") # format rdp taxonomy to fit BIOM format if "rdp_tax_and_bootstrap" in metadata_dict: metadata_dict["rdp_taxonomy"]=[] metadata_dict["rdp_bootstrap"]=[] tax = metadata_dict["rdp_tax_and_bootstrap"].rstrip(";").split(";") for i in range(0,len(tax),2): metadata_dict["rdp_taxonomy"].append(tax[i]) metadata_dict["rdp_bootstrap"].append(tax[i+1].replace("(","").replace(")","")) metadata_dict.pop("rdp_tax_and_bootstrap") # format blast taxonomy to fit BIOM format (one consensus blast_taxonomy and possible multiples blast_affiliation detailed if "blast_taxonomy" in metadata_dict: metadata_dict["blast_taxonomy"] = metadata_dict["blast_taxonomy"].split(";") # check multihit blast : filter non consistent taxonomy hit with blast_taxonomy (if TSV modified), and compute consensus tax (if multihit line suppressed) if metadata_dict["blast_subject"] == "multi-subject" and not multi_hit_dict is None: if not cluster_name in multi_hit_dict: raise Exception("\n"+cluster_name+" has multi-subject tag but is not present in your multi-hit TSV file. Please, provide the original multi-hit TSV file.\n\n") else: metadata_dict["blast_taxonomy"], metadata_dict["blast_affiliations"] = observation_blast_parts(metadata_dict, multi_hit_dict[cluster_name]) if metadata_dict["blast_affiliations"] == []: raise Exception("\nyour multihit TSV file is no more consistent with your abundance TSV file for (at least) "+cluster_name+"\n\n") # no multi tag= blast affiliation is equal to blast_taxonomy else: blast_dict={key.replace("blast_",""):metadata_dict[key] for key in metadata_dict if key.startswith("blast")} metadata_dict["blast_affiliations"]=[blast_dict] # filter blast metadata which are moved to blast_affiliations for metadata in metadata_dict["blast_affiliations"][0]: if not metadata == "taxonomy": metadata_dict.pop("blast_"+metadata) # add cluster and count to clusters_count dict clusters_count[cluster_name] = count_by_sample # ok print clusters_count[cluster_name].keys(), "CDT0#LOT05" in clusters_count[cluster_name], "CDT0#LOT02" in clusters_count[cluster_name] # add cluster and metadata to clusters_metadata dict clusters_metadata[cluster_name] = metadata_dict if not output_fasta is None: Fasta_fh.close() in_fh.close() #add samples to biom for sample_name in samples_names: biom.add_sample( sample_name ) # add to cluster to biom for cluster_name in clusters_count: biom.add_observation( cluster_name, clusters_metadata[cluster_name] ) for sample_name in samples_names: if clusters_count[cluster_name][sample_name] > 0: biom.add_count( cluster_name, sample_name, clusters_count[cluster_name][sample_name] ) # Write BiomIO.write( output_biom, biom )