def __init__(self, pattern="**/summary.json", output_filename=None, verbose=True, **kargs): super().__init__() from sequana import logger logger.level = "INFO" if verbose is False: logger.level = "WARNING" logger.info( "Sequana Summary is still a tool in progress and have been " + " tested with the quality_control pipeline only for now.") self.title = "Sequana multiple summary" self.devtools = DevTools() self.filenames = list(glob.iglob(pattern, recursive=True)) self.summaries = [ReadSummary(filename) for filename in self.filenames] self.projects = [ ReadSummary(filename).data['project'] for filename in self.filenames ] self.create_report_content() self.create_html(output_filename)
def switch_header_to_gi(self, acc): """Kraken will only accept the GI from NCBI so we need to convert the ENA accession to GI numbers""" # Accession may have a version .1, .2 hence this try/except first # without the version and then with the version. # Note also that some accession are different from an earlier version. # For instance, AF525933 is in the virus.txt list from ENA but # the new updated accession ois AH012103 showing that the list and DB # must not be fully synchronised. # http://www.ebi.ac.uk/ena/data/search?query=AF525933 # In such case, the results attribute will be missing that accession, # which needs to be searched for specifically. We cannot now its name # before downloading the fasta. if acc in self.results.keys(): res = self.results[acc] else: try: res = self.results[acc.split(".")[0]] except: logger.warning( "\nUnknown accession (%s). May be an updated version. Checking..." % acc) res = self.ena_id_to_gi_number([acc]) self.results.update(res) res = res[acc] logger.info('Found %s using GI number' % acc) return ">" + res['identifier'] + " " + res['comment']
def random_selection(self, output_filename, nreads=None, expected_coverage=None, reference_length=None): """Select random reads :param nreads: number of reads to select randomly. Must be less than number of available reads in the orignal file. :param expected_coverage: :param reference_length: of expected_coverage and reference_length provided, nreads is replaced automatically. """ assert output_filename != self.filename, \ "output filename should be different from the input filename" self.reset() if expected_coverage and reference_length: mu = self.stats['mean'] nreads = int(expected_coverage * reference_length / mu) assert nreads < len( self), "nreads parameter larger than actual Number of reads" selector = random.sample(range(len(self)), nreads) logger.info("Creating a pacbio BAM file with {} reads".format(nreads)) with pysam.AlignmentFile(output_filename, "wb", template=self.data) as fh: for i, read in enumerate(self.data): if i in selector: fh.write(read)
def _get_df(self): if self._df is None: self.reset() N = 0 all_results = [] for read in self.data: res = [] # count reads N += 1 if (N % 10000) == 0: logger.info("Read %d sequences" % N) #res[0] = read length res.append(read.query_length) # res[1] = GC content c = collections.Counter(read.query_sequence) res.append(100 * (c['g'] + c['G'] + c['c'] + c['C']) / float(sum(c.values()))) # aggregate results all_results.append(res) self._df = pd.DataFrame(all_results, columns=['read_length', 'GC_content']) self.reset() return self._df
def get_most_probable_strand_consensus(rnaseq_folder): """From a sequana rna-seq run folder get the most probable strand, based on the frequecies of counts assigned with '0', '1' or '2' type strandness (featureCounts nomenclature) """ rnaseq_folder = Path(rnaseq_folder) sample_folders = list( set([x.parent for x in rnaseq_folder.glob("*/feature_counts_[012]")])) df = pd.concat([ get_most_probable_strand(sample_folder) for sample_folder in sample_folders ]) logger.info("Strandness probability report:") logger.info(df) probable_strands = df.loc[:, "strand"].unique() if len(probable_strands) == 1: return probable_strands[0] else: raise IOError( f"No consensus on most probable strand. Could be: {probable_strands}" )
def _enrichr(self, category, background=None, verbose=True): if background is None: background = self.background if isinstance(category, list): gene_list = category else: assert category in ['up', 'down', 'all'] gene_list = list(self.rnadiff.gene_lists[category]) if self.mapper is not None: logger.info("Input gene list of {} ids".format(len(gene_list))) #gene_list = [x.replace("gene:", "") for x in gene_list] identifiers = self.mapper.loc[gene_list]['name'].drop_duplicates( ).values logger.info("Mapped gene list of {} ids".format(len(identifiers))) gene_list = list(identifiers) enr = gseapy.enrichr(gene_list=gene_list, gene_sets=self.gene_sets, verbose=verbose, background=background, outdir="test", no_plot=True) return enr
def save_significant_pathways(self, mode, cutoff=0.05, nmax=20, background=None): #pragma: no cover """mode should be up, down or all""" if background is None: background = self.background # select the relevant pathways df = self._enrichr(mode, background).results df = self._get_final_df(df, cutoff=cutoff, nmax=nmax) logger.warning("Found {} pathways to save".format(len(df))) if len(df) == nmax: logger.warning("Restricted pathways to {}".format(nmax)) logger.info("saving {} deregulated pathways".format(len(df))) summaries = {} # save them for ID in df['Term']: summary = self.save_pathway(ID, filename="{}_{}.png".format(ID, mode)) summaries[ID] = summary return summaries
def __init__(self, filename, verbose=False): if filename.endswith(".gz"): raise ValueError("Must be decompressed.") self._fasta = FastxFile(filename) self.filename = filename logger.info("Reading input fasta file...please wait") self._N = len([x for x in FastxFile(filename)])
def _get_df(self): if self._df is None: self.reset() N = 0 all_results = [] for read in self.data: res = [] # count reads N += 1 if (N % 10000) == 0: logger.info("Read %d sequences" %N) #res[0] = read length res.append(read.query_length) # res[1] = GC content c = collections.Counter(read.query_sequence) res.append( 100 * (c['g'] + c['G'] + c['c'] + c['C']) / float(sum(c.values())) ) # aggregate results all_results.append(res) self._df = pd.DataFrame(all_results, columns=['read_length','GC_content']) self.reset() return self._df
def _get_data(self): # return list of lists # each list is made of 3 values: mapq, length, concordance from sequana import Cigar data = [] self.reset() count = 0 for align in self.data: mapq = align.mapq length = align.rlen if self.method in ["blasr", "minimap2"]: this = Cigar(align.cigarstring).stats() S, D, I, M = this[4], this[2], this[1], this[0] concordance = 1 - (D + I + S) / (D + I + M + S) else: this = align.get_cigar_stats()[0] error = this[-1] # suppose to be I + D + X total = this[-1] + this[0] if total: concordance = 1 - (error) / (total) else: concordance = 0 data.append([mapq, length, concordance]) if count % 10000 == 0: logger.info("%s" % count) count += 1 return data
def _parse_data(self): taxonomy = {} logger.info("Reading kraken data") columns = ["status", "taxon", "length"] # we select only col 0,2,3 to save memoty, which is required on very # large files try: # each call to concat in the for loop below # will take time and increase with chunk position. # for 15M reads, this has a big cost. So chunksize set to 1M # is better than 1000 and still reasonable in memory reader = pd.read_csv(self.filename, sep="\t", header=None, usecols=[0, 2, 3], chunksize=1000000) except pd.parser.CParserError: raise NotImplementedError # this section is for the case #only_classified_output when there is no found classified read self.unclassified = N # size of the input data set self.classified = 0 self._df = pd.DataFrame([], columns=columns) self._taxons = self._df.taxon return for chunk in reader: try: self._df self._df = pd.concat([self._df, chunk]) except AttributeError: self._df = chunk self._df.columns = columns count = sum(self._df.taxon == 1) if count: logger.warning("Found %s taxons with root ID (1)" % count) # This gives the list of taxons as index and their amount # above, we select only columns 0, 2, 3 the column are still labelled # 0, 2, 3 in the df self._taxons = self._df.groupby("taxon").size() try: self._taxons.drop(0, inplace=True) except: pass # 0 may not be there self._taxons.sort_values(ascending=False, inplace=True) category = self.df.groupby("status").size() if 'C' in category.index: self.classified = category['C'] else: self.classified = 0 if 'U' in category.index: self.unclassified = category['U'] else: self.unclassified = 0
def __init__(self, filename, force=False, **kwargs): """.. rubric:: constructor :param filename: :param force: even though the file format is not recognised, you can force the instanciation. Then, you can use your own filters. """ vcf = VCFBase(filename, verbose=False, **kwargs) if vcf.version == "4.1": logger.info("Reading VCF v 4.1") self.vcf = VCF_mpileup_4dot1(filename, **kwargs) elif vcf.version == "4.2" and vcf.source.startswith("freeBayes"): logger.info("Reading VCF v 4.2 (freebayes)") from sequana.freebayes_vcf_filter import VCF_freebayes self.vcf = VCF_freebayes(filename, **kwargs) else: print(vcf.version) print(vcf.source) msg = """This VCF file is not recognised. So far we handle version v4.1 with mpileup and v4.2 with freebayes. You may use the force option but not all filters will be recognised""" if force is True: print("VCF version %s not tested" % vcf.version) self.vcf = vcf else: raise ValueError(msg)
def _add_db_in_config(self): """ Add new annotation at the end of snpEff.config file. """ logger.info("Updating configuration file") if not self._check_database(self.ref_name): with open("snpEff.config", "a") as fp: print(self.ref_name + ".genome : " + self.ref_name, file=fp)
def splitter_mapped_unmapped(filename, prefix): logger.info("Creating 2 files (mapped and unmapped reads)") data = sniff(filename) count = 0 flags = [] match = 0 unmatch = 0 logger.info("Please wait while creating output files") with open("{}.unmapped.fastq".format(prefix), "w") as fnosirv: with open("{}.mapped.fastq".format(prefix), "w") as fsirv: for a in data: if a.flag & 256: unmatch += 1 elif a.flag & 4: read = "@{}\n{}\n+\n{}\n".format(a.qname, a.query_sequence, a.qual) assert len(a.query_sequence) == len(a.qual) fnosirv.write(read) unmatch += 1 else: read = "@{}\n{}\n+\n{}\n".format(a.qname, a.query_sequence, a.qual) assert len(a.query_sequence) == len(a.qual) fsirv.write(read) match += 1 flags.append(a.flag) return match, unmatch, flags
def df(self): # RG: ID read group ?? # np: number of passes # rq ? # rs: list 6 numbers ? # za: # zm ID of the ZMW # sn: SNR how is this computed ? # zs # - sn: list of ACGT SNRs. A, C, G, T in that order if self._df is not None: return self._df logger.info("Scanning input file. Please wait") self.reset() N = 0 all_results = [] # This takes 60% of the time...could use cython ? for read in self.data: tags = dict(read.tags) #11% of the time res = [] # count reads N += 1 if (N % 10000) == 0: logger.info("Read %d sequences" %N) # res[0] = read length res.append(read.query_length) # also stored in tags["qe"] - tags["qs"] # collections.counter is slow, let us do it ourself res.append( 100. / read.qlen * sum( [read.query_sequence.count(letter) if read.query_sequence else 0 for letter in "CGcgSs"])) # res[1:4] contains SNR stored in tags['sn'] in the order A, C, G, T try: snr = list(tags['sn']) except: snr = [None] * 4 res = res + snr # res[6] = ZMW name, also stored in tags["zm"] res.append(int(tags['zm'])) res.append(tags['np']) # aggregate results all_results.append(res) self._df = pd.DataFrame(all_results, columns=['read_length','GC_content','snr_A','snr_C','snr_G','snr_T','ZMW', "nb_passes"]) self._df.ZMW = self._df.ZMW.astype(int) if len(self._df.ZMW.unique()) != len(self._df): logger.warning("Found non unique ZMW. This may not be a CCS but " "a subread file. Consider using PacbioSubreads class") self.reset() return self._df
def _get_data(self): # return list of lists # each list is made of 3 values: mapq, length, concordance from sequana import Cigar data = [] self.reset() count = 0 for align in self.data: mapq = align.mapq length = align.rlen if self.method in ["blasr", "minimap2"]: this = Cigar(align.cigarstring).stats() S, D, I, M = this[4] , this[2] , this[1], this[0] concordance = 1 - (D+I+S)/(D + I + M + S) else: this = align.get_cigar_stats()[0] error = this[-1] # suppose to be I + D + X total = this[-1] + this[0] if total:concordance = 1- (error)/(total) else:concordance = 0 data.append([mapq, length, concordance]) if count % 10000 == 0: logger.info("%s" % count) count+=1 return data
def test_analysis(krakendb): file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz") file2 = sequana_data("Hm2_GTGAAA_L005_R2_001.fastq.gz") # Test that database must be provided try: df = taxonomy.main([prog, '--file1', file1]) assert False except: assert True from tempfile import TemporaryDirectory directory = TemporaryDirectory() # If on travis and we could not load the database, use the local one # that must have been downloaded try: df = taxonomy.main([ prog, '--file1', file1, "--database", "toydb", "--file2", file2, "--level", "INFO", "--output-directory", directory.name, "--thread", "1" ]) except: # For travis test HOME = os.getenv('HOME') database = os.sep.join([HOME, '.config', 'sequana', 'kraken_toydb']) df = taxonomy.main([ prog, '--file1', file1, "--database", database, "--file2", file2, "--output-directory", directory.name, "--thread", "1" ]) from sequana import logger logger.info(directory.name)
def download_accession_from_ncbi(self, accession): # a list of accessions in a file # can be a list, a unique string, a filename with 1-column wit accession # to retrieve if isinstance(accession, list): pass elif isinstance(accession, str): if os.path.exists(accession): with open(accession, "r") as fin: accessions = fin.read().split() else: accessions = [accession] from easydev import Progress N = len(accessions) pb = Progress(N) logger.info("Fetching {} accession fasta files from NCBI".format(N)) for i, accession in enumerate(accessions): data = self.eutils.EFetch("nucleotide", rettype="fasta", id=accession, retmode="text") if isinstance(data, int): logger.info( "Could not fetch this accession: {}. continue".format( accession)) print("Could not fetch this accession: {}. continue".format( accession)) else: outname = "{}/library/{}.fa".format(self.dbname, accession) with open(outname, "wb") as fout: fout.write(data) pb.animate(i + 1)
def create_taxonomy_file(self, filename="taxonomy.dat"): logger.info("Please wait while creating the output file. " "This may take a few minutes") from easydev import Progress pb = Progress(len(self.df_nodes)) count = 0 df_names = self.df_names.query("key == 'scientific name'").copy() with open(filename, "w") as fout: for taxid in self.df_nodes.index: row = self.df_nodes.loc[taxid] fout.write("ID : {}\n".format(taxid)) fout.write("PARENT ID : {}\n".format( row.parent)) fout.write("RANK : {}\n".format( row['rank'])) #names = df_names.loc[taxid] #print( fout.write("{:26s}: {}\n".format("SCIENTIFIC NAME", df_names.loc[taxid, "name"])) """ len(names) for k,v in zip(names['key'], names['name']): if k.upper() in ['SCIENTIFIC NAME', 'SYNONYM']: fout.write("{:26s}: {}\n".format(k.upper(), v)) except: k, v = names['key'], names['name'] fout.write("{:26s}: {}\n".format(k.upper(), v)) """ fout.write("//\n") count += 1 pb.animate(count)
def main(args=None): if args is None: args = sys.argv[:] print("Welcome to sequana_vcf_filter") user_options = Options(prog="sequana_vcf_filter") if "--version" in args: import sequana print(sequana.version) sys.exit() elif len(args) == 1 or "--help" in args: user_options.parse_args(["prog", "--help"]) elif len(args) == 2: class SimpleOpt(): pass options = SimpleOpt() options.input_filename = args[1] else: options = user_options.parse_args(args[1:]) # set the level logger.level = options.level vcf = VCF(options.input_filename) vcf.vcf.filter_dict['QUAL'] = options.quality vcf.vcf.apply_indel_filter = options.apply_indel_filter vcf.vcf.apply_dp4_filter = options.apply_dp4_filter vcf.vcf.apply_af1_filter = options.apply_af1_filter vcf.vcf.dp4_minimum_depth = options.minimum_depth vcf.vcf.dp4_minimum_depth_strand = options.minimum_depth_strand vcf.vcf.dp4_minimum_ratio = options.minimum_ratio vcf.vcf.minimum_af1 = options.minimum_af1 vcf.vcf.filter_dict['INFO'] = {} vcf.vcf.filter_dict['QUAL'] = options.quality for this in options.filter: this = this[0] signs = [">", "<", ">=", "<="] for sign in signs: if sign in this: key, value = this.split(sign, 1) key = key.strip() value = sign.strip() + value.strip() vcf.vcf.filter_dict['INFO'][key] = value break logger.info(vcf.vcf.filter_dict) res = vcf.vcf.filter_vcf(options.output_filename, output_filtered=options.output_filtered_filename) print() #print(res) return res
def get_data(self, ontologies, include_negative_enrichment=True, fdr=0.05): if isinstance(ontologies, str): ontologies = [ontologies] else: assert isinstance(ontologies, list) # First, we select the required ontologies and build a common data set all_data = [] for ontology in ontologies: data = self.enrichment[ontology]['result'] if isinstance(data, dict): # there was only one hit, we expect: data = [data] all_data.extend(data) data = all_data # remove unclassified GO terms unclassified = [ x for x in data if x['term']['label'] == "UNCLASSIFIED" ] logger.info("Found {} unclassified".format(len(unclassified))) data = [x for x in data if x['term']['label'] != "UNCLASSIFIED"] df = pd.DataFrame(data) if len(df) == 0: return df else: logger.info("Found {} GO terms".format(len(df))) df = df.query("number_in_list!=0").copy() logger.info( "Found {} GO terms with at least 1 gene in reference".format( len(df))) # extract the ID and label df['id'] = [x['id'] for x in df['term']] df['label'] = [x['label'] for x in df['term']] # some extra information for convenience df["pct_diff_expr"] = df['number_in_list'] * 100 / df[ 'number_in_reference'] df["log2_fold_enrichment"] = pylab.log2(df['fold_enrichment']) df["abs_log2_fold_enrichment"] = abs(pylab.log2(df['fold_enrichment'])) # Some user may want to include GO terms with fold enrichment # significanyly below 1 or not. if include_negative_enrichment is False: df = df.query("fold_enrichment>=1").copy() logger.info( "Found {} GO terms after keeping only positive enrichment". format(len(df))) # filter out FDR>0.05 df = df.query("fdr<=@fdr").copy() logger.info("Found {} GO terms after keeping only FDR<{}".format( len(df), fdr)) return df
def get_taxonomy_biokit(self, ids): """Retrieve taxons given a list of taxons :param list ids: list of taxons as strings or integers. Could also be a single string or a single integer :return: a dataframe .. note:: the first call first loads all taxons in memory and takes a few seconds but subsequent calls are much faster """ # filter the lineage to keep only information from one of the main rank # that is superkingdom, kingdom, phylum, class, order, family, genus and # species ranks = ('kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species') if isinstance(ids, int): ids = [ids] if len(ids) == 0: return pd.DataFrame() logger.info('Retrieving taxon using biokit.Taxonomy') if isinstance(ids, list) is False: ids = [ids] lineage = [self.tax.get_lineage_and_rank(x) for x in ids] # Now, we filter each lineage to keep only relevant ranks # We drop the 'no rank' and create a dictionary # Not nice but works for now results = [] for i, this in enumerate(lineage): default = dict.fromkeys(ranks, ' ') for entry in this: if entry[1] in ranks: default[entry[1]] = entry[0] elif entry[1] == "superkingdom": default["kingdom"] = entry[0] # Scientific name is the last entry tagged has no_rank following # species TODO (check this assumption) # e.g. 351680 and 151529 have same 7 ranks so to differenatiate # them, the scientific name should be used. # By default, we will take the last one. If species or genus, we # repeat the term try: default['name'] = this[-1][0] except: default['name'] = "root (ambigous kingdom)" results.append(default) df = pd.DataFrame.from_records(results) df.index = ids df = df[list(ranks) + ['name']] df.index = df.index.astype(int) return df
def filter(self, identifiers_list=[], min_bp=None, max_bp=None, progressbar=True, output_filename='filtered.fastq'): """Save reads in a new file if there are not in the identifier_list :param int min_bp: ignore reads with length shorter than min_bp :param int max_bp: ignore reads with length above max_bp """ # 7 seconds without identifiers to scan the file # on a 750000 reads if min_bp is None: min_bp = 0 if max_bp is None: max_bp = 1e9 # make sure we are at the beginning self.rewind() output_filename, tozip = self._istozip(output_filename) with open(output_filename, "w") as fout: pb = Progress(self.n_reads) buf = "" filtered = 0 saved = 0 for count, lines in enumerate(grouper(self._fileobj)): identifier = lines[0].split()[0] if lines[0].split()[0].decode() in identifiers_list: filtered += 1 else: N = len(lines[1]) if N <= max_bp and N >= min_bp: buf += "{}{}+\n{}".format( lines[0].decode("utf-8"), lines[1].decode("utf-8"), lines[3].decode("utf-8")) saved += 1 else: filtered += 1 if count % 100000 == 0: fout.write(buf) buf = "" if progressbar is True: pb.animate(count+1) fout.write(buf) if filtered < len(identifiers_list): print("\nWARNING: not all identifiers were found in the fastq file to " + "be filtered.") logger.info("\n{} reads were filtered out and {} saved in {}".format( filtered, saved, output_filename)) if tozip is True: logger.info("Compressing file") self._gzip(output_filename)
def _parse_data(self): taxonomy = {} logger.info("Reading kraken data") columns = ["status", "taxon", "length"] # we select only col 0,2,3 to save memoty, which is required on very # large files try: # each call to concat in the for loop below # will take time and increase with chunk position. # for 15M reads, this has a big cost. So chunksize set to 1M # is better than 1000 and still reasonable in memory reader = pd.read_csv(self.filename, sep="\t", header=None, usecols=[0,2,3], chunksize=1000000) except pd.parser.CParserError: raise NotImplementedError # this section is for the case #only_classified_output when there is no found classified read self.unclassified = N # size of the input data set self.classified = 0 self._df = pd.DataFrame([], columns=columns) self._taxons = self._df.taxon return for chunk in reader: try: self._df self._df = pd.concat([self._df, chunk]) except AttributeError: self._df = chunk self._df.columns = columns count = sum(self._df.taxon == 1) if count: logger.warning("Found %s taxons with root ID (1)" % count) # This gives the list of taxons as index and their amount # above, we select only columns 0, 2, 3 the column are still labelled # 0, 2, 3 in the df self._taxons = self._df.groupby("taxon").size() try: self._taxons.drop(0, inplace=True) except: pass # 0 may not be there self._taxons.sort_values(ascending=False, inplace=True) category = self.df.groupby("status").size() if 'C' in category.index: self.classified = category['C'] else: self.classified = 0 if 'U' in category.index: self.unclassified = category['U'] else: self.unclassified = 0
def download_taxonomic_file(self, overwrite=False): """Loads entire flat file from EBI Do not overwrite the file by default. """ import ftplib from sequana import sequana_config_path if os.path.exists(self.database) and overwrite is False: logger.info( "Found taxonomy.dat file in sequana your path {}".format( sequana_config_path)) return else: logger.info( "Downloading and extracting the taxonomy file from the web. Please be patient." ) if self.source == "ena": url = 'ftp.ebi.ac.uk' else: url = 'ftp.ncbi.nlm.nih.gov' self.ftp = ftplib.FTP(url) self.ftp.login() if self.source == "ena": # for the EBI ftp only: self.ftp.cwd('databases') self.ftp.cwd('pub') self.ftp.cwd('databases') self.ftp.cwd('taxonomy') logger.warning( 'Downloading and saving in %s. This is from ebi and may be behind the NCBI taxonomy' % self.database) self.ftp.retrbinary('RETR taxonomy.dat', open(self.database, 'wb').write) ftp.close() else: self.ftp.cwd('pub') self.ftp.cwd('taxonomy') logger.warning('Downloading and saving in %s from ncbi ftp' % self.database) import tempfile import shutil with tempfile.TemporaryDirectory() as tmpdir: filename = tmpdir + os.sep + "taxdump.tar.gz" self.ftp.retrbinary('RETR taxdump.tar.gz', open(filename, "wb").write) import tarfile tf = tarfile.open(filename) assert "nodes.dmp" in tf.getnames() assert "names.dmp" in tf.getnames() tf.extract("nodes.dmp", tmpdir) tf.extract("names.dmp", tmpdir) ncbi = NCBITaxonomy(tmpdir + os.sep + "names.dmp", tmpdir + os.sep + "nodes.dmp") ncbi.create_taxonomy_file(tmpdir + os.sep + "taxonomy.dat") shutil.move(tmpdir + os.sep + "taxonomy.dat", self.database) self.ftp.close()
def get_taxonomy_biokit(self, ids): """Retrieve taxons given a list of taxons :param list ids: list of taxons as strings or integers. Could also be a single string or a single integer :return: a dataframe .. note:: the first call first loads all taxons in memory and takes a few seconds but subsequent calls are much faster """ # filter the lineage to keep only information from one of the main rank # that is superkingdom, kingdom, phylum, class, order, family, genus and # species ranks = ('kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species') if isinstance(ids, int): ids = [ids] if len(ids) == 0: return pd.DataFrame() logger.info('Retrieving taxon using biokit.Taxonomy') if isinstance(ids, list) is False: ids = [ids] lineage = [self.tax.get_lineage_and_rank(x) for x in ids] # Now, we filter each lineage to keep only relevant ranks # We drop the 'no rank' and create a dictionary # Not nice but works for now results = [] for i, this in enumerate(lineage): default = dict.fromkeys(ranks, ' ') for entry in this: if entry[1] in ranks: default[entry[1]] = entry[0] elif entry[1] == "superkingdom": default["kingdom"] = entry[0] # Scientific name is the last entry tagged has no_rank following # species TODO (check this assumption) # e.g. 351680 and 151529 have same 7 ranks so to differenatiate # them, the scientific name should be used. # By default, we will take the last one. If species or genus, we # repeat the term try: default['name'] = this[-1][0] except: default['name'] = "root (ambigous kingdom)" results.append(default) df = pd.DataFrame.from_records(results) df.index = ids df = df[list(ranks) + ['name']] df.index = df.index.astype(int) return df
def copy_config_from_sequana(module, source="config.yaml", target="config.yaml"): # identify config name from the requested module user_config = module.path + os.sep + source if os.path.exists(user_config): shutil.copy(user_config, target) txt = "copied %s from sequana %s pipeline" logger.info(txt % (source, module.name)) else: logger.warning(user_config + "not found")
def run(self, dbname="multiple", output_prefix="kraken_final"): """Run the hierachical analysis This method does not return anything but creates a set of files: - kraken_final.out - krona_final.html - kraken.png (pie plot of the classified/unclassified reads) .. note:: the databases are run in the order provided in the constructor. """ # list of all output to merge at the end self._list_kraken_output = [] self._list_kraken_input = [] # Iteration over the databases for iteration in range(len(self.databases)): status = self._run_one_analysis(iteration) last_unclassified = self._list_kraken_input[-1] stat = os.stat(last_unclassified) if stat.st_size == 0: break # concatenate all kraken output files file_output_final = self.output_directory + os.sep + "%s.out" % output_prefix with open(file_output_final, 'w') as outfile: for fname in self._list_kraken_output: with open(fname) as infile: for line in infile: outfile.write(line) # create html report logger.info("Analysing results") result = KrakenResults(file_output_final) # TODO: this looks similar to the code in KrakenPipeline. could be factorised result.to_js("%s%s%s.html" % (self.output_directory, os.sep, output_prefix)) result.plot(kind="pie") pylab.savefig(self.output_directory + os.sep + "kraken.png") prefix = self.output_directory + os.sep result.kraken_to_json(prefix + "kraken.json", dbname) result.kraken_to_csv(prefix + "kraken.csv", dbname) # remove kraken intermediate files (including unclassified files) if self.unclassified_output: # Just cp the last unclassified file import shutil shutil.copy2(self._list_kraken_input[-1], self.unclassified_output) if not self.keep_temp_files: for f_temp in self._list_kraken_output: os.remove(f_temp) for f_temp in self._list_kraken_input: os.remove(f_temp)
def _download_sequana_db1(self, verbose=True): dbname = "sequana_db1" from easydev import md5 dir1 = sequana_config_path + os.sep + dbname dir2 = dir1 + os.sep + "taxonomy" self.dv.mkdir(dir1) self.dv.mkdir(dir2) logger.info("Downloading about 8Gb of data (if not already downloaded) from" " Synapse into %s" % dir1) from os.path import exists filename = dir1 + "ena_list.txt" if exists(filename) and md5(filename) == "a9cc6268f3338d1632c4712a412593f2": pass else: self._download_from_synapse('syn6171700', dir1) # database.idx filename = dir1 + "database.idx" if exists(filename) and md5(filename) == "2fa4a99a4f52f2f04c5a965adb1534ac": pass else: self._download_from_synapse('syn6171017', dir1) # database.kdb ; this one is large (8Gb) filename = dir1 + "database.kdb" if exists(filename) and md5(filename) == "ff698696bfc88fe83bc201937cd9cbdf": pass else: self._download_from_synapse('syn6171107', dir1) # Then, the taxonomy directory filename = dir1 + "names.dmp" if exists(filename) and md5(filename) == "10bc7a63c579de02112d125a51fd65d0": pass else: self._download_from_synapse('syn6171286', dir2) filename = dir1 + "nodes.dmp" if exists(filename) and md5(filename) == "a68af5a60434e2067c4a0a16df873980": pass else: self._download_from_synapse('syn6171289', dir2) filename = dir1 + "taxons.txt" if exists(filename) and md5(filename) == "e78fbb43b3b41cbf4511d6af16c0287f": pass else: self._download_from_synapse('syn6171290', dir2) logger.info('done. You should have a kraken DB in %s' % dir1) # The annotations wget("https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv", dir1 + os.sep + "annotations.csv")
def _download_sequana_db1(self, verbose=True): dbname = "sequana_db1" from easydev import md5 dir1 = sequana_config_path + os.sep + dbname dir2 = dir1 + os.sep + "taxonomy" self.dv.mkdir(dir1) self.dv.mkdir(dir2) logger.info("Downloading about 8Gb of data (if not already downloaded) from" " Synapse into %s" % dir1) from os.path import exists filename = dir1 + "ena_list.txt" if exists(filename) and md5(filename) == "a9cc6268f3338d1632c4712a412593f2": pass else: self._download_from_synapse('syn6171700', dir1) # database.idx filename = dir1 + "database.idx" if exists(filename) and md5(filename) == "2fa4a99a4f52f2f04c5a965adb1534ac": pass else: self._download_from_synapse('syn6171017', dir1) # database.kdb ; this one is large (8Gb) filename = dir1 + "database.kdb" if exists(filename) and md5(filename) == "ff698696bfc88fe83bc201937cd9cbdf": pass else: self._download_from_synapse('syn6171107', dir1) # Then, the taxonomy directory filename = dir1 + "names.dmp" if exists(filename) and md5(filename) == "10bc7a63c579de02112d125a51fd65d0": pass else: self._download_from_synapse('syn6171286', dir2) filename = dir1 + "nodes.dmp" if exists(filename) and md5(filename) == "a68af5a60434e2067c4a0a16df873980": pass else: self._download_from_synapse('syn6171289', dir2) filename = dir1 + "taxons.txt" if exists(filename) and md5(filename) == "e78fbb43b3b41cbf4511d6af16c0287f": pass else: self._download_from_synapse('syn6171290', dir2) logger.info('done. You should have a kraken DB in %s' % dir1) # The annotations wget("https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv", dir1 + os.sep + "annotations.csv")
def _get_qualities(self): from sequana import logger logger.info("Extracting qualities") qualities = [] with FastqReader(self.filename) as f: for i, record in enumerate(f): if i < self.max_sample: quality = [ord(x) -33 for x in record.qualities] qualities.append(quality) else: break return qualities
def ena_id_to_gi_number(self, identifiers): # Now, let us convert the ENA accession to NCBI GI number once for all. # We can fetch only at max 200 identifiers: logger.info("Fetching %s identifiers from NCBI" % len(identifiers)) Nbaskets = int(math.ceil(len(identifiers) / 200.)) results = {} from easydev import split_into_chunks for chunk in split_into_chunks(identifiers, Nbaskets): result = self.eutils.accession_to_info(",".join(chunk)) results.update(result) return results
def _download_assembly_report(self, category): assert category in self.category ftp = ftplib.FTP("ftp.ncbi.nlm.nih.gov") ftp.login("anonymous", "anonymous") ftp.cwd("genomes/refseq/{}".format(category)) filename = "assembly_summary.txt" ftp.retrbinary( 'RETR ' + filename, open(filename.replace(".txt", "_{}.txt".format(category)), "wb").write) logger.info(filename)
def _download_minikraken(self, verbose=True): dv = DevTools() base = sequana_config_path + os.sep + "" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) logger.info("Downloading minikraken (4Gb)") filename = base + os.sep + "minikraken.tgz" if os.path.exists(filename) and md5(filename) == "30eab12118158d0b31718106785195e2": logger.warning("%s already present" % filename) else: wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz", filename)
def _download_minikraken(self, verbose=True): dv = DevTools() base = sequana_config_path + os.sep + "" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) logger.info("Downloading minikraken (4Gb)") filename = base + os.sep + "minikraken.tgz" if os.path.exists(filename) and md5(filename) == "30eab12118158d0b31718106785195e2": logger.warning("%s already present" % filename) else: wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz", filename)
def __init__(self, directory=".", prefix=""): self.prefix = prefix self.directory = directory self.sample_name = "undefined" # low quality isoforms filename = "all.polished_lq.fastq" self.lq_isoforms = self.get_file(filename) if self.lq_isoforms: logger.info("Reading {}".format(filename)) self.lq_sequence = FastQ(self.lq_isoforms) # high quality isoforms filename = "all.polished_hq.fastq" self.hq_isoforms = self.get_file(filename) if self.hq_isoforms: logger.info("Reading {}".format(filename)) self.hq_sequence = FastQ(self.hq_isoforms) # General info filename = "file.csv" self.csv = self.get_file(filename) if self.csv: logger.info("Reading {}".format(filename)) self.data = pd.read_csv(self.csv) # CCS fasta sequence #self.ccs = self.get_file("-ccs.tar.gz") filename = "ccs.fasta" self.ccs = self.get_file(filename, noprefix=True) if self.ccs: logger.info("Reading {}".format(filename)) self.ccs = FastA(self.ccs)
def _set_window(self,window): if (window > 0) & (window < 1): self._type_window = 'adapted to genome length : %.1f %% of total length' %(window*100) self._window = int(round(self.__len__() * window)) elif (window >= 1) & ( window <= self.__len__()): self._type_window = 'fixed window length : %d' %(window) self._window = int(window) else: raise ValueError("Incorrect value for window: choose either float ]0,1]" + " (fraction of genome) or integer [1,genome_length] (window size)") logger.info("Computing GC skew") self._compute_skews()
def export(self, filename='test.png'): if self.status is None: logger.error("Upload the tree first with upload() method") export = self.itol.get_itol_export() # Set the format if filename.endswith(".png"): logger.info("Exporting in {} format".format("png")) export.params['format'] = "png" elif filename.endswith(".svg"): logger.info("Exporting in {} format".format("svg")) export.params['format'] = "svg" elif filename.endswith(".pdf"): logger.info("Exporting in {} format".format("pdf")) export.params['format'] = "pdf" elif filename.endswith(".eps"): logger.info("Exporting in {} format".format("eps")) export.params['format'] = "eps" else: raise ValueError("filename must end in pdf, png, svg or eps") export.params.update(**self.params) export.export(filename)
def __init__(self, directory=".", prefix=""): self.prefix = prefix self.directory = directory self.sample_name = "undefined" # low quality isoforms filename = "all.polished_lq.fastq" self.lq_isoforms = self.get_file(filename) if self.lq_isoforms: logger.info("Reading {}".format(filename)) self.lq_sequence = FastQ(self.lq_isoforms) # high quality isoforms filename = "all.polished_hq.fastq" self.hq_isoforms = self.get_file(filename) if self.hq_isoforms: logger.info("Reading {}".format(filename)) self.hq_sequence = FastQ(self.hq_isoforms) # General info filename = "file.csv" self.csv = self.get_file(filename) if self.csv: logger.info("Reading {}".format(filename)) self.data = pd.read_csv(self.csv) # CCS fasta sequence #self.ccs = self.get_file("-ccs.tar.gz") filename = "ccs.fasta" self.ccs = self.get_file(filename, noprefix=True) if self.ccs: logger.info("Reading {}".format(filename)) self.ccs = FastA(self.ccs)
def run(self, dbname="multiple", output_prefix="kraken_final"): """Run the hierachical analysis This method does not return anything but creates a set of files: - kraken_final.out - krona_final.html - kraken.png (pie plot of the classified/unclassified reads) .. note:: the databases are run in the order provided in the constructor. """ # list of all output to merge at the end self._list_kraken_output = [] self._list_kraken_input = [] # Iteration over the databases for iteration in range(len(self.databases)): self._run_one_analysis(iteration) # concatenate all kraken output files file_output_final = self.output_directory + os.sep + "%s.out" % output_prefix with open(file_output_final, 'w') as outfile: for fname in self._list_kraken_output: with open(fname) as infile: for line in infile: outfile.write(line) # create html report logger.info("Analysing results") result = KrakenResults(file_output_final) # TODO: this looks similar to the code in KrakenPipeline. could be factorised result.to_js("%s%s%s.html" % (self.output_directory, os.sep, output_prefix)) result.plot(kind="pie") pylab.savefig(self.output_directory + os.sep + "kraken.png") prefix = self.output_directory + os.sep result.kraken_to_json(prefix + "kraken.json", dbname) result.kraken_to_csv(prefix + "kraken.csv", dbname) # remove kraken intermediate files (including unclassified files) if not self.keep_temp_files: for f_temp in self._list_kraken_output: os.remove(f_temp) for f_temp in self._list_kraken_input: os.remove(f_temp)
def _run_one_analysis(self, iteration): """ Run one analysis """ db = self.databases[iteration] logger.info("Analysing data using database {}".format(db)) # By default, the output contains only classified reads only_classified_output = True # a convenient alias _pathto = lambda x: self.output_directory + x # the output is saved in this file file_kraken_class = _pathto("kraken_%d.out" % iteration) output_filename_unclassified = _pathto("unclassified_%d.fastq" % iteration) file_fastq_unclass = _pathto("unclassified_%d.fastq" % iteration) if iteration == 0: inputs = self.inputs else: # previous results inputs = self._list_kraken_input[iteration-1] # if this is the last iteration (even if iteration is zero), save # classified and unclassified in the final kraken results. if iteration == len(self.databases) -1: only_classified_output = False analysis = KrakenAnalysis(inputs, db, self.threads) analysis.run(output_filename=file_kraken_class, output_filename_unclassified=output_filename_unclassified, only_classified_output=only_classified_output) self._list_kraken_input.append(file_fastq_unclass) self._list_kraken_output.append(file_kraken_class) if self.keep_temp_files: result = KrakenResults(file_kraken_class) result.to_js("%skrona_%d.html" %(self.output_directory, iteration))
def filter_mapq(self, output_filename, threshold_min=0, threshold_max=255): """Select and Write reads within a given range :param str output_filename: name of output file :param int threshold_min: minimum length of the reads to keep :param int threshold_max: maximum length of the reads to keep """ assert threshold_min < threshold_max assert output_filename != self.filename, \ "output filename should be different from the input filename" self.reset() count = 0 with pysam.AlignmentFile(output_filename, "wb", template=self.data) as fh: for read in self.data: if ((read.mapq < threshold_max) & (read.mapq > threshold_min)): fh.write(read) else: pass count += 1 if count % 10000: logger.info("%s sequence processed" % count)
def random_selection(self, output_filename, nreads=None, expected_coverage=None, reference_length=None, read_lengths=None): """Select random reads :param nreads: number of reads to select randomly. Must be less than number of available reads in the orignal file. :param expected_coverage: :param reference_length: if expected_coverage and reference_length provided, nreads is replaced automatically. .. note:: to speed up computation (if you need to call random_selection many times), you can provide the mean read length manually """ assert output_filename != self.filename, \ "output filename should be different from the input filename" if read_lengths is None: self.reset() read_lengths = [read.query_length for i, read in enumerate(self.data)] N = len(read_lengths) if expected_coverage and reference_length: mu = pylab.mean(read_lengths) nreads = int(expected_coverage * reference_length / mu) assert nreads < N, "nreads parameter larger than actual Number of reads" selector = random.sample(range(N), nreads) logger.info("Creating a pacbio BAM file with {} reads".format(nreads)) with pysam.AlignmentFile(output_filename,"wb", template=self.data) as fh: self.reset() for i, read in enumerate(self.data): if i in selector: fh.write(read)
def _download_kraken_toydb(self, verbose=True): """Download the kraken DB toy example from sequana_data into .config/sequana directory Checks the md5 checksums. About 32Mb of data """ dv = DevTools() base = sequana_config_path + os.sep + "kraken_toydb" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) baseurl = "https://github.com/sequana/data/raw/master/" # download only if required logger.info("Downloading the database into %s" % base) md5sums = [ "28661f8baf0514105b0c6957bec0fc6e", "97a39d44ed86cadea470352d6f69748d", "d91a0fcbbc0f4bbac918755b6400dea6", "c8bae69565af2170ece194925b5fdeb9"] filenames = [ "database.idx", "database.kdb", "taxonomy/names.dmp", "taxonomy/nodes.dmp"] for filename, md5sum in zip(filenames, md5sums): url = baseurl + "kraken_toydb/%s" % filename filename = base + os.sep + filename if os.path.exists(filename) and md5(filename) == md5sum: logger.warning("%s already present" % filename) else: logger.info("Downloading %s" % url) wget(url, filename)
def _to_fastX(self, mode, output_filename, threads=2): """ :param mode: fastq or fasta """ # for now, we use samtools # can use bamtools as well but as long and output 10% larger (sequences # are split on 80-characters length) from snakemake import shell cmd = "samtools %s -@ %s %s > %s" % (mode, threads, self.filename, output_filename) logger.info("Please be patient") logger.info("This may be long depending on your input data file: ") logger.info("typically, a minute per 500,000 reads") shell(cmd) logger.info("done")
def stats(self): results = {} if self.data is not None: logger.info("Reading strand") results['strand'] = { "+": sum(self.data.strand == "+"), "-": sum(self.data.strand == "-"), "?": sum(self.data.strand.isnull()) } results['classification'] = { "total_ccs_reads" : len(self.data), "five_prime_reads" : int(self.data.fiveseen.sum()), "three_prime_reads" : int(self.data.threeseen.sum()), "chimera" : int(self.data.chimera.sum()), "polyA_reads" : int(self.data.polyAseen.sum()), } if self.lq_isoforms: logger.info("Reading LQ isoforms") results['lq_isoform'] = self.lq_sequence.stats() # number of if self.hq_isoforms: logger.info("Reading HQ isoforms") results['hq_isoform'] = self.hq_sequence.stats() # number of polished HQ isoform if self.ccs: seq = [ len(read.sequence) for read in self.ccs] results["CCS"] = { "mean_length" : pylab.mean(seq), "number_ccs_bases" : sum(seq), "number_ccs_reads" : len(seq) } self.idents_v = [] self.full_v = [] self.non_full_v = [] self.isoform_lengths = [] for read in self.lq_sequence: ident, full, non_full, length = read['identifier'].decode().split(";") self.idents_v.append(ident) self.full_v.append(int(full.split("=")[1])) self.non_full_v.append(int(non_full.split("=")[1])) self.isoform_lengths.append(int(length.split("=")[1])) return results
def get_df_concordance(self, max_align=-1): """This methods returns a dataframe with Insert, Deletion, Match, Substitution, read length, concordance (see below for a definition) Be aware that the SAM or BAM file must be created using minimap2 and the --cs option to store the CIGAR in a new CS format, which also contains the information about substitution. Other mapper are also handled (e.g. bwa) but the substitution are solely based on the NM tag if it exists. alignment that have no CS tag or CIGAR are ignored. """ from sequana import Cigar count = 0 I, D, M, L, mapq, flags, NM = [], [], [], [], [], [], [] S = [] for i, a in enumerate(self._data): # tags and cigar populated if there is a match # if we use --cs cigar is not populated so we can only look at tags # tags can be an empty list if a.tags is None or len(a.tags) == 0: continue count += 1 mapq.append(a.mapq) L.append(a.qlen) try: NM.append([x[1] for x in a.tags if x[0] == "NM"][0]) except: NM.append(-1) flags.append(a.flag) if 'cs' in dict(a.tags): cs = CS(dict(a.tags)['cs']) S.append(cs['S']) I.append(cs['I']) D.append(cs['D']) M.append(cs['M']) elif a.cigarstring: cigar = Cigar(a.cigarstring).as_dict() I.append(cigar["I"]) D.append(cigar['D']) M.append(cigar['M']) S.append(None) # no info about substitutions in the cigar else: I.append(0) D.append(0) M.append(0) S.append(0) if max_align>0 and count == max_align: break if count % 10000 == 0: logger.debug("Read {} alignments".format(count)) I = np.array(I) D = np.array(D) M = np.array(M) NM = np.array(NM) try: S = np.array(S) C = 1 - (I + D + S)/(S + I + D + M) logger.info("computed Concordance based on minimap2 --cs option") except: logger.info("computed Concordance based on standard CIGAR information using INDEL and NM tag") computed_S = NM - D - I C = 1 - (I + D + computed_S)/(computed_S + I + D + M) df = pd.DataFrame([C, L, I, D, M, mapq, flags, NM, S]) df = df.T df.columns = ["concordance", 'length', "I", "D", "M", "mapq", "flags", "NM", "mismatch"] return df
def _get_df(self): # When scanning the BAM, we can extract the length, SNR of ACGT (still # need to know how to use it). The GC content (note there is no # ambiguity so no S character). The ZMW. Also, from the tags we could # get more # In each alignement, there are lots of information to retrieve. # One could for instance introspect the tags. # - cx: subread local context flags # - ip: vector of length qlen from 0 to 250. This is the IPD (raw frames # or codec V1) # - np: number of passes (1 for subread, variable for CCS) # - pw: vector of length qlen from 0 to 128? This is the PulseWidth (raw # frames or codec V1) # - qs: 0-based start of query in the ZMW read (absent in CCS) # - qe: 0-based end of query in the ZMW read (absent in CCS) # - zm: position/ID of the ZMW # - sn: list of ACGT SNRs. A, C, G, T in that order # - rq: float encoding exepted accuracy # - dq: DeletionQV # - dt: deletion Tag # - iq: insertionQV # - mq: mergeQV # - sq: substituionQV # - st: substituion tag # - RG: ? # See http://pacbiofileformats.readthedocs.io/en/3.0/BAM.html if self._df is None: logger.info("Scanning input file. Please wait") self.reset() N = 0 all_results = [] # This takes 60% of the time...could use cython ? for i, read in enumerate(self.data): tags = dict(read.tags) res = [] # count reads N += 1 if (N % 10000) == 0: logger.info("Read %d sequences" %N) # res[0] = read length res.append(read.query_length) # also stored in tags["qe"] - tags["qs"] res.append(read.reference_length) # also stored in tags["qe"] - tags["qs"] # collections.counter is slow, let us do it ourself if read.query_length and read.query_sequence: res.append( 100. / read.query_length * sum( [read.query_sequence.count(letter) for letter in "CGcgSs"])) else: res.append(None) # res[1:4] contains SNR stored in tags['sn'] in the order A, C, G, T try: snr = list(tags['sn']) except: snr = [None] * 4 res = res + snr # res[6] = ZMW name, also stored in tags["zm"] try: res.append(int(read.qname.split('/')[1])) except: # simulated data may not have the ZMW info, in which #case, we store just a unique ID res.append(i) # aggregate results all_results.append(res) if self._sample and N >= self._sample: break self._df = pd.DataFrame(all_results, columns=['read_length', "reference_length", 'GC_content', 'snr_A','snr_C','snr_G','snr_T','ZMW']) # populate the nb passes from the ZMW grouped = self._df.groupby("ZMW") agg = grouped.agg({"read_length": len}) ZMW = self._df.ZMW.unique() aa = list(pylab.flatten([[agg.loc[this][0]] * agg.loc[this][0] for this in ZMW])) self._df['nb_passes'] = aa self._df['nb_passes'] -= 1 # nb passes starts at 0 self.reset() return self._df
def run_analysis(chrom, options, feature_dict): logger.info("Computing some metrics") if chrom.DOC < 8: logger.warning("The depth of coverage is below 8. sequana_coverage is" " not optimised for such depth. You may want to " " increase the threshold to avoid too many false detections") logger.info(chrom.__str__()) if options.w_median > len(chrom.df) / 4: NW = int(len(chrom.df) / 4) if NW % 2 == 0: NW += 1 logger.warning("median window length is too long. \n" " Setting the window length automatically to a fifth of\n" " the chromosome length ({})".format(NW)) options.w_median = NW # compute the running median, zscore and ROIs for each chunk summarizing the # results in a ChromosomeCovMultiChunk instane logger.info('Using running median (w=%s)' % options.w_median) logger.info("Number of mixture models %s " % options.k) results = chrom.run(options.w_median, options.k, circular=options.circular, binning=options.binning, cnv_delta=options.cnv_clustering) # Print some info related to the fitted mixture models try: mu = results.data[0][0].as_dict()['data']['fit_mu'] sigma = results.data[0][0].as_dict()['data']['fit_sigma'] pi = results.data[0][0].as_dict()['data']['fit_pi'] logger.info("Fitted central distribution (first chunk): mu=%s, sigma=%s, pi=%s" % (round(mu,3), round(sigma,3), round(pi,3))) except: pass # some information about the ROIs found high = chrom.thresholds.high2 low = chrom.thresholds.low2 logger.info("Searching for ROIs (threshold=[{},{}] ; double =[{},{}])".format( chrom.thresholds.low, chrom.thresholds.high, low, high)) ROIs = results.get_rois() # results is a ChromosomeCovMultiChunk instane logger.info("Number of ROIs found: {}".format(len(ROIs.df))) logger.info(" - below average: {}".format(len(ROIs.get_low_rois()))) logger.info(" - above average: {}".format(len(ROIs.get_high_rois()))) # Create directory and save ROIs directory = options.output_directory directory += os.sep + "coverage_reports" directory += os.sep + chrom.chrom_name mkdirs(directory) ROIs.df.to_csv("{}/rois.csv".format(directory)) # save summary and metrics logger.info("Computing extra metrics") summary = results.get_summary() summary.to_json(directory + os.sep + "sequana_summary_coverage.json") logger.info("Evenness: {}".format(summary.data['evenness'])) logger.info("Centralness (3 sigma): {}".format(summary.data['C3'])) logger.info("Centralness (4 sigma): {}".format(summary.data['C4'])) if options.skip_html: return logger.info("Creating report in %s. Please wait" % config.output_dir) if chrom._mode == "chunks": logger.warning(("This chromosome is large. " "Plots in the HTML reports are skipped")) datatable = CoverageModule.init_roi_datatable(ROIs) ChromosomeCoverageModule(chrom, datatable, options={"W": options.w_median, "k": options.k, "ROIs": ROIs, "circular": options.circular}, command=" ".join(["sequana_coverage"] + sys.argv[1:]))
def main(args=None): if args is None: args = sys.argv[:] user_options = Options(prog="sequana") # If --help or no options provided, show the help if len(args) == 1: user_options.parse_args(["prog", "--help"]) else: options = user_options.parse_args(args[1:]) logger.level = options.logging_level if options.download_reference: logger.info("Downloading reference %s from %s\n" % (options.download_reference, options.database)) from bioservices.apps import download_fasta as df df.download_fasta(options.download_reference, method=options.database) if options.download_genbank is None: return if options.download_genbank: logger.info("Downloading genbank %s from %s\n" % (options.download_genbank, options.database)) from sequana.snpeff import download_fasta_and_genbank download_fasta_and_genbank(options.download_genbank, options.download_genbank, genbank=True, fasta=False) return if options.genbank: assert os.path.exists(options.genbank), \ "%s does not exists" % options.genbank logger.info("Reading %s. This may take time depending on " "your input file" % options.input) # Convert BAM to BED if options.input.endswith(".bam"): bedfile = options.input.replace(".bam", ".bed") logger.info("Converting BAM into BED file") shellcmd("bedtools genomecov -d -ibam %s > %s" % (options.input, bedfile)) elif options.input.endswith(".bed"): bedfile = options.input else: raise ValueError("Input file must be a BAM or BED file") # Set the thresholds if options.low_threshold is None: options.low_threshold = -options.threshold if options.high_threshold is None: options.high_threshold = options.threshold # and output directory config.output_dir = options.output_directory config.sample_name = os.path.basename(options.input).split('.')[0] # Now we can create the instance of GenomeCoverage if options.chromosome == -1: chrom_list = [] else: chrom_list = [options.chromosome] gc = GenomeCov(bedfile, options.genbank, options.low_threshold, options.high_threshold, options.double_threshold, options.double_threshold, chunksize=options.chunksize, chromosome_list=chrom_list) # if we have the reference, let us use it if options.reference: logger.info('Computing GC content') gc.compute_gc_content(options.reference, options.w_gc, options.circular) # Now we scan the chromosomes, if len(gc.chrom_names) == 1: logger.warning("There is only one chromosome. Selected automatically.") run_analysis(gc.chr_list[0], options, gc.feature_dict) elif options.chromosome <-1 or options.chromosome > len(gc.chrom_names): msg = "invalid chromosome index; must be in [1;{}]".format(len(gc.chrom_names)) logger.error(msg) sys.exit(1) else: if options.chromosome == -1: chromosomes = gc.chrom_names # take all chromosomes else: # For user, we start at position 1 but in python, we start at zero chromosomes = [gc.chrom_names[options.chromosome-1]] logger.info("There are %s chromosomes/contigs." % len(gc)) for this in gc.chrom_names: data = (this, gc.positions[this]["start"], gc.positions[this]["end"]) logger.info(" {} (starting pos: {}, ending pos: {})".format(*data)) # here we read chromosome by chromosome to save memory. # However, if the data is small. for i, chrom in enumerate(chromosomes): logger.info("==================== analysing chrom/contig %s/%s (%s)" % (i + 1, len(gc), gc.chrom_names[i])) # since we read just one contig/chromosome, the chr_list contains # only one contig, so we access to it with index 0 run_analysis(gc.chr_list[i], options, gc.feature_dict) if options.skip_multiqc is False: logger.info("=========================") logger.info("Creating multiqc report") pathtocfg = sequana_data("multiqc_config.yaml", "../multiqc/") cmd = 'multiqc . -m sequana_coverage -f -c {}'.format(pathtocfg) import subprocess proc = subprocess.Popen(cmd.split(), cwd=options.output_directory) proc.wait()
def _compute_skews(self): ### initialisation = Calculating GC skew and AT skew for first window self._init_sliding_window() GC_content_slide, GC_skew_slide = self._init_list_results() AT_content_slide, AT_skew_slide = self._init_list_results() self._init_cumul_nuc() c = Counter(self._slide_window) dict_counts = {'G' : c['G'], 'C' : c['C'], 'A' : c['A'], 'T' : c['T']} i = 0 # GC sumGC = float(dict_counts['G'] + dict_counts['C']) GC_content_slide[0][i] = sumGC if sumGC > 0: GC_skew_slide[0][i] = (dict_counts['G'] - dict_counts['C']) / sumGC # AT sumAT = float(dict_counts['A'] + dict_counts['T']) AT_content_slide[0][i] = sumAT if sumAT > 0: AT_skew_slide[0][i] = (dict_counts['A'] - dict_counts['T']) / sumAT ### Compute for all genome while(self._seq_right): out_nuc = self._slide_window.popleft() in_nuc = self._seq_right.popleft() self._slide_window.append(in_nuc) i += 1 if i % 500000 == 0: logger.info("%d / %d" % (i, self.__len__())) # if in and out are the same : do nothing, append same result if out_nuc != in_nuc: # remove out from counters if out_nuc in self._dict_nuc: dict_counts[out_nuc] -= 1 if in_nuc in self._dict_nuc: dict_counts[in_nuc] += 1 sumGC = float(dict_counts['G'] + dict_counts['C']) sumAT = float(dict_counts['A'] + dict_counts['T']) # fill results # GC GC_content_slide[0][i] = sumGC if sumGC > 0: GC_skew_slide[0][i] = (dict_counts['G'] - dict_counts['C'])/sumGC # AT AT_content_slide[0][i] = sumAT if sumAT > 0: AT_skew_slide[0][i] = (dict_counts['A'] - dict_counts['T'])/sumAT # cumul if in_nuc in self._dict_nuc: self._cumul[self._dict_nuc[in_nuc]][i+self._window-1] +=1 self._GC_content_slide = GC_content_slide/float(self._window) self._AT_content_slide = AT_content_slide/float(self._window) self._cumul = np.delete(self._cumul, range(self.__len__(),self._cumul.shape[1]),1) self._cumul = np.cumsum(self._cumul,axis=1) ### save result for Z curve self._Xn = list((self._cumul[self._dict_nuc['A']] + self._cumul[self._dict_nuc['G']]) -\ (self._cumul[self._dict_nuc['C']] + self._cumul[self._dict_nuc['T']])) self._Yn = list((self._cumul[self._dict_nuc['A']] + self._cumul[self._dict_nuc['C']]) -\ (self._cumul[self._dict_nuc['G']] + self._cumul[self._dict_nuc['T']])) self._Zn = list((self._cumul[self._dict_nuc['A']] + self._cumul[self._dict_nuc['T']]) -\ (self._cumul[self._dict_nuc['C']] + self._cumul[self._dict_nuc['G']])) self._AT_skew_slide = AT_skew_slide self._GC_skew_slide = GC_skew_slide ### check proportion of ignored nucleotides GC_content_total = (self._cumul[self._dict_nuc['G']][-1] + self._cumul[self._dict_nuc['C']][-1]) / float(self.__len__()) AT_content_total = (self._cumul[self._dict_nuc['A']][-1] + self._cumul[self._dict_nuc['T']][-1]) / float(self.__len__()) self._ignored_nuc = 1.0 - GC_content_total - AT_content_total
def main(args=None): if args is None: args = sys.argv[:] print("Welcome to sequana_vcf_filter") user_options = Options(prog="sequana_vcf_filter") if "--version" in args: import sequana print(sequana.version) sys.exit() elif len(args) == 1 or "--help" in args: user_options.parse_args(["prog", "--help"]) elif len(args) == 2: class SimpleOpt(): pass options = SimpleOpt() options.input_filename = args[1] else: options = user_options.parse_args(args[1:]) # set the level logger.level = options.level vcf = VCF(options.input_filename) vcf.vcf.filter_dict['QUAL'] = options.quality vcf.vcf.apply_indel_filter = options.apply_indel_filter vcf.vcf.apply_dp4_filter = options.apply_dp4_filter vcf.vcf.apply_af1_filter = options.apply_af1_filter vcf.vcf.dp4_minimum_depth = options.minimum_depth vcf.vcf.dp4_minimum_depth_strand = options.minimum_depth_strand vcf.vcf.dp4_minimum_ratio = options.minimum_ratio vcf.vcf.minimum_af1 = options.minimum_af1 vcf.vcf.filter_dict['INFO'] = {} vcf.vcf.filter_dict['QUAL'] = options.quality for this in options.filter: this = this[0] signs = [">", "<", ">=", "<="] for sign in signs: if sign in this: key, value = this.split(sign, 1) key = key.strip() value = sign.strip() + value.strip() vcf.vcf.filter_dict['INFO'][key] = value break logger.info(vcf.vcf.filter_dict) res = vcf.vcf.filter_vcf(options.output_filename, output_filtered=options.output_filtered_filename) print() #print(res) return res