def combine_rerun_with_orig(self): """Special case when a sample with low reads was rerun in an other pool. Run this just before the combine_reads() method of the associated cluster. This method is called on the reruned sampled, not the original.""" # Check we have a rerun # if self.info.get('rerun') is None: return False # Check we are processed # assert self.fasta.count > 0 # Get the original sample # run, pool, num = self.info['rerun']['run'], self.info['rerun']['pool'], self.info['rerun']['num'] orig_sample = illumitag.runs[run][pool-1][num-1] merged = FASTA(orig_sample.base_dir + 'rerun_merged.fasta') # Check we don't merge twice # assert orig_sample.count == orig_sample.fasta.count # Do it # merged.create() merged.add(orig_sample.fasta) merged.add(self.fasta) merged.close() merged.rename_with_num(orig_sample.name + '_read', orig_sample.fasta) merged.remove() # Check # orig_sample.fasta = FASTA(orig_sample.fasta.path) assert orig_sample.count < orig_sample.fasta.count return True
class Silva(Database): """SILVA provides comprehensive, quality checked and regularly updated datasets of aligned small (16S/18S, SSU) and large subunit (23S/28S, LSU) ribosomal RNA (rRNA) sequences for all three domains of life (Bacteria, Archaea and Eukarya). SILVA are the official databases of the software package ARB. https://www.arb-silva.de To install: from seqsearch.databases.silva import silva silva.download() silva.unzip() It will put it in ~/databases/silva_xxx/ """ view_url = "https://www.arb-silva.de/no_cache/download/archive/" base_url = "https://www.arb-silva.de/fileadmin/silva_databases/" short_name = "silva" all_paths = """ /test.txt """ def __init__(self, version, seq_type, base_dir=None): # Attributes # self.version = version self.seq_type = seq_type self.short_name = self.short_name + "_" + self.version # Base directory # if base_dir is None: base_dir = home self.base_dir = base_dir + 'databases/' + self.short_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # URL # self.url = "release_%s/Exports/" % self.version # The database # self.nr99_name = "SILVA_%s_SSURef_Nr99_tax_silva.fasta.gz" % self.version self.nr99_dest = FASTA(self.base_dir + self.nr99_name) self.nr99 = FASTA(self.base_dir + self.nr99_name[:-3]) # The alignment # self.aligned_name = "SILVA_%s_SSURef_Nr99_tax_silva_full_align_trunc.fasta.gz" % self.version self.aligned_dest = FASTA(self.base_dir + self.aligned_name) self.aligned = FASTA(self.base_dir + self.aligned_name[:-3]) def download(self): self.nr99_dest.directory.create(safe=True) self.nr99_dest.remove(safe=True) self.aligned_dest.remove(safe=True) print "\nDownloading", self.base_url + self.url + self.nr99_name wget.download(self.base_url + self.url + self.nr99_name, out=self.nr99_dest.path) print "\nDownloading", self.base_url + self.url + self.aligned_name wget.download(self.base_url + self.url + self.aligned_name, out=self.aligned_dest.path) def unzip(self): self.nr99_dest.ungzip_to(self.nr99) self.nr99.permissions.only_readable() self.aligned_dest.ungzip_to(self.aligned) self.aligned.permissions.only_readable()
fnas_genes = [strip(seq) for seq in fna] print faa, len(set(fnas_genes) ^ set(faas_genes)), "discrepancies" #print "- in fna but not in faa:", [x for x in set(fnas_genes) - set(faas_genes)] #print "- in faa but not in fna:", [x for x in set(faas_genes) - set(fnas_genes)] #print "" fnas_genes = [strip(seq) for fna in fnas for seq in fna] print len(fnas_genes), len(set(fnas_genes)) for genome in faas: out_path = genomes_dir + genome.short_prefix + '.fasta' out_fasta = FASTA(out_path) out_fasta.create() for seq in genome: out_fasta.add_str(str(seq.seq), strip(seq)) out_fasta.close() out_fasta.gzip_to() out_fasta.remove() def lines(): for genome in faas: for gene in genome: name = strip(gene) yield name + '\t' + gene.description[len(name):].rstrip( ' |') + '\n' annotations_path = current_dir + '../ld12/data/annotations.tsv' with open(annotations_path, 'w') as handle: handle.writelines(lines())
class PairedFASTA(object): """Read and write FASTA file pairs without using too much RAM""" format = 'fasta' def __len__(self): return self.count def __iter__(self): return self.parse() def __nonzero__(self): return bool(self.fwd) and bool(self.rev) def __repr__(self): return '<%s object on "%s" and "%s">' % \ (self.__class__.__name__, self.fwd.path, self.rev.path) def __enter__(self): return self.create() def __exit__(self, exc_type, exc_value, traceback): self.close() @property def exists(self): return self.fwd.exists and self.rev.exists def __init__(self, fwd, rev, parent=None): # FASTA objects # self.fwd = FASTA(fwd) self.rev = FASTA(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent @property_cached def count(self): assert self.fwd.count == self.rev.count return self.fwd.count def open(self): self.fwd.open() self.rev.open() def parse(self): return izip(self.fwd.parse(), self.rev.parse()) def close(self): self.fwd.close() self.rev.close() def create(self): self.fwd.create() self.rev.create() return self def add(self, f, r): return self.add_pair((f, r)) def add_pair(self, pair): self.fwd.add_seq(pair[0]) self.rev.add_seq(pair[1]) def remove(self): self.fwd.remove() self.rev.remove() @property def progress(self): """Just like self.parse but display a progress bar""" return tqdm(self, total=len(self)) def subsample(self, down_to, dest_pair=None): # Check size # assert down_to < len(self) # Make new pair of files # if dest_pair is None: dest_fwd_path = self.fwd_path.new_name_insert("subsampled") dest_rev_path = self.rev_path.new_name_insert("subsampled") dest_pair = self.__class__(dest_fwd_path, dest_rev_path) # Do it # dest_pair.create() for pair in isubsample(self, down_to): dest_pair.add_pair(pair) self.subsampled.close() # Did it work # assert len(dest_pair) == down_to #------------------------------- Extensions ------------------------------# def parse_primers(self, *args, **kwargs): fwd_gen = self.fwd.parse_primers(*args, **kwargs) rev_gen = self.rev.parse_primers(*args, **kwargs) generator = izip(fwd_gen, rev_gen) return GenWithLength(generator, len(fwd_gen))
class PairedFASTA(object): """Read and write FASTA file pairs without using too much RAM""" format = 'fasta' def __len__(self): return self.count def __iter__(self): return self.parse() def __nonzero__(self): return bool(self.fwd) and bool(self.rev) def __repr__(self): return '<%s object on "%s" and "%s">' % \ (self.__class__.__name__, self.fwd.path, self.rev.path) def __enter__(self): return self.create() def __exit__(self, exc_type, exc_value, traceback): self.close() @property def exists(self): return self.fwd.exists and self.rev.exists def __init__(self, fwd, rev, parent=None): # FASTA objects # self.fwd = FASTA(fwd) self.rev = FASTA(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent @property_cached def count(self): assert self.fwd.count == self.rev.count return self.fwd.count def open(self): self.fwd.open() self.rev.open() def parse(self): return izip(self.fwd.parse(), self.rev.parse()) def close(self): self.fwd.close() self.rev.close() def create(self): self.fwd.create() self.rev.create() return self def add(self, f, r): return self.add_pair((f,r)) def add_pair(self, pair): self.fwd.add_seq(pair[0]) self.rev.add_seq(pair[1]) def remove(self): self.fwd.remove() self.rev.remove() @property def progress(self): """Just like self.parse but display a progress bar""" return tqdm(self, total=len(self)) def subsample(self, down_to, dest_pair=None): # Check size # assert down_to < len(self) # Make new pair of files # if dest_pair is None: dest_fwd_path = self.fwd_path.new_name_insert("subsampled") dest_rev_path = self.rev_path.new_name_insert("subsampled") dest_pair = self.__class__(dest_fwd_path, dest_rev_path) # Do it # dest_pair.create() for pair in isubsample(self, down_to): dest_pair.add_pair(pair) self.subsampled.close() # Did it work # assert len(dest_pair) == down_to #------------------------------- Extensions ------------------------------# def parse_primers(self, *args, **kwargs): fwd_gen = self.fwd.parse_primers(*args, **kwargs) rev_gen = self.rev.parse_primers(*args, **kwargs) generator = izip(fwd_gen, rev_gen) return GenWithLength(generator, len(fwd_gen))
class Silva(Database): """ SILVA provides comprehensive, quality checked and regularly updated datasets of aligned small (16S/18S, SSU) and large subunit (23S/28S, LSU) ribosomal RNA (rRNA) sequences for all three domains of life (Bacteria, Archaea and Eukarya). SILVA are the official databases of the software package ARB. https://www.arb-silva.de To install: from seqsearch.databases.silva import silva silva.download() silva.unzip() It will put it in ~/databases/silva_xxx/ """ view_url = "https://www.arb-silva.de/no_cache/download/archive/" base_url = "https://www.arb-silva.de/fileadmin/silva_databases/" short_name = "silva" all_paths = """ /test.txt """ def __init__(self, version, seq_type, base_dir=None): # Attributes # self.version = version self.seq_type = seq_type self.short_name = self.short_name + "_" + self.version # Base directory # if base_dir is None: base_dir = home self.base_dir = base_dir + 'databases/' + self.short_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # URL # self.url = "release_%s/Exports/" % self.version # The database # self.nr99_name = "SILVA_%s_SSURef_Nr99_tax_silva.fasta.gz" % self.version self.nr99_dest = FASTA(self.base_dir + self.nr99_name) self.nr99 = FASTA(self.base_dir + self.nr99_name[:-3]) # The alignment # self.aligned_name = "SILVA_%s_SSURef_Nr99_tax_silva_full_align_trunc.fasta.gz" % self.version self.aligned_dest = FASTA(self.base_dir + self.aligned_name) self.aligned = FASTA(self.base_dir + self.aligned_name[:-3]) def download(self): self.nr99_dest.directory.create(safe=True) self.nr99_dest.remove(safe=True) self.aligned_dest.remove(safe=True) import wget print("\nDownloading", self.base_url + self.url + self.nr99_name) wget.download(self.base_url + self.url + self.nr99_name, out=self.nr99_dest.path) print("\nDownloading", self.base_url + self.url + self.aligned_name) wget.download(self.base_url + self.url + self.aligned_name, out=self.aligned_dest.path) def unzip(self): self.nr99_dest.ungzip_to(self.nr99) self.nr99.permissions.only_readable() self.aligned_dest.ungzip_to(self.aligned) self.aligned.permissions.only_readable()
seq = seq.split('[')[0] return seq for faa,fna in zip(faas, fnas): faas_genes = [strip(seq) for seq in faa] fnas_genes = [strip(seq) for seq in fna] print faa, len(set(fnas_genes) ^ set(faas_genes)), "discrepancies" #print "- in fna but not in faa:", [x for x in set(fnas_genes) - set(faas_genes)] #print "- in faa but not in fna:", [x for x in set(faas_genes) - set(fnas_genes)] #print "" fnas_genes = [strip(seq) for fna in fnas for seq in fna] print len(fnas_genes), len(set(fnas_genes)) for genome in faas: out_path = genomes_dir + genome.short_prefix + '.fasta' out_fasta = FASTA(out_path) out_fasta.create() for seq in genome: out_fasta.add_str(str(seq.seq), strip(seq)) out_fasta.close() out_fasta.gzip_to() out_fasta.remove() def lines(): for genome in faas: for gene in genome: name = strip(gene) yield name + '\t' + gene.description[len(name):].rstrip(' |') + '\n' annotations_path = current_dir + '../ld12/data/annotations.tsv' with open(annotations_path, 'w') as handle: handle.writelines(lines())
class Cluster(object): """Analyzes a group of samples.""" all_paths = """ /reads/all_reads.fasta /otus/ /logs/ /report/report.pdf /metadata.csv """ def __repr__(self): return '<%s object "%s" with %i samples>' % (self.__class__.__name__, self.name, len(self.samples)) def __iter__(self): return iter(self.samples) def __len__(self): return len(self.samples) def __getitem__(self, key): if isinstance(key, basestring): return [c for c in self.children if c.short_name == key.lower()][0] elif isinstance(key, int) and hasattr(self.first, 'num'): return [c for c in self.children if c.num == key][0] else: return self.children[key] @property def first(self): return self.children[0] @property def count_seq(self): return sum([len(sample) for sample in self]) def __init__(self, samples, name, base_dir=None): # Save samples # self.name = name self.samples, self.children = samples, samples # Check names are unique # names = [s.short_name for s in samples if s.used] assert len(names) == len(set(names)) # Figure out pools # self.pools = list(set([s.pool for s in self.samples])) self.pools.sort(key = lambda x: x.id_name) # Directory # if base_dir: self.base_dir = base_dir else: self.base_dir = illumitag.view_dir + "clusters/" + self.name + '/' # Loaded # self.loaded = False def load(self): """A second __init__ that is delayed and called only if needed""" # Load the pools and samples # for p in self.pools: p.load() for s in self.samples: s.load() # Dir # self.p = AutoPaths(self.base_dir, self.all_paths) # Figure out if it's a project # if set(self.samples) == set(self.first.pool.project.samples): self.project = self.first.pool.project else: self.project = None # Runner # self.runner = ClusterRunner(self) # FASTA # self.reads = FASTA(self.p.all_reads_fasta) # OTU picking # self.otu_uparse = UparseOTUs(self) self.otu_uclust = UclustOTUs(self) self.otu_cdhit = CdhitOTUs(self) # Preferred # self.otus = self.otu_uparse # Simple reporting # self.reporter = ClusterReporter(self) # Full report # self.report = ClusterReport(self) # Loaded # self.loaded = True # Return self for convenience # return self def run(self, *args, **kwargs): self.runner.run(*args, **kwargs) def run_slurm(self, *args, **kwargs): self.runner.run_slurm(*args, **kwargs) def process_samples(self): for sample in tqdm(self): sample.process() def combine_reads(self): """This is the first function should call. It will combine all the reads of all the samples of this cluster into one big FASTA file.""" paths = [sample.fasta.path for sample in self] shell_output('cat %s > %s' % (' '.join(paths), self.reads)) return self.reads def set_size(self, length): """Trim all sequences to a specific length starting from the end.""" self.size_trimmed = FASTA(new_temp_path()) def trim_iterator(reads): for read in reads: if len(read) < length: continue yield read[-length:] self.size_trimmed.write(trim_iterator(self.reads)) self.size_trimmed.close() # Replace it # self.reads.remove() shutil.move(self.size_trimmed, self.reads) def run_uparse(self): self.otu_uparse.run() @property def metadata(self): return pandas.DataFrame([s.info for s in self], index=[s.short_name for s in self]) def export_metadata(self): self.metadata.to_csv(self.p.metadata, sep='\t', encoding='utf-8')