def __init__(self, source, dest=None): # Basic # self.source = FASTQ(source) self.dest = DirectoryPath(dest) # Default case # if dest is None: self.dest = DirectoryPath(self.source.prefix_path + '.fastqc')
def __init__(self, fwd, rev, parent=None): # FASTQ objects # self.fwd = FASTQ(fwd) self.rev = FASTQ(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent
def load(self): """A second __init__ that is delayed and called only if needed""" # Automatic paths # self.base_dir = DirectoryPath(self.out_dir + self.id_name + '/') self.p = AutoPaths(self.base_dir, self.all_paths) # Special # self.primers = TwoPrimers(self) # Samples dummy # self.info['samples'] = [{"name":self.short_name, "used":1, "group":self.group, "dummy":1, "num":self.num, "fwd":"", "rev":""}] self.samples = Samples(self) self.samples.load() self.loaded = True # Files # if not os.access('/proj/%s' % self.account, os.R_OK): return self.fwd_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name) self.rev_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name) self.gzipped = True if self.fwd_path.endswith('gz') else False self.fwd = FASTQ(self.fwd_path) self.rev = FASTQ(self.rev_path) self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self) # Barcode length # self.bar_len = 0 # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Assembly files as children # self.assembled = Assembled('', self) self.unassembled = Unassembled('', self) self.children = (self.assembled, self.unassembled) self.first = self.assembled # Special case, for when the two reads don't join # self.trim_and_concat = TrimerAndConcactenater(self) # Final # self.trimmed = FASTQ(self.p.trimmed) self.renamed = FASTQ(self.p.renamed) self.fasta = FASTA(self.p.reads_fasta) # Graphs # self.graphs = [getattr(outcome_plots, cls_name)(self) for cls_name in outcome_plots.__all__] # Runner # self.runner = PresampleRunner(self) # Diversity # self.diversity = AlphaDiversity(self) # Report # self.report = SampleReport(self) # Loaded # self.loaded = True # Return self for convenience # return self
class PairedFASTQ(PairedFASTA): """Read and write FASTQ file pairs without using too much RAM.""" format = 'fastq' def __init__(self, fwd, rev, parent=None): # FASTQ objects # self.fwd = FASTQ(fwd) self.rev = FASTQ(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent def validate(self): """Call fastQValidator on these files.""" self.fwd.validator() self.rev.validator()
def load(self): """A second __init__ that is delayed and called only if needed""" # Check files are there # for f in self.sff_files_info: if not os.path.exists(f['path']): raise Exception("No file at %s" % f['path']) # Automatic paths # self.base_dir = self.out_dir + self.id_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Primer # self.primer_regex = re.compile(self.info['primer']) # Raw files # self.raw_fasta = FASTA(self.p.raw_fasta) self.raw_fastq = FASTQ(self.p.raw_fastq) # Standard FASTA # self.reads = FASTA(self.p.reads_fasta) self.fasta = FASTA(self.p.renamed) # Special FASTQ # self.fastq = FASTQ(self.p.reads_fastq) # A shameless hack for cdhit to work # self.renamed = self.fastq # Pre-denoised special case # if self.info['predenoised'] and False: self.sff_files_info = [] self.reads.link_from(self.info['predenoised'], safe=True) # Special submission attributes # self.sra = PyroSampleSRA(self) # Loaded # self.loaded = True # Return self for convenience # return self
class PairedFASTQ(PairedFASTA): """Read and write FASTQ file pairs without using too much RAM""" format = 'fastq' def __init__(self, fwd, rev, parent=None): # FASTQ objects # self.fwd = FASTQ(fwd) self.rev = FASTQ(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent def validate(self): """Call https://github.com/statgen/fastQValidator on these files.""" self.fwd.validate() self.rev.validate()
class FastQC(object): """Takes care of running the FastQC program. See http://www.bioinformatics.babraham.ac.uk/projects/fastqc/ Expects version 0.10.1.""" def __nonzero__(self): return os.path.exists(self.output_dir + 'Images/per_base_quality.png') def __repr__(self): return '<%s object on "%s">' % (self.__class__.__name__, self.source.path) def __init__(self, source, dest=None): # Basic # self.source = FASTQ(source) self.dest = DirectoryPath(dest) # Default case # if dest is None: self.dest = DirectoryPath(self.source.prefix_path + '.fastqc') def run(self, cpus=None): # Check version # assert "v0.10.1" in sh.fastqc101('--version') # Variable threads # if cpus is None: cpus = num_processors # Destination absent # if self.dest is None: sh.fastqc101(self.source, '-q', '-t', cpus) os.remove(self.source.prefix_path + '_fastqc.zip') # Destination given # if self.dest is not None: self.tmp_dir = new_temp_dir() sh.fastqc101(self.source, '-q', '-o', self.tmp_dir, '-t', cpus) created_name = self.source.prefix.split('.')[0] + '_fastqc/' created_dir = self.tmp_dir + created_name if self.dest.exists: shutil.rmtree(self.dest) shutil.move(created_dir, self.dest) self.tmp_dir.remove() # Strange case where created_dir is not renamed to self.dest but put inside instead # if self.dest + created_name in self.dest.flat_directories: raise Exception( "Looks like the shutil.move didn't do what was expected. Should fix." ) # Return # return self.results @property def output_dir(self): if self.dest is None: return self.source.split('.')[0] + '_fastqc/' else: return self.dest @property_cached def results(self): results = FastQCResults(self.output_dir) if not results: raise Exception( "You can't access results from FastQC before running the tool." ) return results
def load(self): # Special case for dummy samples # if self.info.get('dummy'): return # Paths # self.base_dir = self.pool.p.samples_dir + self.bar_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) self.path = str(self.p.orig_fastq) # Distances # self.trim_fwd = self.pool.samples.trim_fwd self.trim_rev = self.pool.samples.trim_rev # Files # self.trimmed = FASTQ(self.p.trimmed) self.renamed = FASTQ(self.p.renamed) self.fasta = FASTA(self.p.reads_fasta) self.raw = PairedFASTQ(self.p.raw_fwd, self.p.raw_rev, self.pool) self.raw_gz = PairedFASTQ(self.p.raw_forward_gz, self.p.raw_reverse_gz, self.pool) # Inherit # self.project = self.pool.project # Loaded # self.loaded = True
class FastQC(object): """Takes care of running the FastQC program. See http://www.bioinformatics.babraham.ac.uk/projects/fastqc/ Expects version 0.10.1.""" def __init__(self, source, dest=None): # Basic # self.source = FASTQ(source) self.dest = DirectoryPath(dest) # Default case # if dest is None: self.dest = DirectoryPath(self.source.prefix_path + '.fastqc') def check(self): assert sh.fastqc('--v', ) def run(self): if self.dest is None: sh.fastqc(self.source, '-q') os.remove(self.source.prefix_path + '_fastqc.zip') if self.dest is not None: if self.dest.exists: self.dest.remove() self.tmp_dir = new_temp_dir() sh.fastqc(self.source, '-q', '-o', self.tmp_dir) created_dir = self.tmp_dir + self.source.prefix.split('.')[0] + '_fastqc/' shutil.move(created_dir, self.dest) self.tmp_dir.remove() return self.results @property def output_dir(self): if self.dest is None: return self.source.split('.')[0] + '_fastqc/' else: return self.dest @property_cached def results(self): results = FastQCResults(self.output_dir) if not results: self.run() return results
class Presample(BarcodeGroup): """A Presample is a clumsy name for a new type of barcoded-sequence files. As we updated the lab protocol, sample are not multiplexed with our traditional 50 barcodes anymore, but with Illumina specific MIDs. The demultiplexing thus happens in their pipeline and we are left with one sample per file. This object is a bit like a *Pool*, a *BarcodeGroup* and a *Sample* all at the same time. In the end it inherits from BarcodeGroup and just emulates the behavior of the other objects.""" all_paths = """ /info.json /uncompressed/fwd.fastq /uncompressed/rev.fastq /logs/ /assembled/ /unassembled/ /fastqc/ /graphs/ /report/report.pdf /quality/trimmed.fastq /quality/renamed.fastq /quality/reads.fasta """ kind = 'presample' def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name) def __str__(self): return self.id_name def __iter__(self): return iter(self.children) def __len__(self): return self.count def __getitem__(self, key): return self.samples[key] @property def seq_len(self): return len(self.fwd.first_read) def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = FilePath(json_path) # Parse # self.info = load_json_path(self.json_path) # Basic # self.account = self.info['uppmax_id'] self.run_num = self.info['run_num'] self.run_label = self.info['run_id'] self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] self.fwd_name = self.info['forward_reads'] self.rev_name = self.info['reverse_reads'] # Own attributes # self.num = self.info['sample_num'] self.label = self.info['sample_id'] self.short_name = self.info['sample'] self.long_name = self.info['sample_name'] self.name = 'run%i_sample%i' % (self.run_num, self.num) self.group = self.info.get('group') self.id_name = "run%03d-sample%02d" % (self.run_num, self.num) self.fwd_mid = self.info['forward_mid'] self.rev_mid = self.info['reverse_mid'] self.used = True # Check name is ASCII # assert all(ord(c) < 128 for c in self.short_name) # Pool dummy # self.pool, self.parent = self, self # Second init # self.loaded = False def load(self): """A second __init__ that is delayed and called only if needed""" # Automatic paths # self.base_dir = DirectoryPath(self.out_dir + self.id_name + '/') self.p = AutoPaths(self.base_dir, self.all_paths) # Special # self.primers = TwoPrimers(self) # Samples dummy # self.info['samples'] = [{"name":self.short_name, "used":1, "group":self.group, "dummy":1, "num":self.num, "fwd":"", "rev":""}] self.samples = Samples(self) self.samples.load() self.loaded = True # Files # if not os.access('/proj/%s' % self.account, os.R_OK): return self.fwd_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name) self.rev_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name) self.gzipped = True if self.fwd_path.endswith('gz') else False self.fwd = FASTQ(self.fwd_path) self.rev = FASTQ(self.rev_path) self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self) # Barcode length # self.bar_len = 0 # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Assembly files as children # self.assembled = Assembled('', self) self.unassembled = Unassembled('', self) self.children = (self.assembled, self.unassembled) self.first = self.assembled # Special case, for when the two reads don't join # self.trim_and_concat = TrimerAndConcactenater(self) # Final # self.trimmed = FASTQ(self.p.trimmed) self.renamed = FASTQ(self.p.renamed) self.fasta = FASTA(self.p.reads_fasta) # Graphs # self.graphs = [getattr(outcome_plots, cls_name)(self) for cls_name in outcome_plots.__all__] # Runner # self.runner = PresampleRunner(self) # Diversity # self.diversity = AlphaDiversity(self) # Report # self.report = SampleReport(self) # Loaded # self.loaded = True # Return self for convenience # return self @property_cached def counts(self): """The OTU counts""" taxa_table = self.project.cluster.otus.taxonomy.comp_tips.taxa_table row = taxa_table.loc[self.short_name].copy() row.sort(ascending=False) return row def join(self): """Uses pandaseq 2.7 to join the foward and reverse reads together. See https://github.com/neufeld/pandaseq""" # Special case for new primers that don't join # rev_primer_name = self.info['primers']['reverse']['name'] not_joining_primers = ("1132R", "1000R") if rev_primer_name in not_joining_primers: print "No overlap special case" self.trim_and_concat.run() return # Special case for primers that highly overlap # high_overlap_primers = ("806R",) if rev_primer_name in high_overlap_primers: print "High overlap special case, using mothur" result = sh.mothur("#make.contigs(ffastq=%s, rfastq=%s);" % (self.uncomrpessed_pair.fwd, self.uncomrpessed_pair.rev)) if "ERROR" in result.stdout: raise Exception("Mothur didn't run correctly") # Move things # #shutil.move(self.tax.centers.prefix_path + '.align', self.mothur_aligned) #shutil.move(self.tax.centers.prefix_path + '.align.report', self.p.mothur_report) return # Default case # command = 'pandaseq27 -T 1 -f %s -r %s -u %s -F 1> %s 2> %s' command = command % (self.fwd, self.rev, self.unassembled.path, self.assembled.path, self.assembled.p.out) shell_call(command) # Because it exits with status 1 https://github.com/neufeld/pandaseq/issues/40 def process(self): """Lorem""" def no_primers_iterator(reads): for read_w_miss in reads: yield read_w_miss.read[read_w_miss.fwd_end_pos:read_w_miss.rev_end_pos] reads = self.assembled.good_primers.len_filtered.parse_primers(mismatches=1) self.trimmed.write(no_primers_iterator(reads)) self.trimmed.rename_with_num(self.name + '_read', self.renamed) self.renamed.to_fasta(self.fasta) def make_mothur_output(self): pass def make_qiime_output(self): pass def make_presample_plots(self): for graph in self.graphs: graph.plot() @property_cached def uncomrpessed_pair(self): """Usefull for a few stupid programs that don't take fastq.gz files such as mothur""" result = PairedFASTQ(self.p.uncompressed_fwd, self.p.uncompressed_rev) if not result.exists: self.fwd.ungzip_to(result.fwd) self.rev.ungzip_to(result.rev) return result
class PairedFASTQ(object): """Read and write FASTQ file pairs without using too much RAM""" def __len__(self): return self.count def __iter__(self): return self.parse() def __enter__(self): return self.create() def __exit__(self, exc_type, exc_value, traceback): self.close() def __repr__(self): return '<%s object on "%s" and "%s">' % \ (self.__class__.__name__, self.fwd.path, self.rev.path) @property def exists(self): return self.fwd.exists def __init__(self, fwd, rev, parent=None): # FASTQ objects # self.fwd = FASTQ(fwd) self.rev = FASTQ(rev) # Extra # self.gzipped = self.fwd.gzipped self.parent = parent @property_cached def count(self): assert self.fwd.count == self.rev.count return self.fwd.count def open(self): self.fwd.open() self.rev.open() def parse(self): return izip(self.fwd.parse(), self.rev.parse()) def close(self): self.fwd.close() self.rev.close() def create(self): self.fwd.create() self.rev.create() return self def add_pair(self, pair): self.fwd.add_seq(pair[0]) self.rev.add_seq(pair[1]) @property def progress(self): """Just like self.parse but display a progress bar""" return tqdm(self, total=len(self)) def subsample(self, down_to, dest_pair=None): # Check size # assert down_to < len(self) # Make new pair of files # if dest_pair is None: dest_fwd_path = self.fwd_path.new_name_insert("subsampled") dest_rev_path = self.rev_path.new_name_insert("subsampled") dest_pair = self.__class__(dest_fwd_path, dest_rev_path) # Do it # dest_pair.create() for pair in isubsample(self, down_to): dest_pair.add_pair(pair) self.subsampled.close() # Did it work # assert len(dest_pair) == down_to
class Pyrosample(object): """A Pyrosample is a legacy object for the few 454 samples we still have and that we need to compare against the new Illumina technology.""" all_paths = """ /info.json /reads.fasta /renamed.fasta /raw/raw.sff /raw/raw.fastq /raw/raw.fasta /raw/raw.qual /raw/manifest.txt /fastq/reads.fastq """ kind = "pyrosample" def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name) def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = FilePath(json_path) # Parse # self.info = load_json_path(self.json_path) # Basic # self.account = "/dev/null" self.run_num = self.info['run_num'] self.run_label = "pyrosample_run_%i" % self.run_num self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] # Own attributes # self.num = self.info['sample_num'] self.short_name = self.info['sample'] self.long_name = self.info['sample_name'] self.name = 'run%i_sample%i' % (self.run_num, self.num) self.group = self.info['group'] self.id_name = "run%03d-sample%02d" % (self.run_num, self.num) # Hard coded attributes # self.machine = "454 GS FLX Titanium" # SFF files # self.sff_files_info = self.info['files'] # Pool dummy # self.pool, self.parent = self, self # Other dummy variables # self.bar_len = 0 self.gzipped = False self.used = True # Loaded # self.loaded = False def load(self): """A second __init__ that is delayed and called only if needed""" # Check files are there # for f in self.sff_files_info: if not os.path.exists(f['path']): raise Exception("No file at %s" % f['path']) # Automatic paths # self.base_dir = self.out_dir + self.id_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Primer # self.primer_regex = re.compile(self.info['primer']) # Raw files # self.raw_fasta = FASTA(self.p.raw_fasta) self.raw_fastq = FASTQ(self.p.raw_fastq) # Standard FASTA # self.reads = FASTA(self.p.reads_fasta) self.fasta = FASTA(self.p.renamed) # Special FASTQ # self.fastq = FASTQ(self.p.reads_fastq) # A shameless hack for cdhit to work # self.renamed = self.fastq # Pre-denoised special case # if self.info['predenoised'] and False: self.sff_files_info = [] self.reads.link_from(self.info['predenoised'], safe=True) # Special submission attributes # self.sra = PyroSampleSRA(self) # Loaded # self.loaded = True # Return self for convenience # return self @property def mate(self): if not 'mate' in self.info: return False run_num = self.info['mate']['run'] pool_num = self.info['mate']['pool'] barcode_num = self.info['mate']['num'] return illumitag.runs[run_num][pool_num-1][barcode_num-1] def extract(self): # Call extraction # shell_output('sffinfo -s %s > %s' % (self.p.raw_sff, self.p.raw_fasta)) shell_output('sffinfo -q %s > %s' % (self.p.raw_sff, self.p.raw_qual)) shell_output('sffinfo -m %s > %s' % (self.p.raw_sff, self.p.manifest)) # Convert # sh.fasta_to_fastq(self.p.raw_fasta, self.p.raw_qual, self.p.raw_fastq) def clean_iterator(self, reads, minlength=400, threshold=21, windowsize=20): for read in reads: # Length # if len(read) < minlength: continue # Primer # match = self.primer_regex.search(str(read.seq)) if not match: continue # PHRED score # scores = read.letter_annotations["phred_quality"] averaged = moving_average(scores, windowsize) discard = False for i,value in enumerate(averaged): if value < threshold: read = read[:i+windowsize-1] if len(read) < minlength: discard = True break if discard: continue # Undetermined bases # if 'N' in read: continue # Remove primer # read = read[match.end():] # Flip them because 454 reads the other end # read = read.reverse_complement() # Return # yield read def clean(self, **kwargs): self.reads.write(self.clean_iterator(self.raw_fastq, **kwargs)) def report_loss(self): print "Before cleaning: %i" % len(self.raw_fastq) print "After cleaning: %i" % len(self.reads) print "Loss: %.2f%%" % (100 * (1 - (len(self.raw_fastq)/len(self.reads)))) def process(self): self.reads.rename_with_num(self.name + '_read', new_path=self.fasta) def make_fastq(self, **kwargs): """In some special cases we want the FASTQ""" self.fastq.write(self.clean_iterator(self.raw_fastq, **kwargs)) self.fastq.rename_with_num(self.name + '_read') print "make_fastq for sample %s completed" % self.id_name
class Sample(FASTQ): """All sequences with the same barcode pair grouped together""" all_paths = """ /orig.fastq /trimmed.fastq /renamed.fastq /reads.fasta /raw/fwd.fastq /raw/rev.fastq /raw/forward.fastq.gz /raw/reverse.fastq.gz """ kind = 'sample' def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.name) def __str__(self): return self.bar_name def __init__(self, info, parent): # Save attributes # self.info = info self.parent = parent self.pool = parent.pool # Basic # self.short_name = info['name'] self.group_name = info.get('group') self.num = int(info['num']) self.used = bool(info['used']) self.fwd_str = info['fwd'] self.rev_str = info['rev'] # Other # self.bar_name = 'barcode%i' % self.num self.name = 'run%i_pool%i_sample%i' % (self.pool.run_num, self.pool.num, self.num) # Special submission attributes # self.sra = SampleSRA(self) # Second init # self.loaded = False def load(self): # Special case for dummy samples # if self.info.get('dummy'): return # Paths # self.base_dir = self.pool.p.samples_dir + self.bar_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) self.path = str(self.p.orig_fastq) # Distances # self.trim_fwd = self.pool.samples.trim_fwd self.trim_rev = self.pool.samples.trim_rev # Files # self.trimmed = FASTQ(self.p.trimmed) self.renamed = FASTQ(self.p.renamed) self.fasta = FASTA(self.p.reads_fasta) self.raw = PairedFASTQ(self.p.raw_fwd, self.p.raw_rev, self.pool) self.raw_gz = PairedFASTQ(self.p.raw_forward_gz, self.p.raw_reverse_gz, self.pool) # Inherit # self.project = self.pool.project # Loaded # self.loaded = True def process(self): def no_primers_iterator(reads): for read in reads: yield read[self.trim_fwd:-self.trim_rev] self.trimmed.write(no_primers_iterator(self)) self.trimmed.rename_with_num(self.name + '_read', self.renamed) self.renamed.to_fasta(self.fasta) assert self.count == self.fasta.count def combine_rerun_with_orig(self): """Special case when a sample with low reads was rerun in an other pool. Run this just before the combine_reads() method of the associated cluster. This method is called on the reruned sampled, not the original.""" # Check we have a rerun # if self.info.get('rerun') is None: return False # Check we are processed # assert self.fasta.count > 0 # Get the original sample # run, pool, num = self.info['rerun']['run'], self.info['rerun']['pool'], self.info['rerun']['num'] orig_sample = illumitag.runs[run][pool-1][num-1] merged = FASTA(orig_sample.base_dir + 'rerun_merged.fasta') # Check we don't merge twice # assert orig_sample.count == orig_sample.fasta.count # Do it # merged.create() merged.add(orig_sample.fasta) merged.add(self.fasta) merged.close() merged.rename_with_num(orig_sample.name + '_read', orig_sample.fasta) merged.remove() # Check # orig_sample.fasta = FASTA(orig_sample.fasta.path) assert orig_sample.count < orig_sample.fasta.count return True @property def json(self): """Regenerate the JSON string from the object including extra info""" result = OrderedDict([(k, self.info[k]) for k in ('name', 'used', 'group', 'num', 'fwd', 'rev')]) result = json.dumps(result) if self.extra_metadata: result = result[:-1] + ',' + json.dumps(self.extra_metadata, indent=4)[1:] result = re.compile(r'\bNaN\b').sub('null', result) return result @property def count_raw_reads(self): """The number of reads the sample originally had right after barcode processing and before any other quality filtering""" return self.pool.good_barcodes.breakdown[self.bar_name]