class Pyrosample(object):
"""A Pyrosample is a legacy object for the few 454 samples we still have and that we need to compare against the new Illumina technology."""
all_paths = """
/info.json
/reads.fasta
/renamed.fasta
/raw/raw.sff
/raw/raw.fastq
/raw/raw.fasta
/raw/raw.qual
/raw/manifest.txt
/fastq/reads.fastq
"""
kind = "pyrosample"
def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name)
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = FilePath(json_path)
# Parse #
self.info = load_json_path(self.json_path)
# Basic #
self.account = "/dev/null"
self.run_num = self.info['run_num']
self.run_label = "pyrosample_run_%i" % self.run_num
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
# Own attributes #
self.num = self.info['sample_num']
self.short_name = self.info['sample']
self.long_name = self.info['sample_name']
self.name = 'run%i_sample%i' % (self.run_num, self.num)
self.group = self.info['group']
self.id_name = "run%03d-sample%02d" % (self.run_num, self.num)
# Hard coded attributes #
self.machine = "454 GS FLX Titanium"
# SFF files #
self.sff_files_info = self.info['files']
# Pool dummy #
self.pool, self.parent = self, self
# Other dummy variables #
self.bar_len = 0
self.gzipped = False
self.used = True
# Loaded #
self.loaded = False
def load(self):
"""A second __init__ that is delayed and called only if needed"""
# Check files are there #
for f in self.sff_files_info:
if not os.path.exists(f['path']): raise Exception("No file at %s" % f['path'])
# Automatic paths #
self.base_dir = self.out_dir + self.id_name + '/'
self.p = AutoPaths(self.base_dir, self.all_paths)
# Make an alias to the json #
self.p.info_json.link_from(self.json_path, safe=True)
# Primer #
self.primer_regex = re.compile(self.info['primer'])
# Raw files #
self.raw_fasta = FASTA(self.p.raw_fasta)
self.raw_fastq = FASTQ(self.p.raw_fastq)
# Standard FASTA #
self.reads = FASTA(self.p.reads_fasta)
self.fasta = FASTA(self.p.renamed)
# Special FASTQ #
self.fastq = FASTQ(self.p.reads_fastq)
# A shameless hack for cdhit to work #
self.renamed = self.fastq
# Pre-denoised special case #
if self.info['predenoised'] and False:
self.sff_files_info = []
self.reads.link_from(self.info['predenoised'], safe=True)
# Special submission attributes #
self.sra = PyroSampleSRA(self)
# Loaded #
self.loaded = True
# Return self for convenience #
return self
@property
def mate(self):
if not 'mate' in self.info: return False
run_num = self.info['mate']['run']
pool_num = self.info['mate']['pool']
barcode_num = self.info['mate']['num']
return illumitag.runs[run_num][pool_num-1][barcode_num-1]
def extract(self):
# Call extraction #
shell_output('sffinfo -s %s > %s' % (self.p.raw_sff, self.p.raw_fasta))
shell_output('sffinfo -q %s > %s' % (self.p.raw_sff, self.p.raw_qual))
shell_output('sffinfo -m %s > %s' % (self.p.raw_sff, self.p.manifest))
# Convert #
sh.fasta_to_fastq(self.p.raw_fasta, self.p.raw_qual, self.p.raw_fastq)
def clean_iterator(self, reads, minlength=400, threshold=21, windowsize=20):
for read in reads:
# Length #
if len(read) < minlength: continue
# Primer #
match = self.primer_regex.search(str(read.seq))
if not match: continue
# PHRED score #
scores = read.letter_annotations["phred_quality"]
averaged = moving_average(scores, windowsize)
discard = False
for i,value in enumerate(averaged):
if value < threshold:
read = read[:i+windowsize-1]
if len(read) < minlength: discard = True
break
if discard: continue
# Undetermined bases #
if 'N' in read: continue
# Remove primer #
read = read[match.end():]
# Flip them because 454 reads the other end #
read = read.reverse_complement()
# Return #
yield read
def clean(self, **kwargs):
self.reads.write(self.clean_iterator(self.raw_fastq, **kwargs))
def report_loss(self):
print "Before cleaning: %i" % len(self.raw_fastq)
print "After cleaning: %i" % len(self.reads)
print "Loss: %.2f%%" % (100 * (1 - (len(self.raw_fastq)/len(self.reads))))
def process(self):
self.reads.rename_with_num(self.name + '_read', new_path=self.fasta)
def make_fastq(self, **kwargs):
"""In some special cases we want the FASTQ"""
self.fastq.write(self.clean_iterator(self.raw_fastq, **kwargs))
self.fastq.rename_with_num(self.name + '_read')
print "make_fastq for sample %s completed" % self.id_name
class Presample(BarcodeGroup):
"""A Presample is a clumsy name for a new type of barcoded-sequence files.
As we updated the lab protocol, sample are not multiplexed with
our traditional 50 barcodes anymore, but with Illumina specific MIDs.
The demultiplexing thus happens in their pipeline and we are left with one
sample per file.
This object is a bit like a *Pool*, a *BarcodeGroup* and a *Sample*
all at the same time. In the end it inherits from BarcodeGroup and
just emulates the behavior of the other objects."""
all_paths = """
/info.json
/uncompressed/fwd.fastq
/uncompressed/rev.fastq
/logs/
/assembled/
/unassembled/
/fastqc/
/graphs/
/report/report.pdf
/quality/trimmed.fastq
/quality/renamed.fastq
/quality/reads.fasta
"""
kind = 'presample'
def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name)
def __str__(self): return self.id_name
def __iter__(self): return iter(self.children)
def __len__(self): return self.count
def __getitem__(self, key): return self.samples[key]
@property
def seq_len(self): return len(self.fwd.first_read)
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = FilePath(json_path)
# Parse #
self.info = load_json_path(self.json_path)
# Basic #
self.account = self.info['uppmax_id']
self.run_num = self.info['run_num']
self.run_label = self.info['run_id']
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
self.fwd_name = self.info['forward_reads']
self.rev_name = self.info['reverse_reads']
# Own attributes #
self.num = self.info['sample_num']
self.label = self.info['sample_id']
self.short_name = self.info['sample']
self.long_name = self.info['sample_name']
self.name = 'run%i_sample%i' % (self.run_num, self.num)
self.group = self.info.get('group')
self.id_name = "run%03d-sample%02d" % (self.run_num, self.num)
self.fwd_mid = self.info['forward_mid']
self.rev_mid = self.info['reverse_mid']
self.used = True
# Check name is ASCII #
assert all(ord(c) < 128 for c in self.short_name)
# Pool dummy #
self.pool, self.parent = self, self
# Second init #
self.loaded = False
def load(self):
"""A second __init__ that is delayed and called only if needed"""
# Automatic paths #
self.base_dir = DirectoryPath(self.out_dir + self.id_name + '/')
self.p = AutoPaths(self.base_dir, self.all_paths)
# Special #
self.primers = TwoPrimers(self)
# Samples dummy #
self.info['samples'] = [{"name":self.short_name, "used":1, "group":self.group,
"dummy":1, "num":self.num, "fwd":"", "rev":""}]
self.samples = Samples(self)
self.samples.load()
self.loaded = True
# Files #
if not os.access('/proj/%s' % self.account, os.R_OK): return
self.fwd_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name)
self.rev_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name)
self.gzipped = True if self.fwd_path.endswith('gz') else False
self.fwd = FASTQ(self.fwd_path)
self.rev = FASTQ(self.rev_path)
self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
# Barcode length #
self.bar_len = 0
# Make an alias to the json #
self.p.info_json.link_from(self.json_path, safe=True)
# Assembly files as children #
self.assembled = Assembled('', self)
self.unassembled = Unassembled('', self)
self.children = (self.assembled, self.unassembled)
self.first = self.assembled
# Special case, for when the two reads don't join #
self.trim_and_concat = TrimerAndConcactenater(self)
# Final #
self.trimmed = FASTQ(self.p.trimmed)
self.renamed = FASTQ(self.p.renamed)
self.fasta = FASTA(self.p.reads_fasta)
# Graphs #
self.graphs = [getattr(outcome_plots, cls_name)(self) for cls_name in outcome_plots.__all__]
# Runner #
self.runner = PresampleRunner(self)
# Diversity #
self.diversity = AlphaDiversity(self)
# Report #
self.report = SampleReport(self)
# Loaded #
self.loaded = True
# Return self for convenience #
return self
@property_cached
def counts(self):
"""The OTU counts"""
taxa_table = self.project.cluster.otus.taxonomy.comp_tips.taxa_table
row = taxa_table.loc[self.short_name].copy()
row.sort(ascending=False)
return row
def join(self):
"""Uses pandaseq 2.7 to join the foward and reverse reads together.
See https://github.com/neufeld/pandaseq"""
# Special case for new primers that don't join #
rev_primer_name = self.info['primers']['reverse']['name']
not_joining_primers = ("1132R", "1000R")
if rev_primer_name in not_joining_primers:
print "No overlap special case"
self.trim_and_concat.run()
return
# Special case for primers that highly overlap #
high_overlap_primers = ("806R",)
if rev_primer_name in high_overlap_primers:
print "High overlap special case, using mothur"
result = sh.mothur("#make.contigs(ffastq=%s, rfastq=%s);" % (self.uncomrpessed_pair.fwd, self.uncomrpessed_pair.rev))
if "ERROR" in result.stdout: raise Exception("Mothur didn't run correctly")
# Move things #
#shutil.move(self.tax.centers.prefix_path + '.align', self.mothur_aligned)
#shutil.move(self.tax.centers.prefix_path + '.align.report', self.p.mothur_report)
return
# Default case #
command = 'pandaseq27 -T 1 -f %s -r %s -u %s -F 1> %s 2> %s'
command = command % (self.fwd, self.rev, self.unassembled.path, self.assembled.path, self.assembled.p.out)
shell_call(command) # Because it exits with status 1 https://github.com/neufeld/pandaseq/issues/40
def process(self):
"""Lorem"""
def no_primers_iterator(reads):
for read_w_miss in reads:
yield read_w_miss.read[read_w_miss.fwd_end_pos:read_w_miss.rev_end_pos]
reads = self.assembled.good_primers.len_filtered.parse_primers(mismatches=1)
self.trimmed.write(no_primers_iterator(reads))
self.trimmed.rename_with_num(self.name + '_read', self.renamed)
self.renamed.to_fasta(self.fasta)
def make_mothur_output(self):
pass
def make_qiime_output(self):
pass
def make_presample_plots(self):
for graph in self.graphs: graph.plot()
@property_cached
def uncomrpessed_pair(self):
"""Usefull for a few stupid programs that don't take fastq.gz files such as mothur"""
result = PairedFASTQ(self.p.uncompressed_fwd, self.p.uncompressed_rev)
if not result.exists:
self.fwd.ungzip_to(result.fwd)
self.rev.ungzip_to(result.rev)
return result
class Sample(FASTQ):
"""All sequences with the same barcode pair grouped together"""
all_paths = """
/orig.fastq
/trimmed.fastq
/renamed.fastq
/reads.fasta
/raw/fwd.fastq
/raw/rev.fastq
/raw/forward.fastq.gz
/raw/reverse.fastq.gz
"""
kind = 'sample'
def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.name)
def __str__(self): return self.bar_name
def __init__(self, info, parent):
# Save attributes #
self.info = info
self.parent = parent
self.pool = parent.pool
# Basic #
self.short_name = info['name']
self.group_name = info.get('group')
self.num = int(info['num'])
self.used = bool(info['used'])
self.fwd_str = info['fwd']
self.rev_str = info['rev']
# Other #
self.bar_name = 'barcode%i' % self.num
self.name = 'run%i_pool%i_sample%i' % (self.pool.run_num, self.pool.num, self.num)
# Special submission attributes #
self.sra = SampleSRA(self)
# Second init #
self.loaded = False
def load(self):
# Special case for dummy samples #
if self.info.get('dummy'): return
# Paths #
self.base_dir = self.pool.p.samples_dir + self.bar_name + '/'
self.p = AutoPaths(self.base_dir, self.all_paths)
self.path = str(self.p.orig_fastq)
# Distances #
self.trim_fwd = self.pool.samples.trim_fwd
self.trim_rev = self.pool.samples.trim_rev
# Files #
self.trimmed = FASTQ(self.p.trimmed)
self.renamed = FASTQ(self.p.renamed)
self.fasta = FASTA(self.p.reads_fasta)
self.raw = PairedFASTQ(self.p.raw_fwd, self.p.raw_rev, self.pool)
self.raw_gz = PairedFASTQ(self.p.raw_forward_gz, self.p.raw_reverse_gz, self.pool)
# Inherit #
self.project = self.pool.project
# Loaded #
self.loaded = True
def process(self):
def no_primers_iterator(reads):
for read in reads:
yield read[self.trim_fwd:-self.trim_rev]
self.trimmed.write(no_primers_iterator(self))
self.trimmed.rename_with_num(self.name + '_read', self.renamed)
self.renamed.to_fasta(self.fasta)
assert self.count == self.fasta.count
def combine_rerun_with_orig(self):
"""Special case when a sample with low reads was rerun in an other pool.
Run this just before the combine_reads() method of the associated cluster.
This method is called on the reruned sampled, not the original."""
# Check we have a rerun #
if self.info.get('rerun') is None: return False
# Check we are processed #
assert self.fasta.count > 0
# Get the original sample #
run, pool, num = self.info['rerun']['run'], self.info['rerun']['pool'], self.info['rerun']['num']
orig_sample = illumitag.runs[run][pool-1][num-1]
merged = FASTA(orig_sample.base_dir + 'rerun_merged.fasta')
# Check we don't merge twice #
assert orig_sample.count == orig_sample.fasta.count
# Do it #
merged.create()
merged.add(orig_sample.fasta)
merged.add(self.fasta)
merged.close()
merged.rename_with_num(orig_sample.name + '_read', orig_sample.fasta)
merged.remove()
# Check #
orig_sample.fasta = FASTA(orig_sample.fasta.path)
assert orig_sample.count < orig_sample.fasta.count
return True
@property
def json(self):
"""Regenerate the JSON string from the object including extra info"""
result = OrderedDict([(k, self.info[k]) for k in ('name', 'used', 'group', 'num', 'fwd', 'rev')])
result = json.dumps(result)
if self.extra_metadata:
result = result[:-1] + ',' + json.dumps(self.extra_metadata, indent=4)[1:]
result = re.compile(r'\bNaN\b').sub('null', result)
return result
@property
def count_raw_reads(self):
"""The number of reads the sample originally had right after barcode processing and before any other quality filtering"""
return self.pool.good_barcodes.breakdown[self.bar_name]