示例#1
0
 def load(self):
     """A second __init__ that is delayed and called only if needed"""
     # Automatic paths #
     self.base_dir = DirectoryPath(self.out_dir + self.id_name + '/')
     self.p = AutoPaths(self.base_dir, self.all_paths)
     # Special #
     self.primers = TwoPrimers(self)
     # Samples dummy #
     self.info['samples'] = [{"name":self.short_name, "used":1, "group":self.group,
                              "dummy":1, "num":self.num, "fwd":"", "rev":""}]
     self.samples = Samples(self)
     self.samples.load()
     self.loaded = True
     # Files #
     if not os.access('/proj/%s' % self.account, os.R_OK): return
     self.fwd_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name)
     self.rev_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name)
     self.gzipped = True if self.fwd_path.endswith('gz') else False
     self.fwd = FASTQ(self.fwd_path)
     self.rev = FASTQ(self.rev_path)
     self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
     # Barcode length #
     self.bar_len = 0
     # Make an alias to the json #
     self.p.info_json.link_from(self.json_path, safe=True)
     # Assembly files as children #
     self.assembled = Assembled('', self)
     self.unassembled = Unassembled('', self)
     self.children = (self.assembled, self.unassembled)
     self.first = self.assembled
     # Special case, for when the two reads don't join #
     self.trim_and_concat = TrimerAndConcactenater(self)
     # Final #
     self.trimmed = FASTQ(self.p.trimmed)
     self.renamed = FASTQ(self.p.renamed)
     self.fasta = FASTA(self.p.reads_fasta)
     # Graphs #
     self.graphs = [getattr(outcome_plots, cls_name)(self) for cls_name in outcome_plots.__all__]
     # Runner #
     self.runner = PresampleRunner(self)
     # Diversity #
     self.diversity = AlphaDiversity(self)
     # Report #
     self.report = SampleReport(self)
     # Loaded #
     self.loaded = True
     # Return self for convenience #
     return self
示例#2
0
 def __init__(self, json_path, out_dir):
     # Attributes #
     self.out_dir = out_dir
     self.json_path = FilePath(json_path)
     # Parse #
     with open(json_path) as handle:
         self.info = json.load(handle)
     # Basic #
     self.account = self.info['uppmax_id']
     self.run_num = self.info['run_num']
     self.run_label = self.info['run_id']
     self.project_short_name = self.info['project']
     self.project_long_name = self.info['project_name']
     self.fwd_name = self.info['forward_reads']
     self.rev_name = self.info['reverse_reads']
     # Own attributes #
     self.num = self.info['sample_num']
     self.label = self.info['sample_id']
     self.short_name = self.info['sample']
     self.long_name = self.info['sample_name']
     self.name = 'run%i_sample%i' % (self.run_num, self.num)
     self.group = self.info['group']
     self.id_name = "run%03d-sample%02d" % (self.run_num, self.num)
     self.fwd_mid = self.info['forward_mid']
     self.rev_mid = self.info['forward_mid']
     # Automatic paths #
     self.base_dir = self.out_dir + self.id_name + '/'
     self.p = AutoPaths(self.base_dir, self.all_paths)
     # Special #
     self.primers = TwoPrimers(self)
     # Samples dummy #
     self.info['samples'] = [{
         "name": self.short_name,
         "used": 1,
         "group": self.group,
         "dummy": 1,
         "num": self.num,
         "fwd": "",
         "rev": ""
     }]
     self.samples = Samples(self)
     self.samples.load()
     # Pool dummy #
     self.pool, self.parent = self, self
     # Files #
     self.fwd_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (
         self.run_label, self.label, self.fwd_name)
     self.rev_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (
         self.run_label, self.label, self.rev_name)
     self.gziped = True if self.fwd_path.endswith('gz') else False
     self.fwd = FASTQ(self.p.fwd)
     self.rev = FASTQ(self.p.rev)
     self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
     # Barcode length #
     self.bar_len = 0
     # Make an alias to the json #
     self.p.info_json.link_from(self.json_path, safe=True)
     # Assembly files as children #
     self.assembled = Assembled(self)
     self.unassembled = Unassembled(self)
     self.children = (self.assembled, self.unassembled)
     self.first = self.assembled
     # Graphs #
     self.graphs = [
         getattr(outcome_plots, cls_name)(self)
         for cls_name in outcome_plots.__all__
     ]
     # Runner #
     self.runner = PresampleRunner(self)
     # Final #
     self.trimmed = FASTQ(self.p.trimmed)
     self.renamed = FASTQ(self.p.renamed)
     self.fasta = FASTA(self.p.reads_fasta)
示例#3
0
class Presample(BarcodeGroup):
    """A Presample is a clumsy name for a new type of barcoded-sequence files.
    As we updated the lab protocol, sample are not multiplexed with
    our traditional 50 barcodes anymore, but with Illumina specific MIDs.
    The demultiplexing thus happens in their pipeline and we are left with one
    sample per file.
    This object is a bit like a *Pool*, a *BarcodeGroup* and a *Sample*
    all at the same time. In the end it inherits from BarcodeGroup and
    just emulates the behavior of the other objects."""

    all_paths = """
    /info.json
    /uncompressed/fwd.fastq
    /uncompressed/rev.fastq
    /logs/
    /assembled/
    /unassembled/
    /fastqc/
    /graphs/
    /report/report.pdf
    /quality/trimmed.fastq
    /quality/renamed.fastq
    /quality/reads.fasta
    """

    kind = 'presample'

    def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name)
    def __str__(self): return self.id_name
    def __iter__(self): return iter(self.children)
    def __len__(self): return self.count
    def __getitem__(self, key): return self.samples[key]

    @property
    def seq_len(self): return len(self.fwd.first_read)

    def __init__(self, json_path, out_dir):
        # Attributes #
        self.out_dir = out_dir
        self.json_path = FilePath(json_path)
        # Parse #
        self.info = load_json_path(self.json_path)
        # Basic #
        self.account = self.info['uppmax_id']
        self.run_num = self.info['run_num']
        self.run_label = self.info['run_id']
        self.project_short_name = self.info['project']
        self.project_long_name = self.info['project_name']
        self.fwd_name = self.info['forward_reads']
        self.rev_name = self.info['reverse_reads']
        # Own attributes #
        self.num = self.info['sample_num']
        self.label = self.info['sample_id']
        self.short_name = self.info['sample']
        self.long_name = self.info['sample_name']
        self.name = 'run%i_sample%i' % (self.run_num, self.num)
        self.group = self.info.get('group')
        self.id_name = "run%03d-sample%02d" % (self.run_num, self.num)
        self.fwd_mid = self.info['forward_mid']
        self.rev_mid = self.info['reverse_mid']
        self.used = True
        # Check name is ASCII #
        assert all(ord(c) < 128 for c in self.short_name)
        # Pool dummy #
        self.pool, self.parent = self, self
        # Second init #
        self.loaded = False

    def load(self):
        """A second __init__ that is delayed and called only if needed"""
        # Automatic paths #
        self.base_dir = DirectoryPath(self.out_dir + self.id_name + '/')
        self.p = AutoPaths(self.base_dir, self.all_paths)
        # Special #
        self.primers = TwoPrimers(self)
        # Samples dummy #
        self.info['samples'] = [{"name":self.short_name, "used":1, "group":self.group,
                                 "dummy":1, "num":self.num, "fwd":"", "rev":""}]
        self.samples = Samples(self)
        self.samples.load()
        self.loaded = True
        # Files #
        if not os.access('/proj/%s' % self.account, os.R_OK): return
        self.fwd_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name)
        self.rev_path = home + "proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name)
        self.gzipped = True if self.fwd_path.endswith('gz') else False
        self.fwd = FASTQ(self.fwd_path)
        self.rev = FASTQ(self.rev_path)
        self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
        # Barcode length #
        self.bar_len = 0
        # Make an alias to the json #
        self.p.info_json.link_from(self.json_path, safe=True)
        # Assembly files as children #
        self.assembled = Assembled('', self)
        self.unassembled = Unassembled('', self)
        self.children = (self.assembled, self.unassembled)
        self.first = self.assembled
        # Special case, for when the two reads don't join #
        self.trim_and_concat = TrimerAndConcactenater(self)
        # Final #
        self.trimmed = FASTQ(self.p.trimmed)
        self.renamed = FASTQ(self.p.renamed)
        self.fasta = FASTA(self.p.reads_fasta)
        # Graphs #
        self.graphs = [getattr(outcome_plots, cls_name)(self) for cls_name in outcome_plots.__all__]
        # Runner #
        self.runner = PresampleRunner(self)
        # Diversity #
        self.diversity = AlphaDiversity(self)
        # Report #
        self.report = SampleReport(self)
        # Loaded #
        self.loaded = True
        # Return self for convenience #
        return self

    @property_cached
    def counts(self):
        """The OTU counts"""
        taxa_table = self.project.cluster.otus.taxonomy.comp_tips.taxa_table
        row = taxa_table.loc[self.short_name].copy()
        row.sort(ascending=False)
        return row

    def join(self):
        """Uses pandaseq 2.7 to join the foward and reverse reads together.
        See https://github.com/neufeld/pandaseq"""
        # Special case for new primers that don't join #
        rev_primer_name = self.info['primers']['reverse']['name']
        not_joining_primers = ("1132R", "1000R")
        if rev_primer_name in not_joining_primers:
            print "No overlap special case"
            self.trim_and_concat.run()
            return
        # Special case for primers that highly overlap #
        high_overlap_primers = ("806R",)
        if rev_primer_name in high_overlap_primers:
            print "High overlap special case, using mothur"
            result = sh.mothur("#make.contigs(ffastq=%s, rfastq=%s);" % (self.uncomrpessed_pair.fwd, self.uncomrpessed_pair.rev))
            if "ERROR" in result.stdout: raise Exception("Mothur didn't run correctly")
            # Move things #
            #shutil.move(self.tax.centers.prefix_path + '.align', self.mothur_aligned)
            #shutil.move(self.tax.centers.prefix_path + '.align.report', self.p.mothur_report)
            return
        # Default case #
        command = 'pandaseq27 -T 1 -f %s -r %s -u %s -F 1> %s 2> %s'
        command = command % (self.fwd, self.rev, self.unassembled.path, self.assembled.path, self.assembled.p.out)
        shell_call(command) # Because it exits with status 1 https://github.com/neufeld/pandaseq/issues/40

    def process(self):
        """Lorem"""
        def no_primers_iterator(reads):
            for read_w_miss in reads:
                yield read_w_miss.read[read_w_miss.fwd_end_pos:read_w_miss.rev_end_pos]
        reads = self.assembled.good_primers.len_filtered.parse_primers(mismatches=1)
        self.trimmed.write(no_primers_iterator(reads))
        self.trimmed.rename_with_num(self.name + '_read', self.renamed)
        self.renamed.to_fasta(self.fasta)

    def make_mothur_output(self):
        pass

    def make_qiime_output(self):
        pass

    def make_presample_plots(self):
        for graph in self.graphs: graph.plot()

    @property_cached
    def uncomrpessed_pair(self):
        """Usefull for a few stupid programs that don't take fastq.gz files such as mothur"""
        result = PairedFASTQ(self.p.uncompressed_fwd, self.p.uncompressed_rev)
        if not result.exists:
            self.fwd.ungzip_to(result.fwd)
            self.rev.ungzip_to(result.rev)
        return result
示例#4
0
class Presample(BarcodeGroup):
    """A Presample is a clumsy name for a new type of barcoded-sequence files.
    As we updated the lab protocol, sample are not multiplexed with
    our traditional 50 barcodes anymore, but with Illumina specific MIDs.
    The demultiplexing thus happens in their pipeline and we are left with one
    sample per file.
    This object is a bit like a *Pool*, a *BarcodeGroup* and a *Sample*."""

    all_paths = """
    /logs/
    /graphs/
    /info.json
    /fwd.fastq
    /rev.fastq
    /quality/trimmed.fastq
    /quality/renamed.fastq
    /quality/reads.fasta
    /assembled/
    /unassembled/
    /fastqc/
    """

    def __repr__(self):
        return '<%s object "%s">' % (self.__class__.__name__, self.id_name)

    def __str__(self):
        return self.id_name

    def __iter__(self):
        return iter(self.children)

    def __len__(self):
        return self.count

    def __getitem__(self, key):
        return self.samples[key]

    @property
    def seq_len(self):
        return len(self.fwd.first_read)

    def __init__(self, json_path, out_dir):
        # Attributes #
        self.out_dir = out_dir
        self.json_path = FilePath(json_path)
        # Parse #
        with open(json_path) as handle:
            self.info = json.load(handle)
        # Basic #
        self.account = self.info['uppmax_id']
        self.run_num = self.info['run_num']
        self.run_label = self.info['run_id']
        self.project_short_name = self.info['project']
        self.project_long_name = self.info['project_name']
        self.fwd_name = self.info['forward_reads']
        self.rev_name = self.info['reverse_reads']
        # Own attributes #
        self.num = self.info['sample_num']
        self.label = self.info['sample_id']
        self.short_name = self.info['sample']
        self.long_name = self.info['sample_name']
        self.name = 'run%i_sample%i' % (self.run_num, self.num)
        self.group = self.info['group']
        self.id_name = "run%03d-sample%02d" % (self.run_num, self.num)
        self.fwd_mid = self.info['forward_mid']
        self.rev_mid = self.info['forward_mid']
        # Automatic paths #
        self.base_dir = self.out_dir + self.id_name + '/'
        self.p = AutoPaths(self.base_dir, self.all_paths)
        # Special #
        self.primers = TwoPrimers(self)
        # Samples dummy #
        self.info['samples'] = [{
            "name": self.short_name,
            "used": 1,
            "group": self.group,
            "dummy": 1,
            "num": self.num,
            "fwd": "",
            "rev": ""
        }]
        self.samples = Samples(self)
        self.samples.load()
        # Pool dummy #
        self.pool, self.parent = self, self
        # Files #
        self.fwd_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (
            self.run_label, self.label, self.fwd_name)
        self.rev_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (
            self.run_label, self.label, self.rev_name)
        self.gziped = True if self.fwd_path.endswith('gz') else False
        self.fwd = FASTQ(self.p.fwd)
        self.rev = FASTQ(self.p.rev)
        self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
        # Barcode length #
        self.bar_len = 0
        # Make an alias to the json #
        self.p.info_json.link_from(self.json_path, safe=True)
        # Assembly files as children #
        self.assembled = Assembled(self)
        self.unassembled = Unassembled(self)
        self.children = (self.assembled, self.unassembled)
        self.first = self.assembled
        # Graphs #
        self.graphs = [
            getattr(outcome_plots, cls_name)(self)
            for cls_name in outcome_plots.__all__
        ]
        # Runner #
        self.runner = PresampleRunner(self)
        # Final #
        self.trimmed = FASTQ(self.p.trimmed)
        self.renamed = FASTQ(self.p.renamed)
        self.fasta = FASTA(self.p.reads_fasta)

    def load(self):
        pass

    def uncompress(self):
        shell_output('gunzip -c %s > %s' % (self.fwd_path, self.fwd))
        shell_output('gunzip -c %s > %s' % (self.rev_path, self.rev))

    def presample_fastqc(self):
        self.fastq.fastqc(self.p.fastqc_dir)

    def process(self):
        def no_primers_iterator(reads):
            for read in reads:
                yield read[self.primers.fwd_len:-self.primers.rev_len]

        reads = self.assembled.good_primers.len_filtered
        self.trimmed.write(no_primers_iterator(reads))
        self.trimmed.rename_with_num(self.name + '_read', self.renamed)
        self.renamed.to_fasta(self.fasta)

    def make_mothur_output(self):
        pass

    def make_qiime_output(self):
        pass

    def make_presample_plots(self):
        for graph in self.graphs:
            graph.plot()