def get_read_pairs(self):
        data_fns = glob.glob(self.data_dir + "/*.fastq") + glob.glob(self.data_dir + "/*.fq")
        fn_pairs = defaultdict(lambda: {1: None, 2: None})
        for data_fn in data_fns:
            head, tail = os.path.split(data_fn)
            root, ext = os.path.splitext(tail)
            # Expect root to end in either 1 or 2
            prefix, which_member = root[:-1], int(root[-1])
            fn_pairs[prefix][which_member] = data_fn

        read_pairs_list = []

        for prefix in sorted(fn_pairs):
            R1_fn = fn_pairs[prefix][1]
            R2_fn = fn_pairs[prefix][2]

            if R1_fn == None or R2_fn == None:
                raise ValueError("unpaired file names in data_dir")

            R1_lines = split_file.piece(R1_fn, self.num_pieces, self.which_piece, "fastq")
            R2_lines = split_file.piece(R2_fn, self.num_pieces, self.which_piece, "fastq")
            read_pairs = fastq.read_pairs(R1_lines, R2_lines, standardize_names=True, ensure_sanger_encoding=True)
            read_pairs_list.append(read_pairs)

        all_read_pairs = chain.from_iterable(read_pairs_list)

        return all_read_pairs
    def get_reads(self):
        ''' A generator over the reads in a piece of each data file.
            Can handle a mixture of different fastq encodings across (but not
            within) files.
        '''
        total_reads = 0
        for file_name in self.data_fns:
            total_reads_from_file = 0
            file_piece = split_file.piece(
                file_name,
                self.num_pieces,
                self.which_piece,
                'fastq',
            )
            for read in fastq.reads(file_piece,
                                    standardize_names=True,
                                    ensure_sanger_encoding=True):
                yield read

                total_reads += 1
                total_reads_from_file += 1
                if total_reads % 10000 == 0:
                    logging.info('{0:,} reads processed'.format(total_reads))

            head, tail = os.path.split(file_name)
            self.summary.append(
                ('Reads in {0}'.format(tail), total_reads_from_file))

        logging.info('{0:,} total reads processed'.format(total_reads))

        self.summary.append(('Total reads', total_reads))
    def get_sorted_sam_lines(self):
        ''' Get this piece's set of SAM lines from the merged SAM file, ensuring
            that the piece boundary doesn't break up a set of mappings to the
            same position.
        '''
        get_position = annotation.make_convertor(self.MappingAnnotation,
                                                 self.PositionAnnotation,
                                                )

        sam_lines = split_file.piece(self.merged_file_names['sorted_clean_sam'],
                                     self.num_pieces,
                                     self.which_piece,
                                     'sam',
                                     key=get_position,
                                    )
        return sam_lines
    def get_reads(self):
        """ A generator over the reads in a piece of each data file.
            Can handle a mixture of different fastq encodings across (but not
            within) files.
        """
        total_reads = 0
        for file_name in self.data_fns:
            total_reads_from_file = 0
            file_piece = split_file.piece(file_name, self.num_pieces, self.which_piece, "fastq")
            for read in fastq.reads(file_piece, standardize_names=True, ensure_sanger_encoding=True):
                yield read

                total_reads += 1
                total_reads_from_file += 1
                if total_reads % 10000 == 0:
                    logging.info("{0:,} reads processed".format(total_reads))

            head, tail = os.path.split(file_name)
            self.summary.append(("Reads in {0}".format(tail), total_reads_from_file))

        logging.info("{0:,} total reads processed".format(total_reads))

        self.summary.append(("Total reads", total_reads))