Example #1
0
def test_append():
	for ext in ["", ".gz"]:  # BZ2 does NOT support append
		text = "AB"
		if ext != "":
			text = text.encode("utf-8")  # On Py3, need to send BYTES, not unicode
		reference = text + text
		print("Trying ext=%s" % ext)
		with temporary_path('truncated.fastq' + ext) as path:
			try:
				os.unlink(path)
			except OSError:
				pass
			with xopen(path, 'a') as f:
				f.write(text)
			with xopen(path, 'a') as f:
				f.write(text)
			with xopen(path, 'r') as f:
				for appended in f:
					pass
				try:
					reference = reference.decode("utf-8")
				except AttributeError:
					pass
				print(appended)
				print(reference)
				assert appended == reference
Example #2
0
def test_append():
    for ext in ["", ".gz"]:  # BZ2 does NOT support append
        text = "AB"
        if ext != "":
            text = text.encode(
                "utf-8")  # On Py3, need to send BYTES, not unicode
        reference = text + text
        print("Trying ext=%s" % ext)
        with temporary_path('truncated.fastq' + ext) as path:
            try:
                os.unlink(path)
            except OSError:
                pass
            with xopen(path, 'a') as f:
                f.write(text)
            with xopen(path, 'a') as f:
                f.write(text)
            with xopen(path, 'r') as f:
                for appended in f:
                    pass
                try:
                    reference = reference.decode("utf-8")
                except AttributeError:
                    pass
                print(appended)
                print(reference)
                assert appended == reference
Example #3
0
def test_xopen():
	f = xopen(uncompressed)
	assert f.readline() == '# a comment\n'
	f.close()

	f = xopen(compressed)
	assert f.readline() == '@first_sequence\n'
	f.close()
 def __call__(self, read1, read2=None):
     if read2 is None:
         if read1.match is None:
             if self.untrimmed_outfile is None and self.untrimmed_path is not None:
                 self.untrimmed_outfile = xopen(self.untrimmed_path, "w")
             if self.untrimmed_outfile is not None:
                 read1.write(self.untrimmed_outfile)
         else:
             name = read1.match.adapter.name
             if name not in self.files:
                 self.files[name] = xopen(self.template.format(name=name), "w")
             read1.write(self.files[name])
     else:
         assert False, "Not supported"  # pragma: no cover
Example #5
0
def trimmed_and_untrimmed_files(
		default_output,
		output_path,
		untrimmed_path,
		discard_trimmed,
		discard_untrimmed
		):
	"""
	Figure out (from command-line parameters) where trimmed and untrimmed reads
	should be written.

	Return a pair (trimmed, untrimmed). The values are either open file-like
	objects or None, in which case no output should be produced. The objects may
	be identical (for example: (sys.stdout, sys.stdout)).

	The parameters are sorted: Later parameters have higher precedence.

	default_output -- If nothing else is specified below, this file-like object
		is returned for both trimmed and untrimmed output.
	output_path -- Path to output file for both trimmed and untrimmed output.
	untrimmed_path -- Path to an output file for untrimmed reads.
	discard_trimmed -- bool, override earlier options.
	discard_untrimmed -- bool, overrides earlier options.
	"""
	
	if discard_trimmed:
		if discard_untrimmed:
			untrimmed = None
		elif untrimmed_path is not None:
			untrimmed = xopen(untrimmed_path, 'w')
		elif output_path is not None:
			untrimmed = xopen(output_path, 'w')
		else:
			untrimmed = default_output
		return (None, untrimmed)
	if discard_untrimmed:
		trimmed = default_output
		if output_path is not None:
			trimmed = xopen(output_path, 'w')
		return (trimmed, None)
	
	trimmed = default_output
	untrimmed = default_output
	if output_path is not None:
		trimmed = untrimmed = xopen(output_path, 'w')
	if untrimmed_path is not None:
		untrimmed = xopen(untrimmed_path, 'w')
	
	return (trimmed, untrimmed)
 def __call__(self, read1, read2=None):
     if read2 is None:
         if read1.match is None:
             if self.untrimmed_outfile is None and self.untrimmed_path is not None:
                 self.untrimmed_outfile = xopen(self.untrimmed_path, 'w')
             if self.untrimmed_outfile is not None:
                 read1.write(self.untrimmed_outfile)
         else:
             name = read1.match.adapter.name
             if name not in self.files:
                 self.files[name] = xopen(self.template.format(name=name),
                                          'w')
             read1.write(self.files[name])
     else:
         assert False, "Not supported"  # pragma: no cover
Example #7
0
def test_xopen():
	for name in files:
		f = xopen(name)
		lines = list(f)
		assert len(lines) == 12
		assert lines[5] == 'AGCCGCTANGACGGGTTGGCCCTTAGACGTATCT\n', name
		f.close()
Example #8
0
	def test_truncated_gz_iter():
		with temporary_path('truncated.gz') as path:
			create_truncated_file(path)
			f = xopen(path, 'r')
			for line in f:
				pass
			f.close()
Example #9
0
 def test_truncated_gz_iter():
     with temporary_path('truncated.gz') as path:
         create_truncated_file(path)
         f = xopen(path, 'r')
         for line in f:
             pass
         f.close()
Example #10
0
def test_xopen_binary():
    for name in files:
        f = xopen(name, 'rb')
        lines = list(f)
        assert len(lines) == 12
        assert lines[5] == b'AGCCGCTANGACGGGTTGGCCCTTAGACGTATCT\n', name
        f.close()
Example #11
0
def create_truncated_file(path):
	# Random text
	text = ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(200))
	f = xopen(path, 'w')
	f.write(text)
	f.close()
	f = open(path, 'a')
	f.truncate(os.stat(path).st_size - 10)
	f.close()
Example #12
0
    def __init__(self, file, line_length=None):
        """
		If line_length is not None, the lines will
		be wrapped after line_length characters.
		"""
        self.line_length = line_length if line_length != 0 else None
        if isinstance(file, str):
            file = xopen(file, 'w')
            self._close_on_exit = True
        self._file = file
Example #13
0
 def __init__(self, *args, **kwargs):
     super(GZipMixin, self).__init__(*args, **kwargs)
     file = args[0]
     if hasattr(file, 'read') and hasattr(file, 'name') and splitext(
             file.name)[1].lower() in ('.bz2', '.xz', '.gz'):
         file = file.name
     if isinstance(file, basestring):
         file = xopen(file)
         self._close_on_exit = True
     self._file = file
Example #14
0
def trimmed_and_untrimmed_files(default_output, output_path, untrimmed_path,
                                discard_trimmed, discard_untrimmed):
    """
	Figure out (from command-line parameters) where trimmed and untrimmed reads
	should be written.

	Return a pair (trimmed, untrimmed). The values are either open file-like
	objects or None, in which case no output should be produced. The objects may
	be identical (for example: (sys.stdout, sys.stdout)).

	The parameters are sorted: Later parameters have higher precedence.

	default_output -- If nothing else is specified below, this file-like object
		is returned for both trimmed and untrimmed output.
	output_path -- Path to output file for both trimmed and untrimmed output.
	untrimmed_path -- Path to an output file for untrimmed reads.
	discard_trimmed -- bool, overrides earlier options.
	discard_untrimmed -- bool, overrides earlier options.
	"""
    if discard_trimmed:
        if discard_untrimmed:
            untrimmed = None
        elif untrimmed_path is not None:
            untrimmed = xopen(untrimmed_path, 'w')
        elif output_path is not None:
            untrimmed = xopen(output_path, 'w')
        else:
            untrimmed = default_output
        return (None, untrimmed)
    if discard_untrimmed:
        trimmed = default_output
        if output_path is not None:
            trimmed = xopen(output_path, 'w')
        return (trimmed, None)

    trimmed = default_output
    untrimmed = default_output
    if output_path is not None:
        trimmed = untrimmed = xopen(output_path, 'w')
    if untrimmed_path is not None:
        untrimmed = xopen(untrimmed_path, 'w')

    return (trimmed, untrimmed)
Example #15
0
def test_context_manager():
    major, minor = sys.version_info[0:2]
    for name in files:
        if major == 2 and minor == 6:
            continue  # Py26 compression libraries do not support context manager protocol.
        with xopen(name, 'rt') as f:
            lines = list(f)
            assert len(lines) == 12
            assert lines[5] == 'AGCCGCTANGACGGGTTGGCCCTTAGACGTATCT\n', name
            f.close()
Example #16
0
def main(args):
    #Global vars
    global raw_read_len
    global min_trim

    raw_read_len = args.raw_read_length
    min_trim = args.min_trim

    paired_reader = seqio.PairedSequenceReader(args.R1,
                                               args.R2,
                                               fileformat="fastq")
    stats = FragmentStats(raw_read_len)

    try:
        out_r1 = xopen.xopen(args.output_prefix + "_R1.fq.gz", "w")
        out_r2 = xopen.xopen(args.output_prefix + "_R2.fq.gz", "w")
        small_fragments = xopen.xopen(args.output_prefix + "_single.fq.gz",
                                      "w")
    except Exception:
        logging.error("An error has occured creating one of the output files")
        sys.exit(1)

    for read1, read2 in paired_reader:
        is_short_fragment = False
        #If both paired-ends were trimmed "confidently"
        r1_len, r2_len = len(read1.sequence), len(read2.sequence)
        if max(r1_len, r2_len) < (raw_read_len - min_trim):
            aligned_r1, aligned_r2 = align_sequences(read1, read2)
            is_short_fragment = is_fragment(aligned_r1, aligned_r2)

        if is_short_fragment:
            stats.add_small_fragment(len(aligned_r1.sequence))
            consensus_fragment = get_consensus(aligned_r1, aligned_r2)
            consensus_fragment.write(small_fragments)
        else:
            stats.add_non_fragment()
            read1.write(out_r1)
            read2.write(out_r2)

    out_r1.close()
    out_r2.close()
    small_fragments.close()
    logging.info(str(stats) + "\n")
Example #17
0
def test_context_manager():
	major, minor = sys.version_info[0:2]
	for name in files:
		if major == 2 and minor == 6:
			continue  # Py26 compression libraries do not support context manager protocol.
		with xopen(name, 'rt') as f:
			lines = list(f)
			assert len(lines) == 12
			assert lines[5] == 'AGCCGCTANGACGGGTTGGCCCTTAGACGTATCT\n', name
			f.close()
Example #18
0
	def __init__(self, file, line_length=None):
		"""
		If line_length is not None, the lines will
		be wrapped after line_length characters.
		"""
		self.line_length = line_length if line_length != 0 else None
		if isinstance(file, str):
			file = xopen(file, 'w')
			self._close_on_exit = True
		self._file = file
Example #19
0
def create_truncated_file(path):
    # Random text
    text = ''.join(
        random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(200))
    f = xopen(path, 'w')
    f.write(text)
    f.close()
    f = open(path, 'a')
    f.truncate(os.stat(path).st_size - 10)
    f.close()
	def __call__(self, read1, read2=None):
		if read2 is None:
			# single-end read
			if read1.match is None:
				if self.untrimmed_outfile is None and self.untrimmed_path is not None:
					self.untrimmed_outfile = xopen(self.untrimmed_path, 'w')
				if self.untrimmed_outfile is not None:
					self.written += 1
					self.written_bp[0] += len(read1)
					read1.write(self.untrimmed_outfile)
			else:
				name = read1.match.adapter.name
				if name not in self.files:
					self.files[name] = xopen(self.template.format(name=name), 'w')
				self.written += 1
				self.written_bp[0] += len(read1)
				read1.write(self.files[name])
			return DISCARD
		else:
			assert False, "Not supported"  # pragma: no cover
Example #21
0
 def __call__(self, read1, read2=None):
     if read2 is None:
         # single-end read
         if read1.match is None:
             if self.untrimmed_outfile is None and self.untrimmed_path is not None:
                 self.untrimmed_outfile = xopen(self.untrimmed_path, 'w')
             if self.untrimmed_outfile is not None:
                 self.written += 1
                 self.written_bp[0] += len(read1)
                 read1.write(self.untrimmed_outfile)
         else:
             name = read1.match.adapter.name
             if name not in self.files:
                 self.files[name] = xopen(self.template.format(name=name),
                                          'w')
             self.written += 1
             self.written_bp[0] += len(read1)
             read1.write(self.files[name])
         return DISCARD
     else:
         assert False, "Not supported"  # pragma: no cover
def main(args):
	#Global vars
	global raw_read_len
	global min_trim

	raw_read_len = args.raw_read_length
	min_trim = args.min_trim

	paired_reader = seqio.PairedSequenceReader(args.R1, args.R2 , fileformat="fastq")
	stats = FragmentStats(raw_read_len)

	try:
		out_r1 = xopen.xopen(args.output_prefix+"_R1.fq.gz", "w")
		out_r2 = xopen.xopen(args.output_prefix+"_R2.fq.gz", "w")
		small_fragments = xopen.xopen(args.output_prefix+"_single.fq.gz","w")
	except Exception:
		logging.error("An error has occured creating one of the output files")
		sys.exit(1)

	for read1, read2 in paired_reader:
		is_short_fragment = False
		#If both paired-ends were trimmed "confidently"
		r1_len, r2_len = len(read1.sequence), len(read2.sequence)
		if max(r1_len,r2_len) < (raw_read_len - min_trim):
			aligned_r1, aligned_r2 = align_sequences(read1,read2)
			is_short_fragment = is_fragment(aligned_r1,aligned_r2)

		if is_short_fragment:
			stats.add_small_fragment( len(aligned_r1.sequence) )
			consensus_fragment = get_consensus(aligned_r1,aligned_r2)
			consensus_fragment.write(small_fragments)
		else:
			stats.add_non_fragment()
			read1.write(out_r1)
			read2.write(out_r2)

	out_r1.close()
	out_r2.close()
	small_fragments.close()
	logging.info(str(stats)+"\n")
	def __init__(self, file, keep_linebreaks=False, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then it is passed to xopen().

		keep_linebreaks -- whether to keep newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file)
		self.fp = file
		self.sequence_class = sequence_class
		self.delivers_qualities = False
		self._delimiter = '\n' if keep_linebreaks else ''
	def __init__(self, file, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		The sequence_class should be a class such as Sequence or
		ColorspaceSequence.
		"""
		if isinstance(file, basestring):
			file = xopen(file)
		self.fp = file
		self.sequence_class = sequence_class
		self.delivers_qualities = True
Example #25
0
	def __init__(self, file, wholefile=False, keep_linebreaks=False, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.
		If wholefile is True, then it is ok to read the entire file
		into memory. This is faster when there are many newlines in
		the file, but may obviously need a lot of memory.
		keep_linebreaks -- whether to keep the newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file, "rb")
		self.fp = file
		self.sequence_class = sequence_class
Example #26
0
	def __init__(self, file, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		if isinstance(file, basestring):
			file = xopen(file, "rb")
		self.fp = file
		self.twoheaders = False
		self.sequence_class = sequence_class
	def __init__(self, file, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		if isinstance(file, basestring):
			file = xopen(file, "r")
		self.fp = file
		self.twoheaders = False
		self.sequence_class = sequence_class
Example #28
0
	def __init__(self, file, wholefile=False, keep_linebreaks=False, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.
		If wholefile is True, then it is ok to read the entire file
		into memory. This is faster when there are many newlines in
		the file, but may obviously need a lot of memory.
		keep_linebreaks -- whether to keep the newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file, "rb")
		self.fp = file
		self.sequence_class = sequence_class
Example #29
0
	def __init__(self, file, keep_linebreaks=False, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then it is passed to xopen().

		keep_linebreaks -- whether to keep newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file)
			self._file_passed = False
		else:
			self._file_passed = True
		self.fp = file
		self.sequence_class = sequence_class
		self.delivers_qualities = False
		self._delimiter = '\n' if keep_linebreaks else ''
Example #30
0
	def __init__(self, file, sequence_class=Sequence):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		The sequence_class should be a class such as Sequence or
		ColorspaceSequence.
		"""
		if isinstance(file, basestring):
			file = xopen(file)
			self._file_passed = False
		else:
			self._file_passed = True
		self.fp = file
		self.sequence_class = sequence_class
		self.delivers_qualities = True
Example #31
0
 def test_truncated_gz():
     with temporary_path('truncated.gz') as path:
         create_truncated_file(path)
         f = xopen(path, 'r')
         f.read()
         f.close()
Example #32
0
def main(cmdlineargs=None, default_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
    logging.basicConfig(level=logging.INFO,
                        format='%(message)s')  #  %(levelname)s
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error(
            "At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")
    input_filename = args[0]

    # Find out which 'mode' we need to use.
    # Default: single-read trimming (neither -p nor -A/-G/-B/-U given)
    paired = False
    if options.paired_output:
        # Modify first read only, keep second in sync (-p given, but not -A/-G/-B/-U).
        # This exists for backwards compatibility ('legacy mode').
        paired = 'first'
    if options.adapters2 or options.front2 or options.anywhere2 or options.cut2:
        # Full paired-end trimming when both -p and -A/-G/-B/-U given
        # Also the read modifications (such as quality trimming) are applied
        # to second read.
        paired = 'both'

    if paired and len(args) == 1:
        parser.error(
            "When paired-end trimming is enabled via -A/-G/-B/-U or -p, "
            "two input files are required.")
    if paired:
        input_paired_filename = args[1]
        quality_filename = None
    else:
        input_paired_filename = None
        if len(args) == 2:
            if args[0].endswith('.qual'):
                parser.error("The QUAL file must be the second argument.")
            quality_filename = args[1]
        else:
            quality_filename = None

    if paired:
        if not options.paired_output:
            parser.error(
                "When paired-end trimming is enabled via -A/-G/-B/-U, "
                "a second output file needs to be specified via -p (--paired-output)."
            )
        if bool(options.untrimmed_output) != bool(
                options.untrimmed_paired_output):
            parser.error(
                "When trimming paired-end reads, you must use either none "
                "or both of the --untrimmed-output/--untrimmed-paired-output options."
            )
    else:
        if options.untrimmed_paired_output:
            parser.error(
                "Option --untrimmed-paired-output can only be used when "
                "trimming paired-end reads (with option -p).")
        if input_filename.endswith('.qual'):
            parser.error("Need a FASTA file in addition to the QUAL file.")
        if options.format is not None and quality_filename is not None:
            parser.error(
                "If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used."
            )

    if options.format is not None and options.format.lower() not in [
            'fasta', 'fastq', 'sra-fastq'
    ]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or "
            "'sra-fastq' (not '{0}').".format(options.format))

    if options.quality_cutoff is not None:
        cutoffs = options.quality_cutoff.split(',')
        if len(cutoffs) == 1:
            try:
                cutoffs = [0, int(cutoffs[0])]
            except ValueError as e:
                parser.error(
                    "Quality cutoff value not recognized: {0}".format(e))
        elif len(cutoffs) == 2:
            try:
                cutoffs = [int(cutoffs[0]), int(cutoffs[1])]
            except ValueError as e:
                parser.error(
                    "Quality cutoff value not recognized: {0}".format(e))
        else:
            parser.error(
                "Expected one value or two values separated by comma for the quality cutoff"
            )
    else:
        cutoffs = None
    writers = []
    too_short_outfile = None  # too short reads go here
    too_short_filter = None
    # TODO pass file name to TooShortReadFilter, add a .close() method?
    if options.minimum_length > 0:
        if options.too_short_output:
            too_short_outfile = xopen(options.too_short_output, 'w')
        else:
            too_short_outfile = None
        too_short_filter = TooShortReadFilter(options.minimum_length,
                                              too_short_outfile,
                                              paired == 'both')
        writers.append(too_short_filter)
    too_long_outfile = None  # too long reads go here
    too_long_filter = None
    if options.maximum_length < sys.maxsize:
        if options.too_long_output is not None:
            too_long_outfile = xopen(options.too_long_output, 'w')
        else:
            too_long_outfile = None
        too_long_filter = TooLongReadFilter(options.maximum_length,
                                            too_long_outfile,
                                            check_second=paired == 'both')
        writers.append(too_long_filter)

    if options.max_n != -1:
        writers.append(
            NContentFilter(options.max_n, check_second=paired == 'both'))

    demultiplexer = None
    if options.output is not None and '{name}' in options.output:
        if options.discard_trimmed:
            parser.error("Do not use --discard-trimmed when demultiplexing.")
        if paired:
            parser.error(
                "Demultiplexing not supported for paired-end files, yet.")
        untrimmed = options.output.format(name='unknown')
        if options.untrimmed_output:
            untrimmed = options.untrimmed_output
        if options.discard_untrimmed:
            untrimmed = None
        demultiplexer = Demultiplexer(options.output, untrimmed)
        writers.append(demultiplexer)
        trimmed_outfile, untrimmed_outfile = None, None
        trimmed_paired_outfile, untrimmed_paired_outfile = None, None
    else:
        trimmed_outfile, untrimmed_outfile = trimmed_and_untrimmed_files(
            default_outfile, options.output, options.untrimmed_output,
            options.discard_trimmed, options.discard_untrimmed)

        trimmed_paired_outfile, untrimmed_paired_outfile = trimmed_and_untrimmed_files(
            None,  # applies when not trimming paired-end data
            options.paired_output,
            options.untrimmed_paired_output,
            options.discard_trimmed,
            options.discard_untrimmed)

        if untrimmed_outfile or untrimmed_paired_outfile:
            writers.append(
                DiscardUntrimmedFilter(untrimmed_outfile,
                                       untrimmed_paired_outfile,
                                       check_second=paired == 'both'))
        writer = DiscardTrimmedFilter(trimmed_outfile,
                                      trimmed_paired_outfile,
                                      check_second=paired == 'both')
        writers.append(writer)
        del writer

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append('_F3')
        options.suffix = "/1"
    if options.zero_cap is None:
        options.zero_cap = options.colorspace
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in colorspace.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in colorspace.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, 'w')
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, 'w')
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, 'w')

    if options.colorspace:
        if options.match_read_wildcards:
            parser.error('IUPAC wildcards not supported in colorspace')
        options.match_adapter_wildcards = False

    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter
    try:
        # TODO refactor this a bit
        def collect(back, anywhere, front):
            adapters = []
            for name, seq, where in gather_adapters(back, anywhere, front):
                if not seq:
                    parser.error("The adapter sequence is empty.")
                adapter = ADAPTER_CLASS(seq,
                                        where,
                                        options.error_rate,
                                        options.overlap,
                                        options.match_read_wildcards,
                                        options.match_adapter_wildcards,
                                        name=name,
                                        indels=options.indels)
                if options.debug:
                    adapter.enable_debug()
                adapters.append(adapter)
            return adapters

        adapters = collect(options.adapters, options.anywhere, options.front)
        adapters2 = collect(options.adapters2, options.anywhere2,
                            options.front2)
    except IOError as e:
        if e.errno == errno.ENOENT:
            parser.error(e)
        raise

    if not adapters and not adapters2 and not cutoffs and \
      options.cut == [] and options.cut2 == [] and \
      options.minimum_length == 0 and \
      options.maximum_length == sys.maxsize and \
      quality_filename is None and \
      options.max_n == -1:
        parser.error("You need to provide at least one adapter sequence.")

    try:
        reader = seqio.open(input_filename,
                            file2=input_paired_filename,
                            qualfile=quality_filename,
                            colorspace=options.colorspace,
                            fileformat=options.format)
    except (seqio.UnknownFileType, IOError) as e:
        parser.error(e)

    # Create the processing pipeline consisting of a list of "modifiers".
    modifiers = []
    if options.cut:
        if len(options.cut) > 2:
            parser.error("You cannot remove bases from more than two ends.")
        if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
            parser.error("You cannot remove bases from the same end twice.")
        for cut in options.cut:
            if cut != 0:
                modifiers.append(UnconditionalCutter(cut))

    if cutoffs:
        modifiers.append(
            QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
    if adapters:
        adapter_cutter = AdapterCutter(adapters, options.times,
                                       options.wildcard_file,
                                       options.info_file, rest_writer,
                                       options.action)
        modifiers.append(adapter_cutter)
    else:
        adapter_cutter = None

    # Modifiers that apply to both reads of paired-end reads
    modifiers_both = []
    if options.trim_n:
        modifiers_both.append(NEndTrimmer())
    if options.length_tag:
        modifiers_both.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append('_F3')
    for suffix in options.strip_suffix:
        modifiers_both.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers_both.append(PrefixSuffixAdder(options.prefix,
                                                options.suffix))
    if options.double_encode:
        modifiers_both.append(DoubleEncoder())
    if options.zero_cap and reader.delivers_qualities:
        modifiers_both.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers_both.append(PrimerTrimmer)
    modifiers.extend(modifiers_both)

    # For paired-end data, create a second processing pipeline.
    # However, if no second-read adapters were given (via -A/-G/-B/-U), we need to
    # be backwards compatible and *no modifications* are done to the second read.
    modifiers2 = []
    if paired == 'both':
        if options.cut2:
            if len(options.cut2) > 2:
                parser.error(
                    "You cannot remove bases from more than two ends.")
            if len(options.cut2
                   ) == 2 and options.cut2[0] * options.cut2[1] > 0:
                parser.error(
                    "You cannot remove bases from the same end twice.")
            for cut in options.cut2:
                if cut != 0:
                    modifiers2.append(UnconditionalCutter(cut))

        if cutoffs:
            modifiers2.append(
                QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
        if adapters2:
            adapter_cutter2 = AdapterCutter(adapters2, options.times, None,
                                            None, None, options.action)
            modifiers2.append(adapter_cutter2)
        else:
            adapter_cutter2 = None
        modifiers2.extend(modifiers_both)

    # Due to backwards compatibility, from here on logging output needs to be
    # sent to standard output instead of standard error if the -o option is used.
    if options.output:
        logger.root.handlers = []
        logging.basicConfig(level=logging.INFO,
                            format='%(message)s',
                            stream=sys.stdout)
    logger.info("This is cutadapt %s with Python %s", __version__,
                platform.python_version())
    logger.info("Command line parameters: %s", " ".join(cmdlineargs))
    logger.info(
        "Trimming %s adapter%s with at most %.1f%% errors in %s mode ...",
        len(adapters) + len(adapters2),
        's' if len(adapters) + len(adapters2) != 1 else '',
        options.error_rate * 100, {
            False: 'single-end',
            'first': 'paired-end legacy',
            'both': 'paired-end'
        }[paired])

    start_time = time.clock()
    try:
        if paired:
            stats = process_paired_reads(reader, modifiers, modifiers2,
                                         writers)
        else:
            stats = process_single_reads(reader, modifiers, writers)
    except KeyboardInterrupt as e:
        print("Interrupted", file=sys.stderr)
        sys.exit(130)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except (seqio.FormatError, EOFError) as e:
        sys.exit("cutadapt: error: {0}".format(e))

    # close open files
    for f in [
            trimmed_outfile, untrimmed_outfile, trimmed_paired_outfile,
            untrimmed_paired_outfile, options.rest_file, options.wildcard_file,
            options.info_file, too_short_outfile, too_long_outfile,
            options.info_file, demultiplexer
    ]:
        if f is not None and f is not sys.stdin and f is not sys.stdout:
            f.close()

    elapsed_time = time.clock() - start_time
    if not options.quiet:
        stats.collect((adapters, adapters2), elapsed_time, modifiers,
                      modifiers2, writers)
        # send statistics to stderr if result was sent to stdout
        stat_file = sys.stderr if options.output is None else None
        with redirect_standard_output(stat_file):
            print_report(stats, (adapters, adapters2))
Example #33
0
def main(cmdlineargs=None, default_outfile=sys.stdout):
	"""
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the default output file to which trimmed reads
	are sent. It can be overriden by using the '-o' parameter.
	"""
	parser = get_option_parser()
	if cmdlineargs is None:
		cmdlineargs = sys.argv[1:]
	options, args = parser.parse_args(args=cmdlineargs)

	if len(args) == 0:
		parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
	elif len(args) > 2:
		parser.error("Too many parameters.")

	input_filename = args[0]

	# If a second file name was given, then we either have single-end reads
	# provided as a pair of .fasta/.qual files or we have paired-end reads.
	quality_filename = None
	input_paired_filename = None
	if len(args) == 2:
		if args[1].endswith('.qual'):
			quality_filename = args[1]
		else:
			input_paired_filename = args[1]
			if not options.paired_output:
				parser.error('You must use --paired-output when trimming paired-end reads.')

	if len(args) == 1 and options.paired_output:
		parser.error("You specified a --paired-output file, but gave only one input file.")
	if options.paired_output and bool(options.untrimmed_output) != bool(options.untrimmed_paired_output):
		parser.error("When trimming paired-end reads, you must use either none "
			"or both of the --untrimmed-output/--untrimmed-paired-output options.")
	if options.untrimmed_paired_output and not options.paired_output:
		parser.error("Option --untrimmed-paired-output can only be used when "
			"trimming paired-end reads (with option --paired-output).")
	if input_filename.endswith('.qual') and quality_filename.endswith('fasta'):
		parser.error("FASTA and QUAL file given, but the FASTA file must be first.")

	if options.format is not None and options.format.lower() not in ['fasta', 'fastq', 'sra-fastq']:
		parser.error("The input file format must be either 'fasta', 'fastq' or 'sra-fastq' (not '{0}').".format(options.format))

	# TODO should this really be an error?
	if options.format is not None and quality_filename is not None:
		parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

	trimmed_outfile, untrimmed_outfile = trimmed_and_untrimmed_files(
		default_outfile,
		options.output,
		options.untrimmed_output,
		options.discard_trimmed,
		options.discard_untrimmed)

	trimmed_paired_outfile, untrimmed_paired_outfile = trimmed_and_untrimmed_files(
		None,  # applies when not trimming paired-end data
		options.paired_output,
		options.untrimmed_paired_output,
		options.discard_trimmed,
		options.discard_untrimmed)

	too_short_outfile = None  # too short reads go here
	if options.too_short_output is not None:
		too_short_outfile = xopen(options.too_short_output, 'w')
	too_long_outfile = None  # too long reads go here
	if options.too_long_output is not None:
		too_long_outfile = xopen(options.too_long_output, 'w')

	read_writer = ProcessedReadWriter(
		options.minimum_length,
		options.maximum_length,
		too_short_outfile,
		too_long_outfile,
		trimmed_outfile, trimmed_paired_outfile,
		untrimmed_outfile, untrimmed_paired_outfile
	)

	if options.maq:
		options.colorspace = True
		options.double_encode = True
		options.trim_primer = True
		options.strip_suffix.append('_F3')
		options.suffix = "/1"
	if options.zero_cap is None:
		options.zero_cap = options.colorspace
	if options.trim_primer and not options.colorspace:
		parser.error("Trimming the primer makes only sense in colorspace.")
	if options.double_encode and not options.colorspace:
		parser.error("Double-encoding makes only sense in colorspace.")
	if options.anywhere and options.colorspace:
		parser.error("Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author).")
	if not (0 <= options.error_rate <= 1.):
		parser.error("The maximum error rate must be between 0 and 1.")
	if options.overlap < 1:
		parser.error("The overlap must be at least 1.")

	if options.rest_file is not None:
		options.rest_file = xopen(options.rest_file, 'w')
		rest_writer = RestFileWriter(options.rest_file)
	else:
		rest_writer = None
	if options.info_file is not None:
		options.info_file = xopen(options.info_file, 'w')
	if options.wildcard_file is not None:
		options.wildcard_file = xopen(options.wildcard_file, 'w')

	adapters = []
	ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

	try:
		for name, seq, where in gather_adapters(options.adapters, options.anywhere, options.front):
			if not seq:
				parser.error("The adapter sequence is empty")
			if not options.indels and where not in (PREFIX, SUFFIX):
				parser.error("Not allowing indels is currently supported only for anchored 5' or 3' adapters.")
			adapter = ADAPTER_CLASS(seq, where, options.error_rate,
				options.overlap, options.match_read_wildcards,
				options.match_adapter_wildcards, name=name, indels=options.indels)
			adapters.append(adapter)
	except IOError as e:
		if e.errno == errno.ENOENT:
			print("Error:", e, file=sys.stderr)
			sys.exit(1)
		raise

	start_time = time.clock()

	if input_paired_filename:
		reader = seqio.PairedSequenceReader(input_filename, input_paired_filename,
			colorspace=options.colorspace, fileformat=options.format)
	else:
		reader = read_sequences(input_filename, quality_filename,
			colorspace=options.colorspace, fileformat=options.format)

	# build up list of modifiers
	modifiers = []
	if not adapters and options.quality_cutoff == 0 and options.cut == 0:
		parser.error("You need to provide at least one adapter sequence.")
	if options.cut:
		modifiers.append(UnconditionalCutter(options.cut))
	if options.quality_cutoff > 0:
		modifiers.append(QualityTrimmer(options.quality_cutoff, options.quality_base))
	if adapters:
		adapter_cutter = RepeatedAdapterCutter(adapters, options.times,
				options.wildcard_file, options.info_file, options.trim,
				rest_writer, options.mask_adapter)
		modifiers.append(adapter_cutter)
	else:
		adapter_cutter = None
	if options.length_tag:
		modifiers.append(LengthTagModifier(options.length_tag))
	if options.strip_f3:
		options.strip_suffix.append('_F3')
	for suffix in options.strip_suffix:
		modifiers.append(SuffixRemover(suffix))
	if options.prefix or options.suffix:
		modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
	if options.double_encode:
		modifiers.append(DoubleEncoder())
	if options.zero_cap and reader.delivers_qualities:
		modifiers.append(ZeroCapper(quality_base=options.quality_base))
	if options.trim_primer:
		modifiers.append(PrimerTrimmer)

	try:
		if input_paired_filename:
			stats = process_paired_reads(reader, modifiers, read_writer)
		else:
			stats = process_single_reads(reader, modifiers, read_writer)
	except KeyboardInterrupt as e:
		print("Interrupted", file=sys.stderr)
		sys.exit(1)
	except IOError as e:
		if e.errno == errno.EPIPE:
			sys.exit(1)
		raise
	except seqio.FormatError as e:
		print("Error:", e, file=sys.stderr)
		sys.exit(1)

	# close open files
	for f in [trimmed_outfile, untrimmed_outfile, trimmed_paired_outfile,
			untrimmed_paired_outfile, options.rest_file, options.wildcard_file, options.info_file,
			too_short_outfile, too_long_outfile, options.info_file]:
		if f is not None and f is not sys.stdin and f is not sys.stdout:
			f.close()

	if not options.quiet:
		# send statistics to stderr if result was sent to stdout
		stat_file = sys.stderr if options.output is None else None
		print_statistics(adapters, time.clock() - start_time, stats,
			options.trim, adapter_cutter.reads_matched if adapter_cutter else 0,
			options.error_rate, read_writer.too_short, read_writer.too_long, cmdlineargs, file=stat_file)
Example #34
0
	def test_truncated_gz():
		with temporary_path('truncated.gz') as path:
			create_truncated_file(path)
			f = xopen(path, 'r')
			f.read()
			f.close()
def main(cmdlineargs=None, trimmed_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	trimmed_outfile is the default output file to which trimmed reads
	are sent. It can be overriden by using the '-o' parameter.
	"""
    parser = HelpfulOptionParser(usage=__doc__, version=__version__)

    parser.add_option(
        "-f",
        "--format",
        default=None,
        help="Input file format; can be either 'fasta', 'fastq' or 'sra-fastq'. "
        "Ignored when reading csfasta/qual files (default: auto-detect "
        "from file name extension).")

    group = OptionGroup(
        parser,
        "Options that influence how the adapters are found",
        description=
        "Each of the following three parameters (-a, -b, -g) can be used "
        "multiple times and in any combination to search for an entire set of "
        "adapters of possibly different types. All of the "
        "given adapters will be searched for in each read, but only the best "
        "matching one will be trimmed (but see the --times option).")
    group.add_option(
        "-a",
        "--adapter",
        action="append",
        metavar="ADAPTER",
        dest="adapters",
        default=[],
        help=
        "Sequence of an adapter that was ligated to the 3' end. The adapter itself and anything that follows is trimmed."
    )
    group.add_option(
        "-b",
        "--anywhere",
        action="append",
        metavar="ADAPTER",
        default=[],
        help=
        "Sequence of an adapter that was ligated to the 5' or 3' end. If the adapter is found within the read or overlapping the 3' end of the read, the behavior is the same as for the -a option. If the adapter overlaps the 5' end (beginning of the read), the initial portion of the read matching the adapter is trimmed, but anything that follows is kept."
    )
    group.add_option(
        "-g",
        "--front",
        action="append",
        metavar="ADAPTER",
        default=[],
        help="Sequence of an adapter that was ligated to the 5' end. If the "
        "adapter sequence starts with the character '^', the adapter is "
        "'anchored'. An anchored adapter must appear in its entirety at the "
        "5' end of the read (it is a prefix of the read). A non-anchored adapter may "
        "appear partially at the 5' end, or it may occur within the read. If it is "
        "found within a read, the sequence preceding the adapter is also trimmed. "
        "In all cases, the adapter itself is trimmed.")
    group.add_option(
        "-e",
        "--error-rate",
        type=float,
        default=0.1,
        help=
        "Maximum allowed error rate (no. of errors divided by the length of the matching region) (default: %default)"
    )
    group.add_option(
        "-n",
        "--times",
        type=int,
        metavar="COUNT",
        default=1,
        help=
        "Try to remove adapters at most COUNT times. Useful when an adapter gets appended multiple times (default: %default)."
    )
    group.add_option(
        "-O",
        "--overlap",
        type=int,
        metavar="LENGTH",
        default=3,
        help=
        "Minimum overlap length. If the overlap between the read and the adapter is shorter than LENGTH, the read is not modified."
        "This reduces the no. of bases trimmed purely due to short random adapter matches (default: %default)."
    )
    group.add_option(
        "--match-read-wildcards",
        action="store_true",
        default=False,
        help=
        "Allow 'N's in the read as matches to the adapter (default: %default)."
    )
    group.add_option(
        "-N",
        "--no-match-adapter-wildcards",
        action="store_false",
        default=True,
        dest='match_adapter_wildcards',
        help=
        "Do not treat 'N' in the adapter sequence as wildcards. This is needed when you want to search for literal 'N' characters."
    )
    parser.add_option_group(group)

    group = OptionGroup(parser, "Options for filtering of processed reads")
    group.add_option(
        "--discard-trimmed",
        "--discard",
        action='store_true',
        default=False,
        help=
        "Discard reads that contain the adapter instead of trimming them. Also use -O in order to avoid throwing away too many randomly matching reads!"
    )
    group.add_option("--discard-untrimmed",
                     "--trimmed-only",
                     action='store_true',
                     default=False,
                     help="Discard reads that do not contain the adapter.")
    group.add_option(
        "-m",
        "--minimum-length",
        type=int,
        default=0,
        metavar="LENGTH",
        help=
        "Discard trimmed reads that are shorter than LENGTH. Reads that are too short even before adapter removal are also discarded. In colorspace, an initial primer is not counted (default: 0)."
    )
    group.add_option("-M",
                     "--maximum-length",
                     type=int,
                     default=sys.maxsize,
                     metavar="LENGTH",
                     help="Discard trimmed reads that are longer than LENGTH. "
                     "Reads that are too long even before adapter removal "
                     "are also discarded. In colorspace, an initial primer "
                     "is not counted (default: no limit).")
    group.add_option(
        "--no-trim",
        dest='trim',
        action='store_false',
        default=True,
        help=
        "Match and redirect reads to output/untrimmed-output as usual, but don't remove the adapters. (default: False. Remove the adapters)"
    )
    parser.add_option_group(group)

    group = OptionGroup(parser,
                        "Options that influence what gets output to where")
    group.add_option(
        "-o",
        "--output",
        default=None,
        metavar="FILE",
        help=
        "Write the modified sequences to this file instead of standard output and send the summary report to standard output. "
        "The format is FASTQ if qualities are available, FASTA otherwise. (default: standard output)"
    )
    group.add_option(
        "--info-file",
        metavar="FILE",
        help=
        "Write information about each read and its adapter matches into FILE. "
        "Currently experimental: Expect the file format to change!")
    group.add_option(
        "-r",
        "--rest-file",
        default=None,
        metavar="FILE",
        help=
        "When the adapter matches in the middle of a read, write the rest (after the adapter) into a file. Use - for standard output."
    )
    group.add_option(
        "--wildcard-file",
        default=None,
        metavar="FILE",
        help=
        "When the adapter has wildcard bases ('N's) write adapter bases matching wildcard "
        "positions to FILE. Use - for standard output. "
        "When there are indels in the alignment, this may occasionally "
        "not be quite accurate.")
    group.add_option(
        "--too-short-output",
        default=None,
        metavar="FILE",
        help=
        "Write reads that are too short (according to length specified by -m) to FILE. (default: discard reads)"
    )
    group.add_option(
        "--too-long-output",
        default=None,
        metavar="FILE",
        help=
        "Write reads that are too long (according to length specified by -M) to FILE. (default: discard reads)"
    )
    group.add_option(
        "--untrimmed-output",
        default=None,
        metavar="FILE",
        help="Write reads that do not contain the adapter to FILE, instead "
        "of writing them to the regular output file. (default: output "
        "to same file as trimmed)")
    group.add_option('-p',
                     "--paired-output",
                     default=None,
                     metavar="FILE",
                     help="Write reads from the paired end input to FILE. ")
    parser.add_option_group(group)

    group = OptionGroup(parser, "Additional modifications to the reads")
    group.add_option(
        "-q",
        "--quality-cutoff",
        type=int,
        default=0,
        metavar="CUTOFF",
        help="Trim low-quality ends from reads before adapter removal. "
        "The algorithm is the same as the one used by BWA "
        "(Subtract CUTOFF from all qualities; "
        "compute partial sums from all indices to the end of the "
        "sequence; cut sequence at the index at which the sum "
        "is minimal) (default: %default)")
    group.add_option(
        "--quality-base",
        type=int,
        default=33,
        help=
        "Assume that quality values are encoded as ascii(quality + QUALITY_BASE). The default (33) is usually correct, "
        "except for reads produced by some versions of the Illumina pipeline, where this should be set to 64. (default: %default)"
    )
    group.add_option("-x",
                     "--prefix",
                     default='',
                     help="Add this prefix to read names")
    group.add_option("-y",
                     "--suffix",
                     default='',
                     help="Add this suffix to read names")
    group.add_option(
        "--strip-suffix",
        action='append',
        default=[],
        help=
        "Remove this suffix from read names if present. Can be given multiple times."
    )
    group.add_option(
        "-c",
        "--colorspace",
        action='store_true',
        default=False,
        help=
        "Colorspace mode: Also trim the color that is adjacent to the found adapter."
    )
    group.add_option(
        "-d",
        "--double-encode",
        action='store_true',
        default=False,
        help=
        "When in color space, double-encode colors (map 0,1,2,3,4 to A,C,G,T,N)."
    )
    group.add_option(
        "-t",
        "--trim-primer",
        action='store_true',
        default=False,
        help="When in color space, trim primer base and the first color "
        "(which is the transition to the first nucleotide)")
    group.add_option(
        "--strip-f3",
        action='store_true',
        default=False,
        help="For color space: Strip the _F3 suffix of read names")
    group.add_option(
        "--maq",
        "--bwa",
        action='store_true',
        default=False,
        help=
        "MAQ- and BWA-compatible color space output. This enables -c, -d, -t, --strip-f3, -y '/1' and -z."
    )
    group.add_option(
        "--length-tag",
        default=None,
        metavar="TAG",
        help=
        "Search for TAG followed by a decimal number in the name of the read "
        "(description/comment field of the FASTA or FASTQ file). Replace the "
        "decimal number with the correct length of the trimmed read. "
        "For example, use --length-tag 'length=' to correct fields "
        "like 'length=123'.")
    group.add_option(
        "--zero-cap",
        "-z",
        action='store_true',
        default=False,
        help=
        "Change negative quality values to zero (workaround to avoid segmentation faults in old BWA versions)"
    )
    parser.add_option_group(group)

    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error(
            "At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")

    input_filename = args[0]
    quality_filename = None
    pe_filename = None
    if len(args) == 2:
        if args[1].endswith('.qual'):
            quality_filename = args[1]
        else:
            pe_filename = args[1]
            if not options.paired_output:
                parser.error(
                    'you must use --paired-output when trimming paired-end reads'
                )

    if len(args) == 1 and options.paired_output:
        parser.error(
            "You specified a --paired-output file, but gave only one input file."
        )
    if input_filename.endswith('.qual') and quality_filename.endswith('fasta'):
        parser.error(
            "FASTA and QUAL file given, but the FASTA file must be first.")

    if options.format is not None and options.format.lower() not in [
            'fasta', 'fastq', 'sra-fastq'
    ]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or 'sra-fastq' (not '{0}')."
            .format(options.format))

    # TODO should this really be an error?
    if options.format is not None and quality_filename is not None:
        parser.error(
            "If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used."
        )

    # default output files (overwritten below)
    too_short_outfile = None  # too short reads go here
    too_long_outfile = None  # too long reads go here
    pe_outfile = None

    if options.output is not None:
        trimmed_outfile = xopen(options.output, 'w')
    untrimmed_outfile = trimmed_outfile  # reads without adapters go here
    if options.untrimmed_output is not None:
        untrimmed_outfile = xopen(options.untrimmed_output, 'w')
    if options.too_short_output is not None:
        too_short_outfile = xopen(options.too_short_output, 'w')
    if options.too_long_output is not None:
        too_long_outfile = xopen(options.too_long_output, 'w')
    if options.paired_output:
        pe_outfile = xopen(options.paired_output, 'w')

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append('_F3')
        options.suffix = "/1"
        options.zero_cap = True
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in color space.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in color space.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with color space reads is currently not supported  (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, 'w')
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, 'w')
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, 'w')

    adapters = []

    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

    def append_adapters(adapter_list, where):
        for seq in adapter_list:
            fields = seq.split('=', 1)
            if len(fields) > 1:
                name, seq = fields
                name = name.strip()
            else:
                name = None
            seq = seq.strip()
            w = where
            if w == FRONT and seq.startswith('^'):
                seq = seq[1:]
                w = PREFIX
            if len(seq) == 0:
                parser.error("The adapter sequence is empty")
            adapter = ADAPTER_CLASS(seq,
                                    w,
                                    options.error_rate,
                                    options.overlap,
                                    options.match_read_wildcards,
                                    options.match_adapter_wildcards,
                                    name=name)
            adapters.append(adapter)

    append_adapters(options.adapters, BACK)
    append_adapters(options.anywhere, ANYWHERE)
    append_adapters(options.front, FRONT)

    # make sure these aren't used by accident
    del options.adapters
    del options.anywhere
    del options.front

    if not adapters and options.quality_cutoff == 0:
        print("You need to provide at least one adapter sequence.",
              file=sys.stderr)
        sys.exit(1)

    modifiers = []
    if options.length_tag:
        modifiers.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append('_F3')
    for suffix in options.strip_suffix:
        modifiers.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
    if options.double_encode:
        modifiers.append(DoubleEncoder())
    if options.zero_cap:
        modifiers.append(ZeroCapper(quality_base=options.quality_base))
    if options.quality_cutoff > 0:
        quality_trimmer = QualityTrimmer(options.quality_cutoff,
                                         options.quality_base)
    else:
        quality_trimmer = None

    adapter_matcher = RepeatedAdapterMatcher(adapters, options.times,
                                             options.wildcard_file,
                                             options.info_file, options.trim)
    readfilter = ReadFilter(options.minimum_length, options.maximum_length,
                            too_short_outfile, too_long_outfile,
                            options.discard_trimmed, options.discard_untrimmed,
                            options.trim_primer)
    start_time = time.clock()
    try:
        reader = read_sequences(input_filename,
                                quality_filename,
                                colorspace=options.colorspace,
                                fileformat=options.format)
        if pe_filename:
            pe_reader = read_sequences(pe_filename,
                                       None,
                                       colorspace=options.colorspace,
                                       fileformat=options.format)
        else:
            pe_reader = None
        (n, total_bp) = process_reads(reader, pe_reader, adapter_matcher,
                                      quality_trimmer, modifiers, readfilter,
                                      trimmed_outfile, untrimmed_outfile,
                                      pe_outfile, rest_writer,
                                      options.trim_primer)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except seqio.FormatError as e:
        print("Error:", e, file=sys.stderr)
        sys.exit(1)
    # close open files
    for f in [
            options.rest_file, options.wildcard_file, options.info_file,
            too_short_outfile, too_long_outfile, options.info_file
    ]:
        if f is not None:
            f.close()
    # send statistics to stderr if result was sent to stdout
    stat_file = sys.stderr if options.output is None else None

    total_quality_trimmed = quality_trimmer.trimmed_bases if quality_trimmer else -1
    print_statistics(adapters,
                     time.clock() - start_time,
                     n,
                     total_bp,
                     total_quality_trimmed,
                     options.trim,
                     adapter_matcher.reads_matched,
                     options.error_rate,
                     readfilter.too_short,
                     readfilter.too_long,
                     file=stat_file)
Example #36
0
 def __init__(self, file):
     if isinstance(file, str):
         file = xopen(file, "w")
         self._close_on_exit = True
     self._file = file
def main(cmdlineargs=None, default_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")

    input_filename = args[0]

    # If a second file name was given, then we either have single-end reads
    # provided as a pair of .fasta/.qual files or we have paired-end reads.
    quality_filename = None
    input_paired_filename = None
    if len(args) == 2:
        if args[0].endswith(".qual"):
            parser.error("The QUAL file must be the second argument.")
        if args[1].endswith(".qual"):
            quality_filename = args[1]
        else:
            input_paired_filename = args[1]
            if not options.paired_output:
                parser.error("You must use --paired-output when trimming paired-end reads.")

    if len(args) == 1 and options.paired_output:
        parser.error("You specified a --paired-output file, but gave only one input file.")
    if options.paired_output and bool(options.untrimmed_output) != bool(options.untrimmed_paired_output):
        parser.error(
            "When trimming paired-end reads, you must use either none "
            "or both of the --untrimmed-output/--untrimmed-paired-output options."
        )
    if options.untrimmed_paired_output and not options.paired_output:
        parser.error(
            "Option --untrimmed-paired-output can only be used when "
            "trimming paired-end reads (with option --paired-output)."
        )
    if input_filename.endswith(".qual"):
        parser.error("Need a FASTA file in addition to the QUAL file.")

    if options.format is not None and options.format.lower() not in ["fasta", "fastq", "sra-fastq"]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or "
            "'sra-fastq' (not '{0}').".format(options.format)
        )

        # TODO should this really be an error?
    if options.format is not None and quality_filename is not None:
        parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

    writers = []
    too_short_outfile = None  # too short reads go here
    too_short_filter = None
    # TODO pass file name to TooShortReadFilter, add a .close() method?
    if options.minimum_length > 0:
        if options.too_short_output:
            too_short_outfile = xopen(options.too_short_output, "w")
        else:
            too_short_outfile = None
        too_short_filter = TooShortReadFilter(options.minimum_length, too_short_outfile)
        writers.append(too_short_filter)
    too_long_outfile = None  # too long reads go here
    too_long_filter = None
    if options.maximum_length < sys.maxsize:
        if options.too_long_output is not None:
            too_long_outfile = xopen(options.too_long_output, "w")
        else:
            too_long_outfile = None
        too_long_filter = TooLongReadFilter(options.maximum_length, too_long_outfile)
        writers.append(too_long_filter)

    demultiplexer = None
    if options.output is not None and "{name}" in options.output:
        if options.discard_trimmed:
            parser.error("Do not use --discard-trimmed when demultiplexing.")
        if input_paired_filename:
            parser.error("Demultiplexing not supported for paired-end files, yet.")
        untrimmed = options.output.format(name="unknown")
        if options.untrimmed_output:
            untrimmed = options.untrimmed_output
        if options.discard_untrimmed:
            untrimmed = None
        demultiplexer = Demultiplexer(options.output, untrimmed)
        writers.append(demultiplexer)
        trimmed_outfile, untrimmed_outfile = None, None
        trimmed_paired_outfile, untrimmed_paired_outfile = None, None
    else:
        trimmed_outfile, untrimmed_outfile = trimmed_and_untrimmed_files(
            default_outfile,
            options.output,
            options.untrimmed_output,
            options.discard_trimmed,
            options.discard_untrimmed,
        )

        trimmed_paired_outfile, untrimmed_paired_outfile = trimmed_and_untrimmed_files(
            None,  # applies when not trimming paired-end data
            options.paired_output,
            options.untrimmed_paired_output,
            options.discard_trimmed,
            options.discard_untrimmed,
        )

        writer = ProcessedReadWriter(
            trimmed_outfile, trimmed_paired_outfile, untrimmed_outfile, untrimmed_paired_outfile
        )
        writers.append(writer)

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append("_F3")
        options.suffix = "/1"
    if options.zero_cap is None:
        options.zero_cap = options.colorspace
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in colorspace.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in colorspace.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.0):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, "w")
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, "w")
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, "w")

    if options.colorspace:
        if options.match_read_wildcards:
            parser.error("IUPAC wildcards not supported in colorspace")
        options.match_adapter_wildcards = False
    adapters = []
    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

    try:
        for name, seq, where in gather_adapters(options.adapters, options.anywhere, options.front):
            if not seq:
                parser.error("The adapter sequence is empty")
            if not options.indels and where not in (PREFIX, SUFFIX):
                parser.error("Not allowing indels is currently supported only for anchored 5' and 3' adapters.")
            adapter = ADAPTER_CLASS(
                seq,
                where,
                options.error_rate,
                options.overlap,
                options.match_read_wildcards,
                options.match_adapter_wildcards,
                name=name,
                indels=options.indels,
            )
            adapters.append(adapter)
    except IOError as e:
        if e.errno == errno.ENOENT:
            print("Error:", e, file=sys.stderr)
            sys.exit(1)
        raise

    if (
        not adapters
        and options.quality_cutoff == 0
        and options.cut == 0
        and options.minimum_length == 0
        and options.maximum_length == sys.maxsize
    ):
        parser.error("You need to provide at least one adapter sequence.")

    if input_paired_filename:
        reader = seqio.PairedSequenceReader(
            input_filename, input_paired_filename, colorspace=options.colorspace, fileformat=options.format
        )
    else:
        reader = read_sequences(
            input_filename, quality_filename, colorspace=options.colorspace, fileformat=options.format
        )

        # Create the processing pipeline as a list of "modifiers".
    modifiers = []
    if options.cut:
        if len(options.cut) > 2:
            parser.error("You cannot remove bases from more than two ends.")
        if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
            parser.error("You cannot remove bases from the same end twice.")
        for cut in options.cut:
            if cut != 0:
                modifiers.append(UnconditionalCutter(cut))

    if options.quality_cutoff > 0:
        modifiers.append(QualityTrimmer(options.quality_cutoff, options.quality_base))
    if adapters:
        adapter_cutter = AdapterCutter(
            adapters, options.times, options.wildcard_file, options.info_file, rest_writer, options.action
        )
        modifiers.append(adapter_cutter)
    else:
        adapter_cutter = None
    if options.length_tag:
        modifiers.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append("_F3")
    for suffix in options.strip_suffix:
        modifiers.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
    if options.double_encode:
        modifiers.append(DoubleEncoder())
    if options.zero_cap and reader.delivers_qualities:
        modifiers.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers.append(PrimerTrimmer)

    start_time = time.clock()
    try:
        if input_paired_filename:
            stats = process_paired_reads(reader, modifiers, writers)
        else:
            stats = process_single_reads(reader, modifiers, writers)
    except KeyboardInterrupt as e:
        print("Interrupted", file=sys.stderr)
        sys.exit(1)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except seqio.FormatError as e:
        print("Error:", e, file=sys.stderr)
        sys.exit(1)

        # close open files
    for f in [
        trimmed_outfile,
        untrimmed_outfile,
        trimmed_paired_outfile,
        untrimmed_paired_outfile,
        options.rest_file,
        options.wildcard_file,
        options.info_file,
        too_short_outfile,
        too_long_outfile,
        options.info_file,
        demultiplexer,
    ]:
        if f is not None and f is not sys.stdin and f is not sys.stdout:
            f.close()

    if not options.quiet:
        # send statistics to stderr if result was sent to stdout
        stat_file = sys.stderr if options.output is None else None
        print_statistics(
            adapters,
            time.clock() - start_time,
            stats,
            options.action,
            adapter_cutter.reads_matched if adapter_cutter else 0,
            options.error_rate,
            too_short_filter.too_short if too_short_filter else 0,
            too_long_filter.too_long if too_long_filter else 0,
            cmdlineargs,
            file=stat_file,
        )
def main(cmdlineargs=None, default_outfile=sys.stdout):
	"""
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
	logging.basicConfig(level=logging.INFO, format='%(message)s')  #  %(levelname)s
	parser = get_option_parser()
	if cmdlineargs is None:
		cmdlineargs = sys.argv[1:]
	options, args = parser.parse_args(args=cmdlineargs)

	if len(args) == 0:
		parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
	elif len(args) > 2:
		parser.error("Too many parameters.")
	input_filename = args[0]

	# Find out which 'mode' we need to use.
	# Default: single-read trimming (neither -p nor -A/-G/-B/-U given)
	paired = False
	if options.paired_output:
		# Modify first read only, keep second in sync (-p given, but not -A/-G/-B/-U).
		# This exists for backwards compatibility ('legacy mode').
		paired = 'first'
	if options.adapters2 or options.front2 or options.anywhere2 or options.cut2:
		# Full paired-end trimming when both -p and -A/-G/-B/-U given
		# Also the read modifications (such as quality trimming) are applied
		# to second read.
		paired = 'both'

	if paired and len(args) == 1:
		parser.error("When paired-end trimming is enabled via -A/-G/-B/-U or -p, "
			"two input files are required.")
	if paired:
		input_paired_filename = args[1]
		quality_filename = None
	else:
		input_paired_filename = None
		if len(args) == 2:
			if args[0].endswith('.qual'):
				parser.error("The QUAL file must be the second argument.")
			quality_filename = args[1]
		else:
			quality_filename = None

	if paired:
		if not options.paired_output:
			parser.error("When paired-end trimming is enabled via -A/-G/-B/-U, "
				"a second output file needs to be specified via -p (--paired-output).")
		if bool(options.untrimmed_output) != bool(options.untrimmed_paired_output):
			parser.error("When trimming paired-end reads, you must use either none "
				"or both of the --untrimmed-output/--untrimmed-paired-output options.")
	else:
		if options.untrimmed_paired_output:
			parser.error("Option --untrimmed-paired-output can only be used when "
				"trimming paired-end reads (with option -p).")
		if input_filename.endswith('.qual'):
			parser.error("Need a FASTA file in addition to the QUAL file.")
		if options.format is not None and quality_filename is not None:
			parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

	if options.format is not None and options.format.lower() not in ['fasta', 'fastq', 'sra-fastq']:
		parser.error("The input file format must be either 'fasta', 'fastq' or "
			"'sra-fastq' (not '{0}').".format(options.format))

	if options.quality_cutoff is not None:
		cutoffs = options.quality_cutoff.split(',')
		if len(cutoffs) == 1:
			try:
				cutoffs = [0, int(cutoffs[0])]
			except ValueError as e:
				parser.error("Quality cutoff value not recognized: {0}".format(e))
		elif len(cutoffs) == 2:
			try:
				cutoffs = [int(cutoffs[0]), int(cutoffs[1])]
			except ValueError as e:
				parser.error("Quality cutoff value not recognized: {0}".format(e))
		else:
			parser.error("Expected one value or two values separated by comma for the quality cutoff")
	else:
		cutoffs = None
	writers = []
	too_short_outfile = None  # too short reads go here
	too_short_filter = None
	# TODO pass file name to TooShortReadFilter, add a .close() method?
	if options.minimum_length > 0:
		if options.too_short_output:
			too_short_outfile = xopen(options.too_short_output, 'w')
		else:
			too_short_outfile = None
		too_short_filter = TooShortReadFilter(options.minimum_length,
			too_short_outfile, paired=='both')
		writers.append(too_short_filter)
	too_long_outfile = None  # too long reads go here
	too_long_filter = None
	if options.maximum_length < sys.maxsize:
		if options.too_long_output is not None:
			too_long_outfile = xopen(options.too_long_output, 'w')
		else:
			too_long_outfile = None
		too_long_filter = TooLongReadFilter(options.maximum_length,
			too_long_outfile, check_second=paired=='both')
		writers.append(too_long_filter)

	if options.max_n != -1:
		writers.append(NContentTrimmer(options.max_n, check_second=paired=='both'))

	demultiplexer = None
	if options.output is not None and '{name}' in options.output:
		if options.discard_trimmed:
			parser.error("Do not use --discard-trimmed when demultiplexing.")
		if paired:
			parser.error("Demultiplexing not supported for paired-end files, yet.")
		untrimmed = options.output.format(name='unknown')
		if options.untrimmed_output:
			untrimmed = options.untrimmed_output
		if options.discard_untrimmed:
			untrimmed = None
		demultiplexer = Demultiplexer(options.output, untrimmed)
		writers.append(demultiplexer)
		trimmed_outfile, untrimmed_outfile = None, None
		trimmed_paired_outfile, untrimmed_paired_outfile = None, None
	else:
		trimmed_outfile, untrimmed_outfile = trimmed_and_untrimmed_files(
			default_outfile,
			options.output,
			options.untrimmed_output,
			options.discard_trimmed,
			options.discard_untrimmed)

		trimmed_paired_outfile, untrimmed_paired_outfile = trimmed_and_untrimmed_files(
			None,  # applies when not trimming paired-end data
			options.paired_output,
			options.untrimmed_paired_output,
			options.discard_trimmed,
			options.discard_untrimmed)

		writer = ProcessedReadWriter(
			trimmed_outfile, trimmed_paired_outfile,
			untrimmed_outfile, untrimmed_paired_outfile,
			check_second=paired=='both'
		)
		writers.append(writer)

	if options.maq:
		options.colorspace = True
		options.double_encode = True
		options.trim_primer = True
		options.strip_suffix.append('_F3')
		options.suffix = "/1"
	if options.zero_cap is None:
		options.zero_cap = options.colorspace
	if options.trim_primer and not options.colorspace:
		parser.error("Trimming the primer makes only sense in colorspace.")
	if options.double_encode and not options.colorspace:
		parser.error("Double-encoding makes only sense in colorspace.")
	if options.anywhere and options.colorspace:
		parser.error("Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author).")
	if not (0 <= options.error_rate <= 1.):
		parser.error("The maximum error rate must be between 0 and 1.")
	if options.overlap < 1:
		parser.error("The overlap must be at least 1.")

	if options.rest_file is not None:
		options.rest_file = xopen(options.rest_file, 'w')
		rest_writer = RestFileWriter(options.rest_file)
	else:
		rest_writer = None
	if options.info_file is not None:
		options.info_file = xopen(options.info_file, 'w')
	if options.wildcard_file is not None:
		options.wildcard_file = xopen(options.wildcard_file, 'w')

	if options.colorspace:
		if options.match_read_wildcards:
			parser.error('IUPAC wildcards not supported in colorspace')
		options.match_adapter_wildcards = False

	ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter
	try:
		# TODO refactor, code duplicated
		adapters = []
		for name, seq, where in gather_adapters(options.adapters, options.anywhere, options.front):
			if not seq:
				parser.error("The adapter sequence is empty.")
			if not options.indels and where not in (PREFIX, SUFFIX):
				parser.error("Not allowing indels is currently supported only for anchored 5' and 3' adapters.")
			adapter = ADAPTER_CLASS(seq, where, options.error_rate,
				options.overlap, options.match_read_wildcards,
				options.match_adapter_wildcards, name=name, indels=options.indels)
			adapters.append(adapter)
		adapters2 = []
		for name, seq, where in gather_adapters(options.adapters2, options.anywhere2, options.front2):
			if not seq:
				parser.error("The adapter sequence is empty.")
			if not options.indels and where != PREFIX:
				parser.error("Not allowing indels is currently supported only for anchored 5' and 3' adapters.")
			adapter = ADAPTER_CLASS(seq, where, options.error_rate,
				options.overlap, options.match_read_wildcards,
				options.match_adapter_wildcards, name=name, indels=options.indels)
			adapters2.append(adapter)
	except IOError as e:
		if e.errno == errno.ENOENT:
			parser.error(e)
		raise

	if not adapters and not adapters2 and not cutoffs and \
			options.cut == [] and options.cut2 == [] and \
			options.minimum_length == 0 and \
			options.maximum_length == sys.maxsize and \
			quality_filename is None and \
			options.max_n == -1:
		parser.error("You need to provide at least one adapter sequence.")

	try:
		reader = seqio.open(input_filename, file2=input_paired_filename,
				qualfile=quality_filename, colorspace=options.colorspace,
				fileformat=options.format)
	except (seqio.UnknownFileType, IOError) as e:
		parser.error(e)

	# Create the processing pipeline consisting of a list of "modifiers".
	modifiers = []
	if options.cut:
		if len(options.cut) > 2:
			parser.error("You cannot remove bases from more than two ends.")
		if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
			parser.error("You cannot remove bases from the same end twice.")
		for cut in options.cut:
			if cut != 0:
				modifiers.append(UnconditionalCutter(cut))

	if cutoffs:
		modifiers.append(QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
	if adapters:
		adapter_cutter = AdapterCutter(adapters, options.times,
				options.wildcard_file, options.info_file,
				rest_writer, options.action)
		modifiers.append(adapter_cutter)
	else:
		adapter_cutter = None

	# Modifiers that apply to both reads of paired-end reads
	modifiers_both = []
	if options.trim_n:
		modifiers_both.append(NEndTrimmer())
	if options.length_tag:
		modifiers_both.append(LengthTagModifier(options.length_tag))
	if options.strip_f3:
		options.strip_suffix.append('_F3')
	for suffix in options.strip_suffix:
		modifiers_both.append(SuffixRemover(suffix))
	if options.prefix or options.suffix:
		modifiers_both.append(PrefixSuffixAdder(options.prefix, options.suffix))
	if options.double_encode:
		modifiers_both.append(DoubleEncoder())
	if options.zero_cap and reader.delivers_qualities:
		modifiers_both.append(ZeroCapper(quality_base=options.quality_base))
	if options.trim_primer:
		modifiers_both.append(PrimerTrimmer)
	modifiers.extend(modifiers_both)

	# For paired-end data, create a second processing pipeline.
	# However, if no second-read adapters were given (via -A/-G/-B/-U), we need to
	# be backwards compatible and *no modifications* are done to the second read.
	modifiers2 = []
	if paired == 'both':
		if options.cut2:
			if len(options.cut2) > 2:
				parser.error("You cannot remove bases from more than two ends.")
			if len(options.cut2) == 2 and options.cut2[0] * options.cut2[1] > 0:
				parser.error("You cannot remove bases from the same end twice.")
			for cut in options.cut2:
				if cut != 0:
					modifiers2.append(UnconditionalCutter(cut))

		if cutoffs:
			modifiers2.append(QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
		if adapters2:
			adapter_cutter2 = AdapterCutter(adapters2, options.times,
					None, None, None, options.action)
			modifiers2.append(adapter_cutter2)
		else:
			adapter_cutter2 = None
		modifiers2.extend(modifiers_both)

	# Due to backwards compatibility, from here on logging output needs to be
	# sent to standard output instead of standard error if the -o option is used.
	if options.output:
		logger.root.handlers = []
		logging.basicConfig(level=logging.INFO, format='%(message)s', stream=sys.stdout)
	logger.info("This is cutadapt %s with Python %s", __version__, platform.python_version())
	logger.info("Command line parameters: %s", " ".join(cmdlineargs))
	logger.info("Trimming %s adapter%s with at most %.1f%% errors in %s mode ...",
		len(adapters) + len(adapters2), 's' if len(adapters) + len(adapters2) != 1 else '',
		options.error_rate * 100,
		{ False: 'single-end', 'first': 'paired-end legacy', 'both': 'paired-end' }[paired])

	start_time = time.clock()
	try:
		if paired:
			stats = process_paired_reads(reader, modifiers, modifiers2, writers)
		else:
			stats = process_single_reads(reader, modifiers, writers)
	except KeyboardInterrupt as e:
		print("Interrupted", file=sys.stderr)
		sys.exit(130)
	except IOError as e:
		if e.errno == errno.EPIPE:
			sys.exit(1)
		raise
	except (seqio.FormatError, EOFError) as e:
		sys.exit("cutadapt: error: {0}".format(e))

	# close open files
	for f in [trimmed_outfile, untrimmed_outfile, trimmed_paired_outfile,
			untrimmed_paired_outfile, options.rest_file, options.wildcard_file,
			options.info_file, too_short_outfile, too_long_outfile,
			options.info_file, demultiplexer]:
		if f is not None and f is not sys.stdin and f is not sys.stdout:
			f.close()

	elapsed_time = time.clock() - start_time
	if not options.quiet:
		stats.collect((adapters, adapters2), elapsed_time,
			modifiers, modifiers2, writers)
		# send statistics to stderr if result was sent to stdout
		stat_file = sys.stderr if options.output is None else None
		with redirect_standard_output(stat_file):
			print_report(stats, (adapters, adapters2))
Example #39
0
def main(cmdlineargs=None, trimmed_outfile=sys.stdout):
	"""
	Main function that evaluates command-line parameters and iterates
	over all reads.

	trimmed_outfile is the default output file to which trimmed reads
	are sent. It can be overriden by using the '-o' parameter.
	"""
	parser = HelpfulOptionParser(usage=__doc__, version=__version__)

	parser.add_option("-f", "--format", default=None,
		help="Input file format; can be either 'fasta', 'fastq' or 'sra-fastq'. "
		"Ignored when reading csfasta/qual files (default: auto-detect "
		"from file name extension).")

	group = OptionGroup(parser, "Options that influence how the adapters are found",
		description="Each of the following three parameters (-a, -b, -g) can be used "
			"multiple times and in any combination to search for an entire set of "
			"adapters of possibly different types. All of the "
			"given adapters will be searched for in each read, but only the best "
			"matching one will be trimmed (but see the --times option).")
	group.add_option("-a", "--adapter", action="append", metavar="ADAPTER", dest="adapters", default=[],
		help="Sequence of an adapter that was ligated to the 3' end. The adapter itself and anything that follows is trimmed.")
	group.add_option("-b", "--anywhere", action="append", metavar="ADAPTER", default=[],
		help="Sequence of an adapter that was ligated to the 5' or 3' end. If the adapter is found within the read or overlapping the 3' end of the read, the behavior is the same as for the -a option. If the adapter overlaps the 5' end (beginning of the read), the initial portion of the read matching the adapter is trimmed, but anything that follows is kept.")
	group.add_option("-g", "--front", action="append", metavar="ADAPTER", default=[],
		help="Sequence of an adapter that was ligated to the 5' end. If the "
		"adapter sequence starts with the character '^', the adapter is "
		"'anchored'. An anchored adapter must appear in its entirety at the "
		"5' end of the read (it is a prefix of the read). A non-anchored adapter may "
		"appear partially at the 5' end, or it may occur within the read. If it is "
		"found within a read, the sequence preceding the adapter is also trimmed. "
		"In all cases, the adapter itself is trimmed.")
	group.add_option("-e", "--error-rate", type=float, default=0.1,
		help="Maximum allowed error rate (no. of errors divided by the length of the matching region) (default: %default)")
	group.add_option("-n", "--times", type=int, metavar="COUNT", default=1,
		help="Try to remove adapters at most COUNT times. Useful when an adapter gets appended multiple times (default: %default).")
	group.add_option("-O", "--overlap", type=int, metavar="LENGTH", default=3,
		help="Minimum overlap length. If the overlap between the read and the adapter is shorter than LENGTH, the read is not modified."
			"This reduces the no. of bases trimmed purely due to short random adapter matches (default: %default).")
	group.add_option("--match-read-wildcards", action="store_true", default=False,
		help="Allow 'N's in the read as matches to the adapter (default: %default).")
	group.add_option("-N", "--no-match-adapter-wildcards", action="store_false",
		default=True, dest='match_adapter_wildcards',
		help="Do not treat 'N' in the adapter sequence as wildcards. This is needed when you want to search for literal 'N' characters.")
	parser.add_option_group(group)

	group = OptionGroup(parser, "Options for filtering of processed reads")
	group.add_option("--discard-trimmed", "--discard", action='store_true', default=False,
		help="Discard reads that contain the adapter instead of trimming them. Also use -O in order to avoid throwing away too many randomly matching reads!")
	group.add_option("--discard-untrimmed", "--trimmed-only", action='store_true', default=False,
		help="Discard reads that do not contain the adapter.")
	group.add_option("-m", "--minimum-length", type=int, default=0, metavar="LENGTH",
		help="Discard trimmed reads that are shorter than LENGTH. Reads that are too short even before adapter removal are also discarded. In colorspace, an initial primer is not counted (default: 0).")
	group.add_option("-M", "--maximum-length", type=int, default=sys.maxsize, metavar="LENGTH",
		help="Discard trimmed reads that are longer than LENGTH. "
			"Reads that are too long even before adapter removal "
			"are also discarded. In colorspace, an initial primer "
			"is not counted (default: no limit).")
	group.add_option("--no-trim", dest='trim', action='store_false', default=True,
		help="Match and redirect reads to output/untrimmed-output as usual, but don't remove the adapters. (default: False. Remove the adapters)")
	parser.add_option_group(group)

	group = OptionGroup(parser, "Options that influence what gets output to where")
	group.add_option("-o", "--output", default=None, metavar="FILE",
		help="Write the modified sequences to this file instead of standard output and send the summary report to standard output. "
		     "The format is FASTQ if qualities are available, FASTA otherwise. (default: standard output)")
	group.add_option("--info-file", metavar="FILE",
		help="Write information about each read and its adapter matches into FILE. "
			"Currently experimental: Expect the file format to change!")
	group.add_option("-r", "--rest-file", default=None, metavar="FILE",
		help="When the adapter matches in the middle of a read, write the rest (after the adapter) into a file. Use - for standard output.")
	group.add_option("--wildcard-file", default=None, metavar="FILE",
		help="When the adapter has wildcard bases ('N's) write adapter bases matching wildcard "
		     "positions to FILE. Use - for standard output. "
		     "When there are indels in the alignment, this may occasionally "
		     "not be quite accurate.")
	group.add_option("--too-short-output", default=None, metavar="FILE",
		help="Write reads that are too short (according to length specified by -m) to FILE. (default: discard reads)")
	group.add_option("--too-long-output", default=None, metavar="FILE",
		help="Write reads that are too long (according to length specified by -M) to FILE. (default: discard reads)")
	group.add_option("--untrimmed-output", default=None, metavar="FILE",
		help="Write reads that do not contain the adapter to FILE, instead "
			"of writing them to the regular output file. (default: output "
			"to same file as trimmed)")
	group.add_option('-p', "--paired-output", default=None, metavar="FILE",
		help="Write reads from the paired end input to FILE. ")
	parser.add_option_group(group)

	group = OptionGroup(parser, "Additional modifications to the reads")
	group.add_option("-q", "--quality-cutoff", type=int, default=0, metavar="CUTOFF",
		help="Trim low-quality ends from reads before adapter removal. "
			"The algorithm is the same as the one used by BWA "
			"(Subtract CUTOFF from all qualities; "
			"compute partial sums from all indices to the end of the "
			"sequence; cut sequence at the index at which the sum "
			"is minimal) (default: %default)")
	group.add_option("--quality-base", type=int, default=33,
		help="Assume that quality values are encoded as ascii(quality + QUALITY_BASE). The default (33) is usually correct, "
		     "except for reads produced by some versions of the Illumina pipeline, where this should be set to 64. (default: %default)")
	group.add_option("-x", "--prefix", default='',
		help="Add this prefix to read names")
	group.add_option("-y", "--suffix", default='',
		help="Add this suffix to read names")
	group.add_option("--strip-suffix", action='append', default=[],
		help="Remove this suffix from read names if present. Can be given multiple times.")
	group.add_option("-c", "--colorspace", action='store_true', default=False,
		help="Colorspace mode: Also trim the color that is adjacent to the found adapter.")
	group.add_option("-d", "--double-encode", action='store_true', default=False,
		help="When in color space, double-encode colors (map 0,1,2,3,4 to A,C,G,T,N).")
	group.add_option("-t", "--trim-primer", action='store_true', default=False,
		help="When in color space, trim primer base and the first color "
			"(which is the transition to the first nucleotide)")
	group.add_option("--strip-f3", action='store_true', default=False,
		help="For color space: Strip the _F3 suffix of read names")
	group.add_option("--maq", "--bwa", action='store_true', default=False,
		help="MAQ- and BWA-compatible color space output. This enables -c, -d, -t, --strip-f3, -y '/1' and -z.")
	group.add_option("--length-tag", default=None, metavar="TAG",
		help="Search for TAG followed by a decimal number in the name of the read "
			"(description/comment field of the FASTA or FASTQ file). Replace the "
			"decimal number with the correct length of the trimmed read. "
			"For example, use --length-tag 'length=' to correct fields "
			"like 'length=123'.")
	group.add_option("--zero-cap", "-z", action='store_true', default=False,
		help="Change negative quality values to zero (workaround to avoid segmentation faults in BWA)")
	parser.add_option_group(group)

	options, args = parser.parse_args(args=cmdlineargs)

	if len(args) == 0:
		parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
	elif len(args) > 2:
		parser.error("Too many parameters.")

	input_filename = args[0]
	quality_filename = None
	pe_filename = None
	if len(args) == 2:
		if args[1].endswith('.qual'):
			quality_filename = args[1]
		else:
			pe_filename = args[1]
			if not options.paired_output:
				parser.error('you must use --paired-output when trimming paired-end reads')

	if len(args) == 1 and options.paired_output:
		parser.error("You specified a --paired-output file, but gave only one input file.")
	if input_filename.endswith('.qual') and quality_filename.endswith('fasta'):
		parser.error("FASTA and QUAL file given, but the FASTA file must be first.")

	if options.format is not None and options.format.lower() not in ['fasta', 'fastq', 'sra-fastq']:
		parser.error("The input file format must be either 'fasta', 'fastq' or 'sra-fastq' (not '{0}').".format(options.format))

	# TODO should this really be an error?
	if options.format is not None and quality_filename is not None:
		parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

	# default output files (overwritten below)
	too_short_outfile = None # too short reads go here
	too_long_outfile = None # too long reads go here
	pe_outfile = None

	if options.output is not None:
		trimmed_outfile = xopen(options.output, 'w')
	untrimmed_outfile = trimmed_outfile # reads without adapters go here
	if options.untrimmed_output is not None:
		untrimmed_outfile = xopen(options.untrimmed_output, 'w')
	if options.too_short_output is not None:
		too_short_outfile = xopen(options.too_short_output, 'w')
	if options.too_long_output is not None:
		too_long_outfile = xopen(options.too_long_output, 'w')
	if options.paired_output:
		pe_outfile = xopen(options.paired_output, 'w')

	if options.maq:
		options.colorspace = True
		options.double_encode = True
		options.trim_primer = True
		options.strip_suffix.append('_F3')
		options.suffix = "/1"
		options.zero_cap = True
	if options.trim_primer and not options.colorspace:
		parser.error("Trimming the primer makes only sense in color space.")
	if options.double_encode and not options.colorspace:
		parser.error("Double-encoding makes only sense in color space.")
	if options.anywhere and options.colorspace:
		parser.error("Using --anywhere with color space reads is currently not supported  (if you think this may be useful, contact the author).")
	if not (0 <= options.error_rate <= 1.):
		parser.error("The maximum error rate must be between 0 and 1.")
	if options.overlap < 1:
		parser.error("The overlap must be at least 1.")

	if options.rest_file is not None:
		options.rest_file = xopen(options.rest_file, 'w')
		rest_writer = RestFileWriter(options.rest_file)
	else:
		rest_writer = None
	if options.info_file is not None:
		options.info_file = xopen(options.info_file, 'w')
	if options.wildcard_file is not None:
		options.wildcard_file = xopen(options.wildcard_file, 'w')

	adapters = []

	ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter
	def append_adapters(adapter_list, where):
		for seq in adapter_list:
			fields = seq.split('=', 1)
			if len(fields) > 1:
				name, seq = fields
				name = name.strip()
			else:
				name = None
			seq = seq.strip()
			w = where
			if w == FRONT and seq.startswith('^'):
				seq = seq[1:]
				w = PREFIX
			if len(seq) == 0:
				parser.error("The adapter sequence is empty")
			adapter = ADAPTER_CLASS(seq, w, options.error_rate,
				options.overlap, options.match_read_wildcards,
				options.match_adapter_wildcards, name=name)
			adapters.append(adapter)

	append_adapters(options.adapters, BACK)
	append_adapters(options.anywhere, ANYWHERE)
	append_adapters(options.front, FRONT)

	# make sure these aren't used by accident
	del options.adapters
	del options.anywhere
	del options.front

	if not adapters and options.quality_cutoff == 0:
		print("You need to provide at least one adapter sequence.", file=sys.stderr)
		sys.exit(1)

	modifiers = []
	if options.length_tag:
		modifiers.append(LengthTagModifier(options.length_tag))
	if options.strip_f3:
		options.strip_suffix.append('_F3')
	for suffix in options.strip_suffix:
		modifiers.append(SuffixRemover(suffix))
	if options.prefix or options.suffix:
		modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
	if options.double_encode:
		modifiers.append(DoubleEncoder())
	if options.zero_cap:
		modifiers.append(ZeroCapper(quality_base=options.quality_base))
	if options.quality_cutoff > 0:
		quality_trimmer = QualityTrimmer(options.quality_cutoff, options.quality_base)
	else:
		quality_trimmer = None

	adapter_matcher = RepeatedAdapterMatcher(adapters, options.times,
				options.wildcard_file, options.info_file, options.trim)
	readfilter = ReadFilter(options.minimum_length, options.maximum_length,
		too_short_outfile, too_long_outfile, options.discard_trimmed,
		options.discard_untrimmed, options.trim_primer)
	start_time = time.clock()
	try:
		reader = read_sequences(input_filename, quality_filename, colorspace=options.colorspace, fileformat=options.format)
		if pe_filename:
			pe_reader = read_sequences(pe_filename, None, colorspace=options.colorspace, fileformat=options.format)
		else:
			pe_reader = None
		(n, total_bp) = process_reads(reader, pe_reader, adapter_matcher, quality_trimmer, modifiers, readfilter, trimmed_outfile, untrimmed_outfile, pe_outfile, rest_writer, options.trim_primer)
	except IOError as e:
		if e.errno == errno.EPIPE:
			sys.exit(1)
		raise
	except seqio.FormatError as e:
		print("Error:", e, file=sys.stderr)
		sys.exit(1)
	# close open files
	for f in [options.rest_file, options.wildcard_file, options.info_file,
			too_short_outfile, too_long_outfile, options.info_file]:
		if f is not None:
			f.close()
	# send statistics to stderr if result was sent to stdout
	stat_file = sys.stderr if options.output is None else None

	total_quality_trimmed = quality_trimmer.trimmed_bases if quality_trimmer else -1
	print_statistics(adapters, time.clock() - start_time,
		n, total_bp, total_quality_trimmed, options.trim, adapter_matcher.reads_matched,
		options.error_rate, readfilter.too_short, readfilter.too_long, file=stat_file)
Example #40
0
def main(cmdlineargs=None, trimmed_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	trimmed_outfile is the default output file to which trimmed reads
	are sent. It can be overriden by using the '-o' parameter.
	"""
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")

    input_filename = args[0]
    quality_filename = None
    pe_filename = None
    if len(args) == 2:
        if args[1].endswith(".qual"):
            quality_filename = args[1]
        else:
            pe_filename = args[1]
            if not options.paired_output:
                parser.error("you must use --paired-output when trimming paired-end reads")

    if len(args) == 1 and options.paired_output:
        parser.error("You specified a --paired-output file, but gave only one input file.")
    if input_filename.endswith(".qual") and quality_filename.endswith("fasta"):
        parser.error("FASTA and QUAL file given, but the FASTA file must be first.")

    if options.format is not None and options.format.lower() not in ["fasta", "fastq", "sra-fastq"]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or 'sra-fastq' (not '{0}').".format(options.format)
        )

        # TODO should this really be an error?
    if options.format is not None and quality_filename is not None:
        parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

        # default output files (overwritten below)
    too_short_outfile = None  # too short reads go here
    too_long_outfile = None  # too long reads go here
    pe_outfile = None

    if options.output is not None:
        trimmed_outfile = xopen(options.output, "w")
    untrimmed_outfile = trimmed_outfile  # reads without adapters go here
    if options.untrimmed_output is not None:
        untrimmed_outfile = xopen(options.untrimmed_output, "w")
    if options.too_short_output is not None:
        too_short_outfile = xopen(options.too_short_output, "w")
    if options.too_long_output is not None:
        too_long_outfile = xopen(options.too_long_output, "w")
    if options.paired_output:
        pe_outfile = xopen(options.paired_output, "w")

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append("_F3")
        options.suffix = "/1"
        options.zero_cap = True
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in color space.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in color space.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with color space reads is currently not supported  (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.0):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, "w")
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, "w")
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, "w")

    adapters = []

    def parse_adapter_name(seq):
        """
		Parse an adapter given as 'name=adapt' into 'name' and 'adapt'.
		"""
        fields = seq.split("=", 1)
        if len(fields) > 1:
            name, seq = fields
            name = name.strip()
        else:
            name = None
        seq = seq.strip()
        return name, seq

    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

    def append_adapters(adapter_list, where):
        for seq in adapter_list:
            name, seq = parse_adapter_name(seq)
            w = where
            if w == FRONT and seq.startswith("^"):
                seq = seq[1:]
                w = PREFIX
            elif not options.indels:
                parser.error("Not allowing indels is currently supported only for anchored 5' adapters.")
            if not seq:
                parser.error("The adapter sequence is empty")
            adapter = ADAPTER_CLASS(
                seq,
                w,
                options.error_rate,
                options.overlap,
                options.match_read_wildcards,
                options.match_adapter_wildcards,
                name=name,
                indels=options.indels,
            )
            adapters.append(adapter)

    append_adapters(options.adapters, BACK)
    append_adapters(options.anywhere, ANYWHERE)
    append_adapters(options.front, FRONT)

    # make sure these aren't used by accident
    del options.adapters
    del options.anywhere
    del options.front

    if not adapters and options.quality_cutoff == 0:
        parser.error("You need to provide at least one adapter sequence.")

    modifiers = []
    if options.length_tag:
        modifiers.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append("_F3")
    for suffix in options.strip_suffix:
        modifiers.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
    if options.double_encode:
        modifiers.append(DoubleEncoder())
    if options.zero_cap:
        modifiers.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers.append(PrimerTrimmer())
    if options.quality_cutoff > 0:
        quality_trimmer = QualityTrimmer(options.quality_cutoff, options.quality_base)
    else:
        quality_trimmer = None

    adapter_matcher = RepeatedAdapterMatcher(
        adapters, options.times, options.wildcard_file, options.info_file, options.trim
    )
    readfilter = ReadFilter(
        options.minimum_length,
        options.maximum_length,
        too_short_outfile,
        too_long_outfile,
        options.discard_trimmed,
        options.discard_untrimmed,
    )
    start_time = time.clock()
    try:
        reader = read_sequences(
            input_filename, quality_filename, colorspace=options.colorspace, fileformat=options.format
        )
        if pe_filename:
            pe_reader = read_sequences(pe_filename, None, colorspace=options.colorspace, fileformat=options.format)
        else:
            pe_reader = None
        (n, total_bp) = process_reads(
            reader,
            pe_reader,
            adapter_matcher,
            quality_trimmer,
            modifiers,
            readfilter,
            trimmed_outfile,
            untrimmed_outfile,
            pe_outfile,
            rest_writer,
        )
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except seqio.FormatError as e:
        print("Error:", e, file=sys.stderr)
        sys.exit(1)
        # close open files
    for f in [
        options.rest_file,
        options.wildcard_file,
        options.info_file,
        too_short_outfile,
        too_long_outfile,
        options.info_file,
    ]:
        if f is not None:
            f.close()
            # send statistics to stderr if result was sent to stdout
    stat_file = sys.stderr if options.output is None else None

    total_quality_trimmed = quality_trimmer.trimmed_bases if quality_trimmer else -1
    print_statistics(
        adapters,
        time.clock() - start_time,
        n,
        total_bp,
        total_quality_trimmed,
        options.trim,
        adapter_matcher.reads_matched,
        options.error_rate,
        readfilter.too_short,
        readfilter.too_long,
        cmdlineargs,
        file=stat_file,
    )
Example #41
0
def main(cmdlineargs=None, trimmed_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	trimmed_outfile is the default output file to which trimmed reads
	are sent. It can be overriden by using the '-o' parameter.
	"""
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error(
            "At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")

    input_filename = args[0]
    quality_filename = None
    pe_filename = None
    if len(args) == 2:
        if args[1].endswith('.qual'):
            quality_filename = args[1]
        else:
            pe_filename = args[1]
            if not options.paired_output:
                parser.error(
                    'you must use --paired-output when trimming paired-end reads'
                )

    if len(args) == 1 and options.paired_output:
        parser.error(
            "You specified a --paired-output file, but gave only one input file."
        )
    if input_filename.endswith('.qual') and quality_filename.endswith('fasta'):
        parser.error(
            "FASTA and QUAL file given, but the FASTA file must be first.")

    if options.format is not None and options.format.lower() not in [
            'fasta', 'fastq', 'sra-fastq'
    ]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or 'sra-fastq' (not '{0}')."
            .format(options.format))

    # TODO should this really be an error?
    if options.format is not None and quality_filename is not None:
        parser.error(
            "If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used."
        )

    # default output files (overwritten below)
    too_short_outfile = None  # too short reads go here
    too_long_outfile = None  # too long reads go here
    pe_outfile = None

    if options.output is not None:
        trimmed_outfile = xopen(options.output, 'w')
    untrimmed_outfile = trimmed_outfile  # reads without adapters go here
    if options.untrimmed_output is not None:
        untrimmed_outfile = xopen(options.untrimmed_output, 'w')
    if options.too_short_output is not None:
        too_short_outfile = xopen(options.too_short_output, 'w')
    if options.too_long_output is not None:
        too_long_outfile = xopen(options.too_long_output, 'w')
    if options.paired_output:
        pe_outfile = xopen(options.paired_output, 'w')

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append('_F3')
        options.suffix = "/1"
        options.zero_cap = True
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in color space.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in color space.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with color space reads is currently not supported  (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, 'w')
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, 'w')
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, 'w')

    adapters = []

    def parse_adapter_name(seq):
        """
		Parse an adapter given as 'name=adapt' into 'name' and 'adapt'.
		"""
        fields = seq.split('=', 1)
        if len(fields) > 1:
            name, seq = fields
            name = name.strip()
        else:
            name = None
        seq = seq.strip()
        return name, seq

    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

    def append_adapters(adapter_list, where):
        for seq in adapter_list:
            name, seq = parse_adapter_name(seq)
            w = where
            if w == FRONT and seq.startswith('^'):
                seq = seq[1:]
                w = PREFIX
            elif not options.indels:
                parser.error(
                    "Not allowing indels is currently supported only for anchored 5' adapters."
                )
            if not seq:
                parser.error("The adapter sequence is empty")
            adapter = ADAPTER_CLASS(seq,
                                    w,
                                    options.error_rate,
                                    options.overlap,
                                    options.match_read_wildcards,
                                    options.match_adapter_wildcards,
                                    name=name,
                                    indels=options.indels)
            adapters.append(adapter)

    append_adapters(options.adapters, BACK)
    append_adapters(options.anywhere, ANYWHERE)
    append_adapters(options.front, FRONT)

    # make sure these aren't used by accident
    del options.adapters
    del options.anywhere
    del options.front

    if not adapters and options.quality_cutoff == 0:
        print("You need to provide at least one adapter sequence.",
              file=sys.stderr)
        sys.exit(1)

    modifiers = []
    if options.length_tag:
        modifiers.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append('_F3')
    for suffix in options.strip_suffix:
        modifiers.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
    if options.double_encode:
        modifiers.append(DoubleEncoder())
    if options.zero_cap:
        modifiers.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers.append(PrimerTrimmer())
    if options.quality_cutoff > 0:
        quality_trimmer = QualityTrimmer(options.quality_cutoff,
                                         options.quality_base)
    else:
        quality_trimmer = None

    adapter_matcher = RepeatedAdapterMatcher(adapters, options.times,
                                             options.wildcard_file,
                                             options.info_file, options.trim)
    readfilter = ReadFilter(options.minimum_length, options.maximum_length,
                            too_short_outfile, too_long_outfile,
                            options.discard_trimmed, options.discard_untrimmed)
    start_time = time.clock()
    try:
        reader = read_sequences(input_filename,
                                quality_filename,
                                colorspace=options.colorspace,
                                fileformat=options.format)
        if pe_filename:
            pe_reader = read_sequences(pe_filename,
                                       None,
                                       colorspace=options.colorspace,
                                       fileformat=options.format)
        else:
            pe_reader = None
        (n, total_bp) = process_reads(reader, pe_reader, adapter_matcher,
                                      quality_trimmer, modifiers, readfilter,
                                      trimmed_outfile, untrimmed_outfile,
                                      pe_outfile, rest_writer)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except seqio.FormatError as e:
        print("Error:", e, file=sys.stderr)
        sys.exit(1)
    # close open files
    for f in [
            options.rest_file, options.wildcard_file, options.info_file,
            too_short_outfile, too_long_outfile, options.info_file
    ]:
        if f is not None:
            f.close()
    # send statistics to stderr if result was sent to stdout
    stat_file = sys.stderr if options.output is None else None

    total_quality_trimmed = quality_trimmer.trimmed_bases if quality_trimmer else -1
    print_statistics(adapters,
                     time.clock() - start_time,
                     n,
                     total_bp,
                     total_quality_trimmed,
                     options.trim,
                     adapter_matcher.reads_matched,
                     options.error_rate,
                     readfilter.too_short,
                     readfilter.too_long,
                     cmdlineargs,
                     file=stat_file)
def main(cmdlineargs=None, default_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)

    if len(args) == 0:
        parser.error(
            "At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")

    input_filename = args[0]

    # If a second file name was given, then we either have single-end reads
    # provided as a pair of .fasta/.qual files or we have paired-end reads.
    quality_filename = None
    input_paired_filename = None
    if len(args) == 2:
        if args[0].endswith('.qual'):
            parser.error("The QUAL file must be the second argument.")
        if args[1].endswith('.qual'):
            quality_filename = args[1]
        else:
            input_paired_filename = args[1]
            if not options.paired_output:
                parser.error(
                    'You must use --paired-output when trimming paired-end reads.'
                )

    if len(args) == 1 and options.paired_output:
        parser.error(
            "You specified a --paired-output file, but gave only one input file."
        )
    if options.paired_output and bool(options.untrimmed_output) != bool(
            options.untrimmed_paired_output):
        parser.error(
            "When trimming paired-end reads, you must use either none "
            "or both of the --untrimmed-output/--untrimmed-paired-output options."
        )
    if options.untrimmed_paired_output and not options.paired_output:
        parser.error(
            "Option --untrimmed-paired-output can only be used when "
            "trimming paired-end reads (with option --paired-output).")
    if input_filename.endswith('.qual'):
        parser.error("Need a FASTA file in addition to the QUAL file.")

    if options.format is not None and options.format.lower() not in [
            'fasta', 'fastq', 'sra-fastq'
    ]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or "
            "'sra-fastq' (not '{0}').".format(options.format))

    # TODO should this really be an error?
    if options.format is not None and quality_filename is not None:
        parser.error(
            "If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used."
        )

    writers = []
    too_short_outfile = None  # too short reads go here
    too_short_filter = None
    # TODO pass file name to TooShortReadFilter, add a .close() method?
    if options.minimum_length > 0:
        if options.too_short_output:
            too_short_outfile = xopen(options.too_short_output, 'w')
        else:
            too_short_outfile = None
        too_short_filter = TooShortReadFilter(options.minimum_length,
                                              too_short_outfile)
        writers.append(too_short_filter)
    too_long_outfile = None  # too long reads go here
    too_long_filter = None
    if options.maximum_length < sys.maxsize:
        if options.too_long_output is not None:
            too_long_outfile = xopen(options.too_long_output, 'w')
        else:
            too_long_outfile = None
        too_long_filter = TooLongReadFilter(options.maximum_length,
                                            too_long_outfile)
        writers.append(too_long_filter)

    demultiplexer = None
    if options.output is not None and '{name}' in options.output:
        if options.discard_trimmed:
            parser.error("Do not use --discard-trimmed when demultiplexing.")
        if input_paired_filename:
            parser.error(
                "Demultiplexing not supported for paired-end files, yet.")
        untrimmed = options.output.format(name='unknown')
        if options.untrimmed_output:
            untrimmed = options.untrimmed_output
        if options.discard_untrimmed:
            untrimmed = None
        demultiplexer = Demultiplexer(options.output, untrimmed)
        writers.append(demultiplexer)
        trimmed_outfile, untrimmed_outfile = None, None
        trimmed_paired_outfile, untrimmed_paired_outfile = None, None
    else:
        trimmed_outfile, untrimmed_outfile = trimmed_and_untrimmed_files(
            default_outfile, options.output, options.untrimmed_output,
            options.discard_trimmed, options.discard_untrimmed)

        trimmed_paired_outfile, untrimmed_paired_outfile = trimmed_and_untrimmed_files(
            None,  # applies when not trimming paired-end data
            options.paired_output,
            options.untrimmed_paired_output,
            options.discard_trimmed,
            options.discard_untrimmed)

        writer = ProcessedReadWriter(trimmed_outfile, trimmed_paired_outfile,
                                     untrimmed_outfile,
                                     untrimmed_paired_outfile)
        writers.append(writer)

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append('_F3')
        options.suffix = "/1"
    if options.zero_cap is None:
        options.zero_cap = options.colorspace
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in colorspace.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in colorspace.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, 'w')
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, 'w')
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, 'w')

    if options.colorspace:
        if options.match_read_wildcards:
            parser.error('IUPAC wildcards not supported in colorspace')
        options.match_adapter_wildcards = False
    adapters = []
    ADAPTER_CLASS = ColorspaceAdapter if options.colorspace else Adapter

    try:
        for name, seq, where in gather_adapters(options.adapters,
                                                options.anywhere,
                                                options.front):
            if not seq:
                parser.error("The adapter sequence is empty")
            if not options.indels and where not in (PREFIX, SUFFIX):
                parser.error(
                    "Not allowing indels is currently supported only for anchored 5' and 3' adapters."
                )
            adapter = ADAPTER_CLASS(seq,
                                    where,
                                    options.error_rate,
                                    options.overlap,
                                    options.match_read_wildcards,
                                    options.match_adapter_wildcards,
                                    name=name,
                                    indels=options.indels)
            adapters.append(adapter)
    except IOError as e:
        if e.errno == errno.ENOENT:
            print("Error:", e, file=sys.stderr)
            sys.exit(1)
        raise

    if not adapters and options.quality_cutoff == 0 and options.cut == 0 and \
      options.minimum_length == 0 and options.maximum_length == sys.maxsize:
        parser.error("You need to provide at least one adapter sequence.")

    if input_paired_filename:
        reader = seqio.PairedSequenceReader(input_filename,
                                            input_paired_filename,
                                            colorspace=options.colorspace,
                                            fileformat=options.format)
    else:
        reader = read_sequences(input_filename,
                                quality_filename,
                                colorspace=options.colorspace,
                                fileformat=options.format)

    # Create the processing pipeline as a list of "modifiers".
    modifiers = []
    if options.cut:
        if len(options.cut) > 2:
            parser.error("You cannot remove bases from more than two ends.")
        if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
            parser.error("You cannot remove bases from the same end twice.")
        for cut in options.cut:
            if cut != 0:
                modifiers.append(UnconditionalCutter(cut))

    if options.quality_cutoff > 0:
        modifiers.append(
            QualityTrimmer(options.quality_cutoff, options.quality_base))
    if adapters:
        adapter_cutter = AdapterCutter(adapters, options.times,
                                       options.wildcard_file,
                                       options.info_file, rest_writer,
                                       options.action)
        modifiers.append(adapter_cutter)
    else:
        adapter_cutter = None
    if options.length_tag:
        modifiers.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append('_F3')
    for suffix in options.strip_suffix:
        modifiers.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers.append(PrefixSuffixAdder(options.prefix, options.suffix))
    if options.double_encode:
        modifiers.append(DoubleEncoder())
    if options.zero_cap and reader.delivers_qualities:
        modifiers.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers.append(PrimerTrimmer)

    start_time = time.clock()
    try:
        if input_paired_filename:
            stats = process_paired_reads(reader, modifiers, writers)
        else:
            stats = process_single_reads(reader, modifiers, writers)
    except KeyboardInterrupt as e:
        print("Interrupted", file=sys.stderr)
        sys.exit(1)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except seqio.FormatError as e:
        print("Error:", e, file=sys.stderr)
        sys.exit(1)

    # close open files
    for f in [
            trimmed_outfile, untrimmed_outfile, trimmed_paired_outfile,
            untrimmed_paired_outfile, options.rest_file, options.wildcard_file,
            options.info_file, too_short_outfile, too_long_outfile,
            options.info_file, demultiplexer
    ]:
        if f is not None and f is not sys.stdin and f is not sys.stdout:
            f.close()

    if not options.quiet:
        # send statistics to stderr if result was sent to stdout
        stat_file = sys.stderr if options.output is None else None
        print_statistics(adapters,
                         time.clock() - start_time,
                         stats,
                         options.action,
                         adapter_cutter.reads_matched if adapter_cutter else 0,
                         options.error_rate,
                         too_short_filter.too_short if too_short_filter else 0,
                         too_long_filter.too_long if too_long_filter else 0,
                         cmdlineargs,
                         file=stat_file)
Example #43
0
def main(cmdlineargs=None, default_outfile=sys.stdout):
    """
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
    parser = get_option_parser()
    if cmdlineargs is None:
        cmdlineargs = sys.argv[1:]
    options, args = parser.parse_args(args=cmdlineargs)
    # Setup logging only if there are not already any handlers (can happen when
    # this function is being called externally such as from unit tests)
    if not logging.root.handlers:
        setup_logging(stdout=bool(options.output), quiet=options.quiet)

    if len(args) == 0:
        parser.error(
            "At least one parameter needed: name of a FASTA or FASTQ file.")
    elif len(args) > 2:
        parser.error("Too many parameters.")
    input_filename = args[0]
    if input_filename.endswith('.qual'):
        parser.error(
            "If a .qual file is given, it must be the second argument.")

    # Find out which 'mode' we need to use.
    # Default: single-read trimming (neither -p nor -A/-G/-B/-U/--interleaved given)
    paired = False
    if options.paired_output:
        # Modify first read only, keep second in sync (-p given, but not -A/-G/-B/-U).
        # This exists for backwards compatibility ('legacy mode').
        paired = 'first'
    # Any of these options switch off legacy mode
    if (options.adapters2 or options.front2 or options.anywhere2
            or options.cut2 or options.interleaved or options.pair_filter
            or options.too_short_paired_output
            or options.too_long_paired_output):
        # Full paired-end trimming when both -p and -A/-G/-B/-U given
        # Read modifications (such as quality trimming) are applied also to second read.
        paired = 'both'

    if paired and len(args) == 1 and not options.interleaved:
        parser.error(
            "When paired-end trimming is enabled via -A/-G/-B/-U or -p, "
            "two input files are required.")
    if options.interleaved and len(args) != 1:
        parser.error("When reading interleaved files, only one input file may "
                     "be given.")
    if not paired:
        if options.untrimmed_paired_output:
            parser.error(
                "Option --untrimmed-paired-output can only be used when "
                "trimming paired-end reads (with option -p).")

    # Assign input_paired_filename and quality_filename
    input_paired_filename = None
    quality_filename = None
    if paired:
        if not options.interleaved:
            input_paired_filename = args[1]
            if not options.paired_output:
                parser.error(
                    "When paired-end trimming is enabled via -A/-G/-B/-U, "
                    "a second output file needs to be specified via -p (--paired-output)."
                )
            if not options.output:
                parser.error(
                    "When you use -p or --paired-output, you must also "
                    "use the -o option.")
            if bool(options.untrimmed_output) != bool(
                    options.untrimmed_paired_output):
                parser.error(
                    "When trimming paired-end reads, you must use either none "
                    "or both of the --untrimmed-output/--untrimmed-paired-output options."
                )
            if options.too_short_output and not options.too_short_paired_output:
                parser.error(
                    "When using --too-short-output with paired-end "
                    "reads, you also need to use --too-short-paired-output")
            if options.too_long_output and not options.too_long_paired_output:
                parser.error(
                    "When using --too-long-output with paired-end "
                    "reads, you also need to use --too-long-paired-output")
    elif len(args) == 2:
        quality_filename = args[1]
        if options.format is not None:
            parser.error(
                "If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used."
            )

    if options.format is not None and options.format.lower() not in [
            'fasta', 'fastq', 'sra-fastq'
    ]:
        parser.error(
            "The input file format must be either 'fasta', 'fastq' or "
            "'sra-fastq' (not '{0}').".format(options.format))

    # Open input file(s)
    try:
        reader = seqio.open(input_filename,
                            file2=input_paired_filename,
                            qualfile=quality_filename,
                            colorspace=options.colorspace,
                            fileformat=options.format,
                            interleaved=options.interleaved)
    except (seqio.UnknownFileType, IOError) as e:
        parser.error(e)

    if options.quality_cutoff is not None:
        cutoffs = options.quality_cutoff.split(',')
        if len(cutoffs) == 1:
            try:
                cutoffs = [0, int(cutoffs[0])]
            except ValueError as e:
                parser.error(
                    "Quality cutoff value not recognized: {0}".format(e))
        elif len(cutoffs) == 2:
            try:
                cutoffs = [int(cutoffs[0]), int(cutoffs[1])]
            except ValueError as e:
                parser.error(
                    "Quality cutoff value not recognized: {0}".format(e))
        else:
            parser.error(
                "Expected one value or two values separated by comma for the quality cutoff"
            )
    else:
        cutoffs = None

    open_writer = functools.partial(seqio.open,
                                    mode='w',
                                    qualities=reader.delivers_qualities,
                                    colorspace=options.colorspace,
                                    interleaved=options.interleaved)

    if options.pair_filter is None:
        options.pair_filter = 'any'
    min_affected = 2 if options.pair_filter == 'both' else 1
    if not paired:
        filter_wrapper = Redirector
    elif paired == 'first':
        filter_wrapper = LegacyPairedRedirector
    elif paired == 'both':
        filter_wrapper = functools.partial(PairedRedirector,
                                           min_affected=min_affected)
    filters = []
    # TODO open_files = []
    too_short_writer = None  # too short reads go here
    # TODO pass file name to TooShortReadFilter, add a .close() method?
    if options.minimum_length > 0:
        if options.too_short_output:
            too_short_writer = open_writer(options.too_short_output,
                                           options.too_short_paired_output)
        filters.append(
            filter_wrapper(too_short_writer,
                           TooShortReadFilter(options.minimum_length)))
    too_long_writer = None  # too long reads go here
    if options.maximum_length < sys.maxsize:
        if options.too_long_output is not None:
            too_long_writer = open_writer(options.too_long_output,
                                          options.too_long_paired_output)
        filters.append(
            filter_wrapper(too_long_writer,
                           TooLongReadFilter(options.maximum_length)))

    if options.max_n != -1:
        filters.append(filter_wrapper(None, NContentFilter(options.max_n)))

    if int(options.discard_trimmed) + int(options.discard_untrimmed) + int(
            options.untrimmed_output is not None) > 1:
        parser.error(
            "Only one of the --discard-trimmed, --discard-untrimmed "
            "and --untrimmed-output options can be used at the same time.")
    demultiplexer = None
    untrimmed_writer = None
    writer = None
    if options.output is not None and '{name}' in options.output:
        if options.discard_trimmed:
            parser.error("Do not use --discard-trimmed when demultiplexing.")
        if paired:
            parser.error(
                "Demultiplexing not supported for paired-end files, yet.")
        untrimmed = options.output.replace('{name}', 'unknown')
        if options.untrimmed_output:
            untrimmed = options.untrimmed_output
        if options.discard_untrimmed:
            untrimmed = None
        demultiplexer = Demultiplexer(options.output,
                                      untrimmed,
                                      qualities=reader.delivers_qualities,
                                      colorspace=options.colorspace)
        filters.append(demultiplexer)
    else:
        # Set up the remaining filters to deal with --discard-trimmed,
        # --discard-untrimmed and --untrimmed-output. These options
        # are mutually exclusive in order to avoid brain damage.
        if options.discard_trimmed:
            filters.append(filter_wrapper(None, DiscardTrimmedFilter()))
        elif options.discard_untrimmed:
            filters.append(filter_wrapper(None, DiscardUntrimmedFilter()))
        elif options.untrimmed_output:
            untrimmed_writer = open_writer(options.untrimmed_output,
                                           options.untrimmed_paired_output)
            filters.append(
                filter_wrapper(untrimmed_writer, DiscardUntrimmedFilter()))

        # Finally, figure out where the reads that passed all the previous
        # filters should go.
        if options.output is not None:
            writer = open_writer(options.output, options.paired_output)
        else:
            writer = open_writer(default_outfile)
        if not paired:
            filters.append(NoFilter(writer))
        else:
            filters.append(PairedNoFilter(writer))

    if options.maq:
        options.colorspace = True
        options.double_encode = True
        options.trim_primer = True
        options.strip_suffix.append('_F3')
        options.suffix = "/1"
    if options.zero_cap is None:
        options.zero_cap = options.colorspace
    if options.trim_primer and not options.colorspace:
        parser.error("Trimming the primer makes only sense in colorspace.")
    if options.double_encode and not options.colorspace:
        parser.error("Double-encoding makes only sense in colorspace.")
    if options.anywhere and options.colorspace:
        parser.error(
            "Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author)."
        )
    if not (0 <= options.error_rate <= 1.):
        parser.error("The maximum error rate must be between 0 and 1.")
    if options.overlap < 1:
        parser.error("The overlap must be at least 1.")

    if options.rest_file is not None:
        options.rest_file = xopen(options.rest_file, 'w')
        rest_writer = RestFileWriter(options.rest_file)
    else:
        rest_writer = None
    if options.info_file is not None:
        options.info_file = xopen(options.info_file, 'w')
    if options.wildcard_file is not None:
        options.wildcard_file = xopen(options.wildcard_file, 'w')

    if options.colorspace:
        if options.match_read_wildcards:
            parser.error('IUPAC wildcards not supported in colorspace')
        options.match_adapter_wildcards = False

    adapter_parser = AdapterParser(
        colorspace=options.colorspace,
        max_error_rate=options.error_rate,
        min_overlap=options.overlap,
        read_wildcards=options.match_read_wildcards,
        adapter_wildcards=options.match_adapter_wildcards,
        indels=options.indels)

    try:
        adapters = adapter_parser.parse_multi(options.adapters,
                                              options.anywhere, options.front)
        adapters2 = adapter_parser.parse_multi(options.adapters2,
                                               options.anywhere2,
                                               options.front2)
    except IOError as e:
        if e.errno == errno.ENOENT:
            parser.error(e)
        raise
    except ValueError as e:
        parser.error(e)
    if options.debug:
        for adapter in adapters + adapters2:
            adapter.enable_debug()

    if not adapters and not adapters2 and not cutoffs and \
      options.nextseq_trim is None and \
      options.cut == [] and options.cut2 == [] and \
      options.minimum_length == 0 and \
      options.maximum_length == sys.maxsize and \
      quality_filename is None and \
      options.max_n == -1 and not options.trim_n:
        parser.error("You need to provide at least one adapter sequence.")

    # Create the single-end processing pipeline (a list of "modifiers")
    modifiers = []
    if options.cut:
        if len(options.cut) > 2:
            parser.error("You cannot remove bases from more than two ends.")
        if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
            parser.error("You cannot remove bases from the same end twice.")
        for cut in options.cut:
            if cut != 0:
                modifiers.append(UnconditionalCutter(cut))

    if options.nextseq_trim is not None:
        modifiers.append(
            NextseqQualityTrimmer(options.nextseq_trim, options.quality_base))

    if cutoffs:
        modifiers.append(
            QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
    if adapters:
        adapter_cutter = AdapterCutter(adapters, options.times,
                                       options.wildcard_file,
                                       options.info_file, rest_writer,
                                       options.action)
        modifiers.append(adapter_cutter)

    # Modifiers that apply to both reads of paired-end reads unless in legacy mode
    modifiers_both = []
    if options.trim_n:
        modifiers_both.append(NEndTrimmer())
    if options.length_tag:
        modifiers_both.append(LengthTagModifier(options.length_tag))
    if options.strip_f3:
        options.strip_suffix.append('_F3')
    for suffix in options.strip_suffix:
        modifiers_both.append(SuffixRemover(suffix))
    if options.prefix or options.suffix:
        modifiers_both.append(PrefixSuffixAdder(options.prefix,
                                                options.suffix))
    if options.double_encode:
        modifiers_both.append(DoubleEncoder())
    if options.zero_cap and reader.delivers_qualities:
        modifiers_both.append(ZeroCapper(quality_base=options.quality_base))
    if options.trim_primer:
        modifiers_both.append(PrimerTrimmer)
    modifiers.extend(modifiers_both)

    # For paired-end data, create a second processing pipeline.
    # However, if no second-read adapters were given (via -A/-G/-B/-U), we need to
    # be backwards compatible and *no modifications* are done to the second read.
    modifiers2 = []
    if paired == 'both':
        if options.cut2:
            if len(options.cut2) > 2:
                parser.error(
                    "You cannot remove bases from more than two ends.")
            if len(options.cut2
                   ) == 2 and options.cut2[0] * options.cut2[1] > 0:
                parser.error(
                    "You cannot remove bases from the same end twice.")
            for cut in options.cut2:
                if cut != 0:
                    modifiers2.append(UnconditionalCutter(cut))

        if cutoffs:
            modifiers2.append(
                QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
        if adapters2:
            adapter_cutter2 = AdapterCutter(adapters2, options.times, None,
                                            None, None, options.action)
            modifiers2.append(adapter_cutter2)
        else:
            adapter_cutter2 = None
        modifiers2.extend(modifiers_both)

    logger.info("This is cutadapt %s with Python %s", __version__,
                platform.python_version())
    logger.info("Command line parameters: %s", " ".join(cmdlineargs))
    logger.info(
        "Trimming %s adapter%s with at most %.1f%% errors in %s mode ...",
        len(adapters) + len(adapters2),
        's' if len(adapters) + len(adapters2) != 1 else '',
        options.error_rate * 100, {
            False: 'single-end',
            'first': 'paired-end legacy',
            'both': 'paired-end'
        }[paired])

    if paired == 'first' and (modifiers_both or cutoffs):
        logger.warning('\n'.join(
            textwrap.wrap(
                'WARNING: Requested read '
                'modifications are applied only to the first '
                'read since backwards compatibility mode is enabled. '
                'To modify both reads, also use any of the -A/-B/-G/-U options. '
                'Use a dummy adapter sequence when necessary: -A XXX')))

    start_time = time.clock()
    try:
        if paired:
            stats = process_paired_reads(reader, modifiers, modifiers2,
                                         filters)
        else:
            stats = process_single_reads(reader, modifiers, filters)
    except KeyboardInterrupt as e:
        print("Interrupted", file=sys.stderr)
        sys.exit(130)
    except IOError as e:
        if e.errno == errno.EPIPE:
            sys.exit(1)
        raise
    except (seqio.FormatError, EOFError) as e:
        sys.exit("cutadapt: error: {0}".format(e))

    # close open files
    for f in [
            writer, untrimmed_writer, options.rest_file, options.wildcard_file,
            options.info_file, too_short_writer, too_long_writer,
            options.info_file, demultiplexer
    ]:
        if f is not None and f is not sys.stdin and f is not sys.stdout:
            f.close()

    elapsed_time = time.clock() - start_time
    if not options.quiet:
        stats.collect((adapters, adapters2), elapsed_time, modifiers,
                      modifiers2, filters)
        # send statistics to stderr if result was sent to stdout
        stat_file = sys.stderr if options.output is None else None
        with redirect_standard_output(stat_file):
            print_report(stats, (adapters, adapters2))
Example #44
0
def main(cmdlineargs=None, default_outfile=sys.stdout):
	"""
	Main function that evaluates command-line parameters and iterates
	over all reads.

	default_outfile is the file to which trimmed reads are sent if the ``-o``
	parameter is not used.
	"""
	parser = get_option_parser()
	if cmdlineargs is None:
		cmdlineargs = sys.argv[1:]
	options, args = parser.parse_args(args=cmdlineargs)
	# Setup logging only if there are not already any handlers (can happen when
	# this function is being called externally such as from unit tests)
	if not logging.root.handlers:
		setup_logging(stdout=bool(options.output), quiet=options.quiet)

	if len(args) == 0:
		parser.error("At least one parameter needed: name of a FASTA or FASTQ file.")
	elif len(args) > 2:
		parser.error("Too many parameters.")
	input_filename = args[0]
	if input_filename.endswith('.qual'):
		parser.error("If a .qual file is given, it must be the second argument.")

	# Find out which 'mode' we need to use.
	# Default: single-read trimming (neither -p nor -A/-G/-B/-U/--interleaved given)
	paired = False
	if options.paired_output:
		# Modify first read only, keep second in sync (-p given, but not -A/-G/-B/-U).
		# This exists for backwards compatibility ('legacy mode').
		paired = 'first'
	# Any of these options switch off legacy mode
	if (options.adapters2 or options.front2 or options.anywhere2 or
		options.cut2 or options.interleaved or options.pair_filter or
		options.too_short_paired_output or options.too_long_paired_output):
		# Full paired-end trimming when both -p and -A/-G/-B/-U given
		# Read modifications (such as quality trimming) are applied also to second read.
		paired = 'both'

	if paired and len(args) == 1 and not options.interleaved:
		parser.error("When paired-end trimming is enabled via -A/-G/-B/-U or -p, "
			"two input files are required.")
	if options.interleaved and len(args) != 1:
		parser.error("When reading interleaved files, only one input file may "
			"be given.")
	if not paired:
		if options.untrimmed_paired_output:
			parser.error("Option --untrimmed-paired-output can only be used when "
				"trimming paired-end reads (with option -p).")

	# Assign input_paired_filename and quality_filename
	input_paired_filename = None
	quality_filename = None
	if paired:
		if not options.interleaved:
			input_paired_filename = args[1]
			if not options.paired_output:
				parser.error("When paired-end trimming is enabled via -A/-G/-B/-U, "
					"a second output file needs to be specified via -p (--paired-output).")
			if not options.output:
				parser.error("When you use -p or --paired-output, you must also "
					"use the -o option.")
			if bool(options.untrimmed_output) != bool(options.untrimmed_paired_output):
				parser.error("When trimming paired-end reads, you must use either none "
					"or both of the --untrimmed-output/--untrimmed-paired-output options.")
			if options.too_short_output and not options.too_short_paired_output:
				parser.error("When using --too-short-output with paired-end "
					"reads, you also need to use --too-short-paired-output")
			if options.too_long_output and not options.too_long_paired_output:
				parser.error("When using --too-long-output with paired-end "
					"reads, you also need to use --too-long-paired-output")
	elif len(args) == 2:
		quality_filename = args[1]
		if options.format is not None:
			parser.error("If a pair of .fasta and .qual files is given, the -f/--format parameter cannot be used.")

	if options.format is not None and options.format.lower() not in ['fasta', 'fastq', 'sra-fastq']:
		parser.error("The input file format must be either 'fasta', 'fastq' or "
			"'sra-fastq' (not '{0}').".format(options.format))

	# Open input file(s)
	try:
		reader = seqio.open(input_filename, file2=input_paired_filename,
				qualfile=quality_filename, colorspace=options.colorspace,
				fileformat=options.format, interleaved=options.interleaved)
	except (seqio.UnknownFileType, IOError) as e:
		parser.error(e)

	if options.quality_cutoff is not None:
		cutoffs = options.quality_cutoff.split(',')
		if len(cutoffs) == 1:
			try:
				cutoffs = [0, int(cutoffs[0])]
			except ValueError as e:
				parser.error("Quality cutoff value not recognized: {0}".format(e))
		elif len(cutoffs) == 2:
			try:
				cutoffs = [int(cutoffs[0]), int(cutoffs[1])]
			except ValueError as e:
				parser.error("Quality cutoff value not recognized: {0}".format(e))
		else:
			parser.error("Expected one value or two values separated by comma for the quality cutoff")
	else:
		cutoffs = None

	open_writer = functools.partial(seqio.open, mode='w',
		qualities=reader.delivers_qualities, colorspace=options.colorspace,
		interleaved=options.interleaved)

	if options.pair_filter is None:
		options.pair_filter = 'any'
	min_affected = 2 if options.pair_filter == 'both' else 1
	if not paired:
		filter_wrapper = Redirector
	elif paired == 'first':
		filter_wrapper = LegacyPairedRedirector
	elif paired == 'both':
		filter_wrapper = functools.partial(PairedRedirector, min_affected=min_affected)
	filters = []
	# TODO open_files = []
	too_short_writer = None  # too short reads go here
	# TODO pass file name to TooShortReadFilter, add a .close() method?
	if options.minimum_length > 0:
		if options.too_short_output:
			too_short_writer = open_writer(options.too_short_output, options.too_short_paired_output)
		filters.append(filter_wrapper(too_short_writer, TooShortReadFilter(options.minimum_length)))
	too_long_writer = None  # too long reads go here
	if options.maximum_length < sys.maxsize:
		if options.too_long_output is not None:
			too_long_writer = open_writer(options.too_long_output, options.too_long_paired_output)
		filters.append(filter_wrapper(too_long_writer, TooLongReadFilter(options.maximum_length)))

	if options.max_n != -1:
		filters.append(filter_wrapper(None, NContentFilter(options.max_n)))

	if int(options.discard_trimmed) + int(options.discard_untrimmed) + int(options.untrimmed_output is not None) > 1:
		parser.error("Only one of the --discard-trimmed, --discard-untrimmed "
			"and --untrimmed-output options can be used at the same time.")
	demultiplexer = None
	untrimmed_writer = None
	writer = None
	if options.output is not None and '{name}' in options.output:
		if options.discard_trimmed:
			parser.error("Do not use --discard-trimmed when demultiplexing.")
		if paired:
			parser.error("Demultiplexing not supported for paired-end files, yet.")
		untrimmed = options.output.replace('{name}', 'unknown')
		if options.untrimmed_output:
			untrimmed = options.untrimmed_output
		if options.discard_untrimmed:
			untrimmed = None
		demultiplexer = Demultiplexer(options.output, untrimmed,
			qualities=reader.delivers_qualities, colorspace=options.colorspace)
		filters.append(demultiplexer)
	else:
		# Set up the remaining filters to deal with --discard-trimmed,
		# --discard-untrimmed and --untrimmed-output. These options
		# are mutually exclusive in order to avoid brain damage.
		if options.discard_trimmed:
			filters.append(filter_wrapper(None, DiscardTrimmedFilter()))
		elif options.discard_untrimmed:
			filters.append(filter_wrapper(None, DiscardUntrimmedFilter()))
		elif options.untrimmed_output:
			untrimmed_writer = open_writer(options.untrimmed_output,
				options.untrimmed_paired_output)
			filters.append(filter_wrapper(untrimmed_writer, DiscardUntrimmedFilter()))

		# Finally, figure out where the reads that passed all the previous
		# filters should go.
		if options.output is not None:
			writer = open_writer(options.output, options.paired_output)
		else:
			writer = open_writer(default_outfile)
		if not paired:
			filters.append(NoFilter(writer))
		else:
			filters.append(PairedNoFilter(writer))

	if options.maq:
		options.colorspace = True
		options.double_encode = True
		options.trim_primer = True
		options.strip_suffix.append('_F3')
		options.suffix = "/1"
	if options.zero_cap is None:
		options.zero_cap = options.colorspace
	if options.trim_primer and not options.colorspace:
		parser.error("Trimming the primer makes only sense in colorspace.")
	if options.double_encode and not options.colorspace:
		parser.error("Double-encoding makes only sense in colorspace.")
	if options.anywhere and options.colorspace:
		parser.error("Using --anywhere with colorspace reads is currently not supported (if you think this may be useful, contact the author).")
	if not (0 <= options.error_rate <= 1.):
		parser.error("The maximum error rate must be between 0 and 1.")
	if options.overlap < 1:
		parser.error("The overlap must be at least 1.")

	if options.rest_file is not None:
		options.rest_file = xopen(options.rest_file, 'w')
		rest_writer = RestFileWriter(options.rest_file)
	else:
		rest_writer = None
	if options.info_file is not None:
		options.info_file = xopen(options.info_file, 'w')
	if options.wildcard_file is not None:
		options.wildcard_file = xopen(options.wildcard_file, 'w')

	if options.colorspace:
		if options.match_read_wildcards:
			parser.error('IUPAC wildcards not supported in colorspace')
		options.match_adapter_wildcards = False

	adapter_parser = AdapterParser(
		colorspace=options.colorspace,
		max_error_rate=options.error_rate,
		min_overlap=options.overlap,
		read_wildcards=options.match_read_wildcards,
		adapter_wildcards=options.match_adapter_wildcards,
		indels=options.indels)

	try:
		adapters = adapter_parser.parse_multi(options.adapters, options.anywhere, options.front)
		adapters2 = adapter_parser.parse_multi(options.adapters2, options.anywhere2, options.front2)
	except IOError as e:
		if e.errno == errno.ENOENT:
			parser.error(e)
		raise
	except ValueError as e:
		parser.error(e)
	if options.debug:
		for adapter in adapters + adapters2:
			adapter.enable_debug()

	if not adapters and not adapters2 and not cutoffs and \
			options.nextseq_trim is None and \
			options.cut == [] and options.cut2 == [] and \
			options.minimum_length == 0 and \
			options.maximum_length == sys.maxsize and \
			quality_filename is None and \
			options.max_n == -1 and not options.trim_n:
		parser.error("You need to provide at least one adapter sequence.")

	# Create the single-end processing pipeline (a list of "modifiers")
	modifiers = []
	if options.cut:
		if len(options.cut) > 2:
			parser.error("You cannot remove bases from more than two ends.")
		if len(options.cut) == 2 and options.cut[0] * options.cut[1] > 0:
			parser.error("You cannot remove bases from the same end twice.")
		for cut in options.cut:
			if cut != 0:
				modifiers.append(UnconditionalCutter(cut))

	if options.nextseq_trim is not None:
		modifiers.append(NextseqQualityTrimmer(options.nextseq_trim, options.quality_base))

	if cutoffs:
		modifiers.append(QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
	if adapters:
		adapter_cutter = AdapterCutter(adapters, options.times,
				options.wildcard_file, options.info_file,
				rest_writer, options.action)
		modifiers.append(adapter_cutter)

	# Modifiers that apply to both reads of paired-end reads unless in legacy mode
	modifiers_both = []
	if options.trim_n:
		modifiers_both.append(NEndTrimmer())
	if options.length_tag:
		modifiers_both.append(LengthTagModifier(options.length_tag))
	if options.strip_f3:
		options.strip_suffix.append('_F3')
	for suffix in options.strip_suffix:
		modifiers_both.append(SuffixRemover(suffix))
	if options.prefix or options.suffix:
		modifiers_both.append(PrefixSuffixAdder(options.prefix, options.suffix))
	if options.double_encode:
		modifiers_both.append(DoubleEncoder())
	if options.zero_cap and reader.delivers_qualities:
		modifiers_both.append(ZeroCapper(quality_base=options.quality_base))
	if options.trim_primer:
		modifiers_both.append(PrimerTrimmer)
	modifiers.extend(modifiers_both)

	# For paired-end data, create a second processing pipeline.
	# However, if no second-read adapters were given (via -A/-G/-B/-U), we need to
	# be backwards compatible and *no modifications* are done to the second read.
	modifiers2 = []
	if paired == 'both':
		if options.cut2:
			if len(options.cut2) > 2:
				parser.error("You cannot remove bases from more than two ends.")
			if len(options.cut2) == 2 and options.cut2[0] * options.cut2[1] > 0:
				parser.error("You cannot remove bases from the same end twice.")
			for cut in options.cut2:
				if cut != 0:
					modifiers2.append(UnconditionalCutter(cut))

		if cutoffs:
			modifiers2.append(QualityTrimmer(cutoffs[0], cutoffs[1], options.quality_base))
		if adapters2:
			adapter_cutter2 = AdapterCutter(adapters2, options.times,
					None, None, None, options.action)
			modifiers2.append(adapter_cutter2)
		else:
			adapter_cutter2 = None
		modifiers2.extend(modifiers_both)

	logger.info("This is cutadapt %s with Python %s", __version__, platform.python_version())
	logger.info("Command line parameters: %s", " ".join(cmdlineargs))
	logger.info("Trimming %s adapter%s with at most %.1f%% errors in %s mode ...",
		len(adapters) + len(adapters2), 's' if len(adapters) + len(adapters2) != 1 else '',
		options.error_rate * 100,
		{ False: 'single-end', 'first': 'paired-end legacy', 'both': 'paired-end' }[paired])

	if paired == 'first' and (modifiers_both or cutoffs):
		logger.warning('\n'.join(textwrap.wrap('WARNING: Requested read '
			'modifications are applied only to the first '
			'read since backwards compatibility mode is enabled. '
			'To modify both reads, also use any of the -A/-B/-G/-U options. '
			'Use a dummy adapter sequence when necessary: -A XXX')))

	start_time = time.clock()
	try:
		if paired:
			stats = process_paired_reads(reader, modifiers, modifiers2, filters)
		else:
			stats = process_single_reads(reader, modifiers, filters)
	except KeyboardInterrupt as e:
		print("Interrupted", file=sys.stderr)
		sys.exit(130)
	except IOError as e:
		if e.errno == errno.EPIPE:
			sys.exit(1)
		raise
	except (seqio.FormatError, EOFError) as e:
		sys.exit("cutadapt: error: {0}".format(e))

	# close open files
	for f in [writer, untrimmed_writer,
			options.rest_file, options.wildcard_file,
			options.info_file, too_short_writer, too_long_writer,
			options.info_file, demultiplexer]:
		if f is not None and f is not sys.stdin and f is not sys.stdout:
			f.close()

	elapsed_time = time.clock() - start_time
	if not options.quiet:
		stats.collect((adapters, adapters2), elapsed_time,
			modifiers, modifiers2, filters)
		# send statistics to stderr if result was sent to stdout
		stat_file = sys.stderr if options.output is None else None
		with redirect_standard_output(stat_file):
			print_report(stats, (adapters, adapters2))
Example #45
0
	def __init__(self, file):
		if isinstance(file, str):
			file = xopen(file, "w")
			self._close_on_exit = True
		self._file = file