/
genobox_trim_h.py
executable file
·445 lines (367 loc) · 16.1 KB
/
genobox_trim_h.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
#!/panvol1/simon/bin/python2.7
from __future__ import division
from Bio.SeqIO.QualityIO import FastqGeneralIterator
import argparse
import genobox_modules
import sys
import random
import string
import cdb
import multiprocessing
import gzip
class FastqTrim:
'''Trim/Filter paired end fastq:
trim 3' trailing bases under (q)
trim sequencing adaptors (Illumina) (a)
filter reads with less than avg. Q quals (m)
filter reads with less than L length (l)
filter reads with Ns (keep_n)
'''
def __init__(self, f, o, l=25, q=20, m=20, keep_n=False, M=20,
a=['GATCGGAAGAGCGGTTCAGCAGGAATGCCGAG', 'ACACTCTTTCCCTACACGACGCTCTTCCGATCT',
'AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT',
'CAAGCAGAAGACGGCATACGAGATCGGTCTCGGCATTCCTGCTGAACCGCTCTTCCGATCT',
'CGGTCTCGGCATTCCTGCTGAACCGCTCTTCCGATCT',
'ACACTCTTTCCCTACACGACGCTCTTCCGATCT'], gz=False):
self.f = f
self.o = o
self.l = l
self.q = q
self.m = m
self.keep_n = keep_n
self.M = M
self.a = a
self.paired = False
self.interleaved = False
self.gz = gz
# set adaptors, readtype, fastq format
self.adaptors = self.set_adaptors()
self.set_readtype()
self.format = genobox_modules.set_filetype(self.f[0], self.gz)
if self.format == 'fastq':
self.fqtype = genobox_modules.set_fqtype(self.f[0], self.gz)
def __repr__(self):
msg = 'FastqTrim(%s, %s, %i, %i, %i, %s, %i, %s, %s, %s)' % ("["+", ".join(self.f)+"]", "["+", ".join(self.o)+"]", self.l, self.q, self.m, self.keep_n, self.M, self.paired, self.interleaved, self.gz)
return msg
def set_readtype(self):
'''Set paired and/or interleaved flags'''
def check_interleaved(self):
'''Check if a file is interleaved paired or single'''
if self.gz:
import gzip
fh = gzip.open(self.f[0], 'rb')
else:
fh = open(self.f[0], 'r')
count = 0
ends = []
for (title, sequence, quality) in FastqGeneralIterator(fh):
ends.append(title[-2:])
count += 1
if count > 9: break
# check that they are interleaved (eg. /1, /2, /1, /2)
interleaved = True
for i,e in enumerate(ends[::2]):
if e != '/1':
interleaved = False
break
for i,e in enumerate(ends[1::2]):
if e != '/2':
interleaved = False
break
if interleaved:
self.paired = True
self.interleaved = True
return self
if len(self.f) == 2:
self.paired = True
self.interleaved = False
elif len(self.f) == 1:
self = check_interleaved(self)
def set_adaptors(self):
'''Set adaptors according to adaptor length requirements'''
adaptors = []
for a in self.a:
if self.M == 0:
adaptor = a
elif (self.M) > len(a):
adaptor = a
else:
adaptor = a[:self.M]
adaptors.append(adaptor)
return adaptors
def filter_adaptor(self, title, sequence, quality):
'''Filters adaptor and Ns in read'''
has_adaptor = False
has_N = False
is_short = False
# search for adaptor in reads
for adaptor in self.adaptors:
hit = sequence.find(adaptor)
if hit > -1:
sequence = sequence[:hit]
quality = quality[:hit]
has_adaptor = True
# check if read contains Ns
if sequence.find('N') > -1:
has_N = True
# check if too small then do not print out
if len(sequence) < self.l:
is_short = True
# return read
if has_N or is_short:
return (None, None, None)
else:
return (title, sequence, quality)
# the actual read trimming function
def trim_qual(self, title, sequence, quality):
'''Trims read based on 3' qualities and average quality'''
# functions for read trimming
def average(values):
'''Computes the arithmetic mean of a list of numbers.'''
return sum(values, 0.0) / len(values)
def illumina2qual(qual_string):
'''Convert illumina qualities (offset 64) to values.'''
qual = []
for q in qual_string:
qual.append(ord(q)-64)
return qual
def sanger2qual(qual_string):
'''Convert sanger qualities (offset 33) to values.'''
qual = []
for q in qual_string:
qual.append(ord(q)-33)
return qual
def trim_index(qual, m):
'''Determine position to trim from'''
qs = qual[::-1]
index = None
for i,q in enumerate(qs):
if q > m:
index = len(qual)-i
break
return index
# check if read is not valid
if title == None:
return (None, None, None)
if self.fqtype == 'Illumina':
qual = illumina2qual(quality)
elif self.fqtype == 'Sanger':
qual = sanger2qual(quality)
else:
raise ValueError('Fastq must be Illumina or Sanger')
# check trailing qualities, should it be trimmed?
t = trim_index(qual, self.q)
if t:
sequence = sequence[:t]
quality = quality[:t]
# do not print out if average quality is less than m or length is less than l
a = average(qual[:t])
if a < self.m or len(sequence) < self.l:
return (None, None, None)
else:
return (title, sequence, quality)
else:
# do not print out if average quality is less than m or length is less than l
a = average(qual)
if a < self.m or len(sequence) < self.l:
return (None, None, None)
else:
return (title, sequence, quality)
def write_pairs(self, f1, f2):
'''Parse through two paired files and only write if both pairs are present'''
def intersect(a, b):
'''Intesection between lists'''
return list(set(a) & set(b))
def rm_files(patterns):
'''Remove files using glob given as list of patterns'''
import glob
import os
for p in patterns:
files = glob.glob(p)
if len(files) == 0:
pass
else:
map(os.remove, files)
def write_out(db_common, f, o):
'''Write out reads'''
if self.gz:
fh = open(f, 'r')
out = gzip.open(o+'.gz', 'wb')
else:
fh = open(f, 'r')
out = open(o, 'w')
written_count = 0
total_count = 0
for (title, sequence, quality) in FastqGeneralIterator(fh):
total_count += 1
if db_common.has_key(title[:-2]):
out.write('@%s\n%s\n+\n%s\n' % (title, sequence, quality))
written_count += 1
sys.stderr.write('%s: Total %i, Written %i (%.1f%%)\n' % (f, total_count, written_count, written_count/total_count*100))
fh.close()
out.close()
def create_db(f, db_fname):
'''Write out db of headers'''
fh = open(f, 'r')
fh_headers = (x.strip()[1:-2] for i, x in enumerate(fh) if not (i % 4))
db = cdb.cdbmake(db_fname, db_fname + '.tmp')
for h in fh_headers:
db.add(h, 'T')
db.finish()
del(db)
## get headers from both trimmed files ##
# strip the /2 or /1 and grab only the headers
# write in dbm to minimze memory usage
# create db's (parallel)
rand = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(36))
db1_fname = 'db1_%s' % rand
db2_fname = 'db2_%s' % rand
jobs = []
p = multiprocessing.Process(target=create_db, args=(f1, db1_fname, ))
p.start()
jobs.append(p)
p = multiprocessing.Process(target=create_db, args=(f2, db2_fname, ))
p.start()
jobs.append(p)
# wait for jobs to finish
for job in jobs:
job.join()
## get headers that are in both trimmed files ##
db1 = cdb.init(db1_fname)
db2 = cdb.init(db2_fname)
common = intersect(db1.keys(), db2.keys())
dbcommon_fname = 'dbcommon_%s' % rand
db_common = cdb.cdbmake(dbcommon_fname, dbcommon_fname + '.tmp')
for h in common:
db_common.add(h, 'T')
db_common.finish()
del(db_common)
## get headers that are in only one trimmed file ##
symdiff = set(db1.keys()).symmetric_difference(set(db2.keys()))
dbdiff_fname = 'dbdiff_%s' % rand
db_diff = cdb.cdbmake(dbdiff_fname, dbdiff_fname + '.tmp')
for h in symdiff:
db_diff.add(h, 'T')
db_diff.finish()
del(db_diff)
## open common db ##
db_common = cdb.init(dbcommon_fname)
jobs = []
p = multiprocessing.Process(target=write_out, args=(db_common, f1, self.o[0]))
p.start()
jobs.append(p)
p = multiprocessing.Process(target=write_out, args=(db_common, f2, self.o[1]))
p.start()
jobs.append(p)
## open single db ##
self.single = [self.o[0]+'.single', self.o[1]+'.single']
db_diff = cdb.init(dbdiff_fname)
p = multiprocessing.Process(target=write_out, args=(db_diff, f1, self.single[0]))
p.start()
jobs.append(p)
p = multiprocessing.Process(target=write_out, args=(db_diff, f2, self.single[1]))
p.start()
jobs.append(p)
# wait for jobs to finish
for job in jobs:
job.join()
rm_files([db1_fname, db2_fname, dbcommon_fname, dbdiff_fname, f1, f2])
def trim(self, paired=False, interleave=False):
'''Start trimming of reads'''
def trim_file(self, f, f_out):
written = 0
total = 0
if self.gz:
fh_in = gzip.open(f, 'rb')
if self.paired:
fh_out = open(f_out, 'w')
else:
fh_out = gzip.open(f_out+'.gz', 'wb')
else:
fh_in = open(f, 'r')
fh_out = open(f_out, 'w')
for (title, sequence, quality) in FastqGeneralIterator(fh_in):
total += 1
(title, sequence, quality) = self.filter_adaptor(title, sequence, quality)
(title, sequence, quality) = self.trim_qual(title, sequence, quality)
if title != None:
if len(sequence) != len(quality):
raise ValueError('sequence and quality not of the same length\n%s\n%s\n' % (sequence, quality))
fh_out.write('@%s\n%s\n+\n%s\n' % (title, sequence, quality))
written += 1
fh_out.close()
fh_in.close()
sys.stderr.write('%s: written %s of %s (%.1f%%)\n' % (f, written, total, written/total))
def trim_interleaved_file(self, f, f_out):
written = 0
total = 0
if self.gz:
fh_in = gzip.open(f, 'rb')
fh_out = [open(f_out[0], 'w'), open(f_out[1], 'w')]
else:
fh_in = open(f, 'r')
fh_out = [open(f_out[0], 'w'), open(f_out[1], 'w')]
for (title, sequence, quality) in FastqGeneralIterator(fh_in):
(title, sequence, quality) = self.filter_adaptor(title, sequence, quality)
(title, sequence, quality) = self.trim_qual(title, sequence, quality)
if title != None:
if len(sequence) != len(quality):
raise ValueError('sequence and quality not of the same length\n%s\n%s\n' % (sequence, quality))
fh_out[total%2].write('@%s\n%s\n+\n%s\n' % (title, sequence, quality))
written += 1
total += 1
fh_out[0].close()
fh_out[1].close()
fh_in.close()
sys.stderr.write('%s: written %s of %s (%.1f%%)\n' % (f, written, total, written/total))
if self.paired:
# set tmp files
rand = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(36))
f_out = ['tmp0.'+rand, 'tmp1.'+rand]
# check if file is interleaved
if self.interleaved:
sys.stderr.write('Trimming paired end interleaved\n')
trim_interleaved_file(self, self.f[0], f_out)
else:
# create and start multiprocess
sys.stderr.write('Trimming paired end\n')
jobs = []
for i in range(2):
p = multiprocessing.Process(target=trim_file, args=(self, self.f[i], f_out[i], ))
p.start()
jobs.append(p)
# wait for jobs to finish
for job in jobs:
job.join()
# write files
self.write_pairs(f_out[0], f_out[1])
else:
sys.stderr.write('Trimming single end\n')
trim_file(self, self.f[0], self.o[0])
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='genobox_trim_h.py',
formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=50,width=140),
description='''Trim SE or PE files for low q bases, adaptor sequences, Ns''',
usage='%(prog)s [options]')
# add the arguments
parser.add_argument('--i', help='input single or paired end files', nargs='+', required=True)
parser.add_argument('--min_length', help='minimum length of a read to keep pairs [25]', type=int, default=25)
parser.add_argument('--min_baseq', help='chomp bases with quality less than [20]', default=20, type=int)
parser.add_argument('--min_avgq', help='minimum average quality of read [20]', default=20, type=int)
parser.add_argument('--adaptors', help='adaptor sequence to clip [Illumina adaptors]', nargs='+', default=['GATCGGAAGAGCGGTTCAGCAGGAATGCCGAG', 'ACACTCTTTCCCTACACGACGCTCTTCCGATCT', 'AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT', 'CAAGCAGAAGACGGCATACGAGATCGGTCTCGGCATTCCTGCTGAACCGCTCTTCCGATCT', 'CGGTCTCGGCATTCCTGCTGAACCGCTCTTCCGATCT', 'ACACTCTTTCCCTACACGACGCTCTTCCGATCT'])
parser.add_argument('--keep_n', help='do not remove sequences containing N', default=False, action='store_true')
parser.add_argument('--min_adaptor_match', help='minimum length of match to adaptor (0=all of adaptor) [3]', default=3, type=int)
parser.add_argument('--o', help='output files', nargs='+', required=True)
parser.add_argument('--gz', help='input files are gzipped [False]', default=False, action='store_true')
parser.add_argument('--log', help='log level [INFO]', default='info')
args = parser.parse_args()
#args = parser.parse_args('--i Kleb-10-213361_2.interleaved.fastq --o Kleb-10-213361_2_1.interleaved.fastq.trim.fastq Kleb-10-213361_2_2.interleaved.fastq.trim.fastq'.split())
#args = parser.parse_args('--i kleb_test_2.fq --l 25 --q 20 --o kleb_test_2.trim.fq'.split())
#args = parser.parse_args('--i test_kleb_1.fq test_kleb_2.fq --min_length 60 --o test_kleb_trim_1.fq test_kleb_trim_2.fq'.split())
#args = parser.parse_args('--i /panvol1/simon/projects/cge/test/test_kleb_1.fq.gz --min_length 25 --min_baseq 20 --min_avgq 20 --min_adaptor_match 20 --o trimmed/test_kleb_1.fq.gz.trim.fq --gz'.split())
#args = parser.parse_args('--i test_1.fq test_2.fq --min_length 25 --min_baseq 20 --min_avgq 20 --min_adaptor_match 20 --o trimmed/test_1.fq.trim.fq trimmed/test_2.fq.trim.fq'.split())
#args = parser.parse_args('--i test_1.fq.gz test_2.fq.gz --min_length 25 --min_baseq 20 --min_avgq 20 --min_adaptor_match 20 --o trimmed/test_1.fq.trim.fq trimmed/test_2.fq.trim.fq --gz'.split())
# create instance
fqtrim = FastqTrim(args.i, args.o, args.min_length, args.min_baseq, args.min_avgq, args.keep_n, args.min_adaptor_match, args.adaptors, args.gz)
# start trimming
fqtrim.trim()