def test_single_iter_overlapping(self):
actual = TestMafOverlapIterator.RecordsOverlapping
items = LocatableOverlapIterator(
[iter(TestMafOverlapIterator.RecordsOverlapping)],
by_barcodes=False)
# one record in one list
records = next(items)
self.assertEqual(len(records), 1)
self.assertEqual(len(records[0]), 1)
self.assertEqual(records[0][0], actual[0])
# two records in one list
records = next(items)
self.assertEqual(len(records), 1)
self.assertEqual(len(records[0]), 2)
self.assertEqual(records[0][0], actual[1])
self.assertEqual(records[0][1], actual[2])
# one record in one list
records = next(items)
self.assertEqual(len(records), 1)
self.assertEqual(len(records[0]), 1)
self.assertEqual(records[0][0], actual[3])
with self.assertRaises(StopIteration):
next(items)
def test_by_barcode_mismatching(self):
left = TestMafOverlapIterator.RecordsNoOverlap
right = TestMafOverlapIterator.RecordsNoOverlap
# add different tumor/normal barcodes to left and right
left = [
DummyRecord(r.chromosome, r.start, r.end, "A", "B") for r in left
]
right = [
DummyRecord(r.chromosome, r.start, r.end, "B", "A") for r in right
]
# iterate by barcode
items = LocatableOverlapIterator([iter(left), iter(right)],
by_barcodes=True)
n = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 2)
if n < len(left):
self.assertEqual(len(records[0]), 1)
self.assertEqual(len(records[1]), 0)
self.assertEqual(records[0][0], left[n])
else:
self.assertEqual(len(records[0]), 0)
self.assertEqual(len(records[1]), 1)
self.assertEqual(records[1][0], right[n - len(left)])
n += 1
self.assertEqual(n, len(left) + len(right))
# ignore barcode
items = LocatableOverlapIterator([iter(left), iter(right)],
by_barcodes=False)
n = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 2)
self.assertEqual(len(records[0]) + len(records[1]), 2)
self.assertEqual(records[0][0], left[n])
self.assertEqual(records[1][0], right[n])
n += 1
self.assertEqual(n, len(left))
def test_single_iter_no_overlap(self):
actual = TestMafOverlapIterator.RecordsNoOverlap
items = LocatableOverlapIterator([iter(actual)], by_barcodes=False)
n = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 1)
self.assertEqual(len(records[0]), 1)
self.assertEqual(actual[i], records[0][0])
n += 1
self.assertEqual(n, len(actual))
def test_two_iter_same(self):
first = TestMafOverlapIterator.RecordsNoOverlap
second = TestMafOverlapIterator.RecordsNoOverlap
items = LocatableOverlapIterator(
[iter(first), iter(second)], by_barcodes=False)
n = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 2)
self.assertEqual(len(records[0]) + len(records[1]), 2)
self.assertEqual(records[0][0], first[n])
self.assertEqual(records[1][0], second[n])
n += 1
self.assertEqual(n, len(first))
def test_record_overlap(self):
# tests if right is fully contained in left
left = DummyRecord("A", 11, 109)
right = DummyRecord("A", 100, 200)
items = LocatableOverlapIterator(
[iter([left]), iter([right])], by_barcodes=False)
# one record in one list
records = next(items)
self.assertEqual(len(records), 2)
self.assertEqual(len(records[0]), 1)
self.assertEqual(len(records[1]), 1)
self.assertEqual(records[0][0], left)
self.assertEqual(records[1][0], right)
with self.assertRaises(StopIteration):
next(items)
def test_two_iter_no_overlap(self):
first = TestMafOverlapIterator.RecordsNoOverlap
second = TestMafOverlapIterator.RecordsSecondNoOverlap
items = LocatableOverlapIterator(
[iter(first), iter(second)], by_barcodes=False)
n = first_i = second_i = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 2)
self.assertEqual(len(records[0]) + len(records[1]), 1)
if len(records[0]) > 0:
self.assertEqual(records[0][0], first[first_i])
first_i += 1
else:
self.assertEqual(records[1][0], second[second_i])
second_i += 1
n += 1
self.assertEqual(n, len(first) + len(second))
self.assertEqual(first_i, len(first))
self.assertEqual(second_i, len(second))
def test_two_iter_second_overlaps_first(self):
first = TestMafOverlapIterator.RecordsNoOverlap
second = TestMafOverlapIterator.RecordsSecondOverlappingFirst
items = LocatableOverlapIterator(
[iter(first), iter(second)], by_barcodes=False)
expected_counts = [1, 1, 2, 1, 1, 1]
n = first_i = second_i = 0
for i, records in enumerate(items):
self.assertEqual(len(records), 2)
self.assertEqual(
len(records[0]) + len(records[1]), expected_counts[n])
if len(records[0]) > 0:
self.assertEqual(records[0][0], first[first_i])
first_i += 1
if len(records[1]) > 0:
self.assertEqual(records[1][0], second[second_i])
second_i += 1
n += 1
self.assertEqual(first_i + second_i, len(first) + len(second))
self.assertEqual(first_i, len(first))
self.assertEqual(second_i, len(second))
def test_empty_iter(self):
items = LocatableOverlapIterator([], by_barcodes=False)
self.assertEqual(len([i for i in items]), 0)
def do_work(self):
"""Main wrapper function for running protect MAF merging"""
# Reader
self.load_readers()
# Header
self.setup_maf_header()
self._scheme = self.maf_header.scheme()
self._columns = get_columns_from_header(self.maf_header)
# Sorter
sorter = MafSorter(max_objects_in_ram=100000,
sort_order_name=BarcodesAndCoordinate.name(),
scheme=self.maf_header.scheme(),
contigs=self.maf_header.contigs())
# Merger
self._merger = MafRecordMerger_1_0_0(self._scheme)
# Overlap iterator
o_iter = LocatableOverlapIterator(
self.maf_readers,
contigs=self.maf_header.contigs(),
peekable_iterator_class=FilteringPeekableIterator)
# ndp filter
ndp_filter = Filters.NormalDepth.setup(self.options['min_n_depth'])
ndp_tag = ndp_filter.tags[0]
# Counts
processed = 0
try:
for record in o_iter:
if processed > 0 and processed % 1000 == 0:
self.logger.info(
"Processed {0} overlapping intervals...".format(
processed))
result = OverlapSet(record, self.callers)
for maf_record in self._merger.merge_records(result):
if maf_record is not None:
# Recheck normal depth
gdc_filters = maf_record['GDC_FILTER'].value
has_tag = ndp_tag in gdc_filters
ndp = ndp_filter.filter(maf_record)
if has_tag != ndp:
if ndp:
gdc_filters.extend(ndp_filter.tags)
else:
gdc_filters = list(
filter(lambda x: x != ndp_filter.tags[0],
gdc_filters))
maf_record["GDC_FILTER"] = get_builder(
"GDC_FILTER",
self._scheme,
value=sorted(gdc_filters))
# Add to sorter
sorter += maf_record
processed += 1
self.logger.info(
"Writing {0} sorted, merged records...".format(processed))
# Writer
self.maf_writer = MafWriter.from_path(
path=self.options['output_maf'],
header=self.maf_header,
validation_stringency=ValidationStringency.Strict)
counter = 0
for record in sorter:
if counter > 0 and counter % 1000 == 0:
self.logger.info(
"Wrote {0} sorted, merged records...".format(counter))
self.maf_writer += record
counter += 1
self.logger.info(
"Finished writing {0} sorted, merged records.".format(counter))
finally:
for reader in self.maf_readers:
reader.close()
sorter.close()
if self.maf_writer:
self.maf_writer.close()