def test_duplicate_fragments_read1(self): # load pair 1 p = list(test_utils.pair1()) p = test_utils.erase_read2(p) p0 = p[0] # insert the pair into the context, twice self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual( 1, len(self.__ctx.emitted.values() [0])) # only one SAM record associated with the key short_name = p0.get_name()[0:-2] self.assertEqual(short_name, self.__ctx.emitted.keys()[0]) self.assertTrue( re.match("\d+\s+%s\s+%d\s+.*" % (p0.tid, p0.pos), self.__ctx.emitted[short_name][0])) # check counter self.assertFalse( self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key( self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_duplicate_fragments_read1_no_discard(self): # load pair 1 and erase its second read p = list(test_utils.pair1()) p = test_utils.erase_read2(p) p0 = p[0] # insert the pair into the context, twice self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(2, len(self.__ctx.emitted.values() [0])) # Two SAM records associated with the key short_name = p0.get_name()[0:-2] self.assertEqual(short_name, self.__ctx.emitted.keys()[0]) flags = map(lambda sam: int(*re.match("(\d+).*", sam).groups(1)), self.__ctx.emitted.values()[0]) # ensure we have one marked as duplicate self.assertEqual( 1, len(filter(lambda flag: flag & sam_flags.SAM_FDP, flags))) # and ensure we have one NOT marked as duplicates self.assertEqual( 1, len(filter(lambda flag: flag & sam_flags.SAM_FDP == 0, flags))) # check counter self.assertFalse( self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key( self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_fragment_with_duplicate_in_pair_1_no_discard(self): # Ensure the reducer catches a fragment duplicate of pair[0] p = list(test_utils.pair1()) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) p = test_utils.erase_read2(p) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) # now ensure that both were emitted, but the fragment is marked as duplicate self.__ensure_pair1_emitted() self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(3, len(self.__ctx.emitted.values()[0])) # 3 SAM records associated with the key (for the pair) # make sure we have a read with the duplicate flag set regexp = "(\d+)\s+.*" flags = [ int(re.match(regexp, value).group(1)) for value in self.__ctx.emitted.values()[0] ] dup_flags = [ flag for flag in flags if flag & sam_flags.SAM_FDP ] self.assertEqual(1, len(dup_flags)) f = dup_flags[0] self.assertTrue( f & sam_flags.SAM_FR1 > 0 ) # ensure the duplicate read is r1 self.assertTrue( f & sam_flags.SAM_FPD == 0 ) # ensure the duplicate read is unpaired # check counter self.assertFalse(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key(self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_duplicate_pairs_no_discard(self): # Two identical pairs. Ensure only one is emitted p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) # add it twice self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual( 4, len(self.__ctx.emitted.values() [0])) # four SAM records associated with the same key flags = map(lambda sam: int(*re.match("(\d+).*", sam).groups(1)), self.__ctx.emitted.values()[0]) # ensure we have two marked as duplicates self.assertEqual( 2, len(filter(lambda flag: flag & sam_flags.SAM_FDP, flags))) # ensure we have two NOT marked as duplicates self.assertEqual( 2, len(filter(lambda flag: flag & sam_flags.SAM_FDP == 0, flags))) # check counter if self.__ctx.counters.has_key(self.__frag_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__frag_counter_name()]) self.assertTrue(self.__ctx.counters.has_key( self.__pair_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__pair_counter_name()])
def test_duplicate_pairs_right_key(self): # Two identical pairs on the right key # Ensure nothing is emitted p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[1]), PAIR_STRING) self.__ctx.add_value(test_utils.make_key(p[1]), PAIR_STRING) # add it twice self.__reducer.reduce(self.__ctx) self.assertEqual(0, len(self.__ctx.emitted.keys())) # check counter if self.__ctx.counters.has_key(self.__pair_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__pair_counter_name()]) if self.__ctx.counters.has_key(self.__frag_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__frag_counter_name()])
def test_fragment_with_duplicate_in_pair_1(self): # Ensure the reducer catches a fragment duplicate of pair[0] p = list(test_utils.pair1()) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) test_utils.erase_read2(p) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) # now ensure that the pair was emitted, but not the fragment self.__ensure_only_pair1_emitted() self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(2, len(self.__ctx.emitted.values()[0])) # two SAM records associated with the key (for the pair) # check counter self.assertFalse(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key(self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_duplicate_pairs(self): # Two identical pairs. Ensure only one is emitted p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) # add it twice self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(2, len(self.__ctx.emitted.values()[0])) # two SAM records associated with the same key self.__ensure_only_pair1_emitted() # check counter if self.__ctx.counters.has_key(self.__frag_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__frag_counter_name()]) self.assertTrue(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__pair_counter_name()])
def test_unmapped2(self): p = test_utils.pair1() p[1].set_mapped(False) p[0].set_mate_mapped(False) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) self.assertEqual(2, len(self.__ctx.emitted.values()[0]))
def test_emit_on_left_key(self): # load pair 1 p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) self.__ensure_only_pair1_emitted()
def test_no_emit_on_right_key(self): # load pair 1 p = test_utils.pair1() # use the SECOND read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[1]), PAIR_STRING) self.__reducer.reduce(self.__ctx) # we should have no output self.assertEqual(0, len(self.__ctx.emitted.keys()))
def test_unmapped2(self): self.pair1[1].set_mapped(False) self.pair1[0].set_mate_mapped(False) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) self.assertEqual(0, len(filter(lambda k: re.match(UNMAPPED_STRING + ":\d+", k), self.ctx.emitted.keys()))) self.assertTrue( test_utils.make_key(self.pair1[0]) in self.ctx.emitted.keys() ) self.assertEqual(1, self.ctx.counters["Test:UNMAPPED READS"])
def test_fragment_with_duplicate_in_pair_2(self): # Ensure the reducer catches a fragment duplicate of pair[1]. p = list(test_utils.pair1()) # Insert the pair into the context self.__ctx.add_value(test_utils.make_key(p[1]), PAIR_STRING) # Remove the first read from the pair, reorder so that the None is at index 1, # the serialize and insert into the context. test_utils.erase_read1(p) self.__ctx.add_value(test_utils.make_key(p[1]), proto.serialize_pair( (p[1], None) )) self.__reducer.reduce(self.__ctx) # now ensure that nothing was emitted. The pair isn't emitted because # the key refers to read2, and the fragment isn't emitted because it's a duplicate of # the one in the pair. self.assertEqual(0, len(self.__ctx.emitted.keys())) # check counter self.assertFalse(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key(self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_duplicate_fragments_read1(self): # load pair 1 p = list(test_utils.pair1()) p = test_utils.erase_read2(p) p0 = p[0] # insert the pair into the context, twice self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(1, len(self.__ctx.emitted.values()[0])) # only one SAM record associated with the key short_name = p0.get_name()[0:-2] self.assertEqual(short_name, self.__ctx.emitted.keys()[0]) self.assertTrue( re.match("\d+\s+%s\s+%d\s+.*" % (p0.tid, p0.pos), self.__ctx.emitted[short_name][0]) ) # check counter self.assertFalse(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key(self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_unmapped1(self): p = test_utils.pair1() p[0].set_mapped(False) p[1].set_mate_mapped(False) # Having an unmapped read before a mapped read is not allowed. This should # raise an exception # The key is meaningless self.__ctx.add_value(test_utils.make_key(p[1]), proto.serialize_pair(p)) self.assertRaises(ValueError, self.__reducer.reduce, self.__ctx)
def test_duplicate_pairs_no_discard(self): # Two identical pairs. Ensure only one is emitted p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) # add it twice self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(4, len(self.__ctx.emitted.values()[0])) # four SAM records associated with the same key flags = map(lambda sam: int(*re.match("(\d+).*", sam).groups(1)), self.__ctx.emitted.values()[0]) # ensure we have two marked as duplicates self.assertEqual(2, len(filter(lambda flag: flag & sam_flags.SAM_FDP, flags)) ) # ensure we have two NOT marked as duplicates self.assertEqual(2, len(filter(lambda flag: flag & sam_flags.SAM_FDP == 0, flags)) ) # check counter if self.__ctx.counters.has_key(self.__frag_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__frag_counter_name()]) self.assertTrue(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__pair_counter_name()])
def test_fragment_with_duplicate_in_pair_2(self): # Ensure the reducer catches a fragment duplicate of pair[1]. p = list(test_utils.pair1()) # Insert the pair into the context self.__ctx.add_value(test_utils.make_key(p[1]), PAIR_STRING) # Remove the first read from the pair, reorder so that the None is at index 1, # the serialize and insert into the context. test_utils.erase_read1(p) self.__ctx.add_value(test_utils.make_key(p[1]), proto.serialize_pair((p[1], None))) self.__reducer.reduce(self.__ctx) # now ensure that nothing was emitted. The pair isn't emitted because # the key refers to read2, and the fragment isn't emitted because it's a duplicate of # the one in the pair. self.assertEqual(0, len(self.__ctx.emitted.keys())) # check counter self.assertFalse( self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key( self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_fragment_with_duplicate_in_pair_1(self): # Ensure the reducer catches a fragment duplicate of pair[0] p = list(test_utils.pair1()) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) test_utils.erase_read2(p) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.reduce(self.__ctx) # now ensure that the pair was emitted, but not the fragment self.__ensure_only_pair1_emitted() self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(2, len( self.__ctx.emitted.values() [0])) # two SAM records associated with the key (for the pair) # check counter self.assertFalse( self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key( self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_duplicate_pairs(self): # Two identical pairs. Ensure only one is emitted p = test_utils.pair1() # use the first read to create the map-reduce key self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) # add it twice self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual( 2, len(self.__ctx.emitted.values() [0])) # two SAM records associated with the same key self.__ensure_only_pair1_emitted() # check counter if self.__ctx.counters.has_key(self.__frag_counter_name()): self.assertEqual(0, self.__ctx.counters[self.__frag_counter_name()]) self.assertTrue(self.__ctx.counters.has_key( self.__pair_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__pair_counter_name()])
def test_emit_reverse_fragment1(self): # None in pair[0]. Fragment in pair[1]. self.pair1 = test_utils.erase_read1(list(self.pair1)) self.pair1[1].set_on_reverse(True) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) expected_key = test_utils.make_key(self.pair1[1]) self.assertEqual(1, len(self.ctx.emitted[expected_key])) unserialized = proto.unserialize_pair(self.ctx.emitted[expected_key][0]) self.assertTrue(unserialized[1] is None) self.assertEqual(self.pair1[1].tid, unserialized[0].tid) self.assertEqual(self.pair1[1].pos, unserialized[0].pos) self.assertTrue(unserialized[0].is_on_reverse())
def test_unmapped2(self): self.pair1[1].set_mapped(False) self.pair1[0].set_mate_mapped(False) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) self.assertEqual( 0, len( filter(lambda k: re.match(UNMAPPED_STRING + ":\d+", k), self.ctx.emitted.keys()))) self.assertTrue( test_utils.make_key(self.pair1[0]) in self.ctx.emitted.keys()) self.assertEqual(1, self.ctx.counters["Test:UNMAPPED READS"])
def test_emit_forward_fragment2(self): # Fragment in pair[0]. None in pair[1] self.pair1 = test_utils.erase_read2(list(self.pair1)) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) expected_key = test_utils.make_key(self.pair1[0]) self.assertEqual(1, len(self.ctx.emitted[expected_key])) unserialized = proto.unserialize_pair(self.ctx.emitted[expected_key][0]) self.assertTrue(unserialized[1] is None) self.assertEqual(self.pair1[0].tid, unserialized[0].tid) self.assertEqual(self.pair1[0].pos, unserialized[0].pos) self.assertTrue(self.ctx.counters.has_key("Test:MAPPED COORDINATES")) self.assertEqual(1, self.ctx.counters["Test:MAPPED COORDINATES"])
def test_fragment_with_duplicate_in_pair_1_no_discard(self): # Ensure the reducer catches a fragment duplicate of pair[0] p = list(test_utils.pair1()) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) p = test_utils.erase_read2(p) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) # now ensure that both were emitted, but the fragment is marked as duplicate self.__ensure_pair1_emitted() self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(3, len( self.__ctx.emitted.values() [0])) # 3 SAM records associated with the key (for the pair) # make sure we have a read with the duplicate flag set regexp = "(\d+)\s+.*" flags = [ int(re.match(regexp, value).group(1)) for value in self.__ctx.emitted.values()[0] ] dup_flags = [flag for flag in flags if flag & sam_flags.SAM_FDP] self.assertEqual(1, len(dup_flags)) f = dup_flags[0] self.assertTrue( f & sam_flags.SAM_FR1 > 0) # ensure the duplicate read is r1 self.assertTrue( f & sam_flags.SAM_FPD == 0) # ensure the duplicate read is unpaired # check counter self.assertFalse( self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key( self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_emit_reverse_fragment1(self): # None in pair[0]. Fragment in pair[1]. self.pair1 = test_utils.erase_read1(list(self.pair1)) self.pair1[1].set_on_reverse(True) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) expected_key = test_utils.make_key(self.pair1[1]) self.assertEqual(1, len(self.ctx.emitted[expected_key])) unserialized = proto.unserialize_pair( self.ctx.emitted[expected_key][0]) self.assertTrue(unserialized[1] is None) self.assertEqual(self.pair1[1].tid, unserialized[0].tid) self.assertEqual(self.pair1[1].pos, unserialized[0].pos) self.assertTrue(unserialized[0].is_on_reverse())
def test_emit_forward_fragment2(self): # Fragment in pair[0]. None in pair[1] self.pair1 = test_utils.erase_read2(list(self.pair1)) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) expected_key = test_utils.make_key(self.pair1[0]) self.assertEqual(1, len(self.ctx.emitted[expected_key])) unserialized = proto.unserialize_pair( self.ctx.emitted[expected_key][0]) self.assertTrue(unserialized[1] is None) self.assertEqual(self.pair1[0].tid, unserialized[0].tid) self.assertEqual(self.pair1[0].pos, unserialized[0].pos) self.assertTrue(self.ctx.counters.has_key("Test:MAPPED COORDINATES")) self.assertEqual(1, self.ctx.counters["Test:MAPPED COORDINATES"])
def test_unmapped1(self): self.pair1[0].set_mapped(False) self.pair1[1].set_mate_mapped(False) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) self.assertTrue( test_utils.make_key(self.pair1[1]) in self.ctx.emitted.keys() ) self.assertEqual(1, len(self.ctx.emitted.values()[0])) unserialized = proto.unserialize_pair(self.ctx.emitted.values()[0][0]) self.assertFalse(unserialized[0] is None) self.assertFalse(unserialized[1] is None) self.assertEqual(self.pair1[1].tid, unserialized[0].tid) self.assertEqual(self.pair1[1].pos, unserialized[0].pos) self.assertEqual(1, self.ctx.counters["Test:UNMAPPED READS"])
def test_unmapped1(self): self.pair1[0].set_mapped(False) self.pair1[1].set_mate_mapped(False) self.link.process(self.pair1) self.assertEqual(1, len(self.ctx.emitted.keys())) self.assertTrue( test_utils.make_key(self.pair1[1]) in self.ctx.emitted.keys()) self.assertEqual(1, len(self.ctx.emitted.values()[0])) unserialized = proto.unserialize_pair(self.ctx.emitted.values()[0][0]) self.assertFalse(unserialized[0] is None) self.assertFalse(unserialized[1] is None) self.assertEqual(self.pair1[1].tid, unserialized[0].tid) self.assertEqual(self.pair1[1].pos, unserialized[0].pos) self.assertEqual(1, self.ctx.counters["Test:UNMAPPED READS"])
def test_duplicate_fragments_read1_no_discard(self): # load pair 1 and erase its second read p = list(test_utils.pair1()) p = test_utils.erase_read2(p) p0 = p[0] # insert the pair into the context, twice self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__ctx.add_value(test_utils.make_key(p[0]), proto.serialize_pair(p)) self.__reducer.discard_duplicates = False self.__reducer.reduce(self.__ctx) self.assertEqual(1, len(self.__ctx.emitted.keys())) self.assertEqual(2, len(self.__ctx.emitted.values()[0])) # Two SAM records associated with the key short_name = p0.get_name()[0:-2] self.assertEqual(short_name, self.__ctx.emitted.keys()[0]) flags = map(lambda sam: int(*re.match("(\d+).*", sam).groups(1)), self.__ctx.emitted.values()[0]) # ensure we have one marked as duplicate self.assertEqual(1, len(filter(lambda flag: flag & sam_flags.SAM_FDP, flags)) ) # and ensure we have one NOT marked as duplicates self.assertEqual(1, len(filter(lambda flag: flag & sam_flags.SAM_FDP == 0, flags)) ) # check counter self.assertFalse(self.__ctx.counters.has_key(self.__pair_counter_name())) self.assertTrue(self.__ctx.counters.has_key(self.__frag_counter_name())) self.assertEqual(1, self.__ctx.counters[self.__frag_counter_name()])
def test_empty_read1(self): # Ensure the reducer raises an exception if the pair[0] is None p = test_utils.erase_read1(list(test_utils.pair1())) self.__ctx.add_value(test_utils.make_key(p[1]), proto.serialize_pair(p)) self.assertRaises(ValueError, self.__reducer.reduce, self.__ctx)