def test_errorState(self): a = InMemoryDROP("a", "a") b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") c.addProducer(b) b.addInput(a) a.setError() self.assertEquals(DROPStates.ERROR, a.status) self.assertEquals(DROPStates.ERROR, b.status) self.assertEquals(DROPStates.ERROR, c.status)
def test_errorState(self): a = InMemoryDROP('a', 'a') b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') c.addProducer(b) b.addInput(a) a.setError() self.assertEqual(DROPStates.ERROR, a.status) self.assertEqual(DROPStates.ERROR, b.status) self.assertEqual(DROPStates.ERROR, c.status)
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ # create file data objects a1 = InMemoryDROP("oid:A1", "uid:A1") a2 = InMemoryDROP("oid:A2", "uid:A2") a3 = InMemoryDROP("oid:A3", "uid:A3") # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum("oid:B1", "uid:B1") b2 = SumupContainerChecksum("oid:B2", "uid:B2") b3 = SumupContainerChecksum("oid:B3", "uid:B3") c1 = InMemoryDROP("oid:C1", "uid:C1") c2 = InMemoryDROP("oid:C2", "uid:C2") c3 = InMemoryDROP("oid:C3", "uid:C3") # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33) e = InMemoryDROP("oid:E", "uid:E") # Wire together dropAList = [a1, a2, a3] dropBList = [b1, b2, b3] dropCList = [c1, c2, c3] for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): # for dropA in dropAList: # this should be parallel for a1.write(" ") a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(" ") a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEquals(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEquals(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEquals(sum_crc, 0) self.assertEquals(sum_crc, dropEData)
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ #create file data objects a1 = InMemoryDROP('oid:A1', 'uid:A1') a2 = InMemoryDROP('oid:A2', 'uid:A2') a3 = InMemoryDROP('oid:A3', 'uid:A3') # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33) e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in zip(dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in zip(dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): #for dropA in dropAList: # this should be parallel for a1.write(' '); a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(' '); a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEqual(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEqual(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)