def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1)
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write("1")
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEquals(c.status, DROPStates.INITIALIZED)
self.assertEquals(b.status, DROPStates.INITIALIZED)
self.assertEquals(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted("a", DROPStates.COMPLETED)
self.assertEquals(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEquals(c.status, DROPStates.COMPLETED)
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1)
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write('1')
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEqual(c.status, DROPStates.INITIALIZED)
self.assertEqual(b.status, DROPStates.INITIALIZED)
self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted('a', DROPStates.COMPLETED)
self.assertEqual(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEqual(c.status, DROPStates.COMPLETED)
def test_errorState(self):
a = InMemoryDROP("a", "a")
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
c.addProducer(b)
b.addInput(a)
a.setError()
self.assertEquals(DROPStates.ERROR, a.status)
self.assertEquals(DROPStates.ERROR, b.status)
self.assertEquals(DROPStates.ERROR, c.status)
def test_errorState(self):
a = InMemoryDROP('a', 'a')
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
c.addProducer(b)
b.addInput(a)
a.setError()
self.assertEqual(DROPStates.ERROR, a.status)
self.assertEqual(DROPStates.ERROR, b.status)
self.assertEqual(DROPStates.ERROR, c.status)
def _testGraph(execMode):
"""
A test graph that looks like this:
|--> B1 --> C1 --|
|--> B2 --> C2 --|
SL_A --> A --|--> B3 --> C3 --| --> D --> E
|--> .. --> .. --|
|--> BN --> CN --|
B and C DROPs are InMemorySleepAndCopyApp DROPs (see above). D is simply a
container. A is a socket listener, so we can actually write to it externally
and watch the progress of the luigi tasks. We give DROPs a long lifespan;
otherwise they will expire and luigi will see it as a failed task (which is
actually right!)
If execMode is given we use that in all DROPs. If it's None we use a mixture
of DROP/EXTERNAL execution modes.
"""
aMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
bMode = execMode if execMode is not None else ExecutionMode.DROP
cMode = execMode if execMode is not None else ExecutionMode.DROP
dMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
eMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
sl_a = SocketListenerApp('oid:SL_A', 'uid:SL_A', executionMode=aMode, lifespan=lifespan)
a = InMemoryDROP('oid:A', 'uid:A', executionMode=aMode, lifespan=lifespan)
d = SumupContainerChecksum('oid:D', 'uid:D', executionMode=dMode, lifespan=lifespan)
e = InMemoryDROP('oid:E', 'uid:E', executionMode=eMode, lifespan=lifespan)
sl_a.addOutput(a)
e.addProducer(d)
for i in xrange(random.SystemRandom().randint(10, 20)):
b = SleepAndCopyApp('oid:B%d' % (i), 'uid:B%d' % (i), executionMode=bMode, lifespan=lifespan)
c = InMemoryDROP('oid:C%d' % (i), 'uid:C%d' % (i), executionMode=cMode, lifespan=lifespan)
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
return sl_a
