def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + six.b(" "))
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e,f]):
a.write('20')
a.setCompleted()
# Check the final results are correct
for drop in [a,b,c,d,e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1)
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write("1")
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEquals(c.status, DROPStates.INITIALIZED)
self.assertEquals(b.status, DROPStates.INITIALIZED)
self.assertEquals(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted("a", DROPStates.COMPLETED)
self.assertEquals(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEquals(c.status, DROPStates.COMPLETED)
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1)
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write('1')
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEqual(c.status, DROPStates.INITIALIZED)
self.assertEqual(b.status, DROPStates.INITIALIZED)
self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted('a', DROPStates.COMPLETED)
self.assertEqual(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEqual(c.status, DROPStates.COMPLETED)
def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in xrange(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + " ")
# Create DROPs
a = InMemoryDROP("oid:A", "uid:A")
b = NumberWriterApp("oid:B", "uid:B")
c = InMemoryDROP("oid:A", "uid:A")
d = OddAndEvenContainerApp("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = InMemoryDROP("oid:F", "uid:F")
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e, f]):
a.write("20")
a.setCompleted()
# Check the final results are correct
for drop in [a, b, c, d, e]:
self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastChar = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastChar = data[-1]
outputDrop.write(self._lastChar)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP("a", "a")
b = LastCharWriterApp("b", "b")
c = SumupContainerChecksum("c", "c")
d = InMemoryDROP("d", "d")
e = InMemoryDROP("e", "e")
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, lambda: a.addConsumer(b))
self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastChar):
self.assertEquals(aStatus, a.status)
self.assertEquals(dStatus, d.status)
self.assertEquals(eStatus, e.status)
self.assertEquals(lastChar, b._lastChar)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write("abcde")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
a.write("fghij")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
a.write("k")
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")
self.assertEquals("ejk", droputils.allDropContents(d))
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = six.indexbytes(data, -1)
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))
checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write('abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
a.write('fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
a.write('k')
with DROPWaiterCtx(self, [d,e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')
self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def test_pg():
"""
A very simple graph that looks like this:
A --> B --> C --> D
"""
a = SocketListenerApp('A', 'A', lifespan=lifespan)
b = InMemoryDROP('B', 'B', lifespan=lifespan)
c = SleepAndCopyApp('C', 'C', lifespan=lifespan)
d = InMemoryDROP('D', 'D', lifespan=lifespan)
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
return a
def container_pg():
'''
Creates the following graph:
|--> B --> D --|
SL --> A --| |--> F --> G --> H --> I
|--> C --> E --|
'''
sl= SocketListenerApp('SL', 'SL', lifespan=lifespan)
a = InMemoryDROP('A', 'A', lifespan=lifespan)
b = SleepAndCopyApp('B', 'B', lifespan=lifespan)
c = SleepAndCopyApp('C', 'C', lifespan=lifespan)
d = InMemoryDROP('D', 'D', lifespan=lifespan)
e = InMemoryDROP('E', 'E', lifespan=lifespan)
f = SimpleBarrierApp('F', 'F', lifespan=lifespan)
g = ContainerDROP('G', 'G', lifespan=lifespan)
h = SleepAndCopyApp('H', 'H', lifespan=lifespan)
i = InMemoryDROP('I', 'I', lifespan=lifespan)
# Wire together
sl.addOutput(a)
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
d.addConsumer(f)
e.addConsumer(f)
f.addOutput(g)
g.addChild(d)
g.addChild(e)
g.addConsumer(h)
h.addOutput(i)
return sl
def _testGraph(execMode):
"""
A test graph that looks like this:
|--> B1 --> C1 --|
|--> B2 --> C2 --|
SL_A --> A --|--> B3 --> C3 --| --> D --> E
|--> .. --> .. --|
|--> BN --> CN --|
B and C DROPs are InMemorySleepAndCopyApp DROPs (see above). D is simply a
container. A is a socket listener, so we can actually write to it externally
and watch the progress of the luigi tasks. We give DROPs a long lifespan;
otherwise they will expire and luigi will see it as a failed task (which is
actually right!)
If execMode is given we use that in all DROPs. If it's None we use a mixture
of DROP/EXTERNAL execution modes.
"""
aMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
bMode = execMode if execMode is not None else ExecutionMode.DROP
cMode = execMode if execMode is not None else ExecutionMode.DROP
dMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
eMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
sl_a = SocketListenerApp('oid:SL_A', 'uid:SL_A', executionMode=aMode, lifespan=lifespan)
a = InMemoryDROP('oid:A', 'uid:A', executionMode=aMode, lifespan=lifespan)
d = SumupContainerChecksum('oid:D', 'uid:D', executionMode=dMode, lifespan=lifespan)
e = InMemoryDROP('oid:E', 'uid:E', executionMode=eMode, lifespan=lifespan)
sl_a.addOutput(a)
e.addProducer(d)
for i in xrange(random.SystemRandom().randint(10, 20)):
b = SleepAndCopyApp('oid:B%d' % (i), 'uid:B%d' % (i), executionMode=bMode, lifespan=lifespan)
c = InMemoryDROP('oid:C%d' % (i), 'uid:C%d' % (i), executionMode=cMode, lifespan=lifespan)
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
return sl_a
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = os.urandom(1025)
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
a.async_execute()
utils.write_to(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEqual(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEqual(data, bContents)
self.assertEqual(crc32(data, 0), dContents)
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = 'shine on you crazy diamond'
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
utils.writeToRemotePort(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEquals(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEquals(data, bContents)
self.assertEquals(crc32(data, 0), dContents)
def lofar_standard_pip_pg():
"""
Lofar simple imaging pipeline
based on
Ger's email and
https://github.com/lofar-astron/lofar-profiling
"""
roots = []
num_time_slice = 3
total_num_subband = 8
num_subb_per_image = 2
Cal_model_oid = "A-team/Calibrator"
dob_cal_model = InMemoryDROP(Cal_model_oid, Cal_model_oid, lifespan=lifespan)
dob_cal_model.location = "catalogue.Groningen"
roots.append(dob_cal_model)
make_source_app_oid = "makesourcedb"
make_source_app = SleepAndCopyApp(make_source_app_oid, make_source_app_oid)
make_source_app.location = dob_cal_model.location
dob_cal_model.addConsumer(make_source_app)
source_db_oid = "CalSource.db"
dob_source_db = InMemoryDROP(source_db_oid, source_db_oid, lifespan=lifespan)
dob_source_db.location = dob_cal_model.location
make_source_app.addOutput(dob_source_db)
GSM_oid = "GlobalSkyModel"
dob_gsm = InMemoryDROP(GSM_oid, GSM_oid, lifespan=lifespan)
dob_gsm.location = "catalogue.Groningen"
roots.append(dob_gsm)
dob_img_dict = dict()
sb_chunks = chunks(range(1, total_num_subband + 1), num_subb_per_image)
for k, img_list in enumerate(sb_chunks):
imger_oid = "AWImager_SB_{0}~{1}".format(img_list[0], img_list[-1])
dob_img = SleepAndCopyApp(imger_oid, imger_oid, lifespan=lifespan)
kk = k + 1
#print k, img_list, kk
img_prod_oid = "Image_SB_{0}~{1}".format(img_list[0], img_list[-1])
dob_img_prod = InMemoryDROP(img_prod_oid, img_prod_oid, lifespan=lifespan)
dob_img.addOutput(dob_img_prod)
dob_img_dict[kk] = (dob_img, dob_img_prod)
for j in range(1, num_time_slice + 1):
sli = "T%02d" % j
for i in range(1, total_num_subband + 1):
# for LBA, assume each subband should have two beams
# one for calibration pipeline, the other for target / imaging pipeline
stri = "%s_SB%02d" % (sli, i)
time_slice_oid = "Socket_{0}".format(stri)
dob_time_slice_sl = SocketListenerApp(time_slice_oid + "_SL", time_slice_oid + "_SL", lifespan=lifespan)
dob_time_slice = InMemoryDROP(time_slice_oid, time_slice_oid, lifespan=lifespan)
dob_time_slice_sl.addOutput(dob_time_slice)
# all the same sub-bands (i) will be on the same node regardless of its time slice j
dob_time_slice.location = "Node%03d.Groningen" % i
roots.append(dob_time_slice_sl)
oid_data_writer = "DataWriter_{0}".format(stri)
dob_ingest = SleepAndCopyApp(oid_data_writer, oid_data_writer, lifespan=lifespan)
dob_ingest.location = dob_time_slice.location
dob_time_slice.addConsumer(dob_ingest)
# assume a single measuremenset has multiple beams
oid = "CalNTgt.MS_{0}".format(stri)
dob_input = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_ingest.addOutput(dob_input)
dob_input.location = dob_ingest.location
#split by beam - calibrator beam and target beam
appOid = "Split_{0}".format(stri)
splitApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
splitApp.location = dob_ingest.location
dob_input.addConsumer(splitApp)
oid = "Cal_BEAM_{0}".format(stri)
dob_cal_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_cal_beam.location = dob_ingest.location
splitApp.addOutput(dob_cal_beam)
oid = "Tgt_BEAM_{0}".format(stri)
dob_tgt_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_tgt_beam.location = dob_ingest.location
splitApp.addOutput(dob_tgt_beam)
# flag the cal beam
oid = "FlgCal.MS_{0}".format(stri)
appOid = "NDPPP_Cal" + stri
dob_cal_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
flagCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_cal_beam.addConsumer(flagCalApp)
dob_source_db.addConsumer(flagCalApp)
flagCalApp.addOutput(dob_cal_flagged)
flagCalApp.location = dob_ingest.location
dob_cal_flagged.location = dob_ingest.location #"C%03d.Groningen" % i
# flag the target beam
oid = "FlgTgt.MS_{0}".format(stri)
appOid = "NDPPP_Tgt" + stri
dob_tgt_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
flagTgtApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_tgt_beam.addConsumer(flagTgtApp)
dob_source_db.addConsumer(flagTgtApp)
flagTgtApp.addOutput(dob_tgt_flagged)
flagTgtApp.location = dob_ingest.location
dob_tgt_flagged.location = dob_ingest.location
# For calibration, each time slice is calibrated independently, and
# the result is an output MS per subband
# solve the gain
oid = "Gain.TBL_{0}".format(stri)
appOid = "BBS_GainCal_{0}".format(stri)
dob_param_tbl = InMemoryDROP(oid, oid, lifespan=lifespan)
gainCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_cal_flagged.addConsumer(gainCalApp)
gainCalApp.addOutput(dob_param_tbl)
gainCalApp.location = dob_ingest.location
dob_param_tbl.location = dob_ingest.location
# or apply the gain to the dataset
oid = "CAL.MS_{0}".format(stri)
appOid = "BBS_ApplyCal_{0}".format(stri)
dob_calibrated = InMemoryDROP(oid, oid, lifespan=lifespan)
applyCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_tgt_flagged.addConsumer(applyCalApp)
dob_param_tbl.addConsumer(applyCalApp)
applyCalApp.addOutput(dob_calibrated)
applyCalApp.location = dob_ingest.location
dob_calibrated.location = dob_ingest.location
# extract local sky model (LSM) from GSM
appOid = "Extract_LSM_{0}".format(stri)
extractLSMApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
extractLSMApp.location = dob_ingest.location
dob_gsm.addConsumer(extractLSMApp)
LSM_oid = "LSM_{0}".format(stri)
dob_lsm = InMemoryDROP(LSM_oid, LSM_oid, lifespan=lifespan)
dob_lsm.location = dob_ingest.location
extractLSMApp.addOutput(dob_lsm)
# convert LSM to source.db
makesource_app_oid = "makesourcedb_{0}".format(stri)
makesource_app = SleepAndCopyApp(makesource_app_oid, makesource_app_oid)
makesource_app.location = dob_ingest.location
dob_lsm.addConsumer(makesource_app)
sourcedb_oid = "LSM_db_{0}".format(stri)
dob_lsmdb = InMemoryDROP(sourcedb_oid, sourcedb_oid, lifespan=lifespan)
dob_lsmdb.location = dob_ingest.location
makesource_app.addOutput(dob_lsmdb)
# add awimager
img_k = (i - 1) / num_subb_per_image + 1
dob_img, dob_img_prod = dob_img_dict[img_k]
dob_calibrated.addConsumer(dob_img)
dob_lsmdb.addConsumer(dob_img)
dob_img.location = dob_calibrated.location #overwrite to the last i
dob_img_prod.location = dob_img.location
return roots
def chiles_pg():
total_bandwidth = 480
num_obs = 8 # the same as num of data island
subband_width = 60 # MHz
num_subb = total_bandwidth / subband_width
subband_dict = collections.defaultdict(list) # for corner turning
img_list = []
start_freq = 940
# this should be removed
roots = []
for i in range(1, num_obs + 1):
stri = "%02d" % i
oid = "Obs_day_{0}".format(stri)
appOid = "Receive_" + oid
dob_obs_sl = SocketListenerApp(oid + "_SL", oid + "_SL", lifespan=lifespan, port=1110+i)
dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_obs_sl.addOutput(dob_obs)
dob_obs.location = "{0}.aws-ec2.sydney".format(i)
roots.append(dob_obs_sl)
for j in range(1, num_subb + 1):
app_oid = "mstransform_{0}_{1}".format(stri, "%02d" % j)
adob_split = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
adob_split.location = dob_obs.location
dob_obs.addConsumer(adob_split)
dob_sboid = "Split_{0}_{1}~{2}MHz".format(stri,
start_freq + subband_width * j,
start_freq + subband_width * (j + 1))
dob_sb = InMemoryDROP(dob_sboid, dob_sboid, lifespan=lifespan)
dob_sb.location = dob_obs.location
adob_split.addOutput(dob_sb)
subband_dict[j].append(dob_sb)
for j, v in subband_dict.items():
oid = "Subband_{0}~{1}MHz".format(start_freq + subband_width * j,
start_freq + subband_width * (j + 1))
appOid = "Combine_" + oid
dob = InMemoryDROP(oid, oid)
dob.location = "{0}.aws-ec2.sydney".format(j % num_obs)
subbandCombineAppDrop = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
subbandCombineAppDrop.addOutput(dob)
for dob_sb in v:
subbandCombineAppDrop.addInput(dob_sb)
app_oid = oid.replace("Subband_", "Clean_")
adob_clean = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
adob_clean.location = dob.location
dob.addConsumer(adob_clean)
img_oid = oid.replace("Subband_", "Image_")
dob_img = InMemoryDROP(img_oid, img_oid, lifespan=lifespan)
dob_img.location = dob.location
adob_clean.addOutput(dob_img)
img_list.append(dob_img)
#container + app that produces it
combineAppDrop = SleepAndCopyApp('Combine', 'Combine')
dob_comb_img_oid = "Combined_image"
dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
dob_comb_img.location = "10.1.1.100:7777"
combineAppDrop.addOutput(dob_comb_img)
for dob_img in img_list:
combineAppDrop.addInput(dob_img)
#concatenate all images
adob_concat_oid = "Concat_image"
adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
adob_concat.location = dob_comb_img.location
dob_comb_img.addConsumer(adob_concat)
# produce cube
dob_cube_oid = "Cube_Day{0}~{1}_{2}~{3}MHz".format(1,
num_obs,
start_freq,
start_freq + total_bandwidth)
dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
dob_cube.location = dob_comb_img.location
adob_concat.addOutput(dob_cube)
return roots
def mwa_fornax_pg():
num_coarse_ch = 24
num_split = 3 # number of time splits per channel
dob_root_sl = SocketListenerApp("MWA_LTA_SL", "MWA_LTA_SL", lifespan=lifespan)
dob_root = InMemoryDROP("MWA_LTA", "MWA_LTA", lifespan=lifespan)
dob_root.location = "Pawsey"
dob_root_sl.addOutput(dob_root)
#container
dob_comb_img_oid = "Combined_image"
combineImgAppOid = "Combine image"
dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
combineImgApp = SleepAndCopyApp(combineImgAppOid, combineImgAppOid)
combineImgApp.addOutput(dob_comb_img)
dob_comb_img.location = "f032.fornax"
for i in range(1, num_coarse_ch + 1):
stri = "%02d" % i
oid = "Subband_{0}".format(stri)
appOid = "Combine_" + oid
dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
combineSubbandApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
combineSubbandApp.addOutput(dob_obs)
dob_obs.location = "f%03d.fornax" % i
oid_ingest = "NGAS_{0}".format(stri)
dob_ingest = SleepAndCopyApp(oid_ingest, oid_ingest, lifespan=lifespan)
dob_ingest.location = "f%03d.fornax:7777" % i
dob_root.addConsumer(dob_ingest)
for j in range(1, num_split + 1):
strj = "%02d" % j
split_oid = "Subband_{0}_Split_{1}".format(stri, strj)
dob_split = InMemoryDROP(split_oid, split_oid, lifespan=lifespan)
dob_split.location = dob_obs.location
dob_ingest.addOutput(dob_split)
combineSubbandApp.addInput(dob_split)
oid_rts = "RTS_{0}".format(stri)
dob_rts = SleepAndCopyApp(oid_rts, oid_rts, lifespan=lifespan)
dob_rts.location = dob_obs.location
dob_obs.addConsumer(dob_rts)
oid_subimg = "Subcube_{0}".format(stri)
dob_subimg = InMemoryDROP(oid_subimg, oid_subimg, lifespan=lifespan)
dob_subimg.location = dob_obs.location
dob_rts.addOutput(dob_subimg)
combineImgApp.addInput(dob_subimg)
#concatenate all images
adob_concat_oid = "Concat_image"
adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
adob_concat.location = dob_comb_img.location
dob_comb_img.addConsumer(adob_concat)
# produce cube
dob_cube_oid = "Cube_30.72MHz"
dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
dob_cube.location = dob_comb_img.location
adob_concat.addOutput(dob_cube)
return dob_root_sl
def test_simple_chain(self):
"""
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
"""
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = kwargs["substring"]
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
buf = ""
for c in line:
if c == " " or c == "\n":
output.write(buf[::-1])
output.write(c)
buf = ""
else:
buf += c
a = InMemoryDROP("oid:A", "uid:A")
b = GrepResult("oid:B", "uid:B", substring="a")
c = InMemoryDROP("oid:C", "uid:C")
d = SortResult("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = RevResult("oid:F", "oid:F")
g = InMemoryDROP("oid:G", "uid:G")
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def pip_cont_img_pg():
"""
PIP continuum imaging pipeline
"""
num_beam = 1
num_time = 2
num_freq = 2
num_facet = 2
num_grid = 4
stokes = ['I', 'Q', 'U', 'V']
dob_root_sl = SocketListenerApp("FullDataset_SL", "FullDataset_SL", lifespan=lifespan)
dob_root = InMemoryDROP("FullDataset", "FullDataset", lifespan=lifespan)
dob_root_sl.addOutput(dob_root)
dob_root.location = "Buf01"
adob_sp_beam = SleepAndCopyApp("SPLT_BEAM", "SPLT_BEAM", lifespan=lifespan)
adob_sp_beam.location = "Buf01"
dob_root.addConsumer(adob_sp_beam)
for i in range(1, num_beam + 1):
uid = i
dob_beam = InMemoryDROP("BEAM_{0}".format(uid), "BEAM_{0}".format(uid), lifespan=lifespan)
adob_sp_beam.addOutput(dob_beam)
adob_sp_time = SleepAndCopyApp("SPLT_TIME_{0}".format(uid), "SPLT_TIME_{0}".format(uid), lifespan=lifespan)
dob_beam.addConsumer(adob_sp_time)
for j in range(1, num_time + 1):
uid = "%d-%d" % (i, j)
dob_time = InMemoryDROP("TIME_{0}".format(uid), "TIME_{0}".format(uid), lifespan=lifespan)
adob_sp_time.addOutput(dob_time)
adob_sp_freq = SleepAndCopyApp("SPLT_FREQ_{0}".format(uid), "SPLT_FREQ_{0}".format(uid), lifespan=lifespan)
dob_time.addConsumer(adob_sp_freq)
for k in range(1, num_freq + 1):
uid = "%d-%d-%d" % (i, j, k)
dob_freq = InMemoryDROP("FREQ_{0}".format(uid), "FREQ_{0}".format(uid), lifespan=lifespan)
adob_sp_freq.addOutput(dob_freq)
adob_sp_facet = SleepAndCopyApp("SPLT_FACET_{0}".format(uid), "SPLT_FACET_{0}".format(uid), lifespan=lifespan)
dob_freq.addConsumer(adob_sp_facet)
for l in range(1, num_facet + 1):
uid = "%d-%d-%d-%d" % (i, j, k, l)
dob_facet = InMemoryDROP("FACET_{0}".format(uid), "FACET_{0}".format(uid), lifespan=lifespan)
adob_sp_facet.addOutput(dob_facet)
adob_ph_rot = SleepAndCopyApp("PH_ROTATN_{0}".format(uid), "PH_ROTATN_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_ph_rot)
dob_ph_rot = InMemoryDROP("PH_ROTD_{0}".format(uid), "PH_ROTD_{0}".format(uid), lifespan=lifespan)
adob_ph_rot.addOutput(dob_ph_rot)
adob_sp_stokes = SleepAndCopyApp("SPLT_STOKES_{0}".format(uid), "SPLT_STOKES_{0}".format(uid), lifespan=lifespan)
dob_ph_rot.addConsumer(adob_sp_stokes)
adob_w_kernel = SleepAndCopyApp("CACL_W_Knl_{0}".format(uid), "CACL_W_Knl_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_w_kernel)
dob_w_knl = InMemoryDROP("W_Knl_{0}".format(uid), "W_Knl_{0}".format(uid), lifespan=lifespan)
adob_w_kernel.addOutput(dob_w_knl)
adob_a_kernel = SleepAndCopyApp("CACL_A_Knl_{0}".format(uid), "CACL_A_Knl_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_a_kernel)
dob_a_knl = InMemoryDROP("A_Knl_{0}".format(uid), "A_Knl_{0}".format(uid), lifespan=lifespan)
adob_a_kernel.addOutput(dob_a_knl)
adob_create_kernel = SleepAndCopyApp("CREATE_Knl_{0}".format(uid), "CREATE_Knl_{0}".format(uid), lifespan=lifespan)
dob_w_knl.addConsumer(adob_create_kernel)
dob_a_knl.addConsumer(adob_create_kernel)
for stoke in stokes:
uid = "%s-%d-%d-%d-%d" % (stoke, i, j, k, l)
#print "uid = {0}".format(uid)
dob_stoke = InMemoryDROP("STOKE_{0}".format(uid), "STOKE_{0}".format(uid), lifespan=lifespan)
adob_sp_stokes.addOutput(dob_stoke)
dob_stoke.addConsumer(adob_create_kernel)
dob_aw = InMemoryDROP("A_{0}".format(uid), "A_{0}".format(uid), lifespan=lifespan)
adob_create_kernel.addOutput(dob_aw)
# we do not do loop yet
griders = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_gridding = SleepAndCopyApp("Gridding_{0}".format(gid), "Gridding_{0}".format(gid), lifespan=lifespan)
dob_stoke.addConsumer(adob_gridding)
dob_gridded_cell = InMemoryDROP("Grided_Cell_{0}".format(gid), "Grided_Cell_{0}".format(gid), lifespan=lifespan)
adob_gridding.addOutput(dob_gridded_cell)
griders.append(dob_gridded_cell)
adob_gridded_bar = SleepAndCopyApp("GRIDDED_BARRIER_{0}".format(uid), "GRIDDED_BARRIER_{0}".format(uid), lifespan=lifespan)
for grider in griders:
grider.addConsumer(adob_gridded_bar)
dob_gridded_stoke = InMemoryDROP("GRIDDED_STOKE_{0}".format(uid), "GRIDDED_STOKE_{0}".format(uid), lifespan=lifespan)
adob_gridded_bar.addOutput(dob_gridded_stoke)
FFTers = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_fft = SleepAndCopyApp("FFT_{0}".format(gid), "FFT_{0}".format(gid), lifespan=lifespan)
dob_gridded_stoke.addConsumer(adob_fft)
dob_ffted_cell = InMemoryDROP("FFTed_Cell_{0}".format(gid), "FFTed_Cell_{0}".format(gid), lifespan=lifespan)
adob_fft.addOutput(dob_ffted_cell)
FFTers.append(dob_ffted_cell)
adob_ffted_bar = SleepAndCopyApp("FFTed_BARRIER_{0}".format(uid), "FFTed_BARRIER_{0}".format(uid), lifespan=lifespan)
for ffter in FFTers:
ffter.addConsumer(adob_ffted_bar)
dob_ffted_stoke = InMemoryDROP("FFTed_STOKE_{0}".format(uid), "FFTed_STOKE_{0}".format(uid), lifespan=lifespan)
adob_ffted_bar.addOutput(dob_ffted_stoke)
cleaners = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_cleaner = SleepAndCopyApp("DECONV_{0}".format(gid), "DECONV_{0}".format(gid), lifespan=lifespan)
dob_ffted_stoke.addConsumer(adob_cleaner)
dob_cleaned_cell = InMemoryDROP("CLEANed_Cell_{0}".format(gid), "CLEANed_Cell_{0}".format(gid), lifespan=lifespan)
adob_cleaner.addOutput(dob_cleaned_cell)
cleaners.append(dob_cleaned_cell)
adob_cleaned_bar = SleepAndCopyApp("CLEANed_BARRIER_{0}".format(uid), "CLEANed_BARRIER_{0}".format(uid), lifespan=lifespan)
for cleaner in cleaners:
cleaner.addConsumer(adob_cleaned_bar)
dob_decon_stoke = InMemoryDROP("CLEANed_STOKE_{0}".format(uid), "CLEANed_STOKE_{0}".format(uid), lifespan=lifespan)
adob_cleaned_bar.addOutput(dob_decon_stoke)
adob_create_prod = SleepAndCopyApp("CRT-PROD_{0}".format(uid), "CRT-PROD_{0}".format(uid), lifespan=lifespan)
dob_decon_stoke.addConsumer(adob_create_prod)
dob_prod = InMemoryDROP("PRODUCT_{0}".format(uid), "PRODUCT_{0}".format(uid), lifespan=lifespan)
adob_create_prod.addOutput(dob_prod)
return dob_root_sl
def test_simple_chain(self):
'''
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
'''
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = six.b(kwargs['substring'])
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = bytearray()
for c in allbytes:
if c == six.b(' ') or c == six.b('\n'):
output.write(buf[::-1])
output.write(c)
buf = bytearray()
else:
buf.append(c)
a = InMemoryDROP('oid:A', 'uid:A')
b = GrepResult('oid:B', 'uid:B', substring="a")
c = InMemoryDROP('oid:C', 'uid:C')
d = SortResult('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = RevResult('oid:F', 'oid:F')
g = InMemoryDROP('oid:G', 'uid:G')
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
