def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(six.b(str(i)) + b" ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + b" ")
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e, f]):
a.write(b'20')
a.setCompleted()
# Check the final results are correct
for drop in [a, b, c, d, e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(b"0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
self.assertEqual(b"1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def test_runGraphSeveralDropsPerDM(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #1 DM #2
=================== ================
| A --> C --> D --|----|-| |
| | | |--> E --> F |
| B --------------|----|-| |
=================== ================
:see: `self.test_runGraphSingleDOPerDOM`
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]},
{"oid":"B", "type":"plain", "storage": "memory"},
{"oid":"C", "type":"app", "app":"dlg.apps.crc.CRCApp"},
{"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}]
g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"},
{"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}]
rels = [DROPRel('D', DROPLinkType.INPUT, 'E'),
DROPRel('B', DROPLinkType.INPUT, 'E')]
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(4, len(dm1._sessions[sessionId].drops))
self.assertEqual(2, len(dm2._sessions[sessionId].drops))
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
a,b,c,d = [dm1._sessions[sessionId].drops[x] for x in ('A', 'B', 'C', 'D')]
e,f = [dm2._sessions[sessionId].drops[x] for x in ('E', 'F')]
with droputils.DROPWaiterCtx(self, f, 5):
a.write(b'a')
a.setCompleted()
b.write(b'a')
b.setCompleted()
for drop in a,b,c,d,e,f:
self.assertEqual(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid))
self.assertEqual(a.checksum, int(droputils.allDropContents(d)))
self.assertEqual(b.checksum + d.checksum, int(droputils.allDropContents(f)))
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_multi_listappendthrashing(self, size=1000, parallel=True):
max_threads = cpu_count(logical=False)
drop_ids = [chr(97 + x) for x in range(max_threads)]
threadpool = ThreadPool(processes=max_threads)
memory_manager = DlgSharedMemoryManager()
session_id = 1
memory_manager.register_session(session_id)
S = InMemoryDROP("S", "S")
X = AverageArraysApp("X", "X")
Z = InMemoryDROP("Z", "Z")
drops = [ListAppendThrashingApp(x, x, size=size) for x in drop_ids]
mdrops = [
InMemoryDROP(chr(65 + x), chr(65 + x)) for x in range(max_threads)
]
if parallel:
# a bit of magic to get the app drops using the processes
_ = [drop.__setattr__("_tp", threadpool) for drop in drops]
_ = [drop.__setattr__("_tp", threadpool) for drop in mdrops]
_ = [drop.__setattr__("_sessID", session_id) for drop in mdrops]
_ = [
memory_manager.register_drop(drop.uid, session_id)
for drop in mdrops
]
X.__setattr__("_tp", threadpool)
Z.__setattr__("_tp", threadpool)
Z.__setattr__("_sessID", session_id)
memory_manager.register_drop(Z.uid, session_id)
_ = [d.addInput(S) for d in drops]
_ = [d.addOutput(m) for d, m in zip(drops, mdrops)]
_ = [X.addInput(m) for m in mdrops]
X.addOutput(Z)
logger.info(
f"Number of inputs/outputs: {len(X.inputs)}, {len(X.outputs)}")
self._test_graph_runs([S, X, Z] + drops + mdrops, S, Z, timeout=200)
# Need to run our 'copy' of the averaging APP
num_array = []
for drop in mdrops:
buf = droputils.allDropContents(drop)
num_array.extend(pickle.loads(buf))
X.marray = num_array
average = X.averageArray()
# Load actual results
graph_result = droputils.allDropContents(Z)
graph_result = pickle.loads(graph_result)
self.assertEqual(graph_result, average)
# Must be called to unlink all shared memory
memory_manager.shutdown_all()
def test_helloworldapp(self):
h = HelloWorldApp('h', 'h')
b = FileDROP('c', 'c')
h.addOutput(b)
b.addProducer(h)
h.execute()
self.assertEqual(six.b(h.greeting), droputils.allDropContents(b))
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = data[-1:]
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(lastByte, b._lastByte)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write(b'abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'e')
a.write(b'fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'j')
a.write(b'k')
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, b'k')
self.assertEqual(b'ejk', droputils.allDropContents(d))
def getInputArrays(self):
ins = self.inputs
if len(ins) != 2:
raise Exception('Precisely two input required for %r' % self)
array = [np.fromstring(droputils.allDropContents(inp)) for inp in ins]
self.series = array
def getInputArrays(self):
ins = self.inputs
if len(ins) != 1:
raise Exception('Precisely one input required for %r' % self)
array = np.fromstring(droputils.allDropContents(ins[0]))
self.signal = np.frombuffer(array)
def _do_test(func, expected_out, *args, **kwargs):
# List with (drop, value) elements
arg_inputs = []
# dict with name: (drop, value) items
kwarg_inputs = {}
translate = lambda x: base64.b64encode(pickle.dumps(x))
i = 0
for arg in args:
si = 'uid_%d' % i
arg_inputs.append(InMemoryDROP(si, si, pydata=translate(arg)))
i += 1
for name, kwarg in kwargs.items():
si = 'uid_%d' % i
kwarg_inputs[name] = (si, InMemoryDROP(si, si, pydata=translate(kwarg)))
i += 1
a = InMemoryDROP('a', 'a', pydata=translate(1))
output = InMemoryDROP('o', 'o')
app = _PyFuncApp('f', 'f', func, func_arg_mapping={name: vals[0] for name, vals in kwarg_inputs.items()})
app.addInput(a)
app.addOutput(output)
for drop in arg_inputs + [x[1] for x in kwarg_inputs.values()]:
app.addInput(drop)
with droputils.DROPWaiterCtx(self, output):
a.setCompleted()
for i in arg_inputs + [x[1] for x in kwarg_inputs.values()]:
i.setCompleted()
self.assertEqual(expected_out, pickle.loads(droputils.allDropContents(output))) # @UndefinedVariable
def test_speadApp(self):
port = 1111
itemId = 0x2000
thread_pool = spead2.ThreadPool()
self._stream = spead2.send.UdpStream(
thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))
a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
b = InMemoryDROP('b', 'b')
a.addOutput(b)
size = 1024
threading.Thread(target=lambda: a.execute()).start()
time.sleep(1)
msg = os.urandom(size)
with DROPWaiterCtx(self, b, timeout=1):
ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
item = ig.add_item(itemId,
'main_data',
'a char array',
shape=(size, ),
format=[('c', 8)])
item.value = msg
self._stream.send_heap(ig.get_heap())
self._stream.send_heap(ig.get_end())
for drop in a, b:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(size, b.size)
self.assertEqual(msg, droputils.allDropContents(b))
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(six.b(msg), droputils.allDropContents(b))
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + b" ")
def assert_message_is_correct(message, command):
a = BashShellApp("a", "a", command=command)
b = FileDROP("b", "b")
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(message.encode("utf8"), droputils.allDropContents(b))
def _test_dynamic_write_withDropType(self, dropType):
"""
Test an AbstractDROP and a simple AppDROP (for checksum calculation)
without an expected drop size (for app compatibility and not
recommended in production)
"""
# NOTE: use_staging required for multiple writes to plasma drops
a = dropType("oid:A", "uid:A", expectedSize=-1, use_staging=True)
b = SumupContainerChecksum("oid:B", "uid:B")
c = InMemoryDROP("oid:C", "uid:C")
b.addInput(a)
b.addOutput(c)
test_crc = 0
with DROPWaiterCtx(self, c):
for _ in range(self._test_num_blocks):
a.write(self._test_block)
test_crc = crc32c(self._test_block, test_crc)
a.setCompleted()
# Read the checksum from c
cChecksum = int(droputils.allDropContents(c))
self.assertNotEqual(a.checksum, 0)
self.assertEqual(a.checksum, test_crc)
self.assertEqual(cChecksum, test_crc)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = io.BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
"""
a = FileDROP("a", "a")
b = DockerApp("b", "b", image="ubuntu:14.04", command="cp %i0 %o0")
c = FileDROP("c", "c")
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_genericScatter(self):
data_in = random.randint(0, 100, size=100)
b = InMemoryDROP("b", "b")
b.write(pickle.dumps(data_in))
s = GenericScatterApp("s", "s", num_of_copies=2)
s.addInput(b)
o1 = InMemoryDROP("o1", "o1")
o2 = InMemoryDROP("o2", "o2")
for x in o1, o2:
s.addOutput(x)
self._test_graph_runs((b, s, o1, o2), b, (o1, o2), timeout=4)
data1 = pickle.loads(droputils.allDropContents(o1))
data2 = pickle.loads(droputils.allDropContents(o2))
data_out = concatenate([data1, data2])
self.assertEqual(data_in.all(), data_out.all())
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = io.BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
def assertMsgIsCorrect(msg, command):
a = DockerApp("a", "a", image="ubuntu:14.04", command=command)
b = FileDROP("b", "b")
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(msg.encode("utf8"), droputils.allDropContents(b))
def test_ddGraph(self):
"""
Graph is using dd to read a file and write to another. This is mainly
to test that the separatorString parameter is working correctly.
"""
sessionId = "lalo"
ddGraph = "graphs/ddTest.graph"
with pkg_resources.resource_stream("test",
ddGraph) as f: # @UndefinedVariable
logger.debug(f"Loading graph: {f}")
graphSpec = json.load(f)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
bs = graphSpec[0]["applicationArgs"]["bs"]["value"]
count = graphSpec[0]["applicationArgs"]["count"]["value"]
self.dim.deploySession(sessionId)
a, c = [
self.dm._sessions[sessionId].drops[x]
for x in ("2022-02-11T08:05:47_-5_0", "2022-02-11T08:05:47_-3_0")
]
data = os.urandom(bs * count)
logger.debug(f"Length of data produced: {len(data)}")
with droputils.DROPWaiterCtx(self, c, 3):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
def assert_message_is_correct(message, command):
a = BashShellApp('a', 'a', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(six.b(message), droputils.allDropContents(b))
def test_helloworldapp(self):
h = HelloWorldApp("h", "h")
b = FileDROP("c", "c")
h.addOutput(b)
b.addProducer(h)
h.execute()
self.assertEqual(h.greeting.encode("utf8"),
droputils.allDropContents(b))
def assert_envvar_is_there(varname, value):
command = "echo -n $%s > %%o0" % (varname)
a = BashShellApp(app_uid, app_uid, dlg_session=session, command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(six.b(value), droputils.allDropContents(b))
def test_NullDROP(self):
"""
Check that the NullDROP is usable for testing
"""
a = NullDROP("A", "A", expectedSize=5)
a.write(b"1234")
a.write(b"5")
allContents = droputils.allDropContents(a)
self.assertFalse(allContents)
def _ngas_and_fs_io(self, command):
a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, c, 100):
a.setCompleted()
self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def _test_runGraphInTwoNMs(
self,
g1,
g2,
rels,
root_data,
leaf_data,
root_oids=("A",),
leaf_oid="C",
expected_failures=[],
sessionId=f"s{random.randint(0, 1000)}",
node_managers=None,
threads=0,
):
"""Utility to run a graph in two Node Managers"""
dm1, dm2 = node_managers or [self._start_dm(threads=threads) for _ in range(2)]
add_test_reprodata(g1)
add_test_reprodata(g2)
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(len(g1), len(dm1._sessions[sessionId].drops))
self.assertEqual(len(g2), len(dm2._sessions[sessionId].drops))
# Run! We wait until c is completed
drops = {}
drops.update(dm1._sessions[sessionId].drops)
drops.update(dm2._sessions[sessionId].drops)
leaf_drop = drops[leaf_oid]
with droputils.DROPWaiterCtx(self, leaf_drop, 2):
for oid in root_oids:
drop = drops[oid]
drop.write(root_data)
drop.setCompleted()
expected_successes = [
drops[oid] for oid in drops if oid not in expected_failures
]
expected_failures = [drops[oid] for oid in drops if oid in expected_failures]
for drop in expected_successes:
self.assertEqual(DROPStates.COMPLETED, drop.status)
for drop in expected_failures:
self.assertEqual(DROPStates.ERROR, drop.status)
leaf_drop_data = None
if leaf_drop not in expected_failures:
leaf_drop_data = droputils.allDropContents(leaf_drop)
if leaf_data is not None:
self.assertEqual(len(leaf_data), len(leaf_drop_data))
self.assertEqual(leaf_data, leaf_drop_data)
sleep(0.1) # just make sure all events have been processed.
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
return leaf_drop_data
def test_listappendthrashing(self, size=1000):
a = InMemoryDROP("a", "a")
b = ListAppendThrashingApp("b", "b", size=size)
self.assertEqual(b.size, size)
c = InMemoryDROP("c", "c")
b.addInput(a)
b.addOutput(c)
self._test_graph_runs((a, b, c), a, c, timeout=4)
data_out = pickle.loads(droputils.allDropContents(c))
self.assertEqual(b.marray, data_out)
def test_initialize(self):
yanda_kwargs = dict(self.kwargs)
yanda_kwargs["mode"] = "YANDA"
yanda_parset = ParameterSetDROP(oid="a", uid="a", **yanda_kwargs)
standard_parset = ParameterSetDROP(oid="b", uid="b", **self.kwargs)
yanda_output = "Cimager=2\nStringParam=param\nBoolparam=True"
standard_output = (
"mode=None\n" + yanda_output +
"\nconfig_data=\niid=-1\nrank=0\nconsumers=['drop-1', 'drop-2']")
yanda_parset.setCompleted()
standard_parset.setCompleted()
self.assertEqual(yanda_output,
allDropContents(yanda_parset).decode("utf-8"))
self.assertEqual(standard_output,
allDropContents(standard_parset).decode("utf-8"))
def test_no_write_to_file_drop(self):
"""Check that FileDrops can be *not* written"""
a = FileDROP("a", "a")
b = SleepAndCopyApp("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
b.addOutput(c)
with DROPWaiterCtx(self, c):
a.setCompleted()
self.assertEqual(droputils.allDropContents(c), b"")
def test_randomarrayapp(self):
i = NullDROP("i", "i")
c = RandomArrayApp("c", "c", keep_array=True)
o = InMemoryDROP("o", "o")
c.addInput(i)
c.addOutput(o)
self._test_graph_runs((i, c, o), i, o)
marray = c._getArray()
data = pickle.loads(droputils.allDropContents(o))
v = marray == data
self.assertEqual(v.all(), True)