def test_cleanupExpiredDrops(self):
with dlm.DataLifecycleManager(checkPeriod=0.5,
cleanupPeriod=2) as manager:
drop = FileDROP('oid:A',
'uid:A1',
expectedSize=1,
lifespan=1,
precious=False)
manager.addDrop(drop)
self._writeAndClose(drop)
# Wait 2 seconds, the DROP is still COMPLETED
time.sleep(0.5)
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertTrue(drop.exists())
# Wait 5 more second, now it should be expired but still there
time.sleep(1)
self.assertEqual(DROPStates.EXPIRED, drop.status)
self.assertTrue(drop.exists())
# Wait 2 more seconds, now it should have been deleted
time.sleep(1)
self.assertEqual(DROPStates.DELETED, drop.status)
self.assertFalse(drop.exists())
def test_helloworldapp(self):
h = HelloWorldApp('h', 'h')
b = FileDROP('c', 'c')
h.addOutput(b)
b.addProducer(h)
h.execute()
self.assertEqual(six.b(h.greeting), droputils.allDropContents(b))
def test_plasma(self):
in_file = '/tmp/test.ms'
out_file = '/tmp/copy.ms'
with tarfile.open('./data/test_ms.tar.gz', 'r') as ref:
ref.extractall('/tmp/')
a = FileDROP('a', 'a', filepath=in_file)
b = MSPlasmaWriter('b', 'b')
c = PlasmaDROP('c', 'c')
d = MSPlasmaReader('d', 'd')
e = FileDROP('e', 'e', filepath=out_file)
b.addInput(a)
b.addOutput(c)
d.addInput(c)
d.addOutput(e)
# Check the MS DATA content is the same as original
with droputils.DROPWaiterCtx(self, e, 5):
a.setCompleted()
self.compare_ms(in_file, out_file)
# check we can go from dataURL to plasma ID
client = plasma.connect("/tmp/plasma")
a = c.dataURL.split('//')[1].decode("hex")
client.get(plasma.ObjectID(a))
def test_helloworldapp(self):
h = HelloWorldApp("h", "h")
b = FileDROP("c", "c")
h.addOutput(b)
b.addProducer(h)
h.execute()
self.assertEqual(h.greeting.encode("utf8"),
droputils.allDropContents(b))
def test_filename_integration(self):
with tempfile.TemporaryDirectory() as tmp_dir:
os.environ["DLG_ROOT"] = tmp_dir
os.environ["DLG_FILE"] = "test_file"
test_drop = FileDROP(oid="a",
uid="a",
filepath="$DLG_FILE",
dirname="$DLG_ROOT")
test_drop.write(b"1234")
self.assertEqual(tmp_dir, test_drop.dirname)
self.assertEqual("test_file", test_drop.filepath)
def test_basic_run(self):
input_ms = os.environ.get('INPUT_MS', '/tmp/output/aa01.ms')
a = FileDROP('1', '1', filepath=input_ms)
b = CImagerDrop('2', '2')
c = FileDROP('3', '3', filepath='image_aa01')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, c, 10000):
a.setCompleted()
def assertFiles(delete_parent_directory, parentDirExists, tempDir=None):
tempDir = tempDir or tempfile.mkdtemp()
a = FileDROP('a', 'a', dirname=tempDir, delete_parent_directory=delete_parent_directory)
a.write(b' ')
a.setCompleted()
self.assertTrue(a.exists())
self.assertTrue(os.path.isdir(tempDir))
a.delete()
self.assertFalse(a.exists())
self.assertEqual(parentDirExists, os.path.isdir(tempDir))
if parentDirExists:
shutil.rmtree(tempDir)
def assertMsgIsCorrect(msg, command):
a = DockerApp("a", "a", image="ubuntu:14.04", command=command)
b = FileDROP("b", "b")
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(msg.encode("utf8"), droputils.allDropContents(b))
def assert_message_is_correct(message, command):
a = BashShellApp("a", "a", command=command)
b = FileDROP("b", "b")
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(message.encode("utf8"), droputils.allDropContents(b))
def assert_message_is_correct(message, command):
a = BashShellApp('a', 'a', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(six.b(message), droputils.allDropContents(b))
def test_basic_run(self):
signal = SignalGenerateAndAverageDrop('1', '1',
internal_port=internal_port,
stream_port=stream_port,
start_freq=int(os.environ.get('START_FREQ', 45991200)),
freq_step=int(os.environ.get('FREQ_STEP', 6400)),
use_gpus=int(os.environ.get('USE_GPUS', 0)),
num_freq_steps=int(os.environ.get('NUM_CHANNELS', 1)),
telescope_model_path='./conf/%s.tm' % tm,
sky_model_file_path="./conf/eor_model_list.csv",
num_time_steps=int(os.environ.get('NUM_TIME_STEPS', 1)))
sink = AveragerSinkDrop('2', '2',
stream_listen_port_start=stream_port,
use_adios2=int(os.environ.get('USE_ADIOS2', 0)),
baseline_exclusion_map_path='./conf/%s_baselines.csv' % tm,
node='127.0.0.1')
drop = InMemoryDROP('3', '3')
drop.addStreamingConsumer(sink)
signal.addOutput(drop)
ms = FileDROP('4', '4', filepath=output)
sink.addOutput(ms)
with droputils.DROPWaiterCtx(self, ms, 1000):
signal.async_execute()
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(six.b(msg), droputils.allDropContents(b))
def assert_envvar_is_there(varname, value):
command = "echo -n $%s > %%o0" % (varname)
a = BashShellApp(app_uid, app_uid, dlg_session=session, command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(six.b(value), droputils.allDropContents(b))
def test_dropCompleteTriggersReplication(self):
with dlm.DataLifecycleManager() as manager:
drop = FileDROP('oid:A', 'uid:A1', expectedSize=1)
manager.addDrop(drop)
self._writeAndClose(drop)
# The call to close() should have turned it into a SOLID object
# because the DLM replicated it
self.assertEqual(DROPPhases.SOLID, drop.phase)
self.assertEqual(2, len(manager.getDropUids(drop)))
# Try the same with a non-precious data object, it shouldn't be replicated
drop = FileDROP('oid:B', 'uid:B1', expectedSize=1, precious=False)
manager.addDrop(drop)
self._writeAndClose(drop)
self.assertEqual(DROPPhases.GAS, drop.phase)
self.assertEqual(1, len(manager.getDropUids(drop)))
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir("/tmp")
dirname = "/tmp/.hidden"
dirname2 = "/tmp/.hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP("a", "a", dirname=dirname)
b = FileDROP("b", "b", dirname=dirname)
c = FileDROP("c", "c", dirname=dirname2)
d = FileDROP("d", "d", dirname=dirname2)
cont1 = DirectoryContainer("e", "e", dirname=dirname)
cont2 = DirectoryContainer("f", "f", dirname=dirname2)
# Paths are absolutely reported
self.assertEqual(os.path.realpath("/tmp/.hidden"), os.path.realpath(cont1.path))
self.assertEqual(
os.path.realpath("/tmp/.hidden/inside"), os.path.realpath(cont2.path)
)
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP("g", "g"))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP("h", "h"))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP("i", "i"))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
"""
a = FileDROP("a", "a")
b = DockerApp("b", "b", image="ubuntu:14.04", command="cp %i0 %o0")
c = FileDROP("c", "c")
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def _ngas_and_fs_io(self, command):
a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, c, 100):
a.setCompleted()
self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def test_expiringNormalDrop(self):
with dlm.DataLifecycleManager(checkPeriod=0.5) as manager:
drop = FileDROP('oid:A', 'uid:A1', expectedSize=1, lifespan=0.5)
manager.addDrop(drop)
# Writing moves the DROP to COMPLETE
self._writeAndClose(drop)
# Wait now, the DROP should be moved by the DLM to EXPIRED
time.sleep(1)
self.assertEqual(DROPStates.EXPIRED, drop.status)
def _ngas_and_fs_io(self, command):
a = NgasDROP(
"HelloWorld.txt", "HelloWorld.txt"
) # not a filesystem-related DROP, we can reference its URL in the command-line
a.ngasSrv = "ngas.ddns.net"
b = DockerApp("b", "b", image="ubuntu:14.04", command=command)
c = FileDROP("c", "c")
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, c, 100):
a.setCompleted()
self.assertEqual(a.dataURL.encode("utf8"),
droputils.allDropContents(c))
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
+-------------|named-pipe|----------+ +-----------|named-pipe|-----------+
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)
def test_dropWroteFromOutside(self):
"""
A different scenario to those tested above, in which the data
represented by the DROP isn't actually written *through* the
DROP. Still, the DROP needs to be moved to COMPLETED once
the data is written, and reading from it should still yield a correct
result
"""
# Write, but not through the DROP
a = FileDROP("A", "A")
filename = a.path
msg = b"a message"
with open(filename, "wb") as f:
f.write(msg)
a.setCompleted()
# Read from the DROP
self.assertEqual(msg, droputils.allDropContents(a))
self.assertIsNotNone(a.checksum)
self.assertEqual(9, a.size)
# The drop now calculates the size thus we can't set it anymore
self.assertRaises(Exception, a.size, len(msg))
def test_no_write_to_file_drop(self):
"""Check that FileDrops can be *not* written"""
a = FileDROP("a", "a")
b = SleepAndCopyApp("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
b.addOutput(c)
with DROPWaiterCtx(self, c):
a.setCompleted()
self.assertEqual(droputils.allDropContents(c), b"")
def _test_working_dir(self, ensureUserAndSwitch):
# the sleep is required to make sure that the docker container exists long enough for
# DALiuGE to get the required information (2 lines of Python code after starting the container!)
a = DockerApp(
"a",
"a",
workingDir="/mydir",
image="ubuntu:14.04",
command="pwd > %o0 && sleep 0.05",
ensureUserAndSwitch=ensureUserAndSwitch,
)
b = FileDROP("b", "b")
a.addOutput(b)
with DROPWaiterCtx(self, b, 10):
a.execute()
self.assertEqual(b"/mydir", droputils.allDropContents(b).strip())
def test_single_pipe(self):
"""
A simple test where two bash apps are connected to each other in a
streaming fashion. The data flows through a pipe which is created by
the framework. The data drop in between acts only as a intermediator
to establish the underlying communication channel.
------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
| echo | | "/a/pipe" | | dc | | |
-----*------- -------------- ------*------ ----------
| |
+-------------|named-pipe|------------+
BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
and redirected to D.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp("a", "a", command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP("b", "b")
c = StreamingInputBashApp("c", "c", command="cat > %o0")
d = FileDROP("d", "d", filepath=output_fname)
a.addOutput(b)
c.addStreamingInput(b)
c.addOutput(d)
# Let's fire the app
with DROPWaiterCtx(self, d, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d):
self.assertEqual(
DROPStates.COMPLETED,
drop.status,
"Drop %r not COMPLETED: %d" % (drop, drop.status),
)
self.assertEqual(
[5, 4, 3, 2, 1], [int(x) for x in droputils.allDropContents(d).split(b"\n")]
)
# Clean up and go
os.remove(output_fname)
def test_lostDrop(self):
with dlm.DataLifecycleManager(checkPeriod=0.5) as manager:
drop = FileDROP('oid:A',
'uid:A1',
expectedSize=1,
lifespan=10,
precious=False)
manager.addDrop(drop)
self._writeAndClose(drop)
# "externally" remove the file, its contents now don't exist
os.unlink(drop.path)
# Let the DLM do its work
time.sleep(1)
# Check that the DROP is marked as LOST
self.assertEqual(DROPPhases.LOST, drop.phase)
def test_with_dynlib(self):
"""
We test the following graph:
A -----> B ----> C ---> D ---> E
| +--> F ---> G ---> H
+------------------> I ---> J
A and C are FileDrops; B is a DynlibApp; D, G and I are CRCApps;
F, E, H and J are InMemoryDrops.
The DynlibApp B copies A into C and F; therefore D, G and I should yield
the same results, meaning that E, H and J should have the same contents.
Similarly, A, C and F should have the same contents.
This graph was experiencing some problems in a MacOS machine. Hopefully
this test will shed some light on that issue and allow us to track it
down and fix it.
"""
# Build drops and wire them together
a, c = (FileDROP(x, x) for x in ('a', 'c'))
b = DynlibApp('b', 'b', lib=test_dynlib._libpath)
d, g, i = (CRCApp(x, x) for x in ('d', 'g', 'i'))
f, e, h, j = (InMemoryDROP(x, x) for x in ('f', 'e', 'h', 'j'))
for data, app in (a, b), (c, d), (f, g), (a, i):
app.addInput(data)
for app, data in (b, c), (b, f), (d, e), (g, h), (i, j):
app.addOutput(data)
# The crc32 is the same used by the CRCApp, see the imports
data = os.urandom(1024)
crc = six.b(str(crc32(data)))
# Execute the graph and check results
with droputils.DROPWaiterCtx(self, (e, h, j), 5):
a.write(data)
a.setCompleted()
# Data and CRCs are the expected ones
for what, who in (data, (a, c, f)), (crc, (e, h, j)):
for drop in who:
self.assertEqual(what, droputils.allDropContents(drop))
def test_DROPFile(self):
"""
This test exercises the DROPFile mechanism to read the data represented by
a given DROP. The DROPFile class will decide whether the data should be read
directly or through the DROP
"""
drop = FileDROP("a", "a", expectedSize=5)
drop.write(b"abcde")
with DROPFile(drop) as f:
self.assertEqual(b"abcde", f.read())
self.assertTrue(drop.isBeingRead())
self.assertIsNotNone(f._io)
self.assertFalse(drop.isBeingRead())
def test_echo(self):
a = FileDROP('a', 'a')
b = BashShellApp('b', 'b', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_parallelHelloWorld(self):
m0 = InMemoryDROP("m0", "m0")
s = GenericScatterApp("s", "s", num_of_copies=4)
greets = ["World", "Solar system", "Galaxy", "Universe"]
m0.write(pickle.dumps(greets))
s.addInput(m0)
m = []
h = []
f = []
for i in range(1, len(greets) + 1, 1):
m.append(InMemoryDROP("m%d" % i, "m%d" % i))
h.append(HelloWorldApp("h%d" % i, "h%d" % i))
f.append(FileDROP("f%d" % i, "f%d" % i))
s.addOutput(m[-1])
h[-1].addInput(m[-1])
h[-1].addOutput(f[-1])
ad = [m0, s]
ad.extend(m)
ad.extend(h)
ad.extend(f)
self._test_graph_runs(ad, m0, f)
for i in range(len(f)):
self.assertEqual(("Hello %s" % greets[i]).encode("utf8"),
droputils.allDropContents(f[i]))
