def test_cleanupExpiredDrops(self): with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager: drop = FileDROP('oid:A', 'uid:A1', expectedSize=1, lifespan=1, precious=False) manager.addDrop(drop) self._writeAndClose(drop) # Wait 2 seconds, the DROP is still COMPLETED time.sleep(0.5) self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertTrue(drop.exists()) # Wait 5 more second, now it should be expired but still there time.sleep(1) self.assertEqual(DROPStates.EXPIRED, drop.status) self.assertTrue(drop.exists()) # Wait 2 more seconds, now it should have been deleted time.sleep(1) self.assertEqual(DROPStates.DELETED, drop.status) self.assertFalse(drop.exists())
def test_helloworldapp(self): h = HelloWorldApp('h', 'h') b = FileDROP('c', 'c') h.addOutput(b) b.addProducer(h) h.execute() self.assertEqual(six.b(h.greeting), droputils.allDropContents(b))
def test_plasma(self): in_file = '/tmp/test.ms' out_file = '/tmp/copy.ms' with tarfile.open('./data/test_ms.tar.gz', 'r') as ref: ref.extractall('/tmp/') a = FileDROP('a', 'a', filepath=in_file) b = MSPlasmaWriter('b', 'b') c = PlasmaDROP('c', 'c') d = MSPlasmaReader('d', 'd') e = FileDROP('e', 'e', filepath=out_file) b.addInput(a) b.addOutput(c) d.addInput(c) d.addOutput(e) # Check the MS DATA content is the same as original with droputils.DROPWaiterCtx(self, e, 5): a.setCompleted() self.compare_ms(in_file, out_file) # check we can go from dataURL to plasma ID client = plasma.connect("/tmp/plasma") a = c.dataURL.split('//')[1].decode("hex") client.get(plasma.ObjectID(a))
def test_helloworldapp(self): h = HelloWorldApp("h", "h") b = FileDROP("c", "c") h.addOutput(b) b.addProducer(h) h.execute() self.assertEqual(h.greeting.encode("utf8"), droputils.allDropContents(b))
def test_filename_integration(self): with tempfile.TemporaryDirectory() as tmp_dir: os.environ["DLG_ROOT"] = tmp_dir os.environ["DLG_FILE"] = "test_file" test_drop = FileDROP(oid="a", uid="a", filepath="$DLG_FILE", dirname="$DLG_ROOT") test_drop.write(b"1234") self.assertEqual(tmp_dir, test_drop.dirname) self.assertEqual("test_file", test_drop.filepath)
def test_basic_run(self): input_ms = os.environ.get('INPUT_MS', '/tmp/output/aa01.ms') a = FileDROP('1', '1', filepath=input_ms) b = CImagerDrop('2', '2') c = FileDROP('3', '3', filepath='image_aa01') b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, c, 10000): a.setCompleted()
def assertFiles(delete_parent_directory, parentDirExists, tempDir=None): tempDir = tempDir or tempfile.mkdtemp() a = FileDROP('a', 'a', dirname=tempDir, delete_parent_directory=delete_parent_directory) a.write(b' ') a.setCompleted() self.assertTrue(a.exists()) self.assertTrue(os.path.isdir(tempDir)) a.delete() self.assertFalse(a.exists()) self.assertEqual(parentDirExists, os.path.isdir(tempDir)) if parentDirExists: shutil.rmtree(tempDir)
def assertMsgIsCorrect(msg, command): a = DockerApp("a", "a", image="ubuntu:14.04", command=command) b = FileDROP("b", "b") a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.execute() self.assertEqual(msg.encode("utf8"), droputils.allDropContents(b))
def assert_message_is_correct(message, command): a = BashShellApp("a", "a", command=command) b = FileDROP("b", "b") a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.async_execute() self.assertEqual(message.encode("utf8"), droputils.allDropContents(b))
def assert_message_is_correct(message, command): a = BashShellApp('a', 'a', command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.async_execute() self.assertEqual(six.b(message), droputils.allDropContents(b))
def test_basic_run(self): signal = SignalGenerateAndAverageDrop('1', '1', internal_port=internal_port, stream_port=stream_port, start_freq=int(os.environ.get('START_FREQ', 45991200)), freq_step=int(os.environ.get('FREQ_STEP', 6400)), use_gpus=int(os.environ.get('USE_GPUS', 0)), num_freq_steps=int(os.environ.get('NUM_CHANNELS', 1)), telescope_model_path='./conf/%s.tm' % tm, sky_model_file_path="./conf/eor_model_list.csv", num_time_steps=int(os.environ.get('NUM_TIME_STEPS', 1))) sink = AveragerSinkDrop('2', '2', stream_listen_port_start=stream_port, use_adios2=int(os.environ.get('USE_ADIOS2', 0)), baseline_exclusion_map_path='./conf/%s_baselines.csv' % tm, node='127.0.0.1') drop = InMemoryDROP('3', '3') drop.addStreamingConsumer(sink) signal.addOutput(drop) ms = FileDROP('4', '4', filepath=output) sink.addOutput(ms) with droputils.DROPWaiterCtx(self, ms, 1000): signal.async_execute()
def assertMsgIsCorrect(msg, command): a = DockerApp('a', 'a', image='ubuntu:14.04', command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.execute() self.assertEqual(six.b(msg), droputils.allDropContents(b))
def assert_envvar_is_there(varname, value): command = "echo -n $%s > %%o0" % (varname) a = BashShellApp(app_uid, app_uid, dlg_session=session, command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.async_execute() self.assertEqual(six.b(value), droputils.allDropContents(b))
def test_dropCompleteTriggersReplication(self): with dlm.DataLifecycleManager() as manager: drop = FileDROP('oid:A', 'uid:A1', expectedSize=1) manager.addDrop(drop) self._writeAndClose(drop) # The call to close() should have turned it into a SOLID object # because the DLM replicated it self.assertEqual(DROPPhases.SOLID, drop.phase) self.assertEqual(2, len(manager.getDropUids(drop))) # Try the same with a non-precious data object, it shouldn't be replicated drop = FileDROP('oid:B', 'uid:B1', expectedSize=1, precious=False) manager.addDrop(drop) self._writeAndClose(drop) self.assertEqual(DROPPhases.GAS, drop.phase) self.assertEqual(1, len(manager.getDropUids(drop)))
def test_directoryContainer(self): """ A small, simple test for the DirectoryContainer DROP that checks it allows only valid children to be added """ # Prepare our playground cwd = os.getcwd() os.chdir("/tmp") dirname = "/tmp/.hidden" dirname2 = "/tmp/.hidden/inside" if not os.path.exists(dirname2): os.makedirs(dirname2) # DROPs involved a = FileDROP("a", "a", dirname=dirname) b = FileDROP("b", "b", dirname=dirname) c = FileDROP("c", "c", dirname=dirname2) d = FileDROP("d", "d", dirname=dirname2) cont1 = DirectoryContainer("e", "e", dirname=dirname) cont2 = DirectoryContainer("f", "f", dirname=dirname2) # Paths are absolutely reported self.assertEqual(os.path.realpath("/tmp/.hidden"), os.path.realpath(cont1.path)) self.assertEqual( os.path.realpath("/tmp/.hidden/inside"), os.path.realpath(cont2.path) ) # Certain children-to-be are rejected self.assertRaises(TypeError, cont1.addChild, NullDROP("g", "g")) self.assertRaises(TypeError, cont1.addChild, InMemoryDROP("h", "h")) self.assertRaises(TypeError, cont1.addChild, ContainerDROP("i", "i")) self.assertRaises(Exception, cont1.addChild, c) self.assertRaises(Exception, cont1.addChild, d) self.assertRaises(Exception, cont2.addChild, a) self.assertRaises(Exception, cont2.addChild, b) # These children are correct cont1.addChild(a) cont1.addChild(b) cont2.addChild(c) cont2.addChild(d) # Revert to previous state shutil.rmtree(dirname, True) os.chdir(cwd)
def test_clientServer(self): """ A client-server duo. The server outputs the data it receives to its output DROP, which in turn is the data held in its input DROP. The graph looks like this: A --|--> B(client) --|--> D |--> C(server) --| C is a server application which B connects to. Therefore C must be started before B, so B knows C's IP address and connects successfully. Although the real writing is done by C, B in this example is also treated as a publisher of D. This way D waits for both applications to finish before proceeding. """ a = FileDROP('a', 'a') b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000') c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0') d = FileDROP('d', 'd') b.addInput(a) b.addOutput(d) c.addInput(a) c.addOutput(d) # Let 'b' handle its interest in c b.handleInterest(c) data = os.urandom(10) with DROPWaiterCtx(self, d, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(d))
def test_simpleCopy(self): """ Simple test for a dockerized application. It copies the contents of one file into another via the command-line cp utility. It then checks that the contents of the target DROP are correct, and that the target file is actually owned by our process. """ a = FileDROP("a", "a") b = DockerApp("b", "b", image="ubuntu:14.04", command="cp %i0 %o0") c = FileDROP("c", "c") b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = os.urandom(10) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEqual(uid, os.stat(c.path).st_uid)
def _ngas_and_fs_io(self, command): a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line b = DockerApp('b', 'b', image="ubuntu:14.04", command=command) c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, c, 100): a.setCompleted() self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def test_expiringNormalDrop(self): with dlm.DataLifecycleManager(checkPeriod=0.5) as manager: drop = FileDROP('oid:A', 'uid:A1', expectedSize=1, lifespan=0.5) manager.addDrop(drop) # Writing moves the DROP to COMPLETE self._writeAndClose(drop) # Wait now, the DROP should be moved by the DLM to EXPIRED time.sleep(1) self.assertEqual(DROPStates.EXPIRED, drop.status)
def _ngas_and_fs_io(self, command): a = NgasDROP( "HelloWorld.txt", "HelloWorld.txt" ) # not a filesystem-related DROP, we can reference its URL in the command-line a.ngasSrv = "ngas.ddns.net" b = DockerApp("b", "b", image="ubuntu:14.04", command=command) c = FileDROP("c", "c") b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, c, 100): a.setCompleted() self.assertEqual(a.dataURL.encode("utf8"), droputils.allDropContents(c))
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | +-------------|named-pipe|----------+ +-----------|named-pipe|-----------+ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)
def test_dropWroteFromOutside(self): """ A different scenario to those tested above, in which the data represented by the DROP isn't actually written *through* the DROP. Still, the DROP needs to be moved to COMPLETED once the data is written, and reading from it should still yield a correct result """ # Write, but not through the DROP a = FileDROP("A", "A") filename = a.path msg = b"a message" with open(filename, "wb") as f: f.write(msg) a.setCompleted() # Read from the DROP self.assertEqual(msg, droputils.allDropContents(a)) self.assertIsNotNone(a.checksum) self.assertEqual(9, a.size) # The drop now calculates the size thus we can't set it anymore self.assertRaises(Exception, a.size, len(msg))
def test_no_write_to_file_drop(self): """Check that FileDrops can be *not* written""" a = FileDROP("a", "a") b = SleepAndCopyApp("b", "b") c = InMemoryDROP("c", "c") a.addConsumer(b) b.addOutput(c) with DROPWaiterCtx(self, c): a.setCompleted() self.assertEqual(droputils.allDropContents(c), b"")
def _test_working_dir(self, ensureUserAndSwitch): # the sleep is required to make sure that the docker container exists long enough for # DALiuGE to get the required information (2 lines of Python code after starting the container!) a = DockerApp( "a", "a", workingDir="/mydir", image="ubuntu:14.04", command="pwd > %o0 && sleep 0.05", ensureUserAndSwitch=ensureUserAndSwitch, ) b = FileDROP("b", "b") a.addOutput(b) with DROPWaiterCtx(self, b, 10): a.execute() self.assertEqual(b"/mydir", droputils.allDropContents(b).strip())
def test_single_pipe(self): """ A simple test where two bash apps are connected to each other in a streaming fashion. The data flows through a pipe which is created by the framework. The data drop in between acts only as a intermediator to establish the underlying communication channel. ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D | | echo | | "/a/pipe" | | dc | | | -----*------- -------------- ------*------ ---------- | | +-------------|named-pipe|------------+ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat and redirected to D. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp("a", "a", command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP("b", "b") c = StreamingInputBashApp("c", "c", command="cat > %o0") d = FileDROP("d", "d", filepath=output_fname) a.addOutput(b) c.addStreamingInput(b) c.addOutput(d) # Let's fire the app with DROPWaiterCtx(self, d, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d): self.assertEqual( DROPStates.COMPLETED, drop.status, "Drop %r not COMPLETED: %d" % (drop, drop.status), ) self.assertEqual( [5, 4, 3, 2, 1], [int(x) for x in droputils.allDropContents(d).split(b"\n")] ) # Clean up and go os.remove(output_fname)
def test_lostDrop(self): with dlm.DataLifecycleManager(checkPeriod=0.5) as manager: drop = FileDROP('oid:A', 'uid:A1', expectedSize=1, lifespan=10, precious=False) manager.addDrop(drop) self._writeAndClose(drop) # "externally" remove the file, its contents now don't exist os.unlink(drop.path) # Let the DLM do its work time.sleep(1) # Check that the DROP is marked as LOST self.assertEqual(DROPPhases.LOST, drop.phase)
def test_with_dynlib(self): """ We test the following graph: A -----> B ----> C ---> D ---> E | +--> F ---> G ---> H +------------------> I ---> J A and C are FileDrops; B is a DynlibApp; D, G and I are CRCApps; F, E, H and J are InMemoryDrops. The DynlibApp B copies A into C and F; therefore D, G and I should yield the same results, meaning that E, H and J should have the same contents. Similarly, A, C and F should have the same contents. This graph was experiencing some problems in a MacOS machine. Hopefully this test will shed some light on that issue and allow us to track it down and fix it. """ # Build drops and wire them together a, c = (FileDROP(x, x) for x in ('a', 'c')) b = DynlibApp('b', 'b', lib=test_dynlib._libpath) d, g, i = (CRCApp(x, x) for x in ('d', 'g', 'i')) f, e, h, j = (InMemoryDROP(x, x) for x in ('f', 'e', 'h', 'j')) for data, app in (a, b), (c, d), (f, g), (a, i): app.addInput(data) for app, data in (b, c), (b, f), (d, e), (g, h), (i, j): app.addOutput(data) # The crc32 is the same used by the CRCApp, see the imports data = os.urandom(1024) crc = six.b(str(crc32(data))) # Execute the graph and check results with droputils.DROPWaiterCtx(self, (e, h, j), 5): a.write(data) a.setCompleted() # Data and CRCs are the expected ones for what, who in (data, (a, c, f)), (crc, (e, h, j)): for drop in who: self.assertEqual(what, droputils.allDropContents(drop))
def test_DROPFile(self): """ This test exercises the DROPFile mechanism to read the data represented by a given DROP. The DROPFile class will decide whether the data should be read directly or through the DROP """ drop = FileDROP("a", "a", expectedSize=5) drop.write(b"abcde") with DROPFile(drop) as f: self.assertEqual(b"abcde", f.read()) self.assertTrue(drop.isBeingRead()) self.assertIsNotNone(f._io) self.assertFalse(drop.isBeingRead())
def test_echo(self): a = FileDROP('a', 'a') b = BashShellApp('b', 'b', command='cp %i0 %o0') c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = os.urandom(10) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEqual(uid, os.stat(c.path).st_uid)
def test_parallelHelloWorld(self): m0 = InMemoryDROP("m0", "m0") s = GenericScatterApp("s", "s", num_of_copies=4) greets = ["World", "Solar system", "Galaxy", "Universe"] m0.write(pickle.dumps(greets)) s.addInput(m0) m = [] h = [] f = [] for i in range(1, len(greets) + 1, 1): m.append(InMemoryDROP("m%d" % i, "m%d" % i)) h.append(HelloWorldApp("h%d" % i, "h%d" % i)) f.append(FileDROP("f%d" % i, "f%d" % i)) s.addOutput(m[-1]) h[-1].addInput(m[-1]) h[-1].addOutput(f[-1]) ad = [m0, s] ad.extend(m) ad.extend(h) ad.extend(f) self._test_graph_runs(ad, m0, f) for i in range(len(f)): self.assertEqual(("Hello %s" % greets[i]).encode("utf8"), droputils.allDropContents(f[i]))