def test_expireAfterUse(self):
"""
Simple test for the expireAfterUse flag. Two DROPs are created with
different values, and after they are used we check whether their data
is still there or not
"""
with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager:
a = DirectoryContainer('a', 'a', precious=False, expireAfterUse=True, dirname=tempfile.mkdtemp())
b_dirname = tempfile.mkdtemp()
b = DirectoryContainer('b', 'b', precious=False, expireAfterUse=False, dirname=b_dirname)
c = BarrierAppDROP('c', 'c')
d = BarrierAppDROP('d', 'd')
a.addConsumer(c)
a.addConsumer(d)
b.addConsumer(c)
b.addConsumer(d)
manager.addDrop(a)
manager.addDrop(b)
manager.addDrop(b)
manager.addDrop(c)
# Make sure all consumers are done
with DROPWaiterCtx(self, [c,d], 1):
a.setCompleted()
b.setCompleted()
# Both directories should be there, but after cleanup A's shouldn't
# be there anymore
self.assertTrue(a.exists())
self.assertTrue(b.exists())
time.sleep(2)
self.assertFalse(a.exists())
self.assertTrue(b.exists())
b.delete()
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def test_to_first_split(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
outputs = []
frequencies = make_groups_of_frequencies(FREQUENCY_GROUPS, 5)
frequencies = frequencies[0]
for group in frequencies:
casa_py_drop = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='casa_py.sh /dfms_root/%i0 /dfms_root/%o0 {0} {1}'.format(group[0], group[1]), user='******')
result = FileDROP(self.get_oid('file'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_to_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_to_s3.sh /dfms_root/%i0 %oDataURL0', user='******')
s3_drop_out = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='{0}_{1}/key123'.format(group[0], group[1]), profile_name='aws-profile')
casa_py_drop.addInput(measurement_set)
casa_py_drop.addOutput(result)
copy_to_s3.addInput(result)
copy_to_s3.addOutput(s3_drop_out)
outputs.append(s3_drop_out)
barrier_drop = BarrierAppDROP(self.get_oid('barrier'), uuid.uuid4())
barrier_drop.addInput(measurement_set)
for output in outputs:
barrier_drop.addInput(output)
with droputils.DROPWaiterCtx(self, barrier_drop, 50000):
s3_drop.setCompleted()
