def test_to_first_split(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
outputs = []
frequencies = make_groups_of_frequencies(FREQUENCY_GROUPS, 5)
frequencies = frequencies[0]
for group in frequencies:
casa_py_drop = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='casa_py.sh /dfms_root/%i0 /dfms_root/%o0 {0} {1}'.format(group[0], group[1]), user='******')
result = FileDROP(self.get_oid('file'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_to_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_to_s3.sh /dfms_root/%i0 %oDataURL0', user='******')
s3_drop_out = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='{0}_{1}/key123'.format(group[0], group[1]), profile_name='aws-profile')
casa_py_drop.addInput(measurement_set)
casa_py_drop.addOutput(result)
copy_to_s3.addInput(result)
copy_to_s3.addOutput(s3_drop_out)
outputs.append(s3_drop_out)
barrier_drop = BarrierAppDROP(self.get_oid('barrier'), uuid.uuid4())
barrier_drop.addInput(measurement_set)
for output in outputs:
barrier_drop.addInput(output)
with droputils.DROPWaiterCtx(self, barrier_drop, 50000):
s3_drop.setCompleted()
def test_start_graph(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
with droputils.DROPWaiterCtx(self, measurement_set, 50000):
s3_drop.setCompleted()
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir('/tmp')
dirname = ".hidden"
dirname2 = ".hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP('a', 'a', dirname=dirname)
b = FileDROP('b', 'b', dirname=dirname)
c = FileDROP('c', 'c', dirname=dirname2)
d = FileDROP('d', 'd', dirname=dirname2)
cont1 = DirectoryContainer('e', 'e', dirname=dirname)
cont2 = DirectoryContainer('f', 'f', dirname=dirname2)
# Paths are absolutely reported
self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path))
self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path))
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g'))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h'))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i'))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir("/tmp")
dirname = ".hidden"
dirname2 = ".hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP("a", "a", dirname=dirname)
b = FileDROP("b", "b", dirname=dirname)
c = FileDROP("c", "c", dirname=dirname2)
d = FileDROP("d", "d", dirname=dirname2)
cont1 = DirectoryContainer("e", "e", dirname=dirname)
cont2 = DirectoryContainer("f", "f", dirname=dirname2)
# Paths are absolutely reported
self.assertEquals(os.path.realpath("/tmp/.hidden"), os.path.realpath(cont1.path))
self.assertEquals(os.path.realpath("/tmp/.hidden/inside"), os.path.realpath(cont2.path))
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP("g", "g"))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP("h", "h"))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP("i", "i"))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
field='deepfield',
mode='frequency',
restfreq='1420.405752MHz',
nchan=-1,
start='',
width='',
interpolation='nearest',
gain=0.1,
imsize=[256],
cell=['1.0arcsec'],
phasecenter='10h01m53.9,+02d24m52s',
weighting='natural',
casapy_path=CASAPY)
image_out = DirectoryContainer(uuid.uuid1(),
uuid.uuid1(),
dirname=CUBE_OUT + CUBE_NAME,
check_exists=False)
cl.addOutput(image_out)
flux.addInput(image_out)
flux.addOutput(flux_out)
for v in VIS:
vis_in = DirectoryContainer(uuid.uuid1(),
uuid.uuid1(),
dirname=v[0])
split_out = DirectoryContainer(uuid.uuid1(),
uuid.uuid1(),
dirname=v[1],
check_exists=False)
vis_in_a.append(vis_in)
def test_expireAfterUse(self):
"""
Simple test for the expireAfterUse flag. Two DROPs are created with
different values, and after they are used we check whether their data
is still there or not
"""
with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager:
a = DirectoryContainer('a', 'a', precious=False, expireAfterUse=True, dirname=tempfile.mkdtemp())
b_dirname = tempfile.mkdtemp()
b = DirectoryContainer('b', 'b', precious=False, expireAfterUse=False, dirname=b_dirname)
c = BarrierAppDROP('c', 'c')
d = BarrierAppDROP('d', 'd')
a.addConsumer(c)
a.addConsumer(d)
b.addConsumer(c)
b.addConsumer(d)
manager.addDrop(a)
manager.addDrop(b)
manager.addDrop(b)
manager.addDrop(c)
# Make sure all consumers are done
with DROPWaiterCtx(self, [c,d], 1):
a.setCompleted()
b.setCompleted()
# Both directories should be there, but after cleanup A's shouldn't
# be there anymore
self.assertTrue(a.exists())
self.assertTrue(b.exists())
time.sleep(2)
self.assertFalse(a.exists())
self.assertTrue(b.exists())
b.delete()
def test_expireAfterUse(self):
"""
Simple test for the expireAfterUse flag. Two DROPs are created with
different values, and after they are used we check whether their data
is still there or not
"""
with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager:
a = DirectoryContainer('a', 'a', precious=False, expireAfterUse=True, dirname=tempfile.mkdtemp())
b_dirname = tempfile.mkdtemp()
b = DirectoryContainer('b', 'b', precious=False, expireAfterUse=False, dirname=b_dirname)
c = BarrierAppDROP('c', 'c')
d = BarrierAppDROP('d', 'd')
a.addConsumer(c)
a.addConsumer(d)
b.addConsumer(c)
b.addConsumer(d)
manager.addDrop(a)
manager.addDrop(b)
manager.addDrop(b)
manager.addDrop(c)
# Make sure all consumers are done
with DROPWaiterCtx(self, [c,d], 1):
a.setCompleted()
b.setCompleted()
# Both directories should be there, but after cleanup A's shouldn't
# be there anymore
self.assertTrue(a.exists())
self.assertTrue(b.exists())
time.sleep(2)
self.assertFalse(a.exists())
self.assertTrue(b.exists())
b.delete()