def test_variflow(self):
p = admit.Project(self.outputDir)
p.setlogginglevel(50)
p.addtask(admit.File_AT(alias='root', file='root', touch=True))
p.addtask(admit.Flow11_AT(alias='flow1'), ['root'])
p.addtask(admit.Flow1N_AT(alias='flow1N', n=2, touch=True, exist=True),
['root'])
p.addtask(admit.Flow11_AT(alias='sub1Nf'), ['flow1N'])
p.addtask(admit.Flow11_AT(alias='sub1Nv'), [('flow1N', 1)])
p.addtask(admit.FlowMN_AT(alias='flowMN', n=2, touch=True, exist=True),
['flow1', 'root', ('flow1N', 1)])
p.addtask(admit.Flow11_AT(alias='subMNf'), ['flowMN'])
p.addtask(admit.FlowN1_AT(alias='subMNv', touch=True), [('flowMN', 0),
('flowMN', 2)])
p.addtask(admit.FlowN1_AT(alias='flowN1', touch=True),
['flow1N', ('flowMN', 1)])
p.addtask(admit.Flow11_AT(alias='subN1'), ['flowN1'])
p.show()
for n in [2, 3, 1, 2]:
if (p.fm.find(lambda at: at._alias == 'flow1N')):
p['flow1N'].setkey('n', n)
if (p.fm.find(lambda at: at._alias == 'flowMN')):
p['flowMN'].setkey('n', n + 1)
p.run()
p.show()
self.assertLessEqual(len(p), 20, "Incorrect task count")
def test_flow2(self):
# Construct a flow: File_AT -> Flow11_AT -> Flow1N_AT
# add first task
task1 = admit.File_AT(touch=True)
tid1 = self.fm.add(task1)
bdp = admit.File_BDP()
task1.addoutput(bdp)
# add second task
task2 = admit.Flow11_AT()
task2.setkey("file", "Flow11.dat")
tid2 = self.fm.add(task2, [(tid1, 0)])
# add third task
task3 = admit.Flow1N_AT()
task3.setkey("file", "Flow1N.dat")
tid3 = self.fm.add(task3, [(tid2, 0)])
task4 = admit.FlowMN_AT()
# test inFlow()
found = self.fm.inFlow(task1)
self.assertTrue(found) # should be true
# test downstream of tid2 (including tid2)
dstream = self.fm.downstream(tid2)
self.assertEquals(dstream, set([tid2, tid3]))
# test stale()
for ds in dstream:
isStale = self.fm._tasks[ds].isstale() # check before mark
if isStale:
self.fm._tasks[ds].markUpToDate()
isStale = self.fm._tasks[ds].isstale() # check after mark
self.assertFalse(isStale) # should not be stale
self.fm.stale(tid2) # call stale()
# after calling stale()
for ds in dstream:
isStale = self.fm._tasks[ds].isstale()
self.assertTrue(
isStale) # all ATs in downstream should be stale now
# clone from tid2 (Flow11_AT) in the flow
cloned = self.fm.clone(tid2)
if (self.verbose):
self.fm.show()
def test_FM_getsetdelitem(self):
""" test FM __getitem__, __setitem__, __delitem__ """
# test add(), connectInputs(), verify(), and show()
#__getitem__(), __delitem__(), __setitem__()
fm = admit.Flow()
# add first task
task1 = admit.File_AT(touch=True)
tid1 = fm.add(task1)
bdp = admit.File_BDP()
task1.addoutput(bdp)
# add another task
task2 = admit.Flow11_AT()
task2.setkey("file", "Flow11.dat")
tid2 = fm.add(task2, [(tid1, 0)])
# test connectInputs
fm.connectInputs()
# check the number of tasks
self.assertEqual(len(fm), 2)
# first task (File_AT)
# fm[tid1] # call __getitem__
self.assertIsInstance(fm[tid1], admit.File_AT)
# second task (Flow11_AT)
# fm[tid2] # call __getitem__
self.assertIsInstance(fm[tid2], admit.Flow11_AT)
# test __setitem__
newtask = admit.FlowMN_AT()
newtask.setkey("file", "FlowMN.txt")
fm[tid2] = newtask # call __setitem__
# check to see if task2 got overwritten
self.assertIsInstance(fm[tid2], admit.FlowMN_AT)
# now restore Flow11_AT
fm[tid2] = task2 # call __setitem__
self.assertIsInstance(fm[tid2], admit.Flow11_AT)
# test __delitem__ (delete Flow11)
at = fm[tid2]
del fm[tid2] # call __delitem__
self.assertEqual(len(fm), 1)
# Add task back.
fm[tid2] = at # call __setitem__
# test verify()
self.assertTrue(fm.verify())
def test_FM_find_replace(self):
""" test FM find() and replace() """
fm = admit.Flow()
# add first task File_AT
task = admit.File_AT(touch=True)
tid1 = fm.add(task)
# add another task Flow11_AT
task = admit.Flow11_AT()
tid2 = fm.add(task, [(tid1, 0)])
task.setkey("file", "Flow11.dat")
# now try to find the tasks
tasks = fm.find(lambda at: at.id() < 100)
# check class types of the tasks
self.assertIsInstance(tasks[0], admit.File_AT)
self.assertIsInstance(tasks[1], admit.Flow11_AT)
# test replace()
task = admit.FlowMN_AT()
fm.replace(tid2, task)
# to find new tasks
tasks = fm.find(lambda at: at.id() < 100)
# check class types of the tasks
self.assertIsInstance(tasks[0], admit.File_AT)
self.assertIsInstance(tasks[1], admit.FlowMN_AT)
# check the number of tasks
self.assertEqual(len(fm), 2)
# Test __contains__()
self.assertFalse(100 in fm)
self.assertTrue(tid1 in fm)
# test __iter__
# we should have two tasks
counter = 0
for t in fm:
counter += 1
self.assertEqual(counter, 2)
def test_task2(self):
# add first task
task1 = admit.File_AT(touch=True)
tid1 = self.fm.add(task1)
bdp = admit.File_BDP()
task1.addoutput(bdp)
# add another task
task2 = admit.Flow11_AT()
task2.setkey("file", "Flow11.dat")
tid2 = self.fm.add(task2, [(tid1, 0)])
# test connectInputs
self.fm.connectInputs()
num = len(self.fm) # should be 2
self.assertTrue(num == 2)
if (self.verbose):
print "The number of tasks in flow:", num
print "The task IDs:", tid1, tid2
# first task (File_AT)
t1 = self.fm[tid1] # call __getitem__
type = isinstance(t1, admit.File_AT)
if (self.verbose):
print "After call __getitem__", t1.id()
self.assertTrue(type)
# second task (Flow11_AT)
t2 = self.fm[tid2] # call __getitem__
type = isinstance(t2, admit.Flow11_AT)
if (self.verbose):
print "After call __getitem__", t2.id()
self.assertTrue(type)
# test __setitem__
newtask = admit.FlowMN_AT()
newtask.setkey("file", "FlowMN.txt")
self.fm[tid2] = newtask # call __setitem__
if (self.verbose):
self.fm.show()
# check to see if task2 got overwritten
t2 = self.fm[tid2]
type = isinstance(t2, admit.FlowMN_AT)
self.assertTrue(type)
# now restore Flow11_AT
self.fm[tid2] = task2 # call __setitem__
if (self.verbose):
self.fm.show()
# check it again
t2 = self.fm[tid2]
type = isinstance(t2, admit.Flow11_AT)
self.assertTrue(type)
# test __delitem__ (delete Flow11)
at = self.fm[tid2]
del self.fm[tid2] # call __delitem__
num = len(self.fm) # should be 1 now
self.assertTrue(num == 1)
# Add task back.
self.fm[tid2] = at # call __setitem__
# test verify()
check = self.fm.verify()
self.assertTrue(check)
def test_task1(self):
# add first task
task = admit.File_AT(touch=True)
tid1 = self.fm.add(task)
if (self.verbose):
print "FM unit test task1 ID:", tid1
# add another task
task = admit.Flow11_AT()
task.setkey("file", "Flow11.dat")
tid2 = self.fm.add(task, [(tid1, 0)])
if (self.verbose):
print "FM unit test task2 ID:", tid2
# now try to find the tasks
tasks = self.fm.find(lambda at: at.id() < 100)
if (self.verbose):
print "Found tasks:", tasks
# check class type of the first task (File_AT)
type = isinstance(tasks[0], admit.File_AT)
if (self.verbose):
print "AT id:", tasks[0].id()
self.assertTrue(type)
# check class type of second task (Flow11_AT)
type = isinstance(tasks[1], admit.Flow11_AT)
if (self.verbose):
print "AT id:", tasks[1].id()
self.assertTrue(type)
# test replace()
task = admit.FlowMN_AT()
self.fm.replace(tid2, task)
# to find new tasks
tasks = self.fm.find(lambda at: at.id() < 100)
if (self.verbose):
print "Found tasks after replace Flow11 with FlowMN:", tasks
# first task (File_AT)
type = isinstance(tasks[0], admit.File_AT)
if (self.verbose):
print "After replace AT id:", tasks[0].id()
self.assertTrue(type)
# second task (FlowMN_AT)
type = isinstance(tasks[1], admit.FlowMN_AT)
if (self.verbose):
print "After replace AT id:", tasks[1].id()
self.assertTrue(type)
# the number of tasks
num = len(self.fm)
if (self.verbose):
print "The number of tasks:", num
self.assertEqual(2, num)
# Test __contains__()
contains = 100 in self.fm # should be false
self.assertFalse(contains)
contains = tid1 in self.fm # should be true
self.assertTrue(contains)
# test __iter__
# we should have two tasks
counter = 0
for t in self.fm:
counter += 1
if (self.verbose):
print "Task id:", t
self.assertEqual(counter, 2)