def dataProvider_testShowAllRoutes(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pShowAllRoutes1 = Proc()
pShowAllRoutes2 = Proc()
pShowAllRoutes3 = Proc()
pShowAllRoutes4 = Proc()
pShowAllRoutes5 = Proc()
pShowAllRoutes6 = Proc()
pShowAllRoutes7 = Proc()
pShowAllRoutes8 = Proc()
pShowAllRoutes9 = Proc()
pShowAllRoutes10 = Proc()
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes6.depends = pShowAllRoutes4, pShowAllRoutes5
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes4.depends = pShowAllRoutes2
pShowAllRoutes2.depends = pShowAllRoutes1
pShowAllRoutes10.depends = pShowAllRoutes1
aAggr = Aggr(pShowAllRoutes7, pShowAllRoutes8, pShowAllRoutes9)
aAggr.starts = [aAggr.pShowAllRoutes7]
aAggr.pShowAllRoutes8.depends = aAggr.pShowAllRoutes7
aAggr.pShowAllRoutes9.depends = aAggr.pShowAllRoutes7
aAggr.depends2 = [pShowAllRoutes3, pShowAllRoutes6]
yield [pShowAllRoutes1, pShowAllRoutes5], [
'DEBUG', '* pShowAllRoutes1 -> pShowAllRoutes10',
'pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes3 -> [aAggr]',
'* pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes4 -> pShowAllRoutes6 -> [aAggr]',
'* pShowAllRoutes5 -> pShowAllRoutes6 -> [aAggr]'
]
def dataProvider_testFlowchart(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pFlowchart1 = Proc()
pFlowchart2 = Proc()
pFlowchart3 = Proc()
pFlowchart4 = Proc()
pFlowchart5 = Proc()
pFlowchart6 = Proc()
pFlowchart7 = Proc()
pFlowchart8 = Proc()
pFlowchart9 = Proc()
pFlowchart10 = Proc()
pFlowchart3.depends = pFlowchart2
pFlowchart6.depends = pFlowchart4, pFlowchart5
pFlowchart3.depends = pFlowchart2
pFlowchart4.depends = pFlowchart2
pFlowchart2.depends = pFlowchart1
pFlowchart10.depends = pFlowchart1
aAggr = Aggr(pFlowchart7, pFlowchart8, pFlowchart9)
aAggr.starts = [aAggr.pFlowchart7]
aAggr.pFlowchart8.depends = aAggr.pFlowchart7
aAggr.pFlowchart9.depends = aAggr.pFlowchart7
aAggr.depends2 = [pFlowchart3, pFlowchart6]
dotfile = path.join(self.testdir, 'test.dot')
fcfile = path.join(self.testdir, 'test.svg')
yield [pFlowchart1, pFlowchart5], fcfile, dotfile, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
yield [pFlowchart1, pFlowchart5], fcfile, None, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
def testSetGetAttr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
pGetAttr4 = Proc()
pGetAttr5 = Proc()
pGetAttr1.args.params = Box(inopts=Box())
pGetAttr2.args.params = Box(inopts=Box())
pGetAttr3.args.params = Box(inopts=Box())
pGetAttr4.args.params = Box(inopts=Box())
pGetAttr5.args.params = Box(inopts=Box())
aggr = Aggr(pGetAttr1,
pGetAttr2,
pGetAttr3,
pGetAttr4,
pGetAttr5,
depends=False)
# nothing delegated yet
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# but when starts and ends changed
aggr.starts = [aggr.pGetAttr1, aggr.pGetAttr2]
aggr.ends = [aggr.pGetAttr5]
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# delegate a short attribute
aggr.delegate('forks', [aggr.pGetAttr2, aggr.pGetAttr3])
aggr.forks = 10
# only 2, 3 changed
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 1, 1])
# change the specific procs
aggr['pGetAttr2', 'pGetAttr4'].forks = 5
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 5, 1])
# fix an attribute
aggr.pGetAttr3.runner = 'dry'
aggr.runner = 'sge'
self.assertListEqual(
[p.config['runner'] for p in aggr._procs.values()],
['sge', 'sge', 'dry', 'sge', 'sge'])
# set input
self.assertRaises(AggrAttributeError, setattr, aggr, 'input', 1)
aggr.input = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[1, 2, '', '', ''])
# reverse it
aggr.input = ['', '']
aggr.input2 = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', '', ''])
aggr[3].input = 'i3'
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', 'i3', ''])
# not suppose to do this
# aggr.input2[2] = ['a', 'b']
#self.assertListEqual([p.config['input'] for p in aggr._procs.values()], [[1,2], [1,2], ['a', 'b'], 'i3', ''])
# similar for depends
#self.assertRaises(AggrAttributeError, setattr, aggr, 'depends', 1)
aggr.depends = [pGetAttr1, pGetAttr2]
self.assertListEqual([p.depends for p in aggr._procs.values()],
[[pGetAttr1], [pGetAttr2], [], [], []])
# reverse it
aggr.depends = [[], []]
aggr.depends2 = [pGetAttr1, pGetAttr2]
self.assertListEqual(
[p.depends for p in aggr._procs.values()],
[[pGetAttr1, pGetAttr2], [pGetAttr1, pGetAttr2], [], [], []])
# set attributes of certain processes
aggr[0].args.params = Box(inopts=Box(a=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(a=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[:2].args.params = Box(inopts=Box(b=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[1, 2].args.params = Box(inopts=Box(c=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
# using proc ids
aggr['pGetAttr1'].args.params = Box(inopts=Box(d=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(d=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1', 'pGetAttr3'].args.params = Box(inopts=Box(e=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1, pGetAttr3'].args.params = Box(inopts=Box(f=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['starts'].args.params = Box(inopts=Box(g=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['ends'].args.params = Box(inopts=Box(h=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.delegate('args.params.inopts', 'pGetAttr3')
aggr.args['params'].inopts = Box(n=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(n=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params['inopts'] = Box(m=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr[0].args.params.inopts = Box(z=1)
# remeber at line 453, inopts refers to the same Box() object, so both pGetAttr1 and 3 will change
self.assertListEqual(
[p.args for p in aggr._procs.values()],
[
Box(params=Box(inopts=Box(z=1))),
Box(params=Box(inopts=Box(z=1))), # <-- not g
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params.inopts.cnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(),
Box(h=1),
])
# something undelegated
aggr.tag = 'abc'
self.assertListEqual([p.tag for p in aggr._procs.values()],
['abc'] * 5)
del aggr._delegates['args.params.inopts']
aggr.delegate('args', 'pGetAttr4')
aggr.args.params.inopts.rnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(rnames=True),
Box(h=1),
])
def dataProvider_testRun(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pRun0 = Proc()
pRun1 = pRun0.copy()
pRun2 = Proc()
pRun3 = Proc()
pRun4 = Proc()
pRun5 = Proc()
pRun6 = Proc()
pRun7 = Proc()
pRun8 = Proc()
pRun9 = Proc()
pRun10 = Proc()
pRun3.depends = pRun2
pRun6.depends = pRun4, pRun5
pRun3.depends = pRun2
pRun4.depends = pRun2
pRun2.depends = pRun1
pRun10.depends = pRun1
aAggr = Aggr(pRun7, pRun8, pRun9)
aAggr.starts = [aAggr.pRun7]
aAggr.pRun8.depends = aAggr.pRun7
aAggr.pRun9.depends = aAggr.pRun7
aAggr.depends2 = [pRun3, pRun6]
yield [pRun1, pRun5], 'profile', 'sge', [
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun1 (pRun0): No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun2: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun3: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun4: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun5: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun6: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun10: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun7.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun8.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun9.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
]
# pRun21 -> pRun22
# pRun23 -/
# \-> pRun24
pRun21 = Proc()
pRun23 = Proc()
pRun22 = Proc()
pRun22.depends = pRun21, pRun23
yield [pRun21], 'profile', 'local', [
"pRun22 won't run as path can't be reached: pRun22 <- pRun23"
]