def dataProvider_testDepends(self):
pDepends1 = Proc()
pDepends2 = Proc()
pDepends3 = Proc()
pDepends4 = Proc()
pDepends5 = Proc()
aggr = Aggr(pDepends1, pDepends2, pDepends3)
aggr.starts = 'pDepends1, pDepends2'
aggr.depends = [aggr.pDepends2, aggr.pDepends3]
yield aggr, [[aggr.pDepends2], [aggr.pDepends3]]
aggr1 = Aggr(pDepends1, pDepends2, pDepends3, pDepends4, pDepends5)
aggr1.starts = 'pDepends1'
aggr1[:1].depends = [['pDepends3', 'pDepends5']]
yield aggr1, [[aggr1.pDepends3, aggr1.pDepends5]]
def dataProvider_testCopy(self):
pCopy1 = Proc()
pCopy2 = Proc()
pCopy3 = Proc()
pCopy4 = Proc()
aggr = Aggr(pCopy1, pCopy2, pCopy3)
aggr.depends = [pCopy4]
yield aggr, 'newtag', True, 'newid'
yield aggr, None, True, None
yield aggr, None, False, None
yield aggr, aggr.pCopy1.tag, False, None, False, False, AggrCopyError, 'Cannot copy process with same id and tag: \'pCopy1.%s\'' % aggr.pCopy1.tag
aggr1 = Aggr(pCopy1, pCopy2, pCopy3, depends=False)
aggr1.starts = [aggr1.pCopy1, aggr1.pCopy2]
aggr1.pCopy3.depends = aggr1.starts
aggr1.module('qc', lambda a: True, lambda a: False)
yield aggr1, None, True, None, True, True
def testSetGetAttr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
pGetAttr4 = Proc()
pGetAttr5 = Proc()
pGetAttr1.args.params = Box(inopts=Box())
pGetAttr2.args.params = Box(inopts=Box())
pGetAttr3.args.params = Box(inopts=Box())
pGetAttr4.args.params = Box(inopts=Box())
pGetAttr5.args.params = Box(inopts=Box())
aggr = Aggr(pGetAttr1,
pGetAttr2,
pGetAttr3,
pGetAttr4,
pGetAttr5,
depends=False)
# nothing delegated yet
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# but when starts and ends changed
aggr.starts = [aggr.pGetAttr1, aggr.pGetAttr2]
aggr.ends = [aggr.pGetAttr5]
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# delegate a short attribute
aggr.delegate('forks', [aggr.pGetAttr2, aggr.pGetAttr3])
aggr.forks = 10
# only 2, 3 changed
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 1, 1])
# change the specific procs
aggr['pGetAttr2', 'pGetAttr4'].forks = 5
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 5, 1])
# fix an attribute
aggr.pGetAttr3.runner = 'dry'
aggr.runner = 'sge'
self.assertListEqual(
[p.config['runner'] for p in aggr._procs.values()],
['sge', 'sge', 'dry', 'sge', 'sge'])
# set input
self.assertRaises(AggrAttributeError, setattr, aggr, 'input', 1)
aggr.input = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[1, 2, '', '', ''])
# reverse it
aggr.input = ['', '']
aggr.input2 = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', '', ''])
aggr[3].input = 'i3'
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', 'i3', ''])
# not suppose to do this
# aggr.input2[2] = ['a', 'b']
#self.assertListEqual([p.config['input'] for p in aggr._procs.values()], [[1,2], [1,2], ['a', 'b'], 'i3', ''])
# similar for depends
#self.assertRaises(AggrAttributeError, setattr, aggr, 'depends', 1)
aggr.depends = [pGetAttr1, pGetAttr2]
self.assertListEqual([p.depends for p in aggr._procs.values()],
[[pGetAttr1], [pGetAttr2], [], [], []])
# reverse it
aggr.depends = [[], []]
aggr.depends2 = [pGetAttr1, pGetAttr2]
self.assertListEqual(
[p.depends for p in aggr._procs.values()],
[[pGetAttr1, pGetAttr2], [pGetAttr1, pGetAttr2], [], [], []])
# set attributes of certain processes
aggr[0].args.params = Box(inopts=Box(a=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(a=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[:2].args.params = Box(inopts=Box(b=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[1, 2].args.params = Box(inopts=Box(c=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
# using proc ids
aggr['pGetAttr1'].args.params = Box(inopts=Box(d=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(d=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1', 'pGetAttr3'].args.params = Box(inopts=Box(e=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1, pGetAttr3'].args.params = Box(inopts=Box(f=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['starts'].args.params = Box(inopts=Box(g=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['ends'].args.params = Box(inopts=Box(h=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.delegate('args.params.inopts', 'pGetAttr3')
aggr.args['params'].inopts = Box(n=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(n=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params['inopts'] = Box(m=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr[0].args.params.inopts = Box(z=1)
# remeber at line 453, inopts refers to the same Box() object, so both pGetAttr1 and 3 will change
self.assertListEqual(
[p.args for p in aggr._procs.values()],
[
Box(params=Box(inopts=Box(z=1))),
Box(params=Box(inopts=Box(z=1))), # <-- not g
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params.inopts.cnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(),
Box(h=1),
])
# something undelegated
aggr.tag = 'abc'
self.assertListEqual([p.tag for p in aggr._procs.values()],
['abc'] * 5)
del aggr._delegates['args.params.inopts']
aggr.delegate('args', 'pGetAttr4')
aggr.args.params.inopts.rnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(rnames=True),
Box(h=1),
])