def dataProvider_testShowAllRoutes(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pShowAllRoutes1 = Proc()
pShowAllRoutes2 = Proc()
pShowAllRoutes3 = Proc()
pShowAllRoutes4 = Proc()
pShowAllRoutes5 = Proc()
pShowAllRoutes6 = Proc()
pShowAllRoutes7 = Proc()
pShowAllRoutes8 = Proc()
pShowAllRoutes9 = Proc()
pShowAllRoutes10 = Proc()
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes6.depends = pShowAllRoutes4, pShowAllRoutes5
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes4.depends = pShowAllRoutes2
pShowAllRoutes2.depends = pShowAllRoutes1
pShowAllRoutes10.depends = pShowAllRoutes1
aAggr = Aggr(pShowAllRoutes7, pShowAllRoutes8, pShowAllRoutes9)
aAggr.starts = [aAggr.pShowAllRoutes7]
aAggr.pShowAllRoutes8.depends = aAggr.pShowAllRoutes7
aAggr.pShowAllRoutes9.depends = aAggr.pShowAllRoutes7
aAggr.depends2 = [pShowAllRoutes3, pShowAllRoutes6]
yield [pShowAllRoutes1, pShowAllRoutes5], [
'DEBUG', '* pShowAllRoutes1 -> pShowAllRoutes10',
'pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes3 -> [aAggr]',
'* pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes4 -> pShowAllRoutes6 -> [aAggr]',
'* pShowAllRoutes5 -> pShowAllRoutes6 -> [aAggr]'
]
def dataProvider_testSetAttr(self):
pSetAttr = Proc()
pSetAttrDepends = Proc()
pSetAttrAggr = Proc()
aSetAttr = Aggr(pSetAttrAggr)
aSetAttr.ends = [aSetAttr.pSetAttrAggr]
yield pSetAttr, '__nosuchattr__', None, None, ProcAttributeError, 'Cannot set attribute for process'
yield pSetAttr, 'profile', 'sge', 'local', None, None, ['WARNING', 'Attribute "profile" is deprecated']
yield pSetAttr, 'envs', {'a': 1}
yield pSetAttr, 'depends', pSetAttr, None, ProcAttributeError, 'Process depends on itself'
yield pSetAttr, 'depends', 1, None, ProcAttributeError, "Process dependents should be 'Proc/Aggr', not: 'int'"
#5
yield pSetAttr, 'depends', pSetAttrDepends, [pSetAttrDepends]
yield pSetAttr, 'depends', aSetAttr, [aSetAttr.pSetAttrAggr]
yield pSetAttr, 'depends', (aSetAttr, pSetAttrDepends), [aSetAttr.pSetAttrAggr, pSetAttrDepends]
yield pSetAttr, 'script', 'file:' + path.abspath(__file__)
yield pSetAttr, 'script', 'file:' + path.relpath(__file__, __folder__), 'file:' + path.abspath(__file__)
#10
yield pSetAttr, 'args', {'a':1}, Box({'a':1})
yield pSetAttr, 'envs', {'a':1}, Box({'a':1})
yield pSetAttr, 'input', 'inkey1:var, inkey2:file'
yield pSetAttr, 'input', [('inval1', 'inval2')], {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
yield pSetAttr, 'input', ['inkey3:var', 'inkey4:file'], {'inkey1:var, inkey2:file': ['inkey3:var', 'inkey4:file']}
yield pSetAttr, 'input', {'inkey1:var': 'inval1', 'inkey2:file': 'inval2'}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
def dataProvider_testDelEnd(self):
p1 = Proc()
p2 = Proc()
p3 = Proc()
aggr = Aggr(p1, p2, p3, depends=False)
aggr.ends = 'p1, p2'
yield aggr, [aggr.p1], [aggr.p2]
yield aggr, ['p1, p2'], []
def dataProvider_testDepends2(self):
pDepends21 = Proc()
pDepends22 = Proc()
pDepends23 = Proc()
pDepends24 = Proc()
pDepends25 = Proc()
aggr = Aggr(pDepends21, pDepends22, pDepends23)
aggr.starts = [pDepends21, pDepends22]
yield aggr, [pDepends24, pDepends25]
def dataProvider_testSetattr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
aggr1 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr2 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
yield aggr1, 'forks', 10, [10] * 3
aggr2.delegate('forks', 'pGetAttr2')
yield aggr2, 'forks', 10, [1, 10, 1]
def dataProvider_testFlowchart(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pFlowchart1 = Proc()
pFlowchart2 = Proc()
pFlowchart3 = Proc()
pFlowchart4 = Proc()
pFlowchart5 = Proc()
pFlowchart6 = Proc()
pFlowchart7 = Proc()
pFlowchart8 = Proc()
pFlowchart9 = Proc()
pFlowchart10 = Proc()
pFlowchart3.depends = pFlowchart2
pFlowchart6.depends = pFlowchart4, pFlowchart5
pFlowchart3.depends = pFlowchart2
pFlowchart4.depends = pFlowchart2
pFlowchart2.depends = pFlowchart1
pFlowchart10.depends = pFlowchart1
aAggr = Aggr(pFlowchart7, pFlowchart8, pFlowchart9)
aAggr.starts = [aAggr.pFlowchart7]
aAggr.pFlowchart8.depends = aAggr.pFlowchart7
aAggr.pFlowchart9.depends = aAggr.pFlowchart7
aAggr.depends2 = [pFlowchart3, pFlowchart6]
dotfile = path.join(self.testdir, 'test.dot')
fcfile = path.join(self.testdir, 'test.svg')
yield [pFlowchart1, pFlowchart5], fcfile, dotfile, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
yield [pFlowchart1, pFlowchart5], fcfile, None, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
def dataProvider_testDelegate(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
aggr1 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr2 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr3 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr4 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr5 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
yield aggr1, 'aggrs.a', 'pGetAttr1', {'aggrs.a': [aggr1.pGetAttr1]}
yield aggr2, 'aggrs.b, aggrs.c', 'pGetAttr1,pGetAttr2', {'aggrs.b': [aggr2.pGetAttr1, aggr2.pGetAttr2], 'aggrs.c': [aggr2.pGetAttr1, aggr2.pGetAttr2]}
yield aggr3, 'aggrs.b, aggrs.c', 'starts', {'aggrs.b': ['starts'], 'aggrs.c': ['starts']}
yield aggr4, 'aggrs.b, aggrs.c', 'ends', {'aggrs.b': ['ends'], 'aggrs.c': ['ends']}
yield aggr5, 'aggrs.a', aggr5.pGetAttr1, {'aggrs.a': [aggr5.pGetAttr1]}
def dataProvider_testInit(self):
pInita = Proc()
pInitb = Proc()
pInitc = Proc()
aggr = Aggr(pInita, pInitb, pInitc)
yield aggr, [pInita, pInitb, pInitc], None, True
yield aggr, [pInita, pInitb, pInitc], 'args', False
def testInit(self, args, kwargs, exception=None, msg=None):
self.maxDiff = None
if exception:
self.assertRaisesRegex(exception, msg, Aggr, *args, **kwargs)
else:
aggr = Aggr(*args, **kwargs)
self.assertEqual(aggr.id,
'aggr' if 'id' not in kwargs else kwargs['id'])
for i, p in enumerate(aggr._procs.values()):
self.assertEqual(
p.tag, kwargs['tag'] if 'tag' in kwargs else utils.uid(
args[i].tag + '@' + aggr.id, 4))
self.assertEqual(p.aggr, aggr.id)
self.assertIs(aggr._procs[p.id], p)
if ('depends' not in kwargs
or kwargs['depends']) and len(args) > 0:
self.assertListEqual(aggr.starts,
[list(aggr._procs.values())[0]])
self.assertListEqual(aggr.ends,
[list(aggr._procs.values())[-1]])
for i, proc in enumerate(aggr._procs.values()):
if i == 0: continue
self.assertIs(proc.depends[0],
list(aggr._procs.values())[i - 1])
else:
self.assertListEqual(aggr.starts, [])
self.assertListEqual(aggr.ends, [])
for i, proc in enumerate(aggr._procs.values()):
if i == 0: continue
self.assertListEqual(proc.depends, [])
def dataProvider_testGetattr(self):
pGetattra = Proc()
pGetattrb = Proc()
pGetattrc = Proc()
aggr = Aggr(pGetattra, pGetattrb, pGetattrc)
proxy = _Proxy(aggr)
yield proxy, ''
yield proxy, 'args'
def dataProvider_testAddEnd(self):
p1 = Proc()
p2 = Proc()
p3 = Proc()
aggr = Aggr(p1, p2, p3, depends=False)
yield aggr, [aggr.p1], [aggr.p1]
yield aggr, [aggr.p2], [aggr.p1, aggr.p2]
yield aggr, ['p1', 'p2'], [aggr.p1, aggr.p2]
yield aggr, ['p1, p2'], [aggr.p1, aggr.p2]
def dataProvider_testAddProc(self): pAddProc1 = Proc() pAddProc2 = Proc() pAddProc3 = Proc() pAddProc4 = Proc(tag = 'new') pAddProc5 = Proc(tag = 'new') pAddProc6 = Proc(tag = 'new') aggr = Aggr(pAddProc1, pAddProc2, pAddProc3) yield aggr, pAddProc4, 'starts', None, True, utils.uid(pAddProc4.tag + '@' + aggr.id, 4) yield aggr, pAddProc5, 'ends', None, True, utils.uid(pAddProc5.tag + '@' + aggr.id, 4) yield aggr, pAddProc6, None, None, False, utils.uid(pAddProc6.tag + '@' + aggr.id, 4)
def dataProvider_testOnOff(self): aggr = Aggr() def on(a): a.id = 'aggr' def off(a): a.id = None def ons(a): return a.id == 'aggr' and a._modules['qc']['status'] == 'on' def offs(a): return a.id == None and a._modules['qc']['status'] == 'off' yield aggr, 'qc', on, off, ons, offs
def dataProvider_testAny2Procs(self):
pAny2Procs1 = Proc()
pAny2Procs2 = Proc()
pAny2Procs3 = Proc()
pAny2Procs4 = Proc()
pAny2Procs51 = Proc(tag='51', id='pAny2Procs5')
pAny2Procs52 = Proc(tag='52', id='pAny2Procs5')
pAny2Procs6 = Proc()
pAny2Procs7 = Proc()
aAggr = Aggr(pAny2Procs6, pAny2Procs7)
aAggr.starts = [aAggr.pAny2Procs6, aAggr.pAny2Procs7]
yield [pAny2Procs1], [pAny2Procs1]
yield ['abc'], [], PyPPLProcFindError, 'Failed to find process'
yield [aAggr], [aAggr.pAny2Procs6, aAggr.pAny2Procs7]
yield ['pAny2Procs5'], [pAny2Procs51, pAny2Procs52]
yield ['pAny2Procs5.51'], [pAny2Procs51]
yield ['pAny2Procs1.notag'], [pAny2Procs1]
yield ['pAny2Procs5', aAggr, [pAny2Procs2, 'pAny2Procs1.notag']], [
pAny2Procs51, pAny2Procs52, pAny2Procs1, pAny2Procs2,
aAggr.pAny2Procs6, aAggr.pAny2Procs7
]
def dataProvider_testGetitem(self):
pSelecta = Proc()
pSelectb = Proc()
pSelectc = Proc()
aggr = Aggr(pSelecta, pSelectb, pSelectc)
yield aggr, 0, Proc
yield aggr, 'pSelecta', Proc
yield aggr, pSelecta, Proc
yield aggr, slice(0, 2), _Proxy
yield aggr, (1, 2), _Proxy
yield aggr, 'starts', _Proxy
yield aggr, 'ends', _Proxy
yield aggr, 'pSelecta, pSelectc', _Proxy
yield aggr, {}, None, AggrKeyError
def dataProvider_testGetattr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
aggr = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
# yield aggr, 'starts', list
# yield aggr, 'ends', list
# yield aggr, '_procs', dict
yield aggr, 'pGetAttr1', Proc
yield aggr, 'pGetAttr2', Proc
yield aggr, 'pGetAttr3', Proc
yield aggr, 'a', _Proxy
yield aggr, 'b', _Proxy
yield aggr, 'aggrs', _Proxy
def dataProvider_testCopy(self):
pCopy1 = Proc()
pCopy2 = Proc()
pCopy3 = Proc()
pCopy4 = Proc()
aggr = Aggr(pCopy1, pCopy2, pCopy3)
aggr.depends = [pCopy4]
yield aggr, 'newtag', True, 'newid'
yield aggr, None, True, None
yield aggr, None, False, None
yield aggr, aggr.pCopy1.tag, False, None, False, False, AggrCopyError, 'Cannot copy process with same id and tag: \'pCopy1.%s\'' % aggr.pCopy1.tag
aggr1 = Aggr(pCopy1, pCopy2, pCopy3, depends=False)
aggr1.starts = [aggr1.pCopy1, aggr1.pCopy2]
aggr1.pCopy3.depends = aggr1.starts
aggr1.module('qc', lambda a: True, lambda a: False)
yield aggr1, None, True, None, True, True
def dataProvider_testDepends(self):
pDepends1 = Proc()
pDepends2 = Proc()
pDepends3 = Proc()
pDepends4 = Proc()
pDepends5 = Proc()
aggr = Aggr(pDepends1, pDepends2, pDepends3)
aggr.starts = 'pDepends1, pDepends2'
aggr.depends = [aggr.pDepends2, aggr.pDepends3]
yield aggr, [[aggr.pDepends2], [aggr.pDepends3]]
aggr1 = Aggr(pDepends1, pDepends2, pDepends3, pDepends4, pDepends5)
aggr1.starts = 'pDepends1'
aggr1[:1].depends = [['pDepends3', 'pDepends5']]
yield aggr1, [[aggr1.pDepends3, aggr1.pDepends5]]
def testIssue31(self):
p = Proc()
#p.runner = 'local'
a = Aggr(p)
a.runner = 'sge'
a.p._readConfig(None, None)
self.assertEqual(a.p.runner, 'sge')
a2 = Aggr(p.copy(id='p2'))
a2.p2.runner = 'local'
a2.runner = 'sge' # make sure it's not overwriting
a2.p2._readConfig(None, None)
self.assertEqual(a2.p2.runner, 'local')
def dataProvider_testSelect(self):
pSelecta = Proc()
pSelectb = Proc()
pSelectc = Proc()
aggr = Aggr(pSelecta, pSelectb, pSelectc)
yield aggr, 0, False, aggr.pSelecta
yield aggr, (0, ), False, [aggr.pSelecta]
yield aggr, 0, True, [aggr.pSelecta]
yield aggr, 1, False, aggr.pSelectb
yield aggr, 'pSelectb', False, aggr.pSelectb
yield aggr, 'pSelectb', True, [aggr.pSelectb]
yield aggr, 'pSelectb,', False, [aggr.pSelectb]
yield aggr, aggr.pSelectb, False, aggr.pSelectb
yield aggr, aggr.pSelectb, True, [aggr.pSelectb]
yield aggr, slice(0, 2), True, [aggr.pSelecta, aggr.pSelectb]
yield aggr, (1, 2), True, [aggr.pSelectb, aggr.pSelectc]
yield aggr, 'starts', True, [aggr.pSelecta]
yield aggr, 'ends', True, [aggr.pSelectc]
yield aggr, 'pSelecta, pSelectc', True, [aggr.pSelecta, aggr.pSelectc]
yield aggr, {}, True, None
|
pBedGetfasta \
pMotifScan[!]
pTFs[*] -- pSimRead /
|
pTFList[*] /
@input:
- TF list file, one per line
- gene list file, one per line
- TF (1st col) list file with motif ids (2nd col). Default: params.tflist.value
"""
aTfbsTfP = Aggr(
pSort.copy(newid = 'pTFs'),
pPromoters,
pSort.copy(newid = 'pTFList'),
pSimRead,
pBedGetfasta,
pMotifScan,
depends = False
)
# defaults
aTfbsTfP.pPromoters.runner = 'local'
delegate
aTfbsTfP.delegate('args.up' , 'pPromoters')
aTfbsTfP.delegate('args.down' , 'pPromoters')
aTfbsTfP.delegate('args.genome' , 'pPromoters')
aTfbsTfP.delegate('args.ref' , 'pBedGetfasta')
aTfbsTfP.delegate('args.pval' , 'pMotifScan')
aTfbsTfP.delegate('args.tfmotifs', 'pMotifScan')
# depends
aTfbsTfP.starts = aTfbsTfP.pTFs, aTfbsTfP.pPromoters, aTfbsTfP.pTFList
def testModule(self):
pModule1 = Proc()
pModule2 = Proc()
pModule3 = Proc()
pModule4 = Proc()
pModule5 = Proc()
pModule6 = Proc()
aggr = Aggr(pModule1,
pModule2,
pModule3,
pModule4,
pModule5,
pModule6,
depends=False)
aggr.module('m1',
starts='pModule1, pModule2',
ends='pModule4',
depends={
'pModule3,': ['starts'],
'pModule4': 'pModule3'
},
ends_shared={'pModule4': 'm2'},
depends_shared={'pModule4': 'm2'})
aggr.module('m2',
starts='pModule3',
ends='pModule4, pModule6',
depends={'pModule4': 'pModule3'},
ends_shared={'pModule4': 'm1'},
depends_shared={'pModule4': 'm1'},
starts_shared={'pModule3': 'm1'})
self.assertEqual(aggr.starts, [])
self.assertEqual(aggr.ends, [])
self.assertEqual(aggr.pModule3.depends, [])
self.assertEqual(aggr.pModule4.depends, [])
aggr.on('m1')
self.assertEqual(aggr.starts, [aggr.pModule1, aggr.pModule2])
self.assertEqual(aggr.ends, [aggr.pModule4])
self.assertEqual(aggr.pModule3.depends, [aggr.pModule1, aggr.pModule2])
self.assertEqual(aggr.pModule4.depends, [aggr.pModule3])
aggr.off('m1')
self.assertEqual(aggr.starts, [])
self.assertEqual(aggr.ends, [])
self.assertEqual(aggr.pModule3.depends, [])
self.assertEqual(aggr.pModule4.depends, [])
aggr.on('m2')
self.assertEqual(aggr.starts, [aggr.pModule3])
self.assertEqual(aggr.ends, [aggr.pModule4, aggr.pModule6])
self.assertEqual(aggr.pModule4.depends, [aggr.pModule3])
aggr.off('m2')
self.assertEqual(aggr.starts, [])
self.assertEqual(aggr.ends, [])
self.assertEqual(aggr.pModule3.depends, [])
aggr.on('m1, m2')
aggr.off('m2')
self.assertEqual(aggr.starts,
[aggr.pModule1, aggr.pModule2, aggr.pModule3])
self.assertEqual(aggr.ends, [aggr.pModule4])
self.assertEqual(aggr.pModule3.depends, [aggr.pModule1, aggr.pModule2])
self.assertEqual(aggr.pModule4.depends, [aggr.pModule3])
def dataProvider_testRun(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pRun0 = Proc()
pRun1 = pRun0.copy()
pRun2 = Proc()
pRun3 = Proc()
pRun4 = Proc()
pRun5 = Proc()
pRun6 = Proc()
pRun7 = Proc()
pRun8 = Proc()
pRun9 = Proc()
pRun10 = Proc()
pRun3.depends = pRun2
pRun6.depends = pRun4, pRun5
pRun3.depends = pRun2
pRun4.depends = pRun2
pRun2.depends = pRun1
pRun10.depends = pRun1
aAggr = Aggr(pRun7, pRun8, pRun9)
aAggr.starts = [aAggr.pRun7]
aAggr.pRun8.depends = aAggr.pRun7
aAggr.pRun9.depends = aAggr.pRun7
aAggr.depends2 = [pRun3, pRun6]
yield [pRun1, pRun5], 'profile', 'sge', [
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun1 (pRun0): No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun2: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun3: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun4: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun5: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun6: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun10: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun7.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun8.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun9.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
]
# pRun21 -> pRun22
# pRun23 -/
# \-> pRun24
pRun21 = Proc()
pRun23 = Proc()
pRun22 = Proc()
pRun22.depends = pRun21, pRun23
yield [pRun21], 'profile', 'local', [
"pRun22 won't run as path can't be reached: pRun22 <- pRun23"
]
from bioprocs.resource import pTxt
from bioprocs.gsea import pExprMat2GCT, pSampleinfo2CLS, pGSEA, pEnrichr
"""
@name:
aExpDir2Deg
@description:
From expfiles to degs with sample info file.
@depends:
```
pExprDir2Matrix[*] \
pRNASeqDEG[!]
pSampleInfo[*] /
```
"""
aExpDir2Deg = Aggr(pExprDir2Matrix,
pFile2Proc.copy(id='pSampleInfo'),
pRNASeqDEG,
depends=False)
# Dependences
aExpDir2Deg.starts = aExpDir2Deg.pExprDir2Matrix, aExpDir2Deg.pSampleInfo
aExpDir2Deg.ends = aExpDir2Deg.pRNASeqDEG
aExpDir2Deg.pRNASeqDEG.depends = aExpDir2Deg.pExprDir2Matrix, aExpDir2Deg.pSampleInfo
# Input
aExpDir2Deg.pRNASeqDEG.input = lambda ch1, ch2: \
ch1.colAt(0).repRow(ch2.length()).cbind(ch2.repRow(ch1.length()))
# Args
aExpDir2Deg.pExprDir2Matrix.args.boxplot = True
aExpDir2Deg.pExprDir2Matrix.args.heatmap = True
aExpDir2Deg.pExprDir2Matrix.args.histplot = True
aExpDir2Deg.pRNASeqDEG.args.maplot = True
aExpDir2Deg.pRNASeqDEG.args.heatmap = True
"""
`pFastMC` : Run multiqc
`pFastq2Sam`: Convert fastq files into sam/bam files.
`pSam2Bam` : Convert sam files into bam files and do some cleaning (sort, markdup)
`pBamRecal` : Recalibrate bam file.
@modules
`ebam` : The aggregation starts with external bam files (off by default)
- `pBam2Fastq[*]` -> `pFastqTrim` -> ...
`fastq`: The aggregation starts with external fastq files (on by default, mutually exclusive with `ebam`)
- `pFastqTrim[*]` -> ...
`qc` : Do qc for the fastq files (off by default)
- `pFastqTrim` -> `pFastQC, pFastMC`
"""
aPrepareBam = Aggr(pBam2Fastq,
pFastqTrim,
pFastQC,
pFastMC,
pFastq2Sam,
pSam2Bam,
pBamRecal,
depends=False)
# starts and ends
aPrepareBam.ends = 'pBamRecal'
# depends
aPrepareBam['pFastq2Sam', ].depends = ['pFastqTrim']
aPrepareBam['pSam2Bam', ].depends = ['pFastq2Sam']
aPrepareBam['pBamRecal', ].depends = ['pSam2Bam']
# delegates
aPrepareBam.delegate('args.ref', 'pFastq2Sam, pBamRecal')
# args
aPrepareBam.pSam2Bam.args.markdup = True
# modules
aPrepareBam.module('ebam',
def dataProvider_testSetattr(self):
pSetattra = Proc()
pSetattra.args.a = Box()
pSetattrb = Proc()
pSetattrb.args.a = Box()
pSetattrc = Proc()
pSetattrc.args.a = Box(b=1)
pSetattrd = Proc()
pSetattre = Proc()
aggr = Aggr(pSetattra, pSetattrb, pSetattrc, pSetattrd, pSetattre)
aggr.delegate('forks', 'pSetattrb')
aggr.delegate('tag', 'ends')
proxy = _Proxy(aggr, check=True)
yield proxy, 'forks', 10, [1, 10, 1, 1, 1]
yield proxy, 'tag', 't', [
aggr.pSetattra.tag, aggr.pSetattra.tag, aggr.pSetattra.tag,
aggr.pSetattra.tag, 't'
]
# depends and input
aggr.starts = 'pSetattra, pSetattrb'
yield proxy, 'depends', ['pSetattrc', 'pSetattrd'], [[aggr.pSetattrc],
[aggr.pSetattrd],
[aggr.pSetattrb],
[aggr.pSetattrc],
[aggr.pSetattrd]]
yield proxy, 'depends2', ['pSetattrc', 'pSetattrd'
], [[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrb], [aggr.pSetattrc],
[aggr.pSetattrd]]
aggr1 = Aggr(pSetattra, pSetattrb, pSetattrc)
aggr1.delegate('args.a.b', 'pSetattra, pSetattrb')
aggr1.delegate('args.a', 'pSetattrb')
proxy1 = _Proxy(aggr1, prefix=['args', 'a'], check=True)
yield proxy1, 'b', 3, [3, 3, 1]
def dataProvider_testModuleFunc(self):
aggr = Aggr()
yield aggr, 'qc', lambda a: True, lambda a: False
def testSetGetAttr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
pGetAttr4 = Proc()
pGetAttr5 = Proc()
pGetAttr1.args.params = Box(inopts=Box())
pGetAttr2.args.params = Box(inopts=Box())
pGetAttr3.args.params = Box(inopts=Box())
pGetAttr4.args.params = Box(inopts=Box())
pGetAttr5.args.params = Box(inopts=Box())
aggr = Aggr(pGetAttr1,
pGetAttr2,
pGetAttr3,
pGetAttr4,
pGetAttr5,
depends=False)
# nothing delegated yet
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# but when starts and ends changed
aggr.starts = [aggr.pGetAttr1, aggr.pGetAttr2]
aggr.ends = [aggr.pGetAttr5]
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# delegate a short attribute
aggr.delegate('forks', [aggr.pGetAttr2, aggr.pGetAttr3])
aggr.forks = 10
# only 2, 3 changed
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 1, 1])
# change the specific procs
aggr['pGetAttr2', 'pGetAttr4'].forks = 5
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 5, 1])
# fix an attribute
aggr.pGetAttr3.runner = 'dry'
aggr.runner = 'sge'
self.assertListEqual(
[p.config['runner'] for p in aggr._procs.values()],
['sge', 'sge', 'dry', 'sge', 'sge'])
# set input
self.assertRaises(AggrAttributeError, setattr, aggr, 'input', 1)
aggr.input = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[1, 2, '', '', ''])
# reverse it
aggr.input = ['', '']
aggr.input2 = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', '', ''])
aggr[3].input = 'i3'
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', 'i3', ''])
# not suppose to do this
# aggr.input2[2] = ['a', 'b']
#self.assertListEqual([p.config['input'] for p in aggr._procs.values()], [[1,2], [1,2], ['a', 'b'], 'i3', ''])
# similar for depends
#self.assertRaises(AggrAttributeError, setattr, aggr, 'depends', 1)
aggr.depends = [pGetAttr1, pGetAttr2]
self.assertListEqual([p.depends for p in aggr._procs.values()],
[[pGetAttr1], [pGetAttr2], [], [], []])
# reverse it
aggr.depends = [[], []]
aggr.depends2 = [pGetAttr1, pGetAttr2]
self.assertListEqual(
[p.depends for p in aggr._procs.values()],
[[pGetAttr1, pGetAttr2], [pGetAttr1, pGetAttr2], [], [], []])
# set attributes of certain processes
aggr[0].args.params = Box(inopts=Box(a=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(a=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[:2].args.params = Box(inopts=Box(b=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[1, 2].args.params = Box(inopts=Box(c=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
# using proc ids
aggr['pGetAttr1'].args.params = Box(inopts=Box(d=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(d=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1', 'pGetAttr3'].args.params = Box(inopts=Box(e=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1, pGetAttr3'].args.params = Box(inopts=Box(f=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['starts'].args.params = Box(inopts=Box(g=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['ends'].args.params = Box(inopts=Box(h=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.delegate('args.params.inopts', 'pGetAttr3')
aggr.args['params'].inopts = Box(n=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(n=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params['inopts'] = Box(m=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr[0].args.params.inopts = Box(z=1)
# remeber at line 453, inopts refers to the same Box() object, so both pGetAttr1 and 3 will change
self.assertListEqual(
[p.args for p in aggr._procs.values()],
[
Box(params=Box(inopts=Box(z=1))),
Box(params=Box(inopts=Box(z=1))), # <-- not g
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params.inopts.cnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(),
Box(h=1),
])
# something undelegated
aggr.tag = 'abc'
self.assertListEqual([p.tag for p in aggr._procs.values()],
['abc'] * 5)
del aggr._delegates['args.params.inopts']
aggr.delegate('args', 'pGetAttr4')
aggr.args.params.inopts.rnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(rnames=True),
Box(h=1),
])
def dataProvider_testSetattr(self):
pSetattra = Proc()
pSetattra.args.a = Box()
pSetattrb = Proc()
pSetattrb.args.a = Box()
pSetattrc = Proc()
pSetattrc.args.a = Box(b=1)
pSetattrd = Proc()
pSetattre = Proc()
aggr = Aggr(pSetattra, pSetattrb, pSetattrc, pSetattrd, pSetattre)
aggr.delegate('forks', 'pSetattrb')
aggr.delegate('tag', 'ends')
yield aggr, 'forks', 10, [1, 10, 1, 1, 1]
yield aggr, 'tag', 't', [
aggr.pSetattra.tag, aggr.pSetattra.tag, aggr.pSetattra.tag,
aggr.pSetattra.tag, 't'
]
# depends and input
aggr.starts = 'pSetattra, pSetattrb'
yield aggr, 'depends', ['pSetattrc', 'pSetattrd'], [[aggr.pSetattrc],
[aggr.pSetattrd],
[aggr.pSetattrb],
[aggr.pSetattrc],
[aggr.pSetattrd]]
yield aggr, 'depends2', ['pSetattrc', 'pSetattrd'
], [[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrb], [aggr.pSetattrc],
[aggr.pSetattrd]]
aggr1 = Aggr(pSetattra, pSetattrb, pSetattrc)
aggr1.delegate('errntry', 'pSetattra, pSetattrb')
aggr1.delegate('args.a', 'pSetattrb')
yield aggr1, 'errntry', 8, [8, 8, 3]