def dataProvider_testSetAttr(self):
pSetAttr = Proc()
pSetAttrDepends = Proc()
pSetAttrAggr = Proc()
aSetAttr = Aggr(pSetAttrAggr)
aSetAttr.ends = [aSetAttr.pSetAttrAggr]
yield pSetAttr, '__nosuchattr__', None, None, ProcAttributeError, 'Cannot set attribute for process'
yield pSetAttr, 'profile', 'sge', 'local', None, None, ['WARNING', 'Attribute "profile" is deprecated']
yield pSetAttr, 'envs', {'a': 1}
yield pSetAttr, 'depends', pSetAttr, None, ProcAttributeError, 'Process depends on itself'
yield pSetAttr, 'depends', 1, None, ProcAttributeError, "Process dependents should be 'Proc/Aggr', not: 'int'"
#5
yield pSetAttr, 'depends', pSetAttrDepends, [pSetAttrDepends]
yield pSetAttr, 'depends', aSetAttr, [aSetAttr.pSetAttrAggr]
yield pSetAttr, 'depends', (aSetAttr, pSetAttrDepends), [aSetAttr.pSetAttrAggr, pSetAttrDepends]
yield pSetAttr, 'script', 'file:' + path.abspath(__file__)
yield pSetAttr, 'script', 'file:' + path.relpath(__file__, __folder__), 'file:' + path.abspath(__file__)
#10
yield pSetAttr, 'args', {'a':1}, Box({'a':1})
yield pSetAttr, 'envs', {'a':1}, Box({'a':1})
yield pSetAttr, 'input', 'inkey1:var, inkey2:file'
yield pSetAttr, 'input', [('inval1', 'inval2')], {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
yield pSetAttr, 'input', ['inkey3:var', 'inkey4:file'], {'inkey1:var, inkey2:file': ['inkey3:var', 'inkey4:file']}
yield pSetAttr, 'input', {'inkey1:var': 'inval1', 'inkey2:file': 'inval2'}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
def dataProvider_testDelEnd(self):
p1 = Proc()
p2 = Proc()
p3 = Proc()
aggr = Aggr(p1, p2, p3, depends=False)
aggr.ends = 'p1, p2'
yield aggr, [aggr.p1], [aggr.p2]
yield aggr, ['p1, p2'], []
pBedGetfasta,
pMotifScan,
depends = False
)
# defaults
aTfbsTfP.pPromoters.runner = 'local'
delegate
aTfbsTfP.delegate('args.up' , 'pPromoters')
aTfbsTfP.delegate('args.down' , 'pPromoters')
aTfbsTfP.delegate('args.genome' , 'pPromoters')
aTfbsTfP.delegate('args.ref' , 'pBedGetfasta')
aTfbsTfP.delegate('args.pval' , 'pMotifScan')
aTfbsTfP.delegate('args.tfmotifs', 'pMotifScan')
# depends
aTfbsTfP.starts = aTfbsTfP.pTFs, aTfbsTfP.pPromoters, aTfbsTfP.pTFList
aTfbsTfP.ends = aTfbsTfP.pMotifScan
aTfbsTfP.pMotifScan.depends = aTfbsTfP.pSimRead, aTfbsTfP.pBedGetfasta
aTfbsTfP.pBedGetfasta.depends = aTfbsTfP.pPromoters
aTfbsTfP.pSimRead.depends = aTfbsTfP.pTFList, aTfbsTfP.pTFs
# input
aTfbsTfP.pTFList.input = [params.tflist.value]
# input size of either pTFs or pTFList should be 1
aTfbsTfP.pSimRead.input = lambda ch1, ch2: [ch1.repRow(l).cbind(ch2.repRow(l)).flatten() for l in [max(ch1.length(), ch2.length())]]
aTfbsTfP.pMotifScan.input = lambda ch1, ch2: [ch1.repRow(l).cbind(ch2.repRow(l)) for l in [max(ch1.length(), ch2.length())]][0]
# args
aTfbsTfP.pBedGetfasta.args.params.name = True
aTfbsTfP.pTFList.args.params.k = 2
aTfbsTfP.pSimRead.args.match = 'lambda line1, line2: -1 if line1[1] == line2[0] else 0 if line1[1] < line2[0] else 1'
aTfbsTfP.pSimRead.args.do = 'lambda line1, line2: fout.write("\\t".join(line1) + "\\n")'
"""
def testSetGetAttr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
pGetAttr4 = Proc()
pGetAttr5 = Proc()
pGetAttr1.args.params = Box(inopts=Box())
pGetAttr2.args.params = Box(inopts=Box())
pGetAttr3.args.params = Box(inopts=Box())
pGetAttr4.args.params = Box(inopts=Box())
pGetAttr5.args.params = Box(inopts=Box())
aggr = Aggr(pGetAttr1,
pGetAttr2,
pGetAttr3,
pGetAttr4,
pGetAttr5,
depends=False)
# nothing delegated yet
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# but when starts and ends changed
aggr.starts = [aggr.pGetAttr1, aggr.pGetAttr2]
aggr.ends = [aggr.pGetAttr5]
self.assertDictEqual(
{
'input': ['starts'],
'depends': ['starts'],
'exdir': ['ends'],
'exhow': ['ends'],
'expart': ['ends'],
'exow': ['ends']
}, aggr._delegates)
# delegate a short attribute
aggr.delegate('forks', [aggr.pGetAttr2, aggr.pGetAttr3])
aggr.forks = 10
# only 2, 3 changed
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 1, 1])
# change the specific procs
aggr['pGetAttr2', 'pGetAttr4'].forks = 5
self.assertListEqual([p.forks for p in aggr._procs.values()],
[1, 10, 10, 5, 1])
# fix an attribute
aggr.pGetAttr3.runner = 'dry'
aggr.runner = 'sge'
self.assertListEqual(
[p.config['runner'] for p in aggr._procs.values()],
['sge', 'sge', 'dry', 'sge', 'sge'])
# set input
self.assertRaises(AggrAttributeError, setattr, aggr, 'input', 1)
aggr.input = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[1, 2, '', '', ''])
# reverse it
aggr.input = ['', '']
aggr.input2 = [1, 2]
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', '', ''])
aggr[3].input = 'i3'
self.assertListEqual([p.config['input'] for p in aggr._procs.values()],
[[1, 2], [1, 2], '', 'i3', ''])
# not suppose to do this
# aggr.input2[2] = ['a', 'b']
#self.assertListEqual([p.config['input'] for p in aggr._procs.values()], [[1,2], [1,2], ['a', 'b'], 'i3', ''])
# similar for depends
#self.assertRaises(AggrAttributeError, setattr, aggr, 'depends', 1)
aggr.depends = [pGetAttr1, pGetAttr2]
self.assertListEqual([p.depends for p in aggr._procs.values()],
[[pGetAttr1], [pGetAttr2], [], [], []])
# reverse it
aggr.depends = [[], []]
aggr.depends2 = [pGetAttr1, pGetAttr2]
self.assertListEqual(
[p.depends for p in aggr._procs.values()],
[[pGetAttr1, pGetAttr2], [pGetAttr1, pGetAttr2], [], [], []])
# set attributes of certain processes
aggr[0].args.params = Box(inopts=Box(a=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(a=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[:2].args.params = Box(inopts=Box(b=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr[1, 2].args.params = Box(inopts=Box(c=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(b=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
# using proc ids
aggr['pGetAttr1'].args.params = Box(inopts=Box(d=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(d=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1', 'pGetAttr3'].args.params = Box(inopts=Box(e=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(e=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['pGetAttr1, pGetAttr3'].args.params = Box(inopts=Box(f=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box(c=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['starts'].args.params = Box(inopts=Box(g=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box())),
])
aggr['ends'].args.params = Box(inopts=Box(h=1))
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(f=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.delegate('args.params.inopts', 'pGetAttr3')
aggr.args['params'].inopts = Box(n=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(n=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params['inopts'] = Box(m=1)
self.assertListEqual([p.args for p in aggr._procs.values()], [
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(g=1))),
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr[0].args.params.inopts = Box(z=1)
# remeber at line 453, inopts refers to the same Box() object, so both pGetAttr1 and 3 will change
self.assertListEqual(
[p.args for p in aggr._procs.values()],
[
Box(params=Box(inopts=Box(z=1))),
Box(params=Box(inopts=Box(z=1))), # <-- not g
Box(params=Box(inopts=Box(m=1))),
Box(params=Box(inopts=Box())),
Box(params=Box(inopts=Box(h=1))),
])
aggr.args.params.inopts.cnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(),
Box(h=1),
])
# something undelegated
aggr.tag = 'abc'
self.assertListEqual([p.tag for p in aggr._procs.values()],
['abc'] * 5)
del aggr._delegates['args.params.inopts']
aggr.delegate('args', 'pGetAttr4')
aggr.args.params.inopts.rnames = True
self.assertListEqual(
[p.args.params.inopts for p in aggr._procs.values()], [
Box(z=1),
Box(z=1),
Box(m=1, cnames=True),
Box(rnames=True),
Box(h=1),
])
- `pBam2Fastq[*]` -> `pFastqTrim` -> ...
`fastq`: The aggregation starts with external fastq files (on by default, mutually exclusive with `ebam`)
- `pFastqTrim[*]` -> ...
`qc` : Do qc for the fastq files (off by default)
- `pFastqTrim` -> `pFastQC, pFastMC`
"""
aPrepareBam = Aggr(pBam2Fastq,
pFastqTrim,
pFastQC,
pFastMC,
pFastq2Sam,
pSam2Bam,
pBamRecal,
depends=False)
# starts and ends
aPrepareBam.ends = 'pBamRecal'
# depends
aPrepareBam['pFastq2Sam', ].depends = ['pFastqTrim']
aPrepareBam['pSam2Bam', ].depends = ['pFastq2Sam']
aPrepareBam['pBamRecal', ].depends = ['pSam2Bam']
# delegates
aPrepareBam.delegate('args.ref', 'pFastq2Sam, pBamRecal')
# args
aPrepareBam.pSam2Bam.args.markdup = True
# modules
aPrepareBam.module('ebam',
starts='pBam2Fastq',
depends={'pFastqTrim': 'pBam2Fastq'})
aPrepareBam.module('fastq', starts='pFastqTrim')
aPrepareBam.module('qc',
ends='pFastQC, pFastMC',
@description:
From expfiles to degs with sample info file.
@depends:
```
pExprDir2Matrix[*] \
pRNASeqDEG[!]
pSampleInfo[*] /
```
"""
aExpDir2Deg = Aggr(pExprDir2Matrix,
pFile2Proc.copy(id='pSampleInfo'),
pRNASeqDEG,
depends=False)
# Dependences
aExpDir2Deg.starts = aExpDir2Deg.pExprDir2Matrix, aExpDir2Deg.pSampleInfo
aExpDir2Deg.ends = aExpDir2Deg.pRNASeqDEG
aExpDir2Deg.pRNASeqDEG.depends = aExpDir2Deg.pExprDir2Matrix, aExpDir2Deg.pSampleInfo
# Input
aExpDir2Deg.pRNASeqDEG.input = lambda ch1, ch2: \
ch1.colAt(0).repRow(ch2.length()).cbind(ch2.repRow(ch1.length()))
# Args
aExpDir2Deg.pExprDir2Matrix.args.boxplot = True
aExpDir2Deg.pExprDir2Matrix.args.heatmap = True
aExpDir2Deg.pExprDir2Matrix.args.histplot = True
aExpDir2Deg.pRNASeqDEG.args.maplot = True
aExpDir2Deg.pRNASeqDEG.args.heatmap = True
"""
@name:
aExpDir2DegGSEA
@description:
From expfiles to degs with sample info file and do GSEA.
