def dataProvider_testFromFile(self):
file1 = path.join(self.testdir, 'testFromFile1.txt')
helpers.writeFile(file1, "a1\tb1\tc1\n" + "a2\tb2\tc2")
outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")])
yield file1, outs, False, 0, '\t'
# head & delimit
file2 = path.join(self.testdir, "testFromFile2.txt")
helpers.writeFile(file2, "a,b,c\n" + "a1,b1,c1\n" + "a2,b2,c2")
outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")])
yield file2, outs, ['a', 'b', 'c'], 0, ','
# skip
file3 = path.join(self.testdir, "testFromFile3.txt")
helpers.writeFile(
file3,
"#a,b,c\n" + "#a,b,c\n" + "b,c\n" + "a1,b1,c1\n" + "a2,b2,c2")
outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")])
yield file3, outs, ['RowNames', 'b', 'c'], 2, ','
# error
file4 = path.join(self.testdir, "testFromFile4.txt")
helpers.writeFile(file4,
"#a,b,c\n" + "b,c,d,e\n" + "a1,b1,c1\n" + "a2,b2,c2")
yield file4, [], ['a'], 1, ',', True
def testUnfold(self, ch, n, outs, exception=False):
ch = Channel.create(ch)
if exception:
self.assertRaises(ValueError, ch.unfold, n)
else:
outs = Channel.create(outs)
self.assertListEqual(ch.unfold(n), outs)
def test_buildinput(tmpdir, caplog):
p10 = Proc()
p10.input = 'a, b:file, '
p10.input = ('1', 'infile')
p10._buildInput()
assert len(p10.input) == 2
assert p10.input['a'] == ('var', ['1'])
assert p10.input['b'] == ('file', ['infile'])
assert p10.size == 1
p10.input = 'a:x:y'
with pytest.raises(ProcInputError):
p10._buildInput()
p101 = Proc()
p101.props.channel = Channel.create([(1, 3), (2, 4)])
p10.depends = p101
p10.input = 'a, b, c'
p10.input = lambda ch: ch.cbind(1).cbind(2)
caplog.clear()
p10._buildInput()
assert 'Not all data are used as input' in caplog.text
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('var', [1, 2])
assert p10.input['b'] == ('var', [3, 4])
assert p10.input['c'] == ('var', [1, 1])
p10.input = 'a:files, b:files, c'
p10.input = Channel.create([['infile1'], ['infile2']])
p10._buildInput()
assert 'No data found for input key "b"' in caplog.text
assert 'No data found for input key "c"' in caplog.text
caplog.clear()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
p10.props.template = TemplateLiquid
p10.props.workdir = tmpdir / 'test_buildinput_p10'
p10.resume = 'resume'
fs.remove(Path(p10.workdir) / 'proc.settings.yaml')
with pytest.raises(ProcInputError):
p10._buildInput()
fs.mkdir(p10.workdir)
p10.props.input = OBox()
p10.input['a'] = ('files', [['infile1'], ['infile2']])
p10.input['b'] = ('files', [[], []])
p10.input['c'] = ('var', ['', ''])
p10._saveSettings()
p10.props.input = None
p10._buildInput()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
def testFromArgv(self, args, outs, exception=False):
import sys
sys.argv = args
if exception:
self.assertRaises(ValueError, Channel.fromArgv)
else:
ch = Channel.fromArgv()
self.assertListEqual(Channel.fromArgv(), outs)
def dataProvider_testFromPairs(self): files1 = [path.join(self.testdir, 'testFromPairs1%s.txt' % i) for i in range(0, 4)] files2 = [path.join(self.testdir, 'testFromPairs2%s.txt' % i) for i in range(0, 4)] for f in files1 + files2: helpers.writeFile(f) yield path.join(self.testdir, 'testFromPairs1?.txt'), Channel.create([(files1[0], files1[1]), (files1[2], files1[3])]) yield path.join(self.testdir, 'testFromPairs2?.txt'), Channel.create([(files2[0], files2[1]), (files2[2], files2[3])])
def testRbind(self, ch, rows, outs, exception=False):
ch = Channel.create(ch)
if not exception:
ch2 = ch.rbind(*rows)
outs = Channel.create(outs)
self.assertListEqual(ch2, outs)
else:
self.assertRaises(ValueError, ch.rbind, *rows)
def test_fromparams():
from pyparam import params
params._test1 = [1, 2, 3]
params._test2 = [4, 4, 4]
assert Channel.fromParams('_test1', '_test2') == [(1, 4), (2, 4), (3, 4)]
params._test3 = 5
params._test4 = 6
assert Channel.fromParams('_test3', '_test4') == [(5, 6)]
with pytest.raises(ValueError):
Channel.fromParams('_test3', '_test1')
def testCbind(self, ch, cols, outs, exception = False): ch = Channel.create(ch) orgcols = deepcopy(cols) if not exception: ch2 = ch.cbind(*cols) outs = Channel.create(outs) self.assertListEqual(ch2, outs) for i, ocol in enumerate(orgcols): self.assertListEqual(Channel.create(ocol), Channel.create(cols[i])) else: self.assertRaises(ValueError, ch.cbind, *cols)
def test_collapse(expand_dirs):
dir1, dir2 = expand_dirs
file1 = str(dir1 / 'testExpand1.txt')
file2 = str(dir1 / 'testExpand2.txt')
file3 = str(dir2 / 'testExpand3.txt')
file4 = str(dir2 / 'testExpand4.txt')
with pytest.raises(ValueError):
Channel.create().collapse(col=0)
assert Channel.create([file1, file2]).collapse() == [(str(dir1), )]
assert Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')
]).collapse(1) == [('a1', str(dir1), 'a2')]
assert Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')
]).collapse(0) == [('', file1, 'a2')]
def testFromParams(self, ps, outs=[], exception=False):
from pyppl.parameters import Parameter, params
outs = Channel.create(outs)
for p in ps:
params._params[p.name] = p
pnames = [p.name for p in ps]
if not exception:
ch = Channel.fromParams(*pnames)
self.assertListEqual(ch, outs)
if ch:
for i, p in enumerate(ps):
self.assertListEqual(getattr(ch, p.name), outs.colAt(i))
else:
self.assertRaises(ValueError, Channel.fromParams, *pnames)
def testCollapse(self, ch, col, outs, exception=False):
if exception:
self.assertRaises(ValueError, ch.collapse, col)
else:
c = ch.collapse(col)
outs = Channel.create(outs)
self.assertListEqual(c, outs)
def testReduceCol(self, ch, func, col, outs, exception=False):
ch = Channel.create(ch)
if not exception:
ch2 = ch.reduceCol(func, col)
self.assertEqual(ch2, outs)
else:
self.assertRaises(TypeError, ch.reduceCol, func, col)
def test_frompairs(tmp_test_dir, pattern, expt, paired_files):
assert Channel.fromPairs(
(tmp_test_dir / 'test_frompairs' / pattern).as_posix()) == [
((tmp_test_dir / 'test_frompairs' / paired_files[e[0]]).as_posix(),
(tmp_test_dir / 'test_frompairs' / paired_files[e[1]]).as_posix())
for e in expt
]
def testExpand(self, ch, col, outs, pattern = '*', t = 'any', sortby = 'name', reverse = False, exception = False): if exception: self.assertRaises(ValueError, ch.expand, col, pattern, t, sortby, reverse) else: c = ch.expand(col, pattern, t, sortby, reverse) outs = Channel.create(outs) self.assertListEqual(c, outs)
def testReduce(self, ch, func, outs, exception=False):
ch = Channel.create(ch)
if not exception:
ch2 = ch.reduce(func)
self.assertTupleEqual(ch2, outs)
else:
self.assertRaises(TypeError, ch.reduce, func)
def testFromPattern(self,
pattern,
outs,
t='any',
sortby='name',
reverse=False):
self.assertListEqual(Channel.fromPattern(pattern, t, sortby, reverse),
outs)
def testAttach(self, ch, names, flatten = False, exception = None):
ch = Channel.create(ch)
if exception:
self.assertRaises(exception, ch.attach, *names, **{'flatten': flatten})
else:
ch.attach(*names, **{'flatten': flatten})
for i, name in enumerate(names):
self.assertListEqual(ch.colAt(i) if not flatten else ch.flatten(i), getattr(ch, name))
def dataProvider_testCollapse(self):
# empty self
yield Channel.create(), 0, [], True
# defaults
dir1 = path.join(self.testdir, 'testCollapse')
file1 = path.join(dir1, 'testCollapse1.txt')
file2 = path.join(dir1, 'testCollapse2.txt')
makedirs(dir1)
helpers.writeFile(file1)
helpers.writeFile(file2)
yield Channel.create([file1, file2]), 0, dir1
# Extra cols
yield Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')]), 1, ('a1', dir1, 'a2')
# No common prefix
yield Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')]), 0, ('', file1, 'a2')
def dataProvider_testCbind(self): yield [], [], [] yield [(1,2), (3,4)], [5, 6], [(1,2,5,6), (3,4,5,6)] yield [(1,2), (3,4)], [(5, 6,)], [(1,2,5,6), (3,4,5,6)] yield [(1,2), (3,4)], [5], [(1,2,5), (3,4,5)] yield [7,8], [[5, 6], Channel.create(4)], [(7,5,4), (8,6,4)] yield [], [21, 22], [(21, 22)] yield [], [(21, 22, )], [(21, 22)] yield [], [[21, 22], (1, 2, 3)], [(21, 1, 2, 3), (22, 1,2,3)] yield [], [[21, 22], [1, 2, 3]], [], True
def testFromFile(self, fn, outs, header = False, skip = 0, delimit = "\t", exception = False): headerFlag = bool(header) if exception: self.assertRaises(ValueError, Channel.fromFile, fn, header, skip, delimit) else: ch = Channel.fromFile(fn, header = headerFlag, skip = skip, delimit = delimit) self.assertListEqual(ch, outs) if headerFlag: for i, h in enumerate(header): self.assertListEqual(getattr(ch, h), outs.colAt(i))
def dataProvider_testExpand(self):
# empty self
yield Channel.create(), 0, []
# defaults
dir1 = path.join(self.testdir, 'testExpand')
file1 = path.join(dir1, 'testExpand1.txt')
file2 = path.join(dir1, 'testExpand2.txt')
makedirs(dir1)
helpers.writeFile(file1)
helpers.writeFile(file2)
yield Channel.create(dir1), 0, [file1, file2]
# extra columns
dir2 = path.join(self.testdir, 'testExpand2')
file3 = path.join(dir2, 'testExpand3.txt')
file4 = path.join(dir2, 'testExpand4.txt')
makedirs(dir2)
helpers.writeFile(file3)
helpers.writeFile(file4)
yield Channel.create(('a', 1, dir2)), 2, [('a', 1, file3),
('a', 1, file4)]
# pattern not exists
yield Channel.create(('a', 1, dir2)), 2, [], 'a.*'
# expand respectively
yield Channel.create([('a', 1, dir1),
('b', 2, dir2)]), 2, Channel.create([
('a', 1, file1),
('a', 1, file2),
('b', 2, file3),
('b', 2, file4),
])
def dataProvider_testFromChannels(self):
ch1 = Channel.create([(1, 2), (3, 4)])
ch2 = Channel.create('a')
ch3 = Channel.create([5, 6])
outs = [(1, 2, 'a', 5), (3, 4, 'a', 6)]
yield [ch1, ch2, ch3], outs
ch1 = Channel.create([])
ch2 = Channel.create([])
yield [ch1, ch2], []
ch1 = Channel.create([])
ch2 = Channel.create(1)
ch3 = Channel.create([1, 2])
yield [ch1, ch2, ch3], [(1, 1), (1, 2)]
def dataProvider_testSplit(self): yield [], True, [] yield [], False, [] yield [1,2,3], True, [[1, 2, 3]] yield [1,2,3], False, [Channel.create([1,2,3])] yield (1,2,3), True, [[1], [2], [3]] yield (1,2,3), False, [Channel.create(1), Channel.create(2), Channel.create(3)] yield [(1,4),(2,5),(3,6)], True, [[1, 2, 3], [4,5,6]] yield [(1,4),(2,5),(3,6)], False, [Channel.create([1,2,3]), Channel.create([4,5,6])]
def test_expand(expand_dirs):
dir1, dir2 = expand_dirs
file1 = str(dir1 / 'testExpand1.txt')
file2 = str(dir1 / 'testExpand2.txt')
file3 = str(dir2 / 'testExpand3.txt')
file4 = str(dir2 / 'testExpand4.txt')
assert Channel.create().expand(col=0) == []
assert Channel.create(dir1).expand(col=0) == [(file1, ), (file2, )]
assert Channel.create(('a', 1, dir2)).expand(col=2) == [('a', 1, file3),
('a', 1, file4)]
assert Channel.create(('a', 1, dir2)).expand(col=2, pattern='a.*') == []
assert Channel.create([('a', 1, dir1),
('b', 2, dir2)]).expand(col=2) == Channel.create([
('a', 1, file1),
('a', 1, file2),
('b', 2, file3),
('b', 2, file4),
])
def testFlatten(self, ch, col, outs, exception=None):
ch = Channel.create(ch)
if exception:
self.assertRaises(exception, ch.flatten, col)
else:
self.assertListEqual(ch.flatten(col), outs)
def testRepRow(self, ch, n, outs):
ch = Channel.create(ch)
outs = Channel.create(outs)
self.assertListEqual(ch.repRow(n), outs)
def dataProvider_testInsert(self):
ch1 = Channel.create([(1, 2), (3, 4)])
ch2 = Channel.create([5, 6])
# 0-3
yield 0, ch1, ch2, [(5, 1, 2), (6, 3, 4)]
yield 1, ch1, ch2, [(1, 5, 2), (3, 6, 4)]
yield -1, ch1, ch2, [(1, 5, 2), (3, 6, 4)]
yield None, ch1, ch2, [(1, 2, 5), (3, 4, 6)]
ch2 = Channel.create(5)
# 4-7
yield 0, ch1, ch2, [(5, 1, 2), (5, 3, 4)]
yield 1, ch1, ch2, [(1, 5, 2), (3, 5, 4)]
yield -1, ch1, ch2, [(1, 5, 2), (3, 5, 4)]
yield None, ch1, ch2, [(1, 2, 5), (3, 4, 5)]
# 8-19
yield 0, ch1, [5, 6], [(5, 1, 2), (6, 3, 4)]
yield 1, ch1, [5, 6], [(1, 5, 2), (3, 6, 4)]
yield -1, ch1, [5, 6], [(1, 5, 2), (3, 6, 4)]
yield None, ch1, [5, 6], [(1, 2, 5), (3, 4, 6)]
yield 0, ch1, (5, 6), [(5, 6, 1, 2), (5, 6, 3, 4)]
yield 1, ch1, (5, 6), [(1, 5, 6, 2), (3, 5, 6, 4)]
yield -1, ch1, (5, 6), [(1, 5, 6, 2), (3, 5, 6, 4)]
yield None, ch1, (5, 6), [(1, 2, 5, 6), (3, 4, 5, 6)]
yield 0, ch1, "a", [('a', 1, 2), ('a', 3, 4)]
yield 1, ch1, "a", [(1, 'a', 2), (3, 'a', 4)]
yield -1, ch1, "a", [(1, 'a', 2), (3, 'a', 4)]
yield None, ch1, "a", [(1, 2, 'a'), (3, 4, 'a')]
# 20-23
yield 0, ch1, [], ch1
yield 1, ch1, [], ch1
yield -1, ch1, [], ch1
yield None, ch1, [], ch1
ch1 = Channel.create()
ch2 = Channel.create([21, 22])
ch3 = 3
ch4 = [41, 42]
ch5 = (51, 52)
ch6 = "a"
# 24-25
yield 1, ch1, ch2, ch2
yield 0, ch1, [ch2, ch3, ch4,
ch5, ch6], [(21, 3, 41, 51, 52, 'a'),
(22, 3, 42, 51, 52, 'a')], True
# 26-29
yield 0, ch1, [], ch1
yield 1, ch1, [], ch1
yield -1, ch1, [], ch1
yield None, ch1, [], ch1
# 30-31
yield None, ch1, [1, [1, 2]], [(1, 1), (1, 2)], True
yield None, ch1, [Channel.create(1),
Channel.create([1, 2])], [(1, 1), (1, 2)], True
# 32 Emptys
yield 1, ch1, [[], 1, [], [2, 3]], [(1, 2), (1, 3)], True
yield None, Channel.create(), [
Channel.create([4, 5]),
Channel.create([1, 2, 3])
], ValueError, True
ch1 = Channel.create([(1, 2), (3, 4)])
yield None, ch1, [1, 2, 3], ValueError
# issue 29
yield 0, Channel.create(), '', [('', )]
def testGet(self, ch, idx, outs, exception=None):
ch = Channel.create(ch)
if exception:
self.assertRaises(exception, ch.get, idx)
else:
self.assertEqual(ch.get(idx), outs)
def testSplit(self, ch, flatten, outs):
ch = Channel.create(ch)
self.assertListEqual(ch.split(flatten), outs)
def dataProvider_testT(self):
yield Channel.create(), []
yield Channel.create([1, 2, 3]), [(1, 2, 3)]
yield Channel.create([(1, 2, 3)]), [(1, ), (2, ), (3, )]