def testUnfold(self, ch, n, outs, exception=False): ch = Channel.create(ch) if exception: self.assertRaises(ValueError, ch.unfold, n) else: outs = Channel.create(outs) self.assertListEqual(ch.unfold(n), outs)
def dataProvider_testFromFile(self): file1 = path.join(self.testdir, 'testFromFile1.txt') helpers.writeFile(file1, "a1\tb1\tc1\n" + "a2\tb2\tc2") outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")]) yield file1, outs, False, 0, '\t' # head & delimit file2 = path.join(self.testdir, "testFromFile2.txt") helpers.writeFile(file2, "a,b,c\n" + "a1,b1,c1\n" + "a2,b2,c2") outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")]) yield file2, outs, ['a', 'b', 'c'], 0, ',' # skip file3 = path.join(self.testdir, "testFromFile3.txt") helpers.writeFile( file3, "#a,b,c\n" + "#a,b,c\n" + "b,c\n" + "a1,b1,c1\n" + "a2,b2,c2") outs = Channel.create([("a1", "b1", "c1"), ("a2", "b2", "c2")]) yield file3, outs, ['RowNames', 'b', 'c'], 2, ',' # error file4 = path.join(self.testdir, "testFromFile4.txt") helpers.writeFile(file4, "#a,b,c\n" + "b,c,d,e\n" + "a1,b1,c1\n" + "a2,b2,c2") yield file4, [], ['a'], 1, ',', True
def test_buildinput(tmpdir, caplog): p10 = Proc() p10.input = 'a, b:file, ' p10.input = ('1', 'infile') p10._buildInput() assert len(p10.input) == 2 assert p10.input['a'] == ('var', ['1']) assert p10.input['b'] == ('file', ['infile']) assert p10.size == 1 p10.input = 'a:x:y' with pytest.raises(ProcInputError): p10._buildInput() p101 = Proc() p101.props.channel = Channel.create([(1, 3), (2, 4)]) p10.depends = p101 p10.input = 'a, b, c' p10.input = lambda ch: ch.cbind(1).cbind(2) caplog.clear() p10._buildInput() assert 'Not all data are used as input' in caplog.text assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('var', [1, 2]) assert p10.input['b'] == ('var', [3, 4]) assert p10.input['c'] == ('var', [1, 1]) p10.input = 'a:files, b:files, c' p10.input = Channel.create([['infile1'], ['infile2']]) p10._buildInput() assert 'No data found for input key "b"' in caplog.text assert 'No data found for input key "c"' in caplog.text caplog.clear() assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('files', [['infile1'], ['infile2']]) assert p10.input['b'] == ('files', [[], []]) assert p10.input['c'] == ('var', ['', '']) p10.props.template = TemplateLiquid p10.props.workdir = tmpdir / 'test_buildinput_p10' p10.resume = 'resume' fs.remove(Path(p10.workdir) / 'proc.settings.yaml') with pytest.raises(ProcInputError): p10._buildInput() fs.mkdir(p10.workdir) p10.props.input = OBox() p10.input['a'] = ('files', [['infile1'], ['infile2']]) p10.input['b'] = ('files', [[], []]) p10.input['c'] = ('var', ['', '']) p10._saveSettings() p10.props.input = None p10._buildInput() assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('files', [['infile1'], ['infile2']]) assert p10.input['b'] == ('files', [[], []]) assert p10.input['c'] == ('var', ['', ''])
def testRbind(self, ch, rows, outs, exception=False): ch = Channel.create(ch) if not exception: ch2 = ch.rbind(*rows) outs = Channel.create(outs) self.assertListEqual(ch2, outs) else: self.assertRaises(ValueError, ch.rbind, *rows)
def dataProvider_testFromPairs(self): files1 = [path.join(self.testdir, 'testFromPairs1%s.txt' % i) for i in range(0, 4)] files2 = [path.join(self.testdir, 'testFromPairs2%s.txt' % i) for i in range(0, 4)] for f in files1 + files2: helpers.writeFile(f) yield path.join(self.testdir, 'testFromPairs1?.txt'), Channel.create([(files1[0], files1[1]), (files1[2], files1[3])]) yield path.join(self.testdir, 'testFromPairs2?.txt'), Channel.create([(files2[0], files2[1]), (files2[2], files2[3])])
def testCbind(self, ch, cols, outs, exception = False): ch = Channel.create(ch) orgcols = deepcopy(cols) if not exception: ch2 = ch.cbind(*cols) outs = Channel.create(outs) self.assertListEqual(ch2, outs) for i, ocol in enumerate(orgcols): self.assertListEqual(Channel.create(ocol), Channel.create(cols[i])) else: self.assertRaises(ValueError, ch.cbind, *cols)
def test_collapse(expand_dirs): dir1, dir2 = expand_dirs file1 = str(dir1 / 'testExpand1.txt') file2 = str(dir1 / 'testExpand2.txt') file3 = str(dir2 / 'testExpand3.txt') file4 = str(dir2 / 'testExpand4.txt') with pytest.raises(ValueError): Channel.create().collapse(col=0) assert Channel.create([file1, file2]).collapse() == [(str(dir1), )] assert Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2') ]).collapse(1) == [('a1', str(dir1), 'a2')] assert Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2') ]).collapse(0) == [('', file1, 'a2')]
def testExpand(self, ch, col, outs, pattern = '*', t = 'any', sortby = 'name', reverse = False, exception = False): if exception: self.assertRaises(ValueError, ch.expand, col, pattern, t, sortby, reverse) else: c = ch.expand(col, pattern, t, sortby, reverse) outs = Channel.create(outs) self.assertListEqual(c, outs)
def testCollapse(self, ch, col, outs, exception=False): if exception: self.assertRaises(ValueError, ch.collapse, col) else: c = ch.collapse(col) outs = Channel.create(outs) self.assertListEqual(c, outs)
def testReduceCol(self, ch, func, col, outs, exception=False): ch = Channel.create(ch) if not exception: ch2 = ch.reduceCol(func, col) self.assertEqual(ch2, outs) else: self.assertRaises(TypeError, ch.reduceCol, func, col)
def testReduce(self, ch, func, outs, exception=False): ch = Channel.create(ch) if not exception: ch2 = ch.reduce(func) self.assertTupleEqual(ch2, outs) else: self.assertRaises(TypeError, ch.reduce, func)
def testAttach(self, ch, names, flatten = False, exception = None): ch = Channel.create(ch) if exception: self.assertRaises(exception, ch.attach, *names, **{'flatten': flatten}) else: ch.attach(*names, **{'flatten': flatten}) for i, name in enumerate(names): self.assertListEqual(ch.colAt(i) if not flatten else ch.flatten(i), getattr(ch, name))
def dataProvider_testCollapse(self): # empty self yield Channel.create(), 0, [], True # defaults dir1 = path.join(self.testdir, 'testCollapse') file1 = path.join(dir1, 'testCollapse1.txt') file2 = path.join(dir1, 'testCollapse2.txt') makedirs(dir1) helpers.writeFile(file1) helpers.writeFile(file2) yield Channel.create([file1, file2]), 0, dir1 # Extra cols yield Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')]), 1, ('a1', dir1, 'a2') # No common prefix yield Channel.create([('a1', file1, 'a2'), ('b1', file2, 'b2')]), 0, ('', file1, 'a2')
def dataProvider_testCbind(self): yield [], [], [] yield [(1,2), (3,4)], [5, 6], [(1,2,5,6), (3,4,5,6)] yield [(1,2), (3,4)], [(5, 6,)], [(1,2,5,6), (3,4,5,6)] yield [(1,2), (3,4)], [5], [(1,2,5), (3,4,5)] yield [7,8], [[5, 6], Channel.create(4)], [(7,5,4), (8,6,4)] yield [], [21, 22], [(21, 22)] yield [], [(21, 22, )], [(21, 22)] yield [], [[21, 22], (1, 2, 3)], [(21, 1, 2, 3), (22, 1,2,3)] yield [], [[21, 22], [1, 2, 3]], [], True
def dataProvider_testExpand(self): # empty self yield Channel.create(), 0, [] # defaults dir1 = path.join(self.testdir, 'testExpand') file1 = path.join(dir1, 'testExpand1.txt') file2 = path.join(dir1, 'testExpand2.txt') makedirs(dir1) helpers.writeFile(file1) helpers.writeFile(file2) yield Channel.create(dir1), 0, [file1, file2] # extra columns dir2 = path.join(self.testdir, 'testExpand2') file3 = path.join(dir2, 'testExpand3.txt') file4 = path.join(dir2, 'testExpand4.txt') makedirs(dir2) helpers.writeFile(file3) helpers.writeFile(file4) yield Channel.create(('a', 1, dir2)), 2, [('a', 1, file3), ('a', 1, file4)] # pattern not exists yield Channel.create(('a', 1, dir2)), 2, [], 'a.*' # expand respectively yield Channel.create([('a', 1, dir1), ('b', 2, dir2)]), 2, Channel.create([ ('a', 1, file1), ('a', 1, file2), ('b', 2, file3), ('b', 2, file4), ])
def dataProvider_testFromChannels(self): ch1 = Channel.create([(1, 2), (3, 4)]) ch2 = Channel.create('a') ch3 = Channel.create([5, 6]) outs = [(1, 2, 'a', 5), (3, 4, 'a', 6)] yield [ch1, ch2, ch3], outs ch1 = Channel.create([]) ch2 = Channel.create([]) yield [ch1, ch2], [] ch1 = Channel.create([]) ch2 = Channel.create(1) ch3 = Channel.create([1, 2]) yield [ch1, ch2, ch3], [(1, 1), (1, 2)]
def testFromParams(self, ps, outs=[], exception=False): from pyppl.parameters import Parameter, params outs = Channel.create(outs) for p in ps: params._params[p.name] = p pnames = [p.name for p in ps] if not exception: ch = Channel.fromParams(*pnames) self.assertListEqual(ch, outs) if ch: for i, p in enumerate(ps): self.assertListEqual(getattr(ch, p.name), outs.colAt(i)) else: self.assertRaises(ValueError, Channel.fromParams, *pnames)
def dataProvider_testSplit(self): yield [], True, [] yield [], False, [] yield [1,2,3], True, [[1, 2, 3]] yield [1,2,3], False, [Channel.create([1,2,3])] yield (1,2,3), True, [[1], [2], [3]] yield (1,2,3), False, [Channel.create(1), Channel.create(2), Channel.create(3)] yield [(1,4),(2,5),(3,6)], True, [[1, 2, 3], [4,5,6]] yield [(1,4),(2,5),(3,6)], False, [Channel.create([1,2,3]), Channel.create([4,5,6])]
def test_expand(expand_dirs): dir1, dir2 = expand_dirs file1 = str(dir1 / 'testExpand1.txt') file2 = str(dir1 / 'testExpand2.txt') file3 = str(dir2 / 'testExpand3.txt') file4 = str(dir2 / 'testExpand4.txt') assert Channel.create().expand(col=0) == [] assert Channel.create(dir1).expand(col=0) == [(file1, ), (file2, )] assert Channel.create(('a', 1, dir2)).expand(col=2) == [('a', 1, file3), ('a', 1, file4)] assert Channel.create(('a', 1, dir2)).expand(col=2, pattern='a.*') == [] assert Channel.create([('a', 1, dir1), ('b', 2, dir2)]).expand(col=2) == Channel.create([ ('a', 1, file1), ('a', 1, file2), ('b', 2, file3), ('b', 2, file4), ])
def dataProvider_testInsert(self): ch1 = Channel.create([(1, 2), (3, 4)]) ch2 = Channel.create([5, 6]) # 0-3 yield 0, ch1, ch2, [(5, 1, 2), (6, 3, 4)] yield 1, ch1, ch2, [(1, 5, 2), (3, 6, 4)] yield -1, ch1, ch2, [(1, 5, 2), (3, 6, 4)] yield None, ch1, ch2, [(1, 2, 5), (3, 4, 6)] ch2 = Channel.create(5) # 4-7 yield 0, ch1, ch2, [(5, 1, 2), (5, 3, 4)] yield 1, ch1, ch2, [(1, 5, 2), (3, 5, 4)] yield -1, ch1, ch2, [(1, 5, 2), (3, 5, 4)] yield None, ch1, ch2, [(1, 2, 5), (3, 4, 5)] # 8-19 yield 0, ch1, [5, 6], [(5, 1, 2), (6, 3, 4)] yield 1, ch1, [5, 6], [(1, 5, 2), (3, 6, 4)] yield -1, ch1, [5, 6], [(1, 5, 2), (3, 6, 4)] yield None, ch1, [5, 6], [(1, 2, 5), (3, 4, 6)] yield 0, ch1, (5, 6), [(5, 6, 1, 2), (5, 6, 3, 4)] yield 1, ch1, (5, 6), [(1, 5, 6, 2), (3, 5, 6, 4)] yield -1, ch1, (5, 6), [(1, 5, 6, 2), (3, 5, 6, 4)] yield None, ch1, (5, 6), [(1, 2, 5, 6), (3, 4, 5, 6)] yield 0, ch1, "a", [('a', 1, 2), ('a', 3, 4)] yield 1, ch1, "a", [(1, 'a', 2), (3, 'a', 4)] yield -1, ch1, "a", [(1, 'a', 2), (3, 'a', 4)] yield None, ch1, "a", [(1, 2, 'a'), (3, 4, 'a')] # 20-23 yield 0, ch1, [], ch1 yield 1, ch1, [], ch1 yield -1, ch1, [], ch1 yield None, ch1, [], ch1 ch1 = Channel.create() ch2 = Channel.create([21, 22]) ch3 = 3 ch4 = [41, 42] ch5 = (51, 52) ch6 = "a" # 24-25 yield 1, ch1, ch2, ch2 yield 0, ch1, [ch2, ch3, ch4, ch5, ch6], [(21, 3, 41, 51, 52, 'a'), (22, 3, 42, 51, 52, 'a')], True # 26-29 yield 0, ch1, [], ch1 yield 1, ch1, [], ch1 yield -1, ch1, [], ch1 yield None, ch1, [], ch1 # 30-31 yield None, ch1, [1, [1, 2]], [(1, 1), (1, 2)], True yield None, ch1, [Channel.create(1), Channel.create([1, 2])], [(1, 1), (1, 2)], True # 32 Emptys yield 1, ch1, [[], 1, [], [2, 3]], [(1, 2), (1, 3)], True yield None, Channel.create(), [ Channel.create([4, 5]), Channel.create([1, 2, 3]) ], ValueError, True ch1 = Channel.create([(1, 2), (3, 4)]) yield None, ch1, [1, 2, 3], ValueError # issue 29 yield 0, Channel.create(), '', [('', )]
def testGet(self, ch, idx, outs, exception=None): ch = Channel.create(ch) if exception: self.assertRaises(exception, ch.get, idx) else: self.assertEqual(ch.get(idx), outs)
def testSplit(self, ch, flatten, outs): ch = Channel.create(ch) self.assertListEqual(ch.split(flatten), outs)
def dataProvider_testT(self): yield Channel.create(), [] yield Channel.create([1, 2, 3]), [(1, 2, 3)] yield Channel.create([(1, 2, 3)]), [(1, ), (2, ), (3, )]
def testSlice(self, ch, start, length, outs): ch = Channel.create(ch) outs = Channel.create(outs) self.assertListEqual(ch.slice(start, length), outs)
def testUnique(self, ch, outs): ch = Channel.create(ch) outs = Channel.create(outs) self.assertListEqual(ch.unique(), outs)
def testRowAt(self, ch, row, outs): ch = Channel.create(ch) outs = Channel.create(outs) self.assertListEqual(ch.rowAt(row), outs)
def testColAt(self, ch, col, outs): ch = Channel.create(ch) outs = Channel.create(outs) self.assertListEqual(ch.colAt(col), outs)
def testRepRow(self, ch, n, outs): ch = Channel.create(ch) outs = Channel.create(outs) self.assertListEqual(ch.repRow(n), outs)
def testFlatten(self, ch, col, outs, exception=None): ch = Channel.create(ch) if exception: self.assertRaises(exception, ch.flatten, col) else: self.assertListEqual(ch.flatten(col), outs)
def testFilterCol(self, ch, func, col, outs): ch = Channel.create(ch) ch2 = ch.filterCol(func, col) outs = Channel.create(outs) self.assertListEqual(ch2, outs)