def test1_70(self):
"""Two chained components, with explicit startcomponent"""
#explicit startcomponent is necessary because the *.txt extension is ambiguous here
luiginlp.run(
Voweleater(inputfile='/tmp/test.txt', startcomponent='Lowercaser'))
self.assertTrue(
testfilecontents('/tmp/test.lowercase.novowels.txt', 'ths s tst'))
import luiginlp from luiginlp.modules.frog import Frog luiginlp.run(Frog(inputfile="test.rst",skip='p'))
import luiginlp from luiginlp.modules.frog import Frog from luiginlp.engine import Parallel, ComponentParameters luiginlp.run(Parallel(component='Frog',inputfiles="test.rst,test2.rst",component_parameters=ComponentParameters(skip='p')))
class ScaleTest(StandardWorkflowComponent):
n = IntParameter()
def accepts(self):
return InputFormat(self,
format_id='txtdir',
extension='txtdir',
directory=True)
def autosetup(self):
return ScaleTestTask
if __name__ == '__main__':
workdir = sys.argv[1]
n = int(sys.argv[2])
if not os.path.exists(workdir):
os.mkdir(workdir)
print("Preparing, making input files", file=sys.stderr)
for i in range(1, n + 1):
print(i, file=sys.stderr)
filename = workdir + '/' + str(i) + '.txt'
if not os.path.exists(filename):
with open(filename, 'w', encoding='utf-8') as f:
f.write("test")
luiginlp.run(ScaleTest(inputfile=workdir, n=n), workers=5)
import luiginlp from luiginlp.modules.frog import Frog luiginlp.run(Frog(inputfile="test.rst", skip='p'))
import luiginlp from luiginlp.modules.frog import Frog luiginlp.run(Frog(inputfile="test.rst"), Frog(inputfile="test2.rst"))
def test3_10(self):
"""Parallelisation on directory input (invokes two chained components, one task per component, for each file)"""
luiginlp.run(LowercaseVoweleaterDir2(inputfile='/tmp/corpus.txtdir'),
workers=5)
def test2_20(self):
"""Parallelisation on directory input (invokes two chained components, one task per component, for each file)"""
luiginlp.run(LowercaseVoweleaterDir2(inputfile='/tmp/corpus.txtdir'))
self.assertTrue(
testdircontents('/tmp/corpus.lcnv.txtdir',
'lowercase.novowels.txt', 'ths s tst'))
def test1_60(self):
"""Two chained tasks in single component"""
luiginlp.run(LowercaseVoweleater(inputfile='/tmp/test.txt'))
self.assertTrue(
testfilecontents('/tmp/test.lowercase.novowels.txt', 'ths s tst'))
def test1_50(self):
"""Single task in single component, task calling external tool"""
luiginlp.run(Voweleater(inputfile='/tmp/test.txt'))
self.assertTrue(
testfilecontents('/tmp/test.novowels.txt', 'THS S TST'))
def test1_40(self):
"""Single task in single component, with setup, passing parameter with autopass"""
luiginlp.run(Lowercaser2(inputfile='/tmp/test.txt', encoding='ascii'))
self.assertTrue(
testfilecontents('/tmp/test.lowercase.txt', 'this is a test'))
def test1_30(self):
"""Single task in single component, with setup"""
luiginlp.run(Lowercaser2(inputfile='/tmp/test.txt'))
self.assertTrue(
testfilecontents('/tmp/test.lowercase.txt', 'this is a test'))
import luiginlp
from luiginlp.modules.frog import Frog
from luiginlp.engine import Parallel, PassParameters
luiginlp.run(
Parallel(component='Frog',
inputfiles="test.rst,test2.rst",
passparameters=PassParameters(skip='p')))
class ScaleTest(StandardWorkflowComponent):
n = IntParameter()
def accepts(self):
return InputFormat(self, format_id='txtdir',extension='txtdir', directory=True)
def autosetup(self):
return ScaleTestTask
if __name__ == '__main__':
workdir = sys.argv[1]
n = int(sys.argv[2])
if not os.path.exists(workdir):
os.mkdir(workdir)
print("Preparing, making input files",file=sys.stderr)
for i in range(1, n+1):
print(i,file=sys.stderr)
filename = workdir + '/' + str(i) + '.txt'
if not os.path.exists(filename):
with open(filename,'w',encoding='utf-8') as f:
f.write("test")
luiginlp.run(ScaleTest(inputfile=workdir,n=n),workers=5)