def workflow(self):
# Rsync a folder
rsync = sl.new_task(
"rsync", RSyncAFolder, self, src_dir_path="data/afolder", dest_dir_path="data/afolder_rsynced"
)
# Run a program that takes 10 minutes (seconds)
run10min = sl.new_task("run10min", Run10MinuteSleep, self)
run10min.in_upstream = rsync.out_destdir
# Do a web request
webreq = sl.new_task("run10min", DoWebRequest, self)
webreq.in_upstream = run10min.out_doneflag
# Split a file
rawdata = sl.new_task("rawdata", ExistingData, self, file_name="acgt.txt")
split = sl.new_task("run10min", SplitAFile, self)
split.in_data = rawdata.out_acgt
# Run the same task on the two splits
dosth1 = sl.new_task("dosth1", DoSomething, self)
dosth1.in_data = split.out_part1
dosth2 = sl.new_task("dosth2", DoSomething, self)
dosth2.in_data = split.out_part2
# Merge the results
merge = sl.new_task("merge", MergeFiles, self)
merge.in_part1 = dosth1.out_data
merge.in_part2 = dosth2.out_data
return locals()[self.task]
def workflow(self):
# Rsync a folder
rsync = sl.new_task('rsync', RSyncAFolder, self,
src_dir_path='data/afolder',
dest_dir_path='data/afolder_rsynced')
# Run a program that takes 10 minutes (seconds)
run10min = sl.new_task('run10min', Run10MinuteSleep, self)
run10min.in_upstream = rsync.out_destdir
# Do a web request
webreq = sl.new_task('run10min', DoWebRequest, self)
webreq.in_upstream = run10min.out_doneflag
# Split a file
rawdata = sl.new_task('rawdata', ExistingData, self,
file_name='acgt.txt')
split = sl.new_task('run10min', SplitAFile, self)
split.in_data = rawdata.out_acgt
# Run the same task on the two splits
dosth1 = sl.new_task('dosth1', DoSomething, self)
dosth1.in_data = split.out_part1
dosth2 = sl.new_task('dosth2', DoSomething, self)
dosth2.in_data = split.out_part2
# Merge the results
merge = sl.new_task('merge', MergeFiles, self)
merge.in_part1 = dosth1.out_data
merge.in_part2 = dosth2.out_data
return locals()[self.task]
def new_task(self, instance_name, cls, **kwargs):
"""
Create new task instance, and link it to the current workflow.
"""
newtask = sciluigi.new_task(instance_name, cls, self, **kwargs)
self._tasks[instance_name] = newtask
return newtask
def new_task(self, instance_name, cls, **kwargs):
'''
Create new task instance, and link it to the current workflow.
'''
newtask = sciluigi.new_task(instance_name, cls, self, **kwargs)
self._tasks[instance_name] = newtask
return newtask