Ejemplo n.º 1
0
    def workflow(self):
        # Rsync a folder
        rsync = sl.new_task(
            "rsync", RSyncAFolder, self, src_dir_path="data/afolder", dest_dir_path="data/afolder_rsynced"
        )

        # Run a program that takes 10 minutes (seconds)
        run10min = sl.new_task("run10min", Run10MinuteSleep, self)
        run10min.in_upstream = rsync.out_destdir

        # Do a web request
        webreq = sl.new_task("run10min", DoWebRequest, self)
        webreq.in_upstream = run10min.out_doneflag

        # Split a file
        rawdata = sl.new_task("rawdata", ExistingData, self, file_name="acgt.txt")

        split = sl.new_task("run10min", SplitAFile, self)
        split.in_data = rawdata.out_acgt

        # Run the same task on the two splits
        dosth1 = sl.new_task("dosth1", DoSomething, self)
        dosth1.in_data = split.out_part1

        dosth2 = sl.new_task("dosth2", DoSomething, self)
        dosth2.in_data = split.out_part2

        # Merge the results
        merge = sl.new_task("merge", MergeFiles, self)
        merge.in_part1 = dosth1.out_data
        merge.in_part2 = dosth2.out_data

        return locals()[self.task]
Ejemplo n.º 2
0
    def workflow(self):
        # Rsync a folder
        rsync = sl.new_task('rsync', RSyncAFolder, self,
                            src_dir_path='data/afolder',
                            dest_dir_path='data/afolder_rsynced')

        # Run a program that takes 10 minutes (seconds)
        run10min = sl.new_task('run10min', Run10MinuteSleep, self)
        run10min.in_upstream = rsync.out_destdir

        # Do a web request
        webreq = sl.new_task('run10min', DoWebRequest, self)
        webreq.in_upstream = run10min.out_doneflag

        # Split a file
        rawdata = sl.new_task('rawdata', ExistingData, self,
                              file_name='acgt.txt')

        split = sl.new_task('run10min', SplitAFile, self)
        split.in_data = rawdata.out_acgt

        # Run the same task on the two splits
        dosth1 = sl.new_task('dosth1', DoSomething, self)
        dosth1.in_data = split.out_part1

        dosth2 = sl.new_task('dosth2', DoSomething, self)
        dosth2.in_data = split.out_part2

        # Merge the results
        merge = sl.new_task('merge', MergeFiles, self)
        merge.in_part1 = dosth1.out_data
        merge.in_part2 = dosth2.out_data

        return locals()[self.task]
Ejemplo n.º 3
0
 def new_task(self, instance_name, cls, **kwargs):
     """
     Create new task instance, and link it to the current workflow.
     """
     newtask = sciluigi.new_task(instance_name, cls, self, **kwargs)
     self._tasks[instance_name] = newtask
     return newtask
Ejemplo n.º 4
0
 def new_task(self, instance_name, cls, **kwargs):
     '''
     Create new task instance, and link it to the current workflow.
     '''
     newtask = sciluigi.new_task(instance_name, cls, self, **kwargs)
     self._tasks[instance_name] = newtask
     return newtask