def main(): "handle args and run workflow" # remove old files print("Removing step files from previous runs...") for i in range(1, 4): stepfile = "step{}.out".format(i) if os.path.exists(stepfile): os.remove(stepfile) sciluigi.run_local(main_task_cls=WF)
'prepare', PrepareFastq, data_path= '/home/aleksandrsl/Desktop/EPAM_project/data/first_try/sequences/') #fastqc = self.new_task('fastqc', FastQC) trimmo = self.new_task('trimmo', TrimmoTaskWParameters) bowtie2 = self.new_task('bowtie2', Bowtie2) samtools = self.new_task('samtools', Samtools) mark_duplicates = self.new_task('mark_duplicates', MarkDuplicates) bqsr = self.new_task('BQSR', BQSR) #fastqc.in_data = prepare.out_data trimmo.in_data = prepare.out_data bowtie2.in_data = trimmo.out_trimmo samtools.in_data = bowtie2.out_bowtie2 mark_duplicates.in_data = samtools.out_data BQSR.in_data = mark_duplicates.out_data # fastqc.in_data = self.paired_fastq return locals() #for name, instance in locals().iteritems(): # if issubclass(type(instance), sl.Task): # log.info('{n}, task id: {taskid}\n{n}, hash: {h}', # n=name, # taskid=instance.task_id, # h=instance.__hash__()) if __name__ == '__main__': sl.run_local(main_task_cls=FastQCWF)
show=self.show) _shotBoundaryDetection = self.new_task( '_shotBoundaryDetection', _ShotBoundaryDetection, workdir=self.workdir) _shotBoundaryDetection.in_video = video.out_put _shotThreading = self.new_task( '_shotThreading', _ShotThreading, workdir=self.workdir) _shotThreading.in_video = video.out_put _shotThreading.in_shot = _shotBoundaryDetection.out_put baselineShots = self.new_task( 'baselineShots', BaselineShots, workdir=self.workdir) baselineShots.in_video = video.out_put baselineShots.in_thread = _shotThreading.out_put return baselineShots if __name__ == '__main__': sciluigi.run_local(main_task_cls=ShotWorkflow)
CreateHtmlReport, dataset_name=self.dataset_name, train_method=self.train_method, train_size='80000', #TODO: Is it? test_size=self.test_size, svm_gamma=self.svm_gamma, svm_cost=self.svm_cost, svm_type=self.svm_type, svm_kernel_type=self.svm_kernel_type, replicate_id='N/A', accounted_project=self.slurm_project) htmlreport.in_signatures = None htmlreport.in_sample_traintest_log = None htmlreport.in_sparse_testdata_log = None htmlreport.in_sparse_traindata_log = None htmlreport.in_traindata = None htmlreport.in_svmmodel = None htmlreport.in_assess_svm_log = None htmlreport.in_assess_svm_plot = None # ======================================================================== # END: Tasks for creating Bioclipse Plugin # ======================================================================== return datareport # ==================================================================================================== if __name__ == '__main__': sl.run_local()
class TestWF(sl.WorkflowTask): task = luigi.Parameter() def workflow(self): t1a = self.new_task('t1a', T1, text='hej_hopp') t1b = self.new_task('t1b', T1, text='hopp_hej') mrg1 = self.new_task('mrg1', Merge) mrg2 = self.new_task('mrg2', Merge) # Workflow definition mrg1.in_data1 = t1a.out_data1 mrg1.in_data2 = t1b.out_data1 mrg2.in_data1 = t1b.out_data1 mrg2.in_data2 = t1a.out_data1 for name, instance in locals().iteritems(): if issubclass(type(instance), sl.Task): log.info('{n}, task id: {taskid}\n{n}, hash: {h}', n=name, taskid=instance.task_id, h=instance.__hash__()) return locals()[self.task] if __name__ == '__main__': sl.run_local(main_task_cls=TestWF, cmdline_args=['--task=mrg2'])
# paired_fastq = (fastq, next(iterator)) # print(paired_fastq) # wf = self.new_task('TestWf', TestWF, paired_fastq=paired_fastq) # tasks.append(wf) # return tasks tasks = [] s = sorted(glob.glob(self.path_to_fastq + '*.fastq.gz')) for paired_fastq in list(zip(s[::2], s[1::2])): n = 0 for head_qual in range(25, 32): trimmo_par = { 'leading': head_qual, 'trailing': 30, 'headcrop': 15, } wf = self.new_task('TestWf', TestWF, paired_fastq=paired_fastq, trimmo_par=trimmo_par, n=n) #print("I'm launched") tasks.append(wf) n += 1 return tasks if __name__ == '__main__': #luigi.run(local_scheduler=True, main_task_cls=MetaWF) sl.run_local(main_task_cls=MetaWF)
with self.in_data().open() as infile, self.out_data().open('w') as outfile: for line in infile: outfile.write(line.lower() + '\n') class MergeFiles(sl.Task): ''' Merge the results of the programs ''' # I/O in_part1 = None in_part2 = None def out_merged(self): return sl.TargetInfo(self, self.in_part1().path + '.merged') # Impl def run(self): self.ex('cat {f1} {f2} > {out}'.format( f1=self.in_part1().path, f2=self.in_part2().path, out=self.out_merged().path)) # ------------------------------------------------------------------------ # Run as script # ------------------------------------------------------------------------ if __name__ == '__main__': sl.run_local(main_task_cls=NGITestWF, cmdline_args=['--task=merge'])
log = logging.getLogger('sciluigi-interface') # ======================================================================== class SimpleWF(sl.WorkflowTask): path_to_fastq = '/home/aleksandrsl/Desktop/EPAM_project/data/first_try/sequences/' # s = sorted(glob.glob(path_to_fastq + '*.fastq.gz')) paired_fastq = list(zip(s[::2], s[1::2]))[0] def workflow(self): trimmo = self.new_task('trimmo', TrimmoTask) fastqc = self.new_task('fastqc', FastqcTask3) trimmo.inp_data = self.paired_fastq fastqc.in_upstream = trimmo.out_trimmo return locals() #for name, instance in locals().iteritems(): # if issubclass(type(instance), sl.Task): # log.info('{n}, task id: {taskid}\n{n}, hash: {h}', # n=name, # taskid=instance.task_id, # h=instance.__hash__()) if __name__ == '__main__': sl.run_local(main_task_cls=SimpleWF, cmdline_args=['--task=fastqc'])
_faceLandmarks.in_tracking = faceTracking.out_put _faceLandmarks.in_model = _dlibModel.out_put _openfaceModel = self.new_task( '_openfaceModel', _OpenfaceModel, workdir=self.workdir) _openface = self.new_task( '_openface', _Openface, workdir=self.workdir) _openface.in_video = video.out_put _openface.in_landmarks = _faceLandmarks.out_put _openface.in_model = _openfaceModel.out_put faceClustering = self.new_task( 'faceClustering', FaceClustering, workdir=self.workdir) faceClustering.in_video = video.out_put faceClustering.in_openface = _openface.out_put return faceClustering if __name__ == '__main__': sciluigi.run_local(main_task_cls=FaceWorkflow)
def out_replaced(self): # As the path to the returned target(info), we # use the path of the foo file: return sl.TargetInfo(self, self.in_foo().path + '.bar.txt') def run(self): with self.in_foo().open() as in_f: with self.out_replaced().open('w') as out_f: # Here we see that we use the parameter self.replacement: out_f.write(in_f.read().replace('foo', self.replacement)) class MyWorkflow(sl.WorkflowTask): def workflow(self): # Initialize tasks: foowriter = self.new_task('foowriter', MyFooWriter) fooreplacer = self.new_task('fooreplacer', MyFooReplacer, replacement='bar') # Here we do the *magic*: Connecting outputs to inputs: fooreplacer.in_foo = foowriter.out_foo # Return the last task(s) in the workflow chain. return fooreplacer if __name__ == '__main__': sl.run_local(main_task_cls=MyWorkflow)
predict_train = self.new_task('predict_train', mm.PredictSVMModel, dataset_name=dataset_name, replicate_id=replicate_id, slurminfo=sl.SlurmInfo( runmode=self.runmode, project=self.slurm_project, partition='core', cores='1', time='4:00:00', jobname='predict_train', threads='1')) predict_train.in_svmmodel = existing_svm_model.out_file predict_train.in_sparse_testdata = existing_traindata_ungzipped.out_file # ------------------------------------------------------------------------ select_idx10 = self.new_task('select_idx10', mm.SelectPercentIndexValue, percent_index=10) select_idx10.in_prediction = predict_train.out_prediction # ------------------------------------------------------------------------ select_idx90 = self.new_task('select_idx90', mm.SelectPercentIndexValue, percent_index=90) select_idx90.in_prediction = predict_train.out_prediction # ------------------------------------------------------------------------ return [select_idx10, select_idx90] if __name__ == '__main__': sl.run_local(main_task_cls=ColoringWorkflow)
# overrides workflow method def workflow(self): # init tasks foowriter = self.new_task('foowriter', FooWriter) foo2bar = self.new_task('foo2bar', Foo2Bar) # connect the tasks into a workflow foo2bar.in_foo = foowriter.out_foo return foo2bar class FooWriter(sl.Task): # out ports - defined as methods def out_foo(self): return sl.TargetInfo(self, 'foo.txt') # defines what the task does def run(self): with self.out_foo().open('w') as outfile: outfile.write('foo\n') class Foo2Bar(sl.Task): in_foo = None def out_bar(self): return sl.TargetInfo(self, self.in_foo().path + '.bar.txt') def run(self): self.ex('sed "s/foo/bar/g" {infile} > {outfile}'.format( infile=self.in_foo().path, outfile=self.out_bar().path)) # main method if __name__ == '__main__': sl.run_local(main_task_cls=MyWorkflow)
runmode = luigi.Parameter() # In-ports in_lowestrmsd = None # Out-ports def out_done(self): return sl.TargetInfo(self, self.in_lowestrmsd().path + '.mainwf_done') # Task action def run(self): with self.in_lowestrmsd().open() as infile: records = sl.recordfile_to_dict(infile) lowest_cost = records['lowest_cost'] self.ex('python wfmm.py MMWorkflow' + ' --dataset-name=%s' % self.dataset_name + ' --run-id=%s' % self.run_id + ' --replicate-id=%s' % self.replicate_id + ' --sampling-method=%s' % self.sampling_method + ' --train-method=%s' % self.train_method + ' --train-size=%s' % self.train_size + ' --test-size=%s' % self.test_size + ' --lin-type=%s' % self.lin_type + ' --lin-cost=%s' % lowest_cost + ' --slurm-project=%s' % self.slurm_project + ' --runmode=%s' % self.runmode) with self.out_done().open('w') as donefile: donefile.write('Done!\n') # ================================================================================ if __name__ == '__main__': sl.run_local()
class TestWF(sl.WorkflowTask): task = luigi.Parameter() def workflow(self): t1a = self.new_task('t1a', T1, text='hej_hopp') t1b = self.new_task('t1b', T1, text='hopp_hej') mrg1 = self.new_task('mrg1', Merge) mrg2 = self.new_task('mrg2', Merge) # Workflow definition mrg1.in_data1 = t1a.out_data1 mrg1.in_data2 = t1b.out_data1 mrg2.in_data1 = t1b.out_data1 mrg2.in_data2 = t1a.out_data1 for name, instance in locals().iteritems(): if issubclass(type(instance), sl.Task): log.info('{n}, task id: {taskid}\n{n}, hash: {h}'.format( n = name, taskid = instance.task_id, h = instance.__hash__())) return locals()[self.task] if __name__ == '__main__': sl.run_local(main_task_cls=TestWF, cmdline_args=['--task=mrg2'])
pyannote_workflows.tasks.person_discovery_2016.Video, corpus_dir=self.corpus_dir, corpus=self.corpus, show=self.show) _shotBoundaryDetection = self.new_task('_shotBoundaryDetection', _ShotBoundaryDetection, workdir=self.workdir) _shotBoundaryDetection.in_video = video.out_put _shotThreading = self.new_task('_shotThreading', _ShotThreading, workdir=self.workdir) _shotThreading.in_video = video.out_put _shotThreading.in_shot = _shotBoundaryDetection.out_put baselineShots = self.new_task('baselineShots', BaselineShots, workdir=self.workdir) baselineShots.in_video = video.out_put baselineShots.in_thread = _shotThreading.out_put return baselineShots if __name__ == '__main__': sciluigi.run_local(main_task_cls=ShotWorkflow)
with self.in_data().open() as infile, self.out_data().open("w") as outfile: for line in infile: outfile.write(line.lower() + "\n") class MergeFiles(sl.Task): """ Merge the results of the programs """ # I/O in_part1 = None in_part2 = None def out_merged(self): return sl.TargetInfo(self, self.in_part1().path + ".merged") # Impl def run(self): self.ex( "cat {f1} {f2} > {out}".format(f1=self.in_part1().path, f2=self.in_part2().path, out=self.out_merged().path) ) # ------------------------------------------------------------------------ # Run as script # ------------------------------------------------------------------------ if __name__ == "__main__": sl.run_local(main_task_cls=NGITestWF, cmdline_args=["--task=merge"])
_dlibModel = self.new_task('_dlibModel', _DLIBModel, workdir=self.workdir) _faceLandmarks.in_video = video.out_put _faceLandmarks.in_tracking = faceTracking.out_put _faceLandmarks.in_model = _dlibModel.out_put _openfaceModel = self.new_task('_openfaceModel', _OpenfaceModel, workdir=self.workdir) _openface = self.new_task('_openface', _Openface, workdir=self.workdir) _openface.in_video = video.out_put _openface.in_landmarks = _faceLandmarks.out_put _openface.in_model = _openfaceModel.out_put faceClustering = self.new_task('faceClustering', FaceClustering, workdir=self.workdir) faceClustering.in_video = video.out_put faceClustering.in_openface = _openface.out_put return faceClustering if __name__ == '__main__': sciluigi.run_local(main_task_cls=FaceWorkflow)
predict_train = self.new_task('predict_train', mm.PredictSVMModel, dataset_name = dataset_name, replicate_id = replicate_id, slurminfo = sl.SlurmInfo( runmode=self.runmode, project=self.slurm_project, partition='core', cores='1', time='4:00:00', jobname='predict_train', threads='1' )) predict_train.in_svmmodel = existing_svm_model.out_file predict_train.in_sparse_testdata = existing_traindata_ungzipped.out_file # ------------------------------------------------------------------------ select_idx10 = self.new_task('select_idx10', mm.SelectPercentIndexValue, percent_index=10) select_idx10.in_prediction = predict_train.out_prediction # ------------------------------------------------------------------------ select_idx90 = self.new_task('select_idx90', mm.SelectPercentIndexValue, percent_index=90) select_idx90.in_prediction = predict_train.out_prediction # ------------------------------------------------------------------------ return [select_idx10, select_idx90] if __name__ == '__main__': sl.run_local(main_task_cls=ColoringWorkflow)