def main():
"handle args and run workflow"
# remove old files
print("Removing step files from previous runs...")
for i in range(1, 4):
stepfile = "step{}.out".format(i)
if os.path.exists(stepfile):
os.remove(stepfile)
sciluigi.run_local(main_task_cls=WF)
'prepare',
PrepareFastq,
data_path=
'/home/aleksandrsl/Desktop/EPAM_project/data/first_try/sequences/')
#fastqc = self.new_task('fastqc', FastQC)
trimmo = self.new_task('trimmo', TrimmoTaskWParameters)
bowtie2 = self.new_task('bowtie2', Bowtie2)
samtools = self.new_task('samtools', Samtools)
mark_duplicates = self.new_task('mark_duplicates', MarkDuplicates)
bqsr = self.new_task('BQSR', BQSR)
#fastqc.in_data = prepare.out_data
trimmo.in_data = prepare.out_data
bowtie2.in_data = trimmo.out_trimmo
samtools.in_data = bowtie2.out_bowtie2
mark_duplicates.in_data = samtools.out_data
BQSR.in_data = mark_duplicates.out_data
# fastqc.in_data = self.paired_fastq
return locals()
#for name, instance in locals().iteritems():
# if issubclass(type(instance), sl.Task):
# log.info('{n}, task id: {taskid}\n{n}, hash: {h}',
# n=name,
# taskid=instance.task_id,
# h=instance.__hash__())
if __name__ == '__main__':
sl.run_local(main_task_cls=FastQCWF)
show=self.show)
_shotBoundaryDetection = self.new_task(
'_shotBoundaryDetection',
_ShotBoundaryDetection,
workdir=self.workdir)
_shotBoundaryDetection.in_video = video.out_put
_shotThreading = self.new_task(
'_shotThreading',
_ShotThreading,
workdir=self.workdir)
_shotThreading.in_video = video.out_put
_shotThreading.in_shot = _shotBoundaryDetection.out_put
baselineShots = self.new_task(
'baselineShots',
BaselineShots,
workdir=self.workdir)
baselineShots.in_video = video.out_put
baselineShots.in_thread = _shotThreading.out_put
return baselineShots
if __name__ == '__main__':
sciluigi.run_local(main_task_cls=ShotWorkflow)
CreateHtmlReport,
dataset_name=self.dataset_name,
train_method=self.train_method,
train_size='80000', #TODO: Is it?
test_size=self.test_size,
svm_gamma=self.svm_gamma,
svm_cost=self.svm_cost,
svm_type=self.svm_type,
svm_kernel_type=self.svm_kernel_type,
replicate_id='N/A',
accounted_project=self.slurm_project)
htmlreport.in_signatures = None
htmlreport.in_sample_traintest_log = None
htmlreport.in_sparse_testdata_log = None
htmlreport.in_sparse_traindata_log = None
htmlreport.in_traindata = None
htmlreport.in_svmmodel = None
htmlreport.in_assess_svm_log = None
htmlreport.in_assess_svm_plot = None
# ========================================================================
# END: Tasks for creating Bioclipse Plugin
# ========================================================================
return datareport
# ====================================================================================================
if __name__ == '__main__':
sl.run_local()
class TestWF(sl.WorkflowTask):
task = luigi.Parameter()
def workflow(self):
t1a = self.new_task('t1a', T1, text='hej_hopp')
t1b = self.new_task('t1b', T1, text='hopp_hej')
mrg1 = self.new_task('mrg1', Merge)
mrg2 = self.new_task('mrg2', Merge)
# Workflow definition
mrg1.in_data1 = t1a.out_data1
mrg1.in_data2 = t1b.out_data1
mrg2.in_data1 = t1b.out_data1
mrg2.in_data2 = t1a.out_data1
for name, instance in locals().iteritems():
if issubclass(type(instance), sl.Task):
log.info('{n}, task id: {taskid}\n{n}, hash: {h}',
n=name,
taskid=instance.task_id,
h=instance.__hash__())
return locals()[self.task]
if __name__ == '__main__':
sl.run_local(main_task_cls=TestWF, cmdline_args=['--task=mrg2'])
# paired_fastq = (fastq, next(iterator))
# print(paired_fastq)
# wf = self.new_task('TestWf', TestWF, paired_fastq=paired_fastq)
# tasks.append(wf)
# return tasks
tasks = []
s = sorted(glob.glob(self.path_to_fastq + '*.fastq.gz'))
for paired_fastq in list(zip(s[::2], s[1::2])):
n = 0
for head_qual in range(25, 32):
trimmo_par = {
'leading': head_qual,
'trailing': 30,
'headcrop': 15,
}
wf = self.new_task('TestWf',
TestWF,
paired_fastq=paired_fastq,
trimmo_par=trimmo_par,
n=n)
#print("I'm launched")
tasks.append(wf)
n += 1
return tasks
if __name__ == '__main__':
#luigi.run(local_scheduler=True, main_task_cls=MetaWF)
sl.run_local(main_task_cls=MetaWF)
with self.in_data().open() as infile, self.out_data().open('w') as outfile:
for line in infile:
outfile.write(line.lower() + '\n')
class MergeFiles(sl.Task):
'''
Merge the results of the programs
'''
# I/O
in_part1 = None
in_part2 = None
def out_merged(self):
return sl.TargetInfo(self, self.in_part1().path + '.merged')
# Impl
def run(self):
self.ex('cat {f1} {f2} > {out}'.format(
f1=self.in_part1().path,
f2=self.in_part2().path,
out=self.out_merged().path))
# ------------------------------------------------------------------------
# Run as script
# ------------------------------------------------------------------------
if __name__ == '__main__':
sl.run_local(main_task_cls=NGITestWF, cmdline_args=['--task=merge'])
log = logging.getLogger('sciluigi-interface')
# ========================================================================
class SimpleWF(sl.WorkflowTask):
path_to_fastq = '/home/aleksandrsl/Desktop/EPAM_project/data/first_try/sequences/' #
s = sorted(glob.glob(path_to_fastq + '*.fastq.gz'))
paired_fastq = list(zip(s[::2], s[1::2]))[0]
def workflow(self):
trimmo = self.new_task('trimmo', TrimmoTask)
fastqc = self.new_task('fastqc', FastqcTask3)
trimmo.inp_data = self.paired_fastq
fastqc.in_upstream = trimmo.out_trimmo
return locals()
#for name, instance in locals().iteritems():
# if issubclass(type(instance), sl.Task):
# log.info('{n}, task id: {taskid}\n{n}, hash: {h}',
# n=name,
# taskid=instance.task_id,
# h=instance.__hash__())
if __name__ == '__main__':
sl.run_local(main_task_cls=SimpleWF, cmdline_args=['--task=fastqc'])
_faceLandmarks.in_tracking = faceTracking.out_put
_faceLandmarks.in_model = _dlibModel.out_put
_openfaceModel = self.new_task(
'_openfaceModel',
_OpenfaceModel,
workdir=self.workdir)
_openface = self.new_task(
'_openface',
_Openface,
workdir=self.workdir)
_openface.in_video = video.out_put
_openface.in_landmarks = _faceLandmarks.out_put
_openface.in_model = _openfaceModel.out_put
faceClustering = self.new_task(
'faceClustering',
FaceClustering,
workdir=self.workdir)
faceClustering.in_video = video.out_put
faceClustering.in_openface = _openface.out_put
return faceClustering
if __name__ == '__main__':
sciluigi.run_local(main_task_cls=FaceWorkflow)
def out_replaced(self):
# As the path to the returned target(info), we
# use the path of the foo file:
return sl.TargetInfo(self, self.in_foo().path + '.bar.txt')
def run(self):
with self.in_foo().open() as in_f:
with self.out_replaced().open('w') as out_f:
# Here we see that we use the parameter self.replacement:
out_f.write(in_f.read().replace('foo', self.replacement))
class MyWorkflow(sl.WorkflowTask):
def workflow(self):
# Initialize tasks:
foowriter = self.new_task('foowriter', MyFooWriter)
fooreplacer = self.new_task('fooreplacer',
MyFooReplacer,
replacement='bar')
# Here we do the *magic*: Connecting outputs to inputs:
fooreplacer.in_foo = foowriter.out_foo
# Return the last task(s) in the workflow chain.
return fooreplacer
if __name__ == '__main__':
sl.run_local(main_task_cls=MyWorkflow)
predict_train = self.new_task('predict_train',
mm.PredictSVMModel,
dataset_name=dataset_name,
replicate_id=replicate_id,
slurminfo=sl.SlurmInfo(
runmode=self.runmode,
project=self.slurm_project,
partition='core',
cores='1',
time='4:00:00',
jobname='predict_train',
threads='1'))
predict_train.in_svmmodel = existing_svm_model.out_file
predict_train.in_sparse_testdata = existing_traindata_ungzipped.out_file
# ------------------------------------------------------------------------
select_idx10 = self.new_task('select_idx10',
mm.SelectPercentIndexValue,
percent_index=10)
select_idx10.in_prediction = predict_train.out_prediction
# ------------------------------------------------------------------------
select_idx90 = self.new_task('select_idx90',
mm.SelectPercentIndexValue,
percent_index=90)
select_idx90.in_prediction = predict_train.out_prediction
# ------------------------------------------------------------------------
return [select_idx10, select_idx90]
if __name__ == '__main__':
sl.run_local(main_task_cls=ColoringWorkflow)
# overrides workflow method
def workflow(self):
# init tasks
foowriter = self.new_task('foowriter', FooWriter)
foo2bar = self.new_task('foo2bar', Foo2Bar)
# connect the tasks into a workflow
foo2bar.in_foo = foowriter.out_foo
return foo2bar
class FooWriter(sl.Task):
# out ports - defined as methods
def out_foo(self):
return sl.TargetInfo(self, 'foo.txt')
# defines what the task does
def run(self):
with self.out_foo().open('w') as outfile:
outfile.write('foo\n')
class Foo2Bar(sl.Task):
in_foo = None
def out_bar(self):
return sl.TargetInfo(self, self.in_foo().path + '.bar.txt')
def run(self):
self.ex('sed "s/foo/bar/g" {infile} > {outfile}'.format(
infile=self.in_foo().path,
outfile=self.out_bar().path))
# main method
if __name__ == '__main__':
sl.run_local(main_task_cls=MyWorkflow)
runmode = luigi.Parameter()
# In-ports
in_lowestrmsd = None
# Out-ports
def out_done(self):
return sl.TargetInfo(self, self.in_lowestrmsd().path + '.mainwf_done')
# Task action
def run(self):
with self.in_lowestrmsd().open() as infile:
records = sl.recordfile_to_dict(infile)
lowest_cost = records['lowest_cost']
self.ex('python wfmm.py MMWorkflow' +
' --dataset-name=%s' % self.dataset_name +
' --run-id=%s' % self.run_id +
' --replicate-id=%s' % self.replicate_id +
' --sampling-method=%s' % self.sampling_method +
' --train-method=%s' % self.train_method +
' --train-size=%s' % self.train_size +
' --test-size=%s' % self.test_size +
' --lin-type=%s' % self.lin_type +
' --lin-cost=%s' % lowest_cost +
' --slurm-project=%s' % self.slurm_project +
' --runmode=%s' % self.runmode)
with self.out_done().open('w') as donefile:
donefile.write('Done!\n')
# ================================================================================
if __name__ == '__main__':
sl.run_local()
class TestWF(sl.WorkflowTask):
task = luigi.Parameter()
def workflow(self):
t1a = self.new_task('t1a', T1, text='hej_hopp')
t1b = self.new_task('t1b', T1, text='hopp_hej')
mrg1 = self.new_task('mrg1', Merge)
mrg2 = self.new_task('mrg2', Merge)
# Workflow definition
mrg1.in_data1 = t1a.out_data1
mrg1.in_data2 = t1b.out_data1
mrg2.in_data1 = t1b.out_data1
mrg2.in_data2 = t1a.out_data1
for name, instance in locals().iteritems():
if issubclass(type(instance), sl.Task):
log.info('{n}, task id: {taskid}\n{n}, hash: {h}'.format(
n = name,
taskid = instance.task_id,
h = instance.__hash__()))
return locals()[self.task]
if __name__ == '__main__':
sl.run_local(main_task_cls=TestWF, cmdline_args=['--task=mrg2'])
pyannote_workflows.tasks.person_discovery_2016.Video,
corpus_dir=self.corpus_dir,
corpus=self.corpus,
show=self.show)
_shotBoundaryDetection = self.new_task('_shotBoundaryDetection',
_ShotBoundaryDetection,
workdir=self.workdir)
_shotBoundaryDetection.in_video = video.out_put
_shotThreading = self.new_task('_shotThreading',
_ShotThreading,
workdir=self.workdir)
_shotThreading.in_video = video.out_put
_shotThreading.in_shot = _shotBoundaryDetection.out_put
baselineShots = self.new_task('baselineShots',
BaselineShots,
workdir=self.workdir)
baselineShots.in_video = video.out_put
baselineShots.in_thread = _shotThreading.out_put
return baselineShots
if __name__ == '__main__':
sciluigi.run_local(main_task_cls=ShotWorkflow)
with self.in_data().open() as infile, self.out_data().open("w") as outfile:
for line in infile:
outfile.write(line.lower() + "\n")
class MergeFiles(sl.Task):
"""
Merge the results of the programs
"""
# I/O
in_part1 = None
in_part2 = None
def out_merged(self):
return sl.TargetInfo(self, self.in_part1().path + ".merged")
# Impl
def run(self):
self.ex(
"cat {f1} {f2} > {out}".format(f1=self.in_part1().path, f2=self.in_part2().path, out=self.out_merged().path)
)
# ------------------------------------------------------------------------
# Run as script
# ------------------------------------------------------------------------
if __name__ == "__main__":
sl.run_local(main_task_cls=NGITestWF, cmdline_args=["--task=merge"])
_dlibModel = self.new_task('_dlibModel',
_DLIBModel,
workdir=self.workdir)
_faceLandmarks.in_video = video.out_put
_faceLandmarks.in_tracking = faceTracking.out_put
_faceLandmarks.in_model = _dlibModel.out_put
_openfaceModel = self.new_task('_openfaceModel',
_OpenfaceModel,
workdir=self.workdir)
_openface = self.new_task('_openface', _Openface, workdir=self.workdir)
_openface.in_video = video.out_put
_openface.in_landmarks = _faceLandmarks.out_put
_openface.in_model = _openfaceModel.out_put
faceClustering = self.new_task('faceClustering',
FaceClustering,
workdir=self.workdir)
faceClustering.in_video = video.out_put
faceClustering.in_openface = _openface.out_put
return faceClustering
if __name__ == '__main__':
sciluigi.run_local(main_task_cls=FaceWorkflow)
predict_train = self.new_task('predict_train',
mm.PredictSVMModel,
dataset_name = dataset_name,
replicate_id = replicate_id,
slurminfo = sl.SlurmInfo(
runmode=self.runmode,
project=self.slurm_project,
partition='core',
cores='1',
time='4:00:00',
jobname='predict_train',
threads='1'
))
predict_train.in_svmmodel = existing_svm_model.out_file
predict_train.in_sparse_testdata = existing_traindata_ungzipped.out_file
# ------------------------------------------------------------------------
select_idx10 = self.new_task('select_idx10',
mm.SelectPercentIndexValue,
percent_index=10)
select_idx10.in_prediction = predict_train.out_prediction
# ------------------------------------------------------------------------
select_idx90 = self.new_task('select_idx90',
mm.SelectPercentIndexValue,
percent_index=90)
select_idx90.in_prediction = predict_train.out_prediction
# ------------------------------------------------------------------------
return [select_idx10, select_idx90]
if __name__ == '__main__':
sl.run_local(main_task_cls=ColoringWorkflow)
