Python Task Exemples

Exemple #1

#================================================================================
#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component('kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(__pipeline__parameters=None, __pipeline__docker_args='--runtime=nvidia --shm-size 8G', __pipeline__log_file_location='/home/molinux01/cchen/ml/logs/test/log_my_test_20200305-193527.txt', __pipeline__config_file_location='/home/molinux01/cchen/ml/pipeline/benchmark_pytorch_config.json', __pipeline__docker_image='cchen/benchmark_pytorch_evaluation', __pipeline__output_files=None, )
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir, args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})

#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(TASK_A_component.component_name, 'TASK_A', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def kronos_component_docker_TASK_A_function(*inargs):

Exemple #2

Fichier : SET_0_pipeline.py Projet : fireofearth/pipeline

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_singularity import component_main as kronos_component_singularity_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
AUGMENT_DATA_component = kronos_component_singularity_main.Component(
    'kronos_component_singularity', component_parent_dir=args.components_dir)
AUGMENT_DATA_task = Task('AUGMENT_DATA', AUGMENT_DATA_component)
AUGMENT_DATA_task.update_comp_args(
    __pipeline__singularity_image=
    '/projects/ovcare/classification/src/docker_augment_data/augmentdata-latest.simg',
    __pipeline__patch_location='/projects/ovcare/classification/1024_Patches',
    __pipeline__augmented_location='/projects/ovcare/classification/1024_Aug',
    __pipeline__parameters=None,
    __pipeline__output_files=None,
)
AUGMENT_DATA_prefix = rm.get_filename_prefix('AUGMENT_DATA')
AUGMENT_DATA_task.update_comp_output_filenames(AUGMENT_DATA_prefix,
                                               rm.outputs_dir, args.no_prefix)
AUGMENT_DATA_task.update_comp_env_vars({})
AUGMENT_DATA_task.update_comp_reqs({'singularity': 'singularity'})

SORT_PATCHES_component = kronos_component_singularity_main.Component(

Exemple #3

Fichier : SET_0_work_file_example.py Projet : fireofearth/pipeline

#================================================================================
#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component('kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(__pipeline__work_location='/home/molinux01/mnursey/workfiles', __pipeline__parameters=None, __pipeline__N=3, __pipeline__output_files=None, __pipeline__docker_image='mnursey/divide_work', __pipeline__data_location='/home/molinux01/mnursey/patches', )
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir, args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})

#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(TASK_A_component.component_name, 'TASK_A', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def kronos_component_docker_TASK_A_function(*inargs):

Exemple #4

Fichier : SET_0_test_run_create_training_files_setup.py Projet : fireofearth/pipeline

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component(
    'kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(
    __pipeline__number_of_patches='1000_1000_1000',
    __pipeline__testing_chunks='6_7',
    __pipeline__parameters=None,
    __pipeline__chunk_file_location=
    '/home/molinux01/cchen/ml/test_data_chunks.txt',
    __pipeline__training_chunks='0_1_2_3_4',
    __pipeline__output_location=
    '/home/molinux01/cchen/ml/test_data_training_chunks.txt',
    __pipeline__seed=1,
    __pipeline__validation_chunks='5',
    __pipeline__docker_image='cchen/create_training_files',
    __pipeline__output_files=None,
)
TASK_A_prefix = rm.get_filename_prefix('TASK_A')

Exemple #5

Fichier : docker.py Projet : fireofearth/pipeline

#================================================================================
#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from AnnoAndImageToPatch import component_main as AnnoAndImageToPatch_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
__TASK_1___component = AnnoAndImageToPatch_main.Component('AnnoAndImageToPatch', component_parent_dir=args.components_dir)
__TASK_1___task = Task('__TASK_1__', __TASK_1___component)
__TASK_1___task.update_comp_args(__pipeline__image_location="('__SHARED__','img_loc')", __pipeline__patch_location="('__SHARED__','patch_loc')", __pipeline__input_files=None, __pipeline__annotation_location="('__SHARED__','anno_loc')", __pipeline__output_files=None, )
__TASK_1___prefix = rm.get_filename_prefix('__TASK_1__')
__TASK_1___task.update_comp_output_filenames(__TASK_1___prefix, rm.outputs_dir, args.no_prefix)
__TASK_1___task.update_comp_env_vars({})
__TASK_1___task.update_comp_reqs({'docker': '__REQUIRED__'})

#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(__TASK_1___component.component_name, '__TASK_1__', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def AnnoAndImageToPatch___TASK_1___function(*inargs):

Exemple #6

Fichier : SET_0_test_run_sort_patches_by_label_setup.py Projet : fireofearth/pipeline

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component(
    'kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(
    __pipeline__patch_location='/home/molinux01/cchen/ml/test_data',
    __pipeline__parameters=None,
    __pipeline__sorted_location='/home/molinux01/cchen/ml/test_data_sorted',
    __pipeline__labels_file_location=
    '/home/molinux01/cchen/ml/test_data_labels_file.txt',
    __pipeline__docker_image='cchen/sort_patches_by_label',
    __pipeline__output_files=None,
)
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir,
                                         args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})

Exemple #7

Fichier : __shared__only___AnnotateImageWorkFlow.py Projet : fireofearth/pipeline

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from AnnoAndImageToPatch import component_main as AnnoAndImageToPatch_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
__TASK_1___component = AnnoAndImageToPatch_main.Component(
    'AnnoAndImageToPatch', component_parent_dir=args.components_dir)
__TASK_1___task = Task('__TASK_1__', __TASK_1___component)
__TASK_1___task.update_comp_args(
    __pipeline__image_location='/aim1/Data/given_data/cancer',
    __pipeline__patch_location='/aim1/Data/eighty_annotations',
    __pipeline__input_files=None,
    __pipeline__annotation_location='/aim1/Data/given_data',
    __pipeline__output_files=None,
)
__TASK_1___prefix = rm.get_filename_prefix('__TASK_1__')
__TASK_1___task.update_comp_output_filenames(__TASK_1___prefix, rm.outputs_dir,
                                             args.no_prefix)
__TASK_1___task.update_comp_env_vars({})
__TASK_1___task.update_comp_reqs({'docker': '/usr/bin/docker'})


#================================================================================

Exemple #8

Fichier : SET_0_downsample_example_setup.py Projet : fireofearth/pipeline

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component(
    'kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(
    __pipeline__config_file_location=
    '/home/molinux01/mnursey/example_pipeline/downsample_config.json',
    __pipeline__docker_image='mnursey/downsample_data',
    __pipeline__parameters=None,
    __pipeline__output_files=None,
)
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir,
                                         args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})


#================================================================================

Exemple #9

Fichier : SET_0_test_run_create_groups_setup.py Projet : fireofearth/pipeline

#================================================================================
#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component('kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(__pipeline__config_file_location='/home/molinux01/cchen/ml/pipeline/create_groups_config.json', __pipeline__docker_image='cchen/create_groups', __pipeline__parameters=None, __pipeline__output_files=None, )
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir, args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})

#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(TASK_A_component.component_name, 'TASK_A', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def kronos_component_docker_TASK_A_function(*inargs):

Exemple #10

#environment preparations
#--------------------------------------------------------------------------------
sys.path.insert(0, args.components_dir)
job_rcs = Queue()

#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main

#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component(
    'kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(
    __pipeline__parameters=None,
    __pipeline__test_results_location='/home/molinux01/cchen/ml/results',
    __pipeline__output_location=
    '/home/molinux01/cchen/ml/formatted_results.txt',
    __pipeline__group_by='test_name',
    __pipeline__docker_image='cchen/format_results',
    __pipeline__output_files=None,
)
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir,
                                         args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})