#--------------------------------------------------------------------------------
AUGMENT_DATA_component = kronos_component_singularity_main.Component(
'kronos_component_singularity', component_parent_dir=args.components_dir)
AUGMENT_DATA_task = Task('AUGMENT_DATA', AUGMENT_DATA_component)
AUGMENT_DATA_task.update_comp_args(
__pipeline__singularity_image=
'/projects/ovcare/classification/src/docker_augment_data/augmentdata-latest.simg',
__pipeline__patch_location='/projects/ovcare/classification/1024_Patches',
__pipeline__augmented_location='/projects/ovcare/classification/1024_Aug',
__pipeline__parameters=None,
__pipeline__output_files=None,
)
AUGMENT_DATA_prefix = rm.get_filename_prefix('AUGMENT_DATA')
AUGMENT_DATA_task.update_comp_output_filenames(AUGMENT_DATA_prefix,
rm.outputs_dir, args.no_prefix)
AUGMENT_DATA_task.update_comp_env_vars({})
AUGMENT_DATA_task.update_comp_reqs({'singularity': 'singularity'})
SORT_PATCHES_component = kronos_component_singularity_main.Component(
'kronos_component_singularity', component_parent_dir=args.components_dir)
SORT_PATCHES_task = Task('SORT_PATCHES', SORT_PATCHES_component)
SORT_PATCHES_task.update_comp_args(
__pipeline__patch_location='/projects/ovcare/classification/1024_Aug',
__pipeline__parameters=None,
__pipeline__sorted_location='/projects/ovcare/classification/1024_Sorted',
__pipeline__labels_file_location=
'/projects/ovcare/classification/labels.txt',
__pipeline__singularity_image=
'/projects/ovcare/classification/src/docker_sort_patches_by_label/sortpatchesbylabel.simg',
__pipeline__output_files=None,
)
job_rcs = Queue()
#================================================================================
#import components
#--------------------------------------------------------------------------------
from kronos_component_docker import component_main as kronos_component_docker_main
#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
TASK_A_component = kronos_component_docker_main.Component('kronos_component_docker', component_parent_dir=args.components_dir)
TASK_A_task = Task('TASK_A', TASK_A_component)
TASK_A_task.update_comp_args(__pipeline__parameters=None, __pipeline__docker_args='--runtime=nvidia --shm-size 8G', __pipeline__log_file_location='/home/molinux01/cchen/ml/logs/test/log_my_test_20200305-193527.txt', __pipeline__config_file_location='/home/molinux01/cchen/ml/pipeline/benchmark_pytorch_config.json', __pipeline__docker_image='cchen/benchmark_pytorch_evaluation', __pipeline__output_files=None, )
TASK_A_prefix = rm.get_filename_prefix('TASK_A')
TASK_A_task.update_comp_output_filenames(TASK_A_prefix, rm.outputs_dir, args.no_prefix)
TASK_A_task.update_comp_env_vars({})
TASK_A_task.update_comp_reqs({'docker': 'docker'})
#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(TASK_A_component.component_name, 'TASK_A', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def kronos_component_docker_TASK_A_function(*inargs):
component_name, task_name, _ = inargs
print '%s for %s started in %s pipeline' % (task_name, component_name, args.pipeline_name)
run_script = rm.generate_script(TASK_A_task, None, None)
job_name = rm.get_filename_prefix(task_name)
job_rcs = Queue()
#================================================================================
#import components
#--------------------------------------------------------------------------------
from AnnoAndImageToPatch import component_main as AnnoAndImageToPatch_main
#================================================================================
#generating tasks
#--------------------------------------------------------------------------------
__TASK_1___component = AnnoAndImageToPatch_main.Component('AnnoAndImageToPatch', component_parent_dir=args.components_dir)
__TASK_1___task = Task('__TASK_1__', __TASK_1___component)
__TASK_1___task.update_comp_args(__pipeline__image_location="('__SHARED__','img_loc')", __pipeline__patch_location="('__SHARED__','patch_loc')", __pipeline__input_files=None, __pipeline__annotation_location="('__SHARED__','anno_loc')", __pipeline__output_files=None, )
__TASK_1___prefix = rm.get_filename_prefix('__TASK_1__')
__TASK_1___task.update_comp_output_filenames(__TASK_1___prefix, rm.outputs_dir, args.no_prefix)
__TASK_1___task.update_comp_env_vars({})
__TASK_1___task.update_comp_reqs({'docker': '__REQUIRED__'})
#================================================================================
#ruffus pipeline
#--------------------------------------------------------------------------------
@ruffus.follows(*[])
@ruffus.parallel(__TASK_1___component.component_name, '__TASK_1__', [])
@ruffus.check_if_uptodate(rm.sentinel_file_exists)
@LogWarnErr(l)
@LogInfo(l)
def AnnoAndImageToPatch___TASK_1___function(*inargs):
component_name, task_name, _ = inargs
print '%s for %s started in %s pipeline' % (task_name, component_name, args.pipeline_name)
run_script = rm.generate_script(__TASK_1___task, None, None)
job_name = rm.get_filename_prefix(task_name)