Пример #1
0
    def gen_all(self):
        dataset_paths = [get_dataset_path(d) for d in DATASETS]

        self.task_ctrl.add(
            Task(verify_lats_lons, dataset_paths, [
                PATHS['output_datadir'] / 'basin_diurnal_cycle_analysis' /
                'verify_lats_lons.txt'
            ]))

        for self.raster_scales in ['small_medium_large', 'sliding']:
            if self.raster_scales == 'small_medium_large':
                self.scales = SCALES
            elif self.raster_scales == 'sliding':
                self.scales = SLIDING_SCALES

            self.hb_raster_cubes_fn = (
                PATHS['output_datadir'] /
                f'basin_diurnal_cycle_analysis/hb_N1280_raster_{self.raster_scales}.nc'
            )
            hb_raster_cubes_task = Task(gen_hydrobasins_raster_cubes,
                                        [get_dataset_path('cmorph')],
                                        [self.hb_raster_cubes_fn],
                                        func_args=[self.scales])
            self.task_ctrl.add(hb_raster_cubes_task)

            for dataset, mode in itertools.product(DATASETS, MODES):
                self.gen_analysis_tasks(dataset, mode)
            self.df_keys = pd.DataFrame(self.df_keys_data)

            for scale, mode in itertools.product(self.scales, MODES):
                self.gen_fig_tasks(scale, mode)

            self.gen_cmorph_vs_datasets_fig_tasks()
Пример #2
0
def gen_task_ctrl():
    tc = TaskControl(__file__)

    year = 2006
    model = 'al508'
    dist_thresh = 100
    dotprod_thresh = 0.05

    inputs = {
        f'al508_diag_{month:02}': fmtp(diag_orog_precip_path_tpl,
                                       model=model,
                                       year=year,
                                       month=month)
        for month in [6, 7, 8]
    }

    inputs['al508_direct'] = fmtp(orog_precip_mean_fields_tpl,
                                  model='al508',
                                  year=year,
                                  season='jja',
                                  dotprod_thresh=dotprod_thresh,
                                  dist_thresh=dist_thresh)
    inputs['ak543_direct'] = fmtp(orog_precip_mean_fields_tpl,
                                  model='ak543',
                                  year=year,
                                  season='jja',
                                  dotprod_thresh=dotprod_thresh,
                                  dist_thresh=dist_thresh)

    tc.add(Task(plot_fig2, inputs, [D23_fig2]))
    tc.add(Task(plot_fig3, inputs, [D23_fig3]))
    tc.add(Task(plot_fig4, inputs, [D23_fig4, D23_fig5]))

    return tc
Пример #3
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    hb_names = [f'S{i}' for i in range(11)]

    output_datadir = PATHS['output_datadir']

    inputs = {f'basin_vector_{hb_name}': output_datadir / 'basin_weighted_analysis' / hb_name / f'hb_{hb_name}.shp'
              for hb_name in hb_names}

    task_ctrl.add(Task(gen_vector_basin_stats,
                       inputs,
                       [PATHS['figsdir'] / 'basin_stats' / 'basin_vector_stats.csv'],
                       func_args=(hb_names,),
                       ))

    resolutions = ['N1280', 'N512', 'N216', 'N96']
    inputs = {f'basin_weights_{res}_{hb_name}': (output_datadir /
                                                 'basin_weighted_analysis' / hb_name / f'weights_{res}_{hb_name}.nc')
              for res in resolutions
              for hb_name in hb_names}

    task_ctrl.add(Task(gen_weights_basin_stats,
                       inputs,
                       [PATHS['figsdir'] / 'basin_stats' / 'basin_weights_stats.csv'],
                       func_args=(resolutions, hb_names),
                       ))

    return task_ctrl
Пример #4
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    for model, hb_name in itertools.product(MODELS, HB_NAMES):
        hb_names = {
            f'hb_name_{ext}': PATHS['output_datadir'] /
            f'raster_vs_hydrobasins/hb_{hb_name}.{ext}'
            for ext in ['shp', 'dbf', 'prj', 'cpg', 'shx']
        }
        input_filenames = {'model': FILENAMES[model]}
        input_filenames.update(hb_names)
        weights_filename = PATHS[
            'output_datadir'] / f'weights_vs_hydrobasins/weights_{model}_{hb_name}.nc'
        task_ctrl.add(
            Task(gen_weights_cube,
                 input_filenames, [weights_filename],
                 func_args=(hb_name, )))
        input_filenames = {
            'model':
            weights_filename,
            'hb_name':
            PATHS['output_datadir'] / f'raster_vs_hydrobasins/hb_{hb_name}.shp'
        }

        # TODO: Has not necessarily been created (if e.g. just querying task_ctrl).
        # TODO: Also, creating loads of output files creates loads of metadata, which takes up space.
        # TODO: Perhaps zip all the pngs together?
        # hb = gpd.read_file(str(input_filenames['hb_name']))
        # output_filenames = {i: PATHS['figsdir'] / 'weights_vs_hydrobasins' / f'{model}_{hb_name}' / f'basin_{i}.png'
        #                     for i in range(len(hb))}
        # task_ctrl.add(Task(plot_weights_cube, input_filenames, output_filenames))

    input_filenames = {
        (model, hb_name):
        (PATHS['output_datadir'] /
         f'weights_vs_hydrobasins/weights_{model}_{hb_name}.nc')
        for model, hb_name in itertools.product(MODELS, HB_NAMES)
    }
    for hb_name in HB_NAMES:
        input_filenames[hb_name] = PATHS[
            'output_datadir'] / f'raster_vs_hydrobasins/hb_{hb_name}.shp'
    output_filenames = [
        PATHS['figsdir'] / 'weights_vs_hydrobasins' / 'weights_cube_table' /
        f'basins_table.pdf'
    ]

    task_ctrl.add(
        Task(plot_weights_cube_table,
             input_filenames,
             output_filenames,
             func_kwargs={
                 'rows': list(zip(HB_NAMES[::-1], [770, 201, 23])),
                 'cols': MODELS
             }))
    return task_ctrl
Пример #5
0
    def basin_area_avg(self, dataset, diurnal_cycle_cube_path, scale, method,
                       mode):
        fn_base = f'basin_diurnal_cycle_analysis/{dataset}/basin_area_avg_' \
                  f'{diurnal_cycle_cube_path.stem}_{mode}_{scale}_{method}'

        task_kwargs = dict(
            dataset=dataset,
            mode=mode,
            basin_scale=scale,
            analysis_order='basin_area_avg',
            method=method,
        )

        df_phase_mag_key = PATHS['output_datadir'] / f'{fn_base}.hdf'
        inputs = {
            'raster_cubes': self.hb_raster_cubes_fn,
            'diurnal_cycle_cubes': diurnal_cycle_cube_path
        }
        if dataset[:7] == 'HadGEM3':
            cube_name = 'amount_of_precip_JJA'
        else:
            cube_name = 'amount_of_precip_jja'
        phase_mag_task = Task(gen_basin_area_avg_phase_mag,
                              inputs, [df_phase_mag_key],
                              func_args=[scale, cube_name, method])
        self.df_keys_data.append({
            **task_kwargs,
            **{
                'type': 'phase_mag',
                'task': phase_mag_task
            }
        })

        phase_mag_cubes_key = PATHS['output_datadir'] / f'{fn_base}.nc'
        phase_mag_cubes_task = Task(gen_phase_mag_map, {
            **inputs,
            **{
                'df_phase_mag': df_phase_mag_key
            }
        }, [phase_mag_cubes_key],
                                    func_args=[scale, cube_name])
        self.df_keys_data.append({
            **task_kwargs,
            **{
                'type': 'phase_mag_cubes',
                'task': phase_mag_cubes_task
            }
        })
        self.task_ctrl.add(phase_mag_task)
        self.task_ctrl.add(phase_mag_cubes_task)
Пример #6
0
    def gen_cmorph_vs_datasets_fig_tasks(self):
        rmses_filename = Path(
            PATHS['output_datadir'] /
            f'basin_diurnal_cycle_analysis/rmses_{self.raster_scales}.pkl')
        inputs = {}
        for mode in MODES:
            selector = ((self.df_keys.method == 'harmonic') &
                        (self.df_keys.type == 'phase_mag_cubes') &
                        (self.df_keys.analysis_order == 'basin_area_avg') &
                        (self.df_keys['mode'] == mode))

            df_cmorph = self.df_keys[selector
                                     & (self.df_keys.dataset == 'cmorph')]

            for dataset in DATASETS[1:]:
                df_dataset = self.df_keys[selector
                                          & (self.df_keys.dataset == dataset)]
                for scale in self.scales:
                    inputs[mode, 'cmorph',
                           scale] = (df_cmorph[df_cmorph.basin_scale == scale].
                                     task.values[0].outputs[0])
                    inputs[mode, dataset, scale] = (
                        df_dataset[df_dataset.basin_scale ==
                                   scale].task.values[0].outputs[0])
        inputs['raster_cubes'] = self.hb_raster_cubes_fn
        self.task_ctrl.add(
            Task(gen_rmses,
                 inputs, [rmses_filename],
                 func_args=(self.scales, )))

        both_filename = Path(f'{self.figsdir}/cmorph_vs/{self.raster_scales}/'
                             f'cmorph_vs_datasets.all.phase_and_mag.png')
        self.task_ctrl.add(
            Task(plot_cmorph_vs_all_datasets, [rmses_filename],
                 [both_filename],
                 func_args=[self.raster_scales]))

        for mode in MODES:
            phase_filename = Path(
                f'{self.figsdir}/cmorph_vs/{self.raster_scales}/'
                f'cmorph_vs_datasets.{mode}.phase.circular_rmse.png')
            mag_filename = Path(
                f'{self.figsdir}/cmorph_vs/{self.raster_scales}/'
                f'cmorph_vs_datasets.{mode}.mag.rmse.png')
            self.task_ctrl.add(
                Task(plot_cmorph_vs_all_datasets2, [rmses_filename],
                     [phase_filename, mag_filename],
                     func_args=[mode, self.raster_scales]))
Пример #7
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    task_ctrl.add(
        Task(
            plot_dem, [],
            [PATHS['figsdir'] / 'orog_precip' / f'dem_asia_with_regions.png']))
    return task_ctrl
Пример #8
0
def gen_task_ctrl():
    tc = TaskControl(__file__)
    # /gws/nopw/j04/cosmic/mmuetz/data/era_interim_orog_precip

    years = [2006]
    # years = [2005, 2006, 2007, 2008]
    models = ['al508', 'ak543']
    months = [6, 7, 8]

    for model, year, month in product(models, years, months):
        # al508a.p9200606.asia_precip.nc
        precip_path = fmtp(precip_path_tpl, model=model, year=year, month=month)
        orog_precip_inputs = {
            'extended_rclim_mask': extended_rclim_mask,
            'land_sea_mask': land_sea_mask,
            'precip': precip_path
        }
        diag_orog_precip_path = fmtp(diag_orog_precip_path_tpl, model=model, year=year, month=month)
        tc.add(Task(calc_orog_precip,
                    orog_precip_inputs,
                    [diag_orog_precip_path],
                    func_args=(month - 1, )))

        orog_precip_fracs_inputs = {
            'extended_rclim_mask': extended_rclim_mask,
            'land_sea_mask': land_sea_mask,
            'orog_precip': diag_orog_precip_path
        }
        diag_orog_precip_frac_path = fmtp(diag_orog_precip_frac_path_tpl, model=model, year=year, month=month)
        tc.add(Task(calc_orog_precip_fracs,
                    orog_precip_fracs_inputs,
                    [diag_orog_precip_frac_path],
                    func_args=(month - 1, )))

    variables = list(product(models, months))
    columns = ['model', 'month']
    combine_inputs = [fmtp(diag_orog_precip_frac_path_tpl, model=model, year=year, month=month)
                      for model, month in variables]
    combine_fracs_output = [diag_combine_frac_path]
    tc.add(Task(combine_orog_precip_fracs,
                combine_inputs,
                combine_fracs_output,
                func_args=(variables, columns)
               ))

    return tc
def gen_task_ctrl():
    tc = TaskControl(__file__)
    models = ['al508', 'ak543']
    # models = ['ak543']
    dist_threshs = [100]
    dotprod_threshs = [0.05]
    months = [6, 7, 8]
    # dist_threshs = [20, 100]
    # dotprod_threshs = [0.05, 0.1]
    year = 2006

    for model, dotprod_thresh, dist_thresh in product(models, dotprod_threshs, dist_threshs):
        # 1 model at a time.
        orog_precip_paths = [fmtp(orog_precip_path_tpl, model=model, year=year, month=month,
                                  dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh)
                             for month in months]

        orog_precip_mean_fields = [fmtp(orog_precip_mean_fields_tpl, model=model, year=year, season='jja',
                                        dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh)]
        tc.add(Task(extract_precip_mean_fields, orog_precip_paths, orog_precip_mean_fields))

    for model, dotprod_thresh, dist_thresh in product(models, dotprod_threshs, dist_threshs):
        orog_precip_mean_fields = [fmtp(orog_precip_mean_fields_tpl, model=model, year=year, season='jja',
                                        dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh)]
        orog_precip_figs = [fmtp(orog_precip_fig_tpl, model=model, year=year, season='jja',
                                 dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh,
                                 precip_type=precip_type)
                            for precip_type in ['orog', 'non_orog', 'ocean', 'orog_frac']]
        tc.add(Task(plot_mean_orog_precip, orog_precip_mean_fields, orog_precip_figs))

    return tc
    for dotprod_thresh, dist_thresh in product(dotprod_threshs, dist_threshs):
        # Compare 2 models.
        orog_precip_paths = {(model, month): fmtp(orog_precip_path_tpl, model=model, year=year, month=month,
                                                  dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh)
                             for model in models
                             for month in months}
        orog_precip_figs = [fmtp(orog_precip_fig_tpl, model='-'.join(models), year=year, season='jja',
                                 dotprod_thresh=dotprod_thresh, dist_thresh=dist_thresh,
                                 precip_type=precip_type)
                            for precip_type in ['orog', 'non_orog', 'ocean']]
        tc.add(Task(plot_compare_mean_orog_precip, orog_precip_paths, orog_precip_figs,
                    func_args=(models, months)))

    return tc
Пример #10
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)

    input_paths = {dataset: get_dataset_path(dataset) for dataset in DATASETS}
    task = Task(plot_gridpoint_mean_precip_asia, input_paths, [
        PATHS['figsdir'] / 'gridpoint_analysis' /
        f'gridpoint_mean_precip_asia.pdf'
    ])
    task_ctrl.add(task)
    return task_ctrl
Пример #11
0
def gen_task_ctrl():
    tc = TaskControl(__file__)
    datadir = PATHS['datadir'] / 'aphrodite_data/025deg'
    for year in ALL_YEARS:
        output_path = datadir / FILE_TPL.format(year=year)
        tc.add(
            Task(download_year, [], [output_path],
                 func_args=(year, datadir),
                 atomic_write=False))
    return tc
Пример #12
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    for fn, args, kwargs in all_plot_gauge_data_gen():
        kwargs_str = '.'.join([f'{k}-{i}' for k, i in kwargs.items()])
        task = Task(fn,
                    [],
                    [PATHS['figsdir'] / 'gauge_data' / f'precip_station_jja_cressman.{kwargs_str}.png'],
                    func_args=args,
                    func_kwargs=kwargs)
        task_ctrl.add(task)
    return task_ctrl
Пример #13
0
def gen_task_ctrl():
    tc = TaskControl(__file__)
    datadir = PATHS['datadir'] / 'aphrodite_data/025deg'
    filenames = [datadir / FILE_TPL.format(year=year) for year in ALL_YEARS]
    tc.add(
        Task(combine_years, filenames, [
            datadir / 'aphrodite_combined_all.nc',
            datadir / 'aphrodite_combined_jja.nc'
        ]))

    return tc
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    aphrodite_dir = Path('aphrodite_data/025deg')
    inputs = {
        ' daily precipitation analysis interpolated onto 0.25deg grids': (PATHS['datadir'] /
                                                                          aphrodite_dir /
                                                                          'APHRO_MA_025deg_V1901.2009.nc')}
    outputs = {
        'asia': PATHS['figsdir'] / 'aphrodite' / 'asia_aphrodite_2009_jja.png',
        'china': PATHS['figsdir'] / 'aphrodite' / 'china_aphrodite_2009_jja.png',
    }

    task_ctrl.add(Task(plot_aphrodite_seasonal_analysis, inputs, outputs, func_args=(True, )))

    outputs = {
        'asia': PATHS['figsdir'] / 'aphrodite' / 'asia_aphrodite_2009_jja.lognorm.png',
        'china': PATHS['figsdir'] / 'aphrodite' / 'china_aphrodite_2009_jja.lognorm.png',
    }

    task_ctrl.add(Task(plot_aphrodite_seasonal_analysis, inputs, outputs, func_args=(False, )))
    return task_ctrl
Пример #15
0
def gen_task_ctrl():
    years = [2019]
    months = range(1, 13)

    task_ctrl = TaskControl(__file__)
    for year, month in itertools.product(years, months):
        filename = (BASEDIR / 'raw' /
                    f'precip_{year}{month:02}' /
                    f'CMORPH_V1.0_ADJ_8km-30min_{year}{month:02}.tar')
        task_ctrl.add(Task(download_year, [], [filename], func_args=(year, month)))

        raw_filename = (BASEDIR / 'raw' /
                        f'precip_{year}{month:02}' /
                        f'CMORPH_V1.0_ADJ_8km-30min_{year}{month:02}.tar')
        nc_filenames = {day: (BASEDIR /
                              f'precip_{year}{month:02}' /
                              f'cmorph_ppt_{year}{month:02}{day:02}.nc')
                        for day in range(1, calendar.monthrange(year, month)[1] + 1)}
        task_ctrl.add(Task(convert_year_month,
                           [raw_filename],
                           nc_filenames,
                           func_args=(year, month)))

        nc_asia_filename = (BASEDIR / f'precip_{year}{month:02}' /
                            f'cmorph_ppt_{year}{month:02}.asia.nc')
        task_ctrl.add(Task(extract_year_month,
                           nc_filenames,
                           [nc_asia_filename],
                           func_args=(year, month)))

        regrid_inputs = {}
        regrid_inputs['target'] = Path(f'/gws/nopw/j04/cosmic/mmuetz/data/u-al508/ap9.pp/precip_200501/al508a.p9200501.asia_precip.nc')
        regrid_inputs['cmorph'] = BASEDIR / f'precip_{year}{month:02}/cmorph_ppt_{year}{month:02}.asia.nc'
        regrid_output = regrid_inputs['cmorph'].parent / (regrid_inputs['cmorph'].stem + '.N1280.nc')
        task_ctrl.add(Task(regrid_asia,
                           regrid_inputs,
                           [regrid_output]))

    return task_ctrl
Пример #16
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)

    for fn, args, kwargs in all_seasonal_analysis_gen():
        task_unique_name = task_unique_name_from_fn_args_kwargs(fn, args, kwargs)
        task = Task(fn,
                    [plot_seasonal_analysis.__file__],
                    [PATHS['figsdir'] / 'seasonal_analysis' / task_unique_name],
                    func_args=args,
                    func_kwargs=kwargs)
        task_ctrl.add(task)

    return task_ctrl
Пример #17
0
 def gen_phase_mag_maps_tasks(self, scale, mode, row):
     phase_filename = Path(
         f'{self.figsdir}/map/{mode}/{row.dataset}_{row.analysis_order}_{row.method}'
         f'.{scale}.phase.png')
     mag_filename = Path(
         f'{self.figsdir}/map/{mode}/{row.dataset}_{row.analysis_order}_{row.method}'
         f'.{scale}.mag.png')
     inputs = {
         'raster_cubes': self.hb_raster_cubes_fn,
         'phase_mag_cubes': row.task.outputs[0]
     }
     self.task_ctrl.add(
         Task(plot_phase_mag,
              inputs, [phase_filename, mag_filename],
              func_args=[scale, mode, row]))
Пример #18
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    start_years = range(1998, 2016)

    inputs = {
        y:
        (PATHS['datadir'] / 'cmorph_data' / '8km-30min' /
         f'cmorph_8km_N1280.{y}06-{y + 3}08.jja.asia_precip_afi.ppt_thresh_0p1.nc'
         )
        for y in start_years
    }
    output = PATHS['figsdir'] / 'cmorph' / 'min_max_data.txt'
    task_ctrl.add(
        Task(calc_cmorph_min_max, inputs, [output], func_args=(start_years, )))

    inputs['full'] = (
        PATHS['datadir'] / 'cmorph_data' / '8km-30min' /
        'cmorph_8km_N1280.199801-201812.jja.asia_precip_afi.ppt_thresh_0p1.nc')
    inputs['min_max'] = output

    output = PATHS['figsdir'] / 'cmorph' / 'cmorph_full_min_max_asia.pdf'
    task_ctrl.add(Task(plot_cmorph_min_max, inputs, [output]))

    return task_ctrl
Пример #19
0
def gen_task_ctrl():
    tc = TaskControl(__file__)
    # pp_files = list((PATHS['datadir'] / 'u-al508' / 'ap8.pp' / 'lowlevel_wind_200901').glob('al508a.p8200901??.pp'))
    # for pp_file in pp_files:
    #     tc.add(Task(convert_wrapper, [pp_file], [pp_file.with_suffix('.nc')]))

    pp_files = list(
        (PATHS['datadir'] / 'u-al508' /
         'ap9.pp').glob('surface_wind_2006??/al508a.p9????????.pp'))
    pp_files += list(
        (PATHS['datadir'] / 'u-ak543' /
         'ap9.pp').glob('surface_wind_2006??/ak543a.p9????????.pp'))
    for pp_file in pp_files:
        tc.add(Task(convert_wrapper, [pp_file], [pp_file.with_suffix('.nc')]))
    return tc
Пример #20
0
def gen_task_ctrl():
    tc = TaskControl(__file__)
    tc.add(
        Task(plot_masks,
             [PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc'],
             [PATHS['figsdir'] / 'experimental' / 'sinclair_orog_masks.png']))
    tc.add(
        Task(calc_extended_orog_mask,
             [PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc'],
             [PATHS['datadir'] / 'experimental' / 'extended_orog_mask.nc']))

    tc.add(
        Task(
            calc_extended_orog_mask_JJA,
            [PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc'],
            [PATHS['datadir'] / 'experimental' / 'extended_orog_mask_JJA.nc']))

    tc.add(
        Task(calc_JJA_clim,
             [PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc'],
             [PATHS['datadir'] / 'experimental' / 'R_clim_JJA.nc']))

    tc.add(
        Task(plot_masks_combined, [
            PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc',
            PATHS['datadir'] / 'experimental' / 'extended_orog_mask.nc'
        ], [
            PATHS['figsdir'] / 'experimental' /
            'sinclair_orog_data_combined.png', PATHS['figsdir'] /
            'experimental' / 'sinclair_orog_masks_combined.png'
        ],
             func_args=(slice(None), )))

    tc.add(
        Task(plot_masks_combined, [
            PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc',
            PATHS['datadir'] / 'experimental' / 'extended_orog_mask_JJA.nc'
        ], [
            PATHS['figsdir'] / 'experimental' /
            'sinclair_orog_data_combined_JJA.png', PATHS['figsdir'] /
            'experimental' / 'sinclair_orog_masks_combined_JJA.png'
        ],
             func_args=(slice(5, 8), )))

    tc.add(
        Task(regrid_to_N1280,
             [PATHS['datadir'] / 'era_interim_orog_precip' / 'R_clim.nc'],
             [PATHS['datadir'] / 'experimental' / 'R_clim.N1280.nc']))

    return tc
Пример #21
0
    def gen_dataset_comparison_tasks(self, scale, mode, row1, row2):
        phase_scatter_filename = Path(
            f'{self.figsdir}/comparison/{mode}/'
            f'{row1.dataset}_{row1.analysis_order}_{row1.method}_vs_'
            f'{row2.dataset}_{row2.analysis_order}_{row2.method}.'
            f'{scale}.phase.png')
        mag_scatter_filename = Path(
            f'{self.figsdir}/comparison/{mode}/'
            f'{row1.dataset}_{row1.analysis_order}_{row1.method}_vs_'
            f'{row2.dataset}_{row2.analysis_order}_{row2.method}.'
            f'{scale}.mag.png')

        inputs = [row1.task.outputs[0], row2.task.outputs[0]]
        self.task_ctrl.add(
            Task(plot_dataset_scatter,
                 inputs, [phase_scatter_filename, mag_scatter_filename],
                 func_args=[scale, mode, row1, row2]))
Пример #22
0
def gen_task_ctrl():
    tc = TaskControl(__file__)

    for model in ['u-al508', 'u-ak543']:
        year = 2006
        for month in [6, 7, 8]:
            nc_files = [
                PATHS['gcosmic'] / 'share' / 'ancils' / 'N1280' / 'qrparm.orog'
            ]
            nc_files.extend(
                sorted((PATHS['datadir'] / model / 'ap9.pp' /
                        f'surface_wind_{year}{month:02}'
                        ).glob(f'{model[2:]}a.p9????????.nc')))
            outpath = (PATHS['datadir'] / model / 'ap9.pp' /
                       f'surface_wind_{year}{month:02}' /
                       f'{model[2:]}a.p9{year}{month:02}.asia.nc')
            tc.add(Task(regrid_extract_asia, nc_files, [outpath]))

    return tc
Пример #23
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)
    task_ctrl.add(
        Task(plot_dem, [], [PATHS['figsdir'] / 'dem' / f'dem_asia.pdf']))
    return task_ctrl
Пример #24
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)

    for basin_scales in ['small_medium_large', 'sliding']:
        hb_raster_cubes_fn = PATHS[
            'output_datadir'] / f'basin_weighted_analysis/hb_N1280_raster_{basin_scales}.nc'
        cmorph_path = get_dataset_path('cmorph')
        # cmorph_path = (PATHS['datadir'] /
        #                'cmorph_data/8km-30min/cmorph_ppt_jja.199801-201812.asia_precip.ppt_thresh_0p1.N1280.nc')
        task_ctrl.add(
            Task(gen_hydrobasins_raster_cubes, [cmorph_path],
                 [hb_raster_cubes_fn],
                 func_args=[
                     SLIDING_SCALES if basin_scales == 'sliding' else SCALES
                 ]))

        if basin_scales == 'small_medium_large':
            hb_names = HB_NAMES
        else:
            hb_names = [f'S{i}' for i in range(11)]
        shp_path_tpl = 'basin_weighted_analysis/{hb_name}/hb_{hb_name}.{ext}'
        for hb_name in hb_names:
            # Creates a few different files with different extensions - need to have them all in outputs
            # so that they are moved to the right place after run by Task.atomic_write.
            task_ctrl.add(
                Task(
                    gen_hydrobasins_files,
                    [],
                    {
                        ext: PATHS['output_datadir'] /
                        shp_path_tpl.format(hb_name=hb_name, ext=ext)
                        for ext in ['shp', 'dbf', 'prj', 'cpg', 'shx']
                    },
                    func_args=[hb_name],
                ))

        # N.B. Need to do this once for one dataset at each resolution.
        # I.e. only need one N1280 res dataset -- u-ak543.
        for dataset, hb_name in itertools.product(DATASETS[:4] + ['aphrodite'],
                                                  hb_names):
            if dataset == 'u-ak543':
                dataset_cube_path = PATHS[
                    'datadir'] / 'u-ak543/ap9.pp/precip_200601/ak543a.p9200601.asia_precip.nc'
            elif dataset[:7] == 'HadGEM3':
                dataset_cube_path = HADGEM_FILENAMES[dataset]
            elif dataset == 'aphrodite':
                dataset_cube_path = PATHS[
                    'datadir'] / 'aphrodite_data/025deg/aphrodite_combined_all.nc'
            input_filenames = {
                dataset:
                dataset_cube_path,
                hb_name:
                PATHS['output_datadir'] /
                shp_path_tpl.format(hb_name=hb_name, ext='shp')
            }

            resolution = DATASET_RESOLUTION[dataset]
            weights_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/{hb_name}/weights_{resolution}_{hb_name}.nc'
            )
            task_ctrl.add(
                Task(gen_weights_cube, input_filenames, [weights_filename]))

        weighted_mean_precip_tpl = 'basin_weighted_analysis/{hb_name}/' \
                                   '{dataset}.{hb_name}.area_weighted.mean_precip.hdf'

        weighted_mean_precip_filenames = defaultdict(list)
        for dataset, hb_name in itertools.product(DATASETS + ['aphrodite'],
                                                  hb_names):
            fmt_kwargs = {'dataset': dataset, 'hb_name': hb_name}
            dataset_path = get_dataset_path(dataset)
            resolution = DATASET_RESOLUTION[dataset]
            weights_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/{hb_name}/weights_{resolution}_{hb_name}.nc'
            )
            weighted_mean_precip_filename = PATHS[
                'output_datadir'] / weighted_mean_precip_tpl.format(
                    **fmt_kwargs)
            weighted_mean_precip_filenames[hb_name].append(
                weighted_mean_precip_filename)

            task_ctrl.add(
                Task(native_weighted_basin_mean_precip_analysis, {
                    'dataset_path': dataset_path,
                    'weights': weights_filename
                }, [weighted_mean_precip_filename]))

        for obs in ['cmorph', 'aphrodite', 'u-al508', 'u-ak543']:
            mean_precip_rmse_data_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/{obs}.mean_precip_all_rmses.{basin_scales}.pkl'
            )
            gen_mean_precip_rmses_inputs = {
                (ds, hb_name): PATHS['output_datadir'] /
                weighted_mean_precip_tpl.format(dataset=ds, hb_name=hb_name)
                for ds, hb_name in itertools.product(DATASETS +
                                                     ['aphrodite'], hb_names)
            }
            task_ctrl.add(
                Task(gen_mean_precip_rmses_corrs,
                     inputs=gen_mean_precip_rmses_inputs,
                     outputs=[mean_precip_rmse_data_filename],
                     func_kwargs={
                         'hb_names': hb_names,
                         'obs': obs
                     }))

            mean_precip_bias_data_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/{obs}.mean_precip_all_bias.{basin_scales}.csv'
            )
            task_ctrl.add(
                Task(gen_mean_precip_highest_percentage_bias,
                     inputs=gen_mean_precip_rmses_inputs,
                     outputs=[mean_precip_bias_data_filename],
                     func_kwargs={
                         'hb_names': hb_names,
                         'obs': obs
                     }))

        for hb_name in hb_names:
            # N.B. out of order.
            max_min_path = PATHS[
                'output_datadir'] / f'basin_weighted_analysis/{hb_name}/mean_precip_max_min.pkl'
            task_ctrl.add(
                Task(calc_mean_precip_max_min,
                     weighted_mean_precip_filenames[hb_name], [max_min_path]))

        weighted_phase_mag_tpl = 'basin_weighted_analysis/{hb_name}/' \
                                 '{dataset}.{hb_name}.{mode}.area_weighted.phase_mag.hdf'

        for dataset, hb_name, mode in itertools.product(
                DATASETS, hb_names, PRECIP_MODES):
            fmt_kwargs = {'dataset': dataset, 'hb_name': hb_name, 'mode': mode}
            if dataset[:7] == 'HadGEM3':
                cube_name = f'{mode}_of_precip_JJA'
            else:
                cube_name = f'{mode}_of_precip_jja'
            dataset_path = get_dataset_path(dataset)
            resolution = DATASET_RESOLUTION[dataset]
            weights_filename = PATHS[
                'output_datadir'] / f'basin_weighted_analysis/{hb_name}/weights_{resolution}_{hb_name}.nc'

            weighted_phase_mag_filename = PATHS[
                'output_datadir'] / weighted_phase_mag_tpl.format(**fmt_kwargs)
            task_ctrl.add(
                Task(native_weighted_basin_diurnal_cycle_analysis, {
                    'diurnal_cycle': dataset_path,
                    'weights': weights_filename
                }, [weighted_phase_mag_filename],
                     func_args=[cube_name]))

        for area_weighted in [True, False]:
            weighted = 'area_weighted' if area_weighted else 'not_area_weighted'

            vrmse_data_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/all_rmses.{weighted}.{basin_scales}.pkl'
            )
            gen_rmses_inputs = {
                (ds, mode, hb_name):
                PATHS['output_datadir'] / weighted_phase_mag_tpl.format(
                    dataset=ds, hb_name=hb_name, mode=mode)
                for ds, mode, hb_name in itertools.product(
                    DATASETS, PRECIP_MODES, hb_names)
            }
            gen_rmses_inputs['raster_cubes'] = hb_raster_cubes_fn

            task_ctrl.add(
                Task(gen_phase_mag_rmses,
                     inputs=gen_rmses_inputs,
                     outputs=[vrmse_data_filename],
                     func_kwargs={
                         'area_weighted': area_weighted,
                         'hb_names': hb_names
                     }))

    return task_ctrl
Пример #25
0
def gen_task_ctrl():
    tc = TaskControl(__file__)

    years = [2006]
    # years = [2005, 2006, 2007, 2008]
    models = ['al508', 'ak543']
    dist_threshs = [50, 100]
    dotprod_threshs = [0.05]
    months = [6, 7, 8]
    # dist_threshs = [20, 100]
    # dotprod_threshs = [0.05, 0.1]

    for dist_thresh in dist_threshs:
        cache_key = fmtp(cache_key_tpl, dist_thresh=dist_thresh)
        tc.add(
            Task(gen_dist_cache, {'orog': orog_path}, [cache_key],
                 func_args=(dist_thresh, )))

    for year, model, dotprod_thresh, dist_thresh in product(
            years, models, dotprod_threshs, dist_threshs):
        cache_key = fmtp(cache_key_tpl, dist_thresh=dist_thresh)
        for month in months:
            surf_wind_path = fmtp(surf_wind_path_tpl,
                                  model=model,
                                  year=year,
                                  month=month)
            orog_mask_path = fmtp(orog_mask_path_tpl,
                                  model=model,
                                  year=year,
                                  month=month,
                                  dotprod_thresh=dotprod_thresh,
                                  dist_thresh=dist_thresh)
            inputs = {
                'orog': orog_path,
                'cache_key': cache_key,
                'surf_wind': surf_wind_path
            }

            tc.add(
                Task(gen_orog_mask,
                     inputs, [orog_mask_path],
                     func_args=(dotprod_thresh, dist_thresh)))

            precip_path = fmtp(precip_path_tpl,
                               model=model,
                               year=year,
                               month=month)
            orog_precip_path = fmtp(orog_precip_path_tpl,
                                    model=model,
                                    year=year,
                                    month=month,
                                    dotprod_thresh=dotprod_thresh,
                                    dist_thresh=dist_thresh)
            orog_precip_inputs = {
                'orog_mask': orog_mask_path,
                'land_sea_mask': land_sea_mask,
                'precip': precip_path
            }
            tc.add(
                Task(calc_orog_precip, orog_precip_inputs, [orog_precip_path]))

            orog_precip_frac_inputs = {
                'orog_mask': orog_mask_path,
                'land_sea_mask': land_sea_mask,
                'orog_precip': orog_precip_path
            }
            orog_precip_frac_path = fmtp(orog_precip_frac_path_tpl,
                                         model=model,
                                         year=year,
                                         month=month,
                                         dotprod_thresh=dotprod_thresh,
                                         dist_thresh=dist_thresh)

            tc.add(
                Task(calc_orog_precip_fracs, orog_precip_frac_inputs,
                     [orog_precip_frac_path]))

    variables = list(product(models, dotprod_threshs, dist_threshs, months))
    columns = ['model', 'dotprod_thresh', 'dist_thresh', 'month']
    combine_inputs = [
        fmtp(orog_precip_frac_path_tpl,
             model=model,
             year=year,
             month=month,
             dotprod_thresh=dotprod_thresh,
             dist_thresh=dist_thresh)
        for model, dotprod_thresh, dist_thresh, month in variables
    ]
    combine_fracs_output = [combine_frac_path]
    tc.add(
        Task(combine_orog_precip_fracs,
             combine_inputs,
             combine_fracs_output,
             func_args=(variables, columns)))

    return tc
Пример #26
0
def gen_task_ctrl():
    task_ctrl = TaskControl(__file__)

    for basin_scales in ['small_medium_large', 'sliding']:
        hb_raster_cubes_fn = PATHS[
            'output_datadir'] / f'basin_weighted_analysis/hb_N1280_raster_{basin_scales}.nc'
        if basin_scales == 'small_medium_large':
            hb_names = HB_NAMES
        else:
            hb_names = [f'S{i}' for i in range(11)]
        shp_path_tpl = 'basin_weighted_analysis/{hb_name}/hb_{hb_name}.{ext}'
        for hb_name in hb_names:
            shp_inputs = {
                ext: PATHS['output_datadir'] /
                shp_path_tpl.format(hb_name=hb_name, ext=ext)
                for ext in ['shp', 'dbf', 'prj', 'cpg', 'shx']
            }
            task_ctrl.add(
                Task(
                    plot_hydrobasins_files,
                    shp_inputs,
                    [
                        PATHS['figsdir'] / 'basin_weighted_analysis' / 'map' /
                        'hydrobasins_size' / f'map_{hb_name}.png'
                    ],
                    func_args=[hb_name],
                ))

        weighted_mean_precip_tpl = 'basin_weighted_analysis/{hb_name}/' \
                                   '{dataset}.{hb_name}.area_weighted.mean_precip.hdf'

        weighted_mean_precip_filenames = defaultdict(list)
        for dataset, hb_name in itertools.product(DATASETS + ['aphrodite'],
                                                  hb_names):
            fmt_kwargs = {'dataset': dataset, 'hb_name': hb_name}
            max_min_path = PATHS[
                'output_datadir'] / f'basin_weighted_analysis/{hb_name}/mean_precip_max_min.pkl'

            weighted_mean_precip_filename = PATHS[
                'output_datadir'] / weighted_mean_precip_tpl.format(
                    **fmt_kwargs)
            weighted_mean_precip_filenames[hb_name].append(
                weighted_mean_precip_filename)
            task_ctrl.add(
                Task(plot_mean_precip, {
                    'weighted': weighted_mean_precip_filename,
                    'raster_cubes': hb_raster_cubes_fn,
                    'mean_precip_max_min': max_min_path
                }, [
                    PATHS['figsdir'] / 'basin_weighted_analysis' / 'map' /
                    'mean_precip' /
                    f'map_{dataset}.{hb_name}.area_weighted.png'
                ],
                     func_args=[dataset, hb_name]))

            if dataset != 'cmorph':
                fmt_kwargs = {'dataset': 'cmorph', 'hb_name': hb_name}
                cmorph_weighted_mean_precip_filename = (
                    PATHS['output_datadir'] /
                    weighted_mean_precip_tpl.format(**fmt_kwargs))
                task_ctrl.add(
                    Task(plot_obs_mean_precip_diff, {
                        'dataset_weighted': weighted_mean_precip_filename,
                        'obs_weighted': cmorph_weighted_mean_precip_filename,
                        'raster_cubes': hb_raster_cubes_fn,
                        'mean_precip_max_min': max_min_path
                    }, [
                        PATHS['figsdir'] / 'basin_weighted_analysis' / 'map' /
                        'cmorph_mean_precip_diff' /
                        f'map_{dataset}.{hb_name}.area_weighted.png'
                    ],
                         func_args=[dataset, hb_name]))

            if dataset not in ['cmorph', 'aphrodite']:
                fmt_kwargs = {'dataset': 'aphrodite', 'hb_name': hb_name}
                obs_weighted_mean_precip_filename = (
                    PATHS['output_datadir'] /
                    weighted_mean_precip_tpl.format(**fmt_kwargs))
                task_ctrl.add(
                    Task(plot_obs_mean_precip_diff, {
                        'dataset_weighted': weighted_mean_precip_filename,
                        'obs_weighted': obs_weighted_mean_precip_filename,
                        'raster_cubes': hb_raster_cubes_fn,
                        'mean_precip_max_min': max_min_path
                    }, [
                        PATHS['figsdir'] / 'basin_weighted_analysis' / 'map' /
                        'cmorph_mean_precip_diff' /
                        f'map_aphrodite_vs_{dataset}.{hb_name}.area_weighted.png'
                    ],
                         func_args=[dataset, hb_name]))

        for obs in ['cmorph', 'aphrodite', 'u-al508', 'u-ak543']:
            mean_precip_rmse_data_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/{obs}.mean_precip_all_rmses.{basin_scales}.pkl'
            )
            task_ctrl.add(
                Task(plot_obs_vs_all_datasets_mean_precip,
                     inputs=[mean_precip_rmse_data_filename],
                     outputs=[
                         PATHS['figsdir'] / 'basin_weighted_analysis' /
                         'cmorph_vs' / 'mean_precip' /
                         f'{obs}_vs_all_datasets.all_{f}.{basin_scales}.pdf'
                         for f in ['rmse', 'corr']
                     ]))

        if basin_scales == 'small_medium_large':
            input_paths = {'raster_cubes': hb_raster_cubes_fn}
            for name, datasets in zip(
                ['', 'full_'],
                (['cmorph', 'u-al508', 'u-ak543'
                  ], ['cmorph', 'u-al508', 'u-am754', 'u-ak543'])):
                paths = {
                    f'weighted_{hb_name}_{dataset}':
                    (PATHS['output_datadir'] / weighted_mean_precip_tpl.format(
                        hb_name=hb_name, dataset=dataset))
                    for hb_name, dataset in itertools.product(
                        hb_names, datasets)
                }
                input_paths.update(paths)
                task_ctrl.add(
                    Task(plot_mean_precip_asia_combined,
                         input_paths, [
                             PATHS['figsdir'] / 'basin_weighted_analysis' /
                             'map' / 'mean_precip_asia_combined' /
                             f'{name}asia_combined_basin_scales.pdf'
                         ],
                         func_args=[datasets, hb_names]))

        weighted_phase_mag_tpl = 'basin_weighted_analysis/{hb_name}/' \
                                 '{dataset}.{hb_name}.{mode}.area_weighted.phase_mag.hdf'

        for dataset, hb_name, mode in itertools.product(
                DATASETS, hb_names, PRECIP_MODES):
            fmt_kwargs = {'dataset': dataset, 'hb_name': hb_name, 'mode': mode}
            weighted_phase_mag_filename = PATHS[
                'output_datadir'] / weighted_phase_mag_tpl.format(**fmt_kwargs)

            task_ctrl.add(
                Task(plot_phase_mag, {
                    'weighted': weighted_phase_mag_filename,
                    'raster_cubes': hb_raster_cubes_fn
                }, [
                    PATHS['figsdir'] / 'basin_weighted_analysis' / 'map' /
                    mode /
                    f'map_{dataset}.{hb_name}.{mode}.area_weighted.{v}.png'
                    for v in ['phase', 'alpha_phase', 'mag']
                ],
                     func_args=[dataset, hb_name, mode]))

        if basin_scales == 'small_medium_large':
            for name, datasets in zip(
                ['', 'full_'],
                (['cmorph', 'u-al508', 'u-ak543'
                  ], ['cmorph', 'u-al508', 'u-am754', 'u-ak543'])):
                for mode in PRECIP_MODES:
                    input_paths = {
                        f'weighted_{hb_name}_{dataset}':
                        (PATHS['output_datadir'] /
                         weighted_phase_mag_tpl.format(
                             hb_name=hb_name, dataset=dataset, mode=mode))
                        for hb_name, dataset in itertools.product(
                            hb_names, datasets)
                    }
                    input_paths.update({'raster_cubes': hb_raster_cubes_fn})
                    task_ctrl.add(
                        Task(plot_phase_alpha_combined,
                             input_paths, [
                                 PATHS['figsdir'] / 'basin_weighted_analysis' /
                                 'map' / 'phase_alpha_combined' /
                                 f'{name}{mode}_phase_alpha_combined_asia.pdf'
                             ],
                             func_args=(datasets, hb_names, mode)))

        for area_weighted in [True, False]:
            weighted = 'area_weighted' if area_weighted else 'not_area_weighted'

            vrmse_data_filename = (
                PATHS['output_datadir'] /
                f'basin_weighted_analysis/all_rmses.{weighted}.{basin_scales}.pkl'
            )

            task_ctrl.add(
                Task(
                    plot_cmorph_vs_all_datasets_phase_mag,
                    [vrmse_data_filename],
                    [
                        PATHS['figsdir'] / 'basin_weighted_analysis' /
                        'cmorph_vs' / 'phase_mag' /
                        f'cmorph_vs_all_datasets.all_rmse.{weighted}.{basin_scales}.pdf'
                    ],
                ))

    return task_ctrl
Пример #27
0
def gen_task_ctrl(test=False):
    tc = TaskControl(__file__)
    models = ['al508', 'ak543']
    months = [6, 7, 8]
    year = 2006
    regions = ['TP_southern_flank', 'sichuan_basin']
    days_in_month = 30

    if test:
        models = models[:1]
        months = months[:1]
        days_in_month = 2
        # regions = regions[:1]

    for model, month, region in product(models, months, regions):
        surf_wind_path = fmtp(surf_wind_path_tpl,
                              model=model,
                              year=year,
                              month=month)
        precip_path = fmtp(precip_path_tpl,
                           model=model,
                           year=year,
                           month=month)
        orog_mask_path = fmtp(orog_mask_path_tpl,
                              model=model,
                              year=year,
                              month=month,
                              dotprod_thresh=0.05,
                              dist_thresh=100)

        raw_data_fig_paths = [
            fmtp(raw_data_fig_tpl,
                 model=model,
                 year=year,
                 month=month,
                 day=day,
                 hour=h,
                 region=region) for day in range(1, days_in_month + 1)
            for h in range(0, 24)
        ]
        tc.add(
            Task(plot_precip_wind_region, {
                'orog': orog_path,
                'surf_wind': surf_wind_path,
                'precip': precip_path,
                'orog_mask': orog_mask_path,
            },
                 raw_data_fig_paths,
                 func_args=(region, month)))

        raw_data_dc_fig_paths = [
            fmtp(raw_data_dc_fig_tpl,
                 model=model,
                 year=year,
                 month=month,
                 hour=h,
                 region=region) for h in range(24)
        ]
        tc.add(
            Task(plot_dc_region, {
                'orog': orog_path,
                'surf_wind': surf_wind_path,
                'precip': precip_path,
                'orog_mask': orog_mask_path,
            },
                 raw_data_dc_fig_paths,
                 func_args=(region, 24)))

        raw_data_dc_anom_wind_fig_paths = [
            fmtp(raw_data_dc_anom_wind_fig_tpl,
                 model=model,
                 year=year,
                 month=month,
                 hour=h,
                 region=region) for h in range(24)
        ]
        tc.add(
            Task(plot_dc_anom_wind_region, {
                'orog': orog_path,
                'surf_wind': surf_wind_path,
                'precip': precip_path,
                'orog_mask': orog_mask_path,
            },
                 raw_data_dc_anom_wind_fig_paths,
                 func_args=(region, 24)))
        tc.add(
            Task(create_animation, raw_data_dc_anom_wind_fig_paths, [
                fmtp(anim_raw_data_dc_anom_wind_fig_tpl,
                     model=model,
                     year=year,
                     month=month,
                     region=region)
            ], (120, )))

    return tc
Пример #28
0
from remake import Task, TaskControl
# from cosmic.task import Task, TaskControl

BSUB_KWARGS = {
    'job_name': 'demo',
    'queue': 'short-serial',
    'max_runtime': '00:10',
    # 'mem': '64000',
}


def task_fn(inputs, outputs):
    print(inputs)
    print(outputs)
    for o in outputs:
        o.touch()


task_ctrl = TaskControl()
task_ctrl.add(Task(task_fn, [], ['task_demo1.out']))
task_ctrl.add(Task(task_fn, [], ['task_demo2.out']))
task_ctrl.add(Task(task_fn, [], ['task_demo3.out']))
task_ctrl.add(Task(task_fn, ['task_demo1.out', 'task_demo2.out', 'task_demo3.out'], ['task_demo4.out']))
task_ctrl.add(Task(task_fn, ['task_demo4.out'], ['task_demo5.out']))
task_ctrl.add(Task(task_fn, ['task_demo1.out', 'task_demo5.out'], ['task_demo6.out']))