Python get_data Examples

Example #1

def main():
    work_dir = "/home/hujk/ploto/esmvaltool/cases/case108/ploto"
    Path(work_dir).mkdir(parents=True, exist_ok=True)

    # recipe
    exp_datasets = bock20_recipe.fig1.exp_datasets
    variables = bock20_recipe.fig1.variables
    variable_additional_datasets = bock20_recipe.fig1.variable_additional_datasets

    # get all datasets
    datasets = get_datasets(
        datasets=exp_datasets,
        variables=variables,
        variable_additional_datasets=variable_additional_datasets)

    # generate fetcher tasks
    fetcher_tasks = []
    for variable in variables:
        fetcher_tasks.extend(
            get_tasks_for_variable(
                datasets=datasets[variable["variable_group"]],
                work_dir=work_dir))

    # run fetcher tasks
    for task in fetcher_tasks:
        get_data(task, config={}, work_dir=work_dir)

Example #2

File: climwip_map_fetcher.py Project: perillaroc/ploto-esmvaltool

def run(exp_dataset, variable):
    work_dir = "/home/hujk/ploto/esmvaltool/cases/case105/ploto"
    Path(work_dir).mkdir(parents=True, exist_ok=True)

    variable = combine_variable(
        variable=variable,
        dataset=exp_dataset,
    )
    add_variable_info(variable)

    data_path = climwip_config.data_path

    task = {
        "products": [{
            "variable": variable,
            "output": {
                "output_directory": "{dataset}/{variable_group}",
                "output_data_source_file": "data_source.yml",
            }
        }],
        "output": {
            "output_directory":
            "{work_dir}" + f"/{diagnostic_name}/fetcher/preproc",
        },
        "config": {
            "data_path": data_path,
        },
    }

    config = {}

    get_data(task=task, work_dir=work_dir, config=config)

Example #3

def main():
    work_dir = "/home/hujk/ploto/esmvaltool/cases/case106/ploto"
    Path(work_dir).mkdir(parents=True, exist_ok=True)

    exp_datasets = deangelis_recipe.f2ext.exp_datasets
    variables = deangelis_recipe.f2ext.variables

    # get all datasets
    datasets = get_datasets(
        datasets=exp_datasets,
        variables=variables,
    )

    # generate fetcher tasks
    fetcher_tasks = []
    for variable in variables:
        fetcher_tasks.extend(
            get_tasks_for_variable(
                datasets=datasets[variable["variable_group"]],
                diagnostic={"diagnostic": diagnostic_name},
                config={"data_path": deangelis_config.data_path},
                work_dir=work_dir))

    for task in fetcher_tasks:
        get_data(task, config={}, work_dir=work_dir)

Example #4

def run(exp_dataset, variable):
    work_dir = "/home/hujk/ploto/esmvaltool/cases/case102/ploto"

    combined_variable = {**exp_dataset, **variable}
    add_variable_info(combined_variable)

    data_path = diurnal_config.data_path

    task = {
        "products": [{
            "variable": combined_variable,
            "output": {
                "output_directory": "{alias}/{variable_group}",
                "output_data_source_file": "data_source.yml",
            }
        }],
        "config": {
            "data_path": data_path,
        },
        "output": {
            "output_directory": "{work_dir}/fetcher/preproc",
        }
    }

    config = {}

    get_data(task=task, work_dir=work_dir, config=config)

Example #5

File: kcs_global_matching_fetcher.py Project: perillaroc/ploto-esmvaltool

def main():
    work_dir = "/home/hujk/ploto/esmvaltool/cases/future/case301/ploto"
    Path(work_dir).mkdir(parents=True, exist_ok=True)

    # recipe
    variable_additional_datasets = kcs_recipe.global_warming.variable_additional_datasets
    variables = kcs_recipe.global_warming.variables

    # get all datasets
    datasets = get_datasets(
        variables=variables,
        variable_additional_datasets=variable_additional_datasets)

    # generate fetcher tasks
    fetcher_tasks = []
    for variable in variables:
        fetcher_tasks.extend(
            get_tasks_for_variable(
                datasets=datasets[variable["variable_group"]],
                work_dir=work_dir))

    # run fetcher tasks
    for task in fetcher_tasks:
        get_data(task, config={}, work_dir=work_dir)