def plot_correlation_diff(sim_label_to_corr, file_for_basemap="", ax=None, cnorm=None, cmap=None):
    lons, lats, bm = analysis.get_basemap_from_hdf(file_path=file_for_basemap)
    x, y = bm(lons, lats)
    im = bm.pcolormesh(x, y, list(sim_label_to_corr.values())[1] - list(sim_label_to_corr.values())[0], cmap=cmap, norm=cnorm)
    ax.set_title("(2) - (1)")
    bm.drawmapboundary(fill_color="0.75")
    return im
def plot_for_simulation(axis=None, sim_path="", cmap=None, cnorm=None,
                        start_year=None, end_year=None, months=None):
    """
    plot a panel for each simulation

    :param axis:
    :param sim_path:
    :param cmap:
    :param cnorm:
    """

    if months is None:
        months = list(range(1, 13))

    lons, lats, bm = analysis.get_basemap_from_hdf(sim_path)

    params = dict(
        path1=sim_path, path2=sim_path,
        start_year=start_year, end_year=end_year,
        varname1="I1", level1=0,
        varname2="AV", level2=0,
        months=months
    )

    corr, i1_clim, av_clim = calculate_correlation_field_for_climatology(**params)

    # convert longitudes to the [-180, 180] range
    lons[lons > 180] -= 360
    corr = maskoceans(lons, lats, corr)

    x, y = bm(lons, lats)

    im = bm.pcolormesh(x, y, corr, norm=cnorm, cmap=cmap, ax=axis)
    bm.drawcoastlines()
    return im, corr
def main(intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
         no_intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
         start_year=1980, end_year=2010, label="CRCM5-HCD-RL-INTFa"):
    """
    Study impact of interflow only for the cases when liquid soil moisture is greater than field capacity
    """

    img_folder = "intf_during_th_gt_thbfc"

    cache_file = "intf_during_th_gt_thbfc_{}_{}-{}.cache".format(ctypes.c_size_t(hash(intf_file + no_intf_file)).value,
                                                             start_year, end_year)


    if not os.path.isdir(img_folder):
        os.mkdir(img_folder)

    with tb.open_file(intf_file) as intf_handle, tb.open_file(no_intf_file) as no_intf_handle:
        # Get runoff tables (level_index == 0 corresponds to runoff from soil)
        traf_table_intf = intf_handle.get_node("/TRAF")
        traf_table_no_intf = no_intf_handle.get_node("/TRAF")

        th_table_intf = intf_handle.get_node("/I1")
        th_table_no_intf = no_intf_handle.get_node("/I1")

        bfc = get_bulk_field_capacity()

        if os.path.isfile(cache_file):
            total_diff = pickle.load(open(cache_file))
        else:
            total_diff = get_runoff_differences_composit(traf_table_intf=traf_table_intf, th_table_intf=th_table_intf,
                                                         traf_table_no_intf=traf_table_no_intf,
                                                         th_table_no_intf=th_table_no_intf, thbfc_field=bfc,
                                                         start_year=start_year, end_year=end_year)

            pickle.dump(total_diff, open(cache_file, "wb"))

        # save the figure
        plt.figure()

        lons, lats, bm = analysis.get_basemap_from_hdf(intf_file)

        x, y = bm(lons, lats)

        clevs = [0.5, 1, 5, 30, 100, 150, 200]
        clevs = [-c for c in reversed(clevs)] + clevs

        cmap = cm.get_cmap("bwr", len(clevs) - 1)
        bn = BoundaryNorm(clevs, len(clevs) - 1)
        im = bm.pcolormesh(x, y, total_diff, cmap=cmap, norm=bn)
        bm.drawcoastlines()
        bm.colorbar(im, ticks=clevs)
        plt.savefig(os.path.join(img_folder, "{}_traf_diff_{}-{}.png".format(label, start_year, end_year)))



    pass
Example #4
0
def demo_interolate_daily_clim():
    import crcm5.analyse_hdf.do_analysis_using_pytables as analysis

    model_data_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
    start_year = 1980
    end_year = 2010

    vmin = -30
    vmax = 30
    vname = "TT_max"
    coef_mod = 1.0e3 * 24 * 3600 if vname == "PR" else 1.0

    # get target lons and lats for testing
    lon, lat, basemap = analysis.get_basemap_from_hdf(
        file_path=model_data_path)

    ans = AnuSplinManager(variable="stmx")
    dates, fields = ans.get_daily_clim_fields_interpolated_to(start_year=start_year,
                                                              end_year=end_year,
                                                              lons_target=lon, lats_target=lat)
    import matplotlib.pyplot as plt

    x, y = basemap(lon, lat)

    margin = 20
    topo = _get_topography()[margin:-margin, margin:-margin]

    # Plot obs data
    plt.figure()
    mean_obs = np.ma.array([fields[i] for i, d in enumerate(dates) if d.month in range(1, 13)]).mean(axis=0)
    im = basemap.pcolormesh(x, y, mean_obs, vmin=vmin, vmax=vmax)
    basemap.colorbar(im)
    basemap.drawcoastlines()
    plt.title("Anusplin")
    print("Obs stdev = {}".format(mean_obs[~mean_obs.mask].std()))

    print("Obs correlations: ", np.corrcoef(mean_obs[~mean_obs.mask], topo[~mean_obs.mask]))

    # Plot model data
    plt.figure()
    dates, fields = analysis.get_daily_climatology(path_to_hdf_file=model_data_path, var_name=vname,
                                                   level=0,
                                                   start_year=start_year, end_year=end_year)

    mean_mod = np.array([fields[i] for i, d in enumerate(dates) if d.month in range(1, 13)]).mean(axis=0) * coef_mod
    im = basemap.pcolormesh(x, y, mean_mod, vmin=vmin, vmax=vmax)
    basemap.colorbar(im)
    basemap.drawcoastlines()
    plt.title("Model")

    print("Model correlations: ", np.corrcoef(mean_mod[~mean_obs.mask], topo[~mean_obs.mask]))
    print("Model stdev = {}".format(mean_mod[~mean_obs.mask].std()))

    plt.show()
def plot_hydrographs():
    plot_utils.apply_plot_params(font_size=14, width_pt=None, width_cm=20, height_cm=20)
    start_year = 1980
    end_year = 2010

    varname = "STFL"

    base_config = RunConfig(start_year=start_year, end_year=end_year,
                            data_path="/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
                            label="NI")

    modif_config = RunConfig(start_year=start_year, end_year=end_year,
                             data_path="/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
                             label="WI")


    r_obj = RPN(GEO_DATA_FILE)
    facc = r_obj.get_first_record_for_name("FAA")
    fldr = r_obj.get_first_record_for_name("FLDR")

    lons, lats, bmp = analysis.get_basemap_from_hdf(file_path=base_config.data_path)

    basin_name_to_out_indices_map, basin_name_to_basin_mask = get_basin_to_outlet_indices_map(lons=lons, lats=lats,
                                                                                              accumulation_areas=facc,
                                                                                              directions=fldr)
    # Calculate the daily mean fields
    dates, stf_base = analysis.get_daily_climatology_for_rconf(base_config, var_name=varname, level=0)
    _, stf_modif = analysis.get_daily_climatology_for_rconf(modif_config, var_name=varname, level=0)

    for bname, (i_out, j_out) in basin_name_to_out_indices_map.items():
        print(bname, i_out, j_out)
        fig = plt.figure()

        gs = GridSpec(2, 1, height_ratios=[1, 0.5], hspace=0.1)

        ax = fig.add_subplot(gs[0, 0])
        ax.plot(dates, stf_base[:, i_out, j_out], "b", lw=2, label=base_config.label)
        ax.plot(dates, stf_modif[:, i_out, j_out], "r", lw=2, label=modif_config.label)
        ax.set_title(bname)
        format_axis(ax)

        # Hide the tick labels from the x-axis of the upper plot
        for tl in ax.xaxis.get_ticklabels():
            tl.set_visible(False)


        ax = fig.add_subplot(gs[1, 0])
        ax.plot(dates, stf_modif[:, i_out, j_out] - stf_base[:, i_out, j_out], "k", lw=2,
                label="{}-{}".format(modif_config.label, base_config.label))
        format_axis(ax)


        fig.savefig(str(IMG_FOLDER.joinpath("{}_{}-{}.png".format(bname, start_year, end_year))))
        plt.close(fig)
def point_comparisons_at_outlets(hdf_folder="/home/huziy/skynet3_rech1/hdf_store"):
    start_year = 1979
    end_year = 1981

    sim_name_to_file_name = {
        # "CRCM5-R": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf",
        # "CRCM5-HCD-R": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf",
        "CRCM5-HCD-RL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf",
        "CRCM5-HCD-RL-INTFL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf",
        # "SANI=10000, ignore THFC":
        # "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf",

        # "CRCM5-HCD-RL-ERA075": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf",
        "SANI=10000": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf"
        # "CRCM5-HCD-RL-ECOCLIMAP": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap.hdf"
    }

    path0 = os.path.join(hdf_folder, list(sim_name_to_file_name.items())[0][1])
    path1 = os.path.join(hdf_folder, list(sim_name_to_file_name.items())[1][1])
    flow_directions = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
    lake_fraction = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_LAKE_FRACTION_NAME)
    slope = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_SLOPE_NAME)

    lons2d, lats2d, _ = analysis.get_basemap_from_hdf(file_path=path0)

    cell_manager = CellManager(flow_directions, lons2d=lons2d, lats2d=lats2d)
    mp_list = cell_manager.get_model_points_of_outlets(lower_accumulation_index_limit=10)

    assert len(mp_list) > 0

    # Get the accumulation indices so that the most important outlets can be identified
    acc_ind_list = [np.sum(cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(mp.ix, mp.jy))
                    for mp in mp_list]

    for mp, acc_ind in zip(mp_list, acc_ind_list):
        mp.acc_index = acc_ind

    mp_list.sort(key=lambda x: x.acc_index)

    # do not take global lake cells into consideration, and discard points with slopes 0 or less
    mp_list = [mp for mp in mp_list if lake_fraction[mp.ix, mp.jy] < 0.6 and slope[mp.ix, mp.jy] >= 0]

    mp_list = mp_list[-12:]  # get 12 most important outlets

    print("The following outlets were chosen for analysis")
    pattern = "({0}, {1}): acc_index = {2} cells; fldr = {3}; lake_fraction = {4}"
    for mp in mp_list:
        print(pattern.format(mp.ix, mp.jy, mp.acc_index, cell_manager.flow_directions[mp.ix, mp.jy],
                             lake_fraction[mp.ix, mp.jy]))

    draw_model_comparison(model_points=mp_list, sim_name_to_file_name=sim_name_to_file_name, hdf_folder=hdf_folder,
                          start_year=start_year, end_year=end_year, cell_manager=cell_manager)
def main():
    start_year = 1980
    end_year = 2010

    base_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    base_label = "CRCM5-L"

    label_to_path = {
        "CRCM5-LI": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    }

    season_to_months = OrderedDict([
        ("Winter", [12, 1, 2]),
        ("Spring", list(range(3, 6))),
        ("Summer", list(range(6, 9))),
        ("Fall", list(range(9, 12)))
    ])

    coords = CoordsAndBg(*analysis.get_basemap_from_hdf(base_path))

    daily_dates, tmax_daily_base = analysis.get_daily_max_climatology(base_path, var_name="TT_max", level=0,
                                                                      start_year=start_year, end_year=end_year)

    _, tmin_daily_base = analysis.get_daily_min_climatology(base_path, var_name="TT_min", level=0,
                                                            start_year=start_year, end_year=end_year)

    label_to_tmax_daily = {}
    label_to_tmin_daily = {}

    for label, the_path in label_to_path.items():
        _, label_to_tmax_daily[label] = analysis.get_daily_max_climatology(the_path, var_name="TT_max", level=0,
                                                                           start_year=start_year, end_year=end_year)

        _, label_to_tmin_daily[label] = analysis.get_daily_min_climatology(the_path, var_name="TT_min", level=0,
                                                                           start_year=start_year, end_year=end_year)


    # Plot results
    plot_impacts_of_intfl_on_seasonal_means(var_name="Tmin", base_label=base_label,
                                            base_data_daily=tmin_daily_base,
                                            label_to_data_daily=label_to_tmin_daily,
                                            coord_obj=coords, season_to_months=season_to_months,
                                            daily_dates=daily_dates)

    plot_impacts_of_intfl_on_seasonal_means(var_name="Tmax", base_label=base_label,
                                            base_data_daily=tmax_daily_base,
                                            label_to_data_daily=label_to_tmax_daily,
                                            coord_obj=coords, season_to_months=season_to_months,
                                            daily_dates=daily_dates)
Example #8
0
def demo_seasonal_mean():
    import matplotlib.pyplot as plt
    import crcm5.analyse_hdf.do_analysis_using_pytables as analysis
    # get target lons and lats for testing
    lon, lat, basemap = analysis.get_basemap_from_hdf(
        file_path="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf")

    x, y = basemap(lon, lat)

    am = AnuSplinManager()
    the_field = am.getMeanFieldForMonthsInterpolatedTo(start_year=1980, end_year=1988, lonstarget=lon, latstarget=lat)

    basemap.pcolormesh(x, y, the_field)
    basemap.colorbar()
    basemap.drawcoastlines()
    plt.show()
Example #9
0
def plot_correlation_diff(sim_label_to_corr,
                          file_for_basemap="",
                          ax=None,
                          cnorm=None,
                          cmap=None):
    lons, lats, bm = analysis.get_basemap_from_hdf(file_path=file_for_basemap)
    x, y = bm(lons, lats)
    im = bm.pcolormesh(x,
                       y,
                       list(sim_label_to_corr.values())[1] -
                       list(sim_label_to_corr.values())[0],
                       cmap=cmap,
                       norm=cnorm)
    ax.set_title("(2) - (1)")
    bm.drawmapboundary(fill_color="0.75")
    return im
Example #10
0
def plot_for_simulation(axis=None,
                        sim_path="",
                        cmap=None,
                        cnorm=None,
                        start_year=None,
                        end_year=None,
                        months=None):
    """
    plot a panel for each simulation

    :param axis:
    :param sim_path:
    :param cmap:
    :param cnorm:
    """

    if months is None:
        months = list(range(1, 13))

    lons, lats, bm = analysis.get_basemap_from_hdf(sim_path)

    params = dict(path1=sim_path,
                  path2=sim_path,
                  start_year=start_year,
                  end_year=end_year,
                  varname1="I1",
                  level1=0,
                  varname2="AV",
                  level2=0,
                  months=months)

    corr, i1_clim, av_clim = calculate_correlation_field_for_climatology(
        **params)

    # convert longitudes to the [-180, 180] range
    lons[lons > 180] -= 360
    corr = maskoceans(lons, lats, corr)

    x, y = bm(lons, lats)

    im = bm.pcolormesh(x, y, corr, norm=cnorm, cmap=cmap, ax=axis)
    bm.drawcoastlines()
    return im, corr
Example #11
0
def demo_seasonal_mean():
    import matplotlib.pyplot as plt
    import crcm5.analyse_hdf.do_analysis_using_pytables as analysis
    # get target lons and lats for testing
    lon, lat, basemap = analysis.get_basemap_from_hdf(
        file_path=
        "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf")

    x, y = basemap(lon, lat)

    am = AnuSplinManager()
    the_field = am.getMeanFieldForMonthsInterpolatedTo(start_year=1980,
                                                       end_year=1988,
                                                       lonstarget=lon,
                                                       latstarget=lat)

    basemap.pcolormesh(x, y, the_field)
    basemap.colorbar()
    basemap.drawcoastlines()
    plt.show()
def main(intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
         no_intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
         start_year=1980, end_year=2010):
    """
    Get runoff difference only for melting periods
    :param intf_file:
    :param no_intf_file:
    :param start_year:
    :param end_year:
    """

    matplotlib.rc("font", size=20)

    img_folder = "intf_during_melting"

    cache_file = "intf_during_melting_{}_{}-{}.cache".format(ctypes.c_size_t(hash(intf_file + no_intf_file)).value,
                                                             start_year, end_year)

    if not os.path.isdir(img_folder):
        os.mkdir(img_folder)

    with tb.open_file(intf_file) as intf_handle, tb.open_file(no_intf_file) as no_intf_handle:
        # Get runoff tables (level_index == 0 corresponds to runoff from soil)
        traf_table_intf = intf_handle.get_node("/TRAF")
        traf_table_no_intf = no_intf_handle.get_node("/TRAF")

        # Get swe to determine when melting is happening
        swe_table_intf = intf_handle.get_node("/I5")
        swe_table_no_intf = no_intf_handle.get_node("/I5")

        if os.path.isfile(cache_file):
            total_diff = pickle.load(open(cache_file))
        else:
            total_diff = get_runoff_diff_composit(swe_table_intf=swe_table_intf,
                                                  traf_table_intf=traf_table_intf,
                                                  swe_table_no_intf=swe_table_no_intf,
                                                  traf_table_no_intf=traf_table_no_intf,
                                                  start_year=start_year, end_year=end_year)

            pickle.dump(total_diff, open(cache_file, "wb"))

        # save the figure
        plt.figure()
        total_diff /= float(end_year - start_year + 1)

        # Convert to mm
        total_diff *= 3 * 60 * 60
        lons, lats, bm = analysis.get_basemap_from_hdf(intf_file)

        x, y = bm(lons, lats)

        clevs = [0.5, 1, 5, 30, 100, 150]
        clevs = [-c for c in reversed(clevs)] + clevs

        cmap = cm.get_cmap("bwr", len(clevs) - 1)
        bn = BoundaryNorm(clevs, len(clevs) - 1)
        im = bm.pcolormesh(x, y, total_diff, cmap=cmap, norm=bn)
        bm.drawcoastlines()
        bm.colorbar(im, ticks=clevs)
        plt.savefig(os.path.join(img_folder, "traf_diff_{}-{}.png".format(start_year, end_year)))

    pass
def main():
    import application_properties
    application_properties.set_current_directory()
    if not IMG_FOLDER.exists():
        IMG_FOLDER.mkdir(parents=True)

    plot_utils.apply_plot_params(font_size=14,
                                 width_pt=None,
                                 width_cm=20,
                                 height_cm=20)
    start_year = 1980
    end_year = 2010

    varname = "TDRA"

    base_config = RunConfig(
        start_year=start_year,
        end_year=end_year,
        data_path=
        "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
        label="NI")

    modif_config = RunConfig(
        start_year=start_year,
        end_year=end_year,
        data_path=
        "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
        label="WI")

    r_obj = RPN(GEO_DATA_FILE)
    facc = r_obj.get_first_record_for_name("FAA")
    fldr = r_obj.get_first_record_for_name("FLDR")
    mg = r_obj.get_first_record_for_name("MG")

    lons, lats, bmp = analysis.get_basemap_from_hdf(
        file_path=base_config.data_path)

    # Calculate the daily mean fields
    dates, daily_clim_base = analysis.get_daily_climatology_for_rconf(
        base_config, var_name=varname, level=0)
    _, daily_clim_modif = analysis.get_daily_climatology_for_rconf(
        modif_config, var_name=varname, level=0)

    _, pr_base = analysis.get_daily_climatology_for_rconf(base_config,
                                                          var_name="PR",
                                                          level=0)
    _, pr_modif = analysis.get_daily_climatology_for_rconf(modif_config,
                                                           var_name="PR",
                                                           level=0)

    _, av_base = analysis.get_daily_climatology_for_rconf(base_config,
                                                          var_name="AV",
                                                          level=0)
    _, av_modif = analysis.get_daily_climatology_for_rconf(modif_config,
                                                           var_name="AV",
                                                           level=0)

    _, intf_rates = analysis.get_daily_climatology_for_rconf(modif_config,
                                                             var_name="INTF",
                                                             level=0)

    # Plot the difference
    fig = plt.figure()
    xx, yy = bmp(lons, lats)

    # daily_clim_base = np.array([f for d, f in zip(dates, daily_clim_base) if d.month not in range(3, 12)])
    # daily_clim_modif = np.array([f for d, f in zip(dates, daily_clim_modif) if d.month not in range(3, 12)])

    mean_base = np.mean(daily_clim_base, axis=0)
    diff = (np.mean(daily_clim_modif, axis=0) - mean_base) * 24 * 3600
    dpr = (pr_modif.sum(axis=0) - pr_base.sum(axis=0)) / pr_base.sum(axis=0)
    dav = (av_modif.sum(axis=0) - av_base.sum(axis=0)) / av_base.sum(axis=0)
    diff = np.ma.masked_where(
        (mg <= 1.0e-3) | (dpr < 0) | (dav > 0) | (diff > 0), diff)

    print("{}-ranges: {}, {}".format(varname, daily_clim_base.min(),
                                     daily_clim_base.max()))
    print("{}-ranges: {}, {}".format(varname, daily_clim_modif.min(),
                                     daily_clim_modif.max()))

    limit_base = np.percentile(daily_clim_base, 90, axis=0)
    limit_modif = np.percentile(daily_clim_modif, 50, axis=0)
    limit_lower_intf = 1.0e-4 / (24.0 * 60.0 * 60.0)

    ndays_base = daily_clim_base > limit_base[np.newaxis, :, :]
    ndays_base = ndays_base.sum(axis=0)

    ndays_modif = daily_clim_modif > limit_base[np.newaxis, :, :]
    ndays_modif = ndays_modif.sum(axis=0)

    diff_days = np.ma.masked_where(
        (mg <= 1.0e-4) | (intf_rates.max() < limit_lower_intf),
        ndays_modif - ndays_base)

    diff_days = np.ma.masked_where(diff_days > 0, diff_days)

    print("diff_days ranges: {} to {}".format(diff_days.min(),
                                              diff_days.max()))

    im = bmp.pcolormesh(xx, yy, diff)
    bmp.colorbar(im)
    bmp.drawcoastlines()

    img_file = IMG_FOLDER.joinpath("{}_{}-{}.png".format(
        varname, start_year, end_year))
    with img_file.open("wb") as imf:
        fig.savefig(imf, bbox_inches="tight")

    plt.show()
def main(
        intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
        no_intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
        start_year=1980,
        end_year=2010):
    """
    Get runoff difference only for melting periods
    :param intf_file:
    :param no_intf_file:
    :param start_year:
    :param end_year:
    """

    matplotlib.rc("font", size=20)

    img_folder = "intf_during_melting"

    cache_file = "intf_during_melting_{}_{}-{}.cache".format(
        ctypes.c_size_t(hash(intf_file + no_intf_file)).value, start_year,
        end_year)

    if not os.path.isdir(img_folder):
        os.mkdir(img_folder)

    with tb.open_file(intf_file) as intf_handle, tb.open_file(
            no_intf_file) as no_intf_handle:
        # Get runoff tables (level_index == 0 corresponds to runoff from soil)
        traf_table_intf = intf_handle.get_node("/TRAF")
        traf_table_no_intf = no_intf_handle.get_node("/TRAF")

        # Get swe to determine when melting is happening
        swe_table_intf = intf_handle.get_node("/I5")
        swe_table_no_intf = no_intf_handle.get_node("/I5")

        if os.path.isfile(cache_file):
            total_diff = pickle.load(open(cache_file))
        else:
            total_diff = get_runoff_diff_composit(
                swe_table_intf=swe_table_intf,
                traf_table_intf=traf_table_intf,
                swe_table_no_intf=swe_table_no_intf,
                traf_table_no_intf=traf_table_no_intf,
                start_year=start_year,
                end_year=end_year)

            pickle.dump(total_diff, open(cache_file, "wb"))

        # save the figure
        plt.figure()
        total_diff /= float(end_year - start_year + 1)

        # Convert to mm
        total_diff *= 3 * 60 * 60
        lons, lats, bm = analysis.get_basemap_from_hdf(intf_file)

        x, y = bm(lons, lats)

        clevs = [0.5, 1, 5, 30, 100, 150]
        clevs = [-c for c in reversed(clevs)] + clevs

        cmap = cm.get_cmap("bwr", len(clevs) - 1)
        bn = BoundaryNorm(clevs, len(clevs) - 1)
        im = bm.pcolormesh(x, y, total_diff, cmap=cmap, norm=bn)
        bm.drawcoastlines()
        bm.colorbar(im, ticks=clevs)
        plt.savefig(
            os.path.join(img_folder,
                         "traf_diff_{}-{}.png".format(start_year, end_year)))

    pass
def compare(paths=None, path_to_control_data=None, control_label="",
            labels=None, varnames=None, levels=None, months_of_interest=None,
            start_year=None, end_year=None):
    """
    Comparing 2D fields
    :param paths: paths to the simulation results
    :param varnames:
    :param labels: Display name for each simulation (number of labels should
     be equal to the number of paths)
    :param path_to_control_data: the path with which the comparison done i.e. a in the following
     formula
            delta = (x - a)/a * 100%

     generates one image file per variable (in the folder images_for_lake-river_paper):
        compare_varname_<control_label>_<label1>_..._<labeln>_startyear_endyear.png

    """
    # get coordinate data  (assumes that all the variables and runs have the same coordinates)
    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path_to_control_data)
    x, y = basemap(lons2d, lats2d)

    lake_fraction = analysis.get_array_from_file(path=path_to_control_data, var_name="lake_fraction")

    if lake_fraction is None:
        lake_fraction = np.zeros(lons2d.shape)

    ncolors = 10
    # +1 to include white
    diff_cmap = cm.get_cmap("RdBu_r", ncolors + 1)

    for var_name, level in zip(varnames, levels):
        sfmt = infovar.get_colorbar_formatter(var_name)
        control_means = analysis.get_mean_2d_fields_for_months(path=path_to_control_data, var_name=var_name,
                                                               months=months_of_interest,
                                                               start_year=start_year, end_year=end_year,
                                                               level=level)

        control_mean = np.mean(control_means, axis=0)
        fig = plt.figure()
        assert isinstance(fig, Figure)
        gs = gridspec.GridSpec(2, len(paths) + 1, wspace=0.5)

        # plot the control
        ax = fig.add_subplot(gs[0, 0])
        assert isinstance(ax, Axes)
        ax.set_title("{0}".format(control_label))
        ax.set_ylabel("Mean: $X_{0}$")
        to_plot = infovar.get_to_plot(var_name, control_mean,
                                      lake_fraction=lake_fraction, mask_oceans=True, lons=lons2d, lats=lats2d)
        # determine colorabr extent and spacing
        field_cmap, field_norm = infovar.get_colormap_and_norm_for(var_name, to_plot, ncolors=ncolors)

        basemap.pcolormesh(x, y, to_plot, cmap=field_cmap, norm=field_norm)
        cb = basemap.colorbar(format=sfmt)

        assert isinstance(cb, Colorbar)
        # cb.ax.set_ylabel(infovar.get_units(var_name))
        units = infovar.get_units(var_name)

        info = "Variable:" \
               "\n{0}" \
               "\nPeriod: {1}-{2}" \
               "\nMonths: {3}" \
               "\nUnits: {4}"

        info = info.format(infovar.get_long_name(var_name), start_year, end_year,
                           ",".join([datetime(2001, m, 1).strftime("%b") for m in months_of_interest]), units)

        ax.annotate(info, xy=(0.1, 0.3), xycoords="figure fraction")

        sel_axes = [ax]

        for the_path, the_label, column in zip(paths, labels, list(range(1, len(paths) + 1))):

            means_for_years = analysis.get_mean_2d_fields_for_months(path=the_path, var_name=var_name,
                                                                     months=months_of_interest,
                                                                     start_year=start_year, end_year=end_year)
            the_mean = np.mean(means_for_years, axis=0)

            # plot the mean value
            ax = fig.add_subplot(gs[0, column])
            sel_axes.append(ax)
            ax.set_title("{0}".format(the_label))
            to_plot = infovar.get_to_plot(var_name, the_mean, lake_fraction=lake_fraction,
                                          mask_oceans=True, lons=lons2d, lats=lats2d)

            basemap.pcolormesh(x, y, to_plot, cmap=field_cmap, norm=field_norm)
            ax.set_ylabel("Mean: $X_{0}$".format(column))
            cb = basemap.colorbar(format=sfmt)
            # cb.ax.set_ylabel(infovar.get_units(var_name))

            # plot the difference
            ax = fig.add_subplot(gs[1, column])
            sel_axes.append(ax)
            ax.set_ylabel("$X_{0} - X_0$".format(column))

            # #Mask only if the previous plot (means) is masked
            thediff = the_mean - control_mean

            if hasattr(to_plot, "mask"):
                to_plot = np.ma.masked_where(to_plot.mask, thediff)
            else:
                to_plot = thediff

            if var_name == "PR":  # convert to mm/day
                to_plot = infovar.get_to_plot(var_name, to_plot, mask_oceans=False)

            vmin = np.ma.min(to_plot)
            vmax = np.ma.max(to_plot)

            d = max(abs(vmin), abs(vmax))
            vmin = -d
            vmax = d

            field_norm, bounds, vmn_nice, vmx_nice = infovar.get_boundary_norm(vmin, vmax, diff_cmap.N,
                                                                               exclude_zero=False)
            basemap.pcolormesh(x, y, to_plot, cmap=diff_cmap, norm=field_norm, vmin=vmn_nice, vmax=vmx_nice)

            cb = basemap.colorbar(format=sfmt)

            t, pval = ttest_ind(means_for_years, control_means, axis=0)
            sig = pval < 0.1
            basemap.contourf(x, y, sig.astype(int), nlevels=2, hatches=["+", None], colors="none")

            # cb.ax.set_ylabel(infovar.get_units(var_name))

        # plot coastlines
        for the_ax in sel_axes:
            basemap.drawcoastlines(ax=the_ax, linewidth=common_plot_params.COASTLINE_WIDTH)

        # depends on the compared simulations and the months of interest
        fig_file_name = "compare_{0}_{1}_{2}_months-{3}.jpeg".format(var_name, control_label,
                                                                     "_".join(labels),
                                                                     "-".join([str(m) for m in months_of_interest]))
        figpath = os.path.join(images_folder, fig_file_name)
        fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
        plt.close(fig)
def main(interflow_data_path="", base_data_path="",
         start_year=1980, end_year=2010, months_of_interest=(4, 5, 6, 7, 8, 9),
         debug_plots=False, dt_seconds=3 * 60 * 60):
    # have a field diff accumulator: dacc
    # rain_regions1 = (PR1 > 0) & (I01[:,:,0] > 273.15)
    # rain_regions2 = (PR2 > 0) & (I02[:,:,0] > 273.15)
    # rain_regions = rain_regions1 & rain_regions2
    # dacc[rain_regions] += TRAF1[rain_regions]  <-- Runoff effect

    traf_diff, prcip_diff, tdra_diff, i1_diff = get_mean_diffs(interflow_data_path=interflow_data_path,
                                                               base_data_path=base_data_path,
                                                               start_year=start_year, end_year=end_year,
                                                               months_of_interest=months_of_interest)

    # Get coordinates of the fields
    lons, lats, bsm = pt_analysis.get_basemap_from_hdf(base_data_path)
    x, y = bsm(lons, lats)

    # Folder containing debug images
    img_dir = "test_intf_extr/exp_avoid_small_incrs_{}-{}".format(start_year, end_year)
    if not os.path.isdir(img_dir):
        os.mkdir(img_dir)


    # Plot surface runoff
    plt.figure()
    clevs = [5, 10, 20, 50]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x, y, traf_diff * dt_seconds / float(end_year - start_year + 1), norm=bn,
                        cmap=cmap)
    plt.colorbar(im)
    plt.title("Surface runoff (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/traf_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year, end_year))

    # Plot precipitation
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x, y, prcip_diff * dt_seconds * 1000.0 / float(end_year - start_year + 1),
                        cmap=cmap, norm=bn)
    plt.colorbar(im)
    plt.title("Precipitation (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/precip_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year, end_year))

    ###Plot drainage
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x, y, tdra_diff * dt_seconds / float(end_year - start_year + 1), cmap=cmap, norm=bn)
    plt.colorbar(im)
    plt.title("Drainage (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/tdra_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year, end_year))

    ###Plot first soil layer moisture
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x, y, i1_diff / float(end_year - start_year + 1), cmap=None, norm=None)
    plt.colorbar(im)
    plt.title("Liquid soil moisture (mm)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/I1_diff_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year, end_year))
Example #17
0
def main():
    season_to_months = get_default_season_to_months_dict()

    # var_names = ["TT", "HU", "PR", "AV", "STFL"]
    # var_names = ["TRAF", "STFL", "TRAF+TDRA"]
    # var_names = ["TT", "PR", "STFL"]
    # var_names = ["TT", "PR", "I5", "STFL", "AV", "AH"]

    var_names = ["TT", ]

    levels = [0, ] * len(var_names)
    multipliers = {
        "PR": 1.0,
        "TRAF+TDRA": 24 * 60 * 60
    }
    name_to_units = {
        "TRAF": "mm/day", "I1": "mm", "PR": "mm/day", "TRAF+TDRA": "mm/day",
        "I5": "mm", "AV": r"${\rm W/m^2}$"
    }

    base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                        "quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5"
    base_label = "CanESM2-CRCM5-NL"

    modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                         "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    modif_label = "CanESM2-CRCM5-L"

    plot_cc_only_for = modif_label

    # plot_cc_only_for = None

    start_year_c = 1980
    end_year_c = 2010

    future_shift_years = 90

    params = dict(
        data_path=base_current_path, start_year=start_year_c, end_year=end_year_c, label=base_label
    )

    base_config_c = RunConfig(**params)
    base_config_f = base_config_c.get_shifted_config(future_shift_years)

    params.update(
        dict(data_path=modif_current_path, label=modif_label)
    )

    modif_config_c = RunConfig(**params)
    modif_config_f = modif_config_c.get_shifted_config(future_shift_years)

    config_dict = OrderedDict([
        ("Current", OrderedDict([(base_label, base_config_c), (modif_label, modif_config_c)])),
        ("Future", OrderedDict([(base_label, base_config_f), (modif_label, modif_config_f)]))
    ])

    # Changes global plot properties mainly figure size and font size
    plot_utils.apply_plot_params(font_size=12, width_cm=25, height_cm=10)


    # Plot the differences
    fig = plt.figure()
    gs = GridSpec(len(var_names), len(season_to_months) + 1, width_ratios=[1., ] * len(season_to_months) + [0.05, ])

    config_dict.fig = fig
    config_dict.gs = gs
    config_dict.label_modif = modif_config_c.label
    config_dict.label_base = base_config_c.label
    config_dict.season_to_months = season_to_months
    config_dict.multipliers = multipliers

    lons, lats, bmp = analysis.get_basemap_from_hdf(base_current_path)
    config_dict.lons = lons
    config_dict.lats = lats
    config_dict.basemap = bmp

    config_dict.name_to_units = name_to_units

    # Calculate and plot seasonal means
    for vname, level, the_row in zip(var_names, levels, list(range(len(levels)))):
        config_dict.the_row = the_row

        _plot_row(vname=vname, level=level, config_dict=config_dict, plot_cc_only_for=plot_cc_only_for, mark_significance=False)

    # Save the image to the file
    if plot_cc_only_for is None:
        img_path = get_image_path(base_config_c, base_config_f, modif_config_c, season_to_months=season_to_months)
    else:

        config_c = base_config_c if base_config_c.label == plot_cc_only_for else modif_config_c
        config_f = base_config_f if base_config_f.label == plot_cc_only_for else modif_config_f

        img_path = get_image_path(config_c, config_f, config_c, season_to_months=season_to_months)
    fig.savefig(img_path, bbox_inches="tight", transparent=True, dpi=common_plot_params.FIG_SAVE_DPI)
    print("saving the plot to: {}".format(img_path))
    plt.close(fig)
def main():
    """

    """

    season_to_months = get_default_season_to_months_dict()

    # season_to_months = OrderedDict([("April", [4, ]), ("May", [5, ]), ("June", [6, ]), ("July", [7, ])])

    var_names = ["TT", "HU", "PR", "AV", "AH", "STFL", "TRAF", "I5", "I0"]

    # var_names = ["TT", "PR"]

    levels = [0, ] * len(var_names)
    multipliers = {
        "PR": 1.0e3 * 24.0 * 3600.,
        "TRAF": 24 * 3600.0
    }

    base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                        "quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5"
    base_label = "CanESM2-CRCM5-NL"

    modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                         "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    modif_label = "CanESM2-CRCM5-L"

    # base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
    # "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    # base_label = "CRCM5-L"
    #
    # modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
    # "quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5"
    # modif_label = "CRCM5-LI"

    start_year_c = 1980
    end_year_c = 2010

    future_shift_years = 90

    params = dict(
        data_path=base_current_path, start_year=start_year_c, end_year=end_year_c, label=base_label
    )

    base_config_c = RunConfig(**params)
    base_config_f = base_config_c.get_shifted_config(future_shift_years)

    params.update(
        dict(data_path=modif_current_path, label=modif_label)
    )

    modif_config_c = RunConfig(**params)
    modif_config_f = modif_config_c.get_shifted_config(future_shift_years)

    config_dict = OrderedDict([
        ("Current", OrderedDict([(base_label, base_config_c), (modif_label, modif_config_c)])),
        ("Future", OrderedDict([(base_label, base_config_f), (modif_label, modif_config_f)]))
    ])

    # Plot the differences
    config_dict.label_modif = modif_config_c.label
    config_dict.label_base = base_config_c.label
    config_dict.season_to_months = season_to_months
    config_dict.multipliers = multipliers

    lons, lats, bmp = analysis.get_basemap_from_hdf(base_current_path)
    config_dict.lons = lons
    config_dict.lats = lats
    config_dict.basemap = bmp

    # Calculate and plot seasonal means
    for vname, level in zip(var_names, levels):
        data = get_data(vname=vname, level=level, config_dict=config_dict)
        _plot_var(vname=vname, level=level, config_dict=config_dict, data_dict=data)
def main(interflow_data_path="",
         base_data_path="",
         start_year=1980,
         end_year=2010,
         months_of_interest=(4, 5, 6, 7, 8, 9),
         debug_plots=False,
         dt_seconds=3 * 60 * 60):
    # have a field diff accumulator: dacc
    # rain_regions1 = (PR1 > 0) & (I01[:,:,0] > 273.15)
    # rain_regions2 = (PR2 > 0) & (I02[:,:,0] > 273.15)
    # rain_regions = rain_regions1 & rain_regions2
    # dacc[rain_regions] += TRAF1[rain_regions]  <-- Runoff effect

    traf_diff, prcip_diff, tdra_diff, i1_diff = get_mean_diffs(
        interflow_data_path=interflow_data_path,
        base_data_path=base_data_path,
        start_year=start_year,
        end_year=end_year,
        months_of_interest=months_of_interest)

    # Get coordinates of the fields
    lons, lats, bsm = pt_analysis.get_basemap_from_hdf(base_data_path)
    x, y = bsm(lons, lats)

    # Folder containing debug images
    img_dir = "test_intf_extr/exp_avoid_small_incrs_{}-{}".format(
        start_year, end_year)
    if not os.path.isdir(img_dir):
        os.mkdir(img_dir)

    # Plot surface runoff
    plt.figure()
    clevs = [5, 10, 20, 50]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x,
                        y,
                        traf_diff * dt_seconds /
                        float(end_year - start_year + 1),
                        norm=bn,
                        cmap=cmap)
    plt.colorbar(im)
    plt.title("Surface runoff (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/traf_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year,
        end_year))

    # Plot precipitation
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x,
                        y,
                        prcip_diff * dt_seconds * 1000.0 /
                        float(end_year - start_year + 1),
                        cmap=cmap,
                        norm=bn)
    plt.colorbar(im)
    plt.title("Precipitation (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/precip_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year,
        end_year))

    ###Plot drainage
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x,
                        y,
                        tdra_diff * dt_seconds /
                        float(end_year - start_year + 1),
                        cmap=cmap,
                        norm=bn)
    plt.colorbar(im)
    plt.title("Drainage (mm/year)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/tdra_diff_traf_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year,
        end_year))

    ###Plot first soil layer moisture
    plt.figure()
    clevs = [1, 2, 5, 10, 20, 40]
    clevs = [-c for c in reversed(clevs)] + clevs
    cmap = cm.get_cmap(name="seismic", lut=len(clevs) - 1)
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    im = bsm.pcolormesh(x,
                        y,
                        i1_diff / float(end_year - start_year + 1),
                        cmap=None,
                        norm=None)
    plt.colorbar(im)
    plt.title("Liquid soil moisture (mm)")
    bsm.drawcoastlines(linewidth=0.5)
    plt.savefig("{}/I1_diff_due_to_intf_{}_{}-{}.jpg".format(
        img_dir, "-".join(str(m) for m in months_of_interest), start_year,
        end_year))
Example #20
0
def main():
    import application_properties
    application_properties.set_current_directory()
    if not IMG_FOLDER.exists():
        IMG_FOLDER.mkdir(parents=True)

    plot_utils.apply_plot_params(font_size=14, width_pt=None, width_cm=20, height_cm=20)
    start_year = 1980
    end_year = 2010

    varname = "TDRA"

    base_config = RunConfig(start_year=start_year, end_year=end_year,
                            data_path="/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
                            label="NI")

    modif_config = RunConfig(start_year=start_year, end_year=end_year,
                             data_path="/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
                             label="WI")


    r_obj = RPN(GEO_DATA_FILE)
    facc = r_obj.get_first_record_for_name("FAA")
    fldr = r_obj.get_first_record_for_name("FLDR")
    mg = r_obj.get_first_record_for_name("MG")

    lons, lats, bmp = analysis.get_basemap_from_hdf(file_path=base_config.data_path)

    # Calculate the daily mean fields
    dates, daily_clim_base = analysis.get_daily_climatology_for_rconf(base_config, var_name=varname, level=0)
    _, daily_clim_modif = analysis.get_daily_climatology_for_rconf(modif_config, var_name=varname, level=0)

    _, pr_base = analysis.get_daily_climatology_for_rconf(base_config, var_name="PR", level=0)
    _, pr_modif = analysis.get_daily_climatology_for_rconf(modif_config, var_name="PR", level=0)

    _, av_base = analysis.get_daily_climatology_for_rconf(base_config, var_name="AV", level=0)
    _, av_modif = analysis.get_daily_climatology_for_rconf(modif_config, var_name="AV", level=0)


    _, intf_rates = analysis.get_daily_climatology_for_rconf(modif_config, var_name="INTF", level=0)

    # Plot the difference
    fig = plt.figure()
    xx, yy = bmp(lons, lats)

    # daily_clim_base = np.array([f for d, f in zip(dates, daily_clim_base) if d.month not in range(3, 12)])
    # daily_clim_modif = np.array([f for d, f in zip(dates, daily_clim_modif) if d.month not in range(3, 12)])

    mean_base = np.mean(daily_clim_base, axis=0)
    diff = (np.mean(daily_clim_modif, axis=0) - mean_base) * 24 * 3600
    dpr = (pr_modif.sum(axis=0) - pr_base.sum(axis=0)) / pr_base.sum(axis=0)
    dav = (av_modif.sum(axis=0) - av_base.sum(axis=0)) / av_base.sum(axis=0)
    diff = np.ma.masked_where((mg <= 1.0e-3) | (dpr < 0) | (dav > 0) | (diff > 0), diff)

    print("{}-ranges: {}, {}".format(varname, daily_clim_base.min(), daily_clim_base.max()))
    print("{}-ranges: {}, {}".format(varname, daily_clim_modif.min(), daily_clim_modif.max()))

    limit_base = np.percentile(daily_clim_base, 90, axis=0)
    limit_modif = np.percentile(daily_clim_modif, 50, axis=0)
    limit_lower_intf = 1.0e-4 / (24.0 * 60.0 * 60.0)


    ndays_base = daily_clim_base > limit_base[np.newaxis, :, :]
    ndays_base = ndays_base.sum(axis=0)

    ndays_modif = daily_clim_modif > limit_base[np.newaxis, :, :]
    ndays_modif = ndays_modif.sum(axis=0)

    diff_days = np.ma.masked_where((mg <= 1.0e-4) | (intf_rates.max() < limit_lower_intf),
                                   ndays_modif - ndays_base)

    diff_days = np.ma.masked_where(diff_days > 0, diff_days)

    print("diff_days ranges: {} to {}".format(diff_days.min(), diff_days.max()))


    im = bmp.pcolormesh(xx, yy, diff)
    bmp.colorbar(im)
    bmp.drawcoastlines()




    img_file = IMG_FOLDER.joinpath("{}_{}-{}.png".format(varname, start_year, end_year))
    with img_file.open("wb") as imf:
        fig.savefig(imf, bbox_inches="tight")

    plt.show()
Example #21
0
def main(start_year=1980, end_year=2010, months=None):
    default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    # default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"

    if months is None:
        months = list(range(1, 13))

    img_folder = os.path.join("interflow_corr_images",
                              os.path.basename(default_path))
    if not os.path.isdir(img_folder):
        os.makedirs(img_folder)

    img_filename = "interflow_correlations_months={}_{}-{}.pdf".format(
        "-".join(str(m) for m in months), start_year, end_year)

    lons, lats, basemap = analysis.get_basemap_from_hdf(file_path=default_path)
    lons[lons > 180] -= 360
    x, y = basemap(lons, lats)

    # Correlate interflow rate and soil moisture
    params = dict(path1=default_path,
                  varname1="INTF",
                  level1=0,
                  path2=default_path,
                  level2=0,
                  varname2="I1",
                  months=months)

    params.update(dict(
        start_year=start_year,
        end_year=end_year,
    ))

    title_list = []
    data_list = []

    corr1, intf_clim, i1_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot1 = maskoceans(lons, lats, corr1)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))
    data_list.append(to_plot1)

    # correlate interflow and precip
    params.update(dict(varname2="PR", level2=0))
    corr2, i1_clim, pr_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot2 = np.ma.masked_where(to_plot1.mask, corr2)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))
    data_list.append(to_plot2)

    # correlate precip and soil moisture
    # params.update(dict(varname1="I1", level1=0))
    # corr3 = calculate_correlation_field_for_climatology(**params)
    # to_plot3 = np.ma.masked_where(to_plot2.mask, corr3)
    # title_list.append("Corr({}, {})".format(params["varname1"], params["varname2"]))
    # data_list.append(to_plot3)

    # correlate evaporation and soil moisture
    params.update(dict(varname2="AV", level2=0, varname1="I1", level1=0))
    corr4, i1_clim, av_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot3 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))
    data_list.append(to_plot3)

    # correlate interflow and evaporation
    params.update(dict(varname2="AV", level2=0, varname1="INTF", level1=0))
    corr4, intf_clim, av_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot4 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))
    data_list.append(to_plot4)

    # TODO: Correlate infiltration and surface runoff

    # Do plotting
    clevels = np.arange(-1, 1.2, 0.2)

    npanels = len(data_list)
    gs = GridSpec(1, npanels + 1, width_ratios=[
        1.0,
    ] * npanels + [
        0.05,
    ])

    fig = plt.figure()
    assert isinstance(fig, Figure)
    fig.set_figheight(1.5 * fig.get_figheight())

    img = None
    for col in range(npanels):
        ax = fig.add_subplot(gs[0, col])
        basemap.drawmapboundary(fill_color="0.75", ax=ax)

        img = basemap.contourf(x,
                               y,
                               data_list[col],
                               levels=clevels,
                               cmap=cm.get_cmap("RdBu_r",
                                                len(clevels) - 1))
        plt.title(title_list[col])
        basemap.drawcoastlines(linewidth=cpp.COASTLINE_WIDTH, ax=ax)

    plt.colorbar(img, cax=fig.add_subplot(gs[0, npanels]))
    fig.savefig(os.path.join(img_folder, img_filename), dpi=cpp.FIG_SAVE_DPI)

    # plot timeseries
    the_mask = corr4 < -0.1
    varname_to_ts = {
        "INTF": get_mean_over(the_mask, intf_clim),
        "LH": get_mean_over(the_mask, av_clim),
        "SM": get_mean_over(the_mask, i1_clim)
    }

    from matplotlib import gridspec

    fig = plt.figure()
    fig.set_figheight(3 * fig.get_figheight())
    gs = gridspec.GridSpec(len(varname_to_ts), 1)

    d0 = datetime(2001, 1, 1)
    dt = timedelta(days=1)
    dates = [d0 + dt * i for i in range(365) if (d0 + dt * i).month in months]
    sfmt = ScalarFormatter()
    dfmt = DateFormatter("%d%b")
    for i, (label, data) in enumerate(varname_to_ts.items()):
        ax = fig.add_subplot(gs[i, 0])
        ax.plot(dates, data, label=label, lw=2)
        ax.grid()

        ax.legend()
        ax.yaxis.set_major_formatter(sfmt)
        if i < len(varname_to_ts) - 1:
            ax.xaxis.set_ticklabels([])
        else:
            ax.xaxis.set_major_formatter(dfmt)

    fig.savefig(os.path.join(
        img_folder,
        "aa_ts_{}_{}.png".format(os.path.basename(default_path),
                                 "-".join(str(m) for m in months))),
                dpi=cpp.FIG_SAVE_DPI)
Example #22
0
def demo_interolate_daily_clim():
    import crcm5.analyse_hdf.do_analysis_using_pytables as analysis

    model_data_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
    start_year = 1980
    end_year = 2010

    vmin = -30
    vmax = 30
    vname = "TT_max"
    coef_mod = 1.0e3 * 24 * 3600 if vname == "PR" else 1.0

    # get target lons and lats for testing
    lon, lat, basemap = analysis.get_basemap_from_hdf(
        file_path=model_data_path)

    ans = AnuSplinManager(variable="stmx")
    dates, fields = ans.get_daily_clim_fields_interpolated_to(
        start_year=start_year,
        end_year=end_year,
        lons_target=lon,
        lats_target=lat)
    import matplotlib.pyplot as plt

    x, y = basemap(lon, lat)

    margin = 20
    topo = _get_topography()[margin:-margin, margin:-margin]

    # Plot obs data
    plt.figure()
    mean_obs = np.ma.array([
        fields[i] for i, d in enumerate(dates) if d.month in range(1, 13)
    ]).mean(axis=0)
    im = basemap.pcolormesh(x, y, mean_obs, vmin=vmin, vmax=vmax)
    basemap.colorbar(im)
    basemap.drawcoastlines()
    plt.title("Anusplin")
    print("Obs stdev = {}".format(mean_obs[~mean_obs.mask].std()))

    print("Obs correlations: ",
          np.corrcoef(mean_obs[~mean_obs.mask], topo[~mean_obs.mask]))

    # Plot model data
    plt.figure()
    dates, fields = analysis.get_daily_climatology(
        path_to_hdf_file=model_data_path,
        var_name=vname,
        level=0,
        start_year=start_year,
        end_year=end_year)

    mean_mod = np.array([
        fields[i] for i, d in enumerate(dates) if d.month in range(1, 13)
    ]).mean(axis=0) * coef_mod
    im = basemap.pcolormesh(x, y, mean_mod, vmin=vmin, vmax=vmax)
    basemap.colorbar(im)
    basemap.drawcoastlines()
    plt.title("Model")

    print("Model correlations: ",
          np.corrcoef(mean_mod[~mean_obs.mask], topo[~mean_obs.mask]))
    print("Model stdev = {}".format(mean_mod[~mean_obs.mask].std()))

    plt.show()
def plot_hydrographs():
    plot_utils.apply_plot_params(font_size=14,
                                 width_pt=None,
                                 width_cm=20,
                                 height_cm=20)
    start_year = 1980
    end_year = 2010

    varname = "STFL"

    base_config = RunConfig(
        start_year=start_year,
        end_year=end_year,
        data_path=
        "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
        label="NI")

    modif_config = RunConfig(
        start_year=start_year,
        end_year=end_year,
        data_path=
        "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
        label="WI")

    r_obj = RPN(GEO_DATA_FILE)
    facc = r_obj.get_first_record_for_name("FAA")
    fldr = r_obj.get_first_record_for_name("FLDR")

    lons, lats, bmp = analysis.get_basemap_from_hdf(
        file_path=base_config.data_path)

    basin_name_to_out_indices_map, basin_name_to_basin_mask = get_basin_to_outlet_indices_map(
        lons=lons, lats=lats, accumulation_areas=facc, directions=fldr)
    # Calculate the daily mean fields
    dates, stf_base = analysis.get_daily_climatology_for_rconf(
        base_config, var_name=varname, level=0)
    _, stf_modif = analysis.get_daily_climatology_for_rconf(modif_config,
                                                            var_name=varname,
                                                            level=0)

    for bname, (i_out, j_out) in basin_name_to_out_indices_map.items():
        print(bname, i_out, j_out)
        fig = plt.figure()

        gs = GridSpec(2, 1, height_ratios=[1, 0.5], hspace=0.1)

        ax = fig.add_subplot(gs[0, 0])
        ax.plot(dates,
                stf_base[:, i_out, j_out],
                "b",
                lw=2,
                label=base_config.label)
        ax.plot(dates,
                stf_modif[:, i_out, j_out],
                "r",
                lw=2,
                label=modif_config.label)
        ax.set_title(bname)
        format_axis(ax)

        # Hide the tick labels from the x-axis of the upper plot
        for tl in ax.xaxis.get_ticklabels():
            tl.set_visible(False)

        ax = fig.add_subplot(gs[1, 0])
        ax.plot(dates,
                stf_modif[:, i_out, j_out] - stf_base[:, i_out, j_out],
                "k",
                lw=2,
                label="{}-{}".format(modif_config.label, base_config.label))
        format_axis(ax)

        fig.savefig(
            str(
                IMG_FOLDER.joinpath("{}_{}-{}.png".format(
                    bname, start_year, end_year))))
        plt.close(fig)
Example #24
0
def main():
    path_to_era_driven = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    path_to_gcm_driven = \
        "/RESCUE/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"

    # Year range ...
    start_year = 1980
    end_year = 2010

    lons, lats, bmp = analysis.get_basemap_from_hdf(file_path=path_to_era_driven)
    bmp_info = BasemapInfo(lons=lons, lats=lats, bmp=bmp)

    seasons_for_mean = OrderedDict([
        ("Winter", (12, 1, 2)),
        ("Spring", range(3, 6)),
        ("Summer", range(6, 9)),
        ("Fall", range(9, 12))
    ])

    seasons_for_max = OrderedDict(
        [("Annual maximum", range(1, 13)), ]
    )

    variables_mean_bfe = ["TT", "PR", "I5", "STFL", "AV"]

    variables_annual_max_bfe = ["I5", ]
    variables_annual_max_bfe.pop()

    level_index_dict = collections.defaultdict(lambda: 0)
    level_index_dict.update({
        "I5": 0,
        "TT": 0,
        "PR": 0
    })

    all_vars = variables_mean_bfe + variables_annual_max_bfe

    plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=25)
    fig = plt.figure()

    nrows = len(all_vars)

    ncols = max(len(seasons_for_mean) + 1, len(seasons_for_max) + 1)
    gs = GridSpec(nrows, ncols,
                  width_ratios=[1., ] * (ncols - 1) + [0.05, ])

    # Plot seasonal mean precipitation and temperature
    finfo_to_season_to_diff = get_bfe_in_seasonal_mean(variables_mean_bfe, level_index_dict=level_index_dict,
                                                       season_to_months=seasons_for_mean,
                                                       path_to_era_driven=path_to_era_driven,
                                                       path_to_gcm_driven=path_to_gcm_driven,
                                                       start_year=start_year, end_year=end_year)

    row = 0
    for vname in variables_mean_bfe:
        row_axes = [fig.add_subplot(gs[row, col]) for col in range(len(seasons_for_mean) + 1)]

        plot_bfe_row_for_var(finfo_to_season_to_diff=finfo_to_season_to_diff,
                             ax_list=row_axes, season_titles=row == 0, varname=vname,
                             basemap_info=bmp_info)

        row += 1

    # plot annual maximum swe
    finfo_to_season_to_diff = get_bfe_in_annual_max(variables_annual_max_bfe, level_index_dict=level_index_dict,
                                                    path_to_era_driven=path_to_era_driven,
                                                    path_to_gcm_driven=path_to_gcm_driven,
                                                    start_year=start_year, end_year=end_year)

    add_season_titles = True
    for vname in variables_annual_max_bfe:
        row_axes = [fig.add_subplot(gs[row, col]) for col in range(ncols)]

        for i, the_ax in enumerate(row_axes[:-1]):
            if i < ncols - len(seasons_for_max) - 1:
                the_ax.set_visible(False)

        row_axes = list(reversed(row_axes[:-1])) + [row_axes[-1], ]

        plot_bfe_row_for_var(finfo_to_season_to_diff=finfo_to_season_to_diff,
                             ax_list=row_axes, season_titles=add_season_titles, varname=vname,
                             basemap_info=bmp_info)

        add_season_titles = False
        row += 1


    # Save the figure to a file
    p = Path(img_folder)
    if not p.is_dir():
        p.mkdir(parents=True)

    img_name = "{}_{}-{}_{}.png".format("-".join(variables_annual_max_bfe + variables_mean_bfe),
                                        start_year, end_year, "-".join(seasons_for_mean.keys()))

    img_path = str(p.joinpath(img_name))
    # fig.tight_layout()
    fig.savefig(img_path, bbox_inches="tight")
    plt.show()
Example #25
0
def main():
    """

    """

    season_to_months = get_default_season_to_months_dict()

    # season_to_months = OrderedDict([("April", [4, ]), ("May", [5, ]), ("June", [6, ]), ("July", [7, ])])

    var_names = ["TT", "HU", "PR", "AV", "AH", "STFL", "TRAF", "I5", "I0"]

    # var_names = ["TT", "PR"]

    levels = [
        0,
    ] * len(var_names)
    multipliers = {"PR": 1.0e3 * 24.0 * 3600., "TRAF": 24 * 3600.0}

    base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                        "quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5"
    base_label = "CanESM2-CRCM5-NL"

    modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                         "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    modif_label = "CanESM2-CRCM5-L"

    # base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
    # "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    # base_label = "CRCM5-L"
    #
    # modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
    # "quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5"
    # modif_label = "CRCM5-LI"

    start_year_c = 1980
    end_year_c = 2010

    future_shift_years = 90

    params = dict(data_path=base_current_path,
                  start_year=start_year_c,
                  end_year=end_year_c,
                  label=base_label)

    base_config_c = RunConfig(**params)
    base_config_f = base_config_c.get_shifted_config(future_shift_years)

    params.update(dict(data_path=modif_current_path, label=modif_label))

    modif_config_c = RunConfig(**params)
    modif_config_f = modif_config_c.get_shifted_config(future_shift_years)

    config_dict = OrderedDict([("Current",
                                OrderedDict([(base_label, base_config_c),
                                             (modif_label, modif_config_c)])),
                               ("Future",
                                OrderedDict([(base_label, base_config_f),
                                             (modif_label, modif_config_f)]))])

    # Plot the differences
    config_dict.label_modif = modif_config_c.label
    config_dict.label_base = base_config_c.label
    config_dict.season_to_months = season_to_months
    config_dict.multipliers = multipliers

    lons, lats, bmp = analysis.get_basemap_from_hdf(base_current_path)
    config_dict.lons = lons
    config_dict.lats = lats
    config_dict.basemap = bmp

    # Calculate and plot seasonal means
    for vname, level in zip(var_names, levels):
        data = get_data(vname=vname, level=level, config_dict=config_dict)
        _plot_var(vname=vname,
                  level=level,
                  config_dict=config_dict,
                  data_dict=data)
def main(
        intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
        no_intf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5",
        start_year=1980,
        end_year=2010,
        label="CRCM5-HCD-RL-INTFa"):
    """
    Study impact of interflow only for the cases when liquid soil moisture is greater than field capacity
    """

    img_folder = "intf_during_th_gt_thbfc"

    cache_file = "intf_during_th_gt_thbfc_{}_{}-{}.cache".format(
        ctypes.c_size_t(hash(intf_file + no_intf_file)).value, start_year,
        end_year)

    if not os.path.isdir(img_folder):
        os.mkdir(img_folder)

    with tb.open_file(intf_file) as intf_handle, tb.open_file(
            no_intf_file) as no_intf_handle:
        # Get runoff tables (level_index == 0 corresponds to runoff from soil)
        traf_table_intf = intf_handle.get_node("/TRAF")
        traf_table_no_intf = no_intf_handle.get_node("/TRAF")

        th_table_intf = intf_handle.get_node("/I1")
        th_table_no_intf = no_intf_handle.get_node("/I1")

        bfc = get_bulk_field_capacity()

        if os.path.isfile(cache_file):
            total_diff = pickle.load(open(cache_file))
        else:
            total_diff = get_runoff_differences_composit(
                traf_table_intf=traf_table_intf,
                th_table_intf=th_table_intf,
                traf_table_no_intf=traf_table_no_intf,
                th_table_no_intf=th_table_no_intf,
                thbfc_field=bfc,
                start_year=start_year,
                end_year=end_year)

            pickle.dump(total_diff, open(cache_file, "wb"))

        # save the figure
        plt.figure()

        lons, lats, bm = analysis.get_basemap_from_hdf(intf_file)

        x, y = bm(lons, lats)

        clevs = [0.5, 1, 5, 30, 100, 150, 200]
        clevs = [-c for c in reversed(clevs)] + clevs

        cmap = cm.get_cmap("bwr", len(clevs) - 1)
        bn = BoundaryNorm(clevs, len(clevs) - 1)
        im = bm.pcolormesh(x, y, total_diff, cmap=cmap, norm=bn)
        bm.drawcoastlines()
        bm.colorbar(im, ticks=clevs)
        plt.savefig(
            os.path.join(
                img_folder,
                "{}_traf_diff_{}-{}.png".format(label, start_year, end_year)))

    pass
Example #27
0
def main(start_year=1980, end_year=2010, months=None, ylabel="",
         fig=None, current_row=0, gs=None):
    default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    # default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"

    if months is None:
        months = list(range(1, 13))

    img_folder = os.path.join("interflow_corr_images", os.path.basename(default_path))
    if not os.path.isdir(img_folder):
        os.makedirs(img_folder)

    img_filename = "4x1_correlations_INTF-PR-I1-Infilt_months={}_{}-{}.png".format("-".join(str(m) for m in months),
                                                                                   start_year, end_year)

    lons, lats, basemap = analysis.get_basemap_from_hdf(file_path=default_path)
    lons[lons > 180] -= 360
    x, y = basemap(lons, lats)

    # Correlate interflow rate and soil moisture
    params = dict(
        path1=default_path,
        varname1="INTF",
        level1=0,

        path2=default_path,
        level2=0,
        varname2="I1",
        months=months
    )

    params.update(dict(
        start_year=start_year, end_year=end_year,
    ))

    title_list = []
    data_list = []

    corr1, intf_clim, i1_clim = calculate_correlation_field_for_climatology(**params)
    to_plot1 = maskoceans(lons, lats, corr1)
    # title_list.append("Corr({}, {})".format(
    # infovar.get_display_label_for_var(params["varname1"]),
    #     infovar.get_display_label_for_var(params["varname2"])))
    # data_list.append(to_plot1)

    # correlate interflow and precip
    params.update(dict(varname2="PR", level2=0))
    corr2, _, pr_clim = calculate_correlation_field_for_climatology(**params)
    to_plot2 = np.ma.masked_where(to_plot1.mask, corr2)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))
    data_list.append(to_plot2)

    # correlate precip and soil moisture
    # params.update(dict(varname1="I1", level1=0))
    # corr3, _, _ = calculate_correlation_field_for_climatology(**params)
    # to_plot3 = np.ma.masked_where(to_plot2.mask, corr3)
    # title_list.append("Corr({}, {})".format(
    #     infovar.get_display_label_for_var(params["varname1"]),
    #     infovar.get_display_label_for_var(params["varname2"])))
    # data_list.append(to_plot3)

    # correlate interflow and infiltration
    # corr4 = calculate_correlation_of_infiltration_rate_with(start_year=start_year,
    #                                                         end_year=end_year,
    #                                                         path_for_infiltration_data=default_path,
    #                                                         path2=default_path,
    #                                                         varname2="INTF",
    #                                                         level2=0, months=months)

    # Correlate interflow rate with latent heat flux
    params = dict(
        path1=default_path,
        varname1="INTF",
        level1=0,

        path2=default_path,
        level2=0,
        varname2="AV",
        months=months,

        start_year=start_year, end_year=end_year,
    )

    corr4, _, _ = calculate_correlation_field_for_climatology(**params)
    to_plot4 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))

    data_list.append(to_plot4)

    # Correlate interflow rate with soil ice
    params = dict(
        path1=default_path,
        varname1="INTF",
        level1=0,

        path2=default_path,
        level2=0,
        varname2="I2",
        months=months,

        start_year=start_year, end_year=end_year,
    )

    corr4, _, _ = calculate_correlation_field_for_climatology(**params)
    to_plot4 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))

    data_list.append(to_plot4)

    # Correlate interflow rate with swe
    params = dict(
        path1=default_path,
        varname1="INTF",
        level1=0,

        path2=default_path,
        level2=0,
        varname2="I5",
        months=months,

        start_year=start_year, end_year=end_year,
    )

    corr4, _, _ = calculate_correlation_field_for_climatology(**params)
    to_plot4 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))

    data_list.append(to_plot4)

    # Correlate LHF rate with swe
    params = dict(
        path1=default_path,
        varname1="AV",
        level1=0,

        path2=default_path,
        level2=0,
        varname2="I5",
        months=months,

        start_year=start_year, end_year=end_year,
    )

    corr4, _, _ = calculate_correlation_field_for_climatology(**params)
    to_plot4 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(
        infovar.get_display_label_for_var(params["varname1"]),
        infovar.get_display_label_for_var(params["varname2"])))

    data_list.append(to_plot4)

    # Do plotting
    clevels = np.arange(-1, 1.2, 0.2)

    npanels = len(data_list)

    if gs is None:
        gs = GridSpec(1, npanels + 1, width_ratios=[1.0, ] * npanels + [0.05, ])

    is_subplot = fig is not None
    fig = plt.figure() if fig is None else fig
    assert isinstance(fig, Figure)
    # fig.set_figheight(1.5 * fig.get_figheight())

    img = None
    for col in range(npanels):

        ax = fig.add_subplot(gs[current_row, col])

        if not col:
            ax.set_ylabel(ylabel)

        basemap.drawmapboundary(fill_color="0.75", ax=ax)

        img = basemap.contourf(x, y, data_list[col], levels=clevels, cmap=cm.get_cmap("RdBu_r", len(clevels) - 1))
        if current_row == 0:
            ax.set_title(title_list[col])
        basemap.drawcoastlines(linewidth=cpp.COASTLINE_WIDTH, ax=ax)

    if not is_subplot:
        plt.colorbar(img, cax=fig.add_subplot(gs[0, npanels]))
        fig.savefig(os.path.join(img_folder, img_filename), dpi=cpp.FIG_SAVE_DPI)

    return img, img_folder
Example #28
0
def main_interflow():
    # Changes global plot properties mainly figure size and font size
    plot_utils.apply_plot_params(font_size=12, width_cm=20)

    # season_to_months = get_default_season_to_months_dict()
    season_to_months = OrderedDict([("January", [1, ]), ("February", [2, ]), ("March", [3, ]), ])

    var_names = ["TT", "HU", "PR", "AV", "TRAF", "I1", "STFL"]

    levels = [0, ] * len(var_names)

    plot_utils.apply_plot_params(font_size=10, width_pt=None, width_cm=20 * len(season_to_months) / 4.0,
                                 height_cm=20 * len(var_names) / 5.0)

    multipliers = {
        "PR": 1.,
        "TRAF": 1.,
        "I1": infovar.soil_layer_widths_26_to_60[0] * 1000.0,
        "TRAF+TDRA": 24 * 60 * 60
    }

    name_to_units = {
        "TRAF": "mm/day", "I1": "mm", "PR": "mm/day", "TRAF+TDRA": "mm/day"
    }

    base_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                        "quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"
    base_label = "CanESM2-CRCM5-L"

    modif_current_path = "/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/" \
                         "quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5"
    modif_label = "CanESM2-CRCM5-LI"

    start_year_c = 1980
    end_year_c = 2010

    future_shift_years = 90

    params = dict(
        data_path=base_current_path, start_year=start_year_c, end_year=end_year_c, label=base_label
    )

    base_config_c = RunConfig(**params)
    base_config_f = base_config_c.get_shifted_config(future_shift_years)

    params.update(
        dict(data_path=modif_current_path, label=modif_label)
    )

    modif_config_c = RunConfig(**params)
    modif_config_f = modif_config_c.get_shifted_config(future_shift_years)

    config_dict = OrderedDict([
        ("Current", OrderedDict([(base_label, base_config_c), (modif_label, modif_config_c)])),
        ("Future", OrderedDict([(base_label, base_config_f), (modif_label, modif_config_f)]))
    ])


    # Plot the differences
    fig = plt.figure()
    gs = GridSpec(len(var_names), len(season_to_months) + 1, width_ratios=[1., ] * len(season_to_months) + [0.05, ])
    config_dict.fig = fig
    config_dict.gs = gs
    config_dict.label_modif = modif_config_c.label
    config_dict.label_base = base_config_c.label
    config_dict.season_to_months = season_to_months
    config_dict.multipliers = multipliers

    lons, lats, bmp = analysis.get_basemap_from_hdf(base_current_path)
    config_dict.lons = lons
    config_dict.lats = lats
    config_dict.basemap = bmp
    config_dict.name_to_units = name_to_units

    # Calculate and plot seasonal means
    for vname, level, the_row in zip(var_names, levels, list(range(len(levels)))):
        config_dict.the_row = the_row

        _plot_row(vname=vname, level=level, config_dict=config_dict)

    # Save the image to the file
    img_path = get_image_path(base_config_c, base_config_f, modif_config_c, season_to_months=season_to_months)
    fig.savefig(img_path, bbox_inches="tight")
def plot_control_and_differences_in_one_panel_for_all_seasons_for_all_vars(
        varnames=None, levels=None,
        season_to_months=None,
        start_year=None,
        end_year=None):
    season_list = list(season_to_months.keys())

    pvalue_max = 0.1

    # crcm5-r vs crcm5-hcd-r
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
    # control_label = "CRCM5-R"
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf", ]
    # labels = ["CRCM5-HCD-R"]

    # crcm5-hcd-rl vs crcm5-hcd-r
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
    # control_label = "CRCM5-HCD-R"
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf", ]
    # labels = ["CRCM5-HCD-RL"]

    # compare simulations with and without interflow
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf"
    # control_label = "CRCM5-HCD-RL"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf", ]
    # labels = ["CRCM5-HCD-RL-INTFL"]

    # very high hydr cond
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
    # control_label = "CRCM5-HCD-RL-INTFL"
    ##
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf", ]
    # labels = ["CRCM5-HCD-RL-INTFL-sani=10000"]

    # Interflow effect
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf"
    # control_label = "CRCM5-HCD-RL"
    # ##
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5", ]
    # labels = ["ITFS"]


    # total lake effect
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5"
    # control_label = "CRCM5-NL"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5", ]
    # labels = ["CRCM5-L2", ]



    # lake effect (lake-atm interactions)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5"
    # control_label = "CRCM5-R"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5", ]
    # labels = ["CRCM5-HCD-R", ]

    # lake effect (lake-river interactions)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
    # control_label = "CRCM5-L1"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5", ]
    # labels = ["CRCM5-HCD-L2", ]


    # interflow effect ()
    control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    control_label = "CRCM5-L2"

    paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5", ]
    labels = ["CRCM5-L2I", ]


    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5", ]
    # labels = ["CRCM5-HCD-RL-INTFb", ]



    # interflow effect (avoid truncation and bigger slopes)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    # control_label = "CRCM5-HCD-RL-INTF"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5", ]
    # labels = ["CRCM5-HCD-RL-INTF-improved", ]
    #

    row_labels = [
        r"{} vs {}".format(s, control_label) for s in labels
    ]
    print(labels)

    # varnames = ["QQ", ]
    # levels = [None, ]

    assert len(levels) == len(varnames)

    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=control_path)
    x, y = basemap(lons2d, lats2d)
    # save the domain properties for reuse
    domain_props = DomainProperties()
    domain_props.basemap = basemap
    domain_props.lons2d = lons2d
    domain_props.lats2d = lats2d
    domain_props.x = x
    domain_props.y = y

    lake_fraction = analysis.get_array_from_file(path=control_path, var_name=infovar.HDF_LAKE_FRACTION_NAME)
    dpth_to_bedrock = analysis.get_array_from_file(path=control_path, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME)

    assert dpth_to_bedrock is not None


    if lake_fraction is None:
        lake_fraction = np.zeros(lons2d.shape)

    ncolors = 10
    # +1 to include white
    diff_cmap = cm.get_cmap("RdBu", ncolors + 1)


    # Do the plotting for each variable
    fig = plt.figure()
    assert isinstance(fig, Figure)

    # plot the control data
    ncols = len(season_list) + 1  # +1 is for the colorbar
    gs = gridspec.GridSpec(len(varnames), ncols, width_ratios=[1.0, ] * (ncols - 1) + [0.07], top=0.95)


    lev_width_3d = np.ones(dpth_to_bedrock.shape + infovar.soil_layer_widths_26_to_60.shape)
    lev_width_3d *= infovar.soil_layer_widths_26_to_60[np.newaxis, np.newaxis, :]
    lev_bot_3d = lev_width_3d.cumsum(axis=2)

    correction = -lev_bot_3d + dpth_to_bedrock[:, :, np.newaxis]
    # Apply the correction only at points where the layer bottom is lower than
    # the bedrock
    lev_width_3d[correction < 0] += correction[correction < 0]
    lev_width_3d[lev_width_3d < 0] = 0


    # plot the plots one file per variable
    for var_name, level, the_row in zip(varnames, levels, list(range(len(varnames)))):
        sfmt = infovar.get_colorbar_formatter(var_name)
        season_to_control_mean = {}
        label_to_season_to_difference = {}
        label_to_season_to_significance = {}

        try:
            # Calculate the difference for each season, and save the results to dictionaries
            # to access later when plotting
            for season, months_of_interest in season_to_months.items():
                print("working on season: {0}".format(season))

                control_means = analysis.get_mean_2d_fields_for_months(path=control_path, var_name=var_name,
                                                                       months=months_of_interest,
                                                                       start_year=start_year, end_year=end_year,
                                                                       level=level)

                control_mean = np.mean(control_means, axis=0)

                control_mean = infovar.get_to_plot(var_name, control_mean,
                                                   lake_fraction=domain_props.lake_fraction,
                                                   lons=lons2d, lats=lats2d, level_width_m=lev_width_3d[:, :, level])

                # multiply by the number of days in a season for PR and TRAF to convert them into mm from mm/day
                if var_name in ["PR", "TRAF", "TDRA"]:
                    control_mean *= get_num_days(months_of_interest)
                    infovar.change_units_to(varnames=[var_name, ], new_units=r"${\rm mm}$")

                season_to_control_mean[season] = control_mean

                print("calculated mean from {0}".format(control_path))

                # calculate the difference for each simulation
                for the_path, the_label in zip(paths, row_labels):
                    modified_means = analysis.get_mean_2d_fields_for_months(path=the_path, var_name=var_name,
                                                                            months=months_of_interest,
                                                                            start_year=start_year, end_year=end_year,
                                                                            level=level)

                    tval, pval = ttest_ind(modified_means, control_means, axis=0, equal_var=False)
                    significance = ((pval <= pvalue_max) & (~control_mean.mask)).astype(int)
                    print("pval ranges: {} to {}".format(pval.min(), pval.max()))

                    modified_mean = np.mean(modified_means, axis=0)
                    if the_label not in label_to_season_to_difference:
                        label_to_season_to_difference[the_label] = OrderedDict()
                        label_to_season_to_significance[the_label] = OrderedDict()

                    modified_mean = infovar.get_to_plot(var_name, modified_mean,
                                                        lake_fraction=domain_props.lake_fraction, lons=lons2d,
                                                        lats=lats2d, level_width_m=lev_width_3d[:, :, level])

                    # multiply by the number of days in a season for PR and TRAF to convert them into mm from mm/day
                    if var_name in ["PR", "TRAF", "TDRA"]:
                        modified_mean *= get_num_days(months_of_interest)

                    diff_vals = modified_mean - control_mean

                    print("diff ranges: min: {0};  max: {1}".format(diff_vals.min(), diff_vals.max()))
                    label_to_season_to_difference[the_label][season] = diff_vals
                    label_to_season_to_significance[the_label][season] = significance

                    print("Calculated mean and diff from {0}".format(the_path))
        except NoSuchNodeError:
            print("Could not find {0}, skipping...".format(var_name))
            continue





        for the_label, data in label_to_season_to_difference.items():
            axes = []
            for col in range(ncols):
                axes.append(fig.add_subplot(gs[the_row, col]))

            # Set season titles
            if the_row == 0:
                for the_season, ax in zip(season_list, axes):
                    ax.set_title(the_season)


            _plot_row(axes, data, the_label, var_name, increments=True, domain_props=domain_props,
                      season_list=season_list, significance=label_to_season_to_significance[the_label])

            var_label = infovar.get_long_display_label_for_var(var_name)
            if var_name in ["I1"]:
                var_label = "{}\n{} layer".format(var_label, ordinal(level + 1))

            axes[0].set_ylabel(var_label)

    fig.suptitle("({}) vs ({})".format(labels[0], control_label), font_properties=FontProperties(weight="bold"))
    folderpath = os.path.join(images_folder, "seasonal_mean_maps/{0}_vs_{1}_for_{2}_{3}-{4}".format(
        "_".join(labels), control_label, "-".join(list(season_to_months.keys())), start_year, end_year))

    if not os.path.isdir(folderpath):
        os.mkdir(folderpath)

    imname = "{0}_{1}.png".format("-".join(varnames), "_".join(labels + [control_label]))
    impath = os.path.join(folderpath, imname)
    fig.savefig(impath, bbox_inches="tight")
Example #30
0
def main():
    start_year = 1980
    end_year = 2010

    base_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    base_label = "CRCM5-L"

    label_to_path = {
        "CRCM5-LI":
        "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    }

    season_to_months = OrderedDict([("Winter", [12, 1, 2]),
                                    ("Spring", list(range(3, 6))),
                                    ("Summer", list(range(6, 9))),
                                    ("Fall", list(range(9, 12)))])

    coords = CoordsAndBg(*analysis.get_basemap_from_hdf(base_path))

    daily_dates, tmax_daily_base = analysis.get_daily_max_climatology(
        base_path,
        var_name="TT_max",
        level=0,
        start_year=start_year,
        end_year=end_year)

    _, tmin_daily_base = analysis.get_daily_min_climatology(
        base_path,
        var_name="TT_min",
        level=0,
        start_year=start_year,
        end_year=end_year)

    label_to_tmax_daily = {}
    label_to_tmin_daily = {}

    for label, the_path in label_to_path.items():
        _, label_to_tmax_daily[label] = analysis.get_daily_max_climatology(
            the_path,
            var_name="TT_max",
            level=0,
            start_year=start_year,
            end_year=end_year)

        _, label_to_tmin_daily[label] = analysis.get_daily_min_climatology(
            the_path,
            var_name="TT_min",
            level=0,
            start_year=start_year,
            end_year=end_year)

    # Plot results
    plot_impacts_of_intfl_on_seasonal_means(
        var_name="Tmin",
        base_label=base_label,
        base_data_daily=tmin_daily_base,
        label_to_data_daily=label_to_tmin_daily,
        coord_obj=coords,
        season_to_months=season_to_months,
        daily_dates=daily_dates)

    plot_impacts_of_intfl_on_seasonal_means(
        var_name="Tmax",
        base_label=base_label,
        base_data_daily=tmax_daily_base,
        label_to_data_daily=label_to_tmax_daily,
        coord_obj=coords,
        season_to_months=season_to_months,
        daily_dates=daily_dates)
Example #31
0
def main():
    path_to_era_driven = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    path_to_gcm_driven = \
        "/RESCUE/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5"

    # Year range ...
    start_year = 1980
    end_year = 2010

    lons, lats, bmp = analysis.get_basemap_from_hdf(
        file_path=path_to_era_driven)
    bmp_info = BasemapInfo(lons=lons, lats=lats, bmp=bmp)

    seasons_for_mean = OrderedDict([("Winter", (12, 1, 2)),
                                    ("Spring", range(3, 6)),
                                    ("Summer", range(6, 9)),
                                    ("Fall", range(9, 12))])

    seasons_for_max = OrderedDict([
        ("Annual maximum", range(1, 13)),
    ])

    variables_mean_bfe = ["TT", "PR", "I5", "STFL", "AV"]

    variables_annual_max_bfe = [
        "I5",
    ]
    variables_annual_max_bfe.pop()

    level_index_dict = collections.defaultdict(lambda: 0)
    level_index_dict.update({"I5": 0, "TT": 0, "PR": 0})

    all_vars = variables_mean_bfe + variables_annual_max_bfe

    plot_utils.apply_plot_params(font_size=12,
                                 width_pt=None,
                                 width_cm=25,
                                 height_cm=25)
    fig = plt.figure()

    nrows = len(all_vars)

    ncols = max(len(seasons_for_mean) + 1, len(seasons_for_max) + 1)
    gs = GridSpec(nrows, ncols, width_ratios=[
        1.,
    ] * (ncols - 1) + [
        0.05,
    ])

    # Plot seasonal mean precipitation and temperature
    finfo_to_season_to_diff = get_bfe_in_seasonal_mean(
        variables_mean_bfe,
        level_index_dict=level_index_dict,
        season_to_months=seasons_for_mean,
        path_to_era_driven=path_to_era_driven,
        path_to_gcm_driven=path_to_gcm_driven,
        start_year=start_year,
        end_year=end_year)

    row = 0
    for vname in variables_mean_bfe:
        row_axes = [
            fig.add_subplot(gs[row, col])
            for col in range(len(seasons_for_mean) + 1)
        ]

        plot_bfe_row_for_var(finfo_to_season_to_diff=finfo_to_season_to_diff,
                             ax_list=row_axes,
                             season_titles=row == 0,
                             varname=vname,
                             basemap_info=bmp_info)

        row += 1

    # plot annual maximum swe
    finfo_to_season_to_diff = get_bfe_in_annual_max(
        variables_annual_max_bfe,
        level_index_dict=level_index_dict,
        path_to_era_driven=path_to_era_driven,
        path_to_gcm_driven=path_to_gcm_driven,
        start_year=start_year,
        end_year=end_year)

    add_season_titles = True
    for vname in variables_annual_max_bfe:
        row_axes = [fig.add_subplot(gs[row, col]) for col in range(ncols)]

        for i, the_ax in enumerate(row_axes[:-1]):
            if i < ncols - len(seasons_for_max) - 1:
                the_ax.set_visible(False)

        row_axes = list(reversed(row_axes[:-1])) + [
            row_axes[-1],
        ]

        plot_bfe_row_for_var(finfo_to_season_to_diff=finfo_to_season_to_diff,
                             ax_list=row_axes,
                             season_titles=add_season_titles,
                             varname=vname,
                             basemap_info=bmp_info)

        add_season_titles = False
        row += 1

    # Save the figure to a file
    p = Path(img_folder)
    if not p.is_dir():
        p.mkdir(parents=True)

    img_name = "{}_{}-{}_{}.png".format(
        "-".join(variables_annual_max_bfe + variables_mean_bfe), start_year,
        end_year, "-".join(seasons_for_mean.keys()))

    img_path = str(p.joinpath(img_name))
    # fig.tight_layout()
    fig.savefig(img_path, bbox_inches="tight")
    plt.show()
Example #32
0
def plot_tmin_tmax_correlations(start_year=1980, end_year=2010, months=None):
    default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    # default_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"

    if months is None:
        months = list(range(1, 13))

    img_folder = os.path.join("interflow_corr_images",
                              os.path.basename(default_path))
    if not os.path.isdir(img_folder):
        os.makedirs(img_folder)

    img_filename = "Tmin_Tmax_interflow_correlations_months={}_{}-{}.jpg".format(
        "-".join(str(m) for m in months), start_year, end_year)

    lons, lats, basemap = analysis.get_basemap_from_hdf(file_path=default_path)
    lons[lons > 180] -= 360
    x, y = basemap(lons, lats)

    # Correlate interflow rate and soil moisture
    params = dict(path1=default_path,
                  varname1="INTF",
                  level1=0,
                  path2=default_path,
                  level2=0,
                  varname2="TT_max",
                  months=months)

    params.update(dict(
        start_year=start_year,
        end_year=end_year,
    ))

    title_list = []
    data_list = []

    corr1, intf_clim, i1_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot1 = maskoceans(lons, lats, corr1)
    title_list.append("Corr({}, {})".format(params["varname1"],
                                            params["varname2"]))
    data_list.append(to_plot1)

    # correlate interflow and precip
    # params.update(dict(varname2="PR", level2=0))
    # corr2 = calculate_correlation_field_for_climatology(**params)
    # to_plot2 = np.ma.masked_where(to_plot1.mask, corr2)
    # title_list.append("Corr({}, {})".format(params["varname1"], params["varname2"]))
    # data_list.append(to_plot2)

    # correlate precip and soil moisture
    # params.update(dict(varname1="I1", level1=0))
    # corr3 = calculate_correlation_field_for_climatology(**params)
    # to_plot3 = np.ma.masked_where(to_plot2.mask, corr3)
    # title_list.append("Corr({}, {})".format(params["varname1"], params["varname2"]))
    # data_list.append(to_plot3)

    # correlate evaporation and soil moisture
    params.update(dict(varname2="TT_min", level2=0, varname1="INTF", level1=0))
    corr4, i1_clim, av_clim = calculate_correlation_field_for_climatology(
        **params)
    to_plot3 = np.ma.masked_where(to_plot1.mask, corr4)
    title_list.append("Corr({}, {})".format(params["varname1"],
                                            params["varname2"]))
    data_list.append(to_plot3)

    # Do plotting
    clevels = np.arange(-1, 1.2, 0.2)

    npanels = len(data_list)
    gs = GridSpec(1, npanels + 1, width_ratios=[
        1.0,
    ] * npanels + [
        0.05,
    ])

    fig = plt.figure()
    assert isinstance(fig, Figure)
    fig.set_figheight(1.5 * fig.get_figheight())

    img = None
    for col in range(npanels):
        ax = fig.add_subplot(gs[0, col])
        basemap.drawmapboundary(fill_color="0.75", ax=ax)

        img = basemap.contourf(x,
                               y,
                               data_list[col],
                               levels=clevels,
                               cmap=cm.get_cmap("RdBu_r",
                                                len(clevels) - 1))
        plt.title(title_list[col])
        basemap.drawcoastlines(linewidth=cpp.COASTLINE_WIDTH, ax=ax)

    plt.colorbar(img, cax=fig.add_subplot(gs[0, npanels]))
    fig.savefig(os.path.join(img_folder, img_filename), dpi=cpp.FIG_SAVE_DPI)
Example #33
0
def plot_control_and_differences_in_one_panel_for_all_seasons_for_all_vars(
        varnames=None,
        levels=None,
        season_to_months=None,
        start_year=None,
        end_year=None):
    season_list = list(season_to_months.keys())

    pvalue_max = 0.1

    # crcm5-r vs crcm5-hcd-r
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
    # control_label = "CRCM5-R"
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf", ]
    # labels = ["CRCM5-HCD-R"]

    # crcm5-hcd-rl vs crcm5-hcd-r
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
    # control_label = "CRCM5-HCD-R"
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf", ]
    # labels = ["CRCM5-HCD-RL"]

    # compare simulations with and without interflow
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf"
    # control_label = "CRCM5-HCD-RL"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf", ]
    # labels = ["CRCM5-HCD-RL-INTFL"]

    # very high hydr cond
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
    # control_label = "CRCM5-HCD-RL-INTFL"
    ##
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf", ]
    # labels = ["CRCM5-HCD-RL-INTFL-sani=10000"]

    # Interflow effect
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf"
    # control_label = "CRCM5-HCD-RL"
    # ##
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5", ]
    # labels = ["ITFS"]

    # total lake effect
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5"
    # control_label = "CRCM5-NL"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5", ]
    # labels = ["CRCM5-L2", ]

    # lake effect (lake-atm interactions)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5"
    # control_label = "CRCM5-R"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5", ]
    # labels = ["CRCM5-HCD-R", ]

    # lake effect (lake-river interactions)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
    # control_label = "CRCM5-L1"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5", ]
    # labels = ["CRCM5-HCD-L2", ]

    # interflow effect ()
    control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
    control_label = "CRCM5-L2"

    paths = [
        "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5",
    ]
    labels = [
        "CRCM5-L2I",
    ]

    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5", ]
    # labels = ["CRCM5-HCD-RL-INTFb", ]

    # interflow effect (avoid truncation and bigger slopes)
    # control_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
    # control_label = "CRCM5-HCD-RL-INTF"
    #
    # paths = ["/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5", ]
    # labels = ["CRCM5-HCD-RL-INTF-improved", ]
    #

    row_labels = [r"{} vs {}".format(s, control_label) for s in labels]
    print(labels)

    # varnames = ["QQ", ]
    # levels = [None, ]

    assert len(levels) == len(varnames)

    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(
        file_path=control_path)
    x, y = basemap(lons2d, lats2d)
    # save the domain properties for reuse
    domain_props = DomainProperties()
    domain_props.basemap = basemap
    domain_props.lons2d = lons2d
    domain_props.lats2d = lats2d
    domain_props.x = x
    domain_props.y = y

    lake_fraction = analysis.get_array_from_file(
        path=control_path, var_name=infovar.HDF_LAKE_FRACTION_NAME)
    dpth_to_bedrock = analysis.get_array_from_file(
        path=control_path, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME)

    assert dpth_to_bedrock is not None

    if lake_fraction is None:
        lake_fraction = np.zeros(lons2d.shape)

    ncolors = 10
    # +1 to include white
    diff_cmap = cm.get_cmap("RdBu", ncolors + 1)

    # Do the plotting for each variable
    fig = plt.figure()
    assert isinstance(fig, Figure)

    # plot the control data
    ncols = len(season_list) + 1  # +1 is for the colorbar
    gs = gridspec.GridSpec(len(varnames),
                           ncols,
                           width_ratios=[
                               1.0,
                           ] * (ncols - 1) + [0.07],
                           top=0.95)

    lev_width_3d = np.ones(dpth_to_bedrock.shape +
                           infovar.soil_layer_widths_26_to_60.shape)
    lev_width_3d *= infovar.soil_layer_widths_26_to_60[np.newaxis,
                                                       np.newaxis, :]
    lev_bot_3d = lev_width_3d.cumsum(axis=2)

    correction = -lev_bot_3d + dpth_to_bedrock[:, :, np.newaxis]
    # Apply the correction only at points where the layer bottom is lower than
    # the bedrock
    lev_width_3d[correction < 0] += correction[correction < 0]
    lev_width_3d[lev_width_3d < 0] = 0

    # plot the plots one file per variable
    for var_name, level, the_row in zip(varnames, levels,
                                        list(range(len(varnames)))):
        sfmt = infovar.get_colorbar_formatter(var_name)
        season_to_control_mean = {}
        label_to_season_to_difference = {}
        label_to_season_to_significance = {}

        try:
            # Calculate the difference for each season, and save the results to dictionaries
            # to access later when plotting
            for season, months_of_interest in season_to_months.items():
                print("working on season: {0}".format(season))

                control_means = analysis.get_mean_2d_fields_for_months(
                    path=control_path,
                    var_name=var_name,
                    months=months_of_interest,
                    start_year=start_year,
                    end_year=end_year,
                    level=level)

                control_mean = np.mean(control_means, axis=0)

                control_mean = infovar.get_to_plot(
                    var_name,
                    control_mean,
                    lake_fraction=domain_props.lake_fraction,
                    lons=lons2d,
                    lats=lats2d,
                    level_width_m=lev_width_3d[:, :, level])

                # multiply by the number of days in a season for PR and TRAF to convert them into mm from mm/day
                if var_name in ["PR", "TRAF", "TDRA"]:
                    control_mean *= get_num_days(months_of_interest)
                    infovar.change_units_to(varnames=[
                        var_name,
                    ],
                                            new_units=r"${\rm mm}$")

                season_to_control_mean[season] = control_mean

                print("calculated mean from {0}".format(control_path))

                # calculate the difference for each simulation
                for the_path, the_label in zip(paths, row_labels):
                    modified_means = analysis.get_mean_2d_fields_for_months(
                        path=the_path,
                        var_name=var_name,
                        months=months_of_interest,
                        start_year=start_year,
                        end_year=end_year,
                        level=level)

                    tval, pval = ttest_ind(modified_means,
                                           control_means,
                                           axis=0,
                                           equal_var=False)
                    significance = ((pval <= pvalue_max) &
                                    (~control_mean.mask)).astype(int)
                    print("pval ranges: {} to {}".format(
                        pval.min(), pval.max()))

                    modified_mean = np.mean(modified_means, axis=0)
                    if the_label not in label_to_season_to_difference:
                        label_to_season_to_difference[the_label] = OrderedDict(
                        )
                        label_to_season_to_significance[
                            the_label] = OrderedDict()

                    modified_mean = infovar.get_to_plot(
                        var_name,
                        modified_mean,
                        lake_fraction=domain_props.lake_fraction,
                        lons=lons2d,
                        lats=lats2d,
                        level_width_m=lev_width_3d[:, :, level])

                    # multiply by the number of days in a season for PR and TRAF to convert them into mm from mm/day
                    if var_name in ["PR", "TRAF", "TDRA"]:
                        modified_mean *= get_num_days(months_of_interest)

                    diff_vals = modified_mean - control_mean

                    print("diff ranges: min: {0};  max: {1}".format(
                        diff_vals.min(), diff_vals.max()))
                    label_to_season_to_difference[the_label][
                        season] = diff_vals
                    label_to_season_to_significance[the_label][
                        season] = significance

                    print("Calculated mean and diff from {0}".format(the_path))
        except NoSuchNodeError:
            print("Could not find {0}, skipping...".format(var_name))
            continue

        for the_label, data in label_to_season_to_difference.items():
            axes = []
            for col in range(ncols):
                axes.append(fig.add_subplot(gs[the_row, col]))

            # Set season titles
            if the_row == 0:
                for the_season, ax in zip(season_list, axes):
                    ax.set_title(the_season)

            _plot_row(axes,
                      data,
                      the_label,
                      var_name,
                      increments=True,
                      domain_props=domain_props,
                      season_list=season_list,
                      significance=label_to_season_to_significance[the_label])

            var_label = infovar.get_long_display_label_for_var(var_name)
            if var_name in ["I1"]:
                var_label = "{}\n{} layer".format(var_label,
                                                  ordinal(level + 1))

            axes[0].set_ylabel(var_label)

    fig.suptitle("({}) vs ({})".format(labels[0], control_label),
                 font_properties=FontProperties(weight="bold"))
    folderpath = os.path.join(
        images_folder, "seasonal_mean_maps/{0}_vs_{1}_for_{2}_{3}-{4}".format(
            "_".join(labels), control_label,
            "-".join(list(season_to_months.keys())), start_year, end_year))

    if not os.path.isdir(folderpath):
        os.mkdir(folderpath)

    imname = "{0}_{1}.png".format("-".join(varnames),
                                  "_".join(labels + [control_label]))
    impath = os.path.join(folderpath, imname)
    fig.savefig(impath, bbox_inches="tight")
Example #34
0
def compare(paths=None,
            path_to_control_data=None,
            control_label="",
            labels=None,
            varnames=None,
            levels=None,
            months_of_interest=None,
            start_year=None,
            end_year=None):
    """
    Comparing 2D fields
    :param paths: paths to the simulation results
    :param varnames:
    :param labels: Display name for each simulation (number of labels should
     be equal to the number of paths)
    :param path_to_control_data: the path with which the comparison done i.e. a in the following
     formula
            delta = (x - a)/a * 100%

     generates one image file per variable (in the folder images_for_lake-river_paper):
        compare_varname_<control_label>_<label1>_..._<labeln>_startyear_endyear.png

    """
    # get coordinate data  (assumes that all the variables and runs have the same coordinates)
    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(
        file_path=path_to_control_data)
    x, y = basemap(lons2d, lats2d)

    lake_fraction = analysis.get_array_from_file(path=path_to_control_data,
                                                 var_name="lake_fraction")

    if lake_fraction is None:
        lake_fraction = np.zeros(lons2d.shape)

    ncolors = 10
    # +1 to include white
    diff_cmap = cm.get_cmap("RdBu_r", ncolors + 1)

    for var_name, level in zip(varnames, levels):
        sfmt = infovar.get_colorbar_formatter(var_name)
        control_means = analysis.get_mean_2d_fields_for_months(
            path=path_to_control_data,
            var_name=var_name,
            months=months_of_interest,
            start_year=start_year,
            end_year=end_year,
            level=level)

        control_mean = np.mean(control_means, axis=0)
        fig = plt.figure()
        assert isinstance(fig, Figure)
        gs = gridspec.GridSpec(2, len(paths) + 1, wspace=0.5)

        # plot the control
        ax = fig.add_subplot(gs[0, 0])
        assert isinstance(ax, Axes)
        ax.set_title("{0}".format(control_label))
        ax.set_ylabel("Mean: $X_{0}$")
        to_plot = infovar.get_to_plot(var_name,
                                      control_mean,
                                      lake_fraction=lake_fraction,
                                      mask_oceans=True,
                                      lons=lons2d,
                                      lats=lats2d)
        # determine colorabr extent and spacing
        field_cmap, field_norm = infovar.get_colormap_and_norm_for(
            var_name, to_plot, ncolors=ncolors)

        basemap.pcolormesh(x, y, to_plot, cmap=field_cmap, norm=field_norm)
        cb = basemap.colorbar(format=sfmt)

        assert isinstance(cb, Colorbar)
        # cb.ax.set_ylabel(infovar.get_units(var_name))
        units = infovar.get_units(var_name)

        info = "Variable:" \
               "\n{0}" \
               "\nPeriod: {1}-{2}" \
               "\nMonths: {3}" \
               "\nUnits: {4}"

        info = info.format(
            infovar.get_long_name(var_name), start_year, end_year, ",".join([
                datetime(2001, m, 1).strftime("%b") for m in months_of_interest
            ]), units)

        ax.annotate(info, xy=(0.1, 0.3), xycoords="figure fraction")

        sel_axes = [ax]

        for the_path, the_label, column in zip(paths, labels,
                                               list(range(1,
                                                          len(paths) + 1))):

            means_for_years = analysis.get_mean_2d_fields_for_months(
                path=the_path,
                var_name=var_name,
                months=months_of_interest,
                start_year=start_year,
                end_year=end_year)
            the_mean = np.mean(means_for_years, axis=0)

            # plot the mean value
            ax = fig.add_subplot(gs[0, column])
            sel_axes.append(ax)
            ax.set_title("{0}".format(the_label))
            to_plot = infovar.get_to_plot(var_name,
                                          the_mean,
                                          lake_fraction=lake_fraction,
                                          mask_oceans=True,
                                          lons=lons2d,
                                          lats=lats2d)

            basemap.pcolormesh(x, y, to_plot, cmap=field_cmap, norm=field_norm)
            ax.set_ylabel("Mean: $X_{0}$".format(column))
            cb = basemap.colorbar(format=sfmt)
            # cb.ax.set_ylabel(infovar.get_units(var_name))

            # plot the difference
            ax = fig.add_subplot(gs[1, column])
            sel_axes.append(ax)
            ax.set_ylabel("$X_{0} - X_0$".format(column))

            # #Mask only if the previous plot (means) is masked
            thediff = the_mean - control_mean

            if hasattr(to_plot, "mask"):
                to_plot = np.ma.masked_where(to_plot.mask, thediff)
            else:
                to_plot = thediff

            if var_name == "PR":  # convert to mm/day
                to_plot = infovar.get_to_plot(var_name,
                                              to_plot,
                                              mask_oceans=False)

            vmin = np.ma.min(to_plot)
            vmax = np.ma.max(to_plot)

            d = max(abs(vmin), abs(vmax))
            vmin = -d
            vmax = d

            field_norm, bounds, vmn_nice, vmx_nice = infovar.get_boundary_norm(
                vmin, vmax, diff_cmap.N, exclude_zero=False)
            basemap.pcolormesh(x,
                               y,
                               to_plot,
                               cmap=diff_cmap,
                               norm=field_norm,
                               vmin=vmn_nice,
                               vmax=vmx_nice)

            cb = basemap.colorbar(format=sfmt)

            t, pval = ttest_ind(means_for_years, control_means, axis=0)
            sig = pval < 0.1
            basemap.contourf(x,
                             y,
                             sig.astype(int),
                             nlevels=2,
                             hatches=["+", None],
                             colors="none")

            # cb.ax.set_ylabel(infovar.get_units(var_name))

        # plot coastlines
        for the_ax in sel_axes:
            basemap.drawcoastlines(
                ax=the_ax, linewidth=common_plot_params.COASTLINE_WIDTH)

        # depends on the compared simulations and the months of interest
        fig_file_name = "compare_{0}_{1}_{2}_months-{3}.jpeg".format(
            var_name, control_label, "_".join(labels),
            "-".join([str(m) for m in months_of_interest]))
        figpath = os.path.join(images_folder, fig_file_name)
        fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
        plt.close(fig)
def draw_model_comparison(model_points=None, stations=None, sim_name_to_file_name=None, hdf_folder=None,
                          start_year=None, end_year=None, cell_manager=None, stfl_name="STFA",
                          drainage_area_reldiff_min=0.1, plot_upstream_area_averaged=True,
                          sim_name_to_color=None):
    """

    :param model_points: list of model point objects
    :param stations: list of stations corresponding to the list of model points
    :param cell_manager: is a CellManager instance which can be provided for better performance if necessary
    len(model_points) == len(stations) if stations is not None.
    if stations is None - then no measured streamflow will be plotted
    """
    assert model_points is None or stations is None or len(stations) == len(model_points)
    label_list = list(sim_name_to_file_name.keys())  # Needed to keep the order the same for all subplots
    path0 = os.path.join(hdf_folder, list(sim_name_to_file_name.items())[0][1])
    flow_directions = analysis.get_array_from_file(path=path0, var_name="flow_direction")
    lake_fraction = analysis.get_array_from_file(path=path0, var_name="lake_fraction")

    # mask lake fraction in the ocean
    lake_fraction = np.ma.masked_where((flow_directions <= 0) | (flow_directions > 128), lake_fraction)

    accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
    area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2)

    # Try to read cell areas im meters if it is not Ok then try in km2
    if area_m2 is not None:
        cell_area_km2 = area_m2 * 1.0e-6
    else:
        cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2)

    print("cell area ranges from {} to {}".format(cell_area_km2.min(), cell_area_km2.max()))

    # print "plotting from {0}".format(path0)
    # plt.pcolormesh(lake_fraction.transpose())
    # plt.colorbar()
    # plt.show()
    # exit()

    file_scores = open("scores_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")
    file_correlations = open("corr_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")
    file_annual_discharge = open("flow_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")

    text_files = [file_scores, file_correlations, file_annual_discharge]
    # write the following columns to the scores file
    header_format = "{0:10s}\t{1:10s}\t{2:10s}\t" + "\t".join(["{" + str(i + 3) + ":10s}"
                                                               for i in range(len(sim_name_to_file_name))])
    line_format = "{0:10s}\t{1:10.1f}\t{2:10.1f}\t" + "\t".join(["{" + str(i + 3) + ":10.1f}"
                                                                 for i in range(len(sim_name_to_file_name))])

    header_ns = ("ID", "DAo", "DAm",) + tuple(["NS({0})".format(key) for key in sim_name_to_file_name])
    file_scores.write(header_format.format(*header_ns) + "\n")

    header_qyear = ("ID", "DAo", "DAm",) + tuple(["Qyear({0})".format(key) for key in label_list]) + \
                   ("Qyear(obs)",)
    header_format_qyear = header_format + "\t{" + str(len(label_list) + 3) + ":10s}"
    file_annual_discharge.write(header_format_qyear.format(*header_qyear) + "\n")

    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0)

    # Create a cell manager if it is not provided
    if cell_manager is None:
        cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2,
                                   lons2d=lons2d, lats2d=lats2d)

    if stations is not None:
        # Get the list of the corresponding model points
        station_to_modelpoint = cell_manager.get_model_points_for_stations(
            station_list=stations,
            lake_fraction=lake_fraction,
            drainaige_area_reldiff_limit=drainage_area_reldiff_min)

        station_list = list(station_to_modelpoint.keys())
        station_list.sort(key=lambda st1: st1.latitude, reverse=True)
        mp_list = [station_to_modelpoint[st] for st in station_list]
    else:
        mp_list = model_points
        station_list = None
        # sort so that the northernmost stations appear uppermost
        mp_list.sort(key=lambda mpt: mpt.latitude, reverse=True)


    # set ids to the model points so they can be distinguished easier
    model_point.set_model_point_ids(mp_list)


    # ###Uncomment the lines below for the validation plot in paper 2
    # brewer2mpl.get_map args: set name  set type  number of colors
    # bmap = brewer2mpl.get_map("Set1", "qualitative", 9)
    # Change the default colors
    # mpl.rcParams["axes.color_cycle"] = bmap.mpl_colors

    # For the streamflow only plot
    ncols = 3
    nrows = max(len(mp_list) // ncols, 1)
    if ncols * nrows < len(mp_list):
        nrows += 1

    figure_stfl = plt.figure(figsize=(4 * ncols, 3 * nrows))
    gs_stfl = gridspec.GridSpec(nrows=nrows, ncols=ncols)
    # a flag which signifies if a legend should be added to the plot, it is needed so we ahve only one legend per plot
    legend_added = False

    ax_stfl = None
    all_years = [y for y in range(start_year, end_year + 1)]

    if station_list is not None:
        processed_stations = station_list
    else:
        processed_stations = [None] * len(mp_list)
    processed_model_points = mp_list
    plot_point_positions_with_upstream_areas(processed_stations, processed_model_points, basemap,
                                             cell_manager, lake_fraction_field=lake_fraction)

    if plot_upstream_area_averaged:
        # create obs data managers
        anusplin_tmin = AnuSplinManager(variable="stmn")
        anusplin_tmax = AnuSplinManager(variable="stmx")
        anusplin_pcp = AnuSplinManager(variable="pcp")

        daily_dates, obs_tmin_fields = anusplin_tmin.get_daily_clim_fields_interpolated_to(
            start_year=start_year, end_year=end_year,
            lons_target=lons2d, lats_target=lats2d)

        _, obs_tmax_fields = anusplin_tmax.get_daily_clim_fields_interpolated_to(
            start_year=start_year, end_year=end_year,
            lons_target=lons2d, lats_target=lats2d)

        _, obs_pcp_fields = anusplin_pcp.get_daily_clim_fields_interpolated_to(
            start_year=start_year, end_year=end_year,
            lons_target=lons2d, lats_target=lats2d)

        swe_path = "/skynet3_rech1/huziy/swe_ross_brown/swe.nc4"
        if not os.path.isfile(os.path.realpath(swe_path)):
            raise IOError("SWE-obs file {} does not exist".format(swe_path))

        swe_manager = SweDataManager(path=swe_path, var_name="SWE")
        obs_swe_daily_clim = swe_manager.get_daily_climatology(start_year, end_year)
        interpolated_obs_swe_clim = swe_manager.interpolate_daily_climatology_to(obs_swe_daily_clim,
                                                                                 lons2d_target=lons2d,
                                                                                 lats2d_target=lats2d)
    values_obs = None

    for i, the_model_point in enumerate(mp_list):

        ax_stfl = figure_stfl.add_subplot(gs_stfl[i // ncols, i % ncols], sharex=ax_stfl)

        assert isinstance(the_model_point, ModelPoint)

        # Check the number of years accessible for the station if the list of stations is given
        the_station = None if station_list is None else station_list[i]
        if the_station is not None:
            assert isinstance(the_station, Station)
            year_list = the_station.get_list_of_complete_years()
            year_list = list(filter(lambda yi: start_year <= yi <= end_year, year_list))

            if len(year_list) < 1:
                continue
        else:
            year_list = all_years

        fig = plt.figure(figsize=(12, 15))

        gs = gridspec.GridSpec(4, 4, wspace=1)


        # plot station position
        ax = fig.add_subplot(gs[3, 0:2])
        upstream_mask = _plot_station_position(ax, the_station, basemap, cell_manager, the_model_point)



        # plot streamflows
        ax = fig.add_subplot(gs[0:2, 0:2])

        dates = None
        model_daily_temp_clim = {}
        model_daily_precip_clim = {}
        model_daily_clim_surf_runoff = {}
        model_daily_clim_subsurf_runoff = {}
        model_daily_clim_swe = {}

        # get model data for the list of years
        simlabel_to_vals = {}
        for label in label_list:
            fname = sim_name_to_file_name[label]

            if hdf_folder is None:
                fpath = fname
            else:
                fpath = os.path.join(hdf_folder, fname)

            if plot_upstream_area_averaged:
                # read temperature data and calculate daily climatologic fileds
                _, model_daily_temp_clim[label] = analysis.get_daily_climatology(
                    path_to_hdf_file=fpath, var_name="TT", level=0, start_year=start_year, end_year=end_year)

                # read modelled precip and calculate daily climatologic fields
                _, model_daily_precip_clim[label] = analysis.get_daily_climatology(
                    path_to_hdf_file=fpath, var_name="PR", level=0, start_year=start_year, end_year=end_year)

                # read modelled surface runoff and calculate daily climatologic fields
                _, model_daily_clim_surf_runoff[label] = analysis.get_daily_climatology(
                    path_to_hdf_file=fpath, var_name="TRAF", level=0, start_year=start_year, end_year=end_year)

                # read modelled subsurface runoff and calculate daily climatologic fields
                _, model_daily_clim_subsurf_runoff[label] = analysis.get_daily_climatology(
                    path_to_hdf_file=fpath, var_name="TDRA", level=0, start_year=start_year, end_year=end_year)

                # read modelled swe and calculate daily climatologic fields
                _, model_daily_clim_swe[label] = analysis.get_daily_climatology(
                    path_to_hdf_file=fpath, var_name="I5", level=0, start_year=start_year, end_year=end_year)

            dates, values_model = analysis.get_daily_climatology_for_a_point(path=fpath,
                                                                             var_name=stfl_name,
                                                                             years_of_interest=year_list,
                                                                             i_index=the_model_point.ix,
                                                                             j_index=the_model_point.jy)

            ax.plot(dates, values_model, label=label, lw=2)

            if sim_name_to_color is None:
                ax_stfl.plot(dates, values_model, label=label, lw=2)
            else:
                ax_stfl.plot(dates, values_model, sim_name_to_color[label], label=label, lw=2)

                print(20 * "!!!")
                print("{} -> {}".format(label, sim_name_to_color[label]))
                print(20 * "!!!")

            simlabel_to_vals[label] = values_model

        if the_station is not None:
            assert isinstance(the_station, Station)
            dates, values_obs = the_station.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=dates,
                                                                                                 years=year_list)

            # To keep the colors consistent for all the variables, the obs Should be plotted last
            ax.plot(dates, values_obs, label="Obs.", lw=2)
            # no ticklabels for streamflow plot
            plt.setp(ax.get_xticklabels(), visible=False)

            if sim_name_to_color is None:
                ax_stfl.plot(dates, values_obs, label="Obs.", lw=2)
            else:
                ax_stfl.plot(dates, values_obs, label="Obs.", lw=2, color=sim_name_to_color["Obs."])

            # Print excesss from streamflow validation
            for label, values_model in simlabel_to_vals.items():
                calclulate_spring_peak_err(dates, values_obs, values_model,
                                           st_id="{}: {}".format(label, the_station.id),
                                           da_mod=the_model_point.accumulation_area,
                                           da_obs=the_station.drainage_km2)





        ax.set_ylabel(r"Streamflow: ${\rm m^3/s}$")
        assert isinstance(ax, Axes)
        assert isinstance(fig, Figure)

        upstream_area_km2 = np.sum(cell_area_km2[upstream_mask == 1])
        # Put some information about the point
        if the_station is not None:
            lf_upstream = lake_fraction[upstream_mask == 1]
            point_info = "{0}".format(the_station.id)
            write_annual_flows_to_txt(label_list, simlabel_to_vals, values_obs, file_annual_discharge,
                                      station_id=the_station.id,
                                      da_obs=the_station.drainage_km2, da_mod=the_model_point.accumulation_area)

        else:
            point_info = "{0}".format(the_model_point.point_id)

        ax.annotate(point_info, (0.8, 0.8), xycoords="axes fraction",
                    bbox=dict(facecolor="white", alpha=0.5),
                    va="top", ha="right")

        ax.legend(loc=(0.0, 1.05), borderaxespad=0, ncol=3)
        ax.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0]))
        ax.xaxis.set_minor_locator(MonthLocator(bymonthday=15))
        ax.xaxis.set_major_locator(MonthLocator())

        ax.grid()

        streamflow_axes = ax  # save streamflow axes for later use

        if not legend_added:
            ax_stfl.legend(loc="lower left", bbox_to_anchor=(0, 1.15), borderaxespad=0, ncol=3)
            ax_stfl.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0]))
            ax_stfl.xaxis.set_minor_locator(MonthLocator(bymonthday=15))
            ax_stfl.xaxis.set_major_locator(MonthLocator())

            ax_stfl.set_ylabel(r"Streamflow ${\rm m^3/s}$")
            legend_added = True

        plt.setp(ax_stfl.get_xmajorticklabels(), visible=False)
        ax_stfl.yaxis.set_major_locator(MaxNLocator(nbins=5))
        sfmt = ScalarFormatter(useMathText=True)
        sfmt.set_powerlimits((-2, 2))
        ax_stfl.yaxis.set_major_formatter(sfmt)
        ax_stfl.grid()

        # annotate streamflow-only panel plot
        ax_stfl.annotate(point_info, (0.05, 0.95), xycoords="axes fraction",
                         bbox=dict(facecolor="white"),
                         va="top", ha="left")


        if plot_upstream_area_averaged:
            # plot temperature comparisons (tmod - daily with anusplin tmin and tmax)
            ax = fig.add_subplot(gs[3, 2:], sharex=streamflow_axes)
            _validate_temperature_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2,
                                                upstream_mask=upstream_mask,
                                                daily_dates=daily_dates,
                                                obs_tmin_clim_fields=obs_tmin_fields,
                                                obs_tmax_clim_fields=obs_tmax_fields,
                                                model_data_dict=model_daily_temp_clim,
                                                simlabel_list=label_list)

            # plot temperature comparisons (tmod - daily with anusplin tmin and tmax)
            ax = fig.add_subplot(gs[2, 2:], sharex=streamflow_axes)
            _validate_precip_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2,
                                           upstream_mask=upstream_mask,
                                           daily_dates=daily_dates,
                                           obs_precip_clim_fields=obs_pcp_fields,
                                           model_data_dict=model_daily_precip_clim,
                                           simlabel_list=label_list)


            # plot mean upstream surface runoff
            ax = fig.add_subplot(gs[0, 2:], sharex=streamflow_axes)
            _plot_upstream_surface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2,
                                          upstream_mask=upstream_mask,
                                          daily_dates=daily_dates,
                                          model_data_dict=model_daily_clim_surf_runoff,
                                          simlabel_list=label_list)


            # plot mean upstream subsurface runoff
            ax = fig.add_subplot(gs[1, 2:], sharex=streamflow_axes, sharey=ax)
            _plot_upstream_subsurface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2,
                                             upstream_mask=upstream_mask,
                                             daily_dates=daily_dates,
                                             model_data_dict=model_daily_clim_subsurf_runoff,
                                             simlabel_list=label_list)

            # plot mean upstream swe comparison
            ax = fig.add_subplot(gs[2, 0:2], sharex=streamflow_axes)
            print("Validating SWE for ", the_station.id, "--" * 20)
            _validate_swe_with_ross_brown(ax, the_model_point, cell_area_km2=cell_area_km2,
                                          upstream_mask=upstream_mask,
                                          daily_dates=daily_dates,
                                          model_data_dict=model_daily_clim_swe,
                                          obs_swe_clim_fields=interpolated_obs_swe_clim,
                                          simlabel_list=label_list)

        if the_station is not None:
            im_name = "comp_point_with_obs_{0}_{1}_{2}.png".format(the_station.id,
                                                                   the_station.source,
                                                                   "_".join(label_list))
            im_folder_path = os.path.join(images_folder, the_station.source)
        else:
            im_name = "comp_point_with_obs_{0}_{1}.png".format(the_model_point.point_id,
                                                               "_".join(label_list))
            im_folder_path = os.path.join(images_folder, "outlets_point_comp")


        # create a folder for a given source of observed streamflow if it does not exist yet
        if not os.path.isdir(im_folder_path):
            os.mkdir(im_folder_path)

        im_path = os.path.join(im_folder_path, im_name)

        if plot_upstream_area_averaged:
            fig.savefig(im_path, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight", transparent=True)

        plt.close(fig)


        # return  # temporary plot only one point

    assert isinstance(figure_stfl, Figure)
    figure_stfl.tight_layout()
    figure_stfl.savefig(os.path.join(images_folder,
                                     "comp_point_with_obs_{0}.png".format("_".join(label_list))),
                        bbox_inches="tight", transparent=True, dpi=cpp.FIG_SAVE_DPI)
    plt.close(figure_stfl)

    # close information text files
    for f in text_files:
        f.close()
def main(intf_file="", no_intf_file="", start_year=1980, end_year=2010, dt_hours=3):
    """
    Do it on a year by year basis
    :param intf_file:
    :param no_intf_file:
    :param start_year:
    :param end_year:

    """
    matplotlib.rc("font", size=20)
    img_folder = "long-rain-events-30y"
    if not os.path.isdir(img_folder):
        os.mkdir(img_folder)

    # Calculate the durations of the longest rain events in both simulations
    no_intf_all_max_durations, no_intf_acc_runoff, intf_all_max_durations, intf_acc_runoff = \
        get_longest_rain_durations_for_files(
            intf_file=intf_file, no_intf_file=no_intf_file, start_year=start_year, end_year=end_year)

    # Debug: visualize
    cmap = cm.get_cmap("rainbow", 20)

    lons, lats, basemap = analysis.get_basemap_from_hdf(file_path=no_intf_file)
    x, y = basemap(lons, lats)

    plt.figure()
    mean_max_durations_nointf = np.mean(no_intf_all_max_durations, axis=0).astype(int)
    im = basemap.pcolormesh(x, y, mean_max_durations_nointf, vmin=0, vmax=50, cmap=cmap)
    basemap.drawcoastlines()
    plt.title("no - intf")
    plt.colorbar(im)
    print(mean_max_durations_nointf.min(), mean_max_durations_nointf.max(), mean_max_durations_nointf.mean())
    plt.savefig(os.path.join(img_folder, "no-intf-durations.png"))

    plt.figure()
    mean_max_durations_intf = np.mean(intf_all_max_durations, axis=0).astype(int)
    im = basemap.pcolormesh(x, y, mean_max_durations_intf, vmin=0, vmax=50, cmap=cmap)
    basemap.drawcoastlines()
    plt.title("intf")
    plt.colorbar(im)
    print(mean_max_durations_intf.min(), mean_max_durations_intf.max(), mean_max_durations_intf.mean())
    plt.savefig(os.path.join(img_folder, "intf-durations.png"))

    # Plot the interflow effect on the longest rain events
    mask = maskoceans(lons, lats, mean_max_durations_intf, inlands=True).mask
    plt.figure()
    clevs = [0.5, 1, 5, 30, 100, 150]
    clevs = [-c for c in reversed(clevs)] + clevs
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    cmap_diff = cm.get_cmap("bwr", len(clevs) - 1)
    diff = np.ma.masked_where(mask, (mean_max_durations_intf - mean_max_durations_nointf) * dt_hours)

    im = basemap.pcolormesh(x, y, diff, cmap=cmap_diff, vmin=clevs[0], vmax=clevs[-1], norm=bn)
    basemap.drawcoastlines()
    plt.title("intf - nointf" + r", $\sum\Delta_{i, j}$ = " + "{}\n".format(diff.sum()))
    cb = plt.colorbar(im)
    cb.ax.set_title("hours")
    plt.savefig(os.path.join(img_folder, "diff_intf-nointf_durations.png"))

    # Plot differences in surface runoff
    plot_surface_runoff_differences(x, y, basemap, mask, no_intf_acc_runoff, intf_acc_runoff, dt_hours=dt_hours,
                                    img_path=os.path.join(img_folder, "runoff_during_long_rain_events.png"))

    # Plot numbers of events of different durations
    plot_nevents_duration_curves(mean_max_durations_nointf[~mask], mean_max_durations_intf[~mask],
                                 img_path=os.path.join(img_folder, "nevents_vs_duration_over_land.png"))

    plot_nevents_duration_curves(mean_max_durations_nointf[mask], mean_max_durations_intf[mask],
                                 img_path=os.path.join(img_folder, "nevents_vs_duration_over_ocean-and-lakes.png"))
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_year=1980, end_year=2010):
    prepare()

    all_markers = ["*", "s", "p", "+", "x", "d", "h"]

    excluded = ["white", "w", "aliceblue", "azure"]
    excluded.extend([ci for ci in colors.cnames if "yellow" in ci])

    all_colors = ["k", "b", "r", "g", "m"] + sorted([ci for ci in colors.cnames if ci not in excluded])

    # Station ids to get from the CEHQ database
    ids_with_lakes_upstream = [
        "104001", "093806", "093801", "081002", "081007", "080718"
    ]

    selected_ids = ids_with_lakes_upstream

    filedir = Path(hdf_folder)
    sim_name_to_file_path = OrderedDict([
        # ("CRCM5-LI", filedir.joinpath("quebec_0.1_crcm5-hcd-r.hdf5").as_posix()),

        ("ERAI-CRCM5-L", filedir.joinpath("quebec_0.1_crcm5-hcd-rl.hdf5").as_posix()),

        # ("CanESM2-CRCM5-NL", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5").as_posix()),

        ("CanESM2-CRCM5-L",
         filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5").as_posix()),

        # ("CanESM2-CRCM5-LI", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5").as_posix()),


    ])

    obs_label = "Obs."
    labels = [obs_label, ] + list(sim_name_to_file_path.keys())

    label_to_marker = dict(zip(labels, all_markers))
    label_to_color = dict(zip(labels, all_colors))

    # Get the list of stations to do the comparison with
    start_date = datetime(start_year, 1, 1)
    end_date = datetime(end_year, 12, 31)
    stations = cehq_station.read_station_data(
        start_date=start_date, end_date=end_date, selected_ids=selected_ids
    )

    # Get geophysical fields from one of the model simulations
    path0 = list(sim_name_to_file_path.values())[0]
    lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0)
    flow_directions = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
    lake_fraction = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_LAKE_FRACTION_NAME)

    accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
    area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2)

    # Try to read cell areas im meters if it is not Ok then try in km2
    if area_m2 is not None:
        cell_area_km2 = area_m2 * 1.0e-6
    else:
        cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2)

    # Create a cell manager if it is not provided
    cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2,
                               lons2d=lons2d, lats2d=lats2d)

    # Get the list of the corresponding model points
    station_to_modelpoint = cell_manager.get_model_points_for_stations(
        station_list=stations,
        lake_fraction=lake_fraction,
        drainaige_area_reldiff_limit=0.1)

    # plot_utils.apply_plot_params(font_size=10, width_cm=20, height_cm=18)
    fig = plt.figure()

    ncols = max([len(rp_list) for et, rp_list in ExtremeProperties.extreme_type_to_return_periods.items()])
    nrows = len(ExtremeProperties.extreme_types)
    gs = GridSpec(nrows, ncols)

    ext_type_to_rp_to_ax = OrderedDict()
    ax_with_legend = None

    label_to_ax_to_xdata = {}
    label_to_ax_to_ydata = {}
    for row, ext_type in enumerate(ExtremeProperties.extreme_types):
        ext_type_to_rp_to_ax[ext_type] = OrderedDict()
        for col, rperiod in enumerate(ExtremeProperties.extreme_type_to_return_periods[ext_type]):
            ax = fig.add_subplot(gs[row, col])
            ext_type_to_rp_to_ax[ext_type][rperiod] = ax

            if col == 0:
                ax.set_ylabel(ext_type)

            if row == nrows - 1 and col == ncols - 1:
                ax_with_legend = ax

            # Set axes labels
            if row == nrows - 1:
                ax.set_xlabel("Observations")

            if col == 0:
                ax.set_ylabel("Model")

            for label in sim_name_to_file_path:

                if label not in label_to_ax_to_xdata:
                    label_to_ax_to_xdata[label] = {ax: []}
                    label_to_ax_to_ydata[label] = {ax: []}
                else:
                    label_to_ax_to_xdata[label][ax] = []
                    label_to_ax_to_ydata[label][ax] = []

            ax.set_xscale("log")
            ax.set_yscale("log")

    print("Initial list of stations:")

    sim_label_to_handle = {}
    for s in stations:
        print("{0}".format(s))
        assert isinstance(s, Station)

        print(len([y for y in s.get_list_of_complete_years() if start_year <= y <= end_year]))
        df_ext_obs = extreme_commons.get_annual_extrema(ts_times=s.dates, ts_vals=s.values,
                                                        start_year=start_year, end_year=end_year)
        mp = station_to_modelpoint[s]

        assert isinstance(mp, ModelPoint)

        years_of_interest = df_ext_obs.index

        label_to_extrema_model = {}



        # label -> ext_type -> [return period -> ret level, return period -> std]
        label_to_return_levels = OrderedDict(
            [(obs_label, OrderedDict())]
        )
        for sim_label, sim_path in sim_name_to_file_path.items():
            label_to_return_levels[sim_label] = OrderedDict()
            label_to_extrema_model[sim_label] = OrderedDict()



        # Calculate the return levels and standard deviations
        for ext_type in ExtremeProperties.extreme_types:

            return_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type]

            # fit GEV distribution and apply non-parametric bootstrap to get std
            label_to_return_levels[obs_label][ext_type] = gevfit.do_gevfit_for_a_point(df_ext_obs[ext_type].values,
                                                                                       extreme_type=ext_type,
                                                                                       return_periods=return_periods)
            return_levels_obs, rl_stds_obs = label_to_return_levels[obs_label][ext_type]


            # get annual extremas for the model output at the points colose to the stations
            for sim_label, sim_path in sim_name_to_file_path.items():
                label_to_return_levels[sim_label] = OrderedDict()

                ext_field = analysis.get_annual_extrema(
                    rconfig=RunConfig(data_path=sim_path, start_year=start_year, end_year=end_year),
                    varname="STFL", months_of_interest=ExtremeProperties.extreme_type_to_month_of_interest[ext_type],
                    n_avg_days=ExtremeProperties.extreme_type_to_n_agv_days[ext_type],
                    high_flow=ext_type == ExtremeProperties.high)

                # Select only those years when obs are available
                ts_data = [v for y, v in zip(range(start_year, end_year + 1), ext_field[:, mp.ix, mp.jy]) if
                           y in years_of_interest]
                ts_data = np.array(ts_data)
                return_levels, rl_stds = gevfit.do_gevfit_for_a_point(ts_data, extreme_type=ext_type,
                                                                      return_periods=return_periods)





                # Do the plotting
                for rp in return_periods:
                    ax = ext_type_to_rp_to_ax[ext_type][rp]
                    ax.set_title("T = {rp}-year".format(rp=rp))

                    # h = ax.errorbar(return_levels_obs[rp], return_levels[rp],
                    # marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label,
                    #                 xerr=rl_stds_obs[rp] * 1.96, yerr=rl_stds[rp] * 1.96)

                    h = ax.scatter(return_levels_obs[rp], return_levels[rp],
                                   marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label)



                    # save the data for maybe further calculation of the correlation coefficients
                    label_to_ax_to_xdata[sim_label][ax].append(return_levels_obs[rp])
                    label_to_ax_to_ydata[sim_label][ax].append(return_levels[rp])

                    sim_label_to_handle[sim_label] = h



    # Calculate the biases
    for sim_label in sim_name_to_file_path:
        for ext_type in ExtremeProperties.extreme_types:
            ret_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type]
            for rp in ret_periods:

                ax = ext_type_to_rp_to_ax[ext_type][rp]
                mod = np.asarray(label_to_ax_to_ydata[sim_label][ax])
                obs = np.asarray(label_to_ax_to_xdata[sim_label][ax])

                bias = np.mean((mod - obs)/obs)
                corr, pv = stats.pearsonr(mod, obs)
                print("({sim_label}) Mean bias for {rp}-year {ext_type}-flow return level is: {bias}; corr={corr:.2f}; corr_pval={corr_pval:2g}".format(
                    sim_label=sim_label, rp=rp, bias=bias, corr=corr, corr_pval=pv,
                    ext_type=ext_type
                ))




    sfmt = ScalarFormatter(useMathText=True)
    sfmt.set_powerlimits((-2, 2))
    for et, rp_to_ax in ext_type_to_rp_to_ax.items():
        for rp, ax in rp_to_ax.items():
            xmin, xmax = ax.get_xlim()
            ymin, ymax = ax.get_ylim()
            x1 = min(xmin, ymin)
            x2 = min(xmax, ymax)
            ax.plot([x1, x2], [x1, x2], "k--")
            # ax.xaxis.set_major_locator(MaxNLocator(nbins=5))
            # ax.yaxis.set_major_locator(MaxNLocator(nbins=5))
            # ax.xaxis.set_major_formatter(sfmt)
            # ax.yaxis.set_major_formatter(sfmt)

    sim_labels = list(sim_name_to_file_path.keys())
    ax_with_legend.legend([sim_label_to_handle[sl] for sl in sim_labels], sim_labels,
                          bbox_to_anchor=(1, -0.25), borderaxespad=0.0, loc="upper right",
                          ncol=2, scatterpoints=1, numpoints=1)

    # Save the plot
    img_file = "{}.eps".format("_".join(sorted(label_to_marker.keys())))
    img_file = img_folder.joinpath(img_file)

    fig.tight_layout()
    with img_file.open("wb") as f:
        fig.savefig(f, bbox_inches="tight")