Esempio n. 1
0
    def get_lake_fraction_for_date(self, the_date=None):

        """
        Get the lake ice cover fraction for the specified date
        :param the_date:
        :return:
        """
        if the_date not in self.cached_data:
            month_date = datetime(the_date.year, the_date.month, 1)

            self.cached_data = {}
            mr = MultiRPN(self.year_month_date_to_file_list[month_date])
            self.cached_data = mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
                                                                          level_kind=self.level_kind)
            mr.close()

        # if still it is not there try looking in the previous month
        if the_date not in self.cached_data:

            month = the_date.month - 1
            year = the_date.year
            if the_date.month == 0:
                month = 12
                year -= 1


            month_date = datetime(year, month, 1)

            mr = MultiRPN(self.year_month_date_to_file_list[month_date])
            self.cached_data.update(mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
                                                                               level_kind=self.level_kind))
            mr.close()

        return self.cached_data[the_date]
Esempio n. 2
0
def test_getting_coordinates_for_the_last_read_record_should_not_fail():
    r = None
    try:

        r = MultiRPN(in_path)
        rec = r.get_first_record_for_name("I5")
        lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
        tools.assert_equals(lons.shape, lats.shape, "Shapes of lon and lat arrays should be the same.")

    finally:
        if r is not None:
            r.close()

        delete_files()
Esempio n. 3
0
def test_get_number_of_records():
    r = None
    try:

        create_files()
        r = MultiRPN("test_?.rpn")
        msg = "Number of records should be equal to the total number in all files"
        tools.assert_equals(r.get_number_of_records(), len(FILE_NAMES), msg)

    finally:
        if r is not None:
            r.close()

        delete_files()
Esempio n. 4
0
def test_can_link_many_files(nfiles=200):
    many_fnames = ["test_{}.rpn".format(i) for i in range(nfiles)]

    r = None
    try:
        create_files(fnames=many_fnames)

        r = MultiRPN("test_*.rpn")

        tools.assert_greater(r.get_number_of_records(), 0, msg="There should be more than 0 records")

    finally:
        if r is not None:
            r.close()

        delete_files(fnames=many_fnames)
Esempio n. 5
0
def test_should_find_all_records():
    r = None
    try:

        create_files()
        r = MultiRPN("test_?.rpn")

        for fn in range(len(FILE_NAMES)):
            rec = r.get_first_record_for_name("T{}".format(fn))
            print("file {} - OK".format(fn))
            print(rec.mean(), rec.shape)

    finally:
        if r is not None:
            r.close()

        delete_files()
Esempio n. 6
0
def extract_runoff_to_nc_process(args):
    in_path, out_path = args

    if os.path.exists(out_path):
        print("Nothing to do for: {}".format(out_path))
        return  # skip files that already exist


    traf_name = "TRAF"
    tdra_name = "TDRA"

    r = MultiRPN(in_path)
    traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
    tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
    r.close()

    nx, ny = list(traf_data.items())[0][1].shape

    with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
        ds.createDimension("lon", nx)
        ds.createDimension("lat", ny)
        ds.createDimension("time", None)

        varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
        varTraf.units = "kg/( m**2 * s )"

        varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
        varTdra.units = "kg/( m**2 * s )"

        timeVar = ds.createVariable("time", "f4", dimensions=("time",))

        sorted_dates = list(sorted(traf_data.keys()))

        timeVar.units = "hours since {0}".format(sorted_dates[0])
        timeVar[:] = nc.date2num(sorted_dates, timeVar.units)

        varTraf[:] = np.array(
            [traf_data[d] for d in sorted_dates]
        )

        varTdra[:] = np.array(
            [tdra_data[d] for d in sorted_dates]
        )
Esempio n. 7
0
def test_get_4d_field():
    r = None
    vname = "T"

    try:

        create_files_with_same_var_for_different_times(vname=vname)
        r = MultiRPN("test_?.rpn")

        recs = r.get_4d_field(vname)
        msg = "Not all records for {} were found".format(vname)
        print(recs.keys())
        tools.assert_equals(len(FILE_NAMES), len(recs), msg)
    finally:

        if r is not None:
            r.close()

        delete_files()
Esempio n. 8
0
def extract_runoff_to_nc_process(args):
    in_path, out_path = args

    if os.path.exists(out_path):
        print("Nothing to do for: {}".format(out_path))
        return  # skip files that already exist


    traf_name = "TRAF"
    tdra_name = "TDRA"

    r = MultiRPN(in_path)
    traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
    tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
    r.close()

    nx, ny = list(traf_data.items())[0][1].shape

    with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
        ds.createDimension("lon", nx)
        ds.createDimension("lat", ny)
        ds.createDimension("time", None)

        varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
        varTraf.units = "kg/( m**2 * s )"

        varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
        varTdra.units = "kg/( m**2 * s )"

        timeVar = ds.createVariable("time", "f4", dimensions=("time",))

        sorted_dates = list(sorted(traf_data.keys()))

        timeVar.units = "hours since {0}".format(sorted_dates[0])
        timeVar[:] = nc.date2num(sorted_dates, timeVar.units)

        varTraf[:] = np.array(
            [traf_data[d] for d in sorted_dates]
        )

        varTdra[:] = np.array(
            [tdra_data[d] for d in sorted_dates]
        )
Esempio n. 9
0
def test_get_list_of_varnames():
    r = None
    vname = "T"

    try:

        create_files_with_same_var_for_different_times(vname=vname)
        r = MultiRPN("test_?.rpn")

        msg = "the list of varnames should contain {}".format(vname)
        vnames = r.get_list_of_varnames()

        tools.assert_in(vname, vnames, msg)
        tools.assert_equal(len(vnames), 1, "There is only one unique field name in the files")

    finally:
        if r is not None:
            r.close()

        delete_files()
Esempio n. 10
0
    def get_lake_fraction_for_date(self, the_date=None):

        """
        Get the lake ice cover fraction for the specified date
        :param the_date:
        :return:
        """
        if the_date not in self.cached_data:
            month_date = datetime(the_date.year, the_date.month, 1)

            self.cached_data = {}
            mr = MultiRPN(self.year_month_date_to_file_list[month_date])
            self.cached_data = mr.get_all_time_records_for_name_and_level(
                varname=self.varname, level=self.level, level_kind=self.level_kind
            )
            mr.close()

        # if still it is not there try looking in the previous month
        if the_date not in self.cached_data:

            month = the_date.month - 1
            year = the_date.year
            if the_date.month == 0:
                month = 12
                year -= 1

            month_date = datetime(year, month, 1)

            mr = MultiRPN(self.year_month_date_to_file_list[month_date])
            self.cached_data.update(
                mr.get_all_time_records_for_name_and_level(
                    varname=self.varname, level=self.level, level_kind=self.level_kind
                )
            )
            mr.close()

        return self.cached_data[the_date]
def get_model_data(station_to_model_point, output_path=None, grid_config=None, basins_of_interest_shp="",
                   cell_manager=None, vname=None):
    lons, lats, bmp = None, None, None
    data_mask = None

    monthly_diagnostics_case = False
    if output_path.name.lower().endswith("diagnostics"):
        fname_pattern = "pm*_moyenne"
        monthly_diagnostics_case = True
    else:
        fname_pattern = "pm*p"

    pattern = re.compile(".*" + 8 * "0" + ".*")

    flist = [f for f in glob.glob(str(output_path.joinpath("*").joinpath(fname_pattern))) if pattern.match(f) is None]
    r = MultiRPN(flist)

    date_to_field = r.get_all_time_records_for_name_and_level(varname=vname)
    lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()

    r.close()

    # get the basemap object
    bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
        lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)

    station_to_model_data = {}  # model data are the pandas timeseries

    stations_to_ignore = []

    for station, model_point in station_to_model_point.items():
        assert isinstance(model_point, ModelPoint)
        assert isinstance(cell_manager, CellManager)
        assert isinstance(station, cehq_station.Station)

        upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(model_point.ix,
                                                                                          model_point.jy)

        # Skip model points and staions with small number of gridcells upstream
        if upstream_mask.sum() <= 1:
            stations_to_ignore.append(station)
            print("Station {} is ignored, because the number of upstream cells is <= 1.".format(station.id))
            continue

        # Skip model points and stations outside the region of interest
        if not data_mask[model_point.ix, model_point.jy]:
            stations_to_ignore.append(station)
            print("Station {} is ignored, because it is outside of the domain of interest.".format(station.id))
            continue

        # Plot station position
        fig = plt.figure()

        ax = plt.gca()

        lons1, lats1 = lons[upstream_mask > 0.5], lats[upstream_mask > 0.5]
        x1, y1 = bmp(lons1, lats1)

        bmp.drawrivers()
        bmp.drawcoastlines(ax=ax)
        bmp.drawcountries(ax=ax, linewidth=0.2)
        bmp.drawstates(linewidth=0.1)
        bmp.readshapefile(basins_of_interest_shp[:-4], "basin", linewidth=2, color="m")

        bmp.scatter(x1, y1, c="g", s=100)
        bmp.scatter(*bmp(lons[model_point.ix, model_point.jy], lats[model_point.ix, model_point.jy]), c="b", s=250)

        fig.tight_layout()
        plt.savefig(str(img_folder.joinpath("{}_position_and_upstream.png".format(station.id))), bbox_inche="tight")
        plt.close(fig)


        res = pd.Series(index=sorted(date_to_field.keys()),
                        data=[date_to_field[d][model_point.ix, model_point.jy] for d in sorted(date_to_field.keys())])

        # get monthly means
        res = res.groupby(lambda d: d.replace(day=15, hour=0)).mean()

        if monthly_diagnostics_case:
            # shift to the end of the month before a previous month, and then shift 15 days to later
            res = res.shift(-2, freq="M").shift(15, freq="D")

        print(res.index[:20])
        print(res.index[-20:])

        station_to_model_data[station] = res


    # Not enough drainage area
    for s in stations_to_ignore:
        del station_to_model_point[s]

    return station_to_model_data
def plot_monthly_clim_in_a_panel(months=None, diag_folder="", vname="STFL",
                                 grid_config=None, basins_of_interest_shp=""):
    """
    Plots climatologies using diagnostics outputs, not samples
    :param months:
    :param diag_folder:
    :param vname:
    """
    if months is None:
        months = list(range(1, 13))


    diag_path = Path(diag_folder)
    month_to_field = OrderedDict()

    lons, lats, bmp = None, None, None
    data_mask = None

    for m in months:

        r = MultiRPN(str(diag_path.joinpath("*{:02d}".format(m)).joinpath("pm*_moyenne")))

        date_to_field = r.get_all_time_records_for_name_and_level()

        the_mean = np.mean([f for f in date_to_field.values()], axis=0)

        the_mean = np.ma.masked_where(the_mean < 0, the_mean)

        month_to_field[m] = the_mean

        if bmp is None:
            lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()

            # get the basemap object
            bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
                lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)

        r.close()



    fig = plt.figure()


    ncols = 3
    nrows = len(months) // ncols + int(len(months) % ncols != 0)

    gs = GridSpec(nrows=nrows, ncols=ncols + 1)


    xx, yy = bmp(lons, lats)

    clevs = [0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000]
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    cmap = cm.get_cmap("jet", len(clevs) - 1)
    for m, field in month_to_field.items():
        row = (m - 1) // ncols
        col = (m - 1) % ncols

        ax = fig.add_subplot(gs[row, col])
        ax.set_title(calendar.month_name[m])

        to_plot = np.ma.masked_where(~data_mask, field)
        im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
        bmp.colorbar(im, extend="max")

        bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
        bmp.drawcoastlines(ax=ax)


    plt.close(fig)

    # plot annual mean
    ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
    fig = plt.figure()
    ax = fig.add_subplot(gs[:, :])
    ax.set_title("Annual mean")

    to_plot = np.ma.masked_where(~data_mask, ann_mean)
    im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
    bmp.colorbar(im, extend="max")

    bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
    bmp.drawcoastlines(ax=ax)

    plt.show()
    plt.close(fig)
Esempio n. 13
0
def main(plot_vals=False):
    varname = "RFAC"
    multiplier = 24 * 3600
    data_folder = Path(
        "/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations"
    )
    day_range = range(19, 22)
    dates_of_interest = [datetime(2013, 6, d) for d in day_range]

    img_folder = Path("calgary_flood/2D")

    if not img_folder.is_dir():
        img_folder.mkdir(parents=True)

    lons, lats, bmp = None, None, None

    the_mask = get_bow_river_basin_mask()

    i_list, j_list = np.where(the_mask > 0.5)
    imin, imax = i_list.min() - 2, i_list.max() + 5
    jmin, jmax = j_list.min() - 2, j_list.max() + 5

    # Calculate daily means
    sim_label_to_date_to_mean = OrderedDict()
    for sim_dir in data_folder.iterdir():
        mr = MultiRPN(str(sim_dir.joinpath("pm*")))
        print(str(sim_dir))
        print(mr.get_number_of_records())

        label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace(
            "frozen", "Frozen").replace("Bow", "")
        sim_label_to_date_to_mean[label] = OrderedDict()

        data = mr.get_4d_field(varname)
        data = {
            d: list(v.items())[0][1]
            for d, v in data.items() if d.day in day_range
        }

        for d in dates_of_interest:
            sim_label_to_date_to_mean[label][d] = np.array([
                field for d1, field in data.items() if d1.day == d.day
            ]).mean(axis=0) * multiplier

        if lons is None:
            lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
            for f in sim_dir.iterdir():
                if f.name.startswith("pm"):
                    r = RPN(str(f))
                    r.get_first_record_for_name(varname=varname)
                    rll = RotatedLatLon(
                        **r.get_proj_parameters_for_the_last_read_rec())
                    bmp = rll.get_basemap_object_for_lons_lats(
                        lons2d=lons[imin:imax, jmin:jmax],
                        lats2d=lats[imin:imax, jmin:jmax],
                        resolution="i")
                    r.close()
                    break

        mr.close()

    # reorder simulations
    sim_label_to_date_to_mean = OrderedDict([
        (k, sim_label_to_date_to_mean[k]) for k in sorted(
            sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
    ])

    key_list = [k for k in sim_label_to_date_to_mean]
    key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
    sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k])
                                             for k in key_list])

    # do the plots (subplots: vertically - simulations, horizontally - days)
    plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
    fig = plt.figure()
    nrows = len(sim_label_to_date_to_mean)
    ncols = len(day_range)
    gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)

    clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]

    base_label = None
    clevs_diff = [1, 5, 10, 20, 50, 100]
    clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
    cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
    cmap_vals = get_cmap_from_ncl_spec_file(
        path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
    xx, yy = bmp(lons, lats)
    title = "Total runoff ({}, mm/day)".format(varname)
    fig.suptitle(title)

    for row, (sim_label,
              date_to_field) in enumerate(sim_label_to_date_to_mean.items()):

        if row == 0 or plot_vals:
            base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
            base_label = sim_label
            plot_label = sim_label
            clevs = clevs_vals
            cmap = cmap_vals
            extend = "max"
        else:
            base_sim = list(sim_label_to_date_to_mean.items())[0][1]
            plot_label = "{}\n-\n{}".format(sim_label, base_label)
            clevs = clevs_diff
            cmap = cmap_diff
            extend = "both"

        bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)

        for col, (the_date, field) in enumerate(date_to_field.items()):

            ax = fig.add_subplot(gs[row, col])
            to_plot = np.ma.masked_where(the_mask < 0.5,
                                         field - base_sim[the_date])
            # cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
            cs = bmp.pcolormesh(xx,
                                yy,
                                to_plot[:-1, :-1],
                                norm=bn,
                                vmin=clevs[0],
                                vmax=clevs[-1],
                                cmap=cmap)

            bmp.drawcoastlines(ax=ax, linewidth=0.3)
            assert isinstance(bmp, Basemap)
            bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
            cb = bmp.colorbar(cs,
                              ax=ax,
                              ticks=clevs,
                              extend=extend,
                              pad="4%",
                              size="10%")

            if plot_vals:
                cb.ax.set_visible(col == ncols - 1 and row == 0)
            else:
                cb.ax.set_visible(col == ncols - 1 and row in (0, 1))

            if col == 0:
                ax.set_ylabel(plot_label)

            if row == 0:
                ax.set_title(the_date.strftime("%b %d"))

    if plot_vals:
        img_file = img_folder.joinpath("{}.png".format(varname))
    else:
        img_file = img_folder.joinpath("{}_diff.png".format(varname))

    fig.savefig(str(img_file))
    plt.close(fig)
Esempio n. 14
0
def get_area_avg_timeseries(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm",
                            level=-1, level_kind=level_kinds.ARBITRARY,
                            varname="", mask=None, mask_lons2d=None, mask_lats2d=None, file_per_var=False) -> pd.Series:
    """
    get the timeseries of area averaged ice fraction
    :rtype : pd.Series
    """

    yearly_ts = []


    lons2d, lats2d = None, None
    samples_dir_p = Path(samples_dir)


    # interpolated mask
    interpolated_mask = None


    for y in range(start_year, end_year + 1):
        files_for_year = []

        mfolders = [f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))]

        for mfolder in mfolders:
            # Select all files containing the varname in the filename
            if file_per_var:
                files_for_year += [str(f) for f in mfolder.iterdir() if varname in f.name]
            else:
                files_for_year += [str(f) for f in mfolder.iterdir() if f.name.startswith(filename_prefix) and f.name[-9:-1] != "0" * 8]


        if len(files_for_year) == 0:
            continue

        mrpn = MultiRPN(files_for_year)
        data = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind)

        if lons2d is None:
            lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec()


        # interpolate the mask using nearest neighbour approach
        if interpolated_mask is None:
            xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(), mask_lats2d.flatten())
            ktree = KDTree(data=list(zip(xs, ys, zs)))

            xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
            dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)

            interpolated_mask = mask.flatten()[inds]
            interpolated_mask.shape = lons2d.shape


        for t, field in data.items():
            data[t] = field[interpolated_mask].mean()

        tlist = [t for t in data.keys()]
        ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
        ser.sort_index(inplace=True)
        yearly_ts.append(ser)
        mrpn.close()

    return pd.concat(yearly_ts), lons2d, lats2d
Esempio n. 15
0
def plot_monthly_clim_in_a_panel(months=None,
                                 diag_folder="",
                                 vname="STFL",
                                 grid_config=None,
                                 basins_of_interest_shp=""):
    """
    Plots climatologies using diagnostics outputs, not samples
    :param months:
    :param diag_folder:
    :param vname:
    """
    if months is None:
        months = list(range(1, 13))

    diag_path = Path(diag_folder)
    month_to_field = OrderedDict()

    lons, lats, bmp = None, None, None
    data_mask = None

    for m in months:

        r = MultiRPN(
            str(
                diag_path.joinpath(
                    "*{:02d}".format(m)).joinpath("pm*_moyenne")))

        date_to_field = r.get_all_time_records_for_name_and_level()

        the_mean = np.mean([f for f in date_to_field.values()], axis=0)

        the_mean = np.ma.masked_where(the_mean < 0, the_mean)

        month_to_field[m] = the_mean

        if bmp is None:
            lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()

            # get the basemap object
            bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
                lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)

        r.close()

    fig = plt.figure()

    ncols = 3
    nrows = len(months) // ncols + int(len(months) % ncols != 0)

    gs = GridSpec(nrows=nrows, ncols=ncols + 1)

    xx, yy = bmp(lons, lats)

    clevs = [
        0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000
    ]
    bn = BoundaryNorm(clevs, len(clevs) - 1)
    cmap = cm.get_cmap("jet", len(clevs) - 1)
    for m, field in month_to_field.items():
        row = (m - 1) // ncols
        col = (m - 1) % ncols

        ax = fig.add_subplot(gs[row, col])
        ax.set_title(calendar.month_name[m])

        to_plot = np.ma.masked_where(~data_mask, field)
        im = bmp.pcolormesh(xx,
                            yy,
                            to_plot,
                            norm=bn,
                            cmap=cmap,
                            vmin=clevs[0],
                            vmax=clevs[-1])
        bmp.colorbar(im, extend="max")

        bmp.readshapefile(basins_of_interest_shp[:-4],
                          "basins",
                          linewidth=2,
                          color="m",
                          ax=ax)
        bmp.drawcoastlines(ax=ax)

    plt.close(fig)

    # plot annual mean
    ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
    fig = plt.figure()
    ax = fig.add_subplot(gs[:, :])
    ax.set_title("Annual mean")

    to_plot = np.ma.masked_where(~data_mask, ann_mean)
    im = bmp.pcolormesh(xx,
                        yy,
                        to_plot,
                        norm=bn,
                        cmap=cmap,
                        vmin=clevs[0],
                        vmax=clevs[-1])
    bmp.colorbar(im, extend="max")

    bmp.readshapefile(basins_of_interest_shp[:-4],
                      "basins",
                      linewidth=2,
                      color="m",
                      ax=ax)
    bmp.drawcoastlines(ax=ax)

    plt.show()
    plt.close(fig)
Esempio n. 16
0
def get_area_avg_timeseries(samples_dir,
                            start_year=-np.Inf,
                            end_year=np.Inf,
                            filename_prefix="pm",
                            level=-1,
                            level_kind=level_kinds.ARBITRARY,
                            varname="",
                            mask=None,
                            mask_lons2d=None,
                            mask_lats2d=None,
                            file_per_var=False) -> pd.Series:
    """
    get the timeseries of area averaged ice fraction
    :rtype : pd.Series
    """

    yearly_ts = []

    lons2d, lats2d = None, None
    samples_dir_p = Path(samples_dir)

    # interpolated mask
    interpolated_mask = None

    for y in range(start_year, end_year + 1):
        files_for_year = []

        mfolders = [
            f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))
        ]

        for mfolder in mfolders:
            # Select all files containing the varname in the filename
            if file_per_var:
                files_for_year += [
                    str(f) for f in mfolder.iterdir() if varname in f.name
                ]
            else:
                files_for_year += [
                    str(f) for f in mfolder.iterdir()
                    if f.name.startswith(filename_prefix)
                    and f.name[-9:-1] != "0" * 8
                ]

        if len(files_for_year) == 0:
            continue

        mrpn = MultiRPN(files_for_year)
        data = mrpn.get_all_time_records_for_name_and_level(
            varname=varname, level=level, level_kind=level_kind)

        if lons2d is None:
            lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec(
            )

        # interpolate the mask using nearest neighbour approach
        if interpolated_mask is None:
            xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(),
                                                      mask_lats2d.flatten())
            ktree = KDTree(data=list(zip(xs, ys, zs)))

            xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(),
                                                      lats2d.flatten())
            dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)

            interpolated_mask = mask.flatten()[inds]
            interpolated_mask.shape = lons2d.shape

        for t, field in data.items():
            data[t] = field[interpolated_mask].mean()

        tlist = [t for t in data.keys()]
        ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
        ser.sort_index(inplace=True)
        yearly_ts.append(ser)
        mrpn.close()

    return pd.concat(yearly_ts), lons2d, lats2d
Esempio n. 17
0
def main():
    #path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
    path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"

    months = [6, 7, 8]

    pm_list = []
    dm_list = []
    for m in months:
        print(path.format(m))

        month_folder = path.format(m)
        for fn in os.listdir(month_folder):

            # if not fn.endswith("moyenne"):
            #    continue

            if fn.startswith("pm"):
                pm_list.append(os.path.join(month_folder, fn))
            elif fn.startswith("dm"):
                dm_list.append(os.path.join(month_folder, fn))

    pm = MultiRPN(pm_list)
    dm = MultiRPN(dm_list)

    tsurf_mean = np.mean([
        field for field in pm.get_all_time_records_for_name_and_level(
            varname="J8").values()
    ],
                         axis=0)
    tair_mean = np.mean([
        field for field in dm.get_all_time_records_for_name_and_level(
            varname="TT", level=1, level_kind=level_kinds.HYBRID).values()
    ],
                        axis=0)

    lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()

    projparams = pm.linked_robj_list[
        0].get_proj_parameters_for_the_last_read_rec()

    rll = RotatedLatLon(**projparams)
    bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)

    xx, yy = bmp(lons, lats)

    plt.figure()
    cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
    bmp.drawcoastlines()
    plt.title("Tsurf")
    plt.colorbar()

    plt.figure()
    bmp.contourf(xx,
                 yy,
                 tair_mean,
                 levels=cs.levels,
                 norm=cs.norm,
                 cmap=cs.cmap)
    bmp.drawcoastlines()
    plt.title("Tair")
    plt.colorbar()

    plt.figure()
    bmp.contourf(xx,
                 yy,
                 tsurf_mean - 273.15 - tair_mean,
                 levels=np.arange(-2, 2.2, 0.2),
                 cmap=cs.cmap)
    bmp.drawcoastlines()
    plt.title("Tsurf - Tair")
    plt.colorbar()

    pm.close()
    dm.close()

    plt.show()
Esempio n. 18
0
def main():
    #path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
    path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"

    months = [6, 7, 8]

    pm_list = []
    dm_list = []
    for m in months:
        print(path.format(m))

        month_folder = path.format(m)
        for fn in os.listdir(month_folder):

            # if not fn.endswith("moyenne"):
            #    continue

            if fn.startswith("pm"):
                pm_list.append(os.path.join(month_folder, fn))
            elif fn.startswith("dm"):
                dm_list.append(os.path.join(month_folder, fn))



    pm = MultiRPN(pm_list)
    dm = MultiRPN(dm_list)

    tsurf_mean = np.mean([field for field in pm.get_all_time_records_for_name_and_level(varname="J8").values()], axis=0)
    tair_mean = np.mean([field for field in dm.get_all_time_records_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID).values()], axis=0)


    lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()

    projparams = pm.linked_robj_list[0].get_proj_parameters_for_the_last_read_rec()

    rll = RotatedLatLon(**projparams)
    bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)

    xx, yy = bmp(lons, lats)


    plt.figure()
    cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
    bmp.drawcoastlines()
    plt.title("Tsurf")
    plt.colorbar()

    plt.figure()
    bmp.contourf(xx, yy, tair_mean, levels=cs.levels, norm=cs.norm, cmap=cs.cmap)
    bmp.drawcoastlines()
    plt.title("Tair")
    plt.colorbar()

    plt.figure()
    bmp.contourf(xx, yy, tsurf_mean - 273.15 - tair_mean, levels=np.arange(-2, 2.2, 0.2), cmap=cs.cmap)
    bmp.drawcoastlines()
    plt.title("Tsurf - Tair")
    plt.colorbar()



    pm.close()
    dm.close()

    plt.show()
Esempio n. 19
0
def main(plot_vals=False):
    varname = "RFAC"
    multiplier = 24 * 3600
    data_folder = Path("/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations")
    day_range = range(19, 22)
    dates_of_interest = [datetime(2013, 6, d) for d in day_range]

    img_folder = Path("calgary_flood/2D")

    if not img_folder.is_dir():
        img_folder.mkdir(parents=True)

    lons, lats, bmp = None, None, None

    the_mask = get_bow_river_basin_mask()

    i_list, j_list = np.where(the_mask > 0.5)
    imin, imax = i_list.min() - 2, i_list.max() + 5
    jmin, jmax = j_list.min() - 2, j_list.max() + 5

    # Calculate daily means
    sim_label_to_date_to_mean = OrderedDict()
    for sim_dir in data_folder.iterdir():
        mr = MultiRPN(str(sim_dir.joinpath("pm*")))
        print(str(sim_dir))
        print(mr.get_number_of_records())

        label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace("frozen", "Frozen").replace("Bow", "")
        sim_label_to_date_to_mean[label] = OrderedDict()

        data = mr.get_4d_field(varname)
        data = {d: list(v.items())[0][1] for d, v in data.items() if d.day in day_range}

        for d in dates_of_interest:
            sim_label_to_date_to_mean[label][d] = (
                np.array([field for d1, field in data.items() if d1.day == d.day]).mean(axis=0) * multiplier
            )

        if lons is None:
            lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
            for f in sim_dir.iterdir():
                if f.name.startswith("pm"):
                    r = RPN(str(f))
                    r.get_first_record_for_name(varname=varname)
                    rll = RotatedLatLon(**r.get_proj_parameters_for_the_last_read_rec())
                    bmp = rll.get_basemap_object_for_lons_lats(
                        lons2d=lons[imin:imax, jmin:jmax], lats2d=lats[imin:imax, jmin:jmax], resolution="i"
                    )
                    r.close()
                    break

        mr.close()

    # reorder simulations
    sim_label_to_date_to_mean = OrderedDict(
        [
            (k, sim_label_to_date_to_mean[k])
            for k in sorted(sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
        ]
    )

    key_list = [k for k in sim_label_to_date_to_mean]
    key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
    sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k]) for k in key_list])

    # do the plots (subplots: vertically - simulations, horizontally - days)
    plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
    fig = plt.figure()
    nrows = len(sim_label_to_date_to_mean)
    ncols = len(day_range)
    gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)

    clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]

    base_label = None
    clevs_diff = [1, 5, 10, 20, 50, 100]
    clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
    cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
    cmap_vals = get_cmap_from_ncl_spec_file(path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
    xx, yy = bmp(lons, lats)
    title = "Total runoff ({}, mm/day)".format(varname)
    fig.suptitle(title)

    for row, (sim_label, date_to_field) in enumerate(sim_label_to_date_to_mean.items()):

        if row == 0 or plot_vals:
            base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
            base_label = sim_label
            plot_label = sim_label
            clevs = clevs_vals
            cmap = cmap_vals
            extend = "max"
        else:
            base_sim = list(sim_label_to_date_to_mean.items())[0][1]
            plot_label = "{}\n-\n{}".format(sim_label, base_label)
            clevs = clevs_diff
            cmap = cmap_diff
            extend = "both"

        bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)

        for col, (the_date, field) in enumerate(date_to_field.items()):

            ax = fig.add_subplot(gs[row, col])
            to_plot = np.ma.masked_where(the_mask < 0.5, field - base_sim[the_date])
            # cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
            cs = bmp.pcolormesh(xx, yy, to_plot[:-1, :-1], norm=bn, vmin=clevs[0], vmax=clevs[-1], cmap=cmap)

            bmp.drawcoastlines(ax=ax, linewidth=0.3)
            assert isinstance(bmp, Basemap)
            bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
            cb = bmp.colorbar(cs, ax=ax, ticks=clevs, extend=extend, pad="4%", size="10%")

            if plot_vals:
                cb.ax.set_visible(col == ncols - 1 and row == 0)
            else:
                cb.ax.set_visible(col == ncols - 1 and row in (0, 1))

            if col == 0:
                ax.set_ylabel(plot_label)

            if row == 0:
                ax.set_title(the_date.strftime("%b %d"))

    if plot_vals:
        img_file = img_folder.joinpath("{}.png".format(varname))
    else:
        img_file = img_folder.joinpath("{}_diff.png".format(varname))

    fig.savefig(str(img_file))
    plt.close(fig)