Exemplo n.º 1
0
def plot_cv_for_return_levels_current_and_future_and_change():
    low_periods = [2, 5]
    high_periods = [10, 30]


    extreme_types = ['high', 'low']
    extreme_type_to_periods = dict(list(zip(extreme_types, [high_periods, low_periods])))


    i_indices, j_indices = data_select.get_indices_from_file()
    current_cvs = {}
    future_cvs = {}
    for extreme_type in extreme_types:
        current_cvs[extreme_type] = {} #to save return level fields for a given extreme type
        future_cvs[extreme_type] = {}

        for return_period in extreme_type_to_periods[extreme_type]:
            c_rl_fields = np.zeros((len(members.current_ids), len(i_indices)))
            f_rl_fields = np.zeros((len(members.current_ids), len(i_indices)))
            for member_num, the_member in enumerate(members.current_ids):
                pars_current = csfb.get_pars_for_member_and_type(the_member, level_type = extreme_type)
                pars_future = csfb.get_pars_for_member_and_type(members.current2future[the_member], level_type = extreme_type)

                #calculate changes for each pair of members and each position
                npos = len(pars_current)
                for pos, pc, pf in zip(range(npos), pars_current, pars_future):
                    if extreme_type == 'high':
                        c = gevfit.get_high_ret_level_stationary(pc, return_period)
                        f = gevfit.get_high_ret_level_stationary(pf, return_period)
                    else:
                        c = gevfit.get_low_ret_level_stationary(pc, return_period)
                        f = gevfit.get_low_ret_level_stationary(pf, return_period)

                    c_rl_fields[member_num, pos] = c
                    f_rl_fields[member_num, pos] = f

                    pass


            current_cvs[extreme_type][return_period] = np.std( c_rl_fields, axis = 0 ) / np.mean( c_rl_fields, axis = 0 )
            future_cvs[extreme_type][return_period] = np.std( f_rl_fields, axis = 0 ) / np.mean( f_rl_fields, axis = 0 )


    #do plotting
    _plot_map_as_subplots(i_indices, j_indices, current_cvs, title = " Current Climate, CV ")
    plt.savefig('cv_for_return_levels_current.png')
    _plot_map_as_subplots(i_indices, j_indices, future_cvs, title = "Future Climate, CV")
    plt.savefig('cv_for_return_levels_future.png')
    pass
Exemplo n.º 2
0
def main():
    plot_utils.apply_plot_params(width_pt=None, font_size=9, aspect_ratio=2.5)
    gs = mpl.gridspec.GridSpec(3,2)
    key_data_list = get_key_data_list()
    i_list, j_list = data_select.get_indices_from_file()
    subplot_count = 0

    for the_type in ["high", "low"]:
        for time_window in ["current", "future"]:
            selected_data = None

            for data in key_data_list:
                if data.time_window == time_window and data.type == the_type:
                    selected_data = data
                    break

            row = subplot_count // 2
            col = subplot_count % 2
            plt.subplot(gs[row, col])
            csfb.plot(selected_data.p_values, i_list, j_list,
                      polar_stereographic.xs, polar_stereographic.ys,
                      units = "", basemap = polar_stereographic.basemap,
                      minmax = (0, 0.25), title = "", # "{0} climate, {1} flow".format(selected_data.time_window, selected_data.type),
                      colorbar_label_format="%.2f", color_map = mpl.cm.get_cmap("jet", 5), upper_limited=True
            )
            subplot_count += 1

    #TODO:add 2 subplots for mean values
    pc = kw_test_for_means()
    plt.subplot(gs[2,0])
    csfb.plot(pc, i_list, j_list,
              polar_stereographic.xs, polar_stereographic.ys,
              units = "", basemap = polar_stereographic.basemap,
              minmax = (0, 0.25), #title = "{0} climate, {1} flow".format("current", "mean"),
              colorbar_label_format="%.2f", color_map = mpl.cm.get_cmap("jet", 5), upper_limited=True
    )

    pf = kw_test_for_means(current_climate=False)
    plt.subplot(gs[2, 1])
    csfb.plot(pf, i_list, j_list,
              polar_stereographic.xs, polar_stereographic.ys,
              units = "", basemap = polar_stereographic.basemap,
              minmax = (0, 0.25), #title = "{0} climate, {1} flow".format("future", "mean"),
              colorbar_label_format="%.2f", color_map = mpl.cm.get_cmap("jet", 5), upper_limited=True
    )


    plt.tight_layout()
    plt.savefig("p_values_kruskalwallis.png")
Exemplo n.º 3
0
def plot_high_flows(period = 10, 
                    imagefile = 'figure.png',
                    pars_set = None,
                    indices_file = 'data/streamflows/hydrosheds_euler9/aex_discharge_1970_01_01_00_00.nc'):
    print('generating %s ' % imagefile)

    plt.clf()
    levs = []
    i_list, j_list = data_select.get_indices_from_file(indices_file)

    #iterate through all grid points
    for pars in pars_set:
        lev = get_high_ret_level_stationary(pars, period)
        if lev < 0:
            print('period = ', period)
            print('pars = ', pars)
            assert False, 'in plot_high_flows'

        assert lev >= 0, pars
        levs.append( lev )
    
    to_plot = np.ma.masked_all(xs.shape)
    for lev, i, j in zip(levs, i_list, j_list):
        assert np.isfinite(lev)
        if isinf(lev):
            print(lev)
        to_plot[i,j] = lev


    nticks = 15
    color_map = mpl.cm.get_cmap('RdBu',nticks)
    int_ticker = LinearLocator(numticks = color_map.N + 1)

    m.pcolormesh(xs, ys, to_plot.copy(), cmap = color_map)
    plt.title('high flow, return period is {0}'.format(period))

    boundaries.plot_basin_boundaries_from_shape(m, plt,  linewidth = 0.5)
    m.drawcoastlines()
#    plot_directions(data_mask = to_plot)
    plt.colorbar( ticks = int_ticker, format = "%.1f" )

    zoom_to_qc()

    plt.savefig(imagefile, bbox_inches = 'tight')
Exemplo n.º 4
0
def plot_data(data = None, imagefile = 'image.png',
              units = 'm**3/s',
              minmax = (None, None),
              indices_file = 'data/streamflows/hydrosheds_euler9/aex_discharge_1970_01_01_00_00.nc',
              color_map = mpl.cm.get_cmap('RdBu', 20),
              ticks_locator = LinearLocator(),
              i_list = None, j_list = None, title = ''):

    if imagefile is not None:
        plt.clf()

    if i_list is None or j_list is None:
        i_list, j_list = data_select.get_indices_from_file(indices_file)
    to_plot = np.ma.masked_all(xs.shape)
    for i, j, value in zip(i_list, j_list, data):
        to_plot[i, j] = value

    plt.title(title)
    m.pcolormesh(xs,ys,to_plot.copy(),  cmap = color_map, edgecolors = 'None',
                    antialiased = True, vmin = minmax[0], vmax = minmax[1])
    m.drawcoastlines()
    boundaries.plot_basin_boundaries_from_shape(m, plotter = plt, linewidth = 0.5)
    draw_meridians_and_parallels(m, 10)

#    plot_directions(data_mask = to_plot)

    cb = plt.colorbar(ticks = ticks_locator, format = "%.1f")
    cb.ax.set_ylabel(units)

    x1, x2, y1, y2 = plot_utils.get_ranges(xs[i_list, j_list], ys[i_list, j_list])
    plt.xlim(x1, x2)
    plt.ylim(y1, y2)

    
    if imagefile is not None:
        plt.savefig(imagefile, bbox_inches = 'tight')
Exemplo n.º 5
0
def plot_signal_to_noise_ratio():
    low_periods = [2, 5]
    high_periods = [10, 30]
    plot_utils.apply_plot_params(width_pt=None, font_size=9, aspect_ratio=2.5)

    extreme_types = ['high', 'low']
    extreme_type_to_periods = dict(list(zip(extreme_types, [high_periods, low_periods])))

    i_indices, j_indices = data_select.get_indices_from_file()
    i_subplot = 0

    #plt.subplots_adjust(wspace = 0.1)
    cMap = mpl.cm.get_cmap('jet', 3)
    cMap.set_over(color = '#FF8C00')

    gs = gridspec.GridSpec(3,2)

    for extreme_type in extreme_types:
        for return_period in extreme_type_to_periods[extreme_type]:

            changes = np.zeros((len(members.current_ids), len(i_indices)))
            for member_num, the_member in enumerate(members.current_ids):

                pars_current = csfb.get_pars_for_member_and_type(the_member, level_type = extreme_type)
                pars_future = csfb.get_pars_for_member_and_type(members.current2future[the_member], level_type = extreme_type)

                
                #calculate changes for each pair of members and each position
                npos = len(pars_current)
                for pos, pc, pf in zip(range(npos), pars_current, pars_future):
                    if extreme_type == 'high':
                        c = gevfit.get_high_ret_level_stationary(pc, return_period)
                        f = gevfit.get_high_ret_level_stationary(pf, return_period)
                    else:
                        c = gevfit.get_low_ret_level_stationary(pc, return_period)
                        f = gevfit.get_low_ret_level_stationary(pf, return_period)

                    changes[member_num, pos] = f - c
                    pass

            #calculate mean and stdev of the obtained changes
            the_mean = np.mean(changes, axis = 0)
            the_std = np.std(changes, axis = 0)

            #change if you want signal to noise ratio, currently it is cv (coefficient of variation 1/(signal-to-noise-ratio))
            the_values = the_std / np.abs(the_mean)
            print(the_values.min(), the_values.max())
            #the_values = the_mean
            to_plot = np.ma.masked_all(csfb.xs.shape)

            max_value = 1.5
            for i, j, value, dev in zip(i_indices, j_indices, the_values, the_std):
                to_plot[i, j] = value


            #shaded = np.ma.masked_where(to_plot != 0, shaded)
            #to_plot = np.ma.masked_where(to_plot == 0, to_plot)


            plot_axes = plt.subplot(gs[i_subplot // 2, i_subplot % 2])
            i_subplot += 1
            print('just before plotting')

            
            basin_boundaries.plot_basin_boundaries_from_shape(csfb.basemap, plotter = plt, linewidth = 1.)
            image = csfb.basemap.pcolormesh(csfb.xs, csfb.ys, to_plot, vmin = 0, vmax = max_value, cmap = cMap,
                                            ax = plot_axes)
            csfb.basemap.drawcoastlines(linewidth = 0.2)

            plot_axes.set_title('T = {0}-year'.format(return_period))
            x_min, x_max, y_min, y_max = plot_utils.get_ranges(csfb.xs[i_indices, j_indices], csfb.ys[i_indices, j_indices])
            plot_axes.set_xlim(x_min, x_max)
            plot_axes.set_ylim(y_min, y_max)




            divider = make_axes_locatable(plot_axes)
            cax = divider.append_axes("right", "8%", pad="3%")


            # extend choices are "both", "min", "max" and "neither"
            cb = plt.colorbar(image, extend = 'max',
                              format = "%.1f", drawedges = True, cax = cax)

            cb.set_ticks(LinearLocator(numticks = 4))
            cb.outline.set_visible(False)



    #plt.show()
    plt.tight_layout()
    plt.savefig('cv_for_changes.png')
Exemplo n.º 6
0
def stationary():

    #data_folder = 'data/streamflows/hydrosheds_euler9'
    data_folder = "data/streamflows/narccap_ccsm-crcm"

    current_data_path_pattern = '%s_discharge_1970_01_01_00_00.nc'
    future_data_path_pattern = '%s_discharge_2041_01_01_00_00.nc'

    i_list, j_list = data_select.get_indices_from_file()

    current_ids = ["ccsm-crcm-current"]
    future_ids = ["ccsm-crcm-future"]
    current2future = dict(list(zip(current_ids, future_ids)))

    current_start_date = datetime(1970, 1, 1, 0, 0)
    current_end_date = datetime(1999, 11, 23,0, 0)

    future_start_date = datetime(2041, 1, 1, 0, 0)
    future_end_date = datetime(2070, 11, 23,0, 0)


    high_return_periods = [10]
    high_start_month = 3
    high_end_month = 7
    high_event_duration = timedelta(days = 1)


    low_return_periods = [2]
    low_start_month = 1
    low_end_month = 5
    low_event_duration = timedelta(days = 15)





    all_return_periods = []
    all_return_periods.extend(high_return_periods)
    all_return_periods.extend(low_return_periods)

    plot_return_levels = False

    extreme_types = ['low', 'high']

    #calculate parameters of the gev distriution for each member
    #calculate and plot return levels
    for extreme_type in extreme_types:
        start_month = low_start_month if extreme_type == 'low' else high_start_month
        end_month = low_end_month if extreme_type == 'low' else high_end_month
        event_duration = low_event_duration if extreme_type == 'low' else high_event_duration

        for current_id in current_ids:
            param_file = 'gev_params_stationary'
            param_file += '_' + current_id + '_' + extreme_type
            data_file = current_data_path_pattern % current_id
            data_file = os.path.join(data_folder, data_file)
            pars_set = optimize_stationary_for_period_and_all_cells(data_file,
                    paramfile = param_file ,
                    high_flow = (extreme_type == 'high'),
                    start_month = start_month,
                    end_month = end_month,
                    start_date = current_start_date,
                    end_date = current_end_date,
                    event_duration = event_duration)

            if plot_return_levels:
                for period in low_return_periods:
                    plot_low_flows(period = period,
                           imagefile = '%drlevel_%s_stationary_%s.png' % (period, extreme_type, current_id),
                           pars_set = pars_set)
                           

        print('Finished optimizing for current climate')

    
        for future_id in future_ids:
            param_file = 'gev_params_stationary'
            param_file += '_' + future_id + '_' + extreme_type
            data_file = future_data_path_pattern % future_id
            data_file = os.path.join(data_folder, data_file)
            pars_set = optimize_stationary_for_period_and_all_cells(data_file,
                    paramfile = param_file ,
                    high_flow = (extreme_type == 'high'),
                    start_month = start_month,
                    end_month = end_month,
                    start_date = future_start_date,
                    end_date = future_end_date,
                    event_duration = event_duration)


            if plot_return_levels:
                for period in low_return_periods:
                    plot_low_flows(period = period, imagefile = '%drlevel_%s_stationary_%s.png' % (period, extreme_type ,future_id),
                           pars_set = pars_set)



    print('Finished optimizing for future climate')
    print('Finished calculating return levels !!!')

    plot_mean_changes = True
    if not plot_mean_changes:
        return

###############################current climate return levels
    print('Calculating mean high flow return levels for current climate ...')
    current_rl_means = {}
    future_rl_means = {}
    the_type_to_periods = {'high': high_return_periods, 'low': low_return_periods}



    keys = []
    for the_type, periods in the_type_to_periods.items():
        for period in periods:
            k = TypePeriodKey()
            k.type = the_type
            k.return_period = period
            keys.append(k)


    for key in keys:
        current_rl_means[key] = []
        future_rl_means[key] = []






    #collect return levels for corresponding type(high , low) and return period for each member and then take mean
    for current_id in current_ids:
        for key in keys:
            future_id = current2future[current_id]
            the_field_current = get_levels_for_type_and_id(current_id, return_period = key.return_period, type = key.type)
            the_field_future = get_levels_for_type_and_id(future_id, return_period = key.return_period, type = key.type)

            assert isinstance(the_field_current, np.ndarray)
            assert isinstance(the_field_future, np.ndarray)

            indices = np.where((the_field_current > 0) & (the_field_future >= 0) )
            to_plot = np.ma.masked_all(the_field_current.shape)
            to_plot[indices] = (the_field_future[indices] - the_field_current[indices]) / the_field_current[indices] * 100.0


            file_name = '{0}-{1}_{2}_{3}yr_change.png'.format(current_id, future_id, key.type, key.return_period)

            delta = np.max(np.abs(to_plot[indices]))
            delta = min(100.0, delta)
            plot_data(data = to_plot, imagefile = file_name,
                  units = '%', minmax = (-125, 125), color_map = my_cm.get_red_blue_colormap(ncolors = 16),
                  title = '{0}-{1}, change, {2}, return period: {3}'.format(current_id,
                  future_id, key.type, key.return_period)
            )


            future_rl_means[key].append(the_field_future)
            current_rl_means[key].append(the_field_current)



            
    for key in keys:

        current_rl_means[key] = np.array(current_rl_means[key])
        current_rl_means[key] = np.ma.masked_where(current_rl_means[key] < 0, current_rl_means[key])

        future_rl_means[key] = np.array(future_rl_means[key])
        future_rl_means[key] = np.ma.masked_where(future_rl_means[key] < 0, future_rl_means[key])


        current_rl_means[key] = np.ma.mean(  current_rl_means[key]  , axis = 0)
        future_rl_means[key] = np.ma.mean(  future_rl_means[key]  , axis = 0)


#        plt.figure()
#        plt.subplot(2,1,1)
#        plt.title('current mean')
#        plot_data(current_rl_means[key], imagefile = None)
#
#        plt.subplot(2,1,2)
#        plt.title('future mean')
#        plot_data(future_rl_means[key], imagefile = None)
#
#        plt.savefig('means_%s_%dyr_rl.png' % (key.type, key.return_period))


###################################################
####Calculate differences between future and current return levels for 10 and 30 year
#### return period.
####

    plot_utils.apply_plot_params(width_pt=None, font_size=9, aspect_ratio=2.5)
    fig = plt.figure()
    assert isinstance(fig, Figure)
    gs = GridSpec(3,2, height_ratios=[1,1,1])
    for i, key in enumerate(keys):
        current = current_rl_means[key]
        future = future_rl_means[key]
        indices = np.where((current > 0) & (future >= 0))
        #to_plot = np.ma.masked_all(current.shape)
        to_plot = (future[indices] - current[indices]) / current[indices] * 100.0

        delta = np.max(np.abs(to_plot))

        min_change = np.min(to_plot)

        if not key.type == "high":
            delta = 100
            lower_limit = 0 if min_change >= 0 else np.floor(min_change / 10.0) * 10
        else:
            delta = 50
            lower_limit = np.floor(min_change / 10.0 ) * 10

        fig.add_subplot(gs[i // 2, i % 2])
        csfb.plot(to_plot,  i_list, j_list, xs, ys,
              imagefile = None, #'%s_%dyr_mean_change.png' % (key.type, key.return_period),
              units = '%', minmax = (-delta, delta),
              color_map = my_cm.get_red_blue_colormap(ncolors = 20), #mpl.cm.get_cmap('RdYlBu',20),
              title = '{0}-year {1} flow'.format( key.return_period, key.type),
              colorbar_tick_locator = LinearLocator(numticks = 11), upper_limited=True,
              impose_lower_limit= lower_limit, basemap=polar_stereographic.basemap
              )

    #gs.update()
    plt.tight_layout(h_pad = 2)
    fig.savefig("ccsm-crcm-rl-changes.png")
Exemplo n.º 7
0
def plot_results():

    cc = "current"
    fc = "future"
    climate_types = [cc, fc]


    folder_path = 'data/streamflows/hydrosheds_euler9/'
    file_path = os.path.join(folder_path, "aex_discharge_1970_01_01_00_00.nc")
    i_indices, j_indices = data_select.get_indices_from_file(file_path)
    xs = polar_stereographic.xs
    ys = polar_stereographic.ys
    basemap = polar_stereographic.basemap
    
    hi = "high"
    lo = "low"
    extreme_types = [hi, lo]

    base_std_name = "{0}_std_dev_{1}"
    base_par_name = {
        cc : "_".join(["{0}"] + members.current_ids),
        fc : "_".join(["{0}"] + members.future_ids)
    }

    extreme_to_return_periods = {
        hi : get_high_return_periods(),
        lo : get_low_return_periods()
    }


    sig_coefs = [1.96
    #    , 1.645
    ]
    sig_levels = ["95 %"
    #    , "90 %"
    ]
    gs = gridspec.GridSpec(3,2)

    for extreme in extreme_types:
        pars_path_current = base_par_name[cc].format(extreme)
        std_path_current = base_std_name.format(extreme, cc)

        pars_path_future = base_par_name[fc].format(extreme)
        std_path_future = base_std_name.format(extreme, fc)

        pars_current = pickle.load(open(pars_path_current))
        stds_current = pickle.load(open(std_path_current))

        pars_future = pickle.load(open(pars_path_future))
        stds_future = pickle.load(open(std_path_future))

        return_periods = extreme_to_return_periods[extreme]
        #plot_utils.apply_plot_params(font_size=15, width_pt=900, aspect_ratio=2.5)
        #plt.figure()


        delta = 50 if extreme == hi else 100
        for row, ret_period in enumerate( return_periods ):
            #calculate changes in return levels
            func = lambda x: gevfit.get_high_ret_level_stationary(x, ret_period)
            rl_c = list(map(func, pars_current))
            rl_f = list(map(func, pars_future))
            
            rl_c = np.array(rl_c)
            rl_f = np.array(rl_f)

            std_c = stds_current[ret_period]
            std_f = stds_future[ret_period]

            in_odf = (std_c > 0) & (std_f > 0) & (rl_c > 0) & (rl_f >= 0)
            change = np.ma.masked_all(rl_c.shape)
            change[in_odf] = (rl_f[in_odf] - rl_c[in_odf]) / rl_c[in_odf] * 100.0


            min_change = np.min((rl_f - rl_c) / rl_c * 100.0)
            if min_change >= 0:
               low_limit = 0
            elif min_change > -10:
               low_limit = -10
            else:
                low_limit = np.floor(min_change / 10.0) * 10

            print("min change = {0}, low limit = {1}".format(min_change, low_limit))


            for sig_coef, sig_name in zip(sig_coefs, sig_levels):
                significance = np.ma.masked_all(rl_c.shape)
                sig_cond = (sig_coef * (std_c + std_f) < np.abs(rl_f - rl_c)) & in_odf
                significance[~sig_cond] = 0#fill with gray non signifcant areas
                change1 = np.ma.masked_where(~sig_cond, change)
                if extreme == hi:
                    plt.subplot(gs[row, 0])
                else:
                    plt.subplot(gs[row, 1])

                csfb.plot(change1 , i_indices, j_indices, xs, ys,
                        title = 'T = {0}-year'.format( ret_period ),
                        color_map = mycolors.get_red_blue_colormap(ncolors = 10),
                        units = '%',
                        basemap = basemap, minmax = (-delta, delta),
                        colorbar_label_format = '%d',
                        upper_limited = True,
                        colorbar_tick_locator = LinearLocator(numticks = 11),
                        not_significant_mask = significance,
                        show_colorbar = True, impose_lower_limit=low_limit
                        )
                #subplot_count += 1
    plt.tight_layout()
    plt.savefig("rl_of_merged_change.png")

    pass
def plot_naveed_data(path = 'data/data_txt_naveed/3_all_for_plotting.csv'):
    """
    plotting data calculated by Naveed
    """
    f = open(path)

    lines = f.readlines()
    f.close()

    #skip header
    while len(lines) > 0:
        line = lines.pop(0)
        if line.startswith('stationSrI'):
            break

    

    folder_path = 'data/streamflows/hydrosheds_euler9/'
    i_indices, j_indices = data_select.get_indices_from_file(folder_path + 'aex_discharge_1970_01_01_00_00.nc')


    return_periods = [2,10,30]
    level_cols = [4,5,6]
    sig95_cols = [9,10,11]
    sig90_cols = [14, 15, 16]

    sets = list(zip(level_cols, return_periods, sig95_cols, sig90_cols))

    subplot_count = 1
    for lev_col, period, sig95_col, sig90_col in sets:
        for sig_col in [sig95_col, sig90_col]:
            ret_lev = np.array(get_column(lev_col, lines))
            plt.subplot(3,2,subplot_count)
            significance = 1 - np.array(get_column(sig_col, lines))
            ret_lev = np.ma.masked_where(significance == 1, ret_lev)

            significance *= 0.75
            significance = np.ma.masked_where(significance == 0, significance)

            delta = 50
            sig_level = '95%' if sig_col == sig95_col else '90%'
            plot(ret_lev , i_indices, j_indices, xs, ys,
                        title = 'T = %d year, conf. (%s)' % (period, sig_level),
                        color_map = mycolors.get_red_blue_colormap(ncolors = 16), units = '%',
                        basemap = basemap, minmax = (-delta, delta),
                        colorbar_label_format = '%d',
                        upper_limited = True, colorbar_tick_locator = LinearLocator(numticks = 9),
                        not_significant_mask = significance, show_colorbar = (subplot_count == 6)
                        )
            subplot_count += 1
    plt.savefig('3_all_for_plotting.pdf', bbox_inches = 'tight')
    
    return

    plt.figure()
    b = Basemap(resolution = 'i')
    plt.subplot(2,1,1)
    medians = get_column(1, lines)
    to_plot = np.ma.masked_all(polar_stereographic.lons.shape)
    for i, j, med in zip(i_indices, j_indices, medians):
        to_plot[i,j] = med

    b.pcolormesh(polar_stereographic.lons, polar_stereographic.lats, to_plot)
    b.drawcoastlines()
    plt.colorbar(shrink = 0.5)

    cond = ~to_plot.mask
    min_lon = polar_stereographic.lons[cond].min()
    max_lon = polar_stereographic.lons[cond].max()

    min_lat = polar_stereographic.lats[cond].min()
    max_lat = polar_stereographic.lats[cond].max()

    marginx = 0.05 * (max_lon - min_lon)
    marginy = 0.05 * (max_lat - min_lat)

    plt.xlim(min_lon - marginx, max_lon + marginx)
    plt.ylim(min_lat - marginy, max_lat + marginy)
    plt.title('median change (%)')


    plt.subplot(2,1,2)
    pvalues = get_column(2, lines)
    to_plot = np.ma.masked_all(polar_stereographic.lons.shape)
    for i, j, pvalue in zip(i_indices, j_indices, pvalues):
        to_plot[i,j] = pvalue


    b.pcolormesh(polar_stereographic.lons, polar_stereographic.lats, to_plot)
    b.drawcoastlines()
    plt.colorbar(shrink = 0.5)
    plt.title('p-value for median change')
    marginx = 0.05 * (max_lon - min_lon)
    marginy = 0.05 * (max_lat - min_lat)
    plt.xlim(min_lon - marginx, max_lon + marginx)
    plt.ylim(min_lat - marginy, max_lat + marginy)
    plt.savefig('median.pdf', bbox_inches = 'tight')


    pass
def plot_naveed_troubled_points(path = 'data/data_txt_naveed/TroubledGridCells_AtLeast3Sims.csv'):
    f = open(path)

    lines = f.readlines()
    if len(lines) == 1:
        lines = lines[0].split('\r')
    f.close()

    #skip header
    while len(lines) > 0:
        line = lines.pop(0)
        if 'stationsri' in line.lower():
            break

    print(lines)

    folder_path = 'data/streamflows/hydrosheds_euler9/'
    i_indices, j_indices = data_select.get_indices_from_file(folder_path + 'aex_discharge_1970_01_01_00_00.nc')


    cols = [1,2,3]
    names = ['C', 'F', 'Both']


    color_map = ListedColormap(['r','b', 'g'])
    #plt.subplots_adjust(hspace = 0.0001)
    for c, name in zip(cols, names):
        plt.subplot(2,2,c)
        to_plot = np.ma.masked_all(xs.shape)
        data = get_column(c, lines)
        print(data)
        for i, j, v in zip(i_indices, j_indices, data):
            if v: #no color for 0
                to_plot[i, j] = v
            else:
                to_plot[i, j] = 3

        basemap.pcolormesh(xs, ys, to_plot.copy(), cmap = color_map,
                          shading = 'flat', rasterized = False )


        plot_basin_boundaries_from_shape(basemap, plotter = plt, linewidth = 1, edge_color = 'k')
        basemap.drawcoastlines(linewidth = 0.5)

        plot_utils.draw_meridians_and_parallels(basemap, step_degrees = 30)

        plt.title(name)

        ymin, ymax = plt.ylim()
        plt.ylim(ymin + 0.05 * (ymax - ymin) , ymax * 0.25)

        xmin, xmax = plt.xlim()
        plt.xlim(xmin + (xmax - xmin) * 0.55, 0.72*xmax)

    cb = plt.colorbar(shrink = 0.5, format = '%d')
    cb.set_ticks([1.25, 2, 2.75])
    cb.set_ticklabels([1,2,3])
    

    plt.savefig('TroubledGridCells_AtLeast3Sims.pdf', bbox_inches = 'tight')

    pass
def calculate_and_plot(return_period = 10,
                       return_level_function = ret_level_getters[0], ax = None):

    save_fig_to_file = (ax is None)
    if return_level_function == gevfit.get_high_ret_level_stationary:
        level_type = 'high'
    else:
        level_type = 'low'

    fig = plt.figure()
    assert isinstance(fig, Figure)


    save_to_txt = False
    current_ids = ["ccsm-crcm-current"] #members.current_ids
    future_ids = ["ccsm-crcm-future"] #members.future_ids
    current2future = dict(list(zip(current_ids, future_ids)))

    #folder_path = 'data/streamflows/hydrosheds_euler9/'
    folder_path = "data/streamflows/narccap_ccsm-crcm"
    coord_file = os.path.join(folder_path, '{0}_discharge_1970_01_01_00_00.nc'.format(current_ids[0]))
    i_indices, j_indices = data_select.get_indices_from_file(coord_file)
    significance_counter = None
    #plt.subplots_adjust(left = 0., hspace = 0.2, wspace = 0.2)



    labels = ["", "(b)", "(c)", "(d)", "(e)", "(f)"]

    ##for querying high flow data for saving to text file
    current_query = None
    future_query = None
    current_highs = None
    future_highs = None
    if level_type == 'high' and save_to_txt:
        high_period_start_month = 3
        high_period_end_month = 7

        current_start_date = datetime(1970,1,1,0,0)
        current_end_date = datetime(1999,12,31,0,0)

        future_start_date = datetime(2041,1,1,0,0)
        future_end_date = datetime(2070,12,31,0,0)

        future_query = QueryObject()
        future_query.start_date = future_start_date
        future_query.end_date = future_end_date
        future_query.event_duration = timedelta(days = 1)
        future_query.start_month = high_period_start_month
        future_query.end_month = high_period_end_month

        current_query = QueryObject()
        current_query.start_date = current_start_date
        current_query.end_date = current_end_date
        current_query.event_duration = timedelta(days = 1)
        current_query.start_month = high_period_start_month
        current_query.end_month = high_period_end_month


    gs = gridspec.GridSpec(3,2)
    current_id_to_changes = {}
    all_current = []
    all_future = []
    all_stds_current = []
    all_stds_future = []
    for k, current_id in enumerate(current_ids):
        if level_type == 'high' and save_to_txt:
            current_path = folder_path + '{0}_discharge_1970_01_01_00_00.nc'.format(current_id)
            future_path = folder_path + '{0}_discharge_2041_01_01_00_00.nc'.format(current2future[current_id])
            current_data, times_current, x_indices, y_indices = data_select.get_data_from_file(current_path)
            future_data, times_future, x_indices, y_indices = data_select.get_data_from_file(future_path)

            current_highs = data_select.get_period_maxima_query(current_data, times_current, current_query)
            future_highs = data_select.get_period_maxima_query(future_data, times_future, future_query)

        #get current return levels
        pars_list = get_pars_for_member_and_type(current_id, level_type)
        return_levels_current = np.zeros(len(pars_list))
        for pos, pars in enumerate(pars_list):
            return_levels_current[pos] = return_level_function(pars, return_period)
        stdevs_current = get_stdevs_for_member_and_type(current_id, level_type)[return_period]


        #get future return levels
        future_id = current2future[current_id]
        pars_list = get_pars_for_member_and_type(future_id, level_type)
        return_levels_future = np.zeros(len(pars_list))
        for pos, pars in enumerate(pars_list):
            return_levels_future[pos] = return_level_function(pars, return_period)
        stdevs_future = get_stdevs_for_member_and_type(future_id, level_type)[return_period]


        change = return_levels_future - return_levels_current
        if significance_counter is None:
            significance_counter = np.zeros( change.shape )


        print('minmax(std_current)')
        print(np.min(stdevs_current), np.max(stdevs_current))
        print('minmax(std_future)')
        print(np.min(stdevs_future), np.max(stdevs_future))

        print('min max min abs(rl_current - rl_future)')
        the_delta = np.abs(return_levels_future - return_levels_current)
        print(np.min(the_delta), np.max(the_delta), np.mean(the_delta))
        
        #stdev = -1 - stands for undefined value
        condition = np.logical_and(np.abs(change) > 1.96 * ( stdevs_current + stdevs_future ),
                                   (stdevs_current >= 0) & (stdevs_future >= 0)
                                   & (return_levels_current > 0)
                                )

        

       
        sign_index = np.where(condition)
        significance_counter[sign_index] += 1


        print(len(sign_index[0]))

        all_current.append(return_levels_current)
        all_future.append(return_levels_future)

        all_stds_current.append(stdevs_current)
        all_stds_future.append(stdevs_future)

        change /= return_levels_current
        change *= 100.0


        min_change = np.min(change)
        print(return_levels_current[change == min_change], return_levels_future[change == min_change], min_change)
        
        if not level_type == "high":
            delta = 100
            lower_limit = 0 if min_change >= 0 else np.floor(min_change / 10.0) * 10
        else:
            delta = 50
            lower_limit = np.floor(min_change / 10.0 ) * 10


        not_significant = np.zeros(change.shape)
        not_significant = np.ma.masked_where(condition, not_significant)


        if level_type == 'high':
            assert np.all(return_levels_current > 0)
            assert np.all(stdevs_current >= 0)
            assert np.all(stdevs_future >= 0)

        #temp change to condition
        #change = np.ma.masked_where(np.logical_not(condition), change)
        print('Plotting: current %s, future %s' % (current_id, future_id))

        current_id_to_changes[current_id] = change
        if ax is None:
            ax = fig.add_subplot(gs[k // 2, k % 2])
        plot(change , i_indices, j_indices, xs, ys,
                    title = "{0}-year {1} flow".format(return_period, level_type), label = labels[k],
                    color_map = mycolors.get_red_blue_colormap(ncolors = 20), units = '%',
                    basemap = basemap, minmax = (-delta, delta),
                    colorbar_label_format = '%d',
                    upper_limited = True, colorbar_tick_locator = LinearLocator(numticks = 11),
                    not_significant_mask = not_significant
                    , impose_lower_limit=lower_limit, ax= ax

                    )
        if return_period == 10 and level_type == 'high' and save_to_txt:
            txt_saver.save_to_file_rls_and_sign(current_id, return_period,
                                      return_levels_current, return_levels_future,
                                      stdevs_current, stdevs_future,
                                      condition, current_highs, future_highs)
    




    plt.subplot(gs[2,1])

    plot_sign_count = False
    plot_significance = True
    if plot_sign_count: #if plotting significance count
        significance_counter = np.ma.masked_where(significance_counter == 0, significance_counter)
        plot(significance_counter, i_indices, j_indices, xs, ys,
             title = 'Significance Count', label = labels[5], minmax = (1,6),
             color_map = mycolors.get_sign_count_cmap(ncolors = 5), basemap = basemap,
             colorbar_tick_locator = MaxNLocator(nbins = 5),
             colorbar_label_format = '%d'
             )

        #TODO plot +/-
        plus_change = None
        minus_change = None

        for current_id, the_change in current_id_to_changes.items():
            if plus_change is None:
                plus_change = (the_change > 0)
                minus_change = (the_change < 0)
            else:
                plus_change = np.logical_and(the_change > 0, plus_change)
                minus_change = np.logical_and(the_change < 0, minus_change)

        #should be at least one member with significant changes
        plus_change = np.logical_and(plus_change, significance_counter > 0)
        minus_change = np.logical_and(minus_change, significance_counter > 0)

        x_interest = xs[i_indices, j_indices]
        y_interest = ys[i_indices, j_indices]


        x_plus = x_interest[plus_change]
        y_plus = y_interest[plus_change]
        x_minus = x_interest[minus_change]
        y_minus = y_interest[minus_change]

        basemap.scatter(x_plus, y_plus, marker = "+", color = "m", s = 15, zorder = 5, linewidth = 1)

        if len(x_minus) > 0:
            basemap.scatter(x_minus, y_minus, marker = "d", zorder = 6)
    else:
        #plot ensemble mean
        all_current = np.array( all_current )
        all_future = np.array( all_future )
        all_stds_current = np.array( all_stds_current )
        all_stds_future = np.array( all_stds_future )


        mean_current = np.mean(all_current, axis = 0)
        mean_future = np.mean(all_future, axis = 0)
        mean_stds_current = np.mean( all_stds_current, axis = 0 )
        mean_stds_future = np.mean( all_stds_future, axis = 0 )

        min_change = np.min((mean_future - mean_current)/mean_current * 100.0)
        if not level_type == "high":
            delta = 100

            lower_limit = 0 if min_change >= 0 else np.floor(min_change / 10.0) * 10
        else:
            delta = 100
            lower_limit = np.floor(min_change / 10.0 ) * 10

        not_significant = np.absolute(mean_future - mean_current) <= 1.96 * (mean_stds_current + mean_stds_future)
        not_significant = not_significant.astype(int)
        print(" sum(not_significant) = ", np.sum(not_significant))
        not_significant = np.ma.masked_where(~(not_significant == 1), not_significant)
        not_significant *= 0.0

        if not plot_significance:
            not_significant = None

        plot((mean_future - mean_current) / mean_current * 100.0, i_indices, j_indices, xs, ys,
                    title = "", label = labels[-1],
                    color_map = mycolors.get_red_blue_colormap(ncolors = 20), units = '%',
                    basemap = basemap, minmax = (-delta, delta),
                    colorbar_label_format = '%d',
                    upper_limited = True, colorbar_tick_locator = LinearLocator(numticks = 11),
                    not_significant_mask = not_significant,
                    impose_lower_limit = lower_limit
                    )



        pass


    if save_fig_to_file:
        plt.tight_layout()
        plt.savefig('%d_%s_change_rl.png' % (return_period, level_type))