示例#1
0
    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset, [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])
        ref_dataset = self.test_dataset
        target_datasets = [self.test_dataset, self.another_test_dataset]
        metrics = [Bias(), Bias()]
        unary_metrics = [TemporalStdDev()]

        self.eval = Evaluation(ref_dataset, target_datasets,
                               metrics + unary_metrics)

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
        self.eval.run()
        out_str = ("<Evaluation - ref_dataset: {}, "
                   "target_dataset(s): {}, "
                   "binary_metric(s): {}, "
                   "unary_metric(s): {}, "
                   "subregion(s): {}>").format(str(
                       self.test_dataset), [str(ds) for ds in target_datasets],
                                               [str(m) for m in metrics],
                                               [str(u)
                                                for u in unary_metrics], None)
        self.assertEqual(str(self.eval), out_str)
示例#2
0
 def test_bias_output_shape(self):
     bias_eval = Evaluation(self.test_dataset, [
                            self.another_test_dataset], [Bias()])
     bias_eval.run()
     input_shape = tuple(self.test_dataset.values.shape)
     bias_results_shape = tuple(bias_eval.results[0][0].shape)
     self.assertEqual(input_shape, bias_results_shape)
示例#3
0
    def test_subregion_unary_result_shape(self):
        bound = Bounds(
                10, 18, 
                100, 108, 
                dt.datetime(2000, 1, 1), dt.datetime(2000, 3, 1))

        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev()],
            [bound]
        )
        new_eval.run()

        # Expected result shape is
        # [
        #   [   
        #       [   # Subregions cause this extra layer
        #           temporalstddev.run(reference),
        #           temporalstddev.run(target1),
        #           temporalstddev.run(target2)
        #       ]
        #   ]
        # ]
        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(type(new_eval.unary_results) == type([]))

        self.assertTrue(len(new_eval.unary_results[0]) == 1)

        self.assertTrue(len(new_eval.unary_results[0][0]) == 3)
示例#4
0
def Taylor_diagram_spatial_pattern_of_multiyear_climatology(
        obs_dataset, obs_name, model_datasets, model_names, file_name):

    # calculate climatological mean fields
    obs_clim_dataset = ds.Dataset(obs_dataset.lats, obs_dataset.lons,
                                  obs_dataset.times,
                                  utils.calc_temporal_mean(obs_dataset))
    model_clim_datasets = []
    for dataset in model_datasets:
        model_clim_datasets.append(
            ds.Dataset(dataset.lats, dataset.lons, dataset.times,
                       utils.calc_temporal_mean(dataset)))

    # Metrics (spatial standard deviation and pattern correlation)
    # determine the metrics
    taylor_diagram = metrics.SpatialPatternTaylorDiagram()

    # create the Evaluation object
    taylor_evaluation = Evaluation(
        obs_clim_dataset,  # Climatological mean of reference dataset for the evaluation
        model_clim_datasets,  # list of climatological means from model datasets for the evaluation
        [taylor_diagram])

    # run the evaluation (bias calculation)
    taylor_evaluation.run()

    taylor_data = taylor_evaluation.results[0]

    plotter.draw_taylor_diagram(
        taylor_data,
        model_names,
        obs_name,
        file_name,
        pos='upper right',
        frameon=False)
示例#5
0
def Taylor_diagram_spatial_pattern_of_multiyear_climatology(
        obs_dataset, obs_name, model_datasets, model_names, file_name):

    # calculate climatological mean fields
    obs_dataset.values = utils.calc_temporal_mean(obs_dataset)
    for dataset in model_datasets:
        dataset.values = utils.calc_temporal_mean(dataset)

    # Metrics (spatial standard deviation and pattern correlation)
    # determine the metrics
    taylor_diagram = metrics.SpatialPatternTaylorDiagram()

    # create the Evaluation object
    taylor_evaluation = Evaluation(
        obs_dataset,  # Reference dataset for the evaluation
        model_datasets,  # list of target datasets for the evaluation
        [taylor_diagram])

    # run the evaluation (bias calculation)
    taylor_evaluation.run()

    taylor_data = taylor_evaluation.results[0]

    plotter.draw_taylor_diagram(taylor_data,
                                model_names,
                                obs_name,
                                file_name,
                                pos='upper right',
                                frameon=False)
示例#6
0
def Taylor_diagram_spatial_pattern_of_multiyear_climatology(
        obs_dataset, obs_name, model_datasets, model_names, file_name):

    # calculate climatological mean fields
    obs_clim_dataset = ds.Dataset(obs_dataset.lats, obs_dataset.lons,
                                  obs_dataset.times,
                                  utils.calc_temporal_mean(obs_dataset))
    model_clim_datasets = []
    for dataset in model_datasets:
        model_clim_datasets.append(
            ds.Dataset(dataset.lats, dataset.lons, dataset.times,
                       utils.calc_temporal_mean(dataset)))

    # Metrics (spatial standard deviation and pattern correlation)
    # determine the metrics
    taylor_diagram = metrics.SpatialPatternTaylorDiagram()

    # create the Evaluation object
    taylor_evaluation = Evaluation(
        obs_clim_dataset,  # Climatological mean of reference dataset for the evaluation
        model_clim_datasets,  # list of climatological means from model datasets for the evaluation
        [taylor_diagram])

    # run the evaluation (bias calculation)
    taylor_evaluation.run()

    taylor_data = taylor_evaluation.results[0]

    plotter.draw_taylor_diagram(taylor_data,
                                model_names,
                                obs_name,
                                file_name,
                                pos='upper right',
                                frameon=False)
示例#7
0
    def test_subregion_unary_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev(), TemporalStdDev()],
            [bound, bound, bound, bound, bound]
        )
        new_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [3, temporalstddev.run(reference).shape],
        #       ]
        # ]

        # 5 = number of subregions
        self.assertTrue(len(new_eval.unary_results) == 5)
        # number of metrics
        self.assertTrue(len(new_eval.unary_results[0]) == 2)
        self.assertTrue(isinstance(new_eval.unary_results, type([])))
        # number of datasets (ref + target)
        self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3)
示例#8
0
def Map_plot_bias_of_multiyear_climatology(obs_dataset, obs_name, model_datasets, model_names,
                                      file_name, row, column):
    '''Draw maps of observed multi-year climatology and biases of models"'''

    # calculate climatology of observation data
    obs_clim = utils.calc_temporal_mean(obs_dataset)
    # determine the metrics
    map_of_bias = metrics.TemporalMeanBias()

    # create the Evaluation object
    bias_evaluation = Evaluation(obs_dataset, # Reference dataset for the evaluation
                                 model_datasets, # list of target datasets for the evaluation
                                 [map_of_bias, map_of_bias])

    # run the evaluation (bias calculation)
    bias_evaluation.run() 

    rcm_bias = bias_evaluation.results[0]

    fig = plt.figure()

    lat_min = obs_dataset.lats.min()
    lat_max = obs_dataset.lats.max()
    lon_min = obs_dataset.lons.min()
    lon_max = obs_dataset.lons.max()

    string_list = list(string.ascii_lowercase) 
    ax = fig.add_subplot(row,column,1)
    m = Basemap(ax=ax, projection ='cyl', llcrnrlat = lat_min, urcrnrlat = lat_max,
            llcrnrlon = lon_min, urcrnrlon = lon_max, resolution = 'l', fix_aspect=False)
    lons, lats = np.meshgrid(obs_dataset.lons, obs_dataset.lats)

    x,y = m(lons, lats)

    m.drawcoastlines(linewidth=1)
    m.drawcountries(linewidth=1)
    m.drawstates(linewidth=0.5, color='w')
    max = m.contourf(x,y,obs_clim,levels = plotter._nice_intervals(obs_dataset.values, 10), extend='both',cmap='PuOr')
    ax.annotate('(a) \n' + obs_name,xy=(lon_min, lat_min))
    cax = fig.add_axes([0.02, 1.-float(1./row), 0.01, 1./row*0.6])
    plt.colorbar(max, cax = cax) 
    clevs = plotter._nice_intervals(rcm_bias, 11)
    for imodel in np.arange(len(model_datasets)):
        ax = fig.add_subplot(row, column,2+imodel)
        m = Basemap(ax=ax, projection ='cyl', llcrnrlat = lat_min, urcrnrlat = lat_max,
                llcrnrlon = lon_min, urcrnrlon = lon_max, resolution = 'l', fix_aspect=False)
        m.drawcoastlines(linewidth=1)
        m.drawcountries(linewidth=1)
        m.drawstates(linewidth=0.5, color='w')
        max = m.contourf(x,y,rcm_bias[imodel,:],levels = clevs, extend='both', cmap='RdBu_r')
        ax.annotate('('+string_list[imodel+1]+')  \n '+model_names[imodel],xy=(lon_min, lat_min))

    cax = fig.add_axes([0.91, 0.1, 0.015, 0.8])
    plt.colorbar(max, cax = cax) 

    plt.subplots_adjust(hspace=0.01,wspace=0.05)

    plt.show()
    fig.savefig(file_name,dpi=600,bbox_inches='tight')
示例#9
0
    def test_result_shape(self):
        bias_eval = Evaluation(self.test_dataset, [
            self.another_test_dataset, self.another_test_dataset,
            self.another_test_dataset
        ], [Bias(), Bias()])
        bias_eval.run()

        # Expected result shape is
        # [bias, bias] where bias.shape[0] = number of datasets
        self.assertTrue(len(bias_eval.results) == 2)
        self.assertTrue(bias_eval.results[0].shape[0] == 3)
示例#10
0
    def test_result_shape(self):
        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset, self.another_test_dataset],
            [Bias(), Bias()]
        )
        bias_eval.run()

        # Expected result shape is
        # [bias, bias] where bias.shape[0] = number of datasets
        self.assertTrue(len(bias_eval.results) == 2)
        self.assertTrue(bias_eval.results[0].shape[0] == 3)
示例#11
0
    def test_unary_result_shape(self):
        new_eval = Evaluation(self.test_dataset, [
            self.another_test_dataset, self.another_test_dataset,
            self.another_test_dataset, self.another_test_dataset
        ], [TemporalStdDev()])
        new_eval.run()

        # Expected result shape is
        # [stddev] where stddev.shape[0] = number of datasets

        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(new_eval.unary_results[0].shape[0] == 5)
示例#12
0
    def setUp(self):
        self.full_evaluation = Evaluation(
            _create_fake_dataset('Ref'),
            [_create_fake_dataset('T1'), _create_fake_dataset('T2')],
            [metrics.TemporalStdDev(), metrics.Bias(), metrics.Bias()]
        )

        self.unary_evaluation = Evaluation(
            None,
            [_create_fake_dataset('T1'), _create_fake_dataset('T2')],
            [metrics.TemporalStdDev()]
        )
示例#13
0
    def test_unary_result_shape(self):
        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset, self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev()]
        )
        new_eval.run()

        # Expected result shape is
        # [stddev] where stddev.shape[0] = number of datasets
        
        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(new_eval.unary_results[0].shape[0] == 5)
示例#14
0
    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value,
                                            self.other_var)
示例#15
0
    def test_unary_result_shape(self):
        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev()]
        )
        new_eval.run()

        # Expected result shape is
        # [
        #   temporalstddev.run(reference),
        #   temporalstddev.run(target1),
        #   temporalstddev.run(target2)
        # ]
        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(len(new_eval.unary_results[0]) == 3)
示例#16
0
    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset, [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
def generate_evaluation_from_config(config_data):
    """ Generate an Evaluation object from configuration data.

    :param config_data: Dictionary of the data parsed from the supplied YAML
        configuration file.
    :type config_data: :func:`dict`

    :returns: An Evaluation object containing the data specified in the
        supplied configuration data.
    """
    # Load datasets
    reference = None
    targets = []
    if config_data['datasets']:
        if 'reference' in config_data['datasets']:
            reference = _load_dataset(config_data['datasets']['reference'])

        if 'targets' in config_data['datasets']:
            targets = [
                _load_dataset(t) for t in config_data['datasets']['targets']
            ]

        reference, targets = _prepare_datasets_for_evaluation(
            reference, targets, config_data)
    # Load metrics
    eval_metrics = []
    if config_data['metrics']:
        eval_metrics = [_load_metric(m)() for m in config_data['metrics']]

    # Load Subregions (if present)
    subregions = None
    if 'subregions' in config_data:
        subregions = [_load_subregion(s) for s in config_data['subregions']]

    return Evaluation(reference, targets, eval_metrics, subregions=subregions)
示例#18
0
    def test_result_shape(self):
        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [Bias()]
        )
        bias_eval.run()

        # Expected result shape is
        # [
        #   [
        #       bias.run(reference, target1)
        #   ],
        #   [
        #       bias.run(reference, target2)
        #   ]
        # ]
        self.assertTrue(len(bias_eval.results) == 2)
        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(len(bias_eval.results[1]) == 1)
示例#19
0
    def test_subregion_result_shape(self):
        bound = Bounds(10, 18, 100, 108, dt.datetime(2000, 1, 1),
                       dt.datetime(2000, 3, 1))

        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset], [Bias()],
            [bound])
        bias_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [number of targets, bias.run(reference, target1).shape]
        #       ]
        #   ],
        self.assertTrue(len(bias_eval.results) == 1)
        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(bias_eval.results[0][0].shape[0] == 2)
        self.assertTrue(isinstance(bias_eval.results, type([])))
    def test_default_data_return(self):
        new_eval = Evaluation(None, [], [])
        default_output = {
            'temporal_time_delta': 999,
            'spatial_regrid_lats': (-90, 90, 1),
            'spatial_regrid_lons': (-180, 180, 1),
            'subset': [-90, 90, -180, 180, "1500-01-01", "2500-01-01"],
        }

        out = writer.generate_evaluation_information(new_eval)

        self.assertEquals(default_output, out)
    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset,
            [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])
        ref_dataset = self.test_dataset
        target_datasets = [self.test_dataset, self.another_test_dataset]
        metrics = [Bias(), Bias()]
        unary_metrics = [TemporalStdDev()]

        self.eval = Evaluation(ref_dataset,
                               target_datasets,
                               metrics + unary_metrics)

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
        self.eval.run()
        out_str = (
            "<Evaluation - ref_dataset: {}, "
            "target_dataset(s): {}, "
            "binary_metric(s): {}, "
            "unary_metric(s): {}, "
            "subregion(s): {}>"
        ).format(
            str(self.test_dataset),
            [str(ds) for ds in target_datasets],
            [str(m) for m in metrics],
            [str(u) for u in unary_metrics],
            None
        )
        self.assertEqual(str(self.eval), out_str)
示例#22
0
def Taylor_diagram_spatial_pattern_of_multiyear_climatology(obs_dataset, obs_name, model_datasets, model_names,
                                      file_name):

    # calculate climatological mean fields
    obs_dataset.values = utils.calc_temporal_mean(obs_dataset)
    for dataset in model_datasets:
        dataset.values = utils.calc_temporal_mean(dataset)

    # Metrics (spatial standard deviation and pattern correlation)
    # determine the metrics
    taylor_diagram = metrics.SpatialPatternTaylorDiagram()

    # create the Evaluation object
    taylor_evaluation = Evaluation(obs_dataset, # Reference dataset for the evaluation
                                 model_datasets, # list of target datasets for the evaluation
                                 [taylor_diagram])

    # run the evaluation (bias calculation)
    taylor_evaluation.run() 

    taylor_data = taylor_evaluation.results[0]

    plotter.draw_taylor_diagram(taylor_data, model_names, obs_name, file_name, pos='upper right',frameon=False)
    def setUpClass(self):
        self.lats = np.array(range(-10, 10, 1))
        self.lons = np.array(range(-20, 20, 1))
        self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(9600))
        self.values = flat_array.reshape(12, 20, 40)

        self.dataset = Dataset(
            self.lats,
            self.lons,
            self.times,
            self.values,
        )

        self.evaluation = Evaluation(self.dataset, [], [])
示例#24
0
    def test_subregion_result_shape(self):
        bound = Bounds(
                10, 18, 
                100, 108, 
                dt.datetime(2000, 1, 1), dt.datetime(2000, 3, 1))

        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [Bias()],
            [bound]
        )
        bias_eval.run()

        # Expected result shape is
        # [
        #   [
        #       [   # Subregions cause this extra layer
        #           bias.run(reference, target1)
        #       ]
        #   ],
        #   [
        #       [
        #           bias.run(reference, target2)
        #       ]
        #   ]
        # ]
        self.assertTrue(len(bias_eval.results) == 2)

        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(type(bias_eval.results[0]) == type([]))
        self.assertTrue(len(bias_eval.results[1]) == 1)
        self.assertTrue(type(bias_eval.results[1]) == type([]))

        self.assertTrue(len(bias_eval.results[0][0]) == 1)
        self.assertTrue(len(bias_eval.results[1][0]) == 1)
    def test_subregion_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [Bias()],
            [bound]
        )
        bias_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [number of targets, bias.run(reference, target1).shape]
        #       ]
        #   ],
        self.assertTrue(len(bias_eval.results) == 1)
        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(bias_eval.results[0][0].shape[0] == 2)
        self.assertTrue(isinstance(bias_eval.results, type([])))
    def test_yearly_temporal_bin(self):
        new_times = np.array(
            [dt.datetime(2000 + x, 1, 1) for x in range(1, 13)])

        dataset = Dataset(
            self.lats,
            self.lons,
            new_times,
            self.values,
        )
        new_eval = Evaluation(dataset, [], [])

        out = writer.generate_evaluation_information(new_eval)

        self.assertEquals(out['temporal_time_delta'], 366)
示例#27
0
    def test_full_init(self):
        self.eval = Evaluation(
                        self.test_dataset,           
                        [self.test_dataset, self.another_test_dataset], 
                        [Bias(), Bias(), TemporalStdDev()])                    

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
    def test_subset_with_multiple_datasets(self):
        new_ds = Dataset(np.arange(0, 20, 1), self.lons, self.times,
                         self.values)
        new_eval = Evaluation(self.dataset, [new_ds], [])

        out = writer.generate_evaluation_information(new_eval)
        subset = out['subset']

        ds_lat_min, ds_lat_max, ds_lon_min, ds_lon_max = self.dataset.spatial_boundaries(
        )
        start, end = self.dataset.temporal_boundaries()

        self.assertEqual(ds_lat_min, subset[0])
        # Check that we actually used the different max lat value that we
        # created by adding 'new_ds'.
        self.assertEqual(max(new_ds.lats), subset[1])
        self.assertEqual(ds_lon_min, subset[2])
        self.assertEqual(ds_lon_max, subset[3])
        self.assertEquals(str(start), subset[4])
        self.assertEquals(str(end), subset[5])
示例#29
0
def Map_plot_bias_of_multiyear_climatology(obs_dataset,
                                           obs_name,
                                           model_datasets,
                                           model_names,
                                           file_name,
                                           row,
                                           column,
                                           map_projection=None):
    '''Draw maps of observed multi-year climatology and biases of models"'''

    # calculate climatology of observation data
    obs_clim = utils.calc_temporal_mean(obs_dataset)
    # determine the metrics
    map_of_bias = metrics.TemporalMeanBias()

    # create the Evaluation object
    bias_evaluation = Evaluation(
        obs_dataset,  # Reference dataset for the evaluation
        model_datasets,  # list of target datasets for the evaluation
        [map_of_bias, map_of_bias])
    # run the evaluation (bias calculation)
    bias_evaluation.run()

    rcm_bias = bias_evaluation.results[0]

    fig = plt.figure()

    lat_min = obs_dataset.lats.min()
    lat_max = obs_dataset.lats.max()
    lon_min = obs_dataset.lons.min()
    lon_max = obs_dataset.lons.max()

    string_list = list(string.ascii_lowercase)
    ax = fig.add_subplot(row, column, 1)
    if map_projection == 'npstere':
        m = Basemap(ax=ax,
                    projection='npstere',
                    boundinglat=lat_min,
                    lon_0=0,
                    resolution='l',
                    fix_aspect=False)
    else:
        m = Basemap(ax=ax,
                    projection='cyl',
                    llcrnrlat=lat_min,
                    urcrnrlat=lat_max,
                    llcrnrlon=lon_min,
                    urcrnrlon=lon_max,
                    resolution='l',
                    fix_aspect=False)
    lons, lats = np.meshgrid(obs_dataset.lons, obs_dataset.lats)

    x, y = m(lons, lats)

    m.drawcoastlines(linewidth=1)
    m.drawcountries(linewidth=1)
    m.drawstates(linewidth=0.5, color='w')
    max = m.contourf(x,
                     y,
                     obs_clim,
                     levels=plotter._nice_intervals(obs_dataset.values, 10),
                     extend='both',
                     cmap='rainbow')
    ax.annotate('(a) \n' + obs_name, xy=(lon_min, lat_min))
    cax = fig.add_axes([0.02, 1. - float(1. / row), 0.01, 1. / row * 0.6])
    plt.colorbar(max, cax=cax)
    clevs = plotter._nice_intervals(rcm_bias, 11)
    for imodel in np.arange(len(model_datasets)):

        ax = fig.add_subplot(row, column, 2 + imodel)
        if map_projection == 'npstere':
            m = Basemap(ax=ax,
                        projection='npstere',
                        boundinglat=lat_min,
                        lon_0=0,
                        resolution='l',
                        fix_aspect=False)
        else:
            m = Basemap(ax=ax,
                        projection='cyl',
                        llcrnrlat=lat_min,
                        urcrnrlat=lat_max,
                        llcrnrlon=lon_min,
                        urcrnrlon=lon_max,
                        resolution='l',
                        fix_aspect=False)
        m.drawcoastlines(linewidth=1)
        m.drawcountries(linewidth=1)
        m.drawstates(linewidth=0.5, color='w')
        max = m.contourf(x,
                         y,
                         rcm_bias[imodel, :],
                         levels=clevs,
                         extend='both',
                         cmap='RdBu_r')
        ax.annotate('(' + string_list[imodel + 1] + ')  \n ' +
                    model_names[imodel],
                    xy=(lon_min, lat_min))

    cax = fig.add_axes([0.91, 0.1, 0.015, 0.8])
    plt.colorbar(max, cax=cax)

    plt.subplots_adjust(hspace=0.01, wspace=0.05)

    fig.savefig(file_name, dpi=600, bbox_inches='tight')
示例#30
0
class TestEvaluation(unittest.TestCase):
    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value,
                                            self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset, [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(-10, 10, -20, 20, dt.datetime(2000, 1, 1),
                       dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

    def test_add_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)

    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [self.another_test_dataset],
                               [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)
    def test_contains_only_target_datasets(self):
        new_eval = Evaluation(None, [self.local_ds], [])
        out = writer.generate_dataset_information(new_eval)

        self.assertTrue('reference' not in out)
        self.assertTrue('targets' in out)
    def setUpClass(self):
        self.bias = metrics.Bias()
        self.tmp_std_dev = metrics.TemporalStdDev()
        loaded_metrics = [self.bias, self.tmp_std_dev]

        self.evaluation = Evaluation(None, [], loaded_metrics)
示例#33
0
class TestEvaluation(unittest.TestCase):

    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value,
                                            self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset,
            [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])
        ref_dataset = self.test_dataset
        target_datasets = [self.test_dataset, self.another_test_dataset]
        metrics = [Bias(), Bias()]
        unary_metrics = [TemporalStdDev()]

        self.eval = Evaluation(ref_dataset,
                               target_datasets,
                               metrics + unary_metrics)

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
        self.eval.run()
        out_str = (
            "<Evaluation - ref_dataset: {}, "
            "target_dataset(s): {}, "
            "binary_metric(s): {}, "
            "unary_metric(s): {}, "
            "subregion(s): {}>"
        ).format(
            str(self.test_dataset),
            [str(ds) for ds in target_datasets],
            [str(m) for m in metrics],
            [str(u) for u in unary_metrics],
            None
        )
        self.assertEqual(str(self.eval), out_str)

    def test_valid_ref_dataset_setter(self):
        self.eval.ref_dataset = self.another_test_dataset
        self.assertEqual(self.eval.ref_dataset.variable,
                         self.another_test_dataset.variable)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(
            lat_min=-10, lat_max=10,
            lon_min=-20, lon_max=20,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_valid_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable,
                         self.variable)

    def test_add_invalid_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.add_dataset('This is an invalid dataset')

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable,
                         self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable,
                         self.other_var)

    def test_add_valid_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_invalid_metric(self):
        with self.assertRaises(TypeError):
            self.eval.add_metric('This is an invalid metric')

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)

    def test_invalid_evaluation_run(self):
        self.eval = Evaluation(None, [], [])
        self.assertEqual(self.eval.run(), None)

    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [
                               self.another_test_dataset], [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)

    def test_result_shape(self):
        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset,
                self.another_test_dataset],
            [Bias(), Bias()]
        )
        bias_eval.run()

        # Expected result shape is
        # [bias, bias] where bias.shape[0] = number of datasets
        self.assertTrue(len(bias_eval.results) == 2)
        self.assertTrue(bias_eval.results[0].shape[0] == 3)

    def test_unary_result_shape(self):
        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset,
                self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev()]
        )
        new_eval.run()

        # Expected result shape is
        # [stddev] where stddev.shape[0] = number of datasets

        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(new_eval.unary_results[0].shape[0] == 5)

    def test_subregion_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [Bias()],
            [bound]
        )
        bias_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [number of targets, bias.run(reference, target1).shape]
        #       ]
        #   ],
        self.assertTrue(len(bias_eval.results) == 1)
        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(bias_eval.results[0][0].shape[0] == 2)
        self.assertTrue(isinstance(bias_eval.results, type([])))

    def test_subregion_unary_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev(), TemporalStdDev()],
            [bound, bound, bound, bound, bound]
        )
        new_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [3, temporalstddev.run(reference).shape],
        #       ]
        # ]

        # 5 = number of subregions
        self.assertTrue(len(new_eval.unary_results) == 5)
        # number of metrics
        self.assertTrue(len(new_eval.unary_results[0]) == 2)
        self.assertTrue(isinstance(new_eval.unary_results, type([])))
        # number of datasets (ref + target)
        self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3)
    def test_empty_metrics_in_evaluation(self):
        new_eval = Evaluation(None, [], [])
        out = writer.generate_metric_information(new_eval)

        self.assertTrue(type(out) == type(list()))
        self.assertTrue(len(out) == 0)
示例#35
0
 def test_invalid_evaluation_run(self):
     self.eval = Evaluation(None, [], [])
     self.assertEqual(self.eval.run(), None)
示例#36
0
    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)
    def setUpClass(self):
        self.lats = np.array([10, 12, 14, 16, 18])
        self.lons = np.array([100, 102, 104, 106, 108])
        self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        self.values = flat_array.reshape(12, 5, 5)
        self.variable = 'var'
        self.units = 'units'
        self.name = 'name'

        self.local_origin = {
            'source': 'local',
            'path': '/a/fake/path.nc',
            'lat_name': 'a lat name',
            'lon_name': 'a lon name',
            'time_name': 'a time name',
            'elevation_index': 2
        }

        self.rcmed_origin = {
            'source': 'rcmed',
            'dataset_id': 4,
            'parameter_id': 14
        }

        self.esgf_origin = {
            'source': 'esgf',
            'dataset_id': 'esgf dataset id',
            'variable': 'var'
        }

        self.dap_origin = {
            'source': 'dap',
            'url': 'a fake url',
        }

        self.local_ds = Dataset(self.lats,
                                self.lons,
                                self.times,
                                self.values,
                                variable=self.variable,
                                units=self.units,
                                name=self.name,
                                origin=self.local_origin)

        self.rcmed_ds = Dataset(self.lats,
                                self.lons,
                                self.times,
                                self.values,
                                variable=self.variable,
                                units=self.units,
                                name=self.name,
                                origin=self.rcmed_origin)

        self.esgf_ds = Dataset(self.lats,
                               self.lons,
                               self.times,
                               self.values,
                               variable=self.variable,
                               units=self.units,
                               name=self.name,
                               origin=self.esgf_origin)

        self.dap_ds = Dataset(self.lats,
                              self.lons,
                              self.times,
                              self.values,
                              variable=self.variable,
                              units=self.units,
                              name=self.name,
                              origin=self.dap_origin)

        self.subregions = [
            Bounds(lat_min=-10, lat_max=10, lon_min=-20, lon_max=20),
            Bounds(lat_min=-5, lat_max=5, lon_min=-15, lon_max=15)
        ]

        self.evaluation = Evaluation(
            self.local_ds, [self.rcmed_ds, self.esgf_ds, self.dap_ds],
            [metrics.Bias(), metrics.TemporalStdDev()],
            subregions=self.subregions)
示例#38
0
def run_evaluation():
    ''' Run an OCW Evaluation.

    *run_evaluation* expects the Evaluation parameters to be POSTed in
    the following format.

    .. sourcecode:: javascript

        {
            reference_dataset: {
                // Id that tells us how we need to load this dataset.
                'data_source_id': 1 == local, 2 == rcmed,

                // Dict of data_source specific identifying information.
                //
                // if data_source_id == 1 == local:
                // {
                //     'id': The path to the local file on the server for loading.
                //     'var_name': The variable data to pull from the file.
                //     'lat_name': The latitude variable name.
                //     'lon_name': The longitude variable name.
                //     'time_name': The time variable name
                //     'name': Optional dataset name
                // }
                //
                // if data_source_id == 2 == rcmed:
                // {
                //     'dataset_id': The dataset id to grab from RCMED.
                //     'parameter_id': The variable id value used by RCMED.
                //     'name': Optional dataset name
                // }
                'dataset_info': {..}
            },

            // The list of target datasets to use in the Evaluation. The data
            // format for the dataset objects should be the same as the
            // reference_dataset above.
            'target_datasets': [{...}, {...}, ...],

            // All the datasets are re-binned to the reference dataset
            // before being added to an experiment. This step (in degrees)
            // is used when re-binning both the reference and target datasets.
            'spatial_rebin_lat_step': The lat degree step. Integer > 0,

            // Same as above, but for lon
            'spatial_rebin_lon_step': The lon degree step. Integer > 0,

            // The temporal resolution to use when doing a temporal re-bin
            // This is a timedelta of days to use so daily == 1, monthly is
            // (1, 31], annual/yearly is (31, 366], and full is anything > 366.
            'temporal_resolution': Integer in range(1, 999),

            // A list of the metric class names to use in the evaluation. The
            // names must match the class name exactly.
            'metrics': [Bias, TemporalStdDev, ...]

            // The bounding values used in the Evaluation. Note that lat values
            // should range from -180 to 180 and lon values from -90 to 90.
            'start_time': start time value in the format '%Y-%m-%d %H:%M:%S',
            'end_time': end time value in the format '%Y-%m-%d %H:%M:%S',
            'lat_min': The minimum latitude value,
            'lat_max': The maximum latitude value,
            'lon_min': The minimum longitude value,
            'lon_max': The maximum longitude value,

            // NOTE: At the moment, subregion support is fairly minimal. This
            // will be addressed in the future. Ideally, the user should be able
            // to load a file that they have locally. That would change the
            // format that this data is passed.
            'subregion_information': Path to a subregion file on the server.
        }
    '''
    # TODO: validate input parameters and return an error if not valid

    eval_time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    data = request.json

    eval_bounds = {
        'start_time': datetime.strptime(data['start_time'], '%Y-%m-%d %H:%M:%S'),
        'end_time': datetime.strptime(data['end_time'], '%Y-%m-%d %H:%M:%S'),
        'lat_min': float(data['lat_min']),
        'lat_max': float(data['lat_max']),
        'lon_min': float(data['lon_min']),
        'lon_max': float(data['lon_max'])
    }

    # Load all the datasets
    ref_dataset = _process_dataset_object(data['reference_dataset'], eval_bounds)

    target_datasets = [_process_dataset_object(obj, eval_bounds)
					   for obj
					   in data['target_datasets']]

    # Normalize the dataset time values so they break on consistent days of the
    # month or time of the day, depending on how they will be rebinned.
    resolution = data['temporal_resolution']
    time_delta = timedelta(days=resolution)

    time_step = 'daily' if resolution == 1 else 'monthly'
    ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, time_step)
    target_datasets = [dsp.normalize_dataset_datetimes(ds, time_step)
                       for ds in target_datasets]

    # Subset the datasets
    start = eval_bounds['start_time']
    end = eval_bounds['end_time']

    # Normalize all the values to the first of the month if we're not
    # dealing with daily data. This will ensure that a valid subregion
    # isn't considered out of bounds do to a dataset's time values
    # being shifted to the first of the month.
    if time_step != 'daily':
        if start.day != 1:
            day_offset = start.day - 1
            start -= timedelta(days=day_offset)

        if end.day != 1:
            day_offset = end.day - 1
            end -= timedelta(days=day_offset)

    subset = Bounds(eval_bounds['lat_min'],
                    eval_bounds['lat_max'],
                    eval_bounds['lon_min'],
                    eval_bounds['lon_max'],
                    start,
                    end)

    ref_dataset = dsp.safe_subset(ref_dataset, subset)
    target_datasets = [dsp.safe_subset(ds, subset)
                       for ds
                       in target_datasets]
    
    # Do temporal re-bin based off of passed resolution
    ref_dataset = dsp.temporal_rebin(ref_dataset, time_delta)
    target_datasets = [dsp.temporal_rebin(ds, time_delta)
					   for ds
					   in target_datasets]

    # Do spatial re=bin based off of reference dataset + lat/lon steps
    lat_step = data['spatial_rebin_lat_step']
    lon_step = data['spatial_rebin_lon_step']
    lat_bins, lon_bins = _calculate_new_latlon_bins(eval_bounds,
													lat_step,
													lon_step)

    ref_dataset = dsp.spatial_regrid(ref_dataset, lat_bins, lon_bins)
    target_datasets =  [dsp.spatial_regrid(ds, lat_bins, lon_bins)
						for ds
						in target_datasets]

    # Load metrics
    loaded_metrics = _load_metrics(data['metrics'])

    # Prime evaluation object with data
    evaluation = Evaluation(ref_dataset, target_datasets, loaded_metrics)

    # Run evaluation
    evaluation.run()

    # Plot
    _generate_evaluation_plots(evaluation, lat_bins, lon_bins, eval_time_stamp)

    return json.dumps({'eval_work_dir': eval_time_stamp})
示例#39
0
def run_evaluation():
    ''' Run an OCW Evaluation.

    *run_evaluation* expects the Evaluation parameters to be POSTed in
    the following format.

    .. sourcecode:: javascript

        {
            reference_dataset: {
                // Id that tells us how we need to load this dataset.
                'data_source_id': 1 == local, 2 == rcmed,

                // Dict of data_source specific identifying information.
                //
                // if data_source_id == 1 == local:
                // {
                //     'id': The path to the local file on the server for loading.
                //     'var_name': The variable data to pull from the file.
                //     'lat_name': The latitude variable name.
                //     'lon_name': The longitude variable name.
                //     'time_name': The time variable name
                //     'name': Optional dataset name
                // }
                //
                // if data_source_id == 2 == rcmed:
                // {
                //     'dataset_id': The dataset id to grab from RCMED.
                //     'parameter_id': The variable id value used by RCMED.
                //     'name': Optional dataset name
                // }
                'dataset_info': {..}
            },

            // The list of target datasets to use in the Evaluation. The data
            // format for the dataset objects should be the same as the
            // reference_dataset above.
            'target_datasets': [{...}, {...}, ...],

            // All the datasets are re-binned to the reference dataset
            // before being added to an experiment. This step (in degrees)
            // is used when re-binning both the reference and target datasets.
            'spatial_rebin_lat_step': The lat degree step. Integer > 0,

            // Same as above, but for lon
            'spatial_rebin_lon_step': The lon degree step. Integer > 0,

            // The temporal resolution to use when doing a temporal re-bin
            // This is a timedelta of days to use so daily == 1, monthly is
            // (1, 31], annual/yearly is (31, 366], and full is anything > 366.
            'temporal_resolution': Integer in range(1, 999),

            // A list of the metric class names to use in the evaluation. The
            // names must match the class name exactly.
            'metrics': [Bias, TemporalStdDev, ...]

            // The bounding values used in the Evaluation. Note that lat values
            // should range from -180 to 180 and lon values from -90 to 90.
            'start_time': start time value in the format '%Y-%m-%d %H:%M:%S',
            'end_time': end time value in the format '%Y-%m-%d %H:%M:%S',
            'lat_min': The minimum latitude value,
            'lat_max': The maximum latitude value,
            'lon_min': The minimum longitude value,
            'lon_max': The maximum longitude value,

            // NOTE: At the moment, subregion support is fairly minimal. This
            // will be addressed in the future. Ideally, the user should be able
            // to load a file that they have locally. That would change the
            // format that this data is passed.
            'subregion_information': Path to a subregion file on the server.
        }
    '''
    # TODO: validate input parameters and return an error if not valid

    eval_time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    data = request.json

    eval_bounds = {
        'start_time': datetime.strptime(data['start_time'], '%Y-%m-%d %H:%M:%S'),
        'end_time': datetime.strptime(data['end_time'], '%Y-%m-%d %H:%M:%S'),
        'lat_min': float(data['lat_min']),
        'lat_max': float(data['lat_max']),
        'lon_min': float(data['lon_min']),
        'lon_max': float(data['lon_max'])
    }

    # Load all the datasets
    ref_dataset = _process_dataset_object(data['reference_dataset'], eval_bounds)

    target_datasets = [_process_dataset_object(obj, eval_bounds)
					   for obj
					   in data['target_datasets']]

    # Normalize the dataset time values so they break on consistent days of the
    # month or time of the day, depending on how they will be rebinned.
    resolution = data['temporal_resolution']
    time_delta = timedelta(days=resolution)

    time_step = 'daily' if resolution == 1 else 'monthly'
    ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, time_step)
    target_datasets = [dsp.normalize_dataset_datetimes(ds, time_step)
                       for ds in target_datasets]

    # Subset the datasets
    start = eval_bounds['start_time']
    end = eval_bounds['end_time']

    # Normalize all the values to the first of the month if we're not
    # dealing with daily data. This will ensure that a valid subregion
    # isn't considered out of bounds do to a dataset's time values
    # being shifted to the first of the month.
    if time_step != 'daily':
        if start.day != 1:
            day_offset = start.day - 1
            start -= timedelta(days=day_offset)

        if end.day != 1:
            day_offset = end.day - 1
            end -= timedelta(days=day_offset)

    subset = Bounds(eval_bounds['lat_min'],
                    eval_bounds['lat_max'],
                    eval_bounds['lon_min'],
                    eval_bounds['lon_max'],
                    start,
                    end)

    ref_dataset = dsp.safe_subset(subset, ref_dataset)
    target_datasets = [dsp.safe_subset(subset, ds)
                       for ds
                       in target_datasets]
    
    # Do temporal re-bin based off of passed resolution
    ref_dataset = dsp.temporal_rebin(ref_dataset, time_delta)
    target_datasets = [dsp.temporal_rebin(ds, time_delta)
					   for ds
					   in target_datasets]

    # Do spatial re=bin based off of reference dataset + lat/lon steps
    lat_step = data['spatial_rebin_lat_step']
    lon_step = data['spatial_rebin_lon_step']
    lat_bins, lon_bins = _calculate_new_latlon_bins(eval_bounds,
													lat_step,
													lon_step)

    ref_dataset = dsp.spatial_regrid(ref_dataset, lat_bins, lon_bins)
    target_datasets =  [dsp.spatial_regrid(ds, lat_bins, lon_bins)
						for ds
						in target_datasets]

    # Load metrics
    loaded_metrics = _load_metrics(data['metrics'])

    # Prime evaluation object with data
    evaluation = Evaluation(ref_dataset, target_datasets, loaded_metrics)

    # Run evaluation
    evaluation.run()

    # Plot
    _generate_evaluation_plots(evaluation, lat_bins, lon_bins, eval_time_stamp)

    return json.dumps({'eval_work_dir': eval_time_stamp})
示例#40
0
class TestEvaluation(unittest.TestCase):
    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value, 
                self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
                        self.test_dataset,           
                        [self.test_dataset, self.another_test_dataset], 
                        [Bias(), Bias(), TemporalStdDev()])                    

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(
                -10, 10, 
                -20, 20, 
                dt.datetime(2000, 1, 1), dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable, 
                self.variable)

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable, 
                self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, 
                self.other_var)

    def test_add_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)
    
    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [self.another_test_dataset], [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)