예제 #1
0
def compute_indicator(args):

    # extract the arguments
    lon_index_start = args[0]
    lat_index = args[1]

    # turn the shared arrays into numpy arrays
    input_data = np.ctypeslib.as_array(input_shared_array)
    input_data = input_data.reshape(input_data_shape)
    gamma_data = np.ctypeslib.as_array(output_gamma_array)
    gamma_data = gamma_data.reshape(output_data_shape)
    pearson_data = np.ctypeslib.as_array(output_pearson_array)
    pearson_data = pearson_data.reshape(output_data_shape)

    for lon_index in range(lons_per_chunk):

        # only process non-empty grid cells, i.e. input_data array contains at least some non-NaN values
        if (isinstance(input_data[:, lon_index, lat_index], np.ma.MaskedArray) and input_data[:, lon_index, lat_index].mask.all()) \
            or np.isnan(input_data[:, lon_index, lat_index]).all() or (input_data[:, lon_index, lat_index] <= 0).all():

#             logger.info('No input_data at lon/lat: {0}/{1}'.format(lon_index_start + lon_index, lat_index))
            pass

        else:  # we have some valid values to work with

            logger.info('Processing longitude/latitude: {}/{}'.format(lon_index_start + lon_index, lat_index))
    
            for scale_index, month_scale in enumerate(month_scales):
            
                # perform a fitting to gamma
                gamma_data[scale_index, :, lon_index, lat_index] = indices.spi_gamma(input_data[:, lon_index, lat_index],
                                                                                      month_scale,
                                                                                      valid_min,
                                                                                      valid_max)

                # perform a fitting to gamma
                pearson_data[scale_index, :, lon_index, lat_index] = indices.spi_pearson(input_data[:, lon_index, lat_index],
                                                                                         month_scale,
                                                                                         valid_min,
                                                                                         valid_max, 
                                                                                         data_start_year, 
                                                                                         data_end_year, 
                                                                                         calibration_start_year, 
                                                                                         calibration_end_year)
                    # slice out the period of record for the x/y point
                    precip_data = precip_dataset.variables[precip_var_name][:, x, y]

                    # only process non-empty grid cells, i.e. the data array contains at least some non-NaN values
                    if (isinstance(precip_data, np.ma.MaskedArray)) and precip_data.mask.all():

                        continue

                    else:  # we have some valid values to work with

                        for month_scale_index, month_scale_var_name in enumerate(sorted(datasets.keys())):

                            # perform the SPI computation (fit to the Pearson distribution) and assign the values into the dataset
                            datasets[month_scale_var_name].variables[month_scale_var_name][
                                :, x, y
                            ] = indices.spi_pearson(
                                precip_data,
                                month_scales[month_scale_index],
                                valid_min,
                                valid_max,
                                data_start_date.year,
                                data_end_date.year,
                                calibration_start_year,
                                calibration_end_year,
                            )

    except Exception, e:
        logger.error("Failed to complete", exc_info=True)
        raise
예제 #3
0
                            
            # loop over the grid cells
            for x in range(precip_dataset.variables[x_dim_name].size):
                for y in range(precip_dataset.variables[y_dim_name].size):
                    
                    logger.info('Processing x/y {}/{}'.format(x, y))
                    
                    # slice out the period of record for the x/y point
                    precip_data = precip_dataset.variables[precip_var_name][:, x, y]
                                           
                    # only process non-empty grid cells, i.e. data array contains at least some non-NaN values
                    if (isinstance(precip_data, np.ma.MaskedArray)) and precip_data.mask.all():
                        
                            continue
                    
                    else:  # we have some valid values to work with
                        
                        # perform the SPI computation (fit to the Gamma distribution) and assign the values into the dataset
                        output_dataset.variables[variable_name][:, x, y] = indices.spi_pearson(precip_data, 
                                                                                               month_scale, 
                                                                                               valid_min, 
                                                                                               valid_max, 
                                                                                               data_start_date.year, 
                                                                                               data_end_date.year, 
                                                                                               calibration_start_year, 
                                                                                               calibration_end_year)
            
    except Exception, e:
        logger.error('Failed to complete', exc_info=True)
        raise
예제 #4
0
def compute_worker(args):
         
    # extract the arguments
    lat_index = args[0]
    
    # turn the shared array into a numpy array
    data = np.ctypeslib.as_array(shared_array)
    data = data.reshape(data_shape)
                 
    # data now expected to be in shape: (indicators, distributions, month_scales, times, lats)
    #
    # with indicator (spi: 0,  spei: 1)
    #      distribution (gamma: 0,  pearson: 1)
    #      month_scales (0, month_scales)
    #
    # with data[0, 0, 0] indicating the longitude slice with shape: (times, lats) with values for precipitation 
    # with data[1, 0, 0] indicating the longitude slice with shape: (times, lats) with values for temperature 
    
    # only process non-empty grid cells, i.e. data array contains at least some non-NaN values
    if (isinstance(data[0, 0, 0, :, lat_index], np.ma.MaskedArray) and data[0, 0, 0, :, lat_index].mask.all()) \
        or np.isnan(data[0, 0, 0, :, lat_index]).all() or (data[0, 0, 0, :, lat_index] <= 0).all():
              
        pass         
                   
    else:  # we have some valid values to work with
              
        logger.info('Processing latitude: {}'.format(lat_index))
              
        for month_index, month_scale in enumerate(month_scales):
            
            # only process month scales after 0 since month_scale = 0 is reserved for the input data 
            if month_index > 0:
                
                # loop over all specified indicators
                for indicator in indicators:

                    # loop over all specified distributions
                    for distribution in distributions:
                        
                        if indicator == 'spi':
                            
                            if distribution == 'gamma':
                                    # perform a fitting to gamma     
                                    data[0, 0, month_index, :, lat_index] = indices.spi_gamma(data[0, 0, 0, :, lat_index],
                                                                                              month_scale, 
                                                                                              valid_min, 
                                                                                              valid_max)
                            elif distribution == 'pearson':
                                    # perform a fitting to Pearson type III     
                                    data[0, 1, month_index, :, lat_index] = indices.spi_pearson(data[0, 0, 0, :, lat_index], 
                                                                                                month_scale, 
                                                                                                valid_min, 
                                                                                                valid_max, 
                                                                                                data_start_year, 
                                                                                                data_end_year, 
                                                                                                calibration_start_year, 
                                                                                                calibration_end_year)
    
                        elif indicator == 'spei':
                            
                            if distribution == 'gamma':
                                    # perform a fitting to gamma     
                                    data[1, 0, month_index, :, lat_index] = indices.spei_gamma(data[0, 0, 0, :, lat_index],
                                                                                               data[0, 0, 1, :, lat_index],
                                                                                               data_start_year,
                                                                                               lats_array[lat_index],
                                                                                               month_scale, 
                                                                                               valid_min, 
                                                                                               valid_max)
                            elif distribution == 'pearson':
                                    # perform a fitting to Pearson type III     
                                    data[1, 1, month_index, :, lat_index] = indices.spei_pearson(data[0, 0, 0, :, lat_index],
                                                                                                 data[0, 0, 1, :, lat_index],
                                                                                                 month_scale, 
                                                                                                 lats_array[lat_index],
                                                                                                 valid_min, 
                                                                                                 valid_max,
                                                                                                 data_start_year,
                                                                                                 data_end_year,
                                                                                                 calibration_start_year, 
                                                                                                 calibration_end_year)
                                    
                            else:
                                raise ValueError('Invalid distribution specified: {}'.format(distribution))
                        else:
                            raise ValueError('Invalid indicator specified: {}'.format(indicator))