def get_indices(self, start, end, period):
stat_p, pval_p = self.test_hypothesis_pearson(start, end)
stat_g, pval_g = self.test_hypothesis_gamma(start, end)
print("pearson:", stat_p, pval_p)
print("gamma:", stat_g, pval_g)
if pval_g > pval_p:
print("using gamma")
sdi = indices.spi(self.get_discharge(start, end), int(period),
indices.Distribution.gamma,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
spi = indices.spi(self.get_precip(start, end), int(period),
indices.Distribution.gamma,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
else:
print("using pearson3")
sdi = indices.spi(self.get_discharge(start, end), int(period),
indices.Distribution.pearson,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
spi = indices.spi(self.get_precip(start, end), int(period),
indices.Distribution.pearson,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
sdi, spi = np.nan_to_num(sdi), np.nan_to_num(spi)
return sdi, spi, self.get_dates(start, end)
def get_indices(self, start, end):
sdi = indices.spi(self.get_discharge(start, end), 12,
indices.Distribution.gamma,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
spi = indices.spi(self.get_precip(start,
end), 12, indices.Distribution.gamma,
self.data_year_start_monthly,
self.data_year_start_monthly,
self.data_year_end_monthly,
compute.Periodicity.monthly)
sdi, spi = np.nan_to_num(sdi), np.nan_to_num(spi)
return sdi, spi, self.get_dates(start, end)
def cal_kernel(self,val):
# values = val
scale = 1
distribution = indices.Distribution.gamma
data_start_year = 2003
calibration_year_initial = 2003
calibration_year_final = 2016
periodicity = compute.Periodicity.monthly
spi = indices.spi(val,
scale,
distribution,
data_start_year,
calibration_year_initial,
calibration_year_final, periodicity
)
return spi
print("Keys: %s" % f.keys())
a_group_key = list(f.keys())[0]
# Get the data
data = list(f[a_group_key])
# Since the dimensions of each matrix is the same, we can simply use the rows/column dimension in the first matrix
rows = len(data[0])
columns = len(data[0][0])
gridded_spi = []
for i in range(0, rows - 1):
row_spi = []
for j in range(0, columns - 1):
pixel_precipitations = []
for clipped_image in data:
pixel_precipitations.append(clipped_image[i][j])
spi = indices.spi(np.array(pixel_precipitations), 90,
indices.Distribution.gamma, 1989, 1989, 2018,
compute.Periodicity.daily)
print('Spi')
print(spi)
row_spi.append(spi)
gridded_spi.append(row_spi)
# Outputing the result as a CSV
with open('spi3.csv', 'w', newline="") as csv_file:
writer = csv.writer(csv_file)
writer.writerow(gridded_spi)
end = time.time()
print('The process took %.2f minutes.' % ((end - start) / 60))
def test_spi(precips_mm_monthly, precips_mm_daily, data_year_start_monthly,
data_year_end_monthly, data_year_start_daily,
calibration_year_start_monthly, calibration_year_end_monthly,
calibration_year_start_daily, calibration_year_end_daily,
spi_1_month_gamma, spi_6_month_gamma, spi_6_month_pearson3):
# confirm that an input array of all NaNs for
# precipitation results in the same array returned
all_nans = np.full(precips_mm_monthly.shape, np.NaN)
computed_spi = indices.spi(all_nans, 1, indices.Distribution.gamma,
data_year_start_monthly,
data_year_start_monthly, data_year_end_monthly,
compute.Periodicity.monthly)
np.testing.assert_allclose(computed_spi,
all_nans.flatten(),
equal_nan=True,
err_msg="SPI/Gamma not handling "
"all-NaN arrays as expected")
# confirm SPI/gamma is being computed as expected
computed_spi = indices.spi(precips_mm_monthly, 1,
indices.Distribution.gamma,
data_year_start_monthly,
data_year_start_monthly, data_year_end_monthly,
compute.Periodicity.monthly)
np.testing.assert_allclose(computed_spi,
spi_1_month_gamma,
atol=0.001,
err_msg="SPI/Gamma values for 1-month "
"scale not computed as expected")
# confirm SPI/gamma is being computed as expected
computed_spi = indices.spi(precips_mm_monthly.flatten(), 6,
indices.Distribution.gamma,
data_year_start_monthly,
data_year_start_monthly, data_year_end_monthly,
compute.Periodicity.monthly)
# confirm SPI/gamma is being computed as expected
np.testing.assert_allclose(computed_spi,
spi_6_month_gamma,
atol=0.001,
err_msg="SPI/Gamma values for 6-month "
"scale not computed as expected")
# confirm we can also call the function with daily data,
# if this completes without error then test passes
indices.spi(precips_mm_daily, 30, indices.Distribution.gamma,
data_year_start_daily, calibration_year_start_daily,
calibration_year_end_daily, compute.Periodicity.daily)
# invalid periodicity argument should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
precips_mm_monthly.flatten(), 6,
indices.Distribution.gamma,
data_year_start_monthly, data_year_start_monthly,
data_year_end_monthly, "unsupported_value")
# invalid distribution argument should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
precips_mm_monthly.flatten(), 6, None,
data_year_start_monthly, data_year_start_monthly,
data_year_end_monthly,
compute.Periodicity.monthly)
# input array argument that's neither 1-D nor 2-D should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
np.array(np.zeros(
(4, 4, 8))), 6, indices.Distribution.gamma,
data_year_start_monthly, data_year_start_monthly,
data_year_end_monthly, compute.Periodicity.daily)
# compute SPI/Pearson at 60-day scale, just make sure it completes without error
# TODO compare against expected results
indices.spi(precips_mm_daily.flatten(), 60, indices.Distribution.pearson,
data_year_start_daily, calibration_year_start_daily,
calibration_year_end_daily, compute.Periodicity.daily)
# confirm SPI/Pearson is being computed as expected
computed_spi = indices.spi(precips_mm_monthly.flatten(), 6,
indices.Distribution.pearson,
data_year_start_monthly,
calibration_year_start_monthly,
calibration_year_end_monthly,
compute.Periodicity.monthly)
np.testing.assert_allclose(computed_spi,
spi_6_month_pearson3,
atol=0.01,
err_msg="SPI/Pearson values for 6-month "
"scale not computed as expected")
# confirm we can compute from daily values without raising an error
indices.spi(precips_mm_daily.flatten(), 60, indices.Distribution.pearson,
data_year_start_daily, calibration_year_start_daily,
calibration_year_end_daily, compute.Periodicity.daily)
# invalid periodicity argument should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
precips_mm_monthly.flatten(), 6,
indices.Distribution.pearson,
data_year_start_monthly,
calibration_year_start_monthly,
calibration_year_end_monthly, "unsupported_value")
def test_spi(self):
# compute SPI/gamma at 1-month scale
computed_spi = indices.spi(self.fixture_precips_mm_monthly, 1,
indices.Distribution.gamma,
self.fixture_data_year_start_monthly,
self.fixture_data_year_start_monthly,
self.fixture_data_year_end_monthly,
compute.Periodicity.monthly)
# confirm SPI/gamma is being computed as expected
np.testing.assert_allclose(
computed_spi,
self.fixture_spi_1_month_gamma,
atol=0.001,
err_msg=
'SPI/Gamma values for 1-month scale not computed as expected')
computed_spi = indices.spi(self.fixture_precips_mm_monthly.flatten(),
6, indices.Distribution.gamma,
self.fixture_data_year_start_monthly,
self.fixture_data_year_start_monthly,
self.fixture_data_year_end_monthly,
compute.Periodicity.monthly)
# confirm SPI/gamma is being computed as expected
np.testing.assert_allclose(
computed_spi,
self.fixture_spi_6_month_gamma,
atol=0.001,
err_msg=
'SPI/Gamma values for 6-month scale not computed as expected')
# confirm we can also call the function with daily data, if this completes without error then test passes
indices.spi(self.fixture_precips_mm_daily, 30,
indices.Distribution.gamma,
self.fixture_data_year_start_daily,
self.fixture_calibration_year_start_daily,
self.fixture_calibration_year_end_daily,
compute.Periodicity.daily)
# invalid periodicity argument should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
self.fixture_precips_mm_monthly.flatten(), 6,
indices.Distribution.gamma,
self.fixture_data_year_start_monthly,
self.fixture_data_year_start_monthly,
self.fixture_data_year_end_monthly,
'unsupported_value')
# input array argument that's neither 1-D nor 2-D should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
np.array(np.zeros((4, 4, 8))), 6,
indices.Distribution.gamma,
self.fixture_data_year_start_monthly,
self.fixture_data_year_start_monthly,
self.fixture_data_year_end_monthly,
compute.Periodicity.daily)
# compute SPI/Pearson at 6-month scale
computed_spi = indices.spi(self.fixture_precips_mm_monthly.flatten(),
6, indices.Distribution.pearson_type3,
self.fixture_data_year_start_monthly,
self.fixture_calibration_year_start_monthly,
self.fixture_calibration_year_end_monthly,
compute.Periodicity.monthly)
# confirm we can compute from daily values without raising an error
indices.spi(self.fixture_precips_mm_daily.flatten(), 60,
indices.Distribution.pearson_type3,
self.fixture_data_year_start_daily,
self.fixture_calibration_year_start_daily,
self.fixture_calibration_year_end_daily,
compute.Periodicity.daily)
# confirm SPI/Pearson is being computed as expected
np.testing.assert_allclose(
computed_spi,
self.fixture_spi_6_month_pearson3,
atol=0.01,
err_msg=
'SPI/Pearson values for 6-month scale not computed as expected')
# invalid periodicity argument should raise a ValueError
np.testing.assert_raises(ValueError, indices.spi,
self.fixture_precips_mm_monthly.flatten(), 6,
indices.Distribution.pearson_type3,
self.fixture_data_year_start_monthly,
self.fixture_calibration_year_start_monthly,
self.fixture_calibration_year_end_monthly,
'unsupported_value')
def _compute_and_write_division(self, div_index):
"""
Computes indices for a single division, writing the output into NetCDF.
:param div_index:
"""
# only process specified divisions
if self.divisions is not None and div_index not in self.divisions:
return
# open the NetCDF files
with netCDF4.Dataset(self.input_file, 'a') as input_divisions, \
netCDF4.Dataset(self.output_file, 'a') as output_divisions:
climdiv_id = input_divisions['division'][div_index]
# only process divisions within CONUS, 101 - 4811
if climdiv_id > 4811:
return
logger.info('Processing indices for division %s', climdiv_id)
# read the division of input temperature values
temperature = input_divisions[self.var_name_temperature][div_index, :] # assuming dims (divisions, time)
# initialize the latitude outside of the valid range, in order to use
# this within a conditional below to verify a valid latitude
latitude = -100.0
# latitudes are only available for certain divisions, make sure we have one for this division index
if div_index < input_divisions['lat'][:].size:
# get the actual latitude value (assumed to be in degrees north)
# for the latitude slice specified by the index
latitude = input_divisions['lat'][div_index]
# only proceed if the latitude value is within valid range
if not np.isnan(latitude) and (latitude < 90.0) and (latitude > -90.0):
# convert temperatures from Fahrenheit to Celsius, if necessary
temperature_units = input_divisions[self.var_name_temperature].units
if temperature_units in ['degree_Fahrenheit',
'degrees Fahrenheit',
'degrees F',
'fahrenheit',
'Fahrenheit',
'F']:
# TODO make sure this application of the ufunc is any faster pylint: disable=fixme
temperature = scipy.constants.convert_temperature(temperature, 'F', 'C')
elif temperature_units not in ['degree_Celsius',
'degrees Celsius',
'degrees C',
'celsius',
'Celsius',
'C']:
raise ValueError('Unsupported temperature units: \'{0}\''.format(temperature_units))
# TODO instead use the numpy.apply_along_axis() function for computing indices such as PET
# that take a single time series array as input (i.e. each division's time series is the initial
# 1-D array argument to the function we'll apply)
logger.info('\tComputing PET for division %s', climdiv_id)
logger.info('\t\tCalculating PET using Thornthwaite method')
# compute PET across all longitudes of the latitude slice
# Thornthwaite PE
pet_time_series = indices.pet(temperature,
latitude_degrees=latitude,
data_start_year=self.data_start_year)
# the above returns PET in millimeters, note this for further consideration
pet_units = 'millimeter'
# write the PET values to NetCDF
lock_output.acquire()
output_divisions['pet'][div_index, :] = np.reshape(pet_time_series, (1, pet_time_series.size))
output_divisions.sync()
lock_output.release()
else:
pet_time_series = np.full(temperature.shape, np.NaN)
pet_units = None
# read the division's input precipitation and available water capacity values
precip_time_series = input_divisions[self.var_name_precip][div_index, :] # assuming dims (divisions, time)
if div_index < input_divisions[self.var_name_soil][:].size:
awc = input_divisions[self.var_name_soil][div_index] # assuming (divisions) dims orientation
# AWC values need to include top inch, values from the soil file do not, so we add top inch here
awc += 1
else:
awc = np.NaN
# compute SPI and SPEI for the current division only if we have valid inputs
if not np.isnan(precip_time_series).all():
# put precipitation into inches if not already
mm_to_inches_multiplier = 0.0393701
possible_mm_units = ['millimeters', 'millimeter', 'mm']
if input_divisions[self.var_name_precip].units in possible_mm_units:
precip_time_series = precip_time_series * mm_to_inches_multiplier
# only compute Palmers if we have PET already
if not np.isnan(pet_time_series).all():
# compute Palmer indices if we have valid inputs
if not np.isnan(awc):
# if PET is in mm, convert to inches
if pet_units in possible_mm_units:
pet_time_series = pet_time_series * mm_to_inches_multiplier
# PET is in mm, convert to inches since the Palmer uses imperial units
pet_time_series = pet_time_series * mm_to_inches_multiplier
logger.info('\tComputing PDSI for division %s', climdiv_id)
# compute Palmer indices
palmer_values = indices.scpdsi(precip_time_series,
pet_time_series,
awc,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year)
scpdsi = palmer_values[0]
pdsi = palmer_values[1]
phdi = palmer_values[2]
pmdi = palmer_values[3]
zindex = palmer_values[4]
# write the Palmer index values to NetCDF
lock_output.acquire()
output_divisions['pdsi'][div_index, :] = np.reshape(pdsi, (1, pdsi.size))
output_divisions['phdi'][div_index, :] = np.reshape(phdi, (1, phdi.size))
output_divisions['pmdi'][div_index, :] = np.reshape(pmdi, (1, pmdi.size))
output_divisions['scpdsi'][div_index, :] = np.reshape(pdsi, (1, scpdsi.size))
output_divisions['zindex'][div_index, :] = np.reshape(zindex, (1, zindex.size))
output_divisions.sync()
lock_output.release()
# process the SPI, SPEI, and PNP at the specified month scales
for months in self.scale_months:
logger.info('\tComputing SPI/SPEI/PNP at %s-month scale for division %s', months, climdiv_id)
# TODO ensure that the precipitation and PET values are using the same units
# compute SPEI/Gamma
spei_gamma = indices.spei(months,
indices.Distribution.gamma,
compute.Periodicity.monthly,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year,
precip_time_series,
pet_mm=pet_time_series)
# compute SPEI/Pearson
spei_pearson = indices.spei(months,
indices.Distribution.pearson_type3,
compute.Periodicity.monthly,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year,
precip_time_series,
pet_mm=pet_time_series)
# compute SPI/Gamma
spi_gamma = indices.spi(precip_time_series,
months,
indices.Distribution.gamma,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year,
compute.Periodicity.monthly)
# compute SPI/Pearson
spi_pearson = indices.spi(precip_time_series,
months,
indices.Distribution.pearson_type3,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year,
compute.Periodicity.monthly)
# compute PNP
pnp = indices.percentage_of_normal(precip_time_series,
months,
self.data_start_year,
self.calibration_start_year,
self.calibration_end_year,
compute.Periodicity.monthly)
# create variable names which should correspond to the appropriate scaled index output variables
scaled_name_suffix = str(months).zfill(2)
spei_gamma_variable_name = 'spei_gamma_' + scaled_name_suffix
spei_pearson_variable_name = 'spei_pearson_' + scaled_name_suffix
spi_gamma_variable_name = 'spi_gamma_' + scaled_name_suffix
spi_pearson_variable_name = 'spi_pearson_' + scaled_name_suffix
pnp_variable_name = 'pnp_' + scaled_name_suffix
# write the SPI, SPEI, and PNP values to NetCDF
lock_output.acquire()
output_divisions[spei_gamma_variable_name][div_index, :] = \
np.reshape(spei_gamma, (1, spei_gamma.size))
output_divisions[spei_pearson_variable_name][div_index, :] = \
np.reshape(spei_pearson, (1, spei_pearson.size))
output_divisions[spi_gamma_variable_name][div_index, :] = \
np.reshape(spi_gamma, (1, spi_gamma.size))
output_divisions[spi_pearson_variable_name][div_index, :] = \
np.reshape(spi_pearson, (1, spi_pearson.size))
output_divisions[pnp_variable_name][div_index, :] = np.reshape(pnp, (1, pnp.size))
output_divisions.sync()
lock_output.release()
def calc_spi(self):
chirps_files = sorted(os.listdir(self.chirps_path))
for station in self.stations:
uri = '{}/wa/istsos/services/{}/procedures/{}'.format(
station['base_url'], station['istsos_service'],
station['name'])
r = requests.get(uri,
auth=(station['istsos_user'],
station['istsos_pwd']))
station['metadata'] = r.json()['data']
r.close()
precipitation = None
total_days = 0
i_proc = 0
i = 0
try:
self.logger.info("MERGING CHIRPS AND {} STATION DATA".format(
station['name']))
i_stat = i
if (station['metadata']['location']['crs']['properties']
['name'].lower() != self.outProj):
# inProj = Proj(
# init=station['metadata']['location'][
# 'crs']['properties']['name'].lower()
# ) NOT WORKING CHECK SCRIPT WHERE DATA UPLOAD
inProj = 'epsg:4326'
outProj_ = Proj(init=self.outProj)
x1 = float(station['metadata']['location']['geometry']
['coordinates'][0])
y1 = float(station['metadata']['location']['geometry']
['coordinates'][1])
lon, lat = transform(inProj, self.outProj, x1, y1)
else:
lat = float(station['metadata']['location']['geometry']
['coordinates'][1])
lon = float(station['metadata']['location']['geometry']
['coordinates'][0])
lat_inds = []
lon_inds = []
for chirp_file in chirps_files:
file_path = os.path.join(self.chirps_path, chirp_file)
if 'v2.0' in file_path:
f = netCDF4.Dataset(file_path)
lats = f.variables['latitude'][:]
lons = f.variables['longitude'][:]
res_deg = 0.05
if not lat_inds or not lon_inds:
while (True):
lat_inds = np.where((lats >= lat - res_deg)
& (lats <= lat + res_deg))
if len(lat_inds[0]) == 1:
break
else:
res_deg -= 0.005
res_deg = 0.05
while (True):
lon_inds = np.where((lons >= lon - res_deg)
& (lons <= lon + res_deg))
if len(lon_inds[0]) == 1:
break
else:
res_deg -= 0.005
n_year_prec = f.variables[
'precip'][:, lat_inds[0][0], lon_inds[0][0]].data
if precipitation is None:
precipitation = n_year_prec
else:
precipitation = np.append(precipitation,
n_year_prec)
day_tmp = datetime.strptime('1981-1-1', '%Y-%m-%d')
days = []
for i in range(len(precipitation)):
days.append(day_tmp)
day_tmp = day_tmp + timedelta(days=1)
pd_index = pd.Index(days)
df = pd.DataFrame(precipitation,
columns=["rain_chirps"],
index=pd_index)
station['ts'].index = station['ts'].index.date
new_df = df.merge(station['ts'][:],
how='outer',
right_index=True,
left_index=True)
max_rain_limit = df['rain_chirps'].max() + (
df['rain_chirps'].max() / 2)
new_df.loc[pd.isnull(new_df['rain']), 'qi'] = 150
new_df.loc[pd.isnull(new_df['rain']),
'rain'] = new_df['rain_chirps']
new_df.loc[new_df['rain'] > max_rain_limit,
'rain'] = new_df['rain_chirps']
new_df.loc[pd.isnull(new_df['rain'])
& pd.isnull(new_df['rain_chirps']), 'qi'] = 100
new_df.loc[pd.isnull(new_df['rain'])
& pd.isnull(new_df['rain_chirps']), 'rain'] = 0
self.logger.info("SPI ELABORATION --> {}".format(
station['name']))
station['spi30'] = {
'data':
spi(new_df['rain'].values, 30, Distribution.pearson, 1981,
1981, new_df.index[-1].year - 1,
compute.Periodicity.daily),
'index':
new_df.index
}
indexes_values = station['spi30']['index'].strftime(
"%Y-%m-%dT00:00:00+0000").tolist()
spi30_values = list(
map(lambda x: self.format_value(x),
station['spi30']['data'].tolist()))
station['spi60'] = {
'data':
spi(new_df['rain'].values, 60, Distribution.pearson, 1981,
1981, new_df.index[-1].year - 1,
compute.Periodicity.daily),
'index':
new_df.index
}
spi60_values = list(
map(lambda x: self.format_value(x),
station['spi60']['data'].tolist()))
try:
# Load of a getobservation template from destination
res = requests.get(
("{}/wa/istsos/services/{}/operations/"
"getobservation/offerings/temporary/procedures/"
"{}/observedproperties/:/eventtime/"
"last?qualityIndex=False").format(
station['base_url'], station['istsos_service'],
station['name']),
params={"qualityIndex": False},
auth=self.basic_auth)
dtemplate = res.json()['data'][0]
res.close()
data_array = []
for i in range(len(indexes_values)):
data_array.append([
indexes_values[i], spi30_values[i], spi60_values[i]
])
dtemplate['result']['DataArray']['elementCount'] = str(
len(data_array))
dtemplate['result']['DataArray']['values'] = data_array
dtemplate["samplingTime"] = {}
dtemplate["samplingTime"][
"beginPosition"] = indexes_values[0]
dtemplate["samplingTime"]["endPosition"] = indexes_values[
-1]
# print(dtemplate['samplingTime'])
# POST data to WA
res = requests.post(
("{}/wa/istsos/services/"
"{}/operations/insertobservation").format(
station['base_url'], station['istsos_service']),
auth=self.basic_auth,
verify=True,
data=json.dumps({
"ForceInsert":
'true',
"AssignedSensorId":
station['metadata']['assignedSensorId'],
"Observation":
dtemplate
}))
res_json = res.json()
res.close()
if res_json['success']:
self.logger.info('SPI UPLOADED TO ISTSOS')
else:
self.logger.error(res_json)
except Exception as e:
msg = ("Error during %s GO syncing: %s" %
(station['name'], str(e)))
self.logger.error(msg)
except Exception as e:
self.logger.error("ERROR {} --> {}".format(
station['name'], str(e)))