def _sim(mean_center_rating, **unused_kwargs):
p = person(mean_center_rating)
count = ma.logical_not(ma.getmaskarray(mean_center_rating)).astype(np.int)
count = np.dot(count, count.T)
weight = np.fmin(count, beta) / beta
sim = p * weight
return sim
def statistic(y, y_hat):
rating_count, predict_count = y.count(), y_hat.count()
common_count = np.sum(
ma.logical_not(
ma.logical_or(ma.getmaskarray(y),
ma.getmaskarray(y_hat))).astype(np.int))
return {
"rating_count": rating_count,
"predict_count": predict_count,
"predict_rate": common_count / rating_count
}
def __backward__(self, hat_rating, rating, adjust_rating, adjust_factor):
_ngb_weight = self._weight * ma.logical_not(
ma.getmaskarray(adjust_rating)).astype(np.int)
_g_l = hat_rating - rating
_adjust_g_l = _g_l / adjust_factor
_adjust_g_l = _adjust_g_l[:, np.newaxis]
_g_m_bias = _g_l
_g_ngb_m_bias = -1.0 * _ngb_weight * _adjust_g_l
_g_ngb_n_bias = np.sum(
_g_l * (1 - np.sum(_ngb_weight, axis=1) / adjust_factor))
_g_weight = adjust_rating * _adjust_g_l
return _g_m_bias, _g_ngb_m_bias, _g_ngb_n_bias, _g_weight
def my_polyfit(x, y, deg, degatt=0):
k = ma.polyfit(x, y, degatt)
res = my_poly(k, x)
resid = ma.abs(res - y)
medresid = ma.median(resid, axis=0)
if y.ndim != 1:
mask = ma.logical_not(ma.sum((resid > 3 * medresid), axis=1).astype('bool'))
else:
mask = (resid < 3 * medresid)
y = y[mask]
x = x[mask]
k = ma.polyfit(x, y, deg)
return(k, mask)
#satellite-based land mask
sland_indices = sland_mask.nonzero()
nsland_pts = len(sland_indices[0])
print("number of satellite-caught land pts: ", nsland_pts)
# satellite-based water points: --------------------------------------
swater_indices = swater_mask.nonzero()
nswater_pts = len(swater_indices[0])
print("number of satellite-caught water pts: ", nswater_pts)
print("fraction of all points which are known after first pass: "******"nobs for round 2: ", nobs, flush=True)
landmask = ma.logical_and(landmask, unknown)
icemask = ma.logical_and(icemask, unknown)
watermask = ma.logical_and(watermask, unknown)
nicepts = len(icemask.nonzero()[0])
nlandpts = len(landmask.nonzero()[0])
nwaterpts = len(watermask.nonzero()[0])
pwater = nwaterpts / float(nobs)
pland = nlandpts / float(nobs)
pice = nicepts / float(nobs)
print("after first pass, nobs, ice, land, water ",
nobs,
areair2 = resize(areair2, sh)
arearf2 = resize(arearf2, sh)
areasum2 = areair2 + arearf2
areair1 = masked_array(zeros(sh), mask = mk)
arearf1 = masked_array(zeros(sh), mask = mk)
# no data -> all rainfed
nodata = logical_and(~mk, logical_or(areasum2.mask, areasum2 == 0))
idx1, idx2, idx3 = where(nodata)
areair1[idx1, idx2, idx3] = 0.
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3]
# has data
hasdata = logical_and(~mk, logical_not(nodata))
idx1, idx2, idx3 = where(hasdata)
areair1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * areair2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * arearf2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
with nc(outputfile, 'w') as f:
f.createDimension('time', sh[0])
tvar = f.createVariable('time', 'i4', 'time')
tvar[:] = range(0, sh[0])
tvar.units = 'years since %d' % year
tvar.long_name = 'time'
f.createDimension('lat', sh[1])
latvar = f.createVariable('lat', 'f8', 'lat')
latvar[:] = lats
latvar.units = 'degrees_north'
def _calc_half_htc(self, prcp_values, temp_values, threshold, start, end):
""" Calculates Selyaninov's hydrothermal coeffcient for a given time period.
Arguments:
prcp_values -- daily total precipitation values
temp_values -- daily mean temperature values
threshold -- threshold temperature (10C, by default)
start -- first index in the time grid, we start analysis from it (inclusive)
end -- last index in the time grid, we stop analysis before it (NOT inclusive)
If start <= stop algorithm runs forth in time. Runs back, otherwise.
Returns:
result -- array of Selyaninov's hydrothermal coefficient values
"""
# The idea is to subtract threshold from temperature values
# and to catch cases when values change sign from negative to positive (a transition)
# i.e., pass x-axis upwards.
# Each crossing corresponds to a possible beginning/ending
# (depends on the direction of the analysis) of the vegetation period.
# We sum values between crossings (in segments) and analyse sums according to Ped' (1951).
# Define a function to detect False->True transition.
trans = lambda a, b: ma.logical_and(ma.logical_not(a), b)
# Create shape for new arrays without time dimmension.
dims = list(temp_values.shape)
_ = dims.pop(0)
# Define some useful arrays.
temp_sums_1 = ma.zeros(dims) # Temperature sums for the first segment.
temp_sums_2 = ma.zeros(dims) # Temperature sums for the second segment.
temp_sums_3 = ma.zeros(dims) # Temperature sums for the third segment.
temp_cnt_1 = ma.zeros(dims) # Temperature counter for the first segment.
temp_cnt_2 = ma.zeros(dims) # Temperature counter for the second segment.
temp_cnt_3 = ma.zeros(dims) # Temperature counter for the third segment.
temp_total = ma.zeros(dims) # Temperature total sum for a vegetation period.
prcp_sums_1 = ma.zeros(dims) # Precipitation sums for the first segment.
prcp_sums_2 = ma.zeros(dims) # Precipitation sums for the second segment.
prcp_sums_3 = ma.zeros(dims) # Precipitation sums for the third segment.
prcp_total = ma.zeros(dims) # Precipitation total sum for a vegetation period.
trans_mask_1 = ma.zeros(dims) # Mask of cells in the first segment state
# (i.e., where the first transition was detected).
trans_mask_2 = ma.zeros(dims) # Mask of cells in the second segment state.
trans_mask_3 = ma.zeros(dims) # Mask of cells in the third segment state.
prev_pos_mask = ma.zeros(dims) # Matrix of previous positive mask state.
step = 1 if start <= end else -1 # Set step depending on start and stop values.
# Search for upward transitions and calculate sums.
for i in range(start, end, step):
cur_temp = temp_values[i]
cur_prcp = prcp_values[i]
temp_deviation = cur_temp - threshold
temp_pos_mask = temp_deviation >= 0 # Mask of positive values.
cur_trans = trans(prev_pos_mask, temp_pos_mask) # Detect negative->positive transition.
prev_pos_mask = temp_pos_mask # Store current positive mask for the next iteration.
# Turn on the third state, if a transition is detected and the second state is on.
# When the state is on, it is never off (provided by a boolean 'or').
trans_mask_3 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_3), trans_mask_2)
# Turn on the second state, if a transition is detected and the first state is on.
trans_mask_2 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_2), trans_mask_1)
# Turn on the first state, if a transition is detected or leave it as is.
trans_mask_1 = ma.logical_or(cur_trans, trans_mask_1)
# Sum values in the first segment.
seg_1_mask = ma.logical_and(trans_mask_1, ma.logical_not(trans_mask_2)) # Only for cells in the first segemnt state.
temp_sums_1[seg_1_mask] += temp_deviation[seg_1_mask]
temp_cnt_1[seg_1_mask] += 1
prcp_sums_1[seg_1_mask] += cur_prcp[seg_1_mask]
# Sum values in the second segment.
seg_2_mask = ma.logical_and(trans_mask_2, ma.logical_not(trans_mask_3)) # Only for cells in the second segemnt state.
temp_sums_2[seg_2_mask] += temp_deviation[seg_2_mask]
temp_cnt_2[seg_2_mask] += 1
prcp_sums_2[seg_2_mask] += cur_prcp[seg_2_mask]
# Sum values in the third segment (in fact, everything after the second segment).
temp_sums_3[trans_mask_3] += temp_deviation[trans_mask_3]
temp_cnt_3[trans_mask_3] += 1
prcp_sums_3[trans_mask_3] += cur_prcp[trans_mask_3]
# Create some intermediate sums.
temp_sums_12 = temp_sums_1 + temp_sums_2 # S1+S2+S3+S4
temp_cnt_12 = temp_cnt_1 + temp_cnt_2
temp_sums_23 = temp_sums_2 + temp_sums_3 # S3+S4+... all the rest
temp_cnt_23 = temp_cnt_2 + temp_cnt_3
temp_sums_123 = temp_sums_12 + temp_sums_3 # S1+S2+S3+S4+... all the rest
temp_cnt_123 = temp_cnt_12 + temp_cnt_3
# Check Ped's conditions and calculate temperature sums for the vegetation period.
# By adding temp_cnt_...*threshold we restore original values of the temperature.
# If vegetation period starts at the first transition point.
start_at_seg_1 = ma.logical_and(temp_sums_1 >= 0, temp_sums_12 >= 0) # S1 >= S2 and S1-S2+S3 >= S4
temp_total[start_at_seg_1] = temp_sums_123[start_at_seg_1] + temp_cnt_123[start_at_seg_1] * threshold
prcp_total[start_at_seg_1] = prcp_sums_1[start_at_seg_1] + prcp_sums_2[start_at_seg_1] + prcp_sums_3[start_at_seg_1]
# If vegetation period starts at the second transition point.
start_at_seg_2 = ma.logical_and(temp_sums_1 < 0, temp_sums_2 >= 0) # S1 < S2 and S3-S4 >= 0
temp_total[start_at_seg_2] = temp_sums_23[start_at_seg_2] + temp_cnt_23[start_at_seg_2] * threshold
prcp_total[start_at_seg_2] = prcp_sums_2[start_at_seg_2] + prcp_sums_3[start_at_seg_2]
# If vegetation period starts at the third transition point.
start_at_seg_3 = ma.logical_or(ma.logical_and(temp_sums_1 < 0, temp_sums_2 < 0), # S1 < S2 and S3 < S4
ma.logical_and(ma.logical_and(temp_sums_1 > 0, temp_sums_2 < 0), # or
temp_sums_12 < 0)) # S3 < S4 and S1 > S2 and S1-S2+S3 < S4
temp_total[start_at_seg_3] = temp_sums_3[start_at_seg_3] + temp_cnt_3[start_at_seg_3] * threshold
prcp_total[start_at_seg_3] = prcp_sums_3[start_at_seg_3]
return (prcp_total, temp_total)