def cifar10_complicated_ensemble_submodel2_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
labels_array[logical_or(labels_array < 1, labels_array > 3)] = 0
return 4
def svhn_complicated_ensemble_v2_submodel2_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
# 1, 2, 3, 4 classes.
labels_array[logical_or(labels_array < 1, labels_array > 4)] = 0
return 5
def nmf(V, Winit, Hinit, tol, timelimit, maxiter):
"""
(W,H) = nmf(V,Winit,Hinit,tol,timelimit,maxiter)
W,H: output solution
Winit,Hinit: initial solution
tol: tolerance for a relative stopping condition
timelimit, maxiter: limit of time and iterations
"""
W = Winit
H = Hinit
initt = time()
gradW = dot(W, dot(H, H.T)) - dot(V, H.T)
gradH = dot(dot(W.T, W), H) - dot(W.T, V)
initgrad = norm(r_[gradW, gradH.T])
print('Init gradient norm %f' % initgrad)
tolW = max(0.001, tol) * initgrad
tolH = tolW
for iter in range(1, maxiter):
# stopping condition
projnorm = norm(r_[gradW[logical_or(gradW < 0, W > 0)],
gradH[logical_or(gradH < 0, H > 0)]])
if projnorm < tol * initgrad or time() - initt > timelimit:
break
(W, gradW, iterW) = nlssubprob(V.T, H.T, W.T, tolW, 1000)
W = W.T
gradW = gradW.T
if iterW == 1:
tolW = 0.1 * tolW
(H, gradH, iterH) = nlssubprob(V, W, H, tolH, 1000)
if iterH == 1:
tolH *= 0.1
if iter % 10 == 0: stdout.write('.')
print('\nIter = %d Final proj-grad norm %f' % (iter, projnorm))
return W, H
def svhn_complicated_ensemble_submodel4_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
labels_array[logical_or(labels_array < 6, labels_array > 7)] = 0
labels_array[labels_array == 6] = 1
labels_array[labels_array == 7] = 2
return 3
def statistic(y, y_hat):
rating_count, predict_count = y.count(), y_hat.count()
common_count = np.sum(
ma.logical_not(
ma.logical_or(ma.getmaskarray(y),
ma.getmaskarray(y_hat))).astype(np.int))
return {
"rating_count": rating_count,
"predict_count": predict_count,
"predict_rate": common_count / rating_count
}
def cifar10_complicated_ensemble_submodel3_labels_manipulation(
labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
labels_array[logical_or(labels_array < 3, labels_array > 5)] = 0
labels_array[labels_array == 3] = 1
labels_array[labels_array == 4] = 2
labels_array[labels_array == 5] = 3
return 4
def svhn_complicated_ensemble_v2_submodel3_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
# 4, 5, 6, 7 classes.
labels_array[logical_or(labels_array < 4, labels_array > 7)] = 0
labels_array[labels_array == 4] = 1
labels_array[labels_array == 5] = 2
labels_array[labels_array == 6] = 3
labels_array[labels_array == 7] = 4
return 5
def nlssubprob(V, W, Hinit, tol, maxiter):
"""
H, grad: output solution and gradient
iter: #iterations used
V, W: constant matrices
Hinit: initial solution
tol: stopping tolerance
maxiter: limit of iterations
"""
H = Hinit
WtV = dot(W.T, V)
WtW = dot(W.T, W)
alpha = 1
beta = 0.1
for iter in range(1, maxiter):
grad = dot(WtW, H) - WtV
projgrad = norm(grad[logical_or(grad < 0, H > 0)])
if projgrad < tol:
break
# search step size
for inner_iter in range(1, 20):
Hn = H - alpha * grad
Hn = where(Hn > 0, Hn, 0)
d = Hn - H
gradd = sum(grad * d)
dQd = sum(dot(WtW, d) * d)
suff_decr = 0.99 * gradd + 0.5 * dQd < 0
if inner_iter == 1:
decr_alpha = not suff_decr
Hp = H
if decr_alpha:
if suff_decr:
H = Hn
break
else:
alpha *= beta
else:
if not suff_decr or (Hp == Hn).all():
H = Hp
break
else:
alpha /= beta
Hp = Hn
if iter == maxiter:
print('Max iter in nlssubprob')
return H, grad, iter
def cifar100_complicated_ensemble_submodel2_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
labels_array[logical_or(labels_array < 20, labels_array > 49)] = -1
for i, label_i in enumerate(range(20, 50)):
labels_array[labels_array == label_i] = i + 1
labels_array[labels_array == -1] = 0
return 31
def caltech_complicated_ensemble_submodel4_labels_manipulation(labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
labels_array[logical_or(labels_array < 62, labels_array > 81)] = -1
for i, label_i in enumerate(range(62, 82)):
labels_array[labels_array == label_i] = i
labels_array[labels_array == -1] = 20
return 21
def omniglot_complicated_ensemble_submodel2_labels_manipulation(
labels_array: ndarray) -> int:
"""
The model's labels manipulator.
:param labels_array: the labels to manipulate.
:return: the number of classes predicted by the model.
"""
# Labels [324 - 648].
labels_array[logical_or(labels_array < 324, labels_array > 648)] = -1
for i, label_i in enumerate(range(324, 649)):
labels_array[labels_array == label_i] = i
labels_array[labels_array == -1] = 325
return 326
def _calc_gsl(self, values, threshold):
""" Calculates GSL for the given values"""
data_shape = values.shape[1:]
gsl_cnt = ma.zeros(data_shape)
warm_cnt = np.zeros(data_shape)
cold_cnt = np.zeros(data_shape)
increment = np.zeros(data_shape)
for i, arr in enumerate(values):
if i < 183: # Take the first half of an year.
mask = arr > threshold
warm_cnt += mask # Count warm days.
warm_cnt *= mask # Reset counter of warm days on a cold one.
increment = ma.logical_or(increment, (warm_cnt == 5)) # Search for cells with 5 consecutive warm days.
else: # Take the second part of an year.
mask = arr < threshold
cold_cnt += mask # Count cold days.
cold_cnt *= mask # Reset counter of cold days on a warm one.
increment = ma.logical_and(increment, ~(cold_cnt == 5)) # Search for cells with 5 consecutive cold days.
gsl_cnt += increment # Count days inside GSL at each cell.
gsl_cnt.mask = values.mask[0] # Take source mask.
return gsl_cnt
#----------------------------------------------------------------
#----------------------------------------------------------------
# Next step, pre-filter based on perfect land filters, perfect ice-free ocean filters
# Exclude 22v because of F15
#
# Land points:
#satellite-based land mask
#Start with false everywhere and then 'or' in the trues:
sland_mask = ma.masked_array(t19h > 3000)
swater_mask = ma.masked_array(t19h > 3000)
#good not-ice (<~ 1.6%), very good not-land (~0.3%), water flag: (98.7%, 1.76 million pts)
tmp = ma.masked_array(delta(t19v, t85v) < -0.090792151111545)
swater_mask = ma.logical_or(swater_mask, tmp)
tmp = ma.masked_array(delta(t19h, t85h) < -0.1756696439150608)
swater_mask = ma.logical_or(swater_mask, tmp)
#Perfect land mask, 200k pts
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19h > 252))
#good land mask, ~140k pts
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h < 166))
sland_mask = ma.logical_or(
sland_mask, ma.masked_array(delta(t19v, t19h) < 0.018077900915434868))
#land mask, ~60k pts
sland_mask = ma.logical_or(
sland_mask, ma.masked_array(delta(t19h, t37v) > 0.05150437355041504))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37v > 261))
area = masked_where(mk, area) # mask
areasum1 = areasum1 * area / 100.
with nc(weightfile) as f:
areair2 = f.variables['irrigated'][:]
arearf2 = f.variables['rainfed'][:]
areair2 = resize(areair2, sh)
arearf2 = resize(arearf2, sh)
areasum2 = areair2 + arearf2
areair1 = masked_array(zeros(sh), mask = mk)
arearf1 = masked_array(zeros(sh), mask = mk)
# no data -> all rainfed
nodata = logical_and(~mk, logical_or(areasum2.mask, areasum2 == 0))
idx1, idx2, idx3 = where(nodata)
areair1[idx1, idx2, idx3] = 0.
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3]
# has data
hasdata = logical_and(~mk, logical_not(nodata))
idx1, idx2, idx3 = where(hasdata)
areair1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * areair2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * arearf2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
with nc(outputfile, 'w') as f:
f.createDimension('time', sh[0])
tvar = f.createVariable('time', 'i4', 'time')
tvar[:] = range(0, sh[0])
tvar.units = 'years since %d' % year
nwaterpts / float(nobs),
nobs,
flush=True)
pwater = nwaterpts / float(nobs)
pland = nlandpts / float(nobs)
pice = nicepts / float(nobs)
#----------------------------------------------------------------
# Next step, pre-filter based on perfect land filters, perfect ice-free ocean filters
# Exclude 22v because of F15
# Land points:
#satellite-based land mask
#Start with false everywhere and then 'or' in the trues:
sland_mask = ma.masked_array(t19h > 3000)
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19v > 265))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19h > 252))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37v > 267))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37h > 260))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85v > 281))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h > 278))
tmp = ma.masked_array(delta(t19h, t37v) > 0.07100180784861249)
sland_mask = ma.logical_or(sland_mask, tmp)
tmp = ma.masked_array(delta(t19v, t19h) < 0.006025966971811622)
sland_mask = ma.logical_or(sland_mask, tmp)
tmp = ma.masked_array(delta(t37v, t37h) < 0.0024425569507810804)
sland_mask = ma.logical_or(sland_mask, tmp)
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19v < 173))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85v < 152))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h < 146))
def _calc_half_htc(self, prcp_values, temp_values, threshold, start, end):
""" Calculates Selyaninov's hydrothermal coeffcient for a given time period.
Arguments:
prcp_values -- daily total precipitation values
temp_values -- daily mean temperature values
threshold -- threshold temperature (10C, by default)
start -- first index in the time grid, we start analysis from it (inclusive)
end -- last index in the time grid, we stop analysis before it (NOT inclusive)
If start <= stop algorithm runs forth in time. Runs back, otherwise.
Returns:
result -- array of Selyaninov's hydrothermal coefficient values
"""
# The idea is to subtract threshold from temperature values
# and to catch cases when values change sign from negative to positive (a transition)
# i.e., pass x-axis upwards.
# Each crossing corresponds to a possible beginning/ending
# (depends on the direction of the analysis) of the vegetation period.
# We sum values between crossings (in segments) and analyse sums according to Ped' (1951).
# Define a function to detect False->True transition.
trans = lambda a, b: ma.logical_and(ma.logical_not(a), b)
# Create shape for new arrays without time dimmension.
dims = list(temp_values.shape)
_ = dims.pop(0)
# Define some useful arrays.
temp_sums_1 = ma.zeros(dims) # Temperature sums for the first segment.
temp_sums_2 = ma.zeros(dims) # Temperature sums for the second segment.
temp_sums_3 = ma.zeros(dims) # Temperature sums for the third segment.
temp_cnt_1 = ma.zeros(dims) # Temperature counter for the first segment.
temp_cnt_2 = ma.zeros(dims) # Temperature counter for the second segment.
temp_cnt_3 = ma.zeros(dims) # Temperature counter for the third segment.
temp_total = ma.zeros(dims) # Temperature total sum for a vegetation period.
prcp_sums_1 = ma.zeros(dims) # Precipitation sums for the first segment.
prcp_sums_2 = ma.zeros(dims) # Precipitation sums for the second segment.
prcp_sums_3 = ma.zeros(dims) # Precipitation sums for the third segment.
prcp_total = ma.zeros(dims) # Precipitation total sum for a vegetation period.
trans_mask_1 = ma.zeros(dims) # Mask of cells in the first segment state
# (i.e., where the first transition was detected).
trans_mask_2 = ma.zeros(dims) # Mask of cells in the second segment state.
trans_mask_3 = ma.zeros(dims) # Mask of cells in the third segment state.
prev_pos_mask = ma.zeros(dims) # Matrix of previous positive mask state.
step = 1 if start <= end else -1 # Set step depending on start and stop values.
# Search for upward transitions and calculate sums.
for i in range(start, end, step):
cur_temp = temp_values[i]
cur_prcp = prcp_values[i]
temp_deviation = cur_temp - threshold
temp_pos_mask = temp_deviation >= 0 # Mask of positive values.
cur_trans = trans(prev_pos_mask, temp_pos_mask) # Detect negative->positive transition.
prev_pos_mask = temp_pos_mask # Store current positive mask for the next iteration.
# Turn on the third state, if a transition is detected and the second state is on.
# When the state is on, it is never off (provided by a boolean 'or').
trans_mask_3 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_3), trans_mask_2)
# Turn on the second state, if a transition is detected and the first state is on.
trans_mask_2 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_2), trans_mask_1)
# Turn on the first state, if a transition is detected or leave it as is.
trans_mask_1 = ma.logical_or(cur_trans, trans_mask_1)
# Sum values in the first segment.
seg_1_mask = ma.logical_and(trans_mask_1, ma.logical_not(trans_mask_2)) # Only for cells in the first segemnt state.
temp_sums_1[seg_1_mask] += temp_deviation[seg_1_mask]
temp_cnt_1[seg_1_mask] += 1
prcp_sums_1[seg_1_mask] += cur_prcp[seg_1_mask]
# Sum values in the second segment.
seg_2_mask = ma.logical_and(trans_mask_2, ma.logical_not(trans_mask_3)) # Only for cells in the second segemnt state.
temp_sums_2[seg_2_mask] += temp_deviation[seg_2_mask]
temp_cnt_2[seg_2_mask] += 1
prcp_sums_2[seg_2_mask] += cur_prcp[seg_2_mask]
# Sum values in the third segment (in fact, everything after the second segment).
temp_sums_3[trans_mask_3] += temp_deviation[trans_mask_3]
temp_cnt_3[trans_mask_3] += 1
prcp_sums_3[trans_mask_3] += cur_prcp[trans_mask_3]
# Create some intermediate sums.
temp_sums_12 = temp_sums_1 + temp_sums_2 # S1+S2+S3+S4
temp_cnt_12 = temp_cnt_1 + temp_cnt_2
temp_sums_23 = temp_sums_2 + temp_sums_3 # S3+S4+... all the rest
temp_cnt_23 = temp_cnt_2 + temp_cnt_3
temp_sums_123 = temp_sums_12 + temp_sums_3 # S1+S2+S3+S4+... all the rest
temp_cnt_123 = temp_cnt_12 + temp_cnt_3
# Check Ped's conditions and calculate temperature sums for the vegetation period.
# By adding temp_cnt_...*threshold we restore original values of the temperature.
# If vegetation period starts at the first transition point.
start_at_seg_1 = ma.logical_and(temp_sums_1 >= 0, temp_sums_12 >= 0) # S1 >= S2 and S1-S2+S3 >= S4
temp_total[start_at_seg_1] = temp_sums_123[start_at_seg_1] + temp_cnt_123[start_at_seg_1] * threshold
prcp_total[start_at_seg_1] = prcp_sums_1[start_at_seg_1] + prcp_sums_2[start_at_seg_1] + prcp_sums_3[start_at_seg_1]
# If vegetation period starts at the second transition point.
start_at_seg_2 = ma.logical_and(temp_sums_1 < 0, temp_sums_2 >= 0) # S1 < S2 and S3-S4 >= 0
temp_total[start_at_seg_2] = temp_sums_23[start_at_seg_2] + temp_cnt_23[start_at_seg_2] * threshold
prcp_total[start_at_seg_2] = prcp_sums_2[start_at_seg_2] + prcp_sums_3[start_at_seg_2]
# If vegetation period starts at the third transition point.
start_at_seg_3 = ma.logical_or(ma.logical_and(temp_sums_1 < 0, temp_sums_2 < 0), # S1 < S2 and S3 < S4
ma.logical_and(ma.logical_and(temp_sums_1 > 0, temp_sums_2 < 0), # or
temp_sums_12 < 0)) # S3 < S4 and S1 > S2 and S1-S2+S3 < S4
temp_total[start_at_seg_3] = temp_sums_3[start_at_seg_3] + temp_cnt_3[start_at_seg_3] * threshold
prcp_total[start_at_seg_3] = prcp_sums_3[start_at_seg_3]
return (prcp_total, temp_total)