コード例 #1
0
ファイル: submodel2.py プロジェクト: Adamantios/NN-Train
def cifar10_complicated_ensemble_submodel2_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    labels_array[logical_or(labels_array < 1, labels_array > 3)] = 0
    return 4
コード例 #2
0
def svhn_complicated_ensemble_v2_submodel2_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    # 1, 2, 3, 4 classes.
    labels_array[logical_or(labels_array < 1, labels_array > 4)] = 0
    return 5
コード例 #3
0
def nmf(V, Winit, Hinit, tol, timelimit, maxiter):
    """
    (W,H) = nmf(V,Winit,Hinit,tol,timelimit,maxiter)
    W,H: output solution
    Winit,Hinit: initial solution
    tol: tolerance for a relative stopping condition
    timelimit, maxiter: limit of time and iterations
    """

    W = Winit
    H = Hinit
    initt = time()

    gradW = dot(W, dot(H, H.T)) - dot(V, H.T)
    gradH = dot(dot(W.T, W), H) - dot(W.T, V)
    initgrad = norm(r_[gradW, gradH.T])
    print('Init gradient norm %f' % initgrad)
    tolW = max(0.001, tol) * initgrad
    tolH = tolW

    for iter in range(1, maxiter):
        # stopping condition
        projnorm = norm(r_[gradW[logical_or(gradW < 0, W > 0)],
                           gradH[logical_or(gradH < 0, H > 0)]])
        if projnorm < tol * initgrad or time() - initt > timelimit:
            break

        (W, gradW, iterW) = nlssubprob(V.T, H.T, W.T, tolW, 1000)
        W = W.T
        gradW = gradW.T

        if iterW == 1:
            tolW = 0.1 * tolW

        (H, gradH, iterH) = nlssubprob(V, W, H, tolH, 1000)
        if iterH == 1:
            tolH *= 0.1

        if iter % 10 == 0: stdout.write('.')

    print('\nIter = %d Final proj-grad norm %f' % (iter, projnorm))
    return W, H
コード例 #4
0
def svhn_complicated_ensemble_submodel4_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    labels_array[logical_or(labels_array < 6, labels_array > 7)] = 0
    labels_array[labels_array == 6] = 1
    labels_array[labels_array == 7] = 2
    return 3
コード例 #5
0
def statistic(y, y_hat):
    rating_count, predict_count = y.count(), y_hat.count()
    common_count = np.sum(
        ma.logical_not(
            ma.logical_or(ma.getmaskarray(y),
                          ma.getmaskarray(y_hat))).astype(np.int))
    return {
        "rating_count": rating_count,
        "predict_count": predict_count,
        "predict_rate": common_count / rating_count
    }
コード例 #6
0
ファイル: submodel3.py プロジェクト: Adamantios/NN-Train
def cifar10_complicated_ensemble_submodel3_labels_manipulation(
        labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    labels_array[logical_or(labels_array < 3, labels_array > 5)] = 0
    labels_array[labels_array == 3] = 1
    labels_array[labels_array == 4] = 2
    labels_array[labels_array == 5] = 3
    return 4
コード例 #7
0
ファイル: submodel3.py プロジェクト: Adamantios/NN-Train
def svhn_complicated_ensemble_v2_submodel3_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    # 4, 5, 6, 7 classes.
    labels_array[logical_or(labels_array < 4, labels_array > 7)] = 0
    labels_array[labels_array == 4] = 1
    labels_array[labels_array == 5] = 2
    labels_array[labels_array == 6] = 3
    labels_array[labels_array == 7] = 4
    return 5
コード例 #8
0
def nlssubprob(V, W, Hinit, tol, maxiter):
    """
    H, grad: output solution and gradient
    iter: #iterations used
    V, W: constant matrices
    Hinit: initial solution
    tol: stopping tolerance
    maxiter: limit of iterations
    """

    H = Hinit
    WtV = dot(W.T, V)
    WtW = dot(W.T, W)

    alpha = 1
    beta = 0.1
    for iter in range(1, maxiter):
        grad = dot(WtW, H) - WtV
        projgrad = norm(grad[logical_or(grad < 0, H > 0)])
        if projgrad < tol:
            break

        # search step size
        for inner_iter in range(1, 20):
            Hn = H - alpha * grad
            Hn = where(Hn > 0, Hn, 0)
            d = Hn - H
            gradd = sum(grad * d)
            dQd = sum(dot(WtW, d) * d)
            suff_decr = 0.99 * gradd + 0.5 * dQd < 0
            if inner_iter == 1:
                decr_alpha = not suff_decr
                Hp = H
            if decr_alpha:
                if suff_decr:
                    H = Hn
                    break
                else:
                    alpha *= beta
            else:
                if not suff_decr or (Hp == Hn).all():
                    H = Hp
                    break
                else:
                    alpha /= beta
                    Hp = Hn

        if iter == maxiter:
            print('Max iter in nlssubprob')
    return H, grad, iter
コード例 #9
0
def cifar100_complicated_ensemble_submodel2_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    labels_array[logical_or(labels_array < 20, labels_array > 49)] = -1

    for i, label_i in enumerate(range(20, 50)):
        labels_array[labels_array == label_i] = i + 1

    labels_array[labels_array == -1] = 0

    return 31
コード例 #10
0
def caltech_complicated_ensemble_submodel4_labels_manipulation(labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    labels_array[logical_or(labels_array < 62, labels_array > 81)] = -1

    for i, label_i in enumerate(range(62, 82)):
        labels_array[labels_array == label_i] = i

    labels_array[labels_array == -1] = 20

    return 21
コード例 #11
0
def omniglot_complicated_ensemble_submodel2_labels_manipulation(
        labels_array: ndarray) -> int:
    """
    The model's labels manipulator.

    :param labels_array: the labels to manipulate.
    :return: the number of classes predicted by the model.
    """
    # Labels [324 - 648].
    labels_array[logical_or(labels_array < 324, labels_array > 648)] = -1

    for i, label_i in enumerate(range(324, 649)):
        labels_array[labels_array == label_i] = i

    labels_array[labels_array == -1] = 325

    return 326
コード例 #12
0
    def _calc_gsl(self, values, threshold):
        """ Calculates GSL for the given values"""

        data_shape = values.shape[1:]
        gsl_cnt = ma.zeros(data_shape)
        warm_cnt = np.zeros(data_shape)
        cold_cnt = np.zeros(data_shape)
        increment = np.zeros(data_shape)
        for i, arr in enumerate(values):
            if i < 183:  # Take the first half of an year.
                mask = arr > threshold
                warm_cnt += mask  # Count warm days.
                warm_cnt *= mask  # Reset counter of warm days on a cold one.
                increment = ma.logical_or(increment, (warm_cnt == 5))  # Search for cells with 5 consecutive warm days.
            else:  # Take the second part of an year.
                mask = arr < threshold
                cold_cnt += mask  # Count cold days.
                cold_cnt *= mask  # Reset counter of cold days on a warm one.
                increment = ma.logical_and(increment, ~(cold_cnt == 5))  # Search for cells with 5 consecutive cold days.
            gsl_cnt += increment  # Count days inside GSL at each cell.

        gsl_cnt.mask = values.mask[0]  # Take source mask.

        return gsl_cnt
コード例 #13
0
#----------------------------------------------------------------

#----------------------------------------------------------------
#  Next step, pre-filter based on perfect land filters, perfect ice-free ocean filters
#  Exclude 22v because of F15
#
# Land points:
#satellite-based land mask
#Start with false everywhere and then 'or' in the trues:
sland_mask = ma.masked_array(t19h > 3000)
swater_mask = ma.masked_array(t19h > 3000)

#good not-ice (<~ 1.6%), very good not-land (~0.3%), water flag: (98.7%, 1.76 million pts)
tmp = ma.masked_array(delta(t19v, t85v) < -0.090792151111545)
swater_mask = ma.logical_or(swater_mask, tmp)
tmp = ma.masked_array(delta(t19h, t85h) < -0.1756696439150608)
swater_mask = ma.logical_or(swater_mask, tmp)

#Perfect land mask, 200k pts
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19h > 252))

#good land mask, ~140k pts
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h < 166))
sland_mask = ma.logical_or(
    sland_mask, ma.masked_array(delta(t19v, t19h) < 0.018077900915434868))
#land mask, ~60k pts
sland_mask = ma.logical_or(
    sland_mask, ma.masked_array(delta(t19h, t37v) > 0.05150437355041504))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37v > 261))
コード例 #14
0
    area = masked_where(mk, area) # mask
    areasum1 = areasum1 * area / 100.

with nc(weightfile) as f:
    areair2 = f.variables['irrigated'][:]
    arearf2 = f.variables['rainfed'][:]

areair2  = resize(areair2, sh)
arearf2  = resize(arearf2, sh)
areasum2 = areair2 + arearf2

areair1 = masked_array(zeros(sh), mask = mk)
arearf1 = masked_array(zeros(sh), mask = mk)

# no data -> all rainfed
nodata = logical_and(~mk, logical_or(areasum2.mask, areasum2 == 0))
idx1, idx2, idx3 = where(nodata)
areair1[idx1, idx2, idx3] = 0.
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3]

# has data
hasdata = logical_and(~mk, logical_not(nodata))
idx1, idx2, idx3 = where(hasdata)
areair1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * areair2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]
arearf1[idx1, idx2, idx3] = areasum1[idx1, idx2, idx3] * arearf2[idx1, idx2, idx3] / areasum2[idx1, idx2, idx3]

with nc(outputfile, 'w') as f:
    f.createDimension('time', sh[0])
    tvar = f.createVariable('time', 'i4', 'time')
    tvar[:] = range(0, sh[0])
    tvar.units = 'years since %d' % year
コード例 #15
0
ファイル: run_algs.py プロジェクト: rgrumbine/concentration
      nwaterpts / float(nobs),
      nobs,
      flush=True)
pwater = nwaterpts / float(nobs)
pland = nlandpts / float(nobs)
pice = nicepts / float(nobs)

#----------------------------------------------------------------
#  Next step, pre-filter based on perfect land filters, perfect ice-free ocean filters
#  Exclude 22v because of F15
# Land points:
#satellite-based land mask
#Start with false everywhere and then 'or' in the trues:
sland_mask = ma.masked_array(t19h > 3000)

sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19v > 265))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19h > 252))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37v > 267))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t37h > 260))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85v > 281))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h > 278))
tmp = ma.masked_array(delta(t19h, t37v) > 0.07100180784861249)
sland_mask = ma.logical_or(sland_mask, tmp)
tmp = ma.masked_array(delta(t19v, t19h) < 0.006025966971811622)
sland_mask = ma.logical_or(sland_mask, tmp)
tmp = ma.masked_array(delta(t37v, t37h) < 0.0024425569507810804)
sland_mask = ma.logical_or(sland_mask, tmp)

sland_mask = ma.logical_or(sland_mask, ma.masked_array(t19v < 173))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85v < 152))
sland_mask = ma.logical_or(sland_mask, ma.masked_array(t85h < 146))
コード例 #16
0
    def _calc_half_htc(self, prcp_values, temp_values, threshold, start, end):
        """ Calculates Selyaninov's hydrothermal coeffcient for a given time period.
        Arguments:
            prcp_values -- daily total precipitation values
            temp_values -- daily mean temperature values
            threshold -- threshold temperature (10C, by default)
            start -- first index in the time grid, we start analysis from it (inclusive)
            end -- last index in the time grid, we stop analysis before it (NOT inclusive)
            If start <= stop algorithm runs forth in time. Runs back, otherwise.
        Returns:
            result -- array of Selyaninov's hydrothermal coefficient values
        """
        # The idea is to subtract threshold from temperature values
        #  and to catch cases when values change sign from negative to positive (a transition)
        #  i.e., pass x-axis upwards.
        # Each crossing corresponds to a possible beginning/ending
        # (depends on the direction of the analysis) of the vegetation period.
        # We sum values between crossings (in segments) and analyse sums according to Ped' (1951).

        # Define a function to detect False->True transition.
        trans = lambda a, b: ma.logical_and(ma.logical_not(a), b)

        # Create shape for new arrays without time dimmension.
        dims = list(temp_values.shape)
        _ = dims.pop(0)

        # Define some useful arrays.
        temp_sums_1 = ma.zeros(dims)  # Temperature sums for the first segment.
        temp_sums_2 = ma.zeros(dims)  # Temperature sums for the second segment.
        temp_sums_3 = ma.zeros(dims)  # Temperature sums for the third segment.
        temp_cnt_1 = ma.zeros(dims)  # Temperature counter for the first segment.
        temp_cnt_2 = ma.zeros(dims)  # Temperature counter for the second segment.
        temp_cnt_3 = ma.zeros(dims)  # Temperature counter for the third segment.
        temp_total = ma.zeros(dims)   # Temperature total sum for a vegetation period.
        prcp_sums_1 = ma.zeros(dims)  # Precipitation sums for the first segment.
        prcp_sums_2 = ma.zeros(dims)  # Precipitation sums for the second segment.
        prcp_sums_3 = ma.zeros(dims)  # Precipitation sums for the third segment.
        prcp_total = ma.zeros(dims)   # Precipitation total sum for a vegetation period.

        trans_mask_1 = ma.zeros(dims)  # Mask of cells in the first segment state
                                      #  (i.e., where the first transition was detected).
        trans_mask_2 = ma.zeros(dims)  # Mask of cells in the second segment state.
        trans_mask_3 = ma.zeros(dims)  # Mask of cells in the third segment state.
        prev_pos_mask = ma.zeros(dims)  # Matrix of previous positive mask state.

        step = 1 if start <= end else -1  # Set step depending on start and stop values.

        # Search for upward transitions and calculate sums.
        for i in range(start, end, step):
            cur_temp = temp_values[i]
            cur_prcp = prcp_values[i]
            temp_deviation = cur_temp - threshold
            temp_pos_mask = temp_deviation >= 0  # Mask of positive values.
            cur_trans = trans(prev_pos_mask, temp_pos_mask)  # Detect negative->positive transition.
            prev_pos_mask = temp_pos_mask  # Store current positive mask for the next iteration.
            # Turn on the third state, if a transition is detected and the second state is on.
            # When the state is on, it is never off (provided by a boolean 'or').
            trans_mask_3 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_3), trans_mask_2)
            # Turn on the second state, if a transition is detected and the first state is on.
            trans_mask_2 = ma.logical_and(ma.logical_or(cur_trans, trans_mask_2), trans_mask_1)
            # Turn on the first state, if a transition is detected or leave it as is.
            trans_mask_1 = ma.logical_or(cur_trans, trans_mask_1)
            # Sum values in the first segment.
            seg_1_mask = ma.logical_and(trans_mask_1, ma.logical_not(trans_mask_2))  # Only for cells in the first segemnt state.
            temp_sums_1[seg_1_mask] += temp_deviation[seg_1_mask]
            temp_cnt_1[seg_1_mask] += 1
            prcp_sums_1[seg_1_mask] += cur_prcp[seg_1_mask]
            # Sum values in the second segment.
            seg_2_mask = ma.logical_and(trans_mask_2, ma.logical_not(trans_mask_3))  # Only for cells in the second segemnt state.
            temp_sums_2[seg_2_mask] += temp_deviation[seg_2_mask]
            temp_cnt_2[seg_2_mask] += 1
            prcp_sums_2[seg_2_mask] += cur_prcp[seg_2_mask]
            # Sum values in the third segment (in fact, everything after the second segment).
            temp_sums_3[trans_mask_3] += temp_deviation[trans_mask_3]
            temp_cnt_3[trans_mask_3] += 1
            prcp_sums_3[trans_mask_3] += cur_prcp[trans_mask_3]

        # Create some intermediate sums.
        temp_sums_12 = temp_sums_1 + temp_sums_2  # S1+S2+S3+S4
        temp_cnt_12 = temp_cnt_1 + temp_cnt_2
        temp_sums_23 = temp_sums_2 + temp_sums_3  # S3+S4+... all the rest
        temp_cnt_23 = temp_cnt_2 + temp_cnt_3
        temp_sums_123 = temp_sums_12 + temp_sums_3  # S1+S2+S3+S4+... all the rest
        temp_cnt_123 = temp_cnt_12 + temp_cnt_3

        # Check Ped's conditions and calculate temperature sums for the vegetation period.
        # By adding temp_cnt_...*threshold we restore original values of the temperature.
        # If vegetation period starts at the first transition point.
        start_at_seg_1 = ma.logical_and(temp_sums_1 >= 0, temp_sums_12 >= 0)  # S1 >= S2 and S1-S2+S3 >= S4
        temp_total[start_at_seg_1] = temp_sums_123[start_at_seg_1] + temp_cnt_123[start_at_seg_1] * threshold
        prcp_total[start_at_seg_1] = prcp_sums_1[start_at_seg_1] + prcp_sums_2[start_at_seg_1] + prcp_sums_3[start_at_seg_1]
        # If vegetation period starts at the second transition point.
        start_at_seg_2 = ma.logical_and(temp_sums_1 < 0, temp_sums_2 >= 0)  # S1 < S2 and S3-S4 >= 0
        temp_total[start_at_seg_2] = temp_sums_23[start_at_seg_2] + temp_cnt_23[start_at_seg_2] * threshold
        prcp_total[start_at_seg_2] = prcp_sums_2[start_at_seg_2] + prcp_sums_3[start_at_seg_2]
        # If vegetation period starts at the third transition point.
        start_at_seg_3 = ma.logical_or(ma.logical_and(temp_sums_1 < 0, temp_sums_2 < 0),  # S1 < S2 and S3 < S4
                                       ma.logical_and(ma.logical_and(temp_sums_1 > 0, temp_sums_2 < 0),  # or
                                                      temp_sums_12 < 0))  # S3 < S4 and S1 > S2 and S1-S2+S3 < S4
        temp_total[start_at_seg_3] = temp_sums_3[start_at_seg_3] + temp_cnt_3[start_at_seg_3] * threshold
        prcp_total[start_at_seg_3] = prcp_sums_3[start_at_seg_3]

        return (prcp_total, temp_total)