def test_subclasses(): # test that subclass is preserved only if subok=True a = VerySimpleSubClass([1, 2, 3, 4]) assert_(type(a) is VerySimpleSubClass) a_view = as_strided(a, shape=(2,), strides=(2 * a.itemsize,)) assert_(type(a_view) is np.ndarray) a_view = as_strided(a, shape=(2,), strides=(2 * a.itemsize,), subok=True) assert_(type(a_view) is VerySimpleSubClass) # test that if a subclass has __array_finalize__, it is used a = SimpleSubClass([1, 2, 3, 4]) a_view = as_strided(a, shape=(2,), strides=(2 * a.itemsize,), subok=True) assert_(type(a_view) is SimpleSubClass) assert_(a_view.info == 'simple finalized') # similar tests for broadcast_arrays b = np.arange(len(a)).reshape(-1, 1) a_view, b_view = broadcast_arrays(a, b) assert_(type(a_view) is np.ndarray) assert_(type(b_view) is np.ndarray) assert_(a_view.shape == b_view.shape) a_view, b_view = broadcast_arrays(a, b, subok=True) assert_(type(a_view) is SimpleSubClass) assert_(a_view.info == 'simple finalized') assert_(type(b_view) is np.ndarray) assert_(a_view.shape == b_view.shape) # and for broadcast_to shape = (2, 4) a_view = broadcast_to(a, shape) assert_(type(a_view) is np.ndarray) assert_(a_view.shape == shape) a_view = broadcast_to(a, shape, subok=True) assert_(type(a_view) is SimpleSubClass) assert_(a_view.info == 'simple finalized') assert_(a_view.shape == shape)
def test_writeable(): # broadcast_to should return a readonly array original = np.array([1, 2, 3]) result = broadcast_to(original, (2, 3)) assert_equal(result.flags.writeable, False) assert_raises(ValueError, result.__setitem__, slice(None), 0) # but the result of broadcast_arrays needs to be writeable (for now), to # preserve backwards compatibility for results in [broadcast_arrays(original), broadcast_arrays(0, original)]: for result in results: assert_equal(result.flags.writeable, True) # keep readonly input readonly original.flags.writeable = False _, result = broadcast_arrays(0, original) assert_equal(result.flags.writeable, False) # regression test for GH6491 shape = (2,) strides = [0] tricky_array = as_strided(np.array(0), shape, strides) other = np.zeros((1,)) first, second = broadcast_arrays(tricky_array, other) assert_(first.shape == second.shape)
def test_broadcast_to_succeeds(): data = [ [np.array(0), (0, ), np.array(0)], [np.array(0), (1, ), np.zeros(1)], [np.array(0), (3, ), np.zeros(3)], [np.ones(1), (1, ), np.ones(1)], [np.ones(1), (2, ), np.ones(2)], [np.ones(1), (1, 2, 3), np.ones((1, 2, 3))], [np.arange(3), (3, ), np.arange(3)], [np.arange(3), (1, 3), np.arange(3).reshape(1, -1)], [np.arange(3), (2, 3), np.array([[0, 1, 2], [0, 1, 2]])], # test if shape is not a tuple [np.ones(0), 0, np.ones(0)], [np.ones(1), 1, np.ones(1)], [np.ones(1), 2, np.ones(2)], # these cases with size 0 are strange, but they reproduce the behavior # of broadcasting with ufuncs (see test_same_as_ufunc above) [np.ones(1), (0, ), np.ones(0)], [np.ones((1, 2)), (0, 2), np.ones((0, 2))], [np.ones((2, 1)), (2, 0), np.ones((2, 0))], ] for input_array, shape, expected in data: actual = broadcast_to(input_array, shape) assert_array_equal(expected, actual)
def test_reference_types(): input_array = np.array('a', dtype=object) expected = np.array(['a'] * 3, dtype=object) actual = broadcast_to(input_array, (3,)) assert_array_equal(expected, actual) actual, _ = broadcast_arrays(input_array, np.ones(3)) assert_array_equal(expected, actual)
def test_reference_types(): input_array = np.array('a', dtype=object) expected = np.array(['a'] * 3, dtype=object) actual = broadcast_to(input_array, (3, )) assert_array_equal(expected, actual) actual, _ = broadcast_arrays(input_array, np.ones(3)) assert_array_equal(expected, actual)
def expand_1d_to_2d_array(array, length, axis=0): """ General utility routine to 'extend arbitrary dimensional array into a higher dimension by duplicating the data along a given 'axis' (default is 0) of size 'length'. Examples:: >>> a = np.array([1, 2, 3, 4]) >>> expand_1d_to_2d_array(a, 4, axis=0) [[1 2 3 4] [1 2 3 4] [1 2 3 4] [1 2 3 4]] >>> a = np.array([1, 2, 3, 4]) >>> expand_1d_to_2d_array(a, 4, axis=1) [[1 1 1 1] [2 2 2 2] [3 3 3 3] [4 4 4 4]] :param array: :param length: :param axis: :return: """ from numpy.lib.stride_tricks import broadcast_to if axis == 0: array_2d = broadcast_to(array, (length, array.size)) # array_2d = np.lib.stride_tricks.as_strided(array_1d, (length, array_1d.size), (0, array_1d.itemsize)) else: reshaped = array.reshape(array.size, 1) array_2d = broadcast_to(reshaped, (array.size, length)) # array_2d = np.lib.stride_tricks.as_strided(array_1d, (array_1d.size, length), (array_1d.itemsize, 0)) return array_2d
def test_writeable(): # broadcast_to should return a readonly array original = np.array([1, 2, 3]) result = broadcast_to(original, (2, 3)) assert_equal(result.flags.writeable, False) assert_raises(ValueError, result.__setitem__, slice(None), 0) # but the result of broadcast_arrays needs to be writeable, to # preserve backwards compatibility for is_broadcast, results in [ ( False, broadcast_arrays(original, ), ), (True, broadcast_arrays(0, original)), ]: for result in results: # This will change to False in a future version if is_broadcast: with assert_warns(FutureWarning): assert_equal(result.flags.writeable, True) with assert_warns(DeprecationWarning): result[:] = 0 # Warning not emitted, writing to the array resets it assert_equal(result.flags.writeable, True) else: # No warning: assert_equal(result.flags.writeable, True) for results in [broadcast_arrays(original), broadcast_arrays(0, original)]: for result in results: # resets the warn_on_write DeprecationWarning result.flags.writeable = True # check: no warning emitted assert_equal(result.flags.writeable, True) result[:] = 0 # keep readonly input readonly original.flags.writeable = False _, result = broadcast_arrays(0, original) assert_equal(result.flags.writeable, False) # regression test for GH6491 shape = (2, ) strides = [0] tricky_array = as_strided(np.array(0), shape, strides) other = np.zeros((1, )) first, second = broadcast_arrays(tricky_array, other) assert_(first.shape == second.shape)
def predict(self, input_x): output = self.predictor(input_x) batch_size, input_channel, input_h, input_w = input_x.shape batch_size, _, grid_h, grid_w = output.shape x, y, w, h, conf, prob = F.split_axis(F.reshape(output, (batch_size, self.predictor.n_boxes, self.predictor.n_classes+5, grid_h, grid_w)), (1, 2, 3, 4, 5), axis=2) x = F.sigmoid(x) # xのactivation y = F.sigmoid(y) # yのactivation conf = F.sigmoid(conf) # confのactivation prob = F.transpose(prob, (0, 2, 1, 3, 4)) prob = F.softmax(prob) # probablitiyのacitivation prob = F.transpose(prob, (0, 2, 1, 3, 4)) # x, y, w, hを絶対座標へ変換 x_shift = Variable(np.broadcast_to(np.arange(grid_w, dtype=np.float32), x.shape)) y_shift = Variable(np.broadcast_to(np.arange(grid_h, dtype=np.float32).reshape(grid_h, 1), y.shape)) w_anchor = Variable(np.broadcast_to(np.reshape(np.array(self.anchors, dtype=np.float32)[:, 0], (self.predictor.n_boxes, 1, 1, 1)), w.shape)) h_anchor = Variable(np.broadcast_to(np.reshape(np.array(self.anchors, dtype=np.float32)[:, 1], (self.predictor.n_boxes, 1, 1, 1)), h.shape)) #x_shift.to_gpu(), y_shift.to_gpu(), w_anchor.to_gpu(), h_anchor.to_gpu() box_x = (x + x_shift) * 1.0 / grid_w box_y = (y + y_shift) * 1.0 / grid_h box_w = F.exp(w) * w_anchor * 1.0 / grid_w box_h = F.exp(h) * h_anchor * 1.0 / grid_h return box_x, box_y, box_w, box_h, conf, prob
def test_writeable(): # broadcast_to should return a readonly array original = np.array([1, 2, 3]) result = broadcast_to(original, (2, 3)) assert_equal(result.flags.writeable, False) assert_raises(ValueError, result.__setitem__, slice(None), 0) # but the result of broadcast_arrays needs to be writeable (for now), to # preserve backwards compatibility for results in [broadcast_arrays(original), broadcast_arrays(0, original)]: for result in results: assert_equal(result.flags.writeable, True) # keep readonly input readonly original.flags.writeable = False _, result = broadcast_arrays(0, original) assert_equal(result.flags.writeable, False)
def test_broadcast_to_raises(): data = [ [(0,), ()], [(1,), ()], [(3,), ()], [(3,), (1,)], [(3,), (2,)], [(3,), (4,)], [(1, 2), (2, 1)], [(1, 1), (1,)], [(1,), -1], [(1,), (-1,)], [(1, 2), (-1, 2)], ] for orig_shape, target_shape in data: arr = np.zeros(orig_shape) assert_raises(ValueError, lambda: broadcast_to(arr, target_shape))
def test_broadcast_to_raises(): data = [ [(0, ), ()], [(1, ), ()], [(3, ), ()], [(3, ), (1, )], [(3, ), (2, )], [(3, ), (4, )], [(1, 2), (2, 1)], [(1, 1), (1, )], [(1, ), -1], [(1, ), (-1, )], [(1, 2), (-1, 2)], ] for orig_shape, target_shape in data: arr = np.zeros(orig_shape) assert_raises(ValueError, lambda: broadcast_to(arr, target_shape))
def _count_reduce_items(arr, axis, keepdims=False, where=True): # fast-path for the default case if where is True: # no boolean mask given, calculate items according to axis if axis is None: axis = tuple(range(arr.ndim)) elif not isinstance(axis, tuple): axis = (axis, ) items = nt.intp(1) for ax in axis: items *= arr.shape[mu.normalize_axis_index(ax, arr.ndim)] else: # TODO: Optimize case when `where` is broadcast along a non-reduction # axis and full sum is more excessive than needed. # guarded to protect circular imports from numpy.lib.stride_tricks import broadcast_to # count True values in (potentially broadcasted) boolean mask items = umr_sum(broadcast_to(where, arr.shape), axis, nt.intp, None, keepdims) return items
def test_broadcast_to_succeeds(): data = [ [np.array(0), (0,), np.array(0)], [np.array(0), (1,), np.zeros(1)], [np.array(0), (3,), np.zeros(3)], [np.ones(1), (1,), np.ones(1)], [np.ones(1), (2,), np.ones(2)], [np.ones(1), (1, 2, 3), np.ones((1, 2, 3))], [np.arange(3), (3,), np.arange(3)], [np.arange(3), (1, 3), np.arange(3).reshape(1, -1)], [np.arange(3), (2, 3), np.array([[0, 1, 2], [0, 1, 2]])], # test if shape is not a tuple [np.ones(0), 0, np.ones(0)], [np.ones(1), 1, np.ones(1)], [np.ones(1), 2, np.ones(2)], # these cases with size 0 are strange, but they reproduce the behavior # of broadcasting with ufuncs (see test_same_as_ufunc above) [np.ones(1), (0,), np.ones(0)], [np.ones((1, 2)), (0, 2), np.ones((0, 2))], [np.ones((2, 1)), (2, 0), np.ones((2, 0))], ] for input_array, shape, expected in data: actual = broadcast_to(input_array, shape) assert_array_equal(expected, actual)
def triu_indices(n, k=0, m=None): """ Return the indices for the upper-triangle of an (n, m) array. Parameters ---------- n : int The size of the arrays for which the returned indices will be valid. k : int, optional Diagonal offset (see `triu` for details). m : int, optional .. versionadded:: 1.9.0 The column dimension of the arrays for which the returned arrays will be valid. By default `m` is taken equal to `n`. Returns ------- inds : tuple, shape(2) of ndarrays, shape(`n`) The indices for the triangle. The returned tuple contains two arrays, each with the indices along one dimension of the array. Can be used to slice a ndarray of shape(`n`, `n`). See also -------- tril_indices : similar function, for lower-triangular. mask_indices : generic function accepting an arbitrary mask function. triu, tril Notes ----- .. versionadded:: 1.4.0 Examples -------- Compute two different sets of indices to access 4x4 arrays, one for the upper triangular part starting at the main diagonal, and one starting two diagonals further right: >>> iu1 = np.triu_indices(4) >>> iu2 = np.triu_indices(4, 2) Here is how they can be used with a sample array: >>> a = np.arange(16).reshape(4, 4) >>> a array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11], [12, 13, 14, 15]]) Both for indexing: >>> a[iu1] array([ 0, 1, 2, ..., 10, 11, 15]) And for assigning values: >>> a[iu1] = -1 >>> a array([[-1, -1, -1, -1], [ 4, -1, -1, -1], [ 8, 9, -1, -1], [12, 13, 14, -1]]) These cover only a small part of the whole array (two diagonals right of the main one): >>> a[iu2] = -10 >>> a array([[ -1, -1, -10, -10], [ 4, -1, -1, -10], [ 8, 9, -1, -1], [ 12, 13, 14, -1]]) """ tri_ = ~tri(n, m, k=k - 1, dtype=bool) return tuple( broadcast_to(inds, tri_.shape)[tri_] for inds in indices(tri_.shape, sparse=True))
def __call__(self, input_x, t): isVola = input_x.volatile output = self.predictor(input_x) batch_size, _, grid_h, grid_w = output.shape if self.predictor.train == True: self.seen += batch_size x, y, w, h, conf, prob = F.split_axis(F.reshape(output, (batch_size, self.predictor.n_boxes, self.predictor.n_classes+5, grid_h, grid_w)), (1, 2, 3, 4, 5), axis=2) x = F.sigmoid(x) # xのactivation y = F.sigmoid(y) # yのactivation conf = F.sigmoid(conf) # confのactivation prob = F.transpose(prob, (0, 2, 1, 3, 4)) prob = F.softmax(prob) # probabilityのactivation # 教師データの用意 tw = np.zeros(w.shape, dtype=np.float32) # wとhが0になるように学習(e^wとe^hは1に近づく -> 担当するbboxの倍率1) th = np.zeros(h.shape, dtype=np.float32) tx = np.tile(0.5, x.shape).astype(np.float32) # 活性化後のxとyが0.5になるように学習() ty = np.tile(0.5, y.shape).astype(np.float32) if self.seen < self.unstable_seen: # centerの存在しないbbox誤差学習スケールは基本0.1 box_learning_scale = np.tile(0.1, x.shape).astype(np.float32) else: box_learning_scale = np.tile(0, x.shape).astype(np.float32) tconf = np.zeros(conf.shape, dtype=np.float32) # confidenceのtruthは基本0、iouがthresh以上のものは学習しない、ただしobjectの存在するgridのbest_boxのみ真のIOUに近づかせる conf_learning_scale = np.tile(0.1, conf.shape).astype(np.float32) tprob = prob.data.copy() # best_anchor以外は学習させない(自身との二乗和誤差 = 0) # 全bboxとtruthのiouを計算(batch単位で計算する) x_shift = Variable(np.broadcast_to(np.arange(grid_w, dtype=np.float32), x.shape[1:]), volatile=isVola) y_shift = Variable(np.broadcast_to(np.arange(grid_h, dtype=np.float32).reshape(grid_h, 1), y.shape[1:]), volatile=isVola) w_anchor = Variable(np.broadcast_to(np.reshape(np.array(self.anchors, dtype=np.float32)[:, 0], (self.predictor.n_boxes, 1, 1, 1)), w.shape[1:]), volatile=isVola) h_anchor = Variable(np.broadcast_to(np.reshape(np.array(self.anchors, dtype=np.float32)[:, 1], (self.predictor.n_boxes, 1, 1, 1)), h.shape[1:]), volatile=isVola) x_shift.to_gpu(), y_shift.to_gpu(), w_anchor.to_gpu(), h_anchor.to_gpu() best_ious = [] for batch in range(batch_size): n_truth_boxes = len(t[batch]) box_x = (x[batch] + x_shift) * 1.0 / grid_w box_y = (y[batch] + y_shift) * 1.0 / grid_h box_w = F.exp(w[batch]) * w_anchor * 1.0 / grid_w box_h = F.exp(h[batch]) * h_anchor * 1.0 / grid_h ious = [] for truth_index in range(n_truth_boxes): truth_box_x = Variable(np.broadcast_to(np.array(t[batch][truth_index]["x"], dtype=np.float32), box_x.shape), volatile=isVola) truth_box_y = Variable(np.broadcast_to(np.array(t[batch][truth_index]["y"], dtype=np.float32), box_y.shape), volatile=isVola) truth_box_w = Variable(np.broadcast_to(np.array(t[batch][truth_index]["w"], dtype=np.float32), box_w.shape), volatile=isVola) truth_box_h = Variable(np.broadcast_to(np.array(t[batch][truth_index]["h"], dtype=np.float32), box_h.shape), volatile=isVola) truth_box_x.to_gpu(), truth_box_y.to_gpu(), truth_box_w.to_gpu(), truth_box_h.to_gpu() ious.append(multi_box_iou(Box(box_x, box_y, box_w, box_h), Box(truth_box_x, truth_box_y, truth_box_w, truth_box_h)).data.get()) ious = np.array(ious) best_ious.append(np.max(ious, axis=0)) best_ious = np.array(best_ious) # 一定以上のiouを持つanchorに対しては、confを0に下げないようにする(truthの周りのgridはconfをそのまま維持)。 tconf[best_ious > self.thresh] = conf.data.get()[best_ious > self.thresh] conf_learning_scale[best_ious > self.thresh] = 0 # objectの存在するanchor boxのみ、x、y、w、h、conf、probを個別修正 abs_anchors = self.anchors / np.array([grid_w, grid_h]) for batch in range(batch_size): for truth_box in t[batch]: truth_w = int(float(truth_box["x"]) * grid_w) truth_h = int(float(truth_box["y"]) * grid_h) truth_n = 0 best_iou = 0.0 for anchor_index, abs_anchor in enumerate(abs_anchors): iou = box_iou(Box(0, 0, float(truth_box["w"]), float(truth_box["h"])), Box(0, 0, abs_anchor[0], abs_anchor[1])) if best_iou < iou: best_iou = iou truth_n = anchor_index # objectの存在するanchorについて、centerを0.5ではなく、真の座標に近づかせる。anchorのスケールを1ではなく真のスケールに近づかせる。学習スケールを1にする。 box_learning_scale[batch, truth_n, :, truth_h, truth_w] = 1.0 tx[batch, truth_n, :, truth_h, truth_w] = float(truth_box["x"]) * grid_w - truth_w ty[batch, truth_n, :, truth_h, truth_w] = float(truth_box["y"]) * grid_h - truth_h tw[batch, truth_n, :, truth_h, truth_w] = np.log(float(truth_box["w"]) * 1.0 / abs_anchors[truth_n][0]) th[batch, truth_n, :, truth_h, truth_w] = np.log(float(truth_box["h"]) * 1.0 / abs_anchors[truth_n][1]) tprob[batch, :, truth_n, truth_h, truth_w] = 0 tprob[batch, int(truth_box["label"]), truth_n, truth_h, truth_w] = 1 # IOUの観測 full_truth_box = Box(float(truth_box["x"]), float(truth_box["y"]), float(truth_box["w"]), float(truth_box["h"])) predicted_box = Box( (x[batch][truth_n][0][truth_h][truth_w].data.get() + truth_w) * 1.0 / grid_w, (y[batch][truth_n][0][truth_h][truth_w].data.get() + truth_h) * 1.0 / grid_h, np.exp(w[batch][truth_n][0][truth_h][truth_w].data.get()) * abs_anchors[truth_n][0], np.exp(h[batch][truth_n][0][truth_h][truth_w].data.get()) * abs_anchors[truth_n][1] ) predicted_iou = box_iou(full_truth_box, predicted_box) tconf[batch, truth_n, :, truth_h, truth_w] = predicted_iou conf_learning_scale[batch, truth_n, :, truth_h, truth_w] = 10.0 # debug prints maps = F.transpose(prob[batch], (2, 3, 1, 0)).data # print("best confidences and best conditional probability and predicted class of each grid:") # for i in range(grid_h): # for j in range(grid_w): # print("%2d" % (int(conf[batch, :, :, i, j].data.max() * 100)), end=" ") # print(" ", end="") # for j in range(grid_w): # print("%2d" % (maps[i][j][int(maps[i][j].max(axis=1).argmax())].argmax()), end=" ") # print(" ", end="") # for j in range(grid_w): # print("%2d" % (maps[i][j][int(maps[i][j].max(axis=1).argmax())].max()*100), end=" ") # print() # # print("best default iou: %.2f predicted iou: %.2f confidence: %.2f class: %s" % (best_iou, predicted_iou, conf[batch][truth_n][0][truth_h][truth_w].data, t[batch][0]["label"])) # print("-------------------------------") #print("seen = %d" % self.seen) # loss計算 tx, ty, tw, th, tconf, tprob = Variable(tx, volatile=isVola), Variable(ty, volatile=isVola), Variable(tw, volatile=isVola), Variable(th, volatile=isVola), Variable(tconf, volatile=isVola), Variable(tprob, volatile=isVola) box_learning_scale, conf_learning_scale = Variable(box_learning_scale, volatile=isVola), Variable(conf_learning_scale, volatile=isVola) tx.to_gpu(), ty.to_gpu(), tw.to_gpu(), th.to_gpu(), tconf.to_gpu(), tprob.to_gpu() box_learning_scale.to_gpu() conf_learning_scale.to_gpu() x_loss = F.sum((tx - x) ** 2 * box_learning_scale) / 2.0 y_loss = F.sum((ty - y) ** 2 * box_learning_scale) / 2.0 w_loss = F.sum((tw - w) ** 2 * box_learning_scale) / 2.0 h_loss = F.sum((th - h) ** 2 * box_learning_scale) / 2.0 c_loss = F.sum((tconf - conf) ** 2 * conf_learning_scale) / 2.0 p_loss = F.sum((tprob - prob) ** 2) / 2.0 print("x_loss: %f y_loss: %f w_loss: %f h_loss: %f c_loss: %f p_loss: %f" % (F.sum(x_loss).data, F.sum(y_loss).data, F.sum(w_loss).data, F.sum(h_loss).data, F.sum(c_loss).data, F.sum(p_loss).data) ) loss = x_loss + y_loss + w_loss + h_loss + c_loss + p_loss return loss
def _create_window_function(cls, window_type: WindowFunction, shape: Tuple, **kwargs) -> np.ndarray: if window_type == WindowFunction.Rectangular: return np.ones(shape) window_size: int = shape[-1] window = window_type.create(window_size, **kwargs).astype(dtype="<f4", order="C") return broadcast_to(window, shape)