Beispiel #1
0
def make_deep_rrnn_rot_relu(size_input, size_mem, n_layers, size_output,
                            size_batch_in, k_in, k_h):
    inputs = [cgt.matrix() for i_layer in xrange(n_layers + 1)]
    outputs = []
    print 'input_size: ', size_input
    for i_layer in xrange(n_layers):
        prev_h = inputs[
            i_layer +
            1]  # note that inputs[0] is the external input, so we add 1
        x = inputs[0] if i_layer == 0 else outputs[i_layer - 1]
        size_x = size_input if i_layer == 0 else size_mem
        size_batch = prev_h.shape[0]

        xform_h_param = nn.TensorParam((2 * k_h, size_mem), name="rotxform")
        xform_h_non = xform_h_param.weight
        xform_h_non.props["is_rotation"] = True

        xform_h_norm = cgt.norm(xform_h_non, axis=1, keepdims=True)
        xform_h = cgt.broadcast('/', xform_h_non, xform_h_norm, "xx,x1")

        add_in_lin = nn.Affine(size_x, size_mem)(x)
        add_in_relu = nn.rectify(add_in_lin)

        prev_h_scaled = nn.scale_mag(prev_h)

        h_in_added = prev_h_scaled + add_in_relu
        inters_h = [h_in_added]

        colon = slice(None, None, None)

        for i in xrange(2 * k_h):
            inter_in = inters_h[-1]
            r_cur = xform_h[i, :]
            #r_cur = cgt.subtensor(xform_h, [i, colon])
            r_cur_2_transpose = cgt.reshape(r_cur, (size_mem, 1))
            r_cur_2 = cgt.reshape(r_cur, (1, size_mem))
            ref_cur = cgt.dot(cgt.dot(inter_in, r_cur_2_transpose), r_cur_2)
            inter_out = inter_in - 2 * ref_cur
            inters_h.append(inter_out)
        next_h = inters_h[-1]
        outputs.append(next_h)

    category_activations = nn.Affine(size_mem, size_output,
                                     name="pred")(outputs[-1])
    logprobs = nn.logsoftmax(category_activations)
    outputs.append(logprobs)

    #print 'len outputs:', len(outputs)
    #print 'len inputs:', len(inputs)

    return nn.Module(inputs, outputs)
Beispiel #2
0
def make_deep_rrnn_rot_relu(size_input, size_mem, n_layers, size_output, size_batch_in, k_in, k_h):
    inputs = [cgt.matrix() for i_layer in xrange(n_layers+1)]
    outputs = []
    print 'input_size: ', size_input
    for i_layer in xrange(n_layers):
        prev_h = inputs[i_layer+1] # note that inputs[0] is the external input, so we add 1
        x = inputs[0] if i_layer==0 else outputs[i_layer-1]
        size_x = size_input if i_layer==0 else size_mem
        size_batch = prev_h.shape[0]

        xform_h_param = nn.TensorParam((2 * k_h, size_mem), name="rotxform")
        xform_h_non = xform_h_param.weight
        xform_h_non.props["is_rotation"] = True

        xform_h_norm = cgt.norm(xform_h_non, axis=1, keepdims=True)
        xform_h = cgt.broadcast('/', xform_h_non, xform_h_norm, "xx,x1")

        add_in_lin = nn.Affine(size_x, size_mem)(x)
        add_in_relu = nn.rectify(add_in_lin)

        prev_h_scaled = nn.scale_mag(prev_h)


        h_in_added = prev_h_scaled + add_in_relu
        inters_h = [h_in_added]

        colon = slice(None, None, None)

        for i in xrange(2 * k_h):
            inter_in = inters_h[-1]
            r_cur = xform_h[i, :]
            #r_cur = cgt.subtensor(xform_h, [i, colon])
            r_cur_2_transpose = cgt.reshape(r_cur, (size_mem, 1))
            r_cur_2 = cgt.reshape(r_cur, (1, size_mem))
            ref_cur = cgt.dot(cgt.dot(inter_in, r_cur_2_transpose), r_cur_2)
            inter_out = inter_in - 2 * ref_cur
            inters_h.append(inter_out)
        next_h = inters_h[-1]
        outputs.append(next_h)


    category_activations = nn.Affine(size_mem, size_output,name="pred")(outputs[-1])
    logprobs = nn.logsoftmax(category_activations)
    outputs.append(logprobs)

    #print 'len outputs:', len(outputs)
    #print 'len inputs:', len(inputs)

    return nn.Module(inputs, outputs)
Beispiel #3
0
def tinyconv_model(X, w, w2, p_drop):
    l1 = nn.conv2d(X, w, kernelshape=(3, 3), pad=(1, 1), stride=(3, 3))
    l1a = nn.dropout(l1, p_drop)
    batchsize, channels, rows, cols = l1.shape
    l1flat = cgt.reshape(l1, [batchsize, channels * rows * cols])
    pyx = nn.softmax(l1flat.dot(w2))
    return l1, pyx
Beispiel #4
0
def tinyconv_model(X, w, w2, p_drop):
    l1 = nn.conv2d(X, w, kernelshape=(3,3), pad=(1,1),stride=(3,3))
    l1a = nn.dropout(l1, p_drop)
    batchsize,channels,rows,cols = l1.shape
    l1flat = cgt.reshape(l1, [batchsize,channels*rows*cols])
    pyx = nn.softmax(l1flat.dot(w2))
    return l1, pyx
Beispiel #5
0
def reshape(x, shp):
    from ..utils import wrap_into_tuple
    import operator
    shp = wrap_into_tuple(shp)
    neg_indices = [(idx, shp_slice)
                   for idx, shp_slice in enumerate(shp) if shp_slice == -1]
    if len(neg_indices) > 1:
        raise ValueError('At most one reshaped dimension can be -1')
    elif len(neg_indices) == 1:
        idx, shp_slice = neg_indices[0]
        neg_size = reduce(
            operator.mul, x.shape, 1) // reduce(operator.mul, shp[:idx] + shp[idx + 1:], 1)
        shp = tuple(shp[:idx] + (neg_size,) + shp[idx + 1:])
        return cgt.reshape(x, shp)
    else:
        return cgt.reshape(x, shp)
Beispiel #6
0
def reshape(x, shp):
    from ..utils import wrap_into_tuple
    import operator
    shp = wrap_into_tuple(shp)
    neg_indices = [(idx, shp_slice) for idx, shp_slice in enumerate(shp)
                   if shp_slice == -1]
    if len(neg_indices) > 1:
        raise ValueError('At most one reshaped dimension can be -1')
    elif len(neg_indices) == 1:
        idx, shp_slice = neg_indices[0]
        neg_size = reduce(operator.mul, x.shape, 1) // reduce(
            operator.mul, shp[:idx] + shp[idx + 1:], 1)
        shp = tuple(shp[:idx] + (neg_size, ) + shp[idx + 1:])
        return cgt.reshape(x, shp)
    else:
        return cgt.reshape(x, shp)
Beispiel #7
0
def denseLayer(nn_input, num_units, activation=rectify, w_init=XavierNormal(), bias_init=Constant(0)):
    """
    Batch by feature input.
    """
    if len(nn_input.shape) > 2:
        nn_input = cgt.reshape(nn_input, [nn_input.shape[0], reduce(lambda x, y: x*y, nn_input.shape[1:])])
    feature_dims = cgt.infer_shape(nn_input)[1]
    return activation(Affine(feature_dims, num_units, weight_init=w_init, bias_init=bias_init)(nn_input))
Beispiel #8
0
def convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden):
    l1a = nn.rectify(nn.conv2d(X, w, kernelshape=(3, 3), pad=(1, 1)))
    l1 = nn.max_pool_2d(l1a, kernelshape=(2, 2), stride=(2, 2))
    l1 = nn.dropout(l1, p_drop_conv)

    l2a = nn.rectify(nn.conv2d(l1, w2, kernelshape=(3, 3), pad=(1, 1)))
    l2 = nn.max_pool_2d(l2a, kernelshape=(2, 2), stride=(2, 2))
    l2 = nn.dropout(l2, p_drop_conv)

    l3a = nn.rectify(nn.conv2d(l2, w3, kernelshape=(3, 3), pad=(1, 1)))
    l3b = nn.max_pool_2d(l3a, kernelshape=(2, 2), stride=(2, 2))
    batchsize, channels, rows, cols = l3b.shape
    l3 = cgt.reshape(l3b, [batchsize, channels * rows * cols])
    l3 = nn.dropout(l3, p_drop_conv)

    l4 = nn.rectify(cgt.dot(l3, w4))
    l4 = nn.dropout(l4, p_drop_hidden)

    pyx = nn.softmax(cgt.dot(l4, w_o))
    return pyx
Beispiel #9
0
def convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden):
    l1a = nn.rectify(nn.conv2d(X, w, kernelshape=(3,3), pad=(1,1)))
    l1 = nn.max_pool_2d(l1a, kernelshape=(2, 2), stride=(2,2))
    l1 = nn.dropout(l1, p_drop_conv)

    l2a = nn.rectify(nn.conv2d(l1, w2, kernelshape=(3,3), pad=(1,1)))
    l2 = nn.max_pool_2d(l2a, kernelshape=(2, 2), stride=(2,2))
    l2 = nn.dropout(l2, p_drop_conv)

    l3a = nn.rectify(nn.conv2d(l2, w3, kernelshape=(3,3), pad=(1,1)))
    l3b = nn.max_pool_2d(l3a, kernelshape=(2, 2), stride=(2,2))
    batchsize,channels,rows,cols = l3b.shape
    l3 = cgt.reshape(l3b, [batchsize, channels*rows*cols])
    l3 = nn.dropout(l3, p_drop_conv)

    l4 = nn.rectify(cgt.dot(l3, w4))
    l4 = nn.dropout(l4, p_drop_hidden)
    
    pyx = nn.softmax(cgt.dot(l4, w_o))
    return pyx
Beispiel #10
0
import cgt
from cgt import nn, utils
import numpy as np, numpy.random as nr
from numpy.linalg import norm
from param_collection import ParamCollection

k_in = 1
size_x = 3
size_mem = 4
size_batch = 4

x = cgt.matrix(fixed_shape=(size_batch, size_x))
prev_h = cgt.matrix(fixed_shape=(size_batch, size_mem))
r_vec = nn.Affine(size_x, 2 * k_in * size_mem)(x)
r_non = cgt.reshape(r_vec, (size_batch, 2 * k_in, size_mem))
r_norm = cgt.norm(r_non, axis=2, keepdims=True)
r = cgt.broadcast('/', r_non, r_norm, "xxx,xx1")
prev_h_3 = cgt.reshape(prev_h, (size_batch, size_mem, 1))
inters = [prev_h_3]

for i in xrange(k_in * 2):
    inter_in = inters[-1]
    r_cur = r[:, i, :]
    r_cur_3_transpose = cgt.reshape(r_cur, (size_batch, 1, size_mem))
    r_cur_3 = cgt.reshape(r_cur, (size_batch, size_mem, 1))
    ref_cur = cgt.batched_matmul(
        r_cur_3, cgt.batched_matmul(r_cur_3_transpose, inter_in))
    inter_out = inter_in - ref_cur
    inters.append(inter_out)
h = inters[-1]
Beispiel #11
0
 elif layer.type == "Pooling":
     param = layer.pooling_param
     X = inputs[0]
     pool_type = {param.MAX: "max", param.AVE: "mean"}[param.pool]
     height_in, width_in = infer_shape(X)[2:4]
     kernel = (param.kernel_size, param.kernel_size) if param.HasField("kernel_size")\
         else (param.kernel_h, param.kernel_w)
     stride = (param.stride, param.stride) if param.HasField("stride")\
         else (param.stride_h, param.stride_w)
     pad = (param.pad, param.pad) if param.HasField("pad")\
         else (param.pad_h, param.pad_w)
     output = [nn.pool(pool_type, X, stride, kernel, pad)]
 elif layer.type == "InnerProduct":
     X = inputs[0]
     if X.ndim == 4:
         X = cgt.reshape(
             X, [X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]])
     param = layer.inner_product_param
     nchanin = infer_shape(X)[1]
     Wshape = (param.num_output, nchanin)
     Wname = layer.param[0].name or layer.name + ":W"
     Wval = np.empty(Wshape, dtype=cgt.floatX)
     W = name2node[Wname] = cgt.shared(Wval,
                                       name=Wname,
                                       fixed_shape_mask="all")
     bshape = (1, param.num_output)
     bname = layer.param[1].name or layer.name + ":b"
     bval = np.empty(bshape, dtype=cgt.floatX)
     b = name2node[bname] = cgt.shared(bval,
                                       name=bname,
                                       fixed_shape_mask="all")
     yname = layer.top[0]
Beispiel #12
0
 elif layer.type == "Pooling":
     param = layer.pooling_param
     X = inputs[0]
     pool_type = {param.MAX : "max", param.AVE : "mean"}[param.pool]
     height_in,width_in = infer_shape(X)[2:4]
     kernel = (param.kernel_size, param.kernel_size) if param.HasField("kernel_size")\
         else (param.kernel_h, param.kernel_w)
     stride = (param.stride, param.stride) if param.HasField("stride")\
         else (param.stride_h, param.stride_w)
     pad = (param.pad, param.pad) if param.HasField("pad")\
         else (param.pad_h, param.pad_w)
     output = [nn.pool(pool_type, X, stride, kernel, pad)]
 elif layer.type == "InnerProduct":
     X = inputs[0]
     if X.ndim == 4:
         X = cgt.reshape(X, [X.shape[0], X.shape[1]*X.shape[2]*X.shape[3]] )
     param = layer.inner_product_param
     nchanin = infer_shape(X)[1]
     Wshape = (param.num_output, nchanin)
     Wname = layer.param[0].name or layer.name+":W"
     Wval = np.empty(Wshape, dtype=cgt.floatX)
     W = name2node[Wname] = cgt.shared(Wval, name=Wname, fixed_shape_mask="all")
     bshape = (1, param.num_output)
     bname = layer.param[1].name or layer.name+":b"
     bval = np.empty(bshape, dtype=cgt.floatX)
     b = name2node[bname] = cgt.shared(bval, name=bname, fixed_shape_mask="all")
     yname = layer.top[0]
     output = [cgt.broadcast("+",X.dot(W), b, "xx,1x")          ]
 elif layer.type == "ReLU":
     output = [nn.rectify(inputs[0])]
 elif layer.type == "Softmax":
Beispiel #13
0
def stack(tensors, axis=0):
    if axis is not 0:
        raise ValueError('only axis=0 is supported under cgt')
    return cgt.concatenate(map(lambda x: cgt.reshape(x, [1] + x.shape), tensors), axis=0)
Beispiel #14
0
import cgt
from cgt import nn, utils
import numpy as np, numpy.random as nr
from numpy.linalg import norm
from param_collection import ParamCollection

k_in = 1
size_x = 3
size_mem = 4
size_batch = 4

x = cgt.matrix(fixed_shape=(size_batch, size_x))
prev_h = cgt.matrix(fixed_shape=(size_batch, size_mem))
r_vec = nn.Affine(size_x, 2 * k_in * size_mem)(x)
r_non = cgt.reshape(r_vec, (size_batch, 2 * k_in, size_mem))
r_norm = cgt.norm(r_non, axis=2, keepdims=True)
r = cgt.broadcast('/', r_non, r_norm, "xxx,xx1")
prev_h_3 = cgt.reshape(prev_h, (size_batch, size_mem, 1))
inters = [prev_h_3]

for i in xrange(k_in * 2):
    inter_in = inters[-1]
    r_cur = r[:, i, :]
    r_cur_3_transpose = cgt.reshape(r_cur, (size_batch, 1, size_mem))
    r_cur_3 = cgt.reshape(r_cur, (size_batch, size_mem, 1))
    ref_cur = cgt.batched_matmul(r_cur_3, cgt.batched_matmul(r_cur_3_transpose, inter_in))
    inter_out = inter_in - ref_cur
    inters.append(inter_out)
h = inters[-1]
    
r_nn = nn.Module([x], [h])
Beispiel #15
0
def stack(tensors, axis=0):
    if axis is not 0:
        raise ValueError('only axis=0 is supported under cgt')
    return cgt.concatenate(map(lambda x: cgt.reshape(x, [1] + x.shape),
                               tensors),
                           axis=0)