Ejemplo n.º 1
0
def create_frcn_model(frcn_fine_tune=False):

    b1 = BranchNode(name="b1")

    imagenet_layers = add_vgg_layers()
    HW = (7, 7)

    frcn_layers = [
        RoiPooling(layers=imagenet_layers, HW=HW,
                   bprop_enabled=frcn_fine_tune),
        Affine(nout=4096,
               init=Gaussian(scale=0.005),
               bias=Constant(.1),
               activation=Rectlin()),
        Dropout(keep=0.5),
        Affine(nout=4096,
               init=Gaussian(scale=0.005),
               bias=Constant(.1),
               activation=Rectlin()),
        Dropout(keep=0.5), b1,
        Affine(nout=21,
               init=Gaussian(scale=0.01),
               bias=Constant(0),
               activation=Softmax())
    ]
    bb_layers = [
        b1,
        Affine(nout=84,
               init=Gaussian(scale=0.001),
               bias=Constant(0),
               activation=Identity())
    ]

    return Model(layers=Tree([frcn_layers, bb_layers]))
Ejemplo n.º 2
0
    def __init__(self,
                 overlapping_classes=None,
                 exclusive_classes=None,
                 analytics_input=True,
                 network_type='conv_net',
                 num_words=60,
                 width=100,
                 lookup_size=0,
                 lookup_dim=0,
                 optimizer=Adam()):
        assert (overlapping_classes is not None) or (exclusive_classes
                                                     is not None)

        self.width = width
        self.num_words = num_words
        self.overlapping_classes = overlapping_classes
        self.exclusive_classes = exclusive_classes
        self.analytics_input = analytics_input
        self.recurrent = network_type == 'lstm'
        self.lookup_size = lookup_size
        self.lookup_dim = lookup_dim

        init = GlorotUniform()
        activation = Rectlin(slope=1E-05)
        gate = Logistic()

        input_layers = self.input_layers(analytics_input, init, activation,
                                         gate)

        if self.overlapping_classes is None:
            output_layers = [
                Affine(len(self.exclusive_classes), init, activation=Softmax())
            ]
        elif self.exclusive_classes is None:
            output_layers = [
                Affine(len(self.overlapping_classes),
                       init,
                       activation=Logistic())
            ]
        else:
            output_branch = BranchNode(name='exclusive_overlapping')
            output_layers = Tree([[
                SkipNode(), output_branch,
                Affine(len(self.exclusive_classes), init, activation=Softmax())
            ],
                                  [
                                      output_branch,
                                      Affine(len(self.overlapping_classes),
                                             init,
                                             activation=Logistic())
                                  ]])
        layers = [
            input_layers,
            # this is where inputs meet, and where we may want to add depth or
            # additional functionality
            Dropout(keep=0.8),
            output_layers
        ]
        super(ClassifierNetwork, self).__init__(layers, optimizer=optimizer)
Ejemplo n.º 3
0
def make_tree(trunk, branch1, branch2, alphas):

    # Make one copy that is the Tree version
    _trunk = [l['layer'](**l['config']) for l in trunk]
    bnode = [BranchNode(name='bnode')]
    _branch1 = [l['layer'](**l['config']) for l in branch1]
    _branch2 = [l['layer'](**l['config']) for l in branch2]
    v1 = Tree([_trunk + bnode + _branch1, bnode + _branch2], alphas)

    # Now a second copy with no sharing as the reference version
    _trunkb = [l['layer'](**l['config']) for l in trunk]
    _branch1b = [l['layer'](**l['config']) for l in branch1]
    _branch2b = [l['layer'](**l['config']) for l in branch2]
    return (v1, _trunkb, _branch1b, _branch2b)
Ejemplo n.º 4
0
      Affine(nout=16, linear_name="b1_l1", **normrelu),
      Affine(nout=10, linear_name="b1_l2", **normsigm)]

p3 = [b2,
      Affine(nout=16, linear_name="b2_l1", **normrelu),
      Affine(nout=10, linear_name="b2_l2", **normsigm)]


# setup cost function as CrossEntropy
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()),
                        GeneralizedCost(costfunc=CrossEntropyBinary()),
                        GeneralizedCost(costfunc=CrossEntropyBinary())],
                 weights=[1, 0., 0.])

# setup optimizer
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding)

# initialize model object
alphas = [1, 0.25, 0.25]
mlp = Model(layers=Tree([p1, p2, p3], alphas=alphas))

# setup standard fit callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)

# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)

logging.getLogger('neon').info("Misclassification error = %.1f%%",
                               (mlp.eval(valid_set, metric=Misclassification())*100))
print('Misclassification error = %.1f%%' % (mlp.eval(valid_set, metric=Misclassification())*100))
Ejemplo n.º 5
0
def build_model(dataset,
                frcn_rois_per_img,
                train_pre_nms_N=12000,
                train_post_nms_N=2000,
                test_pre_nms_N=6000,
                test_post_nms_N=300,
                inference=False):
    """
    Returns the Faster-RCNN model. For inference, also returns a reference to the
    proposal layer.

    Faster-RCNN contains three modules: VGG, the Region Proposal Network (RPN),
    and the Classification Network (ROI-pooling + Fully Connected layers), organized
    as a tree. Tree has 4 branches:

    VGG -> b1 -> Conv (3x3) -> b2 -> Conv (1x1) -> CrossEntropyMulti (objectness label)
                               b2 -> Conv (1x1) -> SmoothL1Loss (bounding box targets)
           b1 -> PropLayer -> ROI -> Affine -> Affine -> b3 -> Affine -> CrossEntropyMulti
                                                         b3 -> Affine -> SmoothL1Loss

    When the model is constructed for inference, several elements are different:
    - The number of regions to keep before and after non-max suppression is (6000, 300) for
      training and (12000, 2000) for inference.
    - The out_shape of the proposalLayer of the network is equal to post_nms_N (number of rois
      to keep after performaing nms). This is configured by passing the inference flag to the
      proposalLayer constructor.

    Arguments:
        dataset (objectlocalization): Dataset object.
        frcn_rois_per_img (int): Number of ROIs per image considered by the classification network.
        inference (bool): Construct the model for inference. Default is False.

    Returns:
        model (Model): Faster-RCNN model.
        proposalLayer (proposalLayer): Reference to proposalLayer in the model.
                                       Returned only for inference=True.
    """
    num_classes = dataset.num_classes

    # define the branch points
    b1 = BranchNode(name="conv_branch")
    b2 = BranchNode(name="rpn_branch")
    b3 = BranchNode(name="roi_branch")

    # define VGG
    VGG = util.add_vgg_layers()

    # define RPN
    rpn_init = dict(strides=1, init=Gaussian(scale=0.01), bias=Constant(0))
    # these references are passed to the ProposalLayer.
    RPN_3x3 = Conv((3, 3, 512), activation=Rectlin(), padding=1, **rpn_init)
    RPN_1x1_obj = Conv((1, 1, 18),
                       activation=PixelwiseSoftmax(c=2),
                       padding=0,
                       **rpn_init)
    RPN_1x1_bbox = Conv((1, 1, 36),
                        activation=Identity(),
                        padding=0,
                        **rpn_init)

    # inference uses different network settings
    if not inference:
        pre_nms_N = train_pre_nms_N  # default 12000
        post_nms_N = train_post_nms_N  # default 2000
    else:
        pre_nms_N = test_pre_nms_N  # default 6000
        post_nms_N = test_post_nms_N  # default 300

    proposalLayer = ProposalLayer([RPN_1x1_obj, RPN_1x1_bbox],
                                  dataset,
                                  pre_nms_N=pre_nms_N,
                                  post_nms_N=post_nms_N,
                                  num_rois=frcn_rois_per_img,
                                  inference=inference)

    # define ROI classification network
    ROI = [
        proposalLayer,
        RoiPooling(HW=(7, 7)),
        Affine(nout=4096,
               init=Gaussian(scale=0.005),
               bias=Constant(.1),
               activation=Rectlin()),
        Dropout(keep=0.5),
        Affine(nout=4096,
               init=Gaussian(scale=0.005),
               bias=Constant(.1),
               activation=Rectlin()),
        Dropout(keep=0.5)
    ]

    ROI_category = Affine(nout=num_classes,
                          init=Gaussian(scale=0.01),
                          bias=Constant(0),
                          activation=Softmax())
    ROI_bbox = Affine(nout=4 * num_classes,
                      init=Gaussian(scale=0.001),
                      bias=Constant(0),
                      activation=Identity())

    # build the model
    # the four branches of the tree mirror the branches listed above
    frcn_tree = Tree([ROI + [b3, ROI_category], [b3, ROI_bbox]])

    model = Model(layers=Tree([
        VGG + [b1, RPN_3x3, b2, RPN_1x1_obj],
        [b2, RPN_1x1_bbox],
        [b1] + [frcn_tree],
    ]))

    if inference:
        return (model, proposalLayer)
    else:
        return model
Ejemplo n.º 6
0
        GeneralizedCost(costfunc=CrossEntropyMulti()),
        GeneralizedCost(costfunc=CrossEntropyMulti()),
        GeneralizedCost(costfunc=CrossEntropyMulti())
    ],
    weights=[1, 0., 0.])  # We only want to consider the CE of the main path

if not args.resume:
    # build the model from scratch and run it

    # Now construct the model
    branch_nodes = [BranchNode(name='branch{}'.format(i)) for i in (0, 1)]
    main1 = main_branch(branch_nodes)
    aux1 = aux_branch(branch_nodes[0], ind=1)
    aux2 = aux_branch(branch_nodes[1], ind=2)

    model = Model(layers=Tree([main1, aux1, aux2], alphas=[1.0, 0.3, 0.3]))

else:
    # load up the save model
    model = Model('serialize_test_2.pkl')
    model.initialize(train, cost=cost)

# configure callbacks
callbacks = Callbacks(model,
                      progress_bar=True,
                      output_file='temp1.h5',
                      serialize=1,
                      history=3,
                      save_path='serialize_test.pkl')

lr_sched = PolySchedule(total_epochs=10, power=0.5)
Ejemplo n.º 7
0
bb_layers = [
    b1,
    bbox_pred,
]

# setup optimizer
opt_w = GradientDescentMomentum(0.001 * learning_rate_scale,
                                0.9,
                                wdecay=0.0005)
opt_b = GradientDescentMomentum(0.002 * learning_rate_scale, 0.9)

optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b})

# setup model

model = Model(layers=Tree([frcn_layers, bb_layers]))

# if training a new model, seed the Alexnet conv layers with pre-trained weights
# otherwise, just load the model file
if args.model_file is None:
    load_imagenet_weights(model, args.data_dir)

cost = Multicost(costs=[
    GeneralizedCost(costfunc=CrossEntropyMulti()),
    GeneralizedCostMask(costfunc=SmoothL1Loss())
],
                 weights=[1, 1])

callbacks = Callbacks(model, **args.callback_args)

model.fit(train_set,
Ejemplo n.º 8
0
def test_branch_model_fork():
    from neon.layers import BranchNode, Tree

    NervanaObject.be = gen_backend("gpu", batch_size=64)
    be = NervanaObject.be
    bnode = BranchNode()
    i1 = inception([(32, ), (32, 32), ('max', 16)])
    top1 = top_branch()
    top2 = top_branch()
    p1 = Sequential(main_branch() + [bnode, i1] + top1)
    p2 = [bnode] + top2

    alpha2 = 0.3
    neon_layer = Tree([p1, p2], alphas=[1.0, alpha2])

    inshape = (3, 224, 224)
    insize = np.prod(inshape)
    inpa = np.random.random((insize, batch_size))
    neon_layer.configure(inshape)
    inp = neon_layer.be.array(inpa)

    neon_layer.allocate()
    print neon_layer.nested_str()
    neon_layer.layers[0].layers[0].prev_layer = True
    neon_layer.allocate_deltas()
    neon_layer.layers[0].layers[0].set_deltas([be.iobuf(inshape)])
    neon_out_dev = neon_layer.fprop(inp)
    neon_out = [d.get() for d in neon_out_dev]

    # Now make the reference pathways:
    main_trunk2 = Sequential(main_branch())
    main_trunk2.configure(inshape)
    main2 = main_trunk2.layers
    main2[0].prev_layer = True
    main2[0].set_deltas([be.iobuf(inshape)])

    branch2 = Sequential(top_branch())
    lbranch2 = branch2.layers
    (b1, b2, b3) = inception_bare(i1, [(32, ), (32, 32), ('max', 16)])

    for bb in (b1, b2, b3, lbranch2):
        oshape = inshape
        for ll in main2 + bb:
            oshape = ll.configure(oshape)

    main1_trunk = neon_layer.layers[0].layers[:8]
    for ll, lo in zip(main2, main1_trunk):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})
        ll.allocate()
        ll.set_deltas([be.iobuf(ll.in_shape)])

    for ll, lo in zip(lbranch2, neon_layer.layers[1].layers[1:]):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})

    for bb in (b1, b2, b3, lbranch2):
        for ll in bb:
            ll.allocate()
            ll.set_deltas([be.iobuf(ll.in_shape)])

    # Create the combined output buffer
    merge_output = be.empty_like(neon_layer.layers[0].layers[9].outputs)

    x = inp
    for ll in main2:
        x = ll.fprop(x)
    main2_out = x

    start = 0
    for bb in (b1, b2, b3):
        xb = main2_out
        for ll in bb:
            xb = ll.fprop(xb)
        end = start + xb.shape[0]
        merge_output[start:end] = xb
        start = end

    x = merge_output

    top_trunk = Sequential(top1).layers
    for ll in top_trunk:
        x = ll.fprop(x)

    neon_out_ref = x.get()
    difference = neon_out_ref - neon_out[0]
    assert np.max(np.abs(difference)) < 1e-7
    print np.max(np.abs(difference))

    # Now do second branch
    neon_out_ref2 = branch2.fprop(main2_out).get()
    difference = neon_out_ref2 - neon_out[1]
    assert np.max(np.abs(difference)) < 1e-7
    print np.max(np.abs(difference))

    print "Beginning Back prop"
    erra = [np.random.random(d.shape) for d in neon_out]
    err = [be.array(d) for d in erra]
    neon_layer.layers[0].layers[0].deltas = be.iobuf(inshape)
    neon_layer.bprop(err)

    bottom_neon_deltas = neon_layer.layers[0].layers[1].deltas.get()
    middle_neon_deltas = neon_layer.layers[1].layers[1].deltas.get()

    err0 = err[0]
    for ll in reversed(top_trunk):
        err0 = ll.bprop(err0)

    err1 = err[1]
    for ll in reversed(lbranch2):
        err1 = ll.bprop(err1)

    for bb, errb in zip((b1, b2, b3),
                        neon_layer.layers[0].layers[-5].error_views):
        for ll in reversed(bb):
            errb = ll.bprop(errb)

    # Now sum up the deltas at the root of the branch layer and compare
    ref_deltas = be.zeros_like(b1[0].deltas)
    ref_deltas[:] = b1[0].deltas + b2[0].deltas + b3[
        0].deltas + alpha2 * lbranch2[0].deltas

    neon_ref_deltas = ref_deltas.get()
    difference = middle_neon_deltas - neon_ref_deltas

    print np.max(np.abs(difference))
    assert np.max(np.abs(difference)) < 1e-8

    x = ref_deltas
    main2[0].deltas = be.iobuf(inshape)

    for ll in reversed(main2):
        x = ll.bprop(x)

    bottom_neon_ref_deltas = main2[1].deltas.get()
    difference = bottom_neon_deltas - bottom_neon_ref_deltas
    print np.max(np.abs(difference))
    assert np.max(np.abs(difference)) < 1e-8
def test_branch_model_fork_cpu(backend_cpu64):
    from neon.layers import BranchNode, Tree
    np.random.seed(0)
    be = NervanaObject.be
    be.bsz = 32
    bnode = BranchNode()
    i1 = inception([(32,), (32, 32), ('max', 16)])
    top1 = top_branch()
    top2 = top_branch()
    p1 = Sequential(main_branch() + [bnode, i1] + top1)
    p2 = [bnode] + top2

    alpha2 = 0.3
    neon_layer = Tree([p1, p2], alphas=[1.0, alpha2])

    inshape = (4, 224, 224)
    insize = np.prod(inshape)
    inpa = np.random.random((insize, batch_size))
    neon_layer.configure(inshape)
    inp = neon_layer.be.array(inpa)

    neon_layer.allocate()

    neon_layer.layers[0].layers[0].prev_layer = True
    neon_layer.allocate_deltas()

    neon_out_dev = neon_layer.fprop(inp)
    neon_out = [d.get() for d in neon_out_dev]

    # Now make the reference pathways:
    main_trunk2 = Sequential(main_branch())
    main_trunk2.configure(inshape)
    main2 = main_trunk2.layers
    main2[0].prev_layer = True
    main2[0].deltas = be.iobuf(inshape)

    branch2 = Sequential(top_branch())
    lbranch2 = branch2.layers
    (b1, b2, b3) = inception_bare(i1, [(32,), (32, 32), ('max', 16)])

    for bb in (b1, b2, b3, lbranch2):
        oshape = inshape
        for ll in main2 + bb:
            oshape = ll.configure(oshape)

    main1_trunk = neon_layer.layers[0].layers[:8]
    for ll, lo in zip(main2, main1_trunk):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})
        ll.allocate()
        temp_deltas = DeltasTree()
        temp_deltas.proc_layer(ll)
        temp_deltas.allocate_buffers()
        ll.set_deltas(temp_deltas)

    for ll, lo in zip(lbranch2, neon_layer.layers[1].layers[1:]):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})

    for bb in (b1, b2, b3, lbranch2):
        for ll in bb:
            ll.allocate()
            temp_deltas = DeltasTree()
            temp_deltas.proc_layer(ll)
            temp_deltas.allocate_buffers()
            ll.set_deltas(temp_deltas)

    # Create the combined output buffer
    merge_output = be.empty_like(neon_layer.layers[0].layers[9].outputs)

    x = inp
    for ll in main2:
        x = ll.fprop(x)
    main2_out = x

    start = 0
    for bb in (b1, b2, b3):
        xb = main2_out
        for ll in bb:
            xb = ll.fprop(xb)
        end = start + xb.shape[0]
        merge_output[start:end] = xb
        start = end

    x = merge_output

    top_trunk = Sequential(top1).layers
    for ll in top_trunk:
        x = ll.fprop(x)

    neon_out_ref = x.get()
    assert allclose_with_out(neon_out_ref, neon_out[0], rtol=0)

    # Now do second branch
    neon_out_ref2 = branch2.fprop(main2_out).get()
    assert allclose_with_out(neon_out_ref2, neon_out[1])

    neon_logger.display("Beginning Back prop")
    erra = [np.random.random(d.shape) for d in neon_out]
    err = [be.array(d) for d in erra]
    neon_layer.layers[0].layers[0].deltas = be.iobuf(inshape)
    neon_layer.bprop(err)

    bottom_neon_deltas = neon_layer.layers[0].layers[1].deltas.get()
    middle_neon_deltas = neon_layer.layers[1].layers[1].deltas.get()

    err0 = err[0]
    for ll in reversed(top_trunk):
        err0 = ll.bprop(err0)

    err1 = err[1]
    for ll in reversed(lbranch2):
        err1 = ll.bprop(err1)

    for bb, errb in zip((b1, b2, b3), neon_layer.layers[0].layers[-5].error_views):
        for ll in reversed(bb):
            errb = ll.bprop(errb)

    # Now sum up the deltas at the root of the branch layer and compare
    ref_deltas = be.zeros_like(b1[0].deltas)
    ref_deltas[:] = alpha2 * lbranch2[0].deltas
    ref_deltas[:] = ref_deltas + b3[0].deltas + b2[0].deltas + b1[0].deltas
    neon_ref_deltas = ref_deltas.get()
    assert allclose_with_out(middle_neon_deltas, neon_ref_deltas, rtol=0)

    x = ref_deltas
    main2[0].deltas = be.iobuf(inshape)

    for ll in reversed(main2):
        x = ll.bprop(x)

    bottom_neon_ref_deltas = main2[1].deltas.get()
    assert allclose_with_out(bottom_neon_deltas, bottom_neon_ref_deltas, rtol=0)
Ejemplo n.º 10
0
def test_branch_model_fork(backend_gpu):
    from neon.layers import BranchNode, Tree
    np.random.seed(0)
    be = NervanaObject.be
    if be.gpu_memory_size < 6.1 * 1024 * 1024 * 1024:
        pytest.skip(msg='Test requires more than 6.1GB')
    be.bsz = 64
    bnode = BranchNode()
    i1 = inception([(32,), (32, 32), ('max', 16)])
    top1 = top_branch()
    top2 = top_branch()
    p1 = Sequential(main_branch() + [bnode, i1] + top1)
    p2 = [bnode] + top2

    alpha2 = 0.3
    neon_layer = Tree([p1, p2], alphas=[1.0, alpha2])

    inshape = (4, 224, 224)
    insize = np.prod(inshape)
    inpa = np.random.random((insize, batch_size))
    neon_layer.configure(inshape)
    inp = neon_layer.be.array(inpa)

    neon_layer.allocate()

    neon_layer.layers[0].layers[0].prev_layer = True
    neon_layer.allocate_deltas()

    neon_out_dev = neon_layer.fprop(inp)
    neon_out = [d.get() for d in neon_out_dev]

    # Now make the reference pathways:
    main_trunk2 = Sequential(main_branch())
    main_trunk2.configure(inshape)
    main2 = main_trunk2.layers
    main2[0].prev_layer = True
    main2[0].deltas = be.iobuf(inshape)

    branch2 = Sequential(top_branch())
    lbranch2 = branch2.layers
    (b1, b2, b3) = inception_bare(i1, [(32,), (32, 32), ('max', 16)])

    for bb in (b1, b2, b3, lbranch2):
        oshape = inshape
        for ll in main2 + bb:
            oshape = ll.configure(oshape)

    main1_trunk = neon_layer.layers[0].layers[:8]
    for ll, lo in zip(main2, main1_trunk):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})
        ll.allocate()
        temp_deltas = DeltasTree()
        temp_deltas.proc_layer(ll)
        temp_deltas.allocate_buffers()
        ll.set_deltas(temp_deltas)

    for ll, lo in zip(lbranch2, neon_layer.layers[1].layers[1:]):
        if ll.has_params:
            ll.set_params({'params': {'W': lo.W.get()}})

    for bb in (b1, b2, b3, lbranch2):
        for ll in bb:
            ll.allocate()
            temp_deltas = DeltasTree()
            temp_deltas.proc_layer(ll)
            temp_deltas.allocate_buffers()
            ll.set_deltas(temp_deltas)

    # Create the combined output buffer
    merge_output = be.empty_like(neon_layer.layers[0].layers[9].outputs)

    x = inp
    for ll in main2:
        x = ll.fprop(x)
    main2_out = x

    start = 0
    for bb in (b1, b2, b3):
        xb = main2_out
        for ll in bb:
            xb = ll.fprop(xb)
        end = start + xb.shape[0]
        merge_output[start:end] = xb
        start = end

    x = merge_output

    top_trunk = Sequential(top1).layers
    for ll in top_trunk:
        x = ll.fprop(x)

    neon_out_ref = x.get()
    assert allclose_with_out(neon_out_ref, neon_out[0], rtol=0)

    # Now do second branch
    neon_out_ref2 = branch2.fprop(main2_out).get()
    assert allclose_with_out(neon_out_ref2, neon_out[1])

    neon_logger.display("Beginning Back prop")
    erra = [np.random.random(d.shape) for d in neon_out]
    err = [be.array(d) for d in erra]
    neon_layer.layers[0].layers[0].deltas = be.iobuf(inshape)
    neon_layer.bprop(err)

    bottom_neon_deltas = neon_layer.layers[0].layers[1].deltas.get()
    middle_neon_deltas = neon_layer.layers[1].layers[1].deltas.get()

    err0 = err[0]
    for ll in reversed(top_trunk):
        err0 = ll.bprop(err0)

    err1 = err[1]
    for ll in reversed(lbranch2):
        err1 = ll.bprop(err1)

    for bb, errb in zip((b1, b2, b3), neon_layer.layers[0].layers[-5].error_views):
        for ll in reversed(bb):
            errb = ll.bprop(errb)

    # Now sum up the deltas at the root of the branch layer and compare
    ref_deltas = be.zeros_like(b1[0].deltas)
    ref_deltas[:] = alpha2 * lbranch2[0].deltas
    ref_deltas[:] = ref_deltas + b3[0].deltas + b2[0].deltas + b1[0].deltas
    neon_ref_deltas = ref_deltas.get()
    assert allclose_with_out(middle_neon_deltas, neon_ref_deltas, rtol=0)

    x = ref_deltas
    main2[0].deltas = be.iobuf(inshape)

    for ll in reversed(main2):
        x = ll.bprop(x)

    bottom_neon_ref_deltas = main2[1].deltas.get()
    assert allclose_with_out(bottom_neon_deltas, bottom_neon_ref_deltas, rtol=0)
Ejemplo n.º 11
0
         }),
     )), ('m_l3', 0, 0, 0, 0, 0, {}), (share, 'shared', (
         (0, 16, 0, 0, 'b2_l1', 0, {'dense'}),
         (0, 10, 0, 0, 'b2_l2', 0, {
             'dense': True,
             'nonlinearity': Logistic(shortcut=True),
             'equal': ['target', 'b2', CrossEntropyBinary()]
         }),
     )), ('m_l4', )), {'target': l_y})

cost = GeneralizedCost(costfunc=CrossEntropyBinary())
cost, layers, tagslice = deepstacks.neon.get_loss(errors, watchpoints, cost)

print layers

network = Tree([network] + layers)

layers = [network]

inputs = deepstacks.neon.get_inputs(network)
targets = deepstacks.neon.get_targets(cost)

print inputs, targets
#assert tuple(inputs)==('image',)

#p1 = [l_in,
#        Affine(nout=100, name="m_l1", **normrelu),
#      b1,
#      Affine(nout=32, name="m_l2", **normrelu),
#      Affine(nout=16, name="m_l3", **normrelu),
#      b2,
Ejemplo n.º 12
0
def test_branch_model_fork():
    from neon.layers import BranchNode, Tree

    NervanaObject.be = gen_backend("gpu", batch_size=64)
    be = NervanaObject.be
    bnode = BranchNode()
    i1 = inception([(32,), (32, 32), ('max', 16)])
    top1 = top_branch()
    top2 = top_branch()
    p1 = Sequential(main_branch() + [bnode, i1] + top1)
    p2 = [bnode] + top2

    alpha2 = 0.3
    neon_layer = Tree([p1, p2], alphas=[1.0, alpha2])

    inshape = (3, 224, 224)
    insize = np.prod(inshape)
    inpa = np.random.random((insize, batch_size))
    neon_layer.configure(inshape)
    inp = neon_layer.be.array(inpa)

    neon_layer.allocate()
    print neon_layer.nested_str()
    neon_layer.layers[0].layers[0].prev_layer = True
    neon_layer.allocate_deltas()
    neon_layer.layers[0].layers[0].set_deltas([be.iobuf(inshape)])
    neon_out_dev = neon_layer.fprop(inp)
    neon_out = [d.get() for d in neon_out_dev]

    # Now make the reference pathways:
    main_trunk2 = Sequential(main_branch())
    main_trunk2.configure(inshape)
    main2 = main_trunk2.layers
    main2[0].prev_layer = True
    main2[0].set_deltas([be.iobuf(inshape)])

    branch2 = Sequential(top_branch())
    lbranch2 = branch2.layers
    (b1, b2, b3) = inception_bare(i1, [(32,), (32, 32), ('max', 16)])

    for bb in (b1, b2, b3, lbranch2):
        oshape = inshape
        for ll in main2 + bb:
            oshape = ll.configure(oshape)

    main1_trunk = neon_layer.layers[0].layers[:8]
    for ll, lo in zip(main2, main1_trunk):
        if ll.has_params:
            ll.set_params(lo.W.get())
        ll.allocate()
        ll.set_deltas([be.iobuf(ll.in_shape)])

    for ll, lo in zip(lbranch2, neon_layer.layers[1].layers[1:]):
        if ll.has_params:
            ll.set_params(lo.W.get())

    for bb in (b1, b2, b3, lbranch2):
        for ll in bb:
            ll.allocate()
            ll.set_deltas([be.iobuf(ll.in_shape)])

    # Create the combined output buffer
    merge_output = be.empty_like(neon_layer.layers[0].layers[9].outputs)

    x = inp
    for ll in main2:
        x = ll.fprop(x)
    main2_out = x

    start = 0
    for bb in (b1, b2, b3):
        xb = main2_out
        for ll in bb:
            xb = ll.fprop(xb)
        end = start + xb.shape[0]
        merge_output[start:end] = xb
        start = end

    x = merge_output

    top_trunk = Sequential(top1).layers
    for ll in top_trunk:
        x = ll.fprop(x)

    neon_out_ref = x.get()
    difference = neon_out_ref - neon_out[0]
    assert np.max(np.abs(difference)) < 1e-7
    print np.max(np.abs(difference))

    # Now do second branch
    neon_out_ref2 = branch2.fprop(main2_out).get()
    difference = neon_out_ref2 - neon_out[1]
    assert np.max(np.abs(difference)) < 1e-7
    print np.max(np.abs(difference))

    print "Beginning Back prop"
    erra = [np.random.random(d.shape) for d in neon_out]
    err = [be.array(d) for d in erra]
    neon_layer.layers[0].layers[0].deltas = be.iobuf(inshape)
    neon_layer.bprop(err)

    bottom_neon_deltas = neon_layer.layers[0].layers[1].deltas.get()
    middle_neon_deltas = neon_layer.layers[1].layers[1].deltas.get()

    err0 = err[0]
    for ll in reversed(top_trunk):
        err0 = ll.bprop(err0)

    err1 = err[1]
    for ll in reversed(lbranch2):
        err1 = ll.bprop(err1)

    for bb, errb in zip((b1, b2, b3), neon_layer.layers[0].layers[-5].error_views):
        for ll in reversed(bb):
            errb = ll.bprop(errb)

    # Now sum up the deltas at the root of the branch layer and compare
    ref_deltas = be.zeros_like(b1[0].deltas)
    ref_deltas[:] = b1[0].deltas + b2[0].deltas + b3[0].deltas + alpha2 * lbranch2[0].deltas

    neon_ref_deltas = ref_deltas.get()
    difference = middle_neon_deltas - neon_ref_deltas

    print np.max(np.abs(difference))
    assert np.max(np.abs(difference)) < 1e-8

    x = ref_deltas
    main2[0].deltas = be.iobuf(inshape)

    for ll in reversed(main2):
        x = ll.bprop(x)

    bottom_neon_ref_deltas = main2[1].deltas.get()
    difference = bottom_neon_deltas - bottom_neon_ref_deltas
    print np.max(np.abs(difference))
    assert np.max(np.abs(difference)) < 1e-8
Ejemplo n.º 13
0
    l_in, (
        (0, 100, 0, 0, 0, 0, {'dense', 'nobias'}),
        (0, 10, 0, 0, 0, 0, {
            'dense': True,
            'nonlinearity': Logistic(shortcut=True),
            'nobias': True
        }),
    ))

# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=CrossEntropyBinary())

cost, extra_layers, tagslice = deepstacks.neon.get_loss(
    errors, watchpoints, cost)

network = Tree([network] + extra_layers)

deepstacks.neon.utils.walk(network)

inputs = deepstacks.neon.get_inputs(network)

assert tuple(inputs) == ('image', )

#print network.get_description()

layers = network

# setup optimizer
optimizer = GradientDescentMomentum(0.1,
                                    momentum_coef=0.9,
                                    stochastic_round=args.rounding)