def CNN(x,c_l1,c_l2,f_l1,f_l2,insize): print "in size ", insize conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1) conv1_shp=get_conv_output_shape(insize,c_l1.get_value().shape,border_mode='valid',subsample=(1,1)) print "conv1 size ", conv1_shp pool1=pool_2d(conv1,(3,3),st=(3,3),ignore_border=True) #default maxpool pool1_shp=get_pool_output_shape(conv1_shp,pool_size=(3,3),st=(3,3),ignore_border=True) print "pool1 size ", pool1_shp lrn1=LRN(pool1,pool1_shp) lrn1_shp=tuple(pool1_shp) print "cross map norm1 size ", lrn1_shp conv2=tensor.nnet.relu(conv2d(lrn1,c_l2)) conv2_shp=get_conv_output_shape(lrn1_shp,c_l2.get_value().shape,border_mode='valid',subsample=(1,1)) print "conv2 size ", conv2_shp pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True) pool2_shp=get_pool_output_shape(conv2_shp,pool_size=(2,2),st=(2,2),ignore_border=True) print "pool2 size ", pool2_shp lrn2=LRN(pool2,pool2_shp) lrn2_shp=tuple(pool2_shp) print "cross map norm2 size " , lrn2_shp fpool2=tensor.flatten(lrn2,outdim=2) full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1)) pyx=tensor.nnet.sigmoid(tensor.dot(full1,f_l2)) return c_l1, c_l2, f_l1, f_l2, pyx
def test_constant_input(self): # Check the AbstractConv Ops for constant inputs input = self.input filters = self.filters topgrad = self.topgrad constant_tensor = self.constant_tensor out_shape = tensor.lvector() # Check the forward Op output = conv.conv2d(constant_tensor, filters) grad_filters = theano.grad(output.sum(), wrt=filters) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type) output = conv.conv2d(input, constant_tensor) grad_input = theano.grad(output.sum(), wrt=input) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) # Check grad wrt weights grad_filters = conv.AbstractConv2d_gradWeights()(constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_filters.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_filters = conv.AbstractConv2d_gradWeights()(input, constant_tensor, out_shape) grad_input = theano.grad(grad_filters.sum(), wrt=input) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) # Check grad wrt inputs grad_input = conv.AbstractConv2d_gradInputs()(constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_input.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()(filters, constant_tensor, out_shape) grad_filters = theano.grad(grad_input.sum(), wrt=filters) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type)
def run_fwd(self, inputs_shape, filters_shape, ref=conv_corr, subsample=(1, 1), verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([ isinstance(n.op, target_op) for n in f.maker.fgraph.toposort() ]) self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace')) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv.AbstractConv2d(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode)
def run_fwd( self, inputs_shape, filters_shape, ref=dnn_conv, subsample=(1, 1), verify_grad=True, mode=mode_without_gpu, border_mode="valid", filter_flip=True, device="cpu", provide_shape=False, target_op=None, ): inputs_val = numpy.random.random(inputs_shape).astype("float32") filters_val = numpy.random.random(filters_shape).astype("float32") if device == "gpu": inputs = gpu_shared(inputs_val) filters = gpu_shared(filters_val) else: inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val)) filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val)) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = "conv" else: conv_mode = "cross" c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d( inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, ) f_ref = theano.function([], c_ref, mode=mode) f = theano.function([], c, mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, "trace")) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad( conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode, )
def CNN(x,c_l1,c_l2,f_l1,f_l2,PP,ims): print ims #------- #conv3D get rid of dependency of the number of input image channel b=numpy.zeros(c_l1.get_value().shape[0]) conv1=tensor.nnet.relu(conv3D(x.dimshuffle(0,2,3,1,'x'),c_l1.dimshuffle(0,2,3,1,'x'),b,d=(1,1,1))) # shuffle dimensions conv1=tensor.sum(conv1,axis=3) #add the dimension of channels conv1=conv1.dimshuffle(0,3,1,2) #shuffle back to same dimension as conv2D #--------- #conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1) conv1_shp=get_conv_output_shape(ims,c_l1.get_value().shape,border_mode='valid',subsample=(1,1)) print conv1_shp #pp=tensor.reshape(conv1,conv1_shp[:2]+(conv1_shp[2]*conv1_shp[3],)) #print pp pool1=pool_2d(conv1,(2,2),st=(2,2),ignore_border=True) #default maxpool pool1_shp=get_pool_output_shape(conv1_shp,pool_size=(2,2),st=(2,2),ignore_border=True) print pool1_shp conv2=tensor.nnet.relu(conv2d(pool1,c_l2)) conv2_shp=get_conv_output_shape(pool1_shp,c_l2.get_value().shape,border_mode='valid',subsample=(1,1)) print conv2_shp #pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True) pool2=spp(conv2,conv2_shp,PP,'max') fpool2=tensor.flatten(pool2,outdim=2) full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1)) pyx=tensor.nnet.softmax(tensor.dot(full1,f_l2)) return c_l1, c_l2, f_l1, f_l2, pyx
def test_grad_types(self): # This function simply tests the behaviour of the AbstractConv # Ops, not their optimizations cpu_input = tensor.ftensor4() cpu_filters = tensor.ftensor4() cpu_topgrad = tensor.ftensor4() gpu_input = gpu_ftensor4() gpu_filters = gpu_ftensor4() gpu_topgrad = gpu_ftensor4() out_shape = tensor.lvector() # Check the gradient of the forward conv2d for input, filters in itertools.product((cpu_input, gpu_input), (cpu_filters, gpu_filters)): output = conv.conv2d(input, filters) grad_input, grad_filters = theano.grad(output.sum(), wrt=(input, filters)) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type) # Check the gradient of gradweight for input, topgrad in itertools.product((cpu_input, gpu_input), (cpu_topgrad, gpu_topgrad)): grad_filters = conv.AbstractConv2d_gradWeights()(input, topgrad, out_shape) grad_input, grad_topgrad = theano.grad(grad_filters.sum(), wrt=(input, topgrad)) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) # Check the gradient of gradinputs for filters, topgrad in itertools.product((cpu_filters, gpu_filters), (cpu_topgrad, gpu_topgrad)): grad_input = conv.AbstractConv2d_gradInputs()(filters, topgrad, out_shape) grad_filters, grad_topgrad = theano.grad(grad_input.sum(), wrt=(filters, topgrad)) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type)
def optimizer_2d(self, input_shapes, direction, include_tags, exclude_tags, op, border_mode='valid', subsample=(1, 1), filter_dilation=(1, 1)): inp1 = theano.shared( np.random.random(input_shapes[0]).astype(theano.config.floatX)) inp2 = theano.shared( np.random.random(input_shapes[1]).astype(theano.config.floatX)) if (direction == 0): conv_op = abstract_conv.conv2d(inp1, inp2, input_shapes[0], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if (direction == 1): conv_op = abstract_conv.conv2d_grad_wrt_weights( inp1, inp2, input_shapes[2], input_shapes[0], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if (direction == 2): conv_op = abstract_conv.conv2d_grad_wrt_inputs( inp1, inp2, input_shapes[2], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) theano.config.metaopt.optimizer_including = include_tags theano.config.metaopt.optimizer_excluding = exclude_tags mode = mode_with_gpu.including('conv_meta') ref_func = theano.function([], conv_op, mode=mode_with_gpu) # All meta optimizer compile a new function. This need to know # the current linker, but this information is not available, # so it use the default mode. with theano.change_flags(mode=mode): conv_func = theano.function([], conv_op, mode=mode) assert any([ isinstance(node.op, op) for node in conv_func.maker.fgraph.toposort() ]) utt.assert_allclose(conv_func(), ref_func())
def test_constant_input(self): # Check the AbstractConv Ops for constant inputs input = self.input filters = self.filters topgrad = self.topgrad constant_tensor = self.constant_tensor out_shape = tensor.lvector() # Check the forward Op output = conv.conv2d(constant_tensor, filters) grad_filters = theano.grad(output.sum(), wrt=filters) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) output = conv.conv2d(input, constant_tensor) grad_input = theano.grad(output.sum(), wrt=input) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) # Check grad wrt weights grad_filters = conv.AbstractConv2d_gradWeights()( constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_filters.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_filters = conv.AbstractConv2d_gradWeights()( input, constant_tensor, out_shape) grad_input = theano.grad(grad_filters.sum(), wrt=input) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) # Check grad wrt inputs grad_input = conv.AbstractConv2d_gradInputs()( constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_input.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()( filters, constant_tensor, out_shape) grad_filters = theano.grad(grad_input.sum(), wrt=filters) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type)
def run_fwd(self, inputs_shape, filters_shape, ref=dnn_conv, subsample=(1, 1), verify_grad=True, mode=mode_without_gpu, border_mode='valid', filter_flip=True, device='cpu', provide_shape=False): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') if device == 'gpu': inputs = gpu_shared(inputs_val) filters = gpu_shared(filters_val) else: inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val)) filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val)) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp) f_ref = theano.function([], c_ref, mode=mode) f = theano.function([], c, mode) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode)
def optimizer_2d(self, input_shapes, direction, include_tags, exclude_tags, op, border_mode='valid', subsample=(1, 1), filter_dilation=(1, 1)): inp1 = theano.shared( np.random.random(input_shapes[0]).astype(theano.config.floatX)) inp2 = theano.shared( np.random.random(input_shapes[1]).astype(theano.config.floatX)) if (direction == 0): conv_op = abstract_conv.conv2d(inp1, inp2, input_shapes[0], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if (direction == 1): conv_op = abstract_conv.conv2d_grad_wrt_weights( inp1, inp2, input_shapes[2], input_shapes[0], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if (direction == 2): conv_op = abstract_conv.conv2d_grad_wrt_inputs( inp1, inp2, input_shapes[2], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) theano.config.metaopt.optimizer_including = include_tags theano.config.metaopt.optimizer_excluding = exclude_tags mode = mode_with_gpu.including('conv_meta') ref_func = theano.function([], conv_op, mode=mode_with_gpu) conv_func = theano.function([], conv_op, mode=mode) assert any([ isinstance(node.op, op) for node in conv_func.maker.fgraph.toposort() ]) utt.assert_allclose(conv_func(), ref_func())
def run_fwd( self, inputs_shape, filters_shape, ref=dnn_conv, subsample=(1, 1), verify_grad=True, mode=mode_without_gpu, border_mode="valid", filter_flip=True, device="cpu", provide_shape=False, ): inputs_val = numpy.random.random(inputs_shape).astype("float32") filters_val = numpy.random.random(filters_shape).astype("float32") if device == "gpu": inputs = gpu_shared(inputs_val) filters = gpu_shared(filters_val) else: inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val)) filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val)) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = "conv" else: conv_mode = "cross" c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d( inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, ) f_ref = theano.function([], c_ref, mode=mode) f = theano.function([], c, mode) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad( conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode, )
def run_fwd(self, inputs_shape, filters_shape, ref=conv_corr, subsample=(1, 1), verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None, check_trace=False, filter_dilation=(1, 1)): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) c = conv.conv2d(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, filter_dilation=filter_dilation) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv.AbstractConv2d(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample, filter_dilation=filter_dilation), [inputs_val, filters_val], mode=mode)
def CNN(x,c_l1,c_l2,f_l1,f_l2,PP,ims): print ims conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1) conv1_shp=get_conv_output_shape(ims,c_l1.get_value().shape,border_mode='valid',subsample=(1,1)) print conv1_shp pp=tensor.reshape(conv1,conv1_shp[:2]+(conv1_shp[2]*conv1_shp[3],)) print pp pool1=pool_2d(conv1,(2,2),st=(2,2),ignore_border=True) #default maxpool pool1_shp=get_pool_output_shape(conv1_shp,pool_size=(2,2),st=(2,2),ignore_border=True) print pool1_shp conv2=tensor.nnet.relu(conv2d(pool1,c_l2)) conv2_shp=get_conv_output_shape(pool1_shp,c_l2.get_value().shape,border_mode='valid',subsample=(1,1)) print conv2_shp #pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True) pool2=spp(conv2,conv2_shp,PP,'max') fpool2=tensor.flatten(pool2,outdim=2) full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1)) pyx=tensor.nnet.sigmoid(tensor.dot(full1,f_l2)) return c_l1, c_l2, f_l1, f_l2, pyx
def optimizer_2d(self, input_shapes, direction, include_tags, exclude_tags, op, border_mode='valid', subsample=(1, 1), filter_dilation=(1, 1)): inp1 = theano.shared(np.random.random(input_shapes[0]).astype(theano.config.floatX)) inp2 = theano.shared(np.random.random(input_shapes[1]).astype(theano.config.floatX)) if(direction == 0): conv_op = abstract_conv.conv2d(inp1, inp2, input_shapes[0], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if(direction == 1): conv_op = abstract_conv.conv2d_grad_wrt_weights(inp1, inp2, input_shapes[2], input_shapes[0], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) if(direction == 2): conv_op = abstract_conv.conv2d_grad_wrt_inputs(inp1, inp2, input_shapes[2], input_shapes[1], border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) theano.config.metaopt.optimizer_including = include_tags theano.config.metaopt.optimizer_excluding = exclude_tags mode = mode_with_gpu.including('conv_meta') ref_func = theano.function([], conv_op, mode=mode_with_gpu) conv_func = theano.function([], conv_op, mode=mode) assert any([isinstance(node.op, op) for node in conv_func.maker.fgraph.toposort()]) utt.assert_allclose(conv_func(), ref_func())
def test_grad_types(self): # This function simply tests the behaviour of the AbstractConv # Ops, not their optimizations input = self.input filters = self.filters topgrad = self.topgrad out_shape = tensor.lvector() output = conv.conv2d(input, filters) grad_input, grad_filters = theano.grad(output.sum(), wrt=(input, filters)) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type) grad_filters = conv.AbstractConv2d_gradWeights()(input, topgrad, out_shape) grad_input, grad_topgrad = theano.grad(grad_filters.sum(), wrt=(input, topgrad)) assert grad_input.type == input.type, (grad_input, grad_input.type, input, input.type) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()(filters, topgrad, out_shape) grad_filters, grad_topgrad = theano.grad(grad_input.sum(), wrt=(filters, topgrad)) assert grad_filters.type == filters.type, (grad_filters, grad_filters.type, filters, filters.type) assert grad_topgrad.type == topgrad.type, (grad_topgrad, grad_topgrad.type, topgrad, topgrad.type)
def test_grad_types(self): # This function simply tests the behaviour of the AbstractConv # Ops, not their optimizations input = self.input filters = self.filters topgrad = self.topgrad out_shape = tensor.lvector() output = conv.conv2d(input, filters) grad_input, grad_filters = theano.grad(output.sum(), wrt=(input, filters)) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) grad_filters = conv.AbstractConv2d_gradWeights()( input, topgrad, out_shape) grad_input, grad_topgrad = theano.grad(grad_filters.sum(), wrt=(input, topgrad)) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()( filters, topgrad, out_shape) grad_filters, grad_topgrad = theano.grad(grad_input.sum(), wrt=(filters, topgrad)) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type)