def buildModel(mtype=1): print "BUILDING MODEL TYPE", mtype, "..." #default settings (Model 1) filters = 64 first_stride = 2 last_filter_multiplier = 16 #specific model type settings (see working notes for details) if mtype == 2: first_stride = 1 elif mtype == 3: filters = 32 last_filter_multiplier = 8 #input layer net = l.InputLayer((None, IM_DIM, IM_SIZE[1], IM_SIZE[0])) #conv layers net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters, filter_size=7, pad='same', stride=first_stride, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) if mtype == 2: net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters, filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 2, filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 4, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 8, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * last_filter_multiplier, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) print "\tFINAL POOL OUT SHAPE:", l.get_output_shape(net) #dense layers net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) #Classification Layer if MULTI_LABEL: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1)) else: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.softmax, W=init.HeNormal(gain=1)) print "...DONE!" #model stats print "MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS" print "MODEL HAS", l.count_params(net), "PARAMS" return net
def buildModel(): print "BUILDING MODEL TYPE..." #default settings filters = 32 first_stride = 2 last_filter_multiplier = 4 #input layer net = l.InputLayer((None, IM_DIM, IM_SIZE[1], IM_SIZE[0])) #conv layers net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters , filter_size=7, pad='same', stride=first_stride, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 2 , filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 4 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 8 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 16 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) #net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 32 , filter_size=7, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) #net = l.MaxPool2DLayer(net, pool_size=2) #print "\tFINAL POOL OUT SHAPE:", l.get_output_shape(net) #dense layers net = l.batch_norm(l.DenseLayer(net, 256, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.DropoutLayer(net, DROPOUT) net = l.batch_norm(l.DenseLayer(net, 256, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.DropoutLayer(net, DROPOUT) #Classification Layer if MULTI_LABEL: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1)) else: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1)) print "...DONE!" #model stats print "MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS" print "MODEL HAS", l.count_params(net), "PARAMS" return net
def __init__(self, incoming, n_units, svi=True, mW_init=linit.HeNormal(), mb_init=linit.Constant([0.]), sW_init=linit.Constant([-5.]), sb_init=linit.Constant([-5.]), actfun=lnl.softmax, **kwargs): """Mixture weights layer with optional weight uncertainty If n_units > 1, this becomes a fully-connected layer. Else, no parameters are added, and the output defaults to weight 1. See ``delfi.neuralnet.layers.FullyConnected`` for docstring """ self.n_units = n_units if n_units > 1: super(MixtureWeightsLayer, self).__init__( incoming, n_units, svi=svi, mW_init=mW_init, mb_init=mb_init, sW_init=sW_init, sb_init=sb_init, actfun=actfun, **kwargs) else: # init of lasagne.layers.Layer super(FullyConnectedLayer, self).__init__(incoming, **kwargs)
def initialization(name): initializations = { 'sigmoid': init.HeNormal(gain=1.0), 'softmax': init.HeNormal(gain=1.0), 'elu': init.HeNormal(gain=1.0), 'relu': init.HeNormal(gain=math.sqrt(2)), 'lrelu': init.HeNormal(gain=math.sqrt(2 / (1 + 0.01**2))), 'vlrelu': init.HeNormal(gain=math.sqrt(2 / (1 + 0.33**2))), 'rectify': init.HeNormal(gain=math.sqrt(2)), 'identity': init.HeNormal(gain=math.sqrt(2)) } return initializations[name]
def __init__(self, incoming, n_units, svi=True, mW_init=linit.HeNormal(), mb_init=linit.Constant([0.]), sW_init=linit.Constant([-5.]), sb_init=linit.Constant([-5.]), actfun=lnl.tanh, **kwargs): """Fully connected layer with optional weight uncertainty Parameters ---------- incoming : lasagne.layers.Layer instance Incoming layer n_units : int Number of units svi : bool Weight uncertainty mW_init : function Function to initialise weights for mean of weight (multiplicative) mb_init : function Function to initialise weights for mean of weight (bias) sW_init : function Function to initialise weights for log std of weight (multiplicative) sb_init : function Function to initialise weights for log std of weight (bias) actfun : function Activation function """ super(FullyConnectedLayer, self).__init__(incoming, **kwargs) self.n_units = n_units self.actfun = actfun self.svi = svi self.mW = self.add_param(mW_init, (self.input_shape[1], self.n_units), name='mW', mp=True, wp=True) self.mb = self.add_param(mb_init, (self.n_units,), name='mb', mp=True, bp=True) if self.svi: self._srng = RandomStreams( lasagne.random.get_rng().randint( 1, 2147462579)) self.sW = self.add_param(sW_init, (self.input_shape[1], self.n_units), name='sW', sp=True, wp=True) self.sb = self.add_param(sb_init, (self.n_units,), name='sb', sp=True, bp=True)
def __init__(self, incoming, n_components, n_dim, svi=True, rank=None, homoscedastic=False, mWs_init=linit.HeNormal(), mbs_init=linit.Constant([0.]), sWs_init=linit.Constant([-5.]), sbs_init=linit.Constant([-5.]), min_precisions=None, **kwargs): """Fully connected layer for mixture precisions, optional weight uncertainty Parameters ---------- incoming : lasagne.layers.Layer instance Incoming layer n_components : int Number of components n_dim : int Dimensionality of output vector svi : bool Weight uncertainty mWs_init : function Function to initialise weights for mean of weight (multiplicative); applied per component mbs_init : function Function to initialise weights for mean of weight (bias); applied per component sWs_init : function Function to initialise weights for log std of weight (multiplicative); applied per component sbs_init : function Function to initialise weights for log std of weight (bias); applied per component min_precisions: 1D numpy array of float32 or None Minimum values for the diagonal elements of the precision matrix, for all components """ super(MixturePrecisionsLayer, self).__init__(incoming, **kwargs) self.n_components = n_components self.rank = rank assert not homoscedastic self.homoscedastic = homoscedastic self.n_dim = n_dim self.svi = svi self.mWs = [ self.add_param(mWs_init, (self.input_shape[1], self.n_dim**2), name='mW' + str(c), mp=True, wp=True) for c in range(n_components) ] self.mbs = [ self.add_param(mbs_init, (self.n_dim**2, ), name='mb' + str(c), mp=True, bp=True) for c in range(n_components) ] if self.svi: self._srng = RandomStreams(lasagne.random.get_rng().randint( 1, 2147462579)) self.sWs = [ self.add_param(sWs_init, (self.input_shape[1], self.n_dim**2), name='sW' + str(c), sp=True, wp=True) for c in range(n_components) ] self.sbs = [ self.add_param(sbs_init, (self.n_dim**2, ), name='sb' + str(c), sp=True, bp=True) for c in range(n_components) ] if min_precisions is not None: assert min_precisions.ndim == 1 and \ min_precisions.size == self.n_dim, "invalid min precisions" min_precisions = min_precisions.astype(dtype) self.min_U_column_norms = np.sqrt(min_precisions) else: self.min_U_column_norms = None
def __init__(self, incoming, n_components, n_dim, svi=True, mWs_init=linit.HeNormal(), mbs_init=linit.Normal(1.), sWs_init=linit.Constant([-5.]), sbs_init=linit.Constant([-5.]), **kwargs): """Fully connected layer for mixture means, optional weight uncertainty Parameters ---------- incoming : lasagne.layers.Layer instance Incoming layer n_components : int Number of components n_dim : int Dimensionality of output vector svi : bool Weight uncertainty mWs_init : function Function to initialise weights for mean of weight (multiplicative); applied per component mbs_init : function Function to initialise weights for mean of weight (bias); applied per component sWs_init : function Function to initialise weights for log std of weight (multiplicative); applied per component sbs_init : function Function to initialise weights for log std of weight (bias); applied per component """ super(MixtureMeansLayer, self).__init__(incoming, **kwargs) self.n_components = n_components self.n_dim = n_dim self.svi = svi self.mWs = [self.add_param(mWs_init, (self.input_shape[1], self.n_dim), name='mW' + str(c), mp=True, wp=True) for c in range(n_components)] self.mbs = [self.add_param(mbs_init, (self.n_dim,), name='mb' + str(c), mp=True, bp=True) for c in range(n_components)] if self.svi: self._srng = RandomStreams( lasagne.random.get_rng().randint( 1, 2147462579)) self.sWs = [self.add_param(sWs_init, (self.input_shape[1], self.n_dim), name='sW' + str(c), sp=True, wp=True) for c in range(n_components)] self.sbs = [self.add_param(sbs_init, (self.n_dim,), name='sb' + str(c), sp=True, bp=True) for c in range(n_components)]
def run(self, shape): return LI.HeNormal(gain=self.gain).sample(shape)