def normalize(input_, output_dim): if normalization == 'off': return input_, None, None #normed = tensor.clip(normed, -3., 3.) #out = G * normed + B #params = [G, B] if normalization == 'bn1': output = (input_ - input_.mean( axis=0, keepdims=True)) / (input_.std( axis=0, keepdims=True) + 1E-6) return output, None, None if normalization == 'bn2': M = shared_floatx_nans((output_dim,)) M.name = 'M' S = shared_floatx_nans((output_dim,)) S.name = 'S' #M = input_.mean(axis=0, keepdims=True) #S = input_.std(axis=0, keepdims=True) output = (input_ - M) / (S + 1E-6) return output, M, S
def _allocate(self): self.W_state = shared_floatx_nans((self.dim, 4*self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim,), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim,), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim,), name='W_cell_to_out') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state") self.initial_cells = shared_floatx_zeros((self.dim,), name="initial_cells") add_role(self.W_state, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) add_role(self.initial_cells, INITIAL_STATE) self.parameters = [ self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_cell_to_out, self.initial_state_, self.initial_cells]
def _allocate(self): self.W_patch = shared_floatx_nans( (np.prod(self.patch_shape), 4 * self.dim), name='W_input') self.W_state = shared_floatx_nans((self.dim, 4 * self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim, ), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim, ), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim, ), name='W_cell_to_out') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim, ), name="initial_state") self.initial_cells = shared_floatx_zeros((self.dim, ), name="initial_cells") add_role(self.W_state, WEIGHT) add_role(self.W_patch, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) add_role(self.initial_cells, INITIAL_STATE) self.parameters = [ self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_patch, self.W_cell_to_out, self.initial_state_, self.initial_cells ]
def _allocate(self): self.W_patch = shared_floatx_nans((np.prod(self.patch_shape) + 4, 4 * self.dim), name="W_patch") self.b = shared_floatx_nans((4 * self.dim,), name="b") self.W_state = shared_floatx_nans((self.dim, 4 * self.dim), name="W_state") # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state") self.initial_cells = shared_floatx_zeros((self.dim,), name="initial_cells") self.initial_location = shared_floatx_zeros((2,), name="initial_location") self.initial_scale = shared_floatx_zeros((2,), name="initial_scale") add_role(self.W_state, WEIGHT) add_role(self.b, BIAS) add_role(self.W_patch, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) add_role(self.initial_cells, INITIAL_STATE) add_role(self.initial_location, INITIAL_STATE) add_role(self.initial_scale, INITIAL_STATE) self.parameters = [ self.W_state, self.W_patch, self.b, self.initial_state_, self.initial_cells, self.initial_location, self.initial_scale, ]
def _allocate(self): W = shared_floatx_nans((self.length, self.dim), name='W_lookup') self.parameters.append(W) add_role(W, WEIGHT) b = shared_floatx_nans((self.dim,), name='b_lookup') self.parameters.append(b) add_role(b, BIAS)
def _allocate(self): W = shared_floatx_nans((self.length, self.dim), name='W_lookup') self.parameters.append(W) add_role(W, WEIGHT) b = shared_floatx_nans((self.dim, ), name='b_lookup') self.parameters.append(b) add_role(b, BIAS)
def _allocate(self): self.padding = shared_floatx_nans((self.dim, ), name='padding') add_role(self.padding, WEIGHT) self.location_component = shared_floatx_nans( (self.max_length, self.dim), name='loc_comp') add_role(self.location_component, WEIGHT) self.parameters = [self.padding, self.location_component]
def _allocate(self): self.W_state = shared_floatx_nans((self.dim, 4*self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim,), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim,), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim,), name='W_cell_to_out') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state") self.initial_cells = shared_floatx_zeros((self.dim,), name="initial_cells") add_role(self.W_state, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) add_role(self.initial_cells, INITIAL_STATE) # gamma gamma_val = 0.1 * numpy.ones((self.dim), dtype=config.floatX) self.gamma = shared(name='gamma', value=gamma_val) add_role(self.gamma, PARAMETER) # beta beta_val = numpy.zeros((self.dim), dtype=config.floatX) self.beta = shared(name='beta', value=beta_val) add_role(self.beta, PARAMETER) self.parameters = [ self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_cell_to_out, self.initial_state_, self.initial_cells, self.gamma, self.beta]
def __allocate(self, input_dim, output_dim0, output_dim1): W = shared_floatx_nans((input_dim, output_dim0, output_dim1), name='W') add_role(W, WEIGHT) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') b = shared_floatx_nans((output_dim0, output_dim1), name='b') add_role(b, BIAS) self.parameters.append(b)
def _allocate(self): super(Convolutional, self)._allocate() log_gamma = shared_floatx_nans((self.num_filters, ), name='log_gamma') beta = shared_floatx_nans((self.num_filters, ), name='beta') self.parameters.append(log_gamma) self.parameters.append(beta) add_role(log_gamma, WEIGHT) add_role(beta, BIAS)
def _allocate(self): self.parameters.append(shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim), name='state_to_gates')) for i in range(2): if self.parameters[i]: add_role(self.parameters[i], WEIGHT)
def _allocate(self): super(Linear, self)._allocate() log_gamma = shared_floatx_nans((self.output_dim, ), name='log_gamma') beta = shared_floatx_nans((self.output_dim, ), name='beta') self.parameters.append(log_gamma) self.parameters.append(beta) add_role(log_gamma, WEIGHT) add_role(beta, BIAS)
def _allocate(self): W = shared_floatx_nans((self.n_out, self.dwin * self.vector_size), name='W') b = shared_floatx_nans((self.n_out, ), name='b') add_role(b, BIAS) add_role(W, WEIGHT) self.parameters.append(W) self.parameters.append(b) self.mlp.allocate()
def _allocate(self): W = shared_floatx_nans((self.input_dim, self.attention_dim), name='W') add_role(W, WEIGHT) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if self.use_bias: b = shared_floatx_nans((1, ), name='b') add_role(b, BIAS) self.parameters.append(b)
def _allocate(self): W = shared_floatx_nans((self.input_dim, self.output_dim), name='W') add_role(W, WEIGHT) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if self.use_bias: b = shared_floatx_nans((self.output_dim,), name='b') add_role(b, BIAS) self.parameters.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self): W = shared_floatx_nans((self.num_filters, self.input_dim, self.filter_length, 1), name='W') add_role(W, FILTER) self.params.append(W) if self.use_bias: b = shared_floatx_nans((self.num_filters, ), name='b') add_role(b, BIAS) self.params.append(b)
def _allocate(self): W = shared_floatx_nans((self.input_dim, self.output_dim), name='W') add_role(W, WEIGHT) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if getattr(self, 'use_bias', True): b = shared_floatx_nans((self.output_dim, ), name='b') add_role(b, BIAS) self.parameters.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self): self.parameters.append(shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim), name='state_to_gates')) self.parameters.append(shared_floatx_zeros((self.dim,), name="initial_state")) for i in range(2): if self.parameters[i]: add_role(self.parameters[i], WEIGHT) add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self): W = shared_floatx_nans( (self.num_filters, self.input_dim, self.filter_length, 1), name='W') add_role(W, FILTER) self.params.append(W) if self.use_bias: b = shared_floatx_nans((self.num_filters, ), name='b') add_role(b, BIAS) self.params.append(b)
def _allocate(self): W = shared_floatx_nans( (self.num_filters, self.num_channels) + self.filter_size, name='W') add_role(W, FILTER) self.params.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if self.use_bias: b = shared_floatx_nans(self.get_dim('output'), name='b') add_role(b, BIAS) self.params.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self): W = shared_floatx_nans((self.num_filters, self.num_channels) + self.filter_size, name='W') add_role(W, FILTERS) self.params.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if self.use_bias: b = shared_floatx_nans(self.get_dim('output'), name='b') add_role(b, BIASES) self.params.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self): self.Wh = shared_floatx_nans((self.dim, self.dim), name='Wh') self.Wi = shared_floatx_nans((self.input_dim, self.dim), name='Wi') self.b = shared_floatx_nans((1,self.dim), name='b') self.params.append(self.Wi) self.params.append(self.Wh) self.params.append(self.b) add_role(self.Wi, WEIGHT) add_role(self.Wh, WEIGHT) add_role(self.b, BIAS)
def _allocate(self): self.parameters.append(shared_floatx_nans((self.dim, self.dim), name='state_to_state')) add_role(self.parameters[-1], WEIGHT) self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim), name='state_to_gates')) add_role(self.parameters[-1], WEIGHT) self.parameters.append(shared_floatx_nans((self.dim,), name="initial_state")) add_role(self.parameters[-1], INITIAL_STATE)
def _allocate(self): W = shared_floatx_nans((self.num_filters, self.num_channels) + self.filter_size, name="W") add_role(W, FILTER) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name="W_norm") if self.use_bias: if self.shared_bias: b = shared_floatx_nans(self.num_filters, name="b") else: b = shared_floatx_nans(self.get_dim("output"), name="b") add_role(b, BIAS) self.parameters.append(b) self.add_auxiliary_variable(b.norm(2), name="b_norm")
def _allocate(self): W = shared_floatx_nans((self.visible_dim, self.hidden_dim), name='Wrbm') add_role(W, WEIGHT) self.params.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') bv = shared_floatx_nans((self.visible_dim,), name='bv') add_role(bv, BIAS) self.params.append(bv) self.add_auxiliary_variable(bv.norm(2), name='bv_norm') bh = shared_floatx_nans((self.hidden_dim,), name='bh') add_role(bh, BIAS) self.params.append(bh) self.add_auxiliary_variable(bh.norm(2), name='bh_norm')
def _allocate(self): self.W_ss = shared_floatx_nans((self.dim, 4*self.dim), name='W_ss') self.W_is = shared_floatx_nans((self.dim,), name='W_is') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state") add_role(self.W_ss, WEIGHT) add_role(self.W_is, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) self.parameters = [ self.W_ss, self.W_is, self.initial_state_]
def _allocate(self): self.W_state = shared_floatx_nans((self.dim, 4 * self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim, ), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim, ), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim, ), name='W_cell_to_out') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim, ), name="initial_state") self.initial_cells = shared_floatx_zeros((self.dim, ), name="initial_cells") add_role(self.W_state, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) add_role(self.initial_cells, INITIAL_STATE) # layer_norm params scale_add = 0.0 scale_mul = 1.0 self.b1 = scale_add * (shared_floatx_zeros( (4 * self.dim), name="b1") + 1) self.b2 = scale_add * (shared_floatx_zeros( (4 * self.dim), name="b2") + 1) self.b3 = scale_add * (shared_floatx_zeros( (1 * self.dim), name="b3") + 1) self.s1 = scale_mul * (shared_floatx_zeros( (4 * self.dim), name="s1") + 1) self.s2 = scale_mul * (shared_floatx_zeros( (4 * self.dim), name="s2") + 1) self.s3 = scale_mul * (shared_floatx_zeros( (1 * self.dim), name="s3") + 1) # biases add_role(self.b1, WEIGHT) add_role(self.b2, WEIGHT) add_role(self.b3, WEIGHT) add_role(self.s1, WEIGHT) add_role(self.s2, WEIGHT) add_role(self.s3, WEIGHT) self.parameters = [ self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_cell_to_out, self.initial_state_, self.initial_cells, self.b1, self.b2, self.b3, self.s1, self.s2, self.s3 ]
def _allocate(self): parameter_shape = [1] + [1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable)] self.population_stats = dict( (stat, self.annotated_statistic( shared_floatx_nans(parameter_shape, name="population_%s" % stat))) for stat in self.stats) self.gamma = shared_floatx_nans(parameter_shape, name='gamma') self.beta = shared_floatx_nans(parameter_shape, name='beta') add_role(self.gamma, WEIGHT) add_role(self.beta, BIAS) self.parameters.append(self.gamma) self.parameters.append(self.beta)
def _allocate(self): self.params.append(shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.params.append(shared_floatx_nans((self.dim, self.dim), name='state_to_update')) self.params.append(shared_floatx_nans((self.dim, self.dim), name='state_to_reset')) self.params.append(shared_floatx_zeros((self.dim,), name="initial_state")) for i in range(3): if self.params[i]: add_role(self.params[i], WEIGHT) add_role(self.params[3], INITIAL_STATE)
def _allocate(self): W = shared_floatx_nans((self.visible_dim, self.hidden_dim), name='Wrbm') add_role(W, WEIGHT) self.params.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') bv = shared_floatx_nans((self.visible_dim, ), name='bv') add_role(bv, BIAS) self.params.append(bv) self.add_auxiliary_variable(bv.norm(2), name='bv_norm') bh = shared_floatx_nans((self.hidden_dim, ), name='bh') add_role(bh, BIAS) self.params.append(bh) self.add_auxiliary_variable(bh.norm(2), name='bh_norm')
def _allocate(self): self.params.append( shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.params.append( shared_floatx_nans((self.dim, self.dim), name='state_to_update')) self.params.append( shared_floatx_nans((self.dim, self.dim), name='state_to_reset')) self.params.append( shared_floatx_zeros((self.dim, ), name="initial_state")) for i in range(3): if self.params[i]: add_role(self.params[i], WEIGHT) add_role(self.params[3], INITIAL_STATE)
def _allocate(self): self.W_rz = shared_floatx_nans((self.dim, 2 * self.dim), name='W_state') self.W_htilde = shared_floatx_nans((self.dim, self.dim), name='W_state') # The underscore is required to prevent collision with # the `initial_state` application method self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state") add_role(self.W_rz, WEIGHT) add_role(self.W_htilde, WEIGHT) add_role(self.initial_state_, INITIAL_STATE) #self.parameters = [self.W_state, self.initial_state_, self.initial_cells] self.parameters = [self.W_rz, self.W_htilde, self.initial_state_]
def _allocate(self): c_dim = self.get_dim('c') zm_dim = self.get_dim('z_mix') # self.c_0 provides the initial state of the canvas self.c_0 = shared_floatx_nans((c_dim,), name='c_0') # self.zm_mean provides the mean of z_mix self.zm_mean = shared_floatx_nans((zm_dim,), name='zm_mean') # self.zm_logvar provides the logvar of z_mix self.zm_logvar = shared_floatx_nans((zm_dim,), name='zm_logvar') add_role(self.c_0, PARAMETER) add_role(self.zm_mean, PARAMETER) add_role(self.zm_logvar, PARAMETER) # add the theano shared variables to our parameter lists self.params.extend([ self.c_0, self.zm_mean, self.zm_logvar ]) return
def _allocate(self): parameter_shape = [1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable)] self.beta = shared_floatx_nans(parameter_shape, name='beta') add_role(self.beta, BIAS) self.parameters.append(self.beta) self.add_auxiliary_variable(self.beta.norm(2), name='beta_norm')
def _allocate(self): """Allocates the single parameter of this brick: the initial element on the stack. """ self.parameters.append( shared_floatx_nans((1, self.stack_dim), name='init_stack')) add_role(self.parameters[-1], INITIAL_STATE)
def _allocate(self): """In addition to the GRU parameters ``state_to_state`` and ``state_to_gates``, add the initial state if the search strategy is "constant". """ self.parameters.append(shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim), name='state_to_gates')) for i in range(2): if self.parameters[i]: add_role(self.parameters[i], WEIGHT) if self.init_strategy == 'constant': self.parameters.append(shared_floatx_zeros((self.dim,), name="initial_state")) add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self): parameter_shape = [1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable)] self.b = shared_floatx_nans(parameter_shape, name='b') add_role(self.b, BIAS) self.parameters.append(self.b) self.add_auxiliary_variable(self.b.norm(2), name='b_norm')
def _allocate(self): self.W_state = shared_floatx_nans((self.dim, 4*self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim,), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim,), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim,), name='W_cell_to_out') add_role(self.W_state, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) self.params = [self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_cell_to_out]
def _allocate(self): parameter_shape = [1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable)] self.gamma = shared_floatx_nans(parameter_shape, name='gamma') add_role(self.gamma, WEIGHT) self.parameters.append(self.gamma) self.add_auxiliary_variable(self.gamma.norm(2), name='gamma_norm')
def _allocate(self): c_dim = self.get_dim('c') self.c_0 = shared_floatx_nans((c_dim,), name='c_0') add_role(self.c_0, PARAMETER) # add the theano shared variables to our parameter lists self.params.extend([ self.c_0 ]) return
def _allocate(self): self.parameters.append(shared_floatx_nans((self.dim, self.dim), name="W")) add_role(self.parameters[0], WEIGHT) self.parameters.append(shared_floatx_zeros((self.dim,), name="initial_state")) add_role(self.parameters[1], INITIAL_STATE)
def _allocate(self): parameter_shape = [1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable)] self.population_stats = dict( (stat, shared_floatx_nans(parameter_shape, name="population_%s" % stat)) for stat in self.stats)
def _allocate(self): self.parameters.append( shared_floatx_nans((self.dim, self.dim), name="W")) add_role(self.parameters[0], WEIGHT) self.parameters.append( shared_floatx_zeros((self.dim, ), name="initial_state")) add_role(self.parameters[1], INITIAL_STATE)
def _allocate(self): """In addition to the GRU parameters ``state_to_state`` and ``state_to_gates``, add the initial state if the search strategy is "constant". """ self.parameters.append( shared_floatx_nans((self.dim, self.dim), name='state_to_state')) self.parameters.append( shared_floatx_nans((self.dim, 2 * self.dim), name='state_to_gates')) for i in range(2): if self.parameters[i]: add_role(self.parameters[i], WEIGHT) if self.init_strategy == 'constant': self.parameters.append( shared_floatx_zeros((self.dim, ), name="initial_state")) add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self): for i in range(self.n_tables): self.tables[i].allocate() self.mlp.allocate() W = shared_floatx_nans((sum(self.n_mot), sum(self.vect_size)), name='W') add_role(W, WEIGHT) self.parameters.append(W)
def _allocate(self): self.W_state = shared_floatx_nans((self.dim, 4 * self.dim), name='W_state') self.W_cell_to_in = shared_floatx_nans((self.dim, ), name='W_cell_to_in') self.W_cell_to_forget = shared_floatx_nans((self.dim, ), name='W_cell_to_forget') self.W_cell_to_out = shared_floatx_nans((self.dim, ), name='W_cell_to_out') add_role(self.W_state, WEIGHT) add_role(self.W_cell_to_in, WEIGHT) add_role(self.W_cell_to_forget, WEIGHT) add_role(self.W_cell_to_out, WEIGHT) self.parameters = [ self.W_state, self.W_cell_to_in, self.W_cell_to_forget, self.W_cell_to_out ]
def _allocate(self): parameter_shape = [ 1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable) ] self.w = shared_floatx_nans(parameter_shape, name='w') add_role(self.w, WEIGHT) self.parameters.append(self.w) self.add_auxiliary_variable(self.w.norm(2), name='w_norm')
def _allocate(self): parameter_shape = [ 1 if broadcast else dim for dim, broadcast in zip(self.shape, self.broadcastable) ] self.population_stats = dict( (stat, shared_floatx_nans(parameter_shape, name="population_%s" % stat)) for stat in self.stats)
def _allocate(self): input_dim = ((self.input_dim, ) if not isinstance(self.input_dim, collections.Sequence) else self.input_dim) broadcastable = (tuple(False for _ in input_dim) if self.broadcastable is None else self.broadcastable) if len(input_dim) != len(broadcastable): raise ValueError("input_dim and broadcastable must be same length") var_dim = tuple( 1 if broadcast else dim for dim, broadcast in equizip(input_dim, broadcastable)) broadcastable = broadcastable # "gamma", from the Ioffe & Szegedy manuscript. self.scale = shared_floatx_nans(var_dim, name='batch_norm_scale', broadcastable=broadcastable) # "beta", from the Ioffe & Szegedy manuscript. self.shift = shared_floatx_nans(var_dim, name='batch_norm_shift', broadcastable=broadcastable) add_role(self.scale, BATCH_NORM_SCALE_PARAMETER) add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER) self.parameters.append(self.scale) self.parameters.append(self.shift) # These aren't technically parameters, in that they should not be # learned using the same cost function as other model parameters. self.population_mean = shared_floatx_zeros( ((self.n_iter, ) if self.n_iter else ()) + var_dim, name='population_mean', broadcastable=((False, ) if self.n_iter else ()) + broadcastable) self.population_stdev = shared_floatx( numpy.ones(((self.n_iter, ) if self.n_iter else ()) + var_dim), name='population_stdev', broadcastable=((False, ) if self.n_iter else ()) + broadcastable) add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN) add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV) # Normally these would get annotated by an AnnotatingList, but they # aren't in self.parameters. add_annotation(self.population_mean, self) add_annotation(self.population_stdev, self)
def _allocate(self): if not isinstance(self.input_dim, collections.Iterable)\ or len(self.input_dim) != 3: raise ValueError("`input_dim` on Convolutional(%s) " "should be iterable and have a inputdim of 3. " "Got %s" %(self.name, str(self.input_dim))) num_channels = self.input_dim[0] W = shared_floatx_nans((self.num_filters, num_channels) + self.filter_size, name='W') add_role(W, FILTER) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if self.use_bias: b = shared_floatx_nans((self.num_filters,), name='b') add_role(b, BIAS) self.parameters.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self): input_dim = ((self.input_dim,) if not isinstance(self.input_dim, collections.Sequence) else self.input_dim) broadcastable = (tuple(False for _ in input_dim) if self.broadcastable is None else self.broadcastable) if len(input_dim) != len(broadcastable): raise ValueError("input_dim and broadcastable must be same length") var_dim = tuple(1 if broadcast else dim for dim, broadcast in equizip(input_dim, broadcastable)) broadcastable = broadcastable # "gamma", from the Ioffe & Szegedy manuscript. self.scale = shared_floatx_nans(var_dim, name='batch_norm_scale', broadcastable=broadcastable) # "beta", from the Ioffe & Szegedy manuscript. self.shift = shared_floatx_nans(var_dim, name='batch_norm_shift', broadcastable=broadcastable) add_role(self.scale, BATCH_NORM_SCALE_PARAMETER) add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER) self.parameters.append(self.scale) self.parameters.append(self.shift) # These aren't technically parameters, in that they should not be # learned using the same cost function as other model parameters. self.population_mean = shared_floatx_zeros(((self.n_iter,) if self.n_iter else ()) + var_dim, name='population_mean', broadcastable=((False,) if self.n_iter else ()) + broadcastable) self.population_stdev = shared_floatx(numpy.ones(((self.n_iter,) if self.n_iter else ()) + var_dim), name='population_stdev', broadcastable=((False,) if self.n_iter else ()) + broadcastable) add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN) add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV) # Normally these would get annotated by an AnnotatingList, but they # aren't in self.parameters. add_annotation(self.population_mean, self) add_annotation(self.population_stdev, self)
def _allocate(self): # W self.parameters.append(shared_floatx_nans((self.dim, self.dim), name="W")) add_role(self.parameters[0], WEIGHT) # gamma gamma_val = 0.1 * numpy.ones((self.dim), dtype=config.floatX) gamma_ = shared(name='gamma', value=gamma_val) add_role(gamma_, PARAMETER) self.parameters.append(gamma_) # beta beta_val = numpy.zeros((self.dim), dtype=config.floatX) beta_ = shared(name='beta', value=beta_val) add_role(beta_, PARAMETER) self.parameters.append(beta_)
def _allocate(self): W = shared_floatx_nans((self.num_filters, self.num_channels) + self.filter_size, name='W') add_role(W, FILTER) self.parameters.append(W) self.add_auxiliary_variable(W.norm(2), name='W_norm') if getattr(self, 'use_bias', True): if self.tied_biases: b = shared_floatx_nans((self.num_filters,), name='b') else: # this error is raised here instead of during initializiation # because ConvolutionalSequence may specify the image size if self.image_size == (None, None) and not self.tied_biases: raise ValueError('Cannot infer bias size without ' 'image_size specified. If you use ' 'variable image_size, you should use ' 'tied_biases=True.') b = shared_floatx_nans(self.get_dim('output'), name='b') add_role(b, BIAS) self.parameters.append(b) self.add_auxiliary_variable(b.norm(2), name='b_norm')