def __init__(self, input_dim, output_dim, hidden_size, init_ranges, **kwargs): linear1 = LinearMaxout(input_dim=input_dim, output_dim=hidden_size, num_pieces=2, name='linear1') linear2 = LinearMaxout(input_dim=hidden_size, output_dim=hidden_size, num_pieces=2, name='linear2') linear3 = Linear(input_dim=hidden_size, output_dim=output_dim) logistic = Logistic() bricks = [ linear1, BatchNormalization(input_dim=hidden_size, name='bn2'), linear2, BatchNormalization(input_dim=hidden_size, name='bnl'), linear3, logistic ] for init_range, b in zip(init_ranges, (linear1, linear2, linear3)): b.biases_init = initialization.Constant(0) b.weights_init = initialization.Uniform(width=init_range) kwargs.setdefault('use_bias', False) super(ConcatenateClassifier, self).__init__([b.apply for b in bricks], **kwargs)
def initialize2(brick, num_feature_maps): fan_in = numpy.prod(brick.filter_size) fan_out = numpy.prod( brick.filter_size) * brick.num_filters / num_feature_maps W_bound = numpy.sqrt(6. / (fan_in + fan_out)) brick.weights_init = initialization.Uniform(width=W_bound) brick.biases_init = initialization.Constant(0) brick.initialize()
def initialize(to_init, width): """ Initialize weights according to Xavier Parameter Initialization :param to_init the block to initialize :param width width of uniform distribution """ to_init.weights_init = initialization.Uniform(width=width) to_init.biases_init = initialization.Constant(0) to_init.initialize()
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size, init_ranges, **kwargs): (visual_init_range, textual_init_range, gbu_init_range, linear_range_1, linear_range_2, linear_range_3) = init_ranges visual_mlp = Sequence([ BatchNormalization(input_dim=visual_dim).apply, Linear(visual_dim, hidden_size, use_bias=False, weights_init=initialization.Uniform( width=visual_init_range)).apply, ], name='visual_mlp') textual_mlp = Sequence([ BatchNormalization(input_dim=textual_dim).apply, Linear(textual_dim, hidden_size, use_bias=False, weights_init=initialization.Uniform( width=textual_init_range)).apply, ], name='textual_mlp') gbu = GatedBimodal( hidden_size, weights_init=initialization.Uniform(width=gbu_init_range)) logistic_mlp = MLPGenreClassifier( hidden_size, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3]) # logistic_mlp = Sequence([ # BatchNormalization(input_dim=hidden_size, name='bn1').apply, # Linear(hidden_size, output_dim, name='linear_output', use_bias=False, # weights_init=initialization.Uniform(width=linear_range_1)).apply, # Logistic().apply #], name='logistic_mlp') children = [visual_mlp, textual_mlp, gbu, logistic_mlp] kwargs.setdefault('use_bias', False) kwargs.setdefault('children', children) super(GatedClassifier, self).__init__(**kwargs)
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size, init_ranges, **kwargs): (visual_range, textual_range, linear_range_1, linear_range_2, linear_range_3) = init_ranges visual_layer = FeedforwardSequence([ BatchNormalization(input_dim=visual_dim).apply, LinearMaxout( input_dim=visual_dim, output_dim=hidden_size, weights_init=initialization.Uniform(width=visual_range), use_bias=False, biases_init=initialization.Constant(0), num_pieces=2).apply ], name='visual_layer') textual_layer = FeedforwardSequence([ BatchNormalization(input_dim=textual_dim).apply, LinearMaxout( input_dim=textual_dim, output_dim=hidden_size, weights_init=initialization.Uniform(width=textual_range), biases_init=initialization.Constant(0), use_bias=False, num_pieces=2).apply ], name='textual_layer') logistic_mlp = MLPGenreClassifier( hidden_size, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3]) # logistic_mlp = Sequence([ # BatchNormalization(input_dim=hidden_size, name='bn1').apply, # Linear(hidden_size, output_dim, name='linear_output', use_bias=False, # weights_init=initialization.Uniform(width=linear_range_1)).apply, # Logistic().apply #], name='logistic_mlp') children = [visual_layer, textual_layer, logistic_mlp] kwargs.setdefault('use_bias', False) kwargs.setdefault('children', children) super(LinearSumClassifier, self).__init__(**kwargs)
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size, init_ranges, **kwargs): (visual_range, textual_range, linear_range_1, linear_range_2, linear_range_3) = init_ranges manager_dim = visual_dim + textual_dim visual_mlp = MLPGenreClassifier( visual_dim, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3], name='visual_mlp') textual_mlp = MLPGenreClassifier( textual_dim, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3], name='textual_mlp') # manager_mlp = MLPGenreClassifier(manager_dim, 2, hidden_size, [ # linear_range_1, linear_range_2, linear_range_3], output_act=Softmax, # name='manager_mlp') bn = BatchNormalization(input_dim=manager_dim, name='bn3') manager_mlp = Sequence([ Linear(manager_dim, 2, name='linear_output', use_bias=False, weights_init=initialization.Uniform( width=linear_range_1)).apply, ], name='manager_mlp') fork = Fork( input_dim=manager_dim, output_dims=[2] * output_dim, prototype=manager_mlp, output_names=['linear_' + str(i) for i in range(output_dim)]) children = [visual_mlp, textual_mlp, fork, bn, NDimensionalSoftmax()] kwargs.setdefault('use_bias', False) kwargs.setdefault('children', children) super(MoEClassifier, self).__init__(**kwargs)
def initialize(to_init, rndstd=0.01): for bricks in to_init: bricks.weights_init = initialization.Uniform(width=0.08) bricks.biases_init = initialization.Constant(0) bricks.initialize()
from blocks.algorithms import StepClipping, GradientDescent, CompositeRule, RMSProp from dataset import T_H5PYDataset source_path = 'dataset/normalized_syllables_rhythm_notes.json-seqlen-30.hdf5' train_dataset = T_H5PYDataset(source_path, which_sets=('train', )) hidden_layer_dim = 1000 x = tensor.lmatrix('syllables') y = tensor.lmatrix('durations') lookup_input = LookupTable(name='lookup_input', length=train_dataset.syllables_vocab_size() + 1, dim=hidden_layer_dim, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) lookup_input.initialize() linear_input = Linear(name='linear_input', input_dim=hidden_layer_dim, output_dim=hidden_layer_dim, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) linear_input.initialize() rnn = SimpleRecurrent(name='hidden', dim=hidden_layer_dim, activation=Tanh(), weights_init=initialization.Uniform(width=0.01)) rnn.initialize()
def initialize_inout(brick, fan_in, fan_out,seed=1): W_bound = numpy.sqrt(6. / (fan_in + fan_out)) brick.weights_init = initialization.Uniform(width=2 * W_bound) brick.biases_init = initialization.Constant(0) brick.initialize()
def get_lookups(self, dim, vocab_list): return [LookupTable(dim=dim, length=vocab, name='lookup' + str(index), weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) for index, vocab in enumerate(vocab_list)]
def __init__(self, input_sources_list, input_sources_vocab_size_list, output_source, output_source_vocab_size, lookup_dim=200, hidden_size=256, recurrent_stack_size=1): self.InputSources = input_sources_list self.InputSourcesVocab = input_sources_vocab_size_list self.OutputSource = output_source self.OutputSourceVocab = output_source_vocab_size inputs = [tensor.lmatrix(source) for source in input_sources_list] output = tensor.lmatrix(output_source) lookups = self.get_lookups(lookup_dim, input_sources_vocab_size_list) for lookup in lookups: lookup.initialize() merge = Merge([lookup.name for lookup in lookups], [lookup.dim for lookup in lookups], hidden_size, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) merge.initialize() linear0 = Linear(input_dim=hidden_size, output_dim=hidden_size, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0), name='linear0') linear0.initialize() recurrent_blocks = [] for i in range(recurrent_stack_size): recurrent_blocks.append(SimpleRecurrent( dim=hidden_size, activation=Tanh(), weights_init=initialization.Uniform(width=0.01), use_bias=False)) for i, recurrent_block in enumerate(recurrent_blocks): recurrent_block.name = 'recurrent'+str(i+1) recurrent_block.initialize() linear_out = Linear(input_dim=hidden_size, output_dim=output_source_vocab_size, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0), name='linear_out') linear_out.initialize() softmax = NDimensionalSoftmax(name='softmax') lookup_outputs = [lookup.apply(input) for lookup, input in zip(lookups, inputs)] m = merge.apply(*lookup_outputs) r = linear0.apply(m) for block in recurrent_blocks: r = block.apply(r) a = linear_out.apply(r) self.Cost = softmax.categorical_cross_entropy(output, a, extra_ndim=1).mean() self.Cost.name = 'cost' y_hat = softmax.apply(a, extra_ndim=1) y_hat.name = 'y_hat' self.ComputationGraph = ComputationGraph(self.Cost) self.Function = None self.MainLoop = None self.Model = Model(y_hat)
def __init__(self, input1_size, input2_size, lookup1_dim=200, lookup2_dim=200, hidden_size=512): self.hidden_size = hidden_size self.input1_size = input1_size self.input2_size = input2_size self.lookup1_dim = lookup1_dim self.lookup2_dim = lookup2_dim x1 = tensor.lmatrix('durations') x2 = tensor.lmatrix('syllables') y = tensor.lmatrix('pitches') lookup1 = LookupTable(dim=self.lookup1_dim, length=self.input1_size, name='lookup1', weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) lookup1.initialize() lookup2 = LookupTable(dim=self.lookup2_dim, length=self.input2_size, name='lookup2', weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) lookup2.initialize() merge = Merge(['lookup1', 'lookup2'], [self.lookup1_dim, self.lookup2_dim], self.hidden_size, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) merge.initialize() recurrent_block = LSTM( dim=self.hidden_size, activation=Tanh(), weights_init=initialization.Uniform(width=0.01) ) #RecurrentStack([LSTM(dim=self.hidden_size, activation=Tanh())] * 3) recurrent_block.initialize() linear = Linear(input_dim=self.hidden_size, output_dim=self.input1_size, weights_init=initialization.Uniform(width=0.01), biases_init=Constant(0)) linear.initialize() softmax = NDimensionalSoftmax() l1 = lookup1.apply(x1) l2 = lookup2.apply(x2) m = merge.apply(l1, l2) h = recurrent_block.apply(m) a = linear.apply(h) y_hat = softmax.apply(a, extra_ndim=1) # ValueError: x must be 1-d or 2-d tensor of floats. Got TensorType(float64, 3D) self.Cost = softmax.categorical_cross_entropy(y, a, extra_ndim=1).mean() self.ComputationGraph = ComputationGraph(self.Cost) self.Model = Model(y_hat)