def mask_layer(spec): layer = MaskLayerImpl( 'MaskLayer', { 'default': BufferStructure('T', 'B', 3, 2), 'mask': BufferStructure('T', 'B', 1) }, NO_CON, NO_CON) return layer, spec
def merge(spec): in_shapes = {'inputs_1': BufferStructure('T', 'B', 3, 2), 'inputs_2': BufferStructure('T', 'B', 3, 4)} layer = MergeLayerImpl('Merge', in_shapes, NO_CON, NO_CON) return layer, spec
def squared_difference_layer(spec): in_shapes = {'inputs_1': BufferStructure('T', 'B', 3, 2), 'inputs_2': BufferStructure('T', 'B', 3, 2) } layer = SquaredDifferenceLayerImpl('SquaredDifferenceLayer', in_shapes, NO_CON, NO_CON) return layer, spec
def highway_layer(spec): in_shapes = { 'H': BufferStructure('T', 'B', 2, 3), 'T': BufferStructure('T', 'B', 2, 3), 'x': BufferStructure('T', 'B', 2, 3) } layer = HighwayLayerImpl('HighwayLayer', in_shapes, NO_CON, NO_CON) return layer, spec
def squared_error_layer(spec): in_shapes = {'default': BufferStructure('T', 'B', 3, 2), 'targets': BufferStructure('T', 'B', 3, 2) } layer = SquaredErrorLayerImpl('SquaredErrorLayer', in_shapes, NO_CON, NO_CON) return layer, spec
def setup(self, kwargs, in_shapes): outputs = OrderedDict() outputs['loss'] = BufferStructure('T', 'B', 1) parameters = OrderedDict() internals = OrderedDict() internals['tmp'] = in_shapes['default'] internals['dsq_activations'] = BufferStructure( *in_shapes['default'].shape, is_backward_only=True) return outputs, parameters, internals
def test_combine_input_sizes_tuples(): assert combine_buffer_structures([BufferStructure(1, 4)]) == \ BufferStructure(1, 4) assert combine_buffer_structures([BufferStructure(4, 1), BufferStructure(4, 3), BufferStructure(4, 6)])\ == BufferStructure(4, 10) assert combine_buffer_structures([BufferStructure(4, 3, 2), BufferStructure(4, 3, 3), BufferStructure(4, 3, 2)]) == \ BufferStructure(4, 3, 7)
def sigmoid_ce_layer(spec): time_steps = spec.get('time_steps', 3) batch_size = spec.get('batch_size', 2) feature_dim = (2, 3, 5) target_shape = (time_steps, batch_size) + feature_dim targets = np.random.randint(0, 2, target_shape) in_shapes = {'default': BufferStructure('T', 'B', *feature_dim), 'targets': BufferStructure('T', 'B', *target_shape[2:])} layer = SigmoidCELayerImpl('SigmoidCELayer', in_shapes, NO_CON, NO_CON) spec['targets'] = targets return layer, spec
def setup(self, kwargs, in_shapes): self.activation = kwargs.get('activation', 'tanh') self.size = kwargs.get('size', in_shapes['default'].feature_size) if not isinstance(self.size, int): raise LayerValidationError('size must be int but was {}'. format(self.size)) in_size = self.in_shapes['default'].feature_size outputs = OrderedDict() outputs['default'] = BufferStructure('T', 'B', self.size, context_size=1) parameters = OrderedDict() parameters['W'] = BufferStructure(self.size, in_size) parameters['R'] = BufferStructure(self.size, self.size) parameters['bias'] = BufferStructure(self.size) parameters['timing'] = BufferStructure(self.size) internals = OrderedDict() internals['Ha'] = BufferStructure('T', 'B', self.size, context_size=1) internals['dHa'] = BufferStructure('T', 'B', self.size, context_size=1, is_backward_only=True) internals['dHb'] = BufferStructure('T', 'B', self.size, context_size=1, is_backward_only=True) return outputs, parameters, internals
def test_layer_constructor(): a = Connection('l', 'default', 'A', 'default') b = Connection('l', 'default', 'B', 'default') c = Connection('l', 'default', 'C', 'default') l = FullyConnectedLayerImpl('LayerName', {'default': BufferStructure('T', 'B', 5)}, {c}, {a, b}, size=8) expected = {'default': BufferStructure('T', 'B', 8)} assert l.out_shapes == expected assert l.in_shapes == {'default': BufferStructure('T', 'B', 5)} assert l.incoming == {c} assert l.outgoing == {a, b} assert l.kwargs == {'size': 8}
def fully_connected_layer_2d(spec): in_shapes = {'default': BufferStructure('T', 'B', 2, 3)} layer = FullyConnectedLayerImpl('FullyConnectedLayer', in_shapes, NO_CON, NO_CON, size=(3, 3, 1), activation=spec['activation']) return layer, spec
def lstm_layer_2d(spec): layer = LstmLayerImpl('LstmLayer', {'default': BufferStructure('T', 'B', 2, 2, 1)}, NO_CON, NO_CON, size=3, activation=spec['activation']) return layer, spec
def rnn_layer_2d(spec): layer = RecurrentLayerImpl('RnnLayer', {'default': BufferStructure('T', 'B', 2, 1, 2)}, NO_CON, NO_CON, size=3, activation=spec['activation']) return layer, spec
def binomial_crossentropy_layer(spec): time_steps = spec.get('time_steps', 3) batch_size = spec.get('batch_size', 2) size = 5 shape = (time_steps, batch_size, size) default = np.random.rand(*shape) targets = np.random.randint(0, 2, shape) in_shapes = {'default': BufferStructure('T', 'B', size), 'targets': BufferStructure('T', 'B', size)} layer = BinomialCrossEntropyLayerImpl('BinomialCrossEntropyError', in_shapes, NO_CON, NO_CON) spec['default'] = default spec['targets'] = targets return layer, spec
def create(source_set, sink_set, layout, connections): def ensure_uniform(l): assert min(l) == max(l) return l[0] sorted_sources = sorted(source_set) flat_sources = list(flatten(sorted_sources)) nesting = convert_to_nested_indices(sorted_sources) # get buffer type for hub and assert its uniform structs = [ BufferStructure.from_layout(get_by_path(layout, s)) for s in flat_sources ] btype = ensure_uniform([s.buffer_type for s in structs]) # max context size context_size = max([s.context_size for s in structs]) hub = Hub(flat_sources, nesting, sorted(sink_set), btype, context_size) hub.setup(connections) hub.sizes = [structs[i].feature_size for i in hub.perm] hub.size = sum(hub.sizes) hub.is_backward_only = ensure_uniform( [structs[i].is_backward_only for i in hub.perm]) return hub
def create_buffer_views_from_layout(layout, buffers, hubs, existing_view=None): if '@slice' in layout: buffer_nr = layout['@hub'] feature_slice = slice(*layout['@slice']) structure = BufferStructure.from_layout(layout) full_buffer = structure.create_from_buffer_hub(buffers[buffer_nr], hubs[buffer_nr], feature_slice) else: full_buffer = None if layout['@type'] == 'BufferView': names, child_buffers = [], [] for n, sub_node in sorted(layout.items(), key=sort_by_index_key): if n.startswith('@'): continue if existing_view: assert n in existing_view c = create_buffer_views_from_layout( sub_node, buffers, hubs, existing_view=existing_view[n]) else: c = create_buffer_views_from_layout(sub_node, buffers, hubs) names.append(n) child_buffers.append(c) if existing_view: return existing_view.adjust(names, child_buffers, full_buffer) else: return BufferView(names, child_buffers, full_buffer) else: # layout['@type'] == 'array': assert full_buffer is not None, layout return full_buffer
def setup(self, kwargs, in_shapes): # 'inputs_1' and 'inputs_2' must have same shape f_shape1 = in_shapes['inputs_1'].feature_shape f_shape2 = in_shapes['inputs_2'].feature_shape if f_shape1 != f_shape2: raise LayerValidationError( "{}: inputs_1 and inputs_2 must have same feature shapes but " "got {} and {}".format(self.name, f_shape1, f_shape2)) outputs = OrderedDict() outputs['default'] = BufferStructure('T', 'B', *f_shape1) internals = OrderedDict() feature_shape = self.in_shapes['inputs_1'].feature_shape internals['diff'] = BufferStructure('T', 'B', *feature_shape) return outputs, OrderedDict(), internals
def create_buffer_views_from_layout(layout, buffers, hubs, existing_view=None): if '@slice' in layout: buffer_nr = layout['@hub'] feature_slice = slice(*layout['@slice']) structure = BufferStructure.from_layout(layout) full_buffer = structure.create_from_buffer_hub( buffers[buffer_nr], hubs[buffer_nr], feature_slice) else: full_buffer = None if layout['@type'] == 'BufferView': names, child_buffers = [], [] for n, sub_node in sorted(layout.items(), key=sort_by_index_key): if n.startswith('@'): continue if existing_view: assert n in existing_view c = create_buffer_views_from_layout( sub_node, buffers, hubs, existing_view=existing_view[n]) else: c = create_buffer_views_from_layout(sub_node, buffers, hubs) names.append(n) child_buffers.append(c) if existing_view: return existing_view.adjust(names, child_buffers, full_buffer) else: return BufferView(names, child_buffers, full_buffer) else: # layout['@type'] == 'array': assert full_buffer is not None, layout return full_buffer
def setup(self, kwargs, in_shapes): # 'default' and 'targets' must have same shape in_shape = in_shapes['default'].feature_shape tar_shape = in_shapes['targets'].feature_shape if in_shape != tar_shape: raise LayerValidationError( "{}: default and targets must have same feature shapes but " "got {} and {}".format(self.name, in_shape, tar_shape)) outputs = OrderedDict() outputs['predictions'] = BufferStructure('T', 'B', *in_shape) outputs['loss'] = BufferStructure('T', 'B', *in_shape) internals = OrderedDict() internals['diff'] = BufferStructure('T', 'B', *in_shape) return outputs, OrderedDict(), internals
def elementwise_layer(spec): layer = ElementwiseLayerImpl('Elementwise', {'default': BufferStructure('T', 'B', 3, 2)}, NO_CON, NO_CON, activation=spec['activation']) return layer, spec
def test_raises_on_unexpected_kwargs(LayerClass): with pytest.raises(LayerValidationError) as excinfo: l = LayerClass('LayerName', {'default': BufferStructure(5, )}, NO_CON, NO_CON, some_foo=16) assert 'some_foo' in excinfo.value.args[0]
def convolution_layer_2d(spec, input_shape=(4, 4, 1), num_filters=1, kernel_size=(2, 2), stride=(1, 1)): x = BufferStructure('T', 'B', *input_shape) layer = Convolution2DLayerImpl('Convolution2DLayer', {'default': x}, NO_CON, NO_CON, num_filters=num_filters, kernel_size=kernel_size, stride=stride, activation=spec['activation']) return layer, spec
def setup(self, kwargs, in_shapes): in_shape = in_shapes['default'].feature_shape tar_shape = in_shapes['targets'].feature_shape if tar_shape != in_shape: raise LayerValidationError('input and targets must have the same ' 'shapes. But got {} != {}' .format(in_shape, tar_shape)) outputs = OrderedDict() outputs['predictions'] = BufferStructure('T', 'B', *in_shape) outputs['loss'] = BufferStructure('T', 'B', *in_shape) internals = OrderedDict() internals['dcee'] = BufferStructure('T', 'B', *in_shape, is_backward_only=True) return outputs, OrderedDict(), internals
def avgpooling_layer_2d(spec): layer = Pooling2DLayerImpl('Pooling2DLayer', {'default': BufferStructure('T', 'B', 4, 4, 1)}, NO_CON, NO_CON, kernel_size=(2, 2), stride=(1, 1), type="avg") return layer, spec
def clockwork_layer_2d(spec): layer = ClockworkLayerImpl('ClockworkRnn', {'default': BufferStructure('T', 'B', 2, 1, 2)}, NO_CON, NO_CON, size=7, activation=spec['activation']) spec['inits'] = {'timing': np.array([1, 1, 2, 2, 3, 3, 5])} return layer, spec
def setup(self, kwargs, in_shapes): if in_shapes['default'] != in_shapes['targets']: raise LayerValidationError("{}: default and targets must have the " "same shapes but got {} and {}" .format(self.name, in_shapes['default'], in_shapes['targets'])) outputs = OrderedDict() outputs['default'] = BufferStructure('T', 'B', 1) feature_shape = in_shapes['default'].feature_shape internals = OrderedDict() internals['cee'] = BufferStructure('T', 'B', *feature_shape) internals['ceed'] = BufferStructure('T', 'B', *feature_shape, is_backward_only=True) return outputs, OrderedDict(), internals
def setup(self, kwargs, in_shapes): self.activation = kwargs.get('activation', 'tanh') assert 'num_filters' in kwargs, "num_filters must be specified " \ " for ConvolutionLayer" assert 'kernel_size' in kwargs, "kernel_size must be specified " \ "for ConvolutionLayer" num_filters = kwargs['num_filters'] kernel_size = kwargs['kernel_size'] stride = kwargs.get('stride', (1, 1)) padding = kwargs.get('padding', 0) assert type(padding) is int and padding >= 0, \ "Invalid padding: {}".format(padding) assert type(kernel_size) in [list, tuple] and \ len(kernel_size) == 2, "Kernel size must be list or tuple of " \ "length 2: {}".format(kernel_size) assert type(stride) in [list, tuple] and len(stride) == 2, \ "Stride must be list or tuple of length 2: {}".format(stride) in_shape = self.in_shapes['default'].feature_shape assert stride[0] >= 0 and stride[1] >= 0, \ "Invalid stride: {}".format(stride) assert isinstance(in_shape, tuple) and len(in_shape) == 3, \ "ConvolutionLayer2D must have 3 dimensional input but input " \ "shape was {}".format(in_shape) self.num_filters = num_filters self.kernel_size = tuple(kernel_size) self.stride = tuple(stride) self.padding = padding kernel_x, kernel_y = self.kernel_size num_input_maps = in_shape[2] output_height = ((in_shape[0] + 2 * padding - kernel_x) // stride[0]) + 1 output_width = ((in_shape[1] + 2 * padding - kernel_y) // stride[1]) + 1 out_shape = (output_height, output_width, num_filters) outputs = OrderedDict() outputs['default'] = BufferStructure('T', 'B', *out_shape) parameters = OrderedDict() parameters['W'] = BufferStructure(num_filters, kernel_x, kernel_y, num_input_maps) parameters['bias'] = BufferStructure(num_filters) internals = OrderedDict() return outputs, parameters, internals
def setup(self, kwargs, in_shapes): assert 'kernel_size' in kwargs, "kernel_size must be specified for " \ "Pooling2D" assert 'type' in kwargs, "type must be specified for Pooling2D" kernel_size = kwargs['kernel_size'] ptype = kwargs['type'] padding = kwargs.get('padding', 0) stride = kwargs.get('stride', (1, 1)) in_shape = self.in_shapes['default'].feature_shape assert ptype in ('max', 'avg') assert type(padding) is int and padding >= 0, \ "Invalid padding: {}".format(padding) assert type(kernel_size) in [list, tuple] and \ len(kernel_size) == 2, "Kernel size must be list or " \ "tuple of length 2: {}".format( kernel_size) assert type(stride) in [list, tuple] and len(stride) == 2, \ "Stride must be list or tuple of length 2: {}".format(stride) assert stride[0] >= 0 and stride[1] >= 0, \ "Invalid stride: {}".format(stride) assert isinstance(in_shape, tuple) and len(in_shape) == 3, \ "PoolingLayer2D must have 3 dimensional input but input " \ "shape was %s" % in_shape self.kernel_size = tuple(kernel_size) self.type = ptype self.padding = padding self.stride = tuple(stride) output_height = ((in_shape[0] + 2 * padding - kernel_size[0]) // stride[0]) + 1 output_width = ((in_shape[1] + 2 * padding - kernel_size[1]) // stride[1]) + 1 assert output_height > 0 and output_width > 0, \ "Evaluated output height and width must be positive but were " \ "({}, {})".format(output_height, output_width) output_shape = (output_height, output_width, in_shape[2]) outputs = OrderedDict() outputs['default'] = BufferStructure('T', 'B', *output_shape) internals = OrderedDict() if self.type == 'max': argmax_shape = outputs['default'].feature_shape internals['argmax'] = BufferStructure('T', 'B', *argmax_shape) return outputs, OrderedDict(), internals
def clockwork_lstm_layer(spec): layer = ClockworkLstmLayerImpl('ClockworkLstm', {'default': BufferStructure('T', 'B', 3)}, NO_CON, NO_CON, size=4, activation=spec['activation']) spec['inits'] = {'timing': np.array([1, 2, 2, 3])} return layer, spec
def softmax_fiddle_layer(spec): time_steps = spec.get('time_steps', 3) batch_size = spec.get('batch_size', 2) feature_dim = (4, ) target_shape = (time_steps, batch_size) + feature_dim targets = np.random.randint(0, 2, target_shape).astype(np.float) targets /= np.clip(targets.sum(2)[:, :, None], 1, 10000) print('TARGETS:', targets) in_shapes = { 'default': BufferStructure('T', 'B', *feature_dim), 'targets': BufferStructure('T', 'B', *target_shape[2:]) } layer = SoftmaxFiddleLayerImpl('SoftmaxFiddleLayer', in_shapes, NO_CON, NO_CON) spec['targets'] = targets return layer, spec
def setup(self, kwargs, in_shapes): outputs = OrderedDict() outputs['loss'] = BufferStructure('T', 'B', 1) parameters = OrderedDict() internals = OrderedDict() internals['tmp'] = in_shapes['default'] return outputs, parameters, internals
def create(source_set, sink_set, layout, connections): def ensure_uniform(l): assert min(l) == max(l) return l[0] sorted_sources = sorted(source_set) flat_sources = list(flatten(sorted_sources)) nesting = convert_to_nested_indices(sorted_sources) # get buffer type for hub and assert its uniform structs = [BufferStructure.from_layout(get_by_path(layout, s)) for s in flat_sources] btype = ensure_uniform([s.buffer_type for s in structs]) # max context size context_size = max([s.context_size for s in structs]) hub = Hub(flat_sources, nesting, sorted(sink_set), btype, context_size) hub.setup(connections) hub.sizes = [structs[i].feature_size for i in hub.perm] hub.size = sum(hub.sizes) hub.is_backward_only = ensure_uniform([structs[i].is_backward_only for i in hub.perm]) return hub