def Convolution(filter_shape, # e.g. (3,3) num_filters=None, # e.g. 64 or None (which means 1 channel and don't add a dimension_ activation=activation_default_or_None, init=init_default_or_glorot_uniform, pad=pad_default_or_False, strides=1, sharing=True, # (must be True currently) bias=bias_default_or_True, init_bias=init_bias_default_or_0, reduction_rank=1, # (must be 1 currently) transpose=False, # (must be False currently) max_temp_mem_size_in_samples=0): #UntestedBranchError("Convolution") activation = _resolve_activation(activation) pad = pad if _is_given(pad ) else _current_default_options.pad bias = bias if _is_given(bias) else _current_default_options.bias # TODO: there must be a Python trick to do this as a function call on locals or so if reduction_rank != 1: NotImplementedError("Convolution: reduction_rank other than 1 currently not supported") if transpose: NotImplementedError("Convolution: transpose option currently not supported") if not sharing: NotImplementedError("Convolution: sharing option currently must be True") output_channels_shape = _as_tuple(num_filters) output_rank = len(output_channels_shape) filter_rank = len(filter_shape) kernel_shape = _INFERRED * reduction_rank + filter_shape # kernel := filter plus reductionDims # parameters bound to this Function #init_kernel = glorot_uniform(filter_rank=-filter_rank, output_rank=1) init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] # expression x = Placeholder(name='convolution_arg') # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) apply_x = convolution (W, x, strides=_as_tuple(strides), sharing=_as_tuple(sharing), auto_padding=_as_tuple(pad), # TODO: can we rename auto_padding to pad? transpose=transpose, max_temp_mem_size_in_samples=max_temp_mem_size_in_samples) if bias: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Convolution', Record(W=W, b=b))
def Recurrence(over, go_backwards=False, initial_state=initial_state_default_or_None): # helper to compute previous value # can take a single Variable/Function or a tuple initial_state = initial_state if _is_given(initial_state) else _current_default_options.initial_state # if initial state is given and a numeric constant, then turn it into a Constant() object if np.isscalar(initial_state): initial_state = Constant(initial_state, shape=(1)) # TODO: This should be automatically done inside the API. def previous_hook(state): if isinstance (state, tuple): # if multiple then apply to each element return tuple([previous_hook(s) for s in state]) # not a tuple: must be a 'scalar', i.e. a single element return past_value (state, initial_state) if not go_backwards else \ future_value(state, initial_state) x = Placeholder(name='recurrence_arg') state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders prev_state = previous_hook(state_forward) # delay (h, c) f_x_h_c = over(x, prev_state) # apply the recurrent over # this returns a Function (x, (h_prev, c_prev)) -> (h, c) h_c = f_x_h_c.outputs replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) } f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it) if _trace_layers: _log_node(h) _log_node(combine([h.owner])) apply_x = combine([h]) # the Function that yielded 'h', so we get to know its inputs # apply_x is a Function x -> h return Block(apply_x, 'Recurrence', Record(over=over))
def Dense(shape, init=init_default_or_glorot_uniform, activation=activation_default_or_None, input_rank=None, map_rank=None, bias=bias_default_or_True, init_bias=init_bias_default_or_0): activation = _resolve_activation(activation) bias = bias if _is_given(bias) else _current_default_options.bias output_shape = _as_tuple(shape) if input_rank is not None and map_rank is not None: raise ValueError( "Dense: input_rank and map_rank cannot be specified at the same time." ) # determine meaning of axes # W gets dimension (input_shape + shape) # where input_shape is determined as: # - by default, equal to the dimensions of the input passed to Dense() # - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over) # - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over) # where input_rank and map_rank are mutuallly exclusive. #output_rank = -len(output_shape) # support outputs with tensor layouts # BUGBUG: Should this be a negative number now, since output is the last axis in Python? output_rank = len(output_shape) # support outputs with tensor layouts # If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank. # The dimension inference may still create multiple axes. input_shape = _INFERRED * (input_rank if input_rank is not None else 1) if input_rank is not None: UntestedBranchError("Dense, input_rank option not implemented") infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules elif map_rank is None: infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims' else: UntestedBranchError("Dense, map_rank option not implemented") infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones # parameters bound to this Function init_weights = _initializer_for(init, Record(output_rank=output_rank)) W = Parameter(input_shape + output_shape, init=init_weights, name='W') b = Parameter(output_shape, init=init_bias, name='b') if bias else None # expression of this function x = Placeholder(name='dense_arg') apply_x = times(x, W, output_rank=output_rank, infer_input_rank_to_map=infer_input_rank_to_map) if b: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Dense', Record(W=W, b=b))
def Dense(shape, init=init_default_or_glorot_uniform, activation=activation_default_or_None, input_rank=None, map_rank=None, bias=bias_default_or_True, init_bias=init_bias_default_or_0): activation = _resolve_activation(activation) bias = bias if _is_given(bias) else _current_default_options.bias output_shape = _as_tuple(shape) if input_rank is not None and map_rank is not None: raise ValueError("Dense: input_rank and map_rank cannot be specified at the same time.") # determine meaning of axes # W gets dimension (input_shape + shape) # where input_shape is determined as: # - by default, equal to the dimensions of the input passed to Dense() # - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over) # - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over) # where input_rank and map_rank are mutuallly exclusive. #output_rank = -len(output_shape) # support outputs with tensor layouts # BUGBUG: Should this be a negative number now, since output is the last axis in Python? output_rank = len(output_shape) # support outputs with tensor layouts # If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank. # The dimension inference may still create multiple axes. input_shape = _INFERRED * (input_rank if input_rank is not None else 1) if input_rank is not None: UntestedBranchError("Dense, input_rank option not implemented") infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules elif map_rank is None: infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims' else: UntestedBranchError("Dense, map_rank option not implemented") infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones # parameters bound to this Function init_weights = _initializer_for(init, Record(output_rank=output_rank)) W = Parameter(input_shape + output_shape, init=init_weights, name='W') b = Parameter( output_shape, init=init_bias, name='b') if bias else None # expression of this function x = Placeholder(name='dense_arg') apply_x = times(x, W, output_rank=output_rank, infer_input_rank_to_map=infer_input_rank_to_map) if b: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Dense', Record(W=W, b=b))