def Dense(shape, init=init_default_or_glorot_uniform, activation=activation_default_or_None, input_rank=None, map_rank=None, bias=bias_default_or_True, init_bias=init_bias_default_or_0): activation = _resolve_activation(activation) bias = bias if _is_given(bias) else _current_default_options.bias output_shape = _as_tuple(shape) if input_rank is not None and map_rank is not None: raise ValueError( "Dense: input_rank and map_rank cannot be specified at the same time." ) # determine meaning of axes # W gets dimension (input_shape + shape) # where input_shape is determined as: # - by default, equal to the dimensions of the input passed to Dense() # - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over) # - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over) # where input_rank and map_rank are mutuallly exclusive. #output_rank = -len(output_shape) # support outputs with tensor layouts # BUGBUG: Should this be a negative number now, since output is the last axis in Python? output_rank = len(output_shape) # support outputs with tensor layouts # If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank. # The dimension inference may still create multiple axes. input_shape = _INFERRED * (input_rank if input_rank is not None else 1) if input_rank is not None: UntestedBranchError("Dense, input_rank option not implemented") infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules elif map_rank is None: infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims' else: UntestedBranchError("Dense, map_rank option not implemented") infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones # parameters bound to this Function init_weights = _initializer_for(init, Record(output_rank=output_rank)) W = Parameter(input_shape + output_shape, init=init_weights, name='W') b = Parameter(output_shape, init=init_bias, name='b') if bias else None # expression of this function x = Placeholder(name='dense_arg') apply_x = times(x, W, output_rank=output_rank, infer_input_rank_to_map=infer_input_rank_to_map) if b: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Dense', Record(W=W, b=b))
def Convolution(filter_shape, # e.g. (3,3) num_filters=None, # e.g. 64 or None (which means 1 channel and don't add a dimension_ activation=activation_default_or_None, init=init_default_or_glorot_uniform, pad=pad_default_or_False, strides=1, sharing=True, # (must be True currently) bias=bias_default_or_True, init_bias=init_bias_default_or_0, reduction_rank=1, # (must be 1 currently) transpose=False, # (must be False currently) max_temp_mem_size_in_samples=0): #UntestedBranchError("Convolution") activation = _resolve_activation(activation) pad = pad if _is_given(pad ) else _current_default_options.pad bias = bias if _is_given(bias) else _current_default_options.bias # TODO: there must be a Python trick to do this as a function call on locals or so if reduction_rank != 1: NotImplementedError("Convolution: reduction_rank other than 1 currently not supported") if transpose: NotImplementedError("Convolution: transpose option currently not supported") if not sharing: NotImplementedError("Convolution: sharing option currently must be True") output_channels_shape = _as_tuple(num_filters) output_rank = len(output_channels_shape) filter_rank = len(filter_shape) kernel_shape = _INFERRED * reduction_rank + filter_shape # kernel := filter plus reductionDims # parameters bound to this Function #init_kernel = glorot_uniform(filter_rank=-filter_rank, output_rank=1) init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] # expression x = Placeholder(name='convolution_arg') # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) apply_x = convolution (W, x, strides=_as_tuple(strides), sharing=_as_tuple(sharing), auto_padding=_as_tuple(pad), # TODO: can we rename auto_padding to pad? transpose=transpose, max_temp_mem_size_in_samples=max_temp_mem_size_in_samples) if bias: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Convolution', Record(W=W, b=b))
def Dense(shape, init=init_default_or_glorot_uniform, activation=activation_default_or_None, input_rank=None, map_rank=None, bias=bias_default_or_True, init_bias=init_bias_default_or_0): activation = _resolve_activation(activation) bias = bias if _is_given(bias) else _current_default_options.bias output_shape = _as_tuple(shape) if input_rank is not None and map_rank is not None: raise ValueError("Dense: input_rank and map_rank cannot be specified at the same time.") # determine meaning of axes # W gets dimension (input_shape + shape) # where input_shape is determined as: # - by default, equal to the dimensions of the input passed to Dense() # - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over) # - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over) # where input_rank and map_rank are mutuallly exclusive. #output_rank = -len(output_shape) # support outputs with tensor layouts # BUGBUG: Should this be a negative number now, since output is the last axis in Python? output_rank = len(output_shape) # support outputs with tensor layouts # If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank. # The dimension inference may still create multiple axes. input_shape = _INFERRED * (input_rank if input_rank is not None else 1) if input_rank is not None: UntestedBranchError("Dense, input_rank option not implemented") infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules elif map_rank is None: infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims' else: UntestedBranchError("Dense, map_rank option not implemented") infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones # parameters bound to this Function init_weights = _initializer_for(init, Record(output_rank=output_rank)) W = Parameter(input_shape + output_shape, init=init_weights, name='W') b = Parameter( output_shape, init=init_bias, name='b') if bias else None # expression of this function x = Placeholder(name='dense_arg') apply_x = times(x, W, output_rank=output_rank, infer_input_rank_to_map=infer_input_rank_to_map) if b: apply_x = apply_x + b apply_x = apply_x >> activation return Block(apply_x, 'Dense', Record(W=W, b=b))