def Convolution(filter_shape, # e.g. (3,3)
num_filters=None, # e.g. 64 or None (which means 1 channel and don't add a dimension_
activation=activation_default_or_None,
init=init_default_or_glorot_uniform,
pad=pad_default_or_False,
strides=1,
sharing=True, # (must be True currently)
bias=bias_default_or_True,
init_bias=init_bias_default_or_0,
reduction_rank=1, # (must be 1 currently)
transpose=False, # (must be False currently)
max_temp_mem_size_in_samples=0):
#UntestedBranchError("Convolution")
activation = _resolve_activation(activation)
pad = pad if _is_given(pad ) else _current_default_options.pad
bias = bias if _is_given(bias) else _current_default_options.bias
# TODO: there must be a Python trick to do this as a function call on locals or so
if reduction_rank != 1:
NotImplementedError("Convolution: reduction_rank other than 1 currently not supported")
if transpose:
NotImplementedError("Convolution: transpose option currently not supported")
if not sharing:
NotImplementedError("Convolution: sharing option currently must be True")
output_channels_shape = _as_tuple(num_filters)
output_rank = len(output_channels_shape)
filter_rank = len(filter_shape)
kernel_shape = _INFERRED * reduction_rank + filter_shape # kernel := filter plus reductionDims
# parameters bound to this Function
#init_kernel = glorot_uniform(filter_rank=-filter_rank, output_rank=1)
init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1))
# BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag
W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ]
b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ]
# expression
x = Placeholder(name='convolution_arg')
# TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras)
apply_x = convolution (W, x,
strides=_as_tuple(strides),
sharing=_as_tuple(sharing),
auto_padding=_as_tuple(pad),
# TODO: can we rename auto_padding to pad?
transpose=transpose,
max_temp_mem_size_in_samples=max_temp_mem_size_in_samples)
if bias:
apply_x = apply_x + b
apply_x = apply_x >> activation
return Block(apply_x, 'Convolution', Record(W=W, b=b))
def Convolution(filter_shape, # e.g. (3,3)
num_filters=None, # e.g. 64 or None (which means 1 channel and don't add a dimension_
activation=activation_default_or_None,
init=init_default_or_glorot_uniform,
pad=pad_default_or_False,
strides=1,
sharing=True, # (must be True currently)
bias=bias_default_or_True,
init_bias=init_bias_default_or_0,
reduction_rank=1, # (must be 1 currently)
transpose=False, # (must be False currently)
max_temp_mem_size_in_samples=0):
#UntestedBranchError("Convolution")
activation = _resolve_activation(activation)
pad = pad if _is_given(pad ) else _current_default_options.pad
bias = bias if _is_given(bias) else _current_default_options.bias
# TODO: there must be a Python trick to do this as a function call on locals or so
if reduction_rank != 1:
NotImplementedError("Convolution: reduction_rank other than 1 currently not supported")
if transpose:
NotImplementedError("Convolution: transpose option currently not supported")
if not sharing:
NotImplementedError("Convolution: sharing option currently must be True")
output_channels_shape = _as_tuple(num_filters)
output_rank = len(output_channels_shape)
filter_rank = len(filter_shape)
kernel_shape = _INFERRED * reduction_rank + filter_shape # kernel := filter plus reductionDims
# parameters bound to this Function
#init_kernel = glorot_uniform(filter_rank=-filter_rank, output_rank=1)
init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1))
# BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag
W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ]
b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ]
# expression
x = Placeholder(name='convolution_arg')
# TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras)
apply_x = convolution (W, x,
strides=_as_tuple(strides),
sharing=_as_tuple(sharing),
auto_padding=_as_tuple(pad),
# TODO: can we rename auto_padding to pad?
transpose=transpose,
max_temp_mem_size_in_samples=max_temp_mem_size_in_samples)
if bias:
apply_x = apply_x + b
apply_x = apply_x >> activation
return Block(apply_x, 'Convolution', Record(W=W, b=b))
def Recurrence(over, go_backwards=False, initial_state=initial_state_default_or_None):
# helper to compute previous value
# can take a single Variable/Function or a tuple
initial_state = initial_state if _is_given(initial_state) else _current_default_options.initial_state
# if initial state is given and a numeric constant, then turn it into a Constant() object
if np.isscalar(initial_state):
initial_state = Constant(initial_state, shape=(1)) # TODO: This should be automatically done inside the API.
def previous_hook(state):
if isinstance (state, tuple): # if multiple then apply to each element
return tuple([previous_hook(s) for s in state])
# not a tuple: must be a 'scalar', i.e. a single element
return past_value (state, initial_state) if not go_backwards else \
future_value(state, initial_state)
x = Placeholder(name='recurrence_arg')
state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders
prev_state = previous_hook(state_forward) # delay (h, c)
f_x_h_c = over(x, prev_state) # apply the recurrent over
# this returns a Function (x, (h_prev, c_prev)) -> (h, c)
h_c = f_x_h_c.outputs
replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) }
f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c
h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it)
if _trace_layers:
_log_node(h)
_log_node(combine([h.owner]))
apply_x = combine([h]) # the Function that yielded 'h', so we get to know its inputs
# apply_x is a Function x -> h
return Block(apply_x, 'Recurrence', Record(over=over))
def Recurrence(over, go_backwards=False, initial_state=initial_state_default_or_None):
# helper to compute previous value
# can take a single Variable/Function or a tuple
initial_state = initial_state if _is_given(initial_state) else _current_default_options.initial_state
# if initial state is given and a numeric constant, then turn it into a Constant() object
if np.isscalar(initial_state):
initial_state = Constant(initial_state, shape=(1)) # TODO: This should be automatically done inside the API.
def previous_hook(state):
if isinstance (state, tuple): # if multiple then apply to each element
return tuple([previous_hook(s) for s in state])
# not a tuple: must be a 'scalar', i.e. a single element
return past_value (state, initial_state) if not go_backwards else \
future_value(state, initial_state)
x = Placeholder(name='recurrence_arg')
state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders
prev_state = previous_hook(state_forward) # delay (h, c)
f_x_h_c = over(x, prev_state) # apply the recurrent over
# this returns a Function (x, (h_prev, c_prev)) -> (h, c)
h_c = f_x_h_c.outputs
replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) }
f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c
h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it)
if _trace_layers:
_log_node(h)
_log_node(combine([h.owner]))
apply_x = combine([h]) # the Function that yielded 'h', so we get to know its inputs
# apply_x is a Function x -> h
return Block(apply_x, 'Recurrence', Record(over=over))
def Dense(shape,
init=init_default_or_glorot_uniform,
activation=activation_default_or_None,
input_rank=None,
map_rank=None,
bias=bias_default_or_True,
init_bias=init_bias_default_or_0):
activation = _resolve_activation(activation)
bias = bias if _is_given(bias) else _current_default_options.bias
output_shape = _as_tuple(shape)
if input_rank is not None and map_rank is not None:
raise ValueError(
"Dense: input_rank and map_rank cannot be specified at the same time."
)
# determine meaning of axes
# W gets dimension (input_shape + shape)
# where input_shape is determined as:
# - by default, equal to the dimensions of the input passed to Dense()
# - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over)
# - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over)
# where input_rank and map_rank are mutuallly exclusive.
#output_rank = -len(output_shape) # support outputs with tensor layouts
# BUGBUG: Should this be a negative number now, since output is the last axis in Python?
output_rank = len(output_shape) # support outputs with tensor layouts
# If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank.
# The dimension inference may still create multiple axes.
input_shape = _INFERRED * (input_rank if input_rank is not None else 1)
if input_rank is not None:
UntestedBranchError("Dense, input_rank option not implemented")
infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules
elif map_rank is None:
infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims'
else:
UntestedBranchError("Dense, map_rank option not implemented")
infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones
# parameters bound to this Function
init_weights = _initializer_for(init, Record(output_rank=output_rank))
W = Parameter(input_shape + output_shape, init=init_weights, name='W')
b = Parameter(output_shape, init=init_bias, name='b') if bias else None
# expression of this function
x = Placeholder(name='dense_arg')
apply_x = times(x,
W,
output_rank=output_rank,
infer_input_rank_to_map=infer_input_rank_to_map)
if b:
apply_x = apply_x + b
apply_x = apply_x >> activation
return Block(apply_x, 'Dense', Record(W=W, b=b))
def Dense(shape, init=init_default_or_glorot_uniform, activation=activation_default_or_None,
input_rank=None, map_rank=None,
bias=bias_default_or_True, init_bias=init_bias_default_or_0):
activation = _resolve_activation(activation)
bias = bias if _is_given(bias) else _current_default_options.bias
output_shape = _as_tuple(shape)
if input_rank is not None and map_rank is not None:
raise ValueError("Dense: input_rank and map_rank cannot be specified at the same time.")
# determine meaning of axes
# W gets dimension (input_shape + shape)
# where input_shape is determined as:
# - by default, equal to the dimensions of the input passed to Dense()
# - if input_rank is given, then the last 'input_rank' dimensions of the input (all others are not reduced over)
# - if map_rank is given, then the all but the first 'map_rank' dimensions of the input (those are not reduced over)
# where input_rank and map_rank are mutuallly exclusive.
#output_rank = -len(output_shape) # support outputs with tensor layouts
# BUGBUG: Should this be a negative number now, since output is the last axis in Python?
output_rank = len(output_shape) # support outputs with tensor layouts
# If input_rank not given then pass a single _INFERRED; map_rank if given will determine the input_rank.
# The dimension inference may still create multiple axes.
input_shape = _INFERRED * (input_rank if input_rank is not None else 1)
if input_rank is not None:
UntestedBranchError("Dense, input_rank option not implemented")
infer_input_rank_to_map = -1 # means map_rank is not specified; input_rank rules
elif map_rank is None:
infer_input_rank_to_map = 0 # neither given: default to 'infer W to use all input dims'
else:
UntestedBranchError("Dense, map_rank option not implemented")
infer_input_rank_to_map = map_rank # infer W to use all input dims except the first static 'map_rank' ones
# parameters bound to this Function
init_weights = _initializer_for(init, Record(output_rank=output_rank))
W = Parameter(input_shape + output_shape, init=init_weights, name='W')
b = Parameter( output_shape, init=init_bias, name='b') if bias else None
# expression of this function
x = Placeholder(name='dense_arg')
apply_x = times(x, W, output_rank=output_rank, infer_input_rank_to_map=infer_input_rank_to_map)
if b:
apply_x = apply_x + b
apply_x = apply_x >> activation
return Block(apply_x, 'Dense', Record(W=W, b=b))
