def Flatten(inputs, axis=0, num_axes=-1, **kwargs):
if not isinstance(inputs, Tensor):
raise RuntimeError('Flatten Operator accepts a Tensor as inputs')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Flatten', **kwargs)
if inputs.shape is not None:
if num_axes == -1: num_axes = len(inputs.shape) - axis
elif num_axes == 0:
raise ValueError('num_axes must > 0 or be -1.')
num_flatten = np.prod(inputs.shape[axis:axis + num_axes])
output.shape = inputs.shape[:axis] + [
num_flatten
] + inputs.shape[axis + num_axes:]
return output
def Reshape(inputs, shape, **kwargs):
if not isinstance(inputs, Tensor):
raise RuntimeError('Reshape Operator accepts a Tensor as inputs')
if not isinstance(shape, tuple) and not isinstance(shape, list):
raise TypeError('Reshape dims must be a tuple or list')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Reshape', **kwargs)
if inputs.shape is not None:
output.shape = [1] * len(shape)
for i, s in enumerate(shape):
if s == -1: output.shape[i] = 1
else: output.shape[i] = s
return output
def Dropout(inputs, prob, scale=True, **kwargs):
"""
:param inputs: a Tensor with any shape
:return: a Tensor of { zero~prob(x) }
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('Dropout Operator accepts a Tensor as inputs')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Dropout', **kwargs)
if inputs.shape is not None:
output.shape = inputs.shape[:]
return output
def Transpose(inputs, perm=None, **kwargs):
if not isinstance(inputs, Tensor):
raise ValueError('Transpose Operator accepts a Tensor as inputs')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
if kwargs['perm'] is None: kwargs['perm'] = [1, 0]
output = Tensor.CreateOperator(nout=1, op_type='Transpose', **kwargs)
if inputs.shape is not None:
if len(inputs.shape) != len(kwargs['perm']):
raise ValueError('input ndim is {}, but perm provide {}'. \
format(len(inputs.shape), len(kwargs['perm'])))
output.shape = inputs.shape[:]
for i, axis in enumerate(kwargs['perm']):
output.shape[i] = inputs.shape[axis]
return output
def Softmax(inputs, axis=1, **kwargs):
"""
:param inputs: a Tensor with any shape
:return: a Tensor { e^(xi) / \sigma e^(xj) }
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('Softmax Operator accepts a Tensor as inputs')
args = locals()
wargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Softmax', **kwargs)
if inputs.shape is not None:
output.shape = inputs.shape[:]
return output
def Tanh(inputs, **kwargs):
"""
:param inputs: a Tensor with any shape
:return: a Tensor of { tanh(x) } Tensor
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('Tanh Operator accepts a Tensor as inputs')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Tanh', **kwargs)
if inputs.shape is not None:
output.shape = inputs.shape[:]
return output
def GlorotUniform(shape, scale=3.0, mode='fan_in', **kwargs):
"""
:param shape: the shape to fill
:param scale: scaling factor (positive float)
:param mode: one of "fan_in", "fan_out", "fan_avg"
:return: a glorot uniform-filled Tensor
"""
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
if not isinstance(shape, Tensor): kwargs['static_shape'] = shape
else:
kwargs['dynamic_shape'] = shape.name
kwargs['extra_inputs'] = shape
del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='GlorotUniform', **kwargs)
output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output
def Fill(shape, value=1.0, **kwargs):
"""
:param shape: the shape to fill
:param value: the value to fill
:return: a value-filled Tensor
"""
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
kwargs['value'] = float(kwargs['value'])
if not isinstance(shape, Tensor): kwargs['static_shape'] = shape
else:
kwargs['dynamic_shape'] = shape.name
kwargs['extra_inputs'] = shape
del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='Fill', **kwargs)
output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output
def OneHot(inputs, depth, on_value=1, off_value=0, **kwargs):
"""
:param inputs: a Tensor with any shape
:param depth: a int defining the depth of the one hot dimension
:param on_value: a int scalar defining the value to fill when indices[j] = i
:param off_value: a int scalar defining the value to fill when indices[j] != i
:return: a one-hot Tensor
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('OneHot Operator accepts a Tensor as inputs')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='OneHot', **kwargs)
if inputs.shape is not None:
output.shape = inputs.shape[:]
output.shape.append(depth)
return output
def Deconv2D(inputs, num_output, kernel_size,
stride=1, pad=0, dilation=1, group=1, **kwargs):
if not isinstance(inputs, list) or len(inputs) < 2:
raise TypeError('Deconv2D Operator accpets a list of at least 2 Tensors')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
if not isinstance(kwargs['kernel_size'], list):
kwargs['kernel_size'] = [kwargs['kernel_size']]
if not isinstance(kwargs['stride'], list):
kwargs['stride'] = [kwargs['stride']]
if not isinstance(kwargs['pad'], list):
kwargs['pad'] = [kwargs['pad']]
if not isinstance(kwargs['dilation'], list):
kwargs['dilation'] = [kwargs['dilation']]
return Tensor.CreateOperator(nout=1, op_type='DeConv', **kwargs)
def Dot(inputs, TransA=False, TransB=False, **kwargs):
"""
:param inputs: a list of 2 Tensors
:return: a Tensor of input[0] \dot input[1]
"""
if not isinstance(inputs, list) or len(inputs) is not 2:
raise RuntimeError('Dot Operator accepts a list of 2 Tensors')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Dot', **kwargs)
if inputs[0].shape is not None and inputs[1].shape is not None:
a_shape = inputs[0].shape[:] if not TransA else inputs[0].shape[::-1]
b_shape = inputs[1].shape[:] if not TransB else inputs[1].shape[::-1]
output.shape = a_shape
output.shape[-1] = b_shape[-1]
return output
def LRelu(inputs, slope=0.2, **kwargs):
"""
:param inputs: a Tensor with any shape
:param slope: a float of the slope
:return: a Tensor of { max(x,0) + slope * min(x, 0) )
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('Relu Operator accepts a Tensor as inputs')
args = locals()
kwargs = args['kwargs']
del args['kwargs']
kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='Relu', **kwargs)
if inputs.shape is not None:
output.shape = inputs.shape[:]
return output
def RandomalUniform(shape, low=-1.0, high=1.0, **kwargs):
"""
:param shape: the shape to fill
:param mean: the low_bound of a uniform distribution
:param std: the high_bound of a uniform distribution
:return: a random uniform-filled Tensor
"""
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
kwargs['low'] = float(kwargs['low'])
kwargs['high'] = float(kwargs['high'])
if not isinstance(shape, Tensor): kwargs['static_shape'] = shape
else:
kwargs['dynamic_shape'] = shape.name
kwargs['extra_inputs'] = shape
del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='RandomUniform', **kwargs)
output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output
def TruncatedNormal(shape, mean=0.0, std=1.0, **kwargs):
"""
:param shape: the shape to fill
:param mean: the mean of a normal distribution
:param std: the std of a normal distribution
:return: a truncated normal-filled Tensor
"""
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
kwargs['low'] = float(mean - 2.0 * std)
kwargs['high'] = float(mean + 2.0 * std)
if not isinstance(shape, Tensor): kwargs['static_shape'] = shape
else:
kwargs['dynamic_shape'] = shape.name
kwargs['extra_inputs'] = shape
del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='TruncatedNormal', **kwargs)
output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output
def InnerProduct(inputs, num_output, axis=1, TransW=True, **kwargs):
"""
:param inputs: a list contains [input, weight] or [input, weight, bias]
:param num_output: a int of the output dim
:param axis a int of the start axis
:param TransW a bool of whether to transpose the weights
:return: a Tensor of { input * weight + bias }
"""
if not isinstance(inputs, list) or len(inputs) < 2:
raise RuntimeError('InnerProduct Operator accpets a list of at least 2 Tensors')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='InnerProduct', **kwargs)
if inputs[0].shape is not None:
output.shape = inputs[0].shape[: axis + 1]
output.shape[axis] = num_output
return output
def SoftmaxCrossEntropy(inputs, axis=1, normalization='FULL', **kwargs):
"""
:param inputs: a list of Tensor contains [input, label]
:param normalization: a str of (UNIT, FULL, BATCH_SIZE, NONE)
:return: a Tensor of loss with the shape (1,)
"""
if not isinstance(inputs, list) or len(inputs) is not 2:
raise RuntimeError('SoftmaxCrossEntropy Operator accpets a list of 2 Tensors')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
output = Tensor.CreateOperator(nout=1, op_type='SoftmaxCrossEntropy', **kwargs)
if inputs[0].shape is not None:
if normalization != 'UNIT': output.shape = [1]
elif all(dim is not None for dim in inputs[0].shape):
outer_dim = int(np.prod(inputs[0].shape[0 : axis]))
inner_dim = int(np.prod(inputs[0].shape[axis + 1 :]))
output.shape = [outer_dim * inner_dim]
else: output.shape = [None]
return output
def MPIBroadcast(inputs, root, mpi_rank=None, **kwargs):
"""
:param inputs: a Tensor which to broadcast
:param root: a int of the root in a broadcast group
:return: a Tensor that be broadcast
"""
if not isinstance(inputs, Tensor):
raise RuntimeError('MPIBroadcast Operator accepts a Tensor as inputs')
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
if mpi_rank is None:
num_nodes = mpi.size()
mpi_rank = [i for i in xrange(0, num_nodes)]
if not isinstance(kwargs['mpi_rank'], list):
kwargs['mpi_rank'] = [kwargs['mpi_rank']]
comm, group = mpi.group(root, incl=mpi_rank)
new_kwargs = {'inputs': kwargs['inputs'], 'mpi_rank': mpi_rank,
'comm': comm, 'group': group}
return Tensor.CreateOperator(nout=1, op_type='MPIBroadcast', **new_kwargs)
def scan(fn, sequences, outputs_info, n_steps=None, axis=0):
"""Run a dynamic loop of the given one step function.
Parameters
----------
fn : lambda
The function to execute at each step.
sequences : Tensor or list or Tensor
The sequences.
outputs_info : Tensor or list of Tensor
The outputs.
n_steps : int or Tensor
The steps of loop.
axis : int
The axis of sequences.
Returns
-------
Tensor or list of Tensor
The outputs.
Examples
--------
>>> x = Tensor('x', dtype='float32')
>>> x.set_value([1, 2, 3, 4, 5])
>>> zero = Tensor('zero', dtype='float32').Variable()
>>> zero.set_value(0)
>>> prefix_sum = scan(fn=lambda x_i, y_i : x_i + y_i, sequences=x,
outputs_info=zero, n_steps=x.shape[0], axis=0)
>>> f = theano.function(outputs=prefix_sum)
>>> print(f())
>>> [ 1. 3. 6. 10. 15.]
"""
if not isinstance(sequences, list): sequences = [sequences]
if not isinstance(outputs_info, list): outputs_info = [outputs_info]
# 1. exact default outputs
fn_nargs = len(inspect.getargspec(fn)[0])
default_outputs = []
for output in outputs_info:
if output is not None: default_outputs.append(output)
if len(sequences) + len(default_outputs) < fn_nargs:
raise RuntimeError('Expect {} args of fn, but at most {} args can be used\n'
'sequences provide {}, outputs_info provide {}.'. \
format(fn_nargs, len(sequences) + len(default_outputs),
len(sequences), len(default_outputs)))
# 2. simulate specific function
fn_inputs = [x for x in sequences] + default_outputs
fn_inputs = copy.deepcopy(fn_inputs)
# clear to avoid importing external expressions into template function
for input in fn_inputs:
input.expressions = {}
outputs = fn(*fn_inputs)
if not isinstance(outputs, tuple): outputs = [outputs]
else: outputs = list(outputs)
if len(outputs) != len(outputs_info):
raise RuntimeError(
'Expect {} outputs of fn, but len of outputs_info is {}.'.format(
len(outputs), len(outputs_info)))
# 3. make GraphDef
graph_def = pb.GraphDef()
all_exprs = {}
for output in outputs:
graph_def.target.extend([output._name])
all_exprs = dict(all_exprs, **output.expressions)
all_exprs = sorted(all_exprs.items(), key=lambda d: d[0])
forward_ops = copy.deepcopy([v for k, v in all_exprs])
graph_def.op.extend(forward_ops)
# 4. exact external inputs
external_inputs = []
internal_outputs = []
internal_inputs = [tensor.name for tensor in fn_inputs]
for op in graph_def.op:
for input in op.input:
if input not in internal_inputs:
if input not in internal_outputs:
external_inputs.append(input)
for output in op.output:
internal_outputs.append(output)
# 5. collect inputs (sequences + default + external)
default_outputs = [
elem.name if elem is not None else '' for elem in outputs_info
]
inputs = fn_inputs + [Tensor(name) for name in external_inputs]
kwargs = {
'axis': axis,
'nseqs': len(sequences),
'default_outputs': default_outputs,
'func_str': str(graph_def)
}
if isinstance(n_steps, int):
kwargs['nsteps'] = n_steps
kwargs['step_type'] = 'Static'
elif isinstance(n_steps, Tensor):
kwargs['extra_inputs'] = [n_steps]
kwargs['step_tensor'] = n_steps.name
kwargs['step_type'] = 'Dynamic'
else:
kwargs['step_type'] = 'Default'
kwargs['inputs_name'] = [t.name for t in inputs]
kwargs['outputs_name'] = [t.name for t in outputs]
return Tensor.CreateOperator(inputs,
existing_outputs=outputs,
op_type='Scan',
**kwargs)
def scan(fn, sequences, outputs_info, n_steps=None, axis=0):
if not isinstance(sequences, list): sequences = [sequences]
if not isinstance(outputs_info, list): outputs_info = [outputs_info]
# 1. exact default outputs
fn_nargs = len(inspect.getargspec(fn)[0])
default_outputs = []
for output in outputs_info:
if output is not None: default_outputs.append(output)
if len(sequences) + len(default_outputs) < fn_nargs:
raise RuntimeError('expect {} fn args, but at most {} args can be used\n'
'sequences provide {}, outputs_info provide {}'. \
format(fn_nargs, len(sequences) + len(default_outputs),
len(sequences), len(default_outputs)))
# 2. simulate specfic function
fn_inputs = [x for x in sequences] + default_outputs
fn_inputs = copy.deepcopy(fn_inputs)
# clear to avoid importing external expressions into template function
for input in fn_inputs:
input.expressions = {}
outputs = fn(*fn_inputs)
if not isinstance(outputs, tuple): outputs = [outputs]
else: outputs = list(outputs)
if len(outputs) != len(outputs_info):
raise RuntimeError(
'fn expect {} outputs, but len of outputs_info is {}'.format(
len(outputs), len(outputs_info)))
# 3. make GraphDef
graph_def = pb.GraphDef()
all_exprs = {}
for output in outputs:
graph_def.target.extend([output._name])
all_exprs = dict(all_exprs, **output.expressions)
all_exprs = sorted(all_exprs.items(), key=lambda d: d[0])
forward_ops = copy.deepcopy([v for k, v in all_exprs])
graph_def.op.extend(forward_ops)
# 4. exact external inputs
external_inputs = []
internal_outputs = []
internal_inputs = [tensor.name for tensor in fn_inputs]
for op in graph_def.op:
for input in op.input:
if input not in internal_inputs:
if input not in internal_outputs:
external_inputs.append(input)
for output in op.output:
internal_outputs.append(output)
# 5. collect inputs (sequences + default + external)
default_outputs = [
elem.name if elem is not None else '' for elem in outputs_info
]
inputs = fn_inputs + [Tensor(name) for name in external_inputs]
kwargs = {
'axis': axis,
'nseqs': len(sequences),
'default_outputs': default_outputs,
'func_str': str(graph_def)
}
if isinstance(n_steps, int):
kwargs['nsteps'] = n_steps
kwargs['step_type'] = 'Static'
elif isinstance(n_steps, Tensor):
kwargs['extra_inputs'] = [n_steps]
kwargs['step_tensor'] = n_steps.name
kwargs['step_type'] = 'Dynamic'
else:
kwargs['step_type'] = 'Default'
kwargs['inputs_name'] = [t.name for t in inputs]
kwargs['outputs_name'] = [t.name for t in outputs]
return Tensor.CreateOperator(inputs,
existing_outputs=outputs,
op_type='Scan',
**kwargs)
