def Flatten(inputs, axis=0, num_axes=-1, **kwargs): if not isinstance(inputs, Tensor): raise RuntimeError('Flatten Operator accepts a Tensor as inputs') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Flatten', **kwargs) if inputs.shape is not None: if num_axes == -1: num_axes = len(inputs.shape) - axis elif num_axes == 0: raise ValueError('num_axes must > 0 or be -1.') num_flatten = np.prod(inputs.shape[axis:axis + num_axes]) output.shape = inputs.shape[:axis] + [ num_flatten ] + inputs.shape[axis + num_axes:] return output
def Reshape(inputs, shape, **kwargs): if not isinstance(inputs, Tensor): raise RuntimeError('Reshape Operator accepts a Tensor as inputs') if not isinstance(shape, tuple) and not isinstance(shape, list): raise TypeError('Reshape dims must be a tuple or list') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Reshape', **kwargs) if inputs.shape is not None: output.shape = [1] * len(shape) for i, s in enumerate(shape): if s == -1: output.shape[i] = 1 else: output.shape[i] = s return output
def Dropout(inputs, prob, scale=True, **kwargs): """ :param inputs: a Tensor with any shape :return: a Tensor of { zero~prob(x) } """ if not isinstance(inputs, Tensor): raise RuntimeError('Dropout Operator accepts a Tensor as inputs') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Dropout', **kwargs) if inputs.shape is not None: output.shape = inputs.shape[:] return output
def Transpose(inputs, perm=None, **kwargs): if not isinstance(inputs, Tensor): raise ValueError('Transpose Operator accepts a Tensor as inputs') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) if kwargs['perm'] is None: kwargs['perm'] = [1, 0] output = Tensor.CreateOperator(nout=1, op_type='Transpose', **kwargs) if inputs.shape is not None: if len(inputs.shape) != len(kwargs['perm']): raise ValueError('input ndim is {}, but perm provide {}'. \ format(len(inputs.shape), len(kwargs['perm']))) output.shape = inputs.shape[:] for i, axis in enumerate(kwargs['perm']): output.shape[i] = inputs.shape[axis] return output
def Softmax(inputs, axis=1, **kwargs): """ :param inputs: a Tensor with any shape :return: a Tensor { e^(xi) / \sigma e^(xj) } """ if not isinstance(inputs, Tensor): raise RuntimeError('Softmax Operator accepts a Tensor as inputs') args = locals() wargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Softmax', **kwargs) if inputs.shape is not None: output.shape = inputs.shape[:] return output
def Tanh(inputs, **kwargs): """ :param inputs: a Tensor with any shape :return: a Tensor of { tanh(x) } Tensor """ if not isinstance(inputs, Tensor): raise RuntimeError('Tanh Operator accepts a Tensor as inputs') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Tanh', **kwargs) if inputs.shape is not None: output.shape = inputs.shape[:] return output
def GlorotUniform(shape, scale=3.0, mode='fan_in', **kwargs): """ :param shape: the shape to fill :param scale: scaling factor (positive float) :param mode: one of "fan_in", "fan_out", "fan_avg" :return: a glorot uniform-filled Tensor """ args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) if not isinstance(shape, Tensor): kwargs['static_shape'] = shape else: kwargs['dynamic_shape'] = shape.name kwargs['extra_inputs'] = shape del kwargs['shape'] output = Tensor.CreateOperator([], nout=1, op_type='GlorotUniform', **kwargs) output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None return output
def Fill(shape, value=1.0, **kwargs): """ :param shape: the shape to fill :param value: the value to fill :return: a value-filled Tensor """ args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) kwargs['value'] = float(kwargs['value']) if not isinstance(shape, Tensor): kwargs['static_shape'] = shape else: kwargs['dynamic_shape'] = shape.name kwargs['extra_inputs'] = shape del kwargs['shape'] output = Tensor.CreateOperator([], nout=1, op_type='Fill', **kwargs) output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None return output
def OneHot(inputs, depth, on_value=1, off_value=0, **kwargs): """ :param inputs: a Tensor with any shape :param depth: a int defining the depth of the one hot dimension :param on_value: a int scalar defining the value to fill when indices[j] = i :param off_value: a int scalar defining the value to fill when indices[j] != i :return: a one-hot Tensor """ if not isinstance(inputs, Tensor): raise RuntimeError('OneHot Operator accepts a Tensor as inputs') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='OneHot', **kwargs) if inputs.shape is not None: output.shape = inputs.shape[:] output.shape.append(depth) return output
def Deconv2D(inputs, num_output, kernel_size, stride=1, pad=0, dilation=1, group=1, **kwargs): if not isinstance(inputs, list) or len(inputs) < 2: raise TypeError('Deconv2D Operator accpets a list of at least 2 Tensors') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) if not isinstance(kwargs['kernel_size'], list): kwargs['kernel_size'] = [kwargs['kernel_size']] if not isinstance(kwargs['stride'], list): kwargs['stride'] = [kwargs['stride']] if not isinstance(kwargs['pad'], list): kwargs['pad'] = [kwargs['pad']] if not isinstance(kwargs['dilation'], list): kwargs['dilation'] = [kwargs['dilation']] return Tensor.CreateOperator(nout=1, op_type='DeConv', **kwargs)
def Dot(inputs, TransA=False, TransB=False, **kwargs): """ :param inputs: a list of 2 Tensors :return: a Tensor of input[0] \dot input[1] """ if not isinstance(inputs, list) or len(inputs) is not 2: raise RuntimeError('Dot Operator accepts a list of 2 Tensors') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Dot', **kwargs) if inputs[0].shape is not None and inputs[1].shape is not None: a_shape = inputs[0].shape[:] if not TransA else inputs[0].shape[::-1] b_shape = inputs[1].shape[:] if not TransB else inputs[1].shape[::-1] output.shape = a_shape output.shape[-1] = b_shape[-1] return output
def LRelu(inputs, slope=0.2, **kwargs): """ :param inputs: a Tensor with any shape :param slope: a float of the slope :return: a Tensor of { max(x,0) + slope * min(x, 0) ) """ if not isinstance(inputs, Tensor): raise RuntimeError('Relu Operator accepts a Tensor as inputs') args = locals() kwargs = args['kwargs'] del args['kwargs'] kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='Relu', **kwargs) if inputs.shape is not None: output.shape = inputs.shape[:] return output
def RandomalUniform(shape, low=-1.0, high=1.0, **kwargs): """ :param shape: the shape to fill :param mean: the low_bound of a uniform distribution :param std: the high_bound of a uniform distribution :return: a random uniform-filled Tensor """ args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) kwargs['low'] = float(kwargs['low']) kwargs['high'] = float(kwargs['high']) if not isinstance(shape, Tensor): kwargs['static_shape'] = shape else: kwargs['dynamic_shape'] = shape.name kwargs['extra_inputs'] = shape del kwargs['shape'] output = Tensor.CreateOperator([], nout=1, op_type='RandomUniform', **kwargs) output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None return output
def TruncatedNormal(shape, mean=0.0, std=1.0, **kwargs): """ :param shape: the shape to fill :param mean: the mean of a normal distribution :param std: the std of a normal distribution :return: a truncated normal-filled Tensor """ args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) kwargs['low'] = float(mean - 2.0 * std) kwargs['high'] = float(mean + 2.0 * std) if not isinstance(shape, Tensor): kwargs['static_shape'] = shape else: kwargs['dynamic_shape'] = shape.name kwargs['extra_inputs'] = shape del kwargs['shape'] output = Tensor.CreateOperator([], nout=1, op_type='TruncatedNormal', **kwargs) output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None return output
def InnerProduct(inputs, num_output, axis=1, TransW=True, **kwargs): """ :param inputs: a list contains [input, weight] or [input, weight, bias] :param num_output: a int of the output dim :param axis a int of the start axis :param TransW a bool of whether to transpose the weights :return: a Tensor of { input * weight + bias } """ if not isinstance(inputs, list) or len(inputs) < 2: raise RuntimeError('InnerProduct Operator accpets a list of at least 2 Tensors') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='InnerProduct', **kwargs) if inputs[0].shape is not None: output.shape = inputs[0].shape[: axis + 1] output.shape[axis] = num_output return output
def SoftmaxCrossEntropy(inputs, axis=1, normalization='FULL', **kwargs): """ :param inputs: a list of Tensor contains [input, label] :param normalization: a str of (UNIT, FULL, BATCH_SIZE, NONE) :return: a Tensor of loss with the shape (1,) """ if not isinstance(inputs, list) or len(inputs) is not 2: raise RuntimeError('SoftmaxCrossEntropy Operator accpets a list of 2 Tensors') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) output = Tensor.CreateOperator(nout=1, op_type='SoftmaxCrossEntropy', **kwargs) if inputs[0].shape is not None: if normalization != 'UNIT': output.shape = [1] elif all(dim is not None for dim in inputs[0].shape): outer_dim = int(np.prod(inputs[0].shape[0 : axis])) inner_dim = int(np.prod(inputs[0].shape[axis + 1 :])) output.shape = [outer_dim * inner_dim] else: output.shape = [None] return output
def MPIBroadcast(inputs, root, mpi_rank=None, **kwargs): """ :param inputs: a Tensor which to broadcast :param root: a int of the root in a broadcast group :return: a Tensor that be broadcast """ if not isinstance(inputs, Tensor): raise RuntimeError('MPIBroadcast Operator accepts a Tensor as inputs') args = locals(); kwargs = args['kwargs'] del args['kwargs']; kwargs = dict(args, **kwargs) if mpi_rank is None: num_nodes = mpi.size() mpi_rank = [i for i in xrange(0, num_nodes)] if not isinstance(kwargs['mpi_rank'], list): kwargs['mpi_rank'] = [kwargs['mpi_rank']] comm, group = mpi.group(root, incl=mpi_rank) new_kwargs = {'inputs': kwargs['inputs'], 'mpi_rank': mpi_rank, 'comm': comm, 'group': group} return Tensor.CreateOperator(nout=1, op_type='MPIBroadcast', **new_kwargs)
def scan(fn, sequences, outputs_info, n_steps=None, axis=0): """Run a dynamic loop of the given one step function. Parameters ---------- fn : lambda The function to execute at each step. sequences : Tensor or list or Tensor The sequences. outputs_info : Tensor or list of Tensor The outputs. n_steps : int or Tensor The steps of loop. axis : int The axis of sequences. Returns ------- Tensor or list of Tensor The outputs. Examples -------- >>> x = Tensor('x', dtype='float32') >>> x.set_value([1, 2, 3, 4, 5]) >>> zero = Tensor('zero', dtype='float32').Variable() >>> zero.set_value(0) >>> prefix_sum = scan(fn=lambda x_i, y_i : x_i + y_i, sequences=x, outputs_info=zero, n_steps=x.shape[0], axis=0) >>> f = theano.function(outputs=prefix_sum) >>> print(f()) >>> [ 1. 3. 6. 10. 15.] """ if not isinstance(sequences, list): sequences = [sequences] if not isinstance(outputs_info, list): outputs_info = [outputs_info] # 1. exact default outputs fn_nargs = len(inspect.getargspec(fn)[0]) default_outputs = [] for output in outputs_info: if output is not None: default_outputs.append(output) if len(sequences) + len(default_outputs) < fn_nargs: raise RuntimeError('Expect {} args of fn, but at most {} args can be used\n' 'sequences provide {}, outputs_info provide {}.'. \ format(fn_nargs, len(sequences) + len(default_outputs), len(sequences), len(default_outputs))) # 2. simulate specific function fn_inputs = [x for x in sequences] + default_outputs fn_inputs = copy.deepcopy(fn_inputs) # clear to avoid importing external expressions into template function for input in fn_inputs: input.expressions = {} outputs = fn(*fn_inputs) if not isinstance(outputs, tuple): outputs = [outputs] else: outputs = list(outputs) if len(outputs) != len(outputs_info): raise RuntimeError( 'Expect {} outputs of fn, but len of outputs_info is {}.'.format( len(outputs), len(outputs_info))) # 3. make GraphDef graph_def = pb.GraphDef() all_exprs = {} for output in outputs: graph_def.target.extend([output._name]) all_exprs = dict(all_exprs, **output.expressions) all_exprs = sorted(all_exprs.items(), key=lambda d: d[0]) forward_ops = copy.deepcopy([v for k, v in all_exprs]) graph_def.op.extend(forward_ops) # 4. exact external inputs external_inputs = [] internal_outputs = [] internal_inputs = [tensor.name for tensor in fn_inputs] for op in graph_def.op: for input in op.input: if input not in internal_inputs: if input not in internal_outputs: external_inputs.append(input) for output in op.output: internal_outputs.append(output) # 5. collect inputs (sequences + default + external) default_outputs = [ elem.name if elem is not None else '' for elem in outputs_info ] inputs = fn_inputs + [Tensor(name) for name in external_inputs] kwargs = { 'axis': axis, 'nseqs': len(sequences), 'default_outputs': default_outputs, 'func_str': str(graph_def) } if isinstance(n_steps, int): kwargs['nsteps'] = n_steps kwargs['step_type'] = 'Static' elif isinstance(n_steps, Tensor): kwargs['extra_inputs'] = [n_steps] kwargs['step_tensor'] = n_steps.name kwargs['step_type'] = 'Dynamic' else: kwargs['step_type'] = 'Default' kwargs['inputs_name'] = [t.name for t in inputs] kwargs['outputs_name'] = [t.name for t in outputs] return Tensor.CreateOperator(inputs, existing_outputs=outputs, op_type='Scan', **kwargs)
def scan(fn, sequences, outputs_info, n_steps=None, axis=0): if not isinstance(sequences, list): sequences = [sequences] if not isinstance(outputs_info, list): outputs_info = [outputs_info] # 1. exact default outputs fn_nargs = len(inspect.getargspec(fn)[0]) default_outputs = [] for output in outputs_info: if output is not None: default_outputs.append(output) if len(sequences) + len(default_outputs) < fn_nargs: raise RuntimeError('expect {} fn args, but at most {} args can be used\n' 'sequences provide {}, outputs_info provide {}'. \ format(fn_nargs, len(sequences) + len(default_outputs), len(sequences), len(default_outputs))) # 2. simulate specfic function fn_inputs = [x for x in sequences] + default_outputs fn_inputs = copy.deepcopy(fn_inputs) # clear to avoid importing external expressions into template function for input in fn_inputs: input.expressions = {} outputs = fn(*fn_inputs) if not isinstance(outputs, tuple): outputs = [outputs] else: outputs = list(outputs) if len(outputs) != len(outputs_info): raise RuntimeError( 'fn expect {} outputs, but len of outputs_info is {}'.format( len(outputs), len(outputs_info))) # 3. make GraphDef graph_def = pb.GraphDef() all_exprs = {} for output in outputs: graph_def.target.extend([output._name]) all_exprs = dict(all_exprs, **output.expressions) all_exprs = sorted(all_exprs.items(), key=lambda d: d[0]) forward_ops = copy.deepcopy([v for k, v in all_exprs]) graph_def.op.extend(forward_ops) # 4. exact external inputs external_inputs = [] internal_outputs = [] internal_inputs = [tensor.name for tensor in fn_inputs] for op in graph_def.op: for input in op.input: if input not in internal_inputs: if input not in internal_outputs: external_inputs.append(input) for output in op.output: internal_outputs.append(output) # 5. collect inputs (sequences + default + external) default_outputs = [ elem.name if elem is not None else '' for elem in outputs_info ] inputs = fn_inputs + [Tensor(name) for name in external_inputs] kwargs = { 'axis': axis, 'nseqs': len(sequences), 'default_outputs': default_outputs, 'func_str': str(graph_def) } if isinstance(n_steps, int): kwargs['nsteps'] = n_steps kwargs['step_type'] = 'Static' elif isinstance(n_steps, Tensor): kwargs['extra_inputs'] = [n_steps] kwargs['step_tensor'] = n_steps.name kwargs['step_type'] = 'Dynamic' else: kwargs['step_type'] = 'Default' kwargs['inputs_name'] = [t.name for t in inputs] kwargs['outputs_name'] = [t.name for t in outputs] return Tensor.CreateOperator(inputs, existing_outputs=outputs, op_type='Scan', **kwargs)