def __init__( self, net, input, target, lr=0.07, adapt=False, rate_dec=0.5, rate_inc=1.2, rate_min=1e-9, rate_max=50 ): super(TrainRprop, self).__init__(net, input, target, lr, adapt) self.rate_inc = rate_inc self.rate_dec = rate_dec self.rate_max = rate_max self.rate_min = rate_min size = tool.np_size(net) self.grad_prev = np.zeros(size) self.rate = np.zeros(size) + lr
def __init__(self, ci, cn, co, property): self.ci = ci self.cn = cn self.co = co self.np = {} for p, shape in property.items(): self.np[p] = np.empty(shape) self.inp = np.zeros(ci) self.out = np.zeros(co) # Property must be change when init Layer self.out_minmax = np.empty([self.co, 2]) # Property will be change when init Net self.inp_minmax = np.empty([self.ci, 2]) self.initf = None
def __init__(self, inp_minmax, co, layers, connect, trainf, errorf): self.inp_minmax = np.asfarray(inp_minmax) self.out_minmax = np.zeros([co, 2]) self.ci = self.inp_minmax.shape[0] self.co = co self.layers = layers self.trainf = trainf self.errorf = errorf self.inp = np.zeros(self.ci) self.out = np.zeros(self.co) # Check connect format assert self.inp_minmax.ndim == 2 assert self.inp_minmax.shape[1] == 2 if len(connect) != len(layers) + 1: raise ValueError("Connect error") # Check connect links tmp = [0] * len(connect) for con in connect: for s in con: if s != -1: tmp[s] += 1 for l, c in enumerate(tmp): if c == 0 and l != len(layers): raise ValueError("Connect error: Lost the signal " + "from the layer " + str(l - 1)) self.connect = connect # Set inp_minmax for all layers for nl, nums_signal in enumerate(self.connect): if nl == len(self.layers): minmax = self.out_minmax else: minmax = self.layers[nl].inp_minmax ni = 0 for ns in nums_signal: t = self.layers[ns].out_minmax if ns != -1 else self.inp_minmax if ni + len(t) > len(minmax): raise ValueError("Connect error: on layer " + str(l - 1)) minmax[ni : ni + len(t)] = t ni += len(t) if ni != len(minmax): raise ValueError("Connect error: Empty inputs on layer " + str(l - 1)) self.init()
def np_get(net): """ Get all network parameters in one array """ size = np_size(net) result = np.zeros(size) start = 0 for l in net.layers: for prop in l.np.values(): result[start: start+prop.size] = prop.flat[:] start += prop.size return result
def ff_grad(net, input, target): """ Calc and accumulate gradient with backpropogete method, for feed-forward neuran networks on each step :Parametrs: net: Net Feed-forward network input: array, shape = N,net.ci Input array target: array, shape = N,net.co Train target deriv: callable Derivative of error function :Returns: grad: list of dict Dradient of net for each layer, format:[{'w':..., 'b':...},{'w':..., 'b':...},...] grad_flat: array All neurons propertys in 1 array (reference of grad) output: array output of network """ grad_flat = np.zeros(np_size(net)) grad = [] st = 0 for i, l in enumerate(net.layers): grad.append({}) for k, v in l.np.items(): grad[i][k] = grad_flat[st: st + v.size] grad[i][k] = grad[i][k].reshape(v.shape) #replacement for " grad[i][k].shape = v.shape" st += v.size output = np.empty(shape = (len(target),)+net.out.shape) i = 0 for inp, tar in zip(input, target): out = net.step(inp) ff_grad_step(net, out, tar, grad) #import sys, traceback #traceback.print_stack() #raise(ValueError("Exit 1 for live {0}".format(grad_flat))) output[i] = out i += 1 #Dirty hack st = 0 for i, l in enumerate(net.layers): for k, v in l.np.items(): grad_flat[st: st + v.size] = grad[i][k].reshape(v.size) st += v.size return grad, grad_flat, output
def __init__(self, ci, cn, transf, max_iter, delta): Layer.__init__(self, ci, cn, cn, {'w': (cn, ci), 'b': cn}) self.max_iter = max_iter self.delta = delta self.transf = transf self.outs = [] if not hasattr(transf, 'out_minmax'): test = np.asfarry([-1e100, -100, -10, -1, 0, 1, 10, 100, 1e100]) val = self.transf(test) self.out_minmax = np.array([val.min(), val.max()] * self.co) else: self.out_minmax = np.asfarray([transf.out_minmax] * self.co) self.initf = None self.s = np.zeros(self.cn)
def ff_grad_step(net, out, tar, grad=None): """ Calc gradient with backpropogete method, for feed-forward neuran networks on each step :Parametrs: net: Net Feed-forward network inp: array, size = net.ci Input array tar: array, size = net.co Train target deriv: callable Derivative of error function grad:list of dict default(None) Grad on previous step :Returns: grad: list of dict Gradient of net for each layer, format:[{'w':..., 'b':...},{'w':..., 'b':...},...] """ delt = [None] * len(net.layers) if grad is None: grad = [] for i, l in enumerate(net.layers): grad.append({}) for k, v in l.np.items(): grad[i][k] = np.zeros(v.shape) e = out - tar # for output layer ln = len(net.layers) - 1 layer = net.layers[ln] delt[ln] = net.errorf.deriv(e) * layer.transf.deriv(layer.s, out) delt[ln] = delt[ln].reshape((delt[ln].size, 1,)) #replacement for " delt[ln].shape = delt[ln].size, 1" grad[ln]['w'] += delt[ln] * layer.inp grad[ln]['b'] += delt[ln].reshape(delt[ln].size) bp = range(len(net.layers) -2, -1, -1) for ln in bp: layer = net.layers[ln] next = ln + 1 dS = np.sum(net.layers[next].np['w'] * delt[next], axis=0) delt[ln] = dS * layer.transf.deriv(layer.s, layer.out) delt[ln] = delt[ln].reshape((delt[ln].size, 1,)) #replacement for " delt[ln].shape = delt[ln].size, 1" grad[ln]['w'] += delt[ln] * layer.inp grad[ln]['b'] += delt[ln].reshape(delt[ln].size) return grad
def __init__(self, ci, cn, transf): Layer.__init__(self, ci, cn, cn, {'w': (cn, ci), 'b': cn}) self.transf = transf if not hasattr(transf, 'out_minmax'): test = np.asfarry([-1e100, -100, -10, -1, 0, 1, 10, 100, 1e100]) val = self.transf(test) self.out_minmax = np.array([val.min(), val.max()] * self.co) else: self.out_minmax = np.asfarray([transf.out_minmax] * self.co) # default init function self.initf = init.initwb_reg #self.initf = init.initwb_nw self.s = np.zeros(self.cn)
def sim(self, input): """ Simulate a neural network :Parameters: input: array like array input vectors :Returns: outputs: array like array output vectors """ input = np.asfarray(input) assert input.ndim == 2 assert input.shape[1] == self.ci output = np.zeros([len(input), self.co]) for inp_num, inp in enumerate(input): output[inp_num, :] = self.step(inp) return output