def gradient(self, x, g, returnError=True): x = np.asarray(x) g = np.asarray(g) probs = self.probs(x) delta = (probs - g) / probs.size penMask = np.ones_like(self.weights) penMask[-1, :] = 0.0 grad = ( util.bias(x).T.dot(delta) + self.elastic * 2.0 * self.penalty * penMask * self.weights / self.weights.size + # L2-norm penalty (1.0 - self.elastic) * self.penalty * penMask * np.sign(self.weights) / self.weights.size) # L1-norm penalty gf = grad.ravel() if returnError: pf = self.weights[:-1, :].ravel() err = ( -np.mean(g * np.log(util.capZero(probs))) + self.elastic * self.penalty * pf.dot(pf) / pf.size + # L2-norm penalty (1.0 - self.elastic) * self.penalty * np.mean(np.abs(pf)) ) # L1-norm penalty return err, gf else: return gf
def error(self, x, g): x = np.asarray(x) g = np.asarray(g) likes = np.log(util.capZero(self.probs(x))) pf = self.weights[:-1, :].ravel() return (-np.mean(g * likes) + self.elastic * self.penalty * pf.dot(pf) / pf.size + # L2-norm penalty (1.0 - self.elastic) * self.penalty * np.mean(np.abs(pf)) ) # L1-norm penalty
def initZmus(self, x): x = np.asarray(x) self.shift = np.mean(x, axis=0) #self.scale = util.capZero(np.std(x, axis=0)) self.scale = np.std(x, axis=0) # best way to handle this? XXX - idfah if np.any(np.isclose(self.scale, 0.0)): print( 'Standardizer Warning: Some dimensions are constant, capping zeros.' ) self.scale = util.capZero(self.scale)
def error(self, x, g): """Compute the negative log likelyhood for given inputs and targets. Error function for Optable interface. Args: x: g: Returns: The scalar negative log likelyhood. """ x = np.asarray(x) g = np.asarray(g) likes = np.log(util.capZero(self.probs(x))) return -np.mean(g * likes)
def gradient(self, x, g, returnError=True): x = np.asarray(x) g = np.asarray(g) probs = self.probs(x) delta = (probs - g) / probs.size grad = util.bias(x).T.dot(delta) gf = grad.ravel() if returnError: err = -np.mean(g * np.log(util.capZero(probs))) return err, gf else: return gf
def probs(self, x): """Compute class probabilities. Args: x: Input data. A numpy array with shape (nObs[,nIn]). Returns: Numpy array with shape (nObs,nCls) containing the probability values. Notes: This is less precise than discrim. Only use probs if you need the class probabilites for each observation. """ # log probability densities logDens = self.logDens(x) # density_i*P(C=i) / sum_j(density_j*P(C=j)) mx = np.max((np.max(logDens), 0.0)) dens = util.capZero(np.exp(logDens-mx)) return dens / dens.sum(axis=1)[:,None]
def plotFreqResponse(self, freqs=None, scale='linear', showCorners=True, label='Frequency Response', ax=None, **kwargs): """Plot the frequency response of the filter. """ if ax is None: fig = plt.figure() ax = fig.add_subplot(1, 1, 1) freqs, responses = self.frequencyResponse(freqs=freqs) freqs = freqs * self.sampRate * 0.5 / np.pi responseMags = np.abs(responses) scale = scale.lower() if scale == 'linear': ax.set_ylabel('Gain') elif scale == 'log': ax.set_ylabel('Gain') ax.set_yscale('symlog') elif scale == 'db': responseMags = 10.0 * np.log10(util.capZero(responseMags**2)) ax.set_ylabel('Gain (dB)') else: raise RuntimeError('Invalid scale: ' + str(scale) + '.') lines = ax.plot(freqs, responseMags, label=label, **kwargs) result = {'ax': ax, 'lines': lines} if showCorners: if scale == 'db': halfPow = 10.0 * np.log10(0.5) halfAmp = 10.0 * np.log10(0.5**2) mn = np.min(responseMags) mx = np.max(responseMags) else: halfPow = np.sqrt(0.5) halfAmp = 0.5 mn = np.min(responseMags) mn = np.min((mn, 0.0)) mx = np.max(responseMags) mx = np.max((mx, 1.0)) halfPowerLines = ax.hlines(halfPow, 0.0, 0.5 * self.sampRate, color='red', linestyle='-.', label='Half Power') result['halfPowerLines'] = halfPowerLines halfAmpLines = ax.hlines(halfAmp, 0.0, 0.5 * self.sampRate, color='orange', linestyle=':', label='Half Amplitude') result['halfAmpLines'] = halfAmpLines cornerLines = ax.vlines((self.lowFreq, self.highFreq), mn, mx, color='violet', linestyle='--', label='Corners') result['cornerLines'] = cornerLines ax.set_xlabel('Frequency (Hz)') ax.set_ylim((0.0, 1.0)) return result
def alopex(optable, stepSize=0.0075, tempInit=10000, tempIter=20, accuracy=0.0, precision=0.0, divergeThresh=1.0e10, maxIter=10000, pTrace=False, tTrace=False, eTrace=False, callback=None, verbose=False, *args, **kwargs): """ ALgorithm Of Pattern EXtraction (ALOPEX) Args: optable: stepSize: Step size. tempInit: tempIter: accuracy: Terminate if current value of the error funciton falls below this value. precision: Terminate if change in the error function falls below this value. divergeThresh: Terminate if the value of the error function exceeds this value. maxIter: Terminate once current iteration reaches this value. pTrace: If True, a list of matrices (one for each parameter matrix) is included in the final results that contains a history of the parameters during optimization. If False (default), then a history is not kept. tTrace: eTrace: If True, an array containing a history of the error function during optimization is included in the final results. If False (default), then a history is not kept. callback: verbose: Print extra information to standard out during the training procedure. args, kwargs: Arguments passed to optable.gradients. Returns: A dictionary containing the following keys: params: A numpy array containing the optimized parameters. error: Final value of the error function. iteration: The number of iterations performed. reason: A string describing the reason for termination. eTrace: A list containing the value of the error function at each iteration. Only returned if eTrace is True. pTrace: A list containing a copy of the parameters at each iteration. Only returned if pTrace is True. Refs: """ params = optable.parameters() # initial error error = optable.error(*args, **kwargs) errorPrev = error # intial temperature temp = tempInit # running correlation corrRun = 0.0 # probability of taking a negative step probs = np.ones_like(params) * 0.5 # weight pertibations dw = np.empty_like(params) dwPrev = np.empty_like(params) paramTrace = [params.copy()] tempTrace = [temp] errorTrace = [error] # termination reason reason = '' iteration = 0 if verbose: print('%d %6f' % (iteration, error)) if callback is not None: callback(optable, iteration, paramTrace, errorTrace) while True: # corr err dw action # + + + - # - - + + # - + - + # + - - - draw = np.random.random(params.shape) stepsNeg = np.where(draw < probs)[0] stepsPos = np.where(draw >= probs)[0] dwPrev[...] = dw dw[stepsNeg] = -stepSize dw[stepsPos] = stepSize params += dw errorPrev = error error = optable.error(*args, **kwargs) # increment iteration counter iteration += 1 if verbose: print('%d %6f' % (iteration, error)) if callback is not None: callback(optable, iteration, paramTrace, errorTrace) # keep parameter history if requested if pTrace: paramTrace.append(params.copy()) # keep temperature trace if requested if tTrace: tempTrace.append(temp) # keep error function history if requested if eTrace: errorTrace.append(error) # terminate if maximum iterations reached if iteration >= maxIter: reason = 'maxiter' break # terminate if desired accuracy reached if error < accuracy: reason = 'accuracy' break # terminate if desired precision reached if np.abs(error - errorPrev) < precision: reason = 'precision' break # terminate if the error function diverges if error > divergeThresh: reason = 'diverge' break # current change in error de = error - errorPrev # correlation metric corr = de * dw corrRun += (np.abs(de) * np.sum(np.abs(dw))) / params.size if (iteration % tempIter) == 0: # new temperature is average correlation # since the previous temperature update temp = corrRun / tempIter # reset running correlation corrRun = 0.0 if verbose: print('Cooling: %f' % temp) # probability of taking negative step # is drawn from the Boltzman Distribution probs[...] = 1.0 / util.capZero(1.0 + np.exp(-corr / temp)) if verbose: print(reason) # save result into a dictionary result = {} result['params'] = params result['error'] = error result['iteration'] = iteration result['reason'] = reason if pTrace: result['pTrace'] = paramTrace if tTrace: result['tTrace'] = tTrace if eTrace: result['eTrace'] = errorTrace return result
def __init__(self, s, sampRate=1.0, order=20, freqs=None, **kwargs): """Construct a new PSD using univariate autoregressive models. Args: s: Numpy array with shape (observations[,dimensions]) containing the signal data to use for generating the PSD estimate. Should be matrix-like. Rows are observations. Columns are optional and, if present, represent dimensions. sampRate: Sampling frequency of data. order: Order of the autoregressive models. freqs: Freqencies at which to estimate the signal power. If freqs is None (default) then estimate the power at all integer frequencies between 1Hz and Nyquist. If freqs is an integer then estimate the power at freqs equally spaced frequencies above DC to Nyquist. If freqs is a list or numpy array, then estimate power at the specified frequencies. """ s = util.colmat(s) nObs, nChan = s.shape # AR model order order = order if freqs is None: freqs = np.arange(1.0, np.floor(sampRate / 2.0)) elif isinstance(freqs, (int, )): freqs = np.linspace(0.0, sampRate / 2.0, freqs + 1)[1:] else: freqs = np.asarray(freqs) freqs = freqs nFreq = len(freqs) # period of sampling frequency dt = 1.0 / float(sampRate) # vector of model orders #orders = np.arange(1, order+1)[None,:] orders = np.arange(order, 0, -1)[None, :] weights = np.empty((order, nChan)) iVar = np.empty(nChan) # for each channel for chanI, chanS in enumerate(s.T): # train an AR model arFit = AutoRegression(chanS[None, ...], order=order, **kwargs) # residual values of AR model resid = arFit.resid(chanS[None, ...])[0] # model weights, ditch bias weights[:, chanI] = arFit.model.weights[:-1].ravel() # innovation variance iVar[chanI] = np.var(resid) # estimate spectrum, vectorized powersDenom = util.capZero( np.abs(1.0 - np.exp(-2.0j * np.pi * freqs[:, None] * orders * dt).dot(weights))**2) powers = (iVar[None, :] * dt) / powersDenom # scale to power density powers /= sampRate ## # for each channel ## for chanI, chanS in enumerate(s.T): ## # train an AR model ## arFit = AutoRegression((chanS,), order=order, *args, **kwargs) ## # predicted and residual values of AR model ## pred, resid = arFit.eval(chanS, returnResid=True) ## # model weights, ditch bias ## weights = arFit.model.weights[:-1,None] ## # innovation variance ## iVar = np.var(resid) ## # estimate spectrum, in a loop ## #for i, f in enumerate(freqs): ## # powers[i,chanI] = (iVar * dt) / (np.abs(1.0 - \ ## # np.sum(weights * np.exp(-2.0j*np.pi*f*orders) * dt)))**2) ## # estimate spectrum, vectorized ## powers[:,chanI] = ((iVar * dt) / ## (np.abs(1.0 - np.sum(weights * ## np.exp(-2.0j*np.pi*freqs[:,None]*orders*dt), axis=0))**2)) ## # estimate spectrum vectorized using sines and cosines instead of complex numbers ## #cs = np.sum(weights[:,None] * np.cos(2.0 * np.pi * freqsNorm*orders)), axis=0) ## #sn = np.sum(weights[:,None] * np.sin(2.0 * np.pi * freqsNorm*orders)), axis=0) ## #powers[:,chanI] = iVar / (sampRate * ((1.0 - cs)**2 + sn**2)) PSDBase.__init__(self, freqs, powers, sampRate)
def gradient(self, x, g, returnError=True): """Compute the gradient of the mean-squared error with respect to the network weights for each layer and given inputs and targets. Useful for optimization routines that make use of first-order gradients. Args: x: g: returnError: If True (default) then also return the mean-squared error. This can improve performance in some optimization routines by avoiding an additional forward pass. Returns: If returnError is True, then return a tuple containing the error followed by a 1d numpy array containing the gradient of the packed weights. If returnError is False, then only return the gradient. """ x = np.asarray(x) g = np.asarray(g) # packed views of the hidden and visible gradient matrices views = util.packedViews(self.layerDims, dtype=self.dtype) pg = views[0] hgs = views[1:-1] vg = views[-1] # forward pass z1 = util.bias(x) z1s = [z1] zPrimes = [] for hw, phi in zip(self.hws, self.transFunc): h = z1.dot(hw) z1 = util.bias(phi(h)) z1s.append(z1) zPrime = phi(h, 1) zPrimes.append(zPrime) v = z1.dot(self.vw) probs = util.softmax(v) # error components delta = util.colmat(probs - g) / probs.size # visible layer gradient vg[...] = z1.T.dot(delta) vg += self.penaltyGradient(-1) # backward pass for hidden layers w = self.vw for l in range(self.nHLayers-1, -1, -1): delta = delta.dot(w[:-1,:].T) * zPrimes[l] hgs[l][...] = z1s[l].T.dot(delta) hgs[l] += self.penaltyGradient(l) w = self.hws[l] if returnError: error = -np.mean(g*np.log(util.capZero(probs))) + self.penaltyError() return error, pg else: return pg