Example #1
0
def gradLinearSP(
        Y,  # Observed response
        S,  # Stimulus
        P  # Parameters
):

    # Extract parameters
    a1, v1, a2, v2, d = P

    # Model nonlinearities
    f1, f2 = logistic, softPlus

    # Derivative of model nonlinearities and cost function
    df1, df2, dfe = dlog, dSP, dllike

    ndim = v1.ndim

    x1 = a1 + tdot(v1, S, 2 * (list(range(ndim)), ))
    r1 = f1(x1)
    dr1 = df1(x1)
    x2 = a2 + (r1 * v2).sum()
    r2 = f2(x2)
    dr2 = df2(x2)

    dy = d * dfe(Y, d * r2)
    dd = dy * r2 / d

    da2 = dy * dr2
    dv2 = dy * dr2 * r1

    da1 = dy * dr2 * (dr1 * v2).sum()
    dv1 = dy * dr2 * tdot(dr1 * S, v2,
                          (list(range(-ndim, 0)), list(range(ndim))))

    return Params([da1, dv1, da2, dv2, dd])
Example #2
0
def gradLog2(
        Y,  # Observed response
        S,  # Stimulus
        P  # Parameters
):

    # Extract parameters
    a1, v1, J1, a2, v2, d = P

    # Model nonlinearities
    f1, f2 = logistic, logistic

    # Derivative of model nonlinearities and cost function
    df1, df2, dfe = dlog, dlog, dllike

    ndim = v1.ndim

    x1 = a1 + tdot(v1, S, 2 *
                   (list(range(ndim)), )) + (tdot(J1, S, 2 *
                                                  (list(range(ndim)), )) *
                                             S).sum(tuple(range(ndim)))
    r1 = f1(x1)
    dr1 = df1(x1)
    x2 = a2 + (r1 * v2).sum()
    r2 = f2(x2)
    dr2 = df2(x2)

    dy = d * dfe(Y, d * r2)
    dd = dy * r2 / d

    da2 = dy * dr2
    dv2 = dy * dr2 * r1

    da1 = dy * dr2 * (dr1 * v2).sum()
    dv1 = dy * dr2 * tdot(dr1 * S, v2,
                          (list(range(-ndim, 0)), list(range(ndim))))
    dJ1 = dy * dr2 * tdot(
        dr1 * S * S.reshape(S.shape[:ndim] + ndim *
                            (1, ) + S.shape[ndim:]), v2,
        (list(range(-ndim, 0)), list(range(ndim))))

    return Params([da1, dv1, dJ1, da2, dv2, dd])
def respLog2(S, P):
    # Inputs:
    #   S - A stimulus
    #   P - Model parameters
    # Output:
    #   r - Response of model for given stimulus

    # Extract parameters
    a1, v1, J, a2, v2, d = P

    # Model nonlinearities
    f1, f2 = logistic, logistic

    ndim = v1.ndim

    # Calculate first layer responses
    r1 = f1(a1 + tdot(v1, S, 2 * (range(ndim), )) +
            (tdot(J, S, 2 * (range(ndim), )) * S).sum(tuple(range(ndim))))

    # Calculate second layer responses
    r2 = f2(a2 + (r1 * v2).sum())

    return d * r2
Example #4
0
def respLinearSP(S, P):
    # Inputs:
    #   S - A stimulus
    #   P - Parameters
    # Output:
    #   r - Response of model for give stimulus

    # Extract parameters
    a1, v1, a2, v2, d = P

    # Model nonlinearities
    f1, f2 = logistic, softPlus

    ndim = v1.ndim

    # Calculate first layer responses
    r1 = f1(a1 + tdot(v1, S, 2 * (list(range(ndim)), )))

    # Calculate second layer responses
    r2 = f2(a2 + (r1 * v2).sum())

    return d * r2
Example #5
0
def reshape_coordinates(coordinates):
    """ reshape coordinates to align with coefficient array """
    return tdot(coordinates, tdot(coordinates, coordinates, axes=0), axes=0)
Example #6
0
def correlation_fct(component):
    X = map(lambda b: peps.make_double_layer(b, o=sigma[component]), a)
    mx = m[component]
    
    tmp = tdot(env.c4[lut[0,-1,-1]], env.t4[lut[0,-1,0]], [1,0])
    tmp = tdot(tmp, env.c1[lut[0,-1,1]], [2,0])
    tmp = tdot(tmp, env.t3[lut[0,0,-1]], [0,2])
    tmp2 = tdot(tmp, X[0], [[3,0],[2,3]])
    uX = tdot(tmp2, env.t1[lut[0,0,1]], [[0,2],[0,1]]).flatten()
    tmp2 = tdot(tmp, A[0], [[3,0],[2,3]])
    uA = tdot(tmp2, env.t1[lut[0,0,1]], [[0,2],[0,1]]).flatten()
    
    vX = [None]*n
    vA = [None]*n
    for j in xrange(n):
        tmp = tdot(env.c3[lut[j,1,-1]], env.t2[lut[j,1,0]], [0,2])
        tmp = tdot(tmp, env.c2[lut[j,1,1]], [1,1])
        tmp = tdot(tmp, env.t3[lut[j,0,-1]], [0,0])
        tmp2 = tdot(tmp, X[j], [[0,2],[1,2]])
        vX[j] = tdot(tmp2, env.t1[lut[j,0,1]], [[0,2],[2,1]]).flatten()
        tmp2 = tdot(tmp, A[j], [[0,2],[1,2]])
        vA[j] = tdot(tmp2, env.t1[lut[j,0,1]], [[0,2],[2,1]]).flatten()
    
    M = [None]*n
    for j in xrange(n):
        tmp = tdot(env.t3[lut[j,0,-1]], A[j], [1,2])
        tmp = tdot(tmp, env.t1[lut[j,0,1]], [2,1])
        M[j] = tmp.transpose([1,3,4,0,2,5]).reshape(len(vX[lut[j,-1,0]]), len(vX[lut[j,1,0]]))
    
    res = np.empty(Lmax+1)
    j = lut[0,1,0]
    for L in xrange(Lmax+1):
        print L
        res[L] = np.dot(uX, vX[j]) / np.dot(uA, vA[j]) - mx[0]*mx[j]
        
        if L < Lmax:
            uX = np.dot(uX, M[j])
            uA = np.dot(uA, M[j])
        
        j = nns[0][j]
    
    return res
def gradDescent(prefix,
                spikes,
                stim,
                jack,
                fsize,
                extrapSteps=10,
                pixelNorm=True,
                filepath=None,
                model='softplus',
                maxIts=None,
                maxHours=None,
                perm=True,
                overwrite=False,
                Njack=4,
                start='rand_rand',
                nlags=1,
                splits=None,
                LRType='DecayRate',
                LRParams={}):

    assert isinstance(prefix, StringType)
    print 'Prefix ' + prefix

    Njack = IntCheck(Njack)
    jack = IntCheck(jack)
    assert jack > 0 and jack <= Njack
    print 'Jack ', jack, 'out of ', Njack

    FSIZE = stim.shape[:-1] + (nlags, )
    print 'Full frame size ', FSIZE
    assert isinstance(fsize, tuple)
    if len(fsize) < len(FSIZE):
        fsize = fsize + (len(FSIZE) - len(fsize)) * (1, )
    print 'Patch frame size ', fsize
    gsize = tuple([F - f + 1 for F, f in zip(FSIZE, fsize)])
    NGRID = prod(gsize)
    ng = len(gsize)
    print 'Grid size ', gsize

    assert model in [
        'softplus', 'linearSoftplus', 'logistic', 'linearLogistic'
    ]
    print 'Model ', model
    if model == 'softplus':
        resp = respSP
        grad = gradSP
        cost = llike
        AlgTag = '_QuadraticSoftPlus'
    elif model == 'linearSoftplus':
        resp = respLinearSP
        grad = gradLinearSP
        cost = llike
        AlgTag = '_LinearSoftPlus'
    elif model == 'logistic':
        resp = respLog2
        grad = gradLog2
        cost = llike
        AlgTag = '_QuadraticLogistic'
    elif model == 'linearLogistic':
        resp = respLinearLog2
        grad = gradLinearLog2
        cost = llike
        AlgTag = '_LinearLogistic'

    extrapSteps = IntCheck(extrapSteps)
    print 'Steps used to estimate error slipe ', extrapSteps

    if pixelNorm:
        print 'Normalizing by pixel statistics'
    else:
        print 'Normalizing by global statistics'

    if filepath in [None, '', './']:
        filepath = ''
        print 'Saving output in current directory'
    else:
        assert isinstance(filepath, StringType)
        filepath = expanduser(filepath)
        assert isdir(filepath)
        print 'Saving output files to ', filepath

    assert isinstance(start, list) or isinstance(start, tuple) or isinstance(
        start, str) or isinstance(start, Params)
    if isinstance(start, str):
        vstart, bstart = start.split('_')
        assert bstart in ['rand', 'sta', 'uniform']
        assert vstart in ['rand', 'stim', 'sta']
        if bstart == 'rand':
            print 'Initializing second layer randomly'
        elif bstart == 'unifrom':
            print 'Initializing second layer uniformly'
        elif bstart == 'sta':
            print 'Initializing second layer using STA'
        if vstart == 'rand':
            print 'Initializing first layer randomly'
        elif vstart == 'stim':
            print 'Initializing first layer using random stimuli'
        elif vstart == 'sta':
            print 'Initializing first layer using STA'
    else:
        print 'Starting parameters given'

    if maxIts is None:
        maxIts = inf
        print 'No limit on iterations'
    else:
        maxIts = IntCheck(maxIts)
        print 'Max iterations ', maxIts

    genesis = time()
    if maxHours is None:
        print 'No limit on runtime'
        eschaton = inf
    else:
        assert isinstance(maxHours, NumType)
        eschaton = genesis + maxHours * 3600
        print 'Max hours ', maxHours

    if isinstance(perm, ndarray):
        print 'Permuting data by given array before division'
    else:
        if perm:
            print 'Randomly divided data sets'
        else:
            print 'Contiguous data sets'

    # Get stimulus shape and size
    Ntrials = stim.shape[-1] - nlags + 1

    # Convert spikes from int
    Y = spikes.astype(float)
    del spikes

    # Drop spikes before first full stimulus
    Y = Y[nlags - 1:]

    # Check that stimulus and responses have the same size
    assert Y.size == Ntrials

    # Size of first layer input
    npix = prod(fsize)

    # Convert stimulus to zero mean and unit stdev
    stim = normStim(stim, pixelNorm)[0]

    Nvalid = Ntrials / Njack
    Ntrials -= Nvalid

    # Randomly permute stimulus and spikes
    if isinstance(perm, ndarray):
        assert perm.size == Ntrials + Nvalid
        p = perm[nlags - 1:]
    elif perm:
        RS = RandomState(0)
        p = RS.permutation(Ntrials + Nvalid - nlags + 1)

    # Split data into training and test sets
    validslice = slice((jack - 1) * Nvalid, jack * Nvalid)
    pv = p[validslice]
    pr = delete(p, validslice)

    # Remove samples that span recordings from training and validation sets
    if splits is not None:
        invalid = array([arange(sp - nlags + 1, sp)
                         for sp in splits]).flatten()
        pv = array([pp for pp in pv if pp not in invalid])
        pr = array([pp for pp in pr if pp not in invalid])

    # Extract stimulus at grid locations
    S = gridStim(stim, fsize, nlags)

    # Divide responses into training and validation sets
    YR = Y[pr]
    YV = Y[pv]

    spikesmean = YR.mean()
    Nspikes = YR.sum()

    # Calcualte error of mean model
    errTrain0 = cost(YR, spikesmean)
    errValid0 = cost(YV, spikesmean)

    stdout.write('Training: {0} frames, {1} spikes\n'.format(Ntrials, Nspikes))
    stdout.write('Validation: {0} frames, {1} spikes\n'.format(
        Nvalid, YV.sum()))
    stdout.flush()

    # Create filenames
    trainBestName = filepath + prefix + AlgTag + '_train_%u.dat' % (jack, )
    validBestName = filepath + prefix + AlgTag + '_valid_%u.dat' % (jack, )
    statusName = filepath + prefix + AlgTag + '_%u.temp' % (jack, )
    errTrainName = filepath + prefix + AlgTag + '_errTrain_%u.dat' % (jack, )
    errValidName = filepath + prefix + AlgTag + '_errValid_%u.dat' % (jack, )

    # Calculate shapes of parameters
    if model in ['softplus', 'logistic']:
        shapes = [(1, ), fsize, 2 * fsize, (1, ), gsize, (1, )]
    else:
        shapes = [(1, ), fsize, (1, ), gsize, (1, )]

    # Check to see if previous run exists
    if exists(statusName):

        stdout.write('Loading previous run\n')
        stdout.flush()
        with open(statusName, 'r') as f:
            its = fromfile(f, count=1, dtype=int)
            errValidMin = fromfile(f, count=1)
        P = Params(trainBestName, shapes)
        PV = Params(validBestName, shapes)
        if its > maxIts:
            maxIts += its
        with open(errValidName, 'r') as f:
            errValidHist = list(fromfile(f))
        if len(errValidHist) > extrapSteps:
            errValidHist = errValidHist[-extrapSteps:]
        with open(errTrainName, 'r') as f:
            errTrain = fromfile(f)[-1]
    else:
        if exists(trainBestName) and not overwrite:
            print 'Output files exist'
            return
        else:

            if isinstance(start, Params):
                # If start is a Params object with the right number of parameters, copy it
                if len(start) == len(shapes):
                    P = start.copy()
                # If start is a linear model, create J from random stimulus combinations
                else:
                    # Linear models have one less parameter
                    assert len(start) == len(shapes) - 1

                    # Create J from randomly weighted stimulus patches
                    RS = RandomState()
                    J = zeros(2 * start[1].shape)
                    for j in pr:
                        r = RS.randn(NGRID).reshape(gsize)
                        J += tdot(S[j, ...], S[j, ...] * r,
                                  (range(-ng, 0), range(-ng, 0)))

                    # Initialize J so that it starts small relative to the linear term
                    J *= 0.00001 * norm(start[1]) / norm(J)

                    # Insert J into P
                    P = start.copy().getParams()
                    P.insert(2, J)
                    P = Params(P)

            # If start is a list/tuple, reshape values and convert to Params
            elif isinstance(start, list) or isinstance(start, tuple):
                assert len(start) == len(shapes)
                for s, p in zip(shapes, start):
                    p.shape = s
                P = Params(start, shapes)
            else:
                # Initialize first layer randomly
                if vstart == 'rand':
                    RS = RandomState()
                    v = RS.randn(npix).reshape(fsize)
                    v /= norm(v)
                    if model in ['softplus', 'logistic']:
                        J = RS.randn(npix, npix)
                        J = J + J.T
                        J /= norm(J)
                        J.shape = 2 * fsize

                # Initialize first layer with random stimuli from training set
                elif vstart == 'stim':
                    RS = RandomState()
                    v = zeros(fsize)
                    for j in pr:
                        r = RS.randn(NGRID).reshape(gsize)
                        v += tdot(S[j, ...], r, (range(-ng, 0), range(ng)))
                    v /= norm(v)
                    if model in ['softplus', 'logistic']:
                        J = zeros(2 * fsize)
                        for j in pr:
                            r = RS.randn(NGRID).reshape(gsize)
                            J += tdot(S[j, ...], S[j, ...] * r,
                                      (range(-ng, 0), range(-ng, 0)))
                        J /= norm(J)

                # Initialize first layer using STA/STC
                elif vstart == 'sta':
                    ES = zeros(fsize)
                    ESY = zeros(fsize)
                    if model in ['softplus', 'logistic']:
                        ESS = zeros(fsize * 2)
                        ESSY = zeros(fsize * 2)
                    for pp in pr:
                        SS = S[pp, ...].sum(-1).sum(-1).sum(-1).sum(-1)
                        ES += SS
                        ESY += SS * Y[pp]
                        if model in ['softplus', 'logistic']:
                            SSS = SS * SS.reshape(SS.shape + 4 * (1, ))
                            ESS += SSS
                            ESSY += SSS * Y[pp]
                    ES /= pr.size
                    ESY /= YR.sum()
                    v = ESY - ES
                    v /= norm(v)
                    if model in ['softplus', 'logistic']:
                        ESS /= pr.size
                        ESSY /= YR.sum()
                        J = (ESSY - ESY * ESY.reshape(ESY.shape + 4 * (1, ))
                             ) - (ESS - ES * ES.reshape(ES.shape + 4 * (1, )))
                        J /= norm(J)

                else:
                    raise Exception('Unsupported initialization')

                # Scale v and J.
                v *= 0.1
                if model in ['softplus', 'logistic']:
                    J *= 0.1

                # Initialize second layer randomly
                if bstart == 'rand':
                    RS = RandomState()
                    v2 = RS.randn(*gsize)
                    v2 /= norm(v2)
                    v2 *= 0.1

                # Initialize second layer uniformly
                elif bstart == 'uniform':
                    v2 = ones(gsize)
                    v2 /= norm(v2)
                    v2 *= 0.1

                # Intialize second layer using STA
                elif bstart == 'sta':
                    ES = zeros(gsize)
                    ESY = zeros(gsize)
                    for pp in pr:
                        xv = tdot(S[pp, ...], v, 2 * (range(4), ))
                        xJ = (tdot(J, S[pp, ...], 2 * (range(4), )) *
                              S[pp, ...]).sum(0).sum(0).sum(0).sum(0)
                        r1 = logistic(xv + xJ)
                        ES += r1
                        ESY += r1 * Y[pp]
                    ES /= pr.size
                    ESY /= YR.sum()
                    v2 = ESY - ES
                    v2 /= norm(v2)
                    v2 *= 0.1

                # Combine intialized parameters into a Params object
                if model in ['softplus', 'logistic']:
                    P = Params([zeros(1), v, J, zeros(1), v2, ones(1)])
                else:
                    P = Params([zeros(1), v, zeros(1), v2, ones(1)])

                # Set d to match mean firing rate on training set
                R = array([resp(S[j, ...], P) for j in pr])
                rmean = R.mean()
                P[-1][:] = spikesmean / rmean
                P = Params(P)

            # Calculate initial error
            R = Resp(S, P, resp)
            errTrain = cost(YR, R[pr]) / errTrain0
            errValid = cost(YV, R[pv]) / errValid0

            # Save initial errors
            with open(errTrainName, 'w') as f:
                errTrain.tofile(f)

            with open(errValidName, 'w') as f:
                errValid.tofile(f)

            errValidHist = [errValid]

            # Save initial values as best so far
            errValidMin = errValid.copy()
            PV = P.copy()

            # Save initial parameters to parameter files
            P.tofile(trainBestName)
            PV.tofile(validBestName)

            # Keep track of the number of iterations
            its = 0

    stdout.write('Beginning optimization\n')
    stdout.flush()
    if model in ['softplus', 'logistic']:
        Pname = ['a1', 'v1', 'J1', 'a2', 'v2', 'd']
    else:
        Pname = ['a1', 'v1', 'a2', 'v2', 'd']

    # Start slope as negative
    slope = -1.

    # Print status
    print '%u Values:' % (its, ),
    for nam, p in zip(Pname, P):
        if p.size == 1:
            print ' %s %.3e' % (nam, p),
        else:
            print ' %s %.3e' % (nam, norm(p)),
    print ''

    errTrainLast = errTrain.copy()
    PLast = P.copy()

    # Select and initialize learning rate rule
    if LRType == 'DecayRate':
        LR = DecayRate(errTrain, its, **LRParams)
    elif LRType == 'BoldDriver':
        LR = BoldDriver(errTrain, **LRParams)
    else:
        LR = LearningRate(errTrain, **LRParams)

    # Run until slope of validation error becomes positive, time runs out,
    # maximum iterations is reached, or learning rate falls to eps
    while ((slope < 0) or (its < extrapSteps)) and (time() < eschaton) and (
            its < maxIts) and (LR.lrate > eps):

        # For each training example, calculate gradient and update parameters
        for j in pr:
            y = Y[j]
            s = S[j, ...]
            P += grad(y, s, P) * LR.lrate

        # Increment to next iteration
        its += 1

        # Calculate current training error and update learning rule
        R = Resp(S, P, resp)
        errTrain = cost(YR, R[pr]) / errTrain0
        LR.update(errTrain)

        # If training error decreases
        if errTrain < errTrainLast:
            # Save new copies of last error and parameters
            errTrainLast = errTrain.copy()
            PLast = P.copy()

            # Calculate validation error
            errValid = cost(YV, R[pv]) / errValid0
            errValidHist.append(errValid)

            # Calculate slope of the validation error
            if len(errValidHist) > extrapSteps:
                errValidHist = errValidHist[-extrapSteps:]
            x = ones((2, len(errValidHist)))
            x[1, :] = arange(len(errValidHist))
            slope = dot(inv(dot(x, x.T)), dot(x, array(errValidHist)))[1]

            # Save current parameters
            P.tofile(trainBestName)

            # Append errors to history files
            with open(errTrainName, 'a') as f:
                errTrain.tofile(f)

            with open(errValidName, 'a') as f:
                errValid.tofile(f)

            # If validation error has reached new minimum
            if errValid < errValidMin:

                # Update best value
                errValidMin = errValid

                # Copy parameters and save to parameter file
                PV = P.copy()
                PV.tofile(validBestName)

                # Output note of improvement
                errDown = errValidMin - errValid
                print '%u: New validation minimum %.5g, down %.3g' % (
                    its, errValidMin, errDown)

            # Save current status
            with open(statusName, 'w') as f:
                array(its).tofile(f)
                errValidMin.tofile(f)

            # Print status
            print '%u Values:' % (its, ),
            for nam, p in zip(Pname, P):
                if p.size == 1:
                    print ' %s %.3e' % (nam, p),
                else:
                    print ' %s %.3e' % (nam, norm(p)),
            print ''
            print 'Slope %.3e' % (slope, )
        else:
            print 'Training error increased: learning rate too high'
            print 'New learning rate %.3e' % LR.lrate
            its -= 1
            P = PLast.copy()

    # If converged, delete status file
    if time() < eschaton and its < maxIts:
        remove(statusName)

    # Note that program has terminated successfully
    stdout.write('Time elapsed {0:.3f} hours\n'.format(
        (time() - genesis) / 3600.))
    stdout.write('Finished\n')
    stdout.flush()