def getAllWeights(edgesList, startingPoint, crimeCounts):
#df = pd.read_csv(distanceFile, names = ["Edge", "Distance"])
#distanceDict = df.set_index('Edge')['Distance'].to_dict()
distanceDict = getDistanceDict(startingPoint, edgesList)
weights = getWeights(distanceDict, crimeCounts)
#writeDict(weights, weightsFile)
return weights
def main():
data = load_crime_data('felonies3.csv', 'new.csv')
lats, lons, intersectionsDict = getIntersections('nodeLocsWithTitles.csv')
crimeCounts = {}
for pt in data:
float_pt = (float(pt[0]), float(pt[1]))
results = getNearestFourPts(pt, lats, lons)
edges = getEdges(results)
distances = getStraightLineDist(float_pt, edges)
if len(distances) > 0:
shortestEdge = min(distances, key=distances.get)
if shortestEdge in crimeCounts:
crimeCounts[shortestEdge] += 1
else:
crimeCounts[shortestEdge] = 1
writeDict(crimeCounts, crimeCountsFile)
distanceDict = getDistanceDict(startingPoint, crimeCounts.keys())
writeDict(distanceDict, distanceFile)
weights = getWeights(distanceDict, crimeCounts)
writeDict(weights, weightsFile)
def moveBorders(data,options):
"""
move parameter-boundaries to save computing power
function borders=moveBorders(data, options)
this function evaluates the likelihood on a much sparser, equally spaced
grid definded by mbStepN and moves the borders in so that that
marginals below tol are taken away from the borders.
this is meant to save computing power by not evaluating the likelihood in
areas where it is practically 0 everywhere.
"""
borders = []
tol = options['maxBorderValue']
d = options['borders'].shape[0]
MBresult = {'X1D':[]}
''' move borders out
should our borders be to tight, e.g. the distribution does not go to zero
at the borders we move them out until this is the case.
TODO it was disabled in MATLAB version. What to do with it?
'''
''' move borders inwards '''
for idx in range(0,d):
if (len(options['mbStepN']) >= idx and options['mbStepN'][idx] >= 2
and options['borders'][idx,0] != options['borders'][idx,1]) :
MBresult['X1D'].append(np.linspace(options['borders'][idx,0], options['borders'][idx,1], options['mbStepN'][idx]))
else:
if (options['borders'][idx,0] != options['borders'][idx,1] and options['expType'] != 'equalAsymptote'):
warnings.warn('MoveBorders: You set only one evaluation for moving the borders!')
MBresult['X1D'].append( np.array([0.5*np.sum(options['borders'][idx])]))
MBresult['weight'] = getWeights(MBresult['X1D'])
#kwargs = {'alpha': None, 'beta':None , 'lambda': None,'gamma':None , 'varscale':None }
#fill_kwargs(kwargs,MBresult['X1D'])
MBresult['Posterior'] = likelihood(data, options, MBresult['X1D'])[0]
integral = sum(np.reshape(MBresult['Posterior'], -1) * np.reshape(MBresult['weight'], -1))
MBresult['Posterior'] /= integral
borders = np.zeros([d,2])
for idx in range(0,d):
(L1D,x,w) = marginalize(MBresult, np.array([idx]))
x1 = x[np.max([np.where(L1D*w >= tol)[0][0] - 1, 0])]
x2 = x[np.min([np.where(L1D*w >= tol)[0][-1]+1, len(x)-1])]
borders[idx,:] = [x1,x2]
return borders
def moveBorders(data,options):
"""
move parameter-boundaries to save computing power
function borders=moveBorders(data, options)
this function evaluates the likelihood on a much sparser, equally spaced
grid definded by mbStepN and moves the borders in so that that
marginals below tol are taken away from the borders.
this is meant to save computing power by not evaluating the likelihood in
areas where it is practically 0 everywhere.
"""
borders = []
tol = options.maxBorderValue
d = options.borders.shape[0]
MBresult = lambda: 0
''' move borders out
should our borders be to tight, e.g. the distribution does not go to zero
at the borders we move them out until this is the case.
TODO it was disabled in MATLAB version. What to do with it?
'''
''' move borders inwards '''
for idx in range(0,d):
if (len(options.mbStepN) >= idx and options.mbStepN[idx] >= 2
and options.borders[idx,0] != options.borders[idx,1]) :
MBresult.X1D[idx] = np.linspace(options.borders[idx,0], options.borders[idx,1], options.mbStepN[idx])
else:
if (options.borders[idx,0] != options.borders[idx,1] and options.expType != 'equalAsymptote'):
warnings.warn('MoveBorders: You set only one evaluation for moving the borders!')
MBresult.X1D[idx] = 0.5*np.sum(options.borders[idx])
MBresult.weight = getWeights(MBresult.X1D)
MBresult.Posterior = likelihood(data, options, MBresult.X1D) # TODO check!
integral = sum(MBresult.Posterior[:] * MBresult.weight[:])
MBresult.Posterior /= integral
borders = np.zeros([d,2])
for idx in range(0,d):
(L1D,x,w) = marginalize(MBresult, idx)
x1 = x[np.max(np.where(L1D*w >= tol)[0] - 1, 1)]
x2 = x[np.min(np.where(L1D*w >= tol)[-1]+1, len(x))]
borders[idx,:] = [x1,x2]
return borders
def moveBorders(data,options):
"""
move parameter-boundaries to save computing power
function borders=moveBorders(data, options)
this function evaluates the likelihood on a much sparser, equally spaced
grid definded by mbStepN and moves the borders in so that that
marginals below tol are taken away from the borders.
this is meant to save computing power by not evaluating the likelihood in
areas where it is practically 0 everywhere.
"""
borders = []
tol = options['maxBorderValue']
d = options['borders'].shape[0]
MBresult = {'X1D':[]}
''' move borders inwards '''
for idx in range(0,d):
if (len(options['mbStepN']) >= idx and options['mbStepN'][idx] >= 2
and options['borders'][idx,0] != options['borders'][idx,1]) :
MBresult['X1D'].append(np.linspace(options['borders'][idx,0], options['borders'][idx,1], options['mbStepN'][idx]))
else:
if (options['borders'][idx,0] != options['borders'][idx,1] and options['expType'] != 'equalAsymptote'):
warnings.warn('MoveBorders: You set only one evaluation for moving the borders!')
MBresult['X1D'].append( np.array([0.5*np.sum(options['borders'][idx])]))
MBresult['weight'] = getWeights(MBresult['X1D'])
#kwargs = {'alpha': None, 'beta':None , 'lambda': None,'gamma':None , 'varscale':None }
#fill_kwargs(kwargs,MBresult['X1D'])
MBresult['Posterior'] = likelihood(data, options, MBresult['X1D'])[0]
integral = sum(np.reshape(MBresult['Posterior'], -1) * np.reshape(MBresult['weight'], -1))
MBresult['Posterior'] /= integral
borders = np.zeros([d,2])
for idx in range(0,d):
(L1D,x,w) = marginalize(MBresult, np.array([idx]))
x1 = x[np.max([np.where(L1D*w >= tol)[0][0] - 1, 0])]
x2 = x[np.min([np.where(L1D*w >= tol)[0][-1]+1, len(x)-1])]
borders[idx,:] = [x1,x2]
return borders
def psignifitCore(data, options):
"""
This is the Core processing of psignifit, call the frontend psignifit!
function result=psignifitCore(data,options)
Data nx3 matrix with values [x, percCorrect, NTrials]
sigmoid should be a handle to a function, which accepts
X,parameters as inputs and gives back a value in [0,1]. ideally
parameters(1) should correspond to the threshold and parameters(2) to
the width (distance containing 95% of the function.
"""
d = len(options['borders'])
result = {'X1D': [], 'marginals': [], 'marginalsX': [], 'marginalsW': []}
'''Choose grid dynamically from data'''
if options['dynamicGrid']:
# get seed from linear regression with logit transform
Seed = getSeed(data,options)
# further optimize the logliklihood to obtain a good estimate of the MAP
if options['expType'] == 'YesNo':
calcSeed = lambda X: -l.logLikelihood(data, options, X[0], X[1], X[2], X[3], X[4])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = Seed)
elif options['expType'] == 'nAFC':
calcSeed = lambda X: -l.logLikelihood(data, options, X[0], X[1], X[2], 1/options['expN'], X[3])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = [Seed[0:2], Seed[4]])
Seed = [Seed[0:2], 1/options['expN'], Seed[3]] #ToDo check whether row or colum vector
result['X1D'] = gridSetting(data,options, Seed)
else: # for types which do not need a MAP estimate
if (options['gridSetType'] == 'priorlike' or options['gridSetType'] == 'STD'
or options['gridSetType'] == 'exp' or options['gridSetType'] == '4power'):
result['X1D'] = gridSetting(data,options)
else: # Use a linear grid
for idx in range(0,d):
# If there is an actual Interval
if options['borders'][idx, 0] < options['borders'][idx,1]:
result['X1D'].append(np.linspace(options['borders'][idx,0], options['borders'][idx,1],
num=options['stepN'][idx]))
# if parameter was fixed
else:
result['X1D'].append(np.array([options['borders'][idx,0]]))
'''Evaluate likelihood and form it into a posterior'''
(result['Posterior'], result['logPmax']) = l.likelihood(data, options, result['X1D'])
result['weight'] = getWeights(result['X1D'])
integral = np.sum(np.array(result['Posterior'][:])*np.array(result['weight'][:]))
result['Posterior'] = result['Posterior']/integral
result['integral'] = integral
'''Compute marginal distributions'''
for idx in range(0,d):
m, mX, mW = marginalize(result, np.array([idx]))
result['marginals'].append(m)
result['marginalsX'].append(mX)
result['marginalsW'].append(mW)
result['marginals'] = np.squeeze(result['marginals'])
result['marginalsX'] = np.squeeze(result['marginalsX'])
result['marginalsW'] = np.squeeze(result['marginalsW'])
'''Find point estimate'''
if (options['estimateType'] in ['MAP','MLE']):
# get MLE estimate
#start at most likely grid point
index = np.where(result['Posterior'] == np.max(result['Posterior'].ravel()))
Fit = np.zeros([d,1])
for idx in range(0,d):
Fit[idx] = result['X1D'][idx][index[idx]]
if options['expType'] == 'YesNo':
fun = lambda X, f: -l.logLikelihood(data, options, [X[0],X[1],X[2],X[3],X[4]])
x0 = deepcopy(Fit)
a = None
elif options['expType'] == 'nAFC':
#def func(X,f):
# return -l.logLikelihood(data,options, [X[0], X[1], X[2], f, X[3]])
#fun = func
fun = lambda X, f: -l.logLikelihood(data,options, [X[0], X[1], X[2], f, X[3]])
x0 = deepcopy(Fit[0:3]) # Fit[3] is excluded
x0 = np.append(x0,deepcopy(Fit[4]))
a = np.array([1/options['expN']])
elif options['expType'] == 'equalAsymptote':
fun = lambda X, f: -l.logLikelihood(data,options,[X[0], X[1], X[2], f, X[3]])
x0 = deepcopy(Fit[0:3])
x0 = np.append(x0,deepcopy(Fit[4]))
a = np.array([np.nan])
else:
raise ValueError('unknown expType')
if options['fastOptim']:
Fit = scipy.optimize.fmin(fun, x0, args = (a,), xtol=0, ftol = 0, maxiter = 100, maxfun=100)
warnings.warn('changed options for optimization')
else:
Fit = scipy.optimize.fmin(fun, x0, args = (a,), disp = True)
if options['expType'] == 'YesNo':
result['Fit'] = deepcopy(Fit)
elif options['expType'] == 'nAFC':
fit = deepcopy(Fit[0:3])
fit = np.append(fit, np.array([1/options['expN']]))
fit = np.append(fit, deepcopy(Fit[3]))
result['Fit'] = fit
elif options['expType'] =='equalAsymptote':
fit = deepcopy(Fit[0:3])
fit = np.append(fit, Fit[2])
fit = np.append(fit, Fit[3])
result['Fit'] = fit
else:
raise ValueError('unknown expType')
par_idx = np.where(np.isnan(options['fixedPars']) == False)
for idx in par_idx:
result['Fit'][idx] = options['fixedPars'][idx]
elif options['estimateType'] == 'mean':
# get mean estimate
Fit = np.zeros([d,1])
for idx in range[0:d]:
Fit[idx] = np.sum(result['marginals'][idx]*result['marginalsW'][idx]*result['marginalsX'][idx])
result['Fit'] = deepcopy(Fit)
Fit = np.empty(Fit.shape)
'''Include input into result'''
result['options'] = options # no copies here, because they are not changing
result['data'] = data
'''Compute confidence intervals'''
if ~options['fastOptim']:
result['conf_Intervals'] = getConfRegion(result)
return result
def psignifitCore(data, options):
"""
This is the Core processing of psignifit, call the frontend psignifit!
function result=psignifitCore(data,options)
Data nx3 matrix with values [x, percCorrect, NTrials]
sigmoid should be a handle to a function, which accepts
X,parameters as inputs and gives back a value in [0,1]. ideally
parameters(1) should correspond to the threshold and parameters(2) to
the width (distance containing 95% of the function.
"""
d = len(options['borders'])
result = {'X1D': [], 'marginals': [], 'marginalsX': [], 'marginalsW': []}
'''Choose grid dynamically from data'''
if options['dynamicGrid']:
# get seed from linear regression with logit transform
Seed = getSeed(data,options)
# further optimize the logliklihood to obtain a good estimate of the MAP
if options['expType'] == 'YesNo':
calcSeed = lambda X: -_l.logLikelihood(data, options, X[0], X[1], X[2], X[3], X[4])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = Seed)
elif options['expType'] == 'nAFC':
calcSeed = lambda X: -_l.logLikelihood(data, options, X[0], X[1], X[2], 1/options['expN'], X[3])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = [Seed[0:2], Seed[4]])
Seed = [Seed[0:2], 1/options['expN'], Seed[3]] #ToDo check whether row or colum vector
result['X1D'] = gridSetting(data,options, Seed)
else: # for types which do not need a MAP estimate
if (options['gridSetType'] == 'priorlike' or options['gridSetType'] == 'STD'
or options['gridSetType'] == 'exp' or options['gridSetType'] == '4power'):
result['X1D'] = gridSetting(data,options)
else: # Use a linear grid
for idx in range(0,d):
# If there is an actual Interval
if options['borders'][idx, 0] < options['borders'][idx,1]:
result['X1D'].append(np.linspace(options['borders'][idx,0], options['borders'][idx,1],
num=options['stepN'][idx]))
# if parameter was fixed
else:
result['X1D'].append(np.array([options['borders'][idx,0]]))
'''Evaluate likelihood and form it into a posterior'''
(result['Posterior'], result['logPmax']) = _l.likelihood(data, options, result['X1D'])
result['weight'] = getWeights(result['X1D'])
integral = np.sum(np.array(result['Posterior'][:])*np.array(result['weight'][:]))
result['Posterior'] = result['Posterior']/integral
result['integral'] = integral
'''Compute marginal distributions'''
for idx in range(0,d):
m, mX, mW = marginalize(result, np.array([idx]))
result['marginals'].append(m)
result['marginalsX'].append(mX)
result['marginalsW'].append(mW)
result['marginals'] = np.squeeze(result['marginals'])
result['marginalsX'] = np.squeeze(result['marginalsX'])
result['marginalsW'] = np.squeeze(result['marginalsW'])
'''Find point estimate'''
if (options['estimateType'] in ['MAP','MLE']):
# get MLE estimate
#start at most likely grid point
index = np.where(result['Posterior'] == np.max(result['Posterior'].ravel()))
Fit = np.zeros([d,1])
for idx in range(0,d):
Fit[idx] = result['X1D'][idx][index[idx]]
if options['expType'] == 'YesNo':
fun = lambda X, f: -_l.logLikelihood(data, options, [X[0],X[1],X[2],X[3],X[4]])
x0 = _deepcopy(Fit)
a = None
elif options['expType'] == 'nAFC':
#def func(X,f):
# return -_l.logLikelihood(data,options, [X[0], X[1], X[2], f, X[3]])
#fun = func
fun = lambda X, f: -_l.logLikelihood(data,options, [X[0], X[1], X[2], f, X[3]])
x0 = _deepcopy(Fit[0:3]) # Fit[3] is excluded
x0 = np.append(x0,_deepcopy(Fit[4]))
a = np.array([1/options['expN']])
elif options['expType'] == 'equalAsymptote':
fun = lambda X, f: -_l.logLikelihood(data,options,[X[0], X[1], X[2], f, X[3]])
x0 = _deepcopy(Fit[0:3])
x0 = np.append(x0,_deepcopy(Fit[4]))
a = np.array([np.nan])
else:
raise ValueError('unknown expType')
if options['fastOptim']:
Fit = scipy.optimize.fmin(fun, x0, args = (a,), xtol=0, ftol = 0, maxiter = 100, maxfun=100)
warnings.warn('changed options for optimization')
else:
Fit = scipy.optimize.fmin(fun, x0, args = (a,), disp = False)
if options['expType'] == 'YesNo':
result['Fit'] = _deepcopy(Fit)
elif options['expType'] == 'nAFC':
fit = _deepcopy(Fit[0:3])
fit = np.append(fit, np.array([1/options['expN']]))
fit = np.append(fit, _deepcopy(Fit[3]))
result['Fit'] = fit
elif options['expType'] =='equalAsymptote':
fit = _deepcopy(Fit[0:3])
fit = np.append(fit, Fit[2])
fit = np.append(fit, Fit[3])
result['Fit'] = fit
else:
raise ValueError('unknown expType')
par_idx = np.where(np.isnan(options['fixedPars']) == False)
for idx in par_idx[0]:
result['Fit'][idx] = options['fixedPars'][idx]
elif options['estimateType'] == 'mean':
# get mean estimate
Fit = np.zeros([d,1])
for idx in range[0:d]:
Fit[idx] = np.sum(result['marginals'][idx]*result['marginalsW'][idx]*result['marginalsX'][idx])
result['Fit'] = _deepcopy(Fit)
Fit = np.empty(Fit.shape)
'''Include input into result'''
result['options'] = options # no copies here, because they are not changing
result['data'] = data
'''Compute confidence intervals'''
if ~options['fastOptim']:
result['conf_Intervals'] = getConfRegion(result)
return result
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from numpy import *
from getWeights import getWeights
t = getWeights()
def equilibrium(rho, u):
(d, nx, ny, nz) = u.shape
cu = zeros((3, 3, 3, nx, ny, nz))
for i in range(3):
for j in range(3):
for k in range(3):
cu[i, j, k] = (i-1)*u[0] + (j-1)*u[1] + (k-1)*u[2]
usqr = 3./2.*(u[0]**2 + u[1]**2 + u[2]**2)
feq = zeros((3, 3, 3, nx, ny, nz))
for i in range(3):
for j in range(3):
for k in range(3):
feq[i, j, k] = rho*t[i,j,k]*(1. + 3.*cu[i,j,k] + 4.5*cu[i,j,k]**2 - usqr)
return feq
def psignifitCore(data, options):
d = len(options.borders)
'''Choose grid dynamically from data'''
if options.dynamicGrid:
# get seed from linear regression with logit transform
Seed = getSeed(data,options)
# further optimize the logliklihood to obtain a good estimate of the MAP
if options.expType == 'YesNo':
calcSeed = lambda X: -logLikelihood(data, options, X[0], X[1], X[2], X[3], X[4])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = Seed)
elif options.expType == 'nAFC':
calcSeed = lambda X: -logLikelihood(data, options, X[0], X[1], X[2], 1/options.expN, X[3])
Seed = scipy.optimize.fmin(func=calcSeed, x0 = [Seed[0:2], Seed[4]])
Seed = [Seed[0:2], 1/options.expN, Seed[3]] #ToDo check whether row or colum vector
result.X1D = gridSetting(data,options, Seed)
else: # for types which do not need a MAP estimate
if (options.gridSetType == 'priorlike' or options.gridSetType == 'STD'
or options.gridSetType == 'exp' or options.gridSetType == '4power'):
result.X1D = gridSetting(data,options)
else: # Use a linear grid
for idx in range[0:d]:
# If there is an actual Interval
if options.borders[idx, 0] < options.borders[idx,1]:
#result.X1D[id] = linspace(bla)
result.X1D[idx] = np.linspace(options.borders[idx,1], options.borders[idx,2],
num=options.stepN[idx])
# if parameter was fixed
else:
result.X1D[idx] = options.borders[idx,0]
'''Evaluate likelihood and form it into a posterior'''
[result.Posterior, result.logPmax] = likelihood(data, options, result.X1D[:])
result.weight = getWeights(result.X1D)
integral = np.sum(np.array(result.Posterior[:])*np.array(result.weight[:]))
result.Posterior = result.Posterior/integral
result.integral = integral
'''Compute marginal distributions'''
for idx in range[0,d]:
[result.marginals[idx], result.marginalsX[idx], result.marginalsW[idx]] = marginalize(result, id)
'''Find point estimate'''
if (options.estimateType == 'MAP' or options.estimateType == 'MLE'):
# get MLE estimate
#start at most likely grid point
(_, idx) = max(result.Posterior[:])
#index = cell(d,1) #ToDo
index = np.unravel_index(idx, result.Posterior.shape)
Fit = np.zeros([d,1])
for idx in range[0:d]:
Fit[idx] = result.X1D[idx](index[idx]) #ToDo the round braces?
if options.expType == 'YesNo':
fun = lambda:X -logLikelihood(data, options, X[0], X[1], X[2], X[3], X[4])
x0 = copy.deepcopy(Fit)
elif options.expType == 'nAFC':
fun = lambda:X -logLikelihood(data,options, X[0], X[1], X[2], 1/options.expN, x[3])
x0 = copy.deepcopy(Fit[0:2])
x0 = np.append(x0,Fit[4])
x0 = np.transpose(x0)
elif options.expType == 'equalAsymptote':
fun = lambda:X -logLikelihood(data,options, X[0], X[1], X[2], np.nan, X[3])
x0 = copy.deepcopy(Fit[0:2])
x0 = np.append(x0,Fit[4])
x0 = np.transpose(x0)
else:
raise ValueError('unknown expType')
if options.fastOptim:
#todo check if dictionary works
optimiseOptions = {'xtol':0, 'ftol':0, 'maxiter': 100, 'maxfun': 100}
# or maybe optimiseOptions = (0,0,100,100)
warnings.warn('changed options for optimization')
else:
optimiseOptions = {'disp':False}
#or maybe optimiseOptions = (_,_,_,_,_,False)
Fit =scipy.optimize.fmin(func=fun, x0, optimiseOptions) #ToDo check if that works this way
if options.expType == 'YesNo':
result.Fit = copy.deepcopy(Fit)
elif options.expType == 'nAFC': #TODO is this row or column vectors?
result.Fit = np.transpose([Fit[0:2], 1/options.expN, Fit[3]])
elif options.expType =='equalAsymptote':
result.Fit = np.transpose([Fit[0:2], Fit[2], Fit[3]])
else:
raise ValueError('unknown expType')
#TODO result.Fit[~np.isnan(options.fixedPars)] = options.fixedPars[~np.isnan(options.fixedPars)]
elif options.estimateType == 'mean':
# get mean estimate
Fit = np.zeros([d,1])
for idx in range[0:d]:
Fit[idx] = np.sum(np.array(result.marginals[idx])*np.array(result.marginalsW[idx])*np.array(result.marginalsX[idx]))
result.Fit = copy.deepcopy(Fit)
clear(Fit)
'''Include input into result'''
result.options = options # no copies here, because they are not changing
result.data = data
'''Compute confidence intervals'''
if ~options.fastOptim:
result.conf_Intervals = getConfRegion(result)
return result
