def outputDir(**kwargs):
ss = paramString(**kwargs)
ss = ss + '_' + timestamp_()
oDir = os.path.join(kwargs['oDir'],ss)
if not os.path.exists(oDir):
os.makedirs(oDir)
return oDir
def compareMargJointLocal(rootDir=None, scppFile=None, nItr=10000, n_expectedSurplus_samples=10000):
if rootDir == None:
raise ValueError("Must specify output directory - oDir")
if scppFile == None:
raise ValueError("Must specify scpp file (pkl)")
rootDir = os.path.realpath(rootDir)
oDir = os.path.join(rootDir, "compareMargLocal_{0}".format(timestamp_()))
if not os.path.exists(oDir):
os.makedirs(oDir)
for itr in xrange(nItr):
pass
def localSearchDominance(nbids = 100, n_samples = 1000, rootDir = ".",
scppFile = None, vfile = None, lfile = None,
njs = 1000, nms = 1000):
scppFile = os.path.realpath(scppFile)
with open(scppFile,'r') as f:
scpp = pickle.load(f)
m = scpp.m()
bundles = listBundles(m)
rootDir = os.path.realpath(rootDir)
oDir = os.path.join(rootDir,timestamp_())
if not os.path.exists(oDir):
os.makedirs(oDir)
table = [["parameter", "value"]]
table.append(["nbids", nbids])
table.append(["n_samples",n_samples])
table.append(["m",m])
table.append(["njs",njs])
table.append(["nms",nms])
table.append(["vfile",vfile])
table.append(["lfile",lfile])
table.append(["sccpFile", scppFile])
pprint_table(sys.stdout,table)
with open(os.path.join(oDir,'params.txt'),'a') as f:
pprint_table(f,table)
jointBidFile = os.path.join(oDir,"jointLocalBids.txt")
condBidFile = os.path.join(oDir,"condLocalBids.txt")
margBidFile = os.path.join(oDir,"margLocalBids.txt")
useExternalValuations = False
if vfile == None and lfile == None:
vfile = os.path.join(oDir,'v.txt')
lfile = os.path.join(oDir,'l.txt')
else:
vmat = numpy.loadtxt(vfile)
lmat = numpy.loadtxt(lfile)
nbids = lmat.shape[0]
useExternalValuations = True
surplusSamples = scpp.sample(n_samples = n_samples)
numpy.savetxt(os.path.join(oDir,'ppSamples.txt'),surplusSamples)
jsdir = os.path.join(oDir,'jointSamples')
if not os.path.exists(jsdir):
os.makedirs(jsdir)
msdir = os.path.join(oDir,'margSamples')
if not os.path.exists(msdir):
os.makedirs(msdir)
es = numpy.zeros((nbids,3))
for i in xrange(nbids):
print 'Bid number {0}'.format(i)
if useExternalValuations:
v = vmat[i,:]
l = lmat[i]
else:
print 'randomizing valuation'
v,l = randomValueVector(m=m)
with open(vfile,'a') as f:
numpy.savetxt(f, v.reshape(1,v.shape[0]))
# print >> f, v
with open(lfile,'a') as f:
numpy.savetxt(f,numpy.atleast_1d(l))
print 'v = {0}'.format(v)
print 'l = {0}'.format(l)
revenue = listRevenue(bundles, v, l)
bundleRevenueDict = {}
for b, r in zip(bundles,revenue):
bundleRevenueDict[tuple(b)] = r
jointSamples = scpp.sample(n_samples = njs)
margSamples = scpp.sampleMarg(n_samples = nms)
numpy.savetxt(
os.path.join(jsdir,'jointSamples_{0:04}.txt'.format(i)),
jointSamples)
numpy.savetxt(
os.path.join(msdir,'margSamples_{0:04}.txt'.format(i)),
margSamples)
initBid = straightMUa(bundles, revenue, scpp)
jbid = jointLocal(bundles,revenue,initBid,jointSamples)
cbid = condLocal(bundles, revenue, initBid, jointSamples)
mbid = margLocal(bundles, revenue, initBid, margSamples)
with open(jointBidFile,'a') as f:
numpy.savetxt(f, jbid.T)
with open(condBidFile,'a') as f:
numpy.savetxt(f, cbid.T)
with open(margBidFile,'a') as f:
numpy.savetxt(f, mbid.T)
es[i,0] = expectedSurplus_(bundleRevenueDict, jbid, surplusSamples)
es[i,1] = expectedSurplus_(bundleRevenueDict, cbid, surplusSamples)
es[i,2] = expectedSurplus_(bundleRevenueDict, mbid, surplusSamples)
with open(os.path.join(oDir,"jointLocalExpectedSurplus.txt"),'a') as f:
numpy.savetxt(f,numpy.atleast_1d(es[i,0]))
with open(os.path.join(oDir,"condLocalExpectedSurplus.txt"),'a') as f:
numpy.savetxt(f,numpy.atleast_1d(es[i,1]))
with open(os.path.join(oDir,"margLocalExpectedSurplus.txt"),'a') as f:
numpy.savetxt(f,numpy.atleast_1d(es[i,2]))
jmean = numpy.mean(es[:,0])
jvar = numpy.var(es[:,0])
cmean = numpy.mean(es[:,1])
cvar = numpy.var(es[:,1])
mmean = numpy.mean(es[:,2])
mvar = numpy.var(es[:,2])
print 'jointLocal Expected Surplus Mean {0}'.format(jmean)
print 'jointLocal Expected Surplus Variance {0}'.format(jvar)
print 'condLocal Expected Surplus Mean {0}'.format(cmean)
print 'condLocal Expected Surplus Variance {0}'.format(cvar)
print 'margLocal Expected Surplus Mean {0}'.format(mmean)
print 'margLocal Expected Surplus Variance {0}'.format(mvar)
with open(os.path.join(oDir,'jointLocalStats.txt'),'a') as f:
print >> f, 'mean ', jmean
print >> f, 'var ', jvar
with open(os.path.join(oDir,'condLocalStats.txt'),'a') as f:
print >> f, 'mean ', cmean
print >> f, 'var ', cvar
with open(os.path.join(oDir,'margLocalStats.txt'),'a') as f:
print >> f, 'mean ', mmean
print >> f, 'var ', mvar
with open(os.path.join(oDir,'stats.txt'),'a') as f:
print >> f, 'jointLocal expected surplus mean: {0}'.format(jmean)
print >> f, 'jointLocal expected surplus variation: {0}'.format(jvar)
print >> f, 'condLocal expected surplus mean: {0}'.format(cmean)
print >> f, 'condLocal expected surplus variation: {0}'.format(cvar)
print >> f, 'margLocal expected surplus mean: {0}'.format(mmean)
print >> f, 'margLocal expected surplus variation: {0}'.format(mvar)
bins = range(int(es.min())-1, int(es.max())+1)
jhist, jbins = numpy.histogram(es[:,0], bins, normed = True)
chist, cbins = numpy.histogram(es[:,1], bins, normed = True)
mhist, mbins = numpy.histogram(es[:,2], bins, normed = True)
f,ax = plt.subplots(3,1, sharex = True)
ax[0].bar( (jbins[:-1]+jbins[1:])/2, jhist, align = 'center')
ax[0].set_title('jointLocal expected surplus')
ax[1].bar( (cbins[:-1]+cbins[1:])/2, chist, align = 'center' )
ax[1].set_title('condLocal expected surplus')
ax[2].bar( (mbins[:-1]+mbins[1:])/2, mhist, align = 'center')
ax[2].set_title('margLocal expected surplus')
plt.savefig(os.path.join(oDir,'expectedSurplus.pdf'))
def jointGmmScpp(**kwargs):
"""
NOTE: EXTRA MODEL IS FIT TO FULL COVAR GMM - regardless of
covariance_type kwarg.
"""
kwargs['oDir'] = kwargs.get('oDir')
if kwargs['oDir'] == None:
raise ValueError("Must specify output directory - oDir.")
kwargs['agentType'] = kwargs.get('agentType')
if kwargs['agentType'] == None:
raise ValueError('Must specify agent type - agentType.')
kwargs['nAgents'] = kwargs.get('nAgents',5)
kwargs['selfIdx'] = kwargs.get('selfIdx',numpy.random.randint(kwargs['nAgents']))
kwargs['nGames'] = kwargs.get('nGames',10000)
kwargs['nklsamples'] = kwargs.get('nklsamples',1000)
kwargs['maxItr'] = kwargs.get('maxItr',100)
kwargs['tol'] = kwargs.get('tol',0.01)
kwargs['aicCompMin'] = kwargs.get('aicCompMin',5)
kwargs['aicCompMax'] = kwargs.get('aicCompMax',21)
kwargs['aicMinCovar'] = kwargs.get('aicMinCovar',0.1)
kwargs['minPrice'] = kwargs.get('minPrice',0)
kwargs['maxPrice'] = kwargs.get('maxPrice',numpy.float('inf'))
kwargs['covariance_type'] = kwargs.get('covariance_type','full')
kwargs['m'] = kwargs.get('m',5)
kwargs['minValuation'] = kwargs.get('vmin',0)
kwargs['maxValuation'] = kwargs.get('vmax',50)
kwargs['parallel'] = kwargs.get('parallel',True)
kwargs['nProc'] = kwargs.get('nProc', multiprocessing.cpu_count())
kwargs['verbose'] = kwargs.get('verbose', True)
kwargs['pltMarg'] = kwargs.get('pltMarg', True)
kwargs['l'] = kwargs.get('l')
kwargs['timeStamp'] = timestamp_()
ps = paramString(**kwargs)
kwargs['oDir'] = os.path.join(kwargs['oDir'],ps)
if not os.path.exists(kwargs['oDir']):
os.makedirs(kwargs['oDir'])
models = numpy.arange(kwargs['aicCompMin'], kwargs['aicCompMax'])
if kwargs['verbose']:
table = []
for k,v in kwargs.iteritems():
table.append([k,str(v)])
pprint_table(sys.stdout, table)
with open(os.path.join(kwargs['oDir'],'params.txt'),'w') as f:
pprint_table(f, table)
with open(os.path.join(kwargs['oDir'],'params.json'),'w') as f:
json.dump(kwargs, f)
kwargs['pricePrediction'] = uniformpp(kwargs['m'],kwargs['minValuation'],kwargs['maxValuation'])
idx2keep = numpy.arange(kwargs['nAgents'])
idx2keep = numpy.delete(idx2keep, kwargs['selfIdx'])
if kwargs['verbose']:
print 'indicies to keep = {0}'.format(idx2keep)
filePostfix = fileNamePostfix(**kwargs)
for itr in xrange(kwargs['maxItr']):
itrStart = time.time()
if kwargs['verbose']:
print 'Iteration {0}'.format(itr+1)
simStart = time.time()
bids = simulateAuction(**kwargs)
simEnd = time.time()
# simFile = os.path.realpath(os.path.join(kwargs['oDir'],"simulationTime_{0}.txt".format(ps)))
simFile = os.path.join(kwargs['oDir'],'simTime_0.01.txt')
# if not simFile:
# with open(os.path.realpath(simFile),'w+') as f:
# numpy.savetxt(f, numpy.atleast_1d(simEnd-simStart))
# else:
# with open(os.path.join(kwargs['oDir'],"simulationTime_{0}.txt".format(ps)),'a+') as f:
with open(simFile,'a+') as f:
numpy.savetxt(f, numpy.atleast_1d(simEnd-simStart))
if kwargs['verbose']:
print 'Simulated {0} auctions in {1} seconds'.format(kwargs['nGames'],simEnd-simStart)
del simStart, simEnd
bidsFile = 'bids_{0:04}_{1}.npy'.format(itr,filePostfix)
with open(os.path.join(kwargs['oDir'], bidsFile),'w') as f:
numpy.save(f, bids)
hob = numpy.max(bids[:,idx2keep,:],1)
hobFile = os.path.join(kwargs['oDir'],'hob_{0:04}_{1}.txt'.format(itr,filePostfix))
with open(hobFile,'w') as f:
numpy.savetxt(f,hob)
del bids
nextpp = jointGMM(covariance_type = kwargs.get('covariance_type'))
temppp, aicValues, compRange = nextpp.aicFit(X=hob, compRange = models, min_covar = kwargs['aicMinCovar'], verbose = kwargs['verbose'])
aicFile = os.path.join(kwargs['oDir'],'aic_{0:03}_{1}.pdf'.format(itr+1,filePostfix))
pltAic(compRange,aicValues,itr,aicFile)
del hob,temppp,compRange
ppFile = os.path.join(kwargs['oDir'], 'gmmScpp_{0:04}_{1}.pkl'.format(itr,filePostfix))
with open(ppFile,'w') as f:
pickle.dump(nextpp,f)
if kwargs['pltMarg']:
oFile = os.path.join(kwargs['oDir'],'marg_{0:04}_{1}.pdf'.format(itr,filePostfix))
nextpp.pltMarg(oFile = oFile)
with open(os.path.join(kwargs['oDir'],'aic_{0:04}_{1}.txt'.format(itr,filePostfix)),'a') as f:
numpy.savetxt(f,numpy.atleast_1d(aicValues).T)
if kwargs['verbose']:
print 'AIC Fit: number of components = {0}'.format(nextpp.n_components)
with open(os.path.join(kwargs['oDir'],'n_components_{0}.txt'.format(filePostfix)), 'a') as f:
numpy.savetxt(f,numpy.atleast_1d(nextpp.n_components))
if itr > 0:
kld = numpy.abs(apprxJointGmmKL(kwargs['pricePrediction'], nextpp,
nSamples = kwargs['nklsamples'], verbose = kwargs['verbose']))
with open(os.path.join(kwargs['oDir'],'kld_{0}.txt'.format(filePostfix)),'a') as f:
numpy.savetxt(f,numpy.atleast_1d(kld))
if kwargs['verbose']:
print 'Symmetric KL Distance = {0}'.format(kld)
itrEnd = time.time()
with open(os.path.join(kwargs['oDir'], "itrTime_{0}.txt".format(filePostfix)),'a') as f:
numpy.savetxt(f, numpy.atleast_1d(itrEnd-itrStart))
kwargs['pricePrediction'] = nextpp
if itr > 0:
if kld < kwargs['tol']:
if kwargs['verbose']:
print 'kld = {0} < tol = {1}'.format(kld, kwargs['tol'])
print 'CONVERGED!'
break
else:
print ''
with open(os.path.join(kwargs['oDir'],'kld_{0}.txt'.format(filePostfix)),'r') as f:
kld = numpy.loadtxt(f, 'float')
f, ax = plt.subplots()
plt.plot(kld,'r-',linewidth=3)
plt.title("Absolute Symmetric K-L Divergence")
plt.xlabel("Iteration")
plt.ylabel(r"|kld|")
plt.savefig(os.path.join(kwargs['oDir'],'kld_{0}.pdf'.format(ps)))
del kld
with open(os.path.join(kwargs['oDir'],'n_components_{0}.txt'.format(filePostfix)),'r') as f:
comp = numpy.loadtxt(f)
f,ax = plt.subplots()
colors = ['#0A0A2A']*len(aicValues)
ax.bar(range(len(comp)), comp, color=colors, align = 'center')
ax.set_ylabel('GMM Model (Number of Components)')
ax.set_xlabel('Iteration')
ax.set_title('Model Selection')
plt.ylim([0,numpy.max(comp) + 0.5])
plt.savefig(os.path.join(kwargs['oDir'],'n_components_{0}.pdf'.format(ps)))
del comp
if kwargs['verbose']:
print 'Simulating {0} auctions after scpp converged.'.format(kwargs['nGames'])
# To check if distribution is SCPP, after convergence simulate
# more bids then evaluate measures of similarity between the resulting
# bids and the scpp candidate.
start = time.time()
extraBids = simulateAuction(**kwargs)
end = time.time()
with open(os.path.join(kwargs['oDir'],'extraBids_{0}.npy'.format(filePostfix)), 'w') as f:
numpy.save(f, extraBids)
if kwargs['verbose']:
print 'Simulated {0} holdout auctions in {1} seconds'.format(kwargs['nGames'],end-start)
extraHob = numpy.max(extraBids[:,idx2keep,:],1)
with open(os.path.join(kwargs['oDir'],'extraHob_{0}.txt'.format(filePostfix)),'w') as f:
numpy.savetxt(f, extraHob)
ll = numpy.sum(kwargs['pricePrediction'].eval(extraHob)[0])
if kwargs['verbose']:
print 'log-likelihood hold out = {0}'.format(ll)
with open(os.path.join(kwargs['oDir'],'extraHobLL_{0}.txt'.format(filePostfix)),'w') as f:
numpy.savetxt(f,numpy.atleast_1d(ll))
# fit another model to held out data and
# compute the last skl between the scpp and
# the extra model
extraModel = jointGMM()
gmm, aicValues, compRange = extraModel.aicFit(X=extraHob, compRange = models, min_covar = kwargs['aicMinCovar'], verbose = kwargs['verbose'])
kld = numpy.abs(apprxJointGmmKL(kwargs['pricePrediction'], extraModel, nSamples = kwargs['nklsamples'], verbose = kwargs['verbose']))
f,ax = plt.subplots()
colors = ['#777777']*len(aicValues)
colors[numpy.argmin(aicValues)] = 'r'
ax.bar(compRange, aicValues, color=colors, align = 'center')
ax.set_ylabel('AIC score')
ax.set_xlabel('GMM Model (Number of Components)')
ax.set_title('Iteration {0}'.format(itr+1))
plt.ylim([0,numpy.max(aicValues) + 0.5])
extraAicFile = "extraAic_{0:04}_{1}.pdf".format(itr+1,ps)
plt.savefig(os.path.join(kwargs['oDir'],extraAicFile))
del gmm,aicValues, compRange
if kwargs['verbose']:
print 'SKL-D between SCPP proposal and extra model = {0}'.format(kld)
extraSkdFile = os.path.join(kwargs['oDir'],'extraHobSkl_{0}.txt'.format(filePostfix))
with open(extraSkdFile,'w') as f:
numpy.savetxt(f, numpy.atleast_1d(kld))