def updateParameters(self, articlePicked, click, userID): featureDimension = len(articlePicked.featureVector) T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.T_A += np.outer(T_X, T_X) self.T_b += click*T_X Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum)) Xi_Matirx.T[userID] = articlePicked.featureVector W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx)) #print np.shape(W_X) self.W_A += np.outer(W_X, W_X) self.W_b += click * W_X self.UserTheta = matrixize(np.dot(np.linalg.inv(self.T_A), self.T_b), len(articlePicked.featureVector)) self.W = matrixize(np.dot(np.linalg.inv(self.W_A), self.W_b), self.userNum) #self.W = normalize(self.W, axis=0, norm='l1') #print 'A', self.W_A #print 'b', self.W_b ''' plt.pcolor(self.W_b) plt.colorbar plt.show() ''' self.CoTheta = np.dot(self.UserTheta, self.W) self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector))) self.CCA = np.dot(np.dot(self.BigW , np.linalg.inv(self.T_A)), np.transpose(self.BigW)) self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta) self.W_CCA = np.dot(np.dot(self.BigTheta , np.linalg.inv(self.W_A)), np.transpose(self.BigTheta))
def updateParameters(self, articlePicked, click, userID): self.counter +=1 self.Wlong = vectorize(self.W) featureDimension = len(articlePicked.featureVector) T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(T_X, T_X) self.b += click*T_X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum)) Xi_Matirx.T[userID] = articlePicked.featureVector W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx)) self.batchGradient +=evaluateGradient(W_X, click, self.Wlong, self.lambda_, self.regu ) if self.counter%self.windowSize ==0: self.Wlong -= 1/(float(self.counter/self.windowSize)+1)*self.batchGradient self.W = matrixize(self.Wlong, self.userNum) self.W = normalize(self.W, axis=0, norm='l1') #print 'SVD', self.W self.batchGradient = np.zeros(self.userNum*self.userNum) # Use Ridge regression to fit W ''' plt.pcolor(self.W_b) plt.colorbar plt.show() ''' if self.W.T[userID].any() <0 or self.W.T[userID].any()>1: print self.W.T[userID] self.CoTheta = np.dot(self.UserTheta, self.W) self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector))) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW)) self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def updateParameters(self, articlePicked, click, userID):
self.counter +=1
self.Wlong = vectorize(self.W)
featureDimension = len(articlePicked.featureVector)
T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID]))
self.A += np.outer(T_X, T_X)
self.b += click*T_X
self.AInv = np.linalg.inv(self.A)
self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector))
Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum))
Xi_Matirx.T[userID] = articlePicked.featureVector
W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx))
W_X_current = np.dot(np.transpose(self.UserTheta), articlePicked.featureVector)
self.W_X_arr[userID].append(W_X_current)
self.W_y_arr[userID].append(click)
def fun(w):
w = np.asarray(w)
res = np.sum((np.dot(self.W_X_arr[userID], w) - self.W_y_arr[userID])**2, axis = 0) + self.lambda_*np.linalg.norm(w)
return res
def fun(w,X,Y):
w = np.asarray(w)
res = np.sum((np.dot(X, w) - Y)**2, axis = 0) + self.lambda_*np.linalg.norm(w)
return res
'''
def fprime(w):
w = np.asarray(w)
res = self.W_X_arr[userID]*(np.dot(np.transpose(self.W_X_arr[userID]),w) - self.W_y_arr[userID]) + self.lambda_*w
return res
'''
'''
if self.counter%self.windowSize ==0:
current = self.W.T[userID]
res = minimize(fun, current, constraints = getcons(len(self.W)), method ='SLSQP', bounds=getbounds(len(self.W)), options={'disp': False})
if res.x.any()>1 or res.x.any <0:
print 'error'
print res.x
self.W.T[userID] = res.x
'''
if self.counter%self.windowSize ==0:
for i in range(len(self.W)):
if len(self.W[i]) !=0:
def fun(w):
w = np.asarray(w)
res = np.sum((np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2, axis = 0) + self.lambda_*np.linalg.norm(w)
return res
current = self.W.T[i]
res = minimize(fun, current, constraints = getcons(len(self.W)), method ='SLSQP', bounds=getbounds(len(self.W)), options={'disp': False})
self.W.T[i] = res.x
self.windowSize = self.windowSize*2
self.CoTheta = np.dot(self.UserTheta, self.W)
self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector)))
self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def __init__(self, featureDimension, lambda_, userNum, W, windowSize,
RankoneInverse, WRegu):
self.windowSize = windowSize
self.counter = 1
self.RankoneInverse = RankoneInverse
self.WRegu = WRegu
self.L1Regu = True
self.userNum = userNum
self.lambda_ = lambda_
# Basic stat in estimating Theta
self.A = lambda_ * np.identity(n=featureDimension * userNum)
self.b = np.zeros(featureDimension * userNum)
self.UserTheta = np.zeros(shape=(featureDimension, userNum))
# self.UserTheta = np.random.random((featureDimension, userNum))
self.AInv = np.linalg.inv(self.A)
# self.W = np.random.random((userNum, userNum))
# self.W = np.identity(n = userNum)
self.W = W
self.Wlong = vectorize(self.W)
self.batchGradient = np.zeros(userNum * userNum)
self.CoTheta = np.dot(self.UserTheta, self.W)
self.BigW = np.kron(np.transpose(self.W),
np.identity(n=featureDimension))
self.CCA = np.identity(n=featureDimension * userNum)
self.BigTheta = np.kron(np.identity(n=userNum), self.UserTheta)
self.W_X_arr = []
self.W_y_arr = []
for i in range(userNum):
self.W_X_arr.append([])
self.W_y_arr.append([])
def updateParameters(self, articlePicked_FeatureVector, click):
additionalFeatureVector = vectorize(
np.outer(self.userFeature, articlePicked_FeatureVector))
LinUCBUserStruct.updateParameters(self, articlePicked_FeatureVector,
click)
self.B += np.outer(articlePicked_FeatureVector,
additionalFeatureVector)
def updateParameters(self, articlePicked, click, userID):
X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID]))
temp1 = np.outer(articlePicked.featureVector,
articlePicked.featureVector)
temp2 = np.outer(self.W.T[userID], self.W.T[userID])
self.A += np.kron(temp2, temp1)
self.b += click * X
self.sA += temp1
self.outerW += temp2
if self.RankoneInverse:
temp = np.dot(self.AInv, X)
self.AInv = self.AInv - (np.outer(
temp, temp)) / (1.0 + np.dot(np.transpose(X), temp))
else:
#self.AInv = np.linalg.inv(self.A)
self.sAInv = np.linalg.inv(self.sA)
temI = 1.0 / self.lambda_ * np.identity(n=self.featureDimension *
self.userNum)
self.AInv = np.kron(self.WPrimeInv, self.sAInv)
self.UserTheta = matrixize(np.dot(self.AInv, self.b),
len(articlePicked.featureVector))
self.CoTheta = np.dot(self.UserTheta, self.W)
self.CCA = np.dot(np.dot(self.BigW, self.AInv),
np.transpose(self.BigW))
def __init__(self, featureDimension, lambda_, eta_, userNum, windowSize =20): self.windowSize = windowSize self.counter = 0 self.userNum = userNum self.lambda_ = lambda_ # Basic stat in estimating Theta self.A = lambda_*np.identity(n = featureDimension*userNum) self.b = np.zeros(featureDimension*userNum) self.UserTheta = np.zeros(shape = (featureDimension, userNum)) #self.UserTheta = np.random.random((featureDimension, userNum)) self.AInv = np.linalg.inv(self.A) #self.W = np.random.random((userNum, userNum)) self.W = np.identity(n = userNum) self.Wlong = vectorize(self.W) self.batchGradient = np.zeros(userNum*userNum) self.CoTheta = np.dot(self.UserTheta, self.W) self.BigW = np.kron(np.transpose(self.W), np.identity(n=featureDimension)) self.CCA = np.identity(n = featureDimension*userNum) self.BigTheta = np.kron(np.identity(n=userNum) , self.UserTheta) self.W_X_arr = [] self.W_y_arr = [] for i in range(userNum): self.W_X_arr.append([]) self.W_y_arr.append([])
def updateParameters(self, articlePicked, click, userID): X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X self.UserTheta = matrixize(np.dot(np.linalg.inv(self.A), self.b), len(articlePicked.featureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , np.linalg.inv(self.A)), np.transpose(self.BigW))
def updateParameters(self, articlePicked, click, userID): X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def getProb(self, alpha, article, userID): TempFeatureM = np.zeros(shape =(len(article.featureVector), self.userNum)) TempFeatureM.T[userID] = article.featureVector TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.featureVector) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) pta = mean + alpha * var return pta
def getProb_plot(self, alpha, alpha2, article, userID): TempFeatureM = np.zeros(shape =(len(article.V), self.userNum)) TempFeatureM.T[userID] = article.V TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.V) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) var2 = np.sqrt(np.dot(np.dot(self.CoTheta.T[userID][self.context_dimension:], article.A2Inv), self.CoTheta.T[userID][self.context_dimension:])) pta = mean + alpha * var + alpha2*var2 return pta, mean, alpha*var
def getProb(self, alpha, article, userID): TempFeatureM = np.zeros(shape =(len(article.featureVector), self.userNum)) TempFeatureM.T[userID] = article.featureVector TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.featureVector) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) pta = mean + alpha * var #pta = mean + alpha * var return pta
def getProb_plot(self, alpha, alpha2, article, userID): TempFeatureM = np.zeros(shape =(len(article.V), self.userNum)) TempFeatureM.T[userID] = article.V TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.V) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) var2 = np.sqrt(np.dot(np.dot(self.CoTheta.T[userID][self.context_dimension:], article.A2Inv), self.CoTheta.T[userID][self.context_dimension:])) pta = mean + alpha * var + alpha2*var2 return pta, mean, alpha*var
def getProb(self, alpha, article_FeatureVector,userID): x = article_FeatureVector z = vectorize(np.outer(self.users[userID].userFeature, article_FeatureVector)) temp =np.dot(np.dot(np.dot( self.A_zInv , np.transpose( self.users[userID].B)) , self.users[userID].AInv), x ) mean = np.dot(self.users[userID].UserTheta, x)+ np.dot(self.beta, z) s_t = np.dot(np.dot(z, self.A_zInv), z) + np.dot(np.dot(x, self.users[userID].AInv), x) -2* np.dot(z, temp)+ np.dot(np.dot( np.dot(x, self.users[userID].AInv) , self.users[userID].B ) ,temp) var = np.sqrt(s_t) pta = mean + alpha * var return pta
def getProb(self, alpha, article_FeatureVector,userID): x = article_FeatureVector z = vectorize(np.outer(self.users[userID].userFeature, article_FeatureVector)) temp =np.dot(np.dot(np.dot( self.A_zInv , np.transpose( self.users[userID].B)) , self.users[userID].AInv), x ) mean = np.dot(self.users[userID].UserTheta, x)+ np.dot(self.beta, z) s_t = np.dot(np.dot(z, self.A_zInv), z) + np.dot(np.dot(x, self.users[userID].AInv), x) -2* np.dot(z, temp)+ np.dot(np.dot( np.dot(x, self.users[userID].AInv) , self.users[userID].B ) ,temp) var = np.sqrt(s_t) pta = mean + alpha * var return pta
def updateParameters(self, articlePicked_FeatureVector, click, userID): X = vectorize(np.outer(articlePicked_FeatureVector, self.W.T[userID])) outer = np.outer(X, X) self.A += outer self.b += click*X self.AInv = self.AInv - float(np.dot(self.AInv, np.dot(outer, self.AInv)))/(1.0+np.dot(np.transpose(X), np.dot(self.AInv, X) )) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked_FeatureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW, self.AInv), np.transpose(self.BigW))
def updateParameters(self, articlePicked, click, userID):
Xi_Matirx = np.zeros(shape=(self.featureDimension, self.userNum))
Xi_Matirx.T[userID] = articlePicked.featureVector
X = vectorize(np.dot(np.transpose(self.theta), Xi_Matirx))
#self.W = matrixize(np.dot(np.linalg.inv(self.A), self.b), self.userNum)
#fun = lambda x = np.identity(self.userNum): self.f1 + (1/2.0)*(np.dot( np.dot(articlePicked.featureVector, self.theta) ,x.T[userID]) - click)**2 + (self.eta_/2.0)*np.matrix.trace(np.dot(np.transpose(x), x))
def fun(x):
x = np.asarray(x)
x = x.reshape(self.userNum, self.userNum)
return self.f1 + (1 / 2.0) * (np.dot(
np.dot(articlePicked.featureVector, self.theta),
x.T[userID]) - click)**2 + (self.eta_ / 2.0) * np.matrix.trace(
np.dot(np.transpose(x), x))
def fprime(x):
x = np.asarray(x)
return np.dot(self.A + np.outer(X, X), x) - (self.b + click * X)
cons = (
{
'type': 'ineq',
'fun': lambda x: x
},
{
'type': 'ineq',
'fun': lambda x: 1 - x
},
{
'type': 'eq',
'fun': lambda x: np.sum(
np.square(np.sum(x, axis=1) - 1)
) # I'm not sure whether to sum in row or column, but this should do the work.
})
'''
res = minimize(fun, self.W, constraints = cons, method ='SLSQP', jac = fprime, options={'disp': False})
self.W = res.x.reshape(self.userNum, self.userNum)
#self.W = np.transpose(self.W)
'''
#print self.W[0]
#Normalization
#self.W = normalize(self.W, axis=0, norm='l1')
self.A += np.outer(X, X)
self.b += click * X
self.W = matrixize(np.dot(np.linalg.inv(self.A), self.b), self.userNum)
self.f1 += (1 / 2.0) * (np.dot(
np.dot(articlePicked.featureVector, self.theta),
self.W.T[userID]))**2
self.CoTheta = np.dot(self.theta, self.W)
self.CCA = np.dot(np.dot(self.BigTheta, np.linalg.inv(self.A)),
np.transpose(self.BigTheta))
def updateParameters(self, articlePicked, click, userID): featureVectorM = np.zeros(shape =(len(articlePicked.featureVector), self.userNum)) featureVectorM.T[userID] = articlePicked.featureVector featureVectorV = vectorize(featureVectorM) CoFeaV = np.dot(self.STBigWInv, featureVectorV) self.A += np.outer(CoFeaV, CoFeaV) self.b += click * CoFeaV self.AInv = np.linalg.inv(self.A) self.theta = np.dot(self.AInv, self.b)
def updateParameters(self, articlePicked, click, userID, update='inv'):
featureVectorM = np.zeros(
shape=(len(articlePicked.contextFeatureVector), self.userNum))
featureVectorM.T[userID] = articlePicked.contextFeatureVector
featureVectorV = vectorize(featureVectorM)
CoFeaV = np.dot(self.STBigWInv, featureVectorV)
self.A = self.A + np.outer(CoFeaV, CoFeaV)
self.b = self.b + click * CoFeaV
self.AInv = np.linalg.inv(self.A)
self.theta = np.dot(self.AInv, self.b)
def updateParameters(self, articlePicked, click, userID): X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X if self.RankoneInverse: temp = np.dot(self.AInv, X) self.AInv = self.AInv - (np.outer(temp,temp))/(1.0+np.dot(np.transpose(X),temp)) else: self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def updateParameters(self, articlePicked, click, userID):
featureDimension = len(articlePicked.featureVector)
T_X = vectorize(np.outer(articlePicked.featureVector,
self.W.T[userID]))
self.T_A += np.outer(T_X, T_X)
self.T_b += click * T_X
Xi_Matirx = np.zeros(shape=(featureDimension, self.userNum))
Xi_Matirx.T[userID] = articlePicked.featureVector
W_X = vectorize(np.dot(np.transpose(self.UserTheta), Xi_Matirx))
#print np.shape(W_X)
self.W_A += np.outer(W_X, W_X)
self.W_b += click * W_X
self.UserTheta = matrixize(np.dot(np.linalg.inv(self.T_A), self.T_b),
len(articlePicked.featureVector))
self.W = matrixize(np.dot(np.linalg.inv(self.W_A), self.W_b),
self.userNum)
#self.W = normalize(self.W, axis=0, norm='l1')
#print 'A', self.W_A
#print 'b', self.W_b
'''
plt.pcolor(self.W_b)
plt.colorbar
plt.show()
'''
self.CoTheta = np.dot(self.UserTheta, self.W)
self.BigW = np.kron(np.transpose(self.W),
np.identity(n=len(articlePicked.featureVector)))
self.CCA = np.dot(np.dot(self.BigW, np.linalg.inv(self.T_A)),
np.transpose(self.BigW))
self.BigTheta = np.kron(np.identity(n=self.userNum), self.UserTheta)
self.W_CCA = np.dot(np.dot(self.BigTheta, np.linalg.inv(self.W_A)),
np.transpose(self.BigTheta))
def updateParameters(self, articlePicked, click, userID, update='Inv'): X = vectorize(np.outer(articlePicked.contextFeatureVector, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X if update == 'Inv': self.AInv = np.linalg.inv(self.A) else: self.AInv = self.AInv - float(np.dot(self.AInv, np.dot(outer, self.AInv)))/(1.0+np.dot(np.transpose(X), np.dot(self.AInv, X) )) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.contextFeatureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def getProb(self, alpha, alpha2, article, userID): if alpha == -1: alpha = 0.1*np.sqrt(np.log(self.time+1))+0.1*(1-0.8**self.time) alpha2 = 0.1*np.sqrt(np.log(article.time+1))+0.1*(1-0.8**article.time) TempFeatureM = np.zeros(shape =(len(article.V), self.userNum)) TempFeatureM.T[userID] = article.V TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.V) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) var2 = np.sqrt(np.dot(np.dot(self.CoTheta.T[userID][self.context_dimension:], article.A2Inv), self.CoTheta.T[userID][self.context_dimension:])) pta = mean + alpha * var + alpha2*var2 return pta
def updateParameters(self, articlePicked, click, userID):
X = vectorize(np.outer(articlePicked, self.W.T[userID]))
self.A += np.outer(X, X)
self.b += click * X
if self.RankoneInverse:
temp = np.dot(self.AInv, X)
self.AInv = self.AInv - (np.outer(temp, temp)) / (1.0 + np.dot(np.transpose(X), temp))
else:
self.AInv = np.linalg.inv(self.A)
self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked))
self.CoTheta = np.dot(self.UserTheta, self.W)
self.CCA = np.dot(np.dot(self.BigW, self.AInv), np.transpose(self.BigW))
def getProb(self, alpha, alpha2, article, userID): if alpha == -1: alpha = 0.1*np.sqrt(np.log(self.time+1))+0.1*(1-0.8**self.time) alpha2 = 0.1*np.sqrt(np.log(article.time+1))+0.1*(1-0.8**article.time) TempFeatureM = np.zeros(shape =(len(article.V), self.userNum)) TempFeatureM.T[userID] = article.V TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.V) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) var2 = np.sqrt(np.dot(np.dot(self.CoTheta.T[userID][self.context_dimension:], article.A2Inv), self.CoTheta.T[userID][self.context_dimension:])) pta = mean + alpha * var + alpha2*var2 return pta
def LateUpdate(self): featureDimension = self.featureVectorMatrix.shape[0] current_A = np.zeros(shape = (featureDimension* self.userNum, featureDimension*self.userNum)) current_b = np.zeros(featureDimension*self.userNum) for i in range(self.userNum): X = vectorize(np.outer(self.featureVectorMatrix.T[i], self.W.T[i])) XS = np.outer(X, X) current_A += XS current_b += self.reward[i] * X self.A += current_A self.b += current_b self.UserTheta = matrixize(np.dot(np.linalg.inv(self.A), self.b), featureDimension) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , np.linalg.inv(self.A)), np.transpose(self.BigW))
def getProb(self,alpha , article, userID): featureVectorM = np.zeros(shape =(len(article.featureVector), self.userNum)) featureVectorM.T[userID] = article.featureVector featureVectorV = vectorize(featureVectorM) CoFeaV = np.dot(self.STBigWInv, featureVectorV) mean = np.dot(np.transpose(self.theta), CoFeaV) a = np.dot(CoFeaV, self.AInv) var = np.sqrt( np.dot( np.dot(CoFeaV, self.AInv) , CoFeaV)) pta = mean + alpha * var return pta
def getProb(self,alpha , article, userID): featureVectorM = np.zeros(shape =(len(article.featureVector), self.userNum)) featureVectorM.T[userID] = article.featureVector featureVectorV = vectorize(featureVectorM) CoFeaV = np.dot(self.STBigWInv, featureVectorV) mean = np.dot(np.transpose(self.theta), CoFeaV) a = np.dot(CoFeaV, self.AInv) var = np.sqrt( np.dot( np.dot(CoFeaV, self.AInv) , CoFeaV)) pta = mean + alpha * var return pta
def updateParameters(self, articlePicked_FeatureVector, click, userID):
X = vectorize(np.outer(articlePicked_FeatureVector, self.W.T[userID]))
# self.A += np.outer(X, X)
self.b += click * X
self.AInv = self.AInv - np.dot(
self.AInv, np.dot(np.outer(X, X), self.AInv)) / float(
1.0 + np.dot(np.dot(np.transpose(X), self.AInv), X))
self.UserTheta = matrixize(np.dot(self.AInv, self.b),
len(articlePicked_FeatureVector))
self.CoTheta = np.dot(self.UserTheta, self.W)
self.CCA = np.dot(np.dot(self.BigW, self.AInv),
np.transpose(self.BigW))
def LateUpdate(self): featureDimension = self.contextFeatureVectorMatrix.shape[0] current_A = np.zeros(shape = (featureDimension* self.userNum, featureDimension*self.userNum)) current_b = np.zeros(featureDimension*self.userNum) for i in range(self.userNum): X = vectorize(np.outer(self.contextFeatureVectorMatrix.T[i], self.W.T[i])) XS = np.outer(X, X) current_A += XS current_b += self.reward[i] * X self.A += current_A self.b += current_b self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), featureDimension) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def getProb(self, alpha, article, userID): TempFeatureM = np.zeros(shape =(len(article.contextFeatureVector), self.userNum)) TempFeatureM.T[userID] = article.contextFeatureVector TempFeatureV = vectorize(TempFeatureM) mean = np.dot(self.CoTheta.T[userID], article.contextFeatureVector) var = np.sqrt(np.dot(np.dot(TempFeatureV, self.CCA), TempFeatureV)) self.alpha_t = 0.01*np.sqrt(np.log(np.linalg.det(self.A)/float(self.sigma * self.lambda_) )) + np.sqrt(self.lambda_) #pta = mean + alpha * var # use emprically tuned alpha pta = mean + self.alpha_t *var # use the theoretically computed alpha_t return pta
def updateParameters(self, articlePicked, click, userID): featureVectorM = np.zeros(shape =(len(articlePicked.featureVector), self.userNum)) featureVectorM.T[userID] = articlePicked.featureVector featureVectorV = vectorize(featureVectorM) CoFeaV = np.dot(self.STBigWInv, featureVectorV) self.A = self.A + np.outer(CoFeaV, CoFeaV) self.b = self.b + click * CoFeaV if self.RankoneInverse: temp = np.dot(self.AInv, CoFeaV) self.AInv = self.AInv - (np.outer(temp,temp))/(1.0+np.dot(np.transpose(CoFeaV),temp)) else: self.AInv = np.linalg.inv(self.A) self.theta = np.dot(self.AInv, self.b)
def updateParameters(self, articles, clicks, userID): self.time += len(articles) for article, click in zip(articles, clicks): if article.id in self.count[userID]: self.count[userID][article.id] += 1 else: self.count[userID][article.id] = 1 X = vectorize(np.outer(article.V, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articles[0].V)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def updateParameters(self, articles, clicks, userID): self.time += len(articles) for article, click in zip(articles, clicks): if article.id in self.count[userID]: self.count[userID][article.id] += 1 else: self.count[userID][article.id] = 1 X = vectorize(np.outer(article.V, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articles[0].V)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def updateParameters(self, articlePicked, click, userID): featureVectorM = np.zeros(shape =(len(articlePicked.featureVector), self.userNum)) featureVectorM.T[userID] = articlePicked.featureVector featureVectorV = vectorize(featureVectorM) CoFeaV = np.dot(self.STBigWInv, featureVectorV) self.A = self.A + np.outer(CoFeaV, CoFeaV) self.b = self.b + click * CoFeaV if self.RankoneInverse: temp = np.dot(self.AInv, CoFeaV) self.AInv = self.AInv - (np.outer(temp,temp))/(1.0+np.dot(np.transpose(CoFeaV),temp)) else: self.AInv = np.linalg.inv(self.A) self.theta = np.dot(self.AInv, self.b)
def updateParameters(self, articlePicked, click, userID, update='Inv'): #self.currentW = np.dot(self.currentW, self.W) #X = vectorize(np.outer(articlePicked.contextFeatureVector, self.currentW.T[userID])) X = vectorize(np.outer(articlePicked.contextFeatureVector, self.W.T[userID])) self.A += np.outer(X, X) self.b += click*X if update == 'Inv': self.AInv = np.linalg.inv(self.A) else: self.AInv = self.AInv - float(np.dot(self.AInv, np.dot(outer, self.AInv)))/(1.0+np.dot(np.transpose(X), np.dot(self.AInv, X) )) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.contextFeatureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def decide(self, pool_articles, userID, k = 1):
# MEAN
art_features = np.empty([len(pool_articles), len(pool_articles[0].contextFeatureVector)*self.n_users])
for i in range(len(pool_articles)):
TempFeatureM = np.zeros(shape =(len(pool_articles[0].contextFeatureVector), self.n_users))
TempFeatureM.T[userID] = pool_articles[i].contextFeatureVector
art_features[i, :] = vectorize(TempFeatureM)
CoFeaV = np.dot(art_features, self.USERS.STBigWInv)
mean_matrix = np.dot(CoFeaV, self.USERS.theta)
var_matrix = np.sqrt(np.dot(np.dot(CoFeaV, self.USERS.AInv), CoFeaV.T).clip(0))
pta_matrix = mean_matrix + self.alpha*np.diag(var_matrix)
pool_positions = np.argsort(pta_matrix)[(k*-1):]
articles = []
for i in range(k):
articles.append(pool_articles[pool_positions[i]])
return articles
def updateParameters(self, articlePicked_FeatureVector, click, userID): z = vectorize( np.outer(self.users[userID].userFeature, articlePicked_FeatureVector)) temp = np.dot(np.transpose(self.users[userID].B), self.users[userID].AInv) self.A_z += np.dot(temp, self.users[userID].B) self.b_z +=np.dot(temp, self.users[userID].b) self.users[userID].updateParameters(articlePicked_FeatureVector, click) temp = np.dot(np.transpose(self.users[userID].B), self.users[userID].AInv) self.A_z = self.A_z + np.outer(z,z) - np.dot(temp, self.users[userID].B) self.b_z =self.b_z+ click*z - np.dot(temp, self.users[userID].b) self.A_zInv = np.linalg.inv(self.A_z) self.beta =np.dot(self.A_zInv, self.b_z) self.users[userID].updateTheta(self.beta)
def updateParameters(self, articlePicked, click, userID): articlePicked_FeatureVector = articlePicked.featureVector z = vectorize( np.outer(self.users[userID].userFeature, articlePicked_FeatureVector)) temp = np.dot(np.transpose(self.users[userID].B), self.users[userID].AInv) self.A_z += np.dot(temp, self.users[userID].B) self.b_z +=np.dot(temp, self.users[userID].b) self.users[userID].updateParameters(articlePicked, click) temp = np.dot(np.transpose(self.users[userID].B), self.users[userID].AInv) self.A_z = self.A_z + np.outer(z,z) - np.dot(temp, self.users[userID].B) self.b_z =self.b_z+ click*z - np.dot(temp, self.users[userID].b) self.A_zInv = np.linalg.inv(self.A_z) self.beta =np.dot(self.A_zInv, self.b_z) self.users[userID].updateTheta(self.beta)
def updateParameters(self, articlePicked, click, userID):
Xi_Matirx = np.zeros(shape = (self.featureDimension, self.userNum))
Xi_Matirx.T[userID] = articlePicked.featureVector
X = vectorize( np.dot(np.transpose(self.theta), Xi_Matirx))
#self.W = matrixize(np.dot(np.linalg.inv(self.A), self.b), self.userNum)
#fun = lambda x = np.identity(self.userNum): self.f1 + (1/2.0)*(np.dot( np.dot(articlePicked.featureVector, self.theta) ,x.T[userID]) - click)**2 + (self.eta_/2.0)*np.matrix.trace(np.dot(np.transpose(x), x))
def fun(x):
x = np.asarray(x)
x = x.reshape(self.userNum, self.userNum)
return self.f1 + (1/2.0)*(np.dot( np.dot(articlePicked.featureVector, self.theta) ,x.T[userID]) - click)**2 + (self.eta_/2.0)*np.matrix.trace(np.dot(np.transpose(x), x))
def fprime(x):
x = np.asarray(x)
return np.dot(self.A+ np.outer(X, X), x) -(self.b + click*X)
cons = ({'type' : 'ineq',
'fun' : lambda x: x },
{'type' : 'ineq',
'fun' : lambda x: 1-x},
{'type': 'eq',
'fun': lambda x : np.sum(np.square(np.sum(x, axis = 1)-1)) # I'm not sure whether to sum in row or column, but this should do the work.
}
)
'''
res = minimize(fun, self.W, constraints = cons, method ='SLSQP', jac = fprime, options={'disp': False})
self.W = res.x.reshape(self.userNum, self.userNum)
#self.W = np.transpose(self.W)
'''
#print self.W[0]
#Normalization
#self.W = normalize(self.W, axis=0, norm='l1')
self.A += np.outer(X, X)
self.b += click * X
self.W = matrixize(np.dot(np.linalg.inv(self.A), self.b), self.userNum)
self.f1 +=(1/2.0)*(np.dot( np.dot(articlePicked.featureVector, self.theta) ,self.W.T[userID]))**2
self.CoTheta = np.dot(self.theta, self.W)
self.CCA = np.dot(np.dot(self.BigTheta, np.linalg.inv(self.A)), np.transpose(self.BigTheta))
def updateParameters(self, articlePicked, click, userID): X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) temp1 = np.outer(articlePicked.featureVector, articlePicked.featureVector) temp2 = np.outer(self.W.T[userID], self.W.T[userID]) self.A += np.kron(temp2, temp1) self.b += click*X self.sA += temp1 self.outerW += temp2 if self.RankoneInverse: temp = np.dot(self.AInv, X) self.AInv = self.AInv - (np.outer(temp,temp))/(1.0+np.dot(np.transpose(X),temp)) else: #self.AInv = np.linalg.inv(self.A) self.sAInv = np.linalg.inv(self.sA) temI = 1.0/self.lambda_*np.identity(n = self.featureDimension*self.userNum) self.AInv = np.kron(self.WPrimeInv, self.sAInv) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) self.CoTheta = np.dot(self.UserTheta, self.W) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
def decide(self, pool_articles, userID, k=1):
# MEAN
art_features = np.empty(
[len(pool_articles),
len(pool_articles[0].contextFeatureVector)])
for i in range(len(pool_articles)):
art_features[i, :] = pool_articles[i].contextFeatureVector
user_features = self.USERS.CoTheta.T[userID]
mean_matrix = np.dot(art_features, user_features)
# VARIANCE
art_temp_features = np.empty([
len(pool_articles),
len(pool_articles[0].contextFeatureVector) * self.n_users
])
for i in range(len(pool_articles)):
TempFeatureM = np.zeros(
shape=(len(pool_articles[0].contextFeatureVector),
self.n_users))
TempFeatureM.T[userID] = pool_articles[i].contextFeatureVector
art_temp_features[i, :] = vectorize(TempFeatureM)
var_matrix = np.sqrt(
np.dot(np.dot(art_temp_features, self.USERS.CCA),
art_temp_features.T))
#self.USERS.calculateAlphaT()
if self.use_alpha_t:
self.USERS.calculateAlphaT()
pta_matrix = mean_matrix + self.USERS.alpha_t * np.diag(var_matrix)
else:
pta_matrix = mean_matrix + self.alpha * np.diag(var_matrix)
pool_positions = np.argsort(pta_matrix)[(k * -1):]
articles = []
for i in range(k):
articles.append(pool_articles[pool_positions[i]])
return articles
def updateParameters(self, featureVector, click, userID):
self.counter +=1
self.Wlong = vectorize(self.W)
featureDimension = len(featureVector)
T_X = vectorize(np.outer(featureVector, self.W.T[userID]))
self.A += np.outer(T_X, T_X)
self.b += click*T_X
if self.RankoneInverse:
temp = np.dot(self.AInv, T_X)
self.AInv = self.AInv - (np.outer(temp,temp))/(1.0+np.dot(np.transpose(T_X),temp))
else:
self.AInv = np.linalg.inv(self.A)
self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(featureVector))
Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum))
Xi_Matirx.T[userID] = featureVector
W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx))
W_X_current = np.dot(np.transpose(self.UserTheta), featureVector)
self.W_X_arr[userID].append(W_X_current)
self.W_y_arr[userID].append(click)
#print self.windowSize
if self.counter%self.windowSize ==0:
for i in range(len(self.W)):
if len(self.W_X_arr[i]) !=0:
def fun(w):
w = np.asarray(w)
return np.sum((np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2, axis = 0) + self.lambda_*np.linalg.norm(w)**2
def fun_l1(w):
w = np.asarray(w)
return np.sum((np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2, axis = 0) + self.lambda_*np.linalg.norm(w,1)**2
def evaluateGradient(w):
w = np.asarray(w)
X = np.asarray(self.W_X_arr[i])
y = np.asarray(self.W_y_arr[i])
grad = np.dot(np.transpose(X) , ( np.dot(X,w)- y)) + self.lambda_ * w
return 2*grad
def fun_WRegu(w):
w = np.asarray(w)
return np.sum((np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2, axis = 0) + self.lambda_*np.linalg.norm(w - self.W.T[i])**2
def evaluateGradient_WRegu(w):
w = np.asarray(w)
X = np.asarray(self.W_X_arr[i])
y = np.asarray(self.W_y_arr[i])
grad = np.dot(np.transpose(X) , ( np.dot(X,w)- y)) + self.lambda_ * (w - self.W.T[i])
return 2*grad
current = self.W.T[i]
if self.L1Regu:
res = minimize(fun_l1, current, constraints = getcons(len(self.W)), method ='SLSQP', bounds=getbounds(len(self.W)), options={'disp': False})
elif self.WRegu:
res = minimize(fun_WRegu, current, constraints = getcons(len(self.W)), method ='SLSQP', jac = evaluateGradient_WRegu, bounds=getbounds(len(self.W)), options={'disp': False})
else:
res = minimize(fun, current, constraints = getcons(len(self.W)), method ='SLSQP', jac = evaluateGradient, bounds=getbounds(len(self.W)), options={'disp': False})
self.W.T[i] = res.x
if self.windowSize<2000:
self.windowSize = self.windowSize*2
self.CoTheta = np.dot(self.UserTheta, self.W)
self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(featureVector)))
self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def updateParameters(self, articlePicked, click): article_FeatureVector = articlePicked.featureVector additionalFeatureVector = vectorize(np.outer(self.userFeature, article_FeatureVector)) LinUCBUserStruct.updateParameters(self, articlePicked, click) self.B +=np.outer(article_FeatureVector, additionalFeatureVector)
def updateParameters(self, featureVector, click, userID):
self.counter += 1
self.Wlong = vectorize(self.W)
featureDimension = len(featureVector)
T_X = vectorize(np.outer(featureVector, self.W.T[userID]))
self.A += np.outer(T_X, T_X)
self.b += click * T_X
if self.RankoneInverse:
temp = np.dot(self.AInv, T_X)
self.AInv = self.AInv - (np.outer(
temp, temp)) / (1.0 + np.dot(np.transpose(T_X), temp))
else:
self.AInv = np.linalg.inv(self.A)
self.UserTheta = matrixize(np.dot(self.AInv, self.b),
len(featureVector))
Xi_Matirx = np.zeros(shape=(featureDimension, self.userNum))
Xi_Matirx.T[userID] = featureVector
W_X = vectorize(np.dot(np.transpose(self.UserTheta), Xi_Matirx))
W_X_current = np.dot(np.transpose(self.UserTheta), featureVector)
self.W_X_arr[userID].append(W_X_current)
self.W_y_arr[userID].append(click)
# print self.windowSize
if self.counter % self.windowSize == 0:
for i in range(len(self.W)):
if len(self.W_X_arr[i]) != 0:
def fun(w):
w = np.asarray(w)
return np.sum(
(np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2,
axis=0) + self.lambda_ * np.linalg.norm(w)**2
def fun_l1(w):
w = np.asarray(w)
return np.sum(
(np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2,
axis=0) + self.lambda_ * np.linalg.norm(w, 1)**2
def evaluateGradient(w):
w = np.asarray(w)
X = np.asarray(self.W_X_arr[i])
y = np.asarray(self.W_y_arr[i])
grad = np.dot(np.transpose(X),
(np.dot(X, w) - y)) + self.lambda_ * w
return 2 * grad
def fun_WRegu(w):
w = np.asarray(w)
return np.sum(
(np.dot(self.W_X_arr[i], w) - self.W_y_arr[i])**2,
axis=0
) + self.lambda_ * np.linalg.norm(w - self.W.T[i])**2
def evaluateGradient_WRegu(w):
w = np.asarray(w)
X = np.asarray(self.W_X_arr[i])
y = np.asarray(self.W_y_arr[i])
grad = np.dot(np.transpose(X),
(np.dot(X, w) -
y)) + self.lambda_ * (w - self.W.T[i])
return 2 * grad
current = self.W.T[i]
if self.L1Regu:
res = minimize(fun_l1,
current,
constraints=getcons(len(self.W)),
method='SLSQP',
bounds=getbounds(len(self.W)),
options={'disp': False})
elif self.WRegu:
res = minimize(fun_WRegu,
current,
constraints=getcons(len(self.W)),
method='SLSQP',
jac=evaluateGradient_WRegu,
bounds=getbounds(len(self.W)),
options={'disp': False})
else:
res = minimize(fun,
current,
constraints=getcons(len(self.W)),
method='SLSQP',
jac=evaluateGradient,
bounds=getbounds(len(self.W)),
options={'disp': False})
self.W.T[i] = res.x
if self.windowSize < 2000:
self.windowSize = self.windowSize * 2
self.CoTheta = np.dot(self.UserTheta, self.W)
self.BigW = np.kron(np.transpose(self.W),
np.identity(n=len(featureVector)))
self.CCA = np.dot(np.dot(self.BigW, self.AInv),
np.transpose(self.BigW))
self.BigTheta = np.kron(np.identity(n=self.userNum), self.UserTheta)
