def print_actor_vector(actorIdInput,model):
if model>1 or model<0:
print "please enter model value 0 or 1"
exit()
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME)
actualResult={}
actorVector={}
recentTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp desc')[0]#the most recent tag from database
oldTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp asc')[0]#the oldest tag from database
#converting to milliseconds
recentTag = int(datetime.strptime(str(recentTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
oldTag = int(datetime.strptime(str(oldTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
actor = loadDatabase.queryActioner(con, 'select * from actorinfo where actorid='+str(actorIdInput))[0]
noOfActors=loadDatabase.queryActioner(con,'select count(*) from actorinfo')
movies=loadDatabase.queryActioner(con,'select * from movieactor where actorid='+str(actor[0]))
actorFinalVectortf={}
actorFinalVectoridf={}
#this block calculates tf for all the tags and maintains it in dictionary actorVector
for movie in movies:
actorrank=loadDatabase.queryActioner(con,'select actorrank from movieactor where actorid='+str(actor[0])+'and movieid='+str(movie[0]))[0][0]
totalcount=0
actualResult[movie[0]]={}
tags = loadDatabase.queryActioner(con, 'select * from mltags where movieid=' + str(movie[0]))
for tag in tags:
if(tag[2] in actualResult[movie[0]]):
actualResult[movie[0]][tag[2]]=(actualResult[movie[0]][tag[2]][0]+1,actualResult[movie[0]][tag[2]][1]+(float(int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
else:
actualResult[movie[0]][tag[2]]=(1,float((int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
for tag in actualResult[movie[0]].keys():
if (tag in actorVector):
actorVector[tag] = actorVector[tag] + (float(actualResult[movie[0]][tag][0]) / totalcount) *(float(1)/actorrank)*actualResult[movie[0]][tag][1]
else:
actorVector[tag] = (float(actualResult[movie[0]][tag][0] )/ totalcount) * (float(1)/actorrank) *(actualResult[movie[0]][tag][1])
#this block calculates the idf for all the tags and multiplies with tf
for tags in actorVector.keys():
moviesForTags=loadDatabase.queryActioner(con,'select distinct(movieid) from mltags where tagid='+str(tags))
tagname=loadDatabase.queryActioner(con,'select tag from genometags where tagid='+str(tags))
string1=''
for i in moviesForTags[:-1]:
string1+=str(i[0])+' or movieid='
string1+=str(moviesForTags[-1][0])
actorsForTags=loadDatabase.queryActioner(con,'select distinct(actorid) from movieactor where movieid='+string1)
idf=math.log(noOfActors[0][0]/(len(actorsForTags)))/math.log(2)
actorFinalVectortf[tagname[0]]=actorVector[tags]
actorFinalVectoridf[tagname[0]]=actorVector[tags]*idf
#final sorting according to score
if model==0:
sorted_tf = sorted(actorFinalVectortf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_tf
else:
sorted_idf = sorted(actorFinalVectoridf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_idf
def print_genre_vector(genre,model):
if model>1 or model<0:
print "please enter model number 0 or 1"
exit()
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME);
actualResult = {}
recentTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp desc')[0]#the most recent tag from database
oldTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp asc')[0]#the oldest tag from database
# converting to milliseconds
recentTag = int(datetime.strptime(str(recentTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
oldTag = int(datetime.strptime(str(oldTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
movies = loadDatabase.queryActioner(con, 'select * from mlmovies where genre like \'%' + genre+'%\'')
totalMovies=loadDatabase.queryActioner(con,'select * from mlmovies')
noOfGenres={}
finalGenreVectortf={}
finalGenreVectoridf={}
totalcount = 0
# this block calculates tf for all the tags and maintains it in dictionary actualResult
for movie in totalMovies:
genres = loadDatabase.queryActioner(con, 'select genre from mlmovies where movieid=' + str(movie[0]))[0][0].split('|')
for i in genres:
noOfGenres[i]=True
for movie in movies:
tags = loadDatabase.queryActioner(con, 'select * from mltags where movieid=' + str(movie[0]))
for tag in tags:
if(tag[2] in actualResult):
actualResult[tag[2]]=(actualResult[tag[2]][0]+1,actualResult[tag[2]][1]+(float(int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
else:
actualResult[tag[2]]=(1,float((int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
# this block calculates the idf for all the tags and multiplies with tf
for tag in actualResult.keys():
noOfIndividualGenres = {}
moviesForTags = loadDatabase.queryActioner(con, 'select distinct(movieid) from mltags where tagid=' + str(tag))
tagname = loadDatabase.queryActioner(con, 'select tag from genometags where tagid=' + str(tag))
string1 = ''
for i in moviesForTags[:-1]:
string1 += str(i[0]) + ' or movieid='
string1 += str(moviesForTags[-1][0])
genres= (loadDatabase.queryActioner(con,'select distinct(genre) from mlmovies where movieid=' + string1))
for g in genres:
for k in g[0].split('|'):
noOfIndividualGenres[k]=True
idf=math.log(len(noOfGenres.keys())/len(noOfIndividualGenres.keys()))/math.log(2)
finalGenreVectortf[tagname[0]]=(float(actualResult[tag][0])/totalcount)*actualResult[tag][1]
finalGenreVectoridf[tagname[0]]=(float(actualResult[tag][0])/totalcount)*actualResult[tag][1]*idf
# final sorting according to score
if model==0:
sorted_tf = sorted(finalGenreVectortf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_tf
else:
sorted_idf=sorted(finalGenreVectoridf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_idf
def differentiate_genre_tf_idf(genre1,genre2):
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME)
noOfMoviesBothGenre=loadDatabase.queryActioner(con,'select distinct(movieid) from mlmovies where genre like \'%'+genre1+'%\' or genre like \'%'+genre2+'%\'')#total number of movies in both the genres
totalTagsGenre1=loadDatabase.queryActioner(con,'select mltags.tagid from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 + '%\'')#tags in genre 1
movieVector={}
tags = loadDatabase.queryActioner(con, 'select distinct(mltags.tagid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 + '%\'')
for tag in tags:
tfnum=loadDatabase.queryActioner(con,'select mltags.tagid from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 + '%\' and tagid='+str(tag[0]))#Calculation of Numerator of TF
tf=float(len(tfnum))/len(totalTagsGenre1)#tf cacluation
idfden=loadDatabase.queryActioner(con,'select distinct(mltags.movieid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 +'%\' or genre like\'%'+genre2+'%\' and tagid='+str(tag[0]))#Calculation of denominator of IDF
idf=float(len(noOfMoviesBothGenre))/len(idfden)#idf calculation
tagname=loadDatabase.queryActioner(con, 'select tag from genometags where tagid=' + str(tag[0]))#selecting tags for getting tagname
movieVector[tagname[0]]=tf*math.log(idf)
sorted_x = sorted(movieVector.items(), key=operator.itemgetter(1), reverse=True)
print sorted_x
def differentiate_genre_pdiff1(genre1,genre2):
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME)
noOfMovies=loadDatabase.queryActioner(con,'select distinct(movieid) from mlmovies where genre like \'%'+genre1+'%\' or genre like \'%'+genre2+'%\'')#total number of movies in both the genres
noOfMoviesGenre1=loadDatabase.queryActioner(con,'select distinct(movieid) from mlmovies where genre like \'%'+genre1+'%\'')#movies in genre 1
tags = loadDatabase.queryActioner(con, 'select distinct(mltags.tagid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 + '%\'')#tags from genre1
rj={}
mj={}
finalVector={}
for tag in tags:
rj[tag[0]] = len(loadDatabase.queryActioner(con,'select distinct(mlmovies.movieid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like\'%' + genre1 + '%\' and tagid=' + str(tag[0])))#calculating r1,j
mj[tag[0]] = len(loadDatabase.queryActioner(con,'select distinct (mlmovies.movieid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%' + genre1 + '%\' or genre like \'%' + genre2 + '%\' and tagid=' + str(tag[0])))#calculating m1,j
tagname = loadDatabase.queryActioner(con, 'select tag from genometags where tagid=' + str(tag[0]))#getting name of the tag
firstTermnum = (float(rj[tag[0]]) / (len(noOfMoviesGenre1) - rj[tag[0]]))
firstTermDenom = (float(mj[tag[0]] - rj[tag[0]]) / abs(len(noOfMovies) - mj[tag[0]] - len(noOfMoviesGenre1) + rj[tag[0]]))
secondTerm = abs((float(rj[tag[0]]) / len(noOfMoviesGenre1)) - (float(mj[tag[0]] - rj[tag[0]]) / (len(noOfMovies) - len(noOfMoviesGenre1))))
finalVector[tagname[0]] = (math.log(1+firstTermnum / firstTermDenom)) * secondTerm
sorted_x = sorted(finalVector.items(), key=operator.itemgetter(1), reverse=True)#sorting finally
print sorted_x
def differentiate_genre_pdiff2(genre1,genre2):
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME)
totalMovies=loadDatabase.queryActioner(con,'select distinct(movieid) from mlmovies where genre like\'%'+genre1+'%\''' or genre like\'%'+genre2+'%\'')#total number of movies
totalMoviesGenre2=loadDatabase.queryActioner(con,'select distinct(movieid) from mlmovies where genre like\'%'+genre2+'%\'')#total number of movies in genre 2
rj={}
mj={}
finalVector={}
tags=loadDatabase.queryActioner(con,'select distinct(mltags.tagid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%'+genre1+'%\'')#getting tags from genre1
for tag in tags:
rj[tag[0]]=len(loadDatabase.queryActioner(con,'select distinct(mlmovies.movieid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like\'%'+genre2+'%\' and tagid!='+str(tag[0])))
mj[tag[0]]=len(loadDatabase.queryActioner(con,'select distinct (mlmovies.movieid) from mlmovies inner join mltags on (mltags.movieid = mlmovies.movieid) where genre like \'%'+genre1 +'%\' or genre like \'%'+ genre2+'%\' and tagid!='+str(tag[0])))
for tag in rj.keys():
tagname=loadDatabase.queryActioner(con, 'select tag from genometags where tagid=' + str(tag))#getting name of tag
FirstTermnum=(float(rj[tag])/(len(totalMoviesGenre2)-rj[tag]))
denominatorFirstTerm=(float(mj[tag]-rj[tag])/(len(totalMovies)-mj[tag]-len(totalMoviesGenre2)+rj[tag]))
secondTerm=abs((float(rj[tag])/len(totalMoviesGenre2))-(float(mj[tag]-rj[tag])/(len(totalMovies)-len(totalMoviesGenre2))))
finalVector[tagname[0]]=math.log(FirstTermnum/denominatorFirstTerm)*secondTerm
sorted_x = sorted(finalVector.items(), key=operator.itemgetter(1), reverse=True)
print sorted_x
def print_user_vector(userid,model):
if model>1 or model<0:
print "please enter model number 0 or 1"
exit()
con = loadDatabase.getOpenConnection('postgres', PASSWORD, DATABASE_NAME)
actualResult = {}
userVectortf = {}
userVectoridf={}
recentTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp desc')[0]#the most recent tag from database
oldTag = loadDatabase.queryActioner(con, 'select timestamp from mltags order by timestamp asc')[0]#the oldest tag from database
# converting to milliseconds
recentTag = int(datetime.strptime(str(recentTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
oldTag = int(datetime.strptime(str(oldTag[0]), '%Y-%m-%d %H:%M:%S').strftime('%s'))
tags = loadDatabase.queryActioner(con, 'select * from mltags where userid='+str(userid))
noOfUsers=loadDatabase.queryActioner(con,'select count(*) from mlusers')
totalcount=0
# this block calculates tf for all the tags and maintains it in dictionary actorVector
for tag in tags:
if(tag[2] in actualResult):
actualResult[tag[2]]=(actualResult[tag[2]][0]+1,actualResult[tag[2]][1]+(float(int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
else:
actualResult[tag[2]]=(1,float((int(datetime.strptime(str(tag[3]),'%Y-%m-%d %H:%M:%S').strftime('%s')))-oldTag)/(recentTag-oldTag))
totalcount+=1
# this block calculates the idf for all the tags and multiplies with tf
for tag in actualResult.keys():
tagname = loadDatabase.queryActioner(con, 'select tag from genometags where tagid=' + str(tag))
usersForTags = len(loadDatabase.queryActioner(con, 'select distinct(userid) from mltags where tagid=' + str(tag)))
idf=math.log(noOfUsers[0][0]/usersForTags)/math.log(2)
userVectortf[tagname[0]] = (float(actualResult[tag][0]) / totalcount) * actualResult[tag][1]
userVectoridf[tagname[0]]=(float(actualResult[tag][0])/totalcount)*actualResult[tag][1]*idf
# final sorting according to score
if model==0:
sorted_tf = sorted(userVectortf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_tf
else:
sorted_idf = sorted(userVectoridf.items(), key=operator.itemgetter(1), reverse=True)
print sorted_idf
def task1_b(genre, model):
con = MySQLdb.connect(user='******',
passwd='haha123',
host='127.0.0.1',
db="mwdb")
actornames = loadDatabase.queryActioner(con,
'select * from imdb_actor_info')
actor_actornames = {}
for i in actornames:
actor_actornames[i[0]] = i[1]
movies = loadDatabase.queryActioner(
con, 'select distinct(movieid) from mlmovies where genres like \'%' +
genre + '%\'')
actors = loadDatabase.queryActioner(
con,
'select distinct(movie_actor.actorid) from movie_actor inner join mlmovies on (movie_actor.movieid = mlmovies.movieid) where genres like \'%'
+ genre + '%\'')
finalarray = []
if model == 0 or model == 1:
for movie in movies:
movieList = []
noOfActors = loadDatabase.queryActioner(
con, 'select count(actorid) from movie_actor where movieid =' +
str(movie[0]))[0][0]
for actor in actors:
#print float((loadDatabase.queryActioner(con, 'select count(*) from mltags where tagid ='+str(tag[0])+' and movieid='+str(movie[0])))[0][0])
idf = float(len(movies)) / (loadDatabase.queryActioner(
con,
'select count(distinct(movieid)) from movie_actor where actorid ='
+ str(actor[0])))[0][0]
movieList.append((float((loadDatabase.queryActioner(
con, 'select count(*) from movie_actor where actorid =' +
str(actor[0]) + ' and movieid=' + str(movie[0])))[0][0]) /
noOfActors) * idf)
finalarray.append(movieList)
x = numpy.array(finalarray)
x = numpy.matrix.transpose(x)
if model == 1:
x = numpy.cov(x)
U, s, V = numpy.linalg.svd(x, full_matrices=True)
latent_semantics = U[:, 0:3]
transpose = numpy.matrix.transpose(latent_semantics)
for (key, topic) in enumerate(transpose):
string = 'topic' + str(key) + '='
sorted_actors = numpy.argsort(transpose[key])[::-1]
for actor in sorted_actors:
string += actor_actornames[actors[actor][0]] + '*' + str(
transpose[key][actor])
print string
print '--------------'
if model == 2:
for movie in movies:
movieList = []
for actor in actors:
#print float((loadDatabase.queryActioner(con, 'select count(*) from mltags where tagid ='+str(tag[0])+' and movieid='+str(movie[0])))[0][0])
#idf=float(len(movies))/(loadDatabase.queryActioner(con, 'select count(distinct(movieid)) from movieactor where actorid ='+str(actor[0])))[0][0]
movieList.append(
loadDatabase.queryActioner(
con,
'select count(*) from movie_actor where actorid =' +
str(actor[0]) + ' and movieid=' + str(movie[0]))[0][0])
finalarray.append(movieList)
#print (finalarray)
x = numpy.array(finalarray)
#x = numpy.matrix.transpose(x)
model = lda.LDA(n_topics=4, n_iter=100, random_state=1)
model.fit(x)
topic_word = model.topic_word_
string = ''
for key, topic in enumerate(topic_word):
string = str([
actor_actornames[actors[i][0]]
for i in numpy.argsort(topic)[::-1]
])
print 'topic' + str(key) + string
def task1_a(genre, model):
con = MySQLdb.connect(user='******',
passwd='haha123',
host='127.0.0.1',
db="mwdb")
tags = loadDatabase.queryActioner(
con, 'select distinct(tagid) from mltags'
) #collecting all the distinct tags from mltags
tagnames = loadDatabase.queryActioner(con, 'select * from genome')
tag_tagnames = {}
for i in tagnames:
tag_tagnames[i[0]] = i[
1] #maintaining a dictionary for tag to tagname reference
movies = loadDatabase.queryActioner(
con, 'select distinct(movieid) from mlmovies where genres like \'%' +
genre + '%\'') #selecting all the movies with the particular genre
finalarray = [] #this is the final movie to tag tf-idf score vector
#
#We are looping over each movies and making a movie to tag tf-idf score matrix.
#if there are no tags for the movies then the score is zero.
#
if model == 1 or model == 0:
for movie in movies:
movieList = []
noOfTags = loadDatabase.queryActioner(
con, 'select count(tagid) from mltags where movieid =' +
str(movie[0]))[0][0]
if noOfTags == 0:
finalarray.append([0.0 for tag in tags])
else:
for tag in tags:
#print float((loadDatabase.queryActioner(con, 'select count(*) from mltags where tagid ='+str(tag[0])+' and movieid='+str(movie[0])))[0][0])
idf = float(len(movies)) / (loadDatabase.queryActioner(
con,
'select count(distinct(movieid)) from mltags where tagid ='
+ str(tag[0])))[0][0]
movieList.append((float((loadDatabase.queryActioner(
con, 'select count(*) from mltags where tagid =' +
str(tag[0]) + ' and movieid=' + str(movie[0])))[0][0])
/ noOfTags) * idf)
finalarray.append(movieList)
x = numpy.array(finalarray)
x = numpy.matrix.transpose(x) #dimensionality reduction on x
if model == 1:
x = numpy.cov(x) #covariance of x
U, s, V = numpy.linalg.svd(x, full_matrices=True)
latent_semantics = U[:, 0:3]
transpose = numpy.matrix.transpose(latent_semantics)
for (key, topic) in enumerate(transpose):
string = 'topic' + str(key) + '='
sorted_tags = numpy.argsort(transpose[key])[::-1]
for tag in sorted_tags:
string += tag_tagnames[tags[tag][0]] + '*' + str(
transpose[key][tag])
print string
print '--------------'
if model == 2:
for movie in movies:
movieList = []
for tag in tags:
movieList.append(
loadDatabase.queryActioner(
con, 'select count(*) from mltags where tagid=' +
str(tag[0]) + ' and movieid=' + str(movie[0]))[0][0])
finalarray.append(movieList)
x = numpy.array(finalarray)
model = lda.LDA(n_topics=4, n_iter=100, random_state=1)
model.fit(x)
topic_word = model.topic_word_
for key, topic in enumerate(topic_word):
string = str([[tag_tagnames[tags[i][0]]]
for i in numpy.argsort(topic)[::-1]])
print 'topic' + str(key) + string