def session_factory(options='',output_buffer_size=8096):
system = platform.system()
path = None
if (system == 'Linux') or (system == 'Darwin'):
# find the MATLAB-root path:
locations = os.environ.get("PATH").split(os.pathsep)
for location in locations:
candidate = os.path.join(location, 'matlab')
if os.path.isfile(candidate):
path = candidate
break
executable = os.path.realpath(path)
basedir = os.path.dirname(os.path.dirname(executable))
exec_and_options = " ".join([executable,options])
session = MatlabSession(basedir,exec_and_options,output_buffer_size)
elif system =='Windows':
locations = os.environ.get("PATH").split(os.pathsep)
for location in locations:
candidate = os.path.join(location, 'matlab.exe')
if os.path.isfile(candidate):
path = candidate
break
executable = os.path.realpath(path)
basedir = os.path.dirname(os.path.dirname(executable))
session = MatlabSession(path=basedir,bufsize=output_buffer_size)
else:
raise NotSupportedException(
'Not supported on the {platform}-platform'.format(
platform=system))
return session
def readConstantly(self):
if enableEyeCircuit:
from pymatlab.matlab import MatlabSession
import time
initialized = 0
if enableEyeCircuit:
import os
print("Opening MATLAB session...")
self.session = MatlabSession('matlab -nojvm -nodisplay')
print("Attempting to connect to DAQ...")
self.session.run("cd " + os.path.join(os.getcwd(), "daq"))
self.session.run("Init") #Initializes DAQ and starts recording
initialized = int(self.session.getvalue("Init_initialized"))
if initialized == 0:
print(
"DAQ Initialization failed. Ensure that the DAQ software is installed and try closing all open MATLAB Processes."
)
self.deactivate()
else:
print("Connected to DAQ")
self.readyState.value = 1
print("Calibrating Eye Circuit...")
self.calibrate()
self.session.run("oldVal = 0")
while self.activeState():
if enableEyeCircuit:
self.session.run(
"signal = monitor(Init_NI,Init_sampleRate,nMin,cLine,nMax,oldVal);"
)
self.session.run("oldVal = signal;")
self.direction.value = int(self.session.getvalue("signal"))
time.sleep(0.01)
self.count += 1
if self.count > 100:
self.session.run("refreshDevice(Init_NI);")
self.count = 0
if enableEyeCircuit:
print("Closing MATLAB Session...")
if initialized == 1:
self.session.run("Cleanup;")
self.session.close()
print("MATLAB Session Closed!")
def start(self,user,session):
""" inits the session with matlab
"""
self.user=user
self.sessionId=session
self.logger.debug("Starting matlab session %s",BCIConn.MATLAB_STR)
self.session = MatlabSession(BCIConn.MATLAB_STR)
self.logger.info("Matlab session started")
self.session.putvalue('user',self.user)
self.session.putvalue('session',self.sessionId)
self.session.run(BCIConn.INIT)
self.logger.debug("Matlab brainz object init with user %i and session %i",self.user,self.sessionId)
def matlabanovan(y,group,**args):
from pymatlab.matlab import MatlabSession
session = MatlabSession()
session.close()
session = MatlabSession('matlab -nojvm -nodisplay')
session.PutValue('y',y)
session.PutValue('group', group)
session.PutValue('varargin' **args)
session.run('anovan(y, group, varargin)')
def readConstantly(self):
if enableEyeCircuit:
from pymatlab.matlab import MatlabSession
import time
initialized = 0
if enableEyeCircuit:
import os
print("Opening MATLAB session...")
self.session = MatlabSession('matlab -nojvm -nodisplay')
print("Attempting to connect to DAQ...")
self.session.run("cd "+os.path.join(os.getcwd(),"daq"))
self.session.run("Init") #Initializes DAQ and starts recording
initialized = int(self.session.getvalue("Init_initialized"))
if initialized == 0:
print("DAQ Initialization failed. Ensure that the DAQ software is installed and try closing all open MATLAB Processes.")
self.deactivate()
else:
print("Connected to DAQ")
self.readyState.value = 1
print("Calibrating Eye Circuit...")
self.calibrate()
self.session.run("oldVal = 0")
while self.activeState():
if enableEyeCircuit:
self.session.run("signal = monitor(Init_NI,Init_sampleRate,nMin,cLine,nMax,oldVal);")
self.session.run("oldVal = signal;")
self.direction.value = int(self.session.getvalue("signal"))
time.sleep(0.01)
self.count += 1
if self.count > 100:
self.session.run("refreshDevice(Init_NI);")
self.count = 0
if enableEyeCircuit:
print("Closing MATLAB Session...")
if initialized == 1:
self.session.run("Cleanup;")
self.session.close()
print("MATLAB Session Closed!")
def remote_session_factory(hostname,remote_path):
system = platform.system()
path = None
if (system == 'Linux') or (system == 'Darwin'):
locations = os.environ.get("PATH").split(os.pathsep)
for location in locations:
candidate = os.path.join(location, 'matlab')
if os.path.isfile(candidate):
path = candidate
break
executable = os.path.realpath(path)
basedir = os.path.dirname(os.path.dirname(executable))
session = MatlabSession(
basedir,
"ssh {host} '/bin/csh -c {full_path}'".format(
host=hostname,
full_path=remote_path)
)
else:
raise NotSupportedException(
'Not supported on the {platform}-platform'.format(
platform=system))
return session
def _start_matlab(self):
self.m = MatlabSession('/usr/local/MATLAB/R2011a/bin/matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('"+os.path.abspath(self.algopath)+"'))")
class Whisperer(object):
savepath = SAVEPATH
algopath = ALGOPATH
def _start_matlab(self):
self.m = MatlabSession('/usr/local/MATLAB/R2011a/bin/matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('"+os.path.abspath(self.algopath)+"'))")
def __init__(self,db,im=None):
self.m = None
self.db = db
self.im = im
def _put(self, name, value):
self.m.putvalue(name, value)
def _run(self, command):
self.m.run(command)
def _get(self, name):
return self.m.getvalue(name)
def _close_matlab(self):
if self.m:
self.m.close()
self.m = None
@matlab
def create_titles_vector(self):
db = self.db
db_ratings = db(db.ratings).select(db.ratings.ALL, distinct=db.ratings.imovie).sort(lambda x: x.imovie)
titles = None
if self.im:
titles = list()
for x in db_ratings:
movie = x.imovie
titles.append(movie.title)
if titles:
self._run("titles = cell({},1)".format(len(titles)))
for i, title in enumerate(titles):
if title is None:
title = 'None'
title = title.replace("'", "''")
self._run("titles{{{0}}}='{1}'".format(i+1, title))
self._run("save('"+os.path.join(self.savepath, 'titles')+"', 'titles')")
@matlab
def create_features_vector(self):
db = self.db
db_features = db(db.movies_features).select(db.movies_features.ALL, distinct=db.movies_features.feature).sort(lambda x: x.feature)
features_vector = list()
for x in db_features:
features_vector.append(x.feature.name)
self._run("features = cell({},1)".format(len(features_vector)))
for i, feature in enumerate(features_vector):
feature_escaped = feature.replace("'", "''")
self._run("features{{{0}}}='{1}'".format(i+1, feature_escaped))
self._run("save('"+os.path.join(self.savepath, 'features')+"', 'features')")
@matlab
def create_matlab_matrices(self, type=None, force=False, *args, **vars):
c_icm = True
c_urm = True
urm_filepath = os.path.join(self.savepath, 'urm.mat')
icm_filepath = os.path.join(self.savepath, 'icm.mat')
try:
urm_date = datetime.datetime.fromtimestamp(os.path.getctime(urm_filepath))
except OSError:
urm_date = None
try:
icm_date = datetime.datetime.fromtimestamp(os.path.getctime(icm_filepath))
except OSError:
icm_date = None
if type == 'urm':
c_icm = False
elif type == 'icm':
c_urm = False
if c_urm:
filepath = os.path.join(self.savepath, 'urm.mat')
if not urm_date or force:
users, movies, ratings, urm_dimensions = self._create_urm()
self._put('urm_users', users)
self._put('urm_movies', movies)
self._put('urm_ratings', ratings)
self._put('urm_dimensions', urm_dimensions)
self._run("urm = sparse(urm_users, urm_movies, urm_ratings, urm_dimensions(1), urm_dimensions(2))")
self._run("save('"+os.path.join(self.savepath, 'urm')+"', 'urm')")
self._run("clear")
if c_icm:
filepath = os.path.join(self.savepath, 'icm.mat')
if not icm_date or force:
icm_items, icm_features, icm_occurrencies, icm_dimensions = self._create_icm()
self._put('icm_items', icm_items)
self._put('icm_features', icm_features)
self._put('icm_occurrencies', icm_occurrencies)
self._put('icm_dimensions', icm_dimensions)
self._run("icm = sparse(icm_items, icm_features, icm_occurrencies, icm_dimensions(1), icm_dimensions(2))")
self._run("save('"+os.path.join(self.savepath, 'icm')+"', 'icm')")
self._run("clear")
def _create_urm(self, users=None, items=None, ratings=None):
"""Return the user rating matrix"""
db = self.db
db_ratings = db(db.ratings).select(db.ratings.ALL, distinct=True)
users = [float(x.iuser) for x in db_ratings]
movies = [float(x.imovie) for x in db_ratings]
ratings = [round(x.rating * 5, 0) for x in db_ratings]
users = numpy.array(users)
movies = numpy.array(movies)
ratings = numpy.array(ratings)
max_user = db.ratings.iuser.max()
max_movies = db.ratings.imovie.max()
dimensions = [float(db().select(max_user).first()[max_user]), float(db().select(max_movies).first()[max_movies])]
dimensions = numpy.array(dimensions)
return users, movies, ratings, dimensions
def _create_icm(self, items=None, metadata=None):
"""Returns the item content matrix"""
db = self.db
if not items and not metadata:
entries = db(db.movies_features).select(db.movies_features.ALL, distinct=True)
items = [float(x.movie.id) for x in entries]
features = [float(x.feature.id) for x in entries]
occurrencies = [float(x.times) for x in entries]
max_movies = db.movies_features.movie.max()
max_features = db.movies_features.feature.max()
dimensions = numpy.array([float(db().select(max_movies).first()[max_movies]), float(db().select(max_features).first()[max_features])])
return numpy.array(items), numpy.array(features), numpy.array(occurrencies), dimensions
def create_userprofile(self, user):
db = self.db
entries = db(db.ratings.iuser==user).select(db.ratings.ALL, cacheable=True)
ratings = [round(x.rating * 5, 0) for x in entries]
movies = [float(x.imovie) for x in entries]
max_movies = db.ratings.imovie.max()
dimension = [float(db().select(max_movies).first()[max_movies])]
# up = numpy.zeros((1,self.db.query(func.max(Item.id)).one()[0]))
# for r in ratings:
# up[0][r.item.id-1] = r.rating
return numpy.array(ratings), numpy.array(movies), numpy.array(dimension)
@matlab
def _create_model(self, alg, param=None, *args, **kwargs):
self._run("load('"+os.path.join(self.savepath, 'urm.mat')+"')")
self._run("load('"+os.path.join(self.savepath, 'icm.mat')+"')")
self._run("param = struct()")
if param:
for k,v in param.iteritems():
self._run("param."+str(k)+" = "+str(v))
self._run("["+alg+"_model] = createModel_"+alg+"(urm, icm, param)")
self._run("save('"+os.path.join(self.savepath, alg+'_model')+"', '"+alg+"_model')")
def create_model(self, algname='AsySVD'):
"""Create a model and saves it the ALGORITHMS/saved directory"""
#function [model] = createModel_ALGORITHM_NAME(URM,ICM,param)
db = self.db
alg = self._get_model_name(algname)
self.create_matlab_matrices(force=True)
self._create_model(alg)
@matlab
def _get_rec(self, algname, user, **param):
"""Return a recommendation using the matlab engine"""
#function [recomList] = onLineRecom_ALGORITHM_NAME (userProfile, model,param)
ratings, movies, dimension = self.create_userprofile(user)
alg = self._get_model_name(algname)
self._put('ratings', ratings)
self._put('movies', movies)
self._put('dimension', dimension)
self._run("up = sparse(1, movies, ratings, 1, dimension(1))")
self._run("param = struct()")
for k,v in param.iteritems():
self._run("param."+str(k)+" = "+str(v))
self._run("load('"+os.path.join(self.savepath, alg+'_model')+"', '"+alg+"_model')")
self._run("[rec] = onLineRecom_"+algname+"(up, "+alg+"_model, param)")
return self._get("rec")
def get_rec(self, algname, user, max=10, **param):
"""Wrapper aroung the real recommendation getter to set parameters"""
if algname == 'AsySVD':
param = dict(param, userToTest=user)
rec = self._get_rec(algname, user, **param)
rec = numpy.squeeze(rec)
indexes = bottleneck.argpartsort(rec, rec.size-max, axis=0)[-max:]
rec = [(rec[index], index) for index in indexes]
rec.sort(reverse=True)
return rec
@classmethod
def get_algnames(self):
"""Return a list of algorithms in the system"""
algs = list()
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
algs.append(f[12:-2])
algs.sort()
return algs
@classmethod
def _get_model_name(self, algname=None):
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
if f[12:-2] == algname:
for mf in files:
if mf.startswith('createModel'):
return mf[12:-2]
return None
@classmethod
def get_models_info(self):
"""Return a dict of algorithms in which there is a model created and the time the model was created"""
algnames = self.get_algnames()
algs = dict()
for root, dirs, files in os.walk(self.savepath):
for f in files:
for algname in algnames:
if f[:-10] in self._get_model_name(algname) and f[:-10]!='':
algo =f[:-10]
path = os.path.join(root, f)
algs[algname] = datetime.datetime.fromtimestamp(os.path.getmtime(path))
return algs
@classmethod
def get_matrices_info(cls):
matrices = dict()
for matrice in ALL_MATRICES:
try:
matrices[matrice] = datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(cls.savepath, matrice+'.mat')))
except:
matrices[matrice] = None
return matrices
@classmethod
def get_matrices_path(cls):
matrices = dict()
for matrice in ALL_MATRICES:
try:
matrices[matrice] = os.path.join(cls.savepath, matrice+'.mat')
except:
matrices[matrice] = None
return matrices
@classmethod
def delete_matrice(cls, matrice):
matrices_path = cls.get_matrices_path()
os.remove(matrices_path[matrice])
def _start_matlab(self):
self.m = MatlabSession('matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('"+os.path.abspath(self.algopath)+"'))")
class Whisperer(object):
savepath = config.SAVEPATH
algopath = config.ALGOPATH
def _start_matlab(self):
self.m = MatlabSession('matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('"+os.path.abspath(self.algopath)+"'))")
def __init__(self):
self.m = None
self.db = Session()
def _put(self, name, value):
self.m.putvalue(name, value)
def _run(self, command):
self.m.run(command)
def _get(self, name):
return self.m.getvalue(name)
def _close_matlab(self):
if self.m:
self.m.close()
self.m = None
def create_urm(self, users=None, items=None, ratings=None):
"""Return the user rating matrix"""
if not users:
users = self.db.query(User).all()
if not items:
items = self.db.query(Item).all()
if not ratings:
ratings = self.db.query(Rating).all()
urm = numpy.zeros((self.db.query(func.max(User.id)).one()[0],self.db.query(func.max(Item.id)).one()[0]))
for r in ratings:
urm[r.user_id-1][r.item_id-1] = r.rating
return urm
def create_icm(self, items=None, metadatas=None):
"""Returns the item content matrix"""
if not items:
items = self.db.query(Item).all()
if not metadatas:
metadatas = self.db.query(Metadata).all()
icm = numpy.zeros((self.db.query(func.max(Metadata.id)).one()[0],self.db.query(func.max(Item.id)).one()[0]))
for i in items:
for m in i.metadatas:
icm[m.id-1][i.id-1] = 1
return icm
def create_userprofile(self, user, items=None):
if not items:
items = self.db.query(Item).all()
ratings = self.db.query(Rating).all()
up = numpy.zeros((1,self.db.query(func.max(Item.id)).one()[0]))
for r in ratings:
up[0][r.item.id-1] = r.rating
return up
@matlab
def create_model(self, algname, urm=None, icm=None):
"""Create a model and saves it the ALGORITHMS/saved directory"""
#function [model] = createModel_ALGORITHM_NAME(URM,ICM,param)
if not urm:
urm = self.create_urm()
if not icm:
icm = self.create_icm()
alg = self._get_model_name(algname)
self._put('urm', urm)
self._put('icm', icm)
self._run("["+alg+"_model] = createModel_"+alg+"(urm, icm)")
self._run("save('"+os.path.join(self.savepath, alg+'_model')+"', '"+alg+"_model')")
@matlab
def _get_rec(self, algname, user, **param):
"""Return a recommendation using the matlab engine"""
#function [recomList] = onLineRecom_ALGORITHM_NAME (userProfile, model,param)
up = self.create_userprofile(user)
alg = self._get_model_name(algname)
self._put('up', up)
self._run("param = struct()")
for k,v in param.iteritems():
self._run("param."+str(k)+" = "+str(v))
self._run("load('"+os.path.join(self.savepath, alg+'_model')+"', '"+alg+"_model')")
self._run("[rec] = onLineRecom_"+algname+"(up, "+alg+"_model, param)")
return self._get("rec")
def get_rec(self, algname, user, **param):
"""Wrapper aroung the real recommendation getter to set parameters"""
if algname == 'AsySVD':
param = dict(param, userToTest=user.id)
return self._get_rec(algname, user, **param)
@classmethod
def get_algnames(self):
"""Return a list of algorithms in the system"""
algs = list()
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
algs.append(f[12:-2])
algs.sort()
return algs
@classmethod
def _get_model_name(self, algname=None):
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
if f[12:-2] == algname:
for mf in files:
if mf.startswith('createModel'):
return mf[12:-2]
return None
@classmethod
def get_models_info(self):
"""Return a dict of algorithms in which there is a model created and the time the model was created"""
algnames = self.get_algnames()
algs = dict()
for root, dirs, files in os.walk(self.savepath):
for f in files:
for algname in algnames:
if f[:-10] in self._get_model_name(algname):
algo =f[:-10]
path = os.path.join(root, f)
algs[algname] = datetime.datetime.fromtimestamp(os.path.getmtime(path))
return algs
@matlab
def load_urm(self):
self._run("load('urmFull.mat')")
self._run("A=full(urm)")
urm = self._get("A")
#force matlab to close and free memory
self._close_matlab()
print 'Out of matlab!'
for (row,col),value in numpy.ndenumerate(urm):
print 'processing rating %i for user %i and item %i' % (value, row, col)
if value is not 0:
self.db.add(Rating(user_id=row+1, item_id=col+1, rating=value))
self.db.flush()
self.db.commit()
#for i,row in enumerate(urm):
# print 'processing row: %s' % (i)
# self.db.add(User(name="netflix%s" % (i)))
# self.db.flush()
#urm[r.user_id-1][r.item_id-1]
@matlab
def load_titles(self, filename='/tmp/movie_titles.txt'):
self._run("load('titles.mat')")
titles = self._get('titles')
l = list()
for row in titles:
l.append(''.join(row).rstrip())
f = open(filename, 'w')
for item in l:
f.write("%s\n" % item)
f.close()
return l
def do_something(self):
print 'URM'
print self.create_urm()
print 'ICM'
print self.create_icm()
print 'run AsySVD'
#print self.create_model('random')
#print self.create_model('cosineIIknn')
#print 'get rec'
print self.get_rec('random', self.db.query(User).filter(User.id==9).first())
def _start_matlab(self):
self.m = MatlabSession(
'/usr/local/MATLAB/R2011a/bin/matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('" + os.path.abspath(self.algopath) +
"'))")
class Whisperer(object):
savepath = SAVEPATH
algopath = ALGOPATH
def _start_matlab(self):
self.m = MatlabSession(
'/usr/local/MATLAB/R2011a/bin/matlab -nosplash -nodisplay')
self.m.run("addpath(genpath('" + os.path.abspath(self.algopath) +
"'))")
def __init__(self, db, im=None):
self.m = None
self.db = db
self.im = im
def _put(self, name, value):
self.m.putvalue(name, value)
def _run(self, command):
self.m.run(command)
def _get(self, name):
return self.m.getvalue(name)
def _close_matlab(self):
if self.m:
self.m.close()
self.m = None
@matlab
def create_titles_vector(self):
db = self.db
db_ratings = db(db.ratings).select(
db.ratings.ALL,
distinct=db.ratings.imovie).sort(lambda x: x.imovie)
titles = None
if self.im:
titles = list()
for x in db_ratings:
movie = x.imovie
titles.append(movie.title)
if titles:
self._run("titles = cell({},1)".format(len(titles)))
for i, title in enumerate(titles):
if title is None:
title = 'None'
title = title.replace("'", "''")
self._run("titles{{{0}}}='{1}'".format(i + 1, title))
self._run("save('" + os.path.join(self.savepath, 'titles') +
"', 'titles')")
@matlab
def create_features_vector(self):
db = self.db
db_features = db(db.movies_features).select(
db.movies_features.ALL,
distinct=db.movies_features.feature).sort(lambda x: x.feature)
features_vector = list()
for x in db_features:
features_vector.append(x.feature.name)
self._run("features = cell({},1)".format(len(features_vector)))
for i, feature in enumerate(features_vector):
feature_escaped = feature.replace("'", "''")
self._run("features{{{0}}}='{1}'".format(i + 1, feature_escaped))
self._run("save('" + os.path.join(self.savepath, 'features') +
"', 'features')")
@matlab
def create_matlab_matrices(self, type=None, force=False, *args, **vars):
c_icm = True
c_urm = True
urm_filepath = os.path.join(self.savepath, 'urm.mat')
icm_filepath = os.path.join(self.savepath, 'icm.mat')
try:
urm_date = datetime.datetime.fromtimestamp(
os.path.getctime(urm_filepath))
except OSError:
urm_date = None
try:
icm_date = datetime.datetime.fromtimestamp(
os.path.getctime(icm_filepath))
except OSError:
icm_date = None
if type == 'urm':
c_icm = False
elif type == 'icm':
c_urm = False
if c_urm:
filepath = os.path.join(self.savepath, 'urm.mat')
if not urm_date or force:
users, movies, ratings, urm_dimensions = self._create_urm()
self._put('urm_users', users)
self._put('urm_movies', movies)
self._put('urm_ratings', ratings)
self._put('urm_dimensions', urm_dimensions)
self._run(
"urm = sparse(urm_users, urm_movies, urm_ratings, urm_dimensions(1), urm_dimensions(2))"
)
self._run("save('" + os.path.join(self.savepath, 'urm') +
"', 'urm')")
self._run("clear")
if c_icm:
filepath = os.path.join(self.savepath, 'icm.mat')
if not icm_date or force:
icm_items, icm_features, icm_occurrencies, icm_dimensions = self._create_icm(
)
self._put('icm_items', icm_items)
self._put('icm_features', icm_features)
self._put('icm_occurrencies', icm_occurrencies)
self._put('icm_dimensions', icm_dimensions)
self._run(
"icm = sparse(icm_items, icm_features, icm_occurrencies, icm_dimensions(1), icm_dimensions(2))"
)
self._run("save('" + os.path.join(self.savepath, 'icm') +
"', 'icm')")
self._run("clear")
def _create_urm(self, users=None, items=None, ratings=None):
"""Return the user rating matrix"""
db = self.db
db_ratings = db(db.ratings).select(db.ratings.ALL, distinct=True)
users = [float(x.iuser) for x in db_ratings]
movies = [float(x.imovie) for x in db_ratings]
ratings = [round(x.rating * 5, 0) for x in db_ratings]
users = numpy.array(users)
movies = numpy.array(movies)
ratings = numpy.array(ratings)
max_user = db.ratings.iuser.max()
max_movies = db.ratings.imovie.max()
dimensions = [
float(db().select(max_user).first()[max_user]),
float(db().select(max_movies).first()[max_movies])
]
dimensions = numpy.array(dimensions)
return users, movies, ratings, dimensions
def _create_icm(self, items=None, metadata=None):
"""Returns the item content matrix"""
db = self.db
if not items and not metadata:
entries = db(db.movies_features).select(db.movies_features.ALL,
distinct=True)
items = [float(x.movie.id) for x in entries]
features = [float(x.feature.id) for x in entries]
occurrencies = [float(x.times) for x in entries]
max_movies = db.movies_features.movie.max()
max_features = db.movies_features.feature.max()
dimensions = numpy.array([
float(db().select(max_movies).first()[max_movies]),
float(db().select(max_features).first()[max_features])
])
return numpy.array(items), numpy.array(features), numpy.array(
occurrencies), dimensions
def create_userprofile(self, user):
db = self.db
entries = db(db.ratings.iuser == user).select(db.ratings.ALL,
cacheable=True)
ratings = [round(x.rating * 5, 0) for x in entries]
movies = [float(x.imovie) for x in entries]
max_movies = db.ratings.imovie.max()
dimension = [float(db().select(max_movies).first()[max_movies])]
# up = numpy.zeros((1,self.db.query(func.max(Item.id)).one()[0]))
# for r in ratings:
# up[0][r.item.id-1] = r.rating
return numpy.array(ratings), numpy.array(movies), numpy.array(
dimension)
@matlab
def _create_model(self, alg, param=None, *args, **kwargs):
self._run("load('" + os.path.join(self.savepath, 'urm.mat') + "')")
self._run("load('" + os.path.join(self.savepath, 'icm.mat') + "')")
self._run("param = struct()")
if param:
for k, v in param.iteritems():
self._run("param." + str(k) + " = " + str(v))
self._run("[" + alg + "_model] = createModel_" + alg +
"(urm, icm, param)")
self._run("save('" + os.path.join(self.savepath, alg + '_model') +
"', '" + alg + "_model')")
def create_model(self, algname='AsySVD'):
"""Create a model and saves it the ALGORITHMS/saved directory"""
#function [model] = createModel_ALGORITHM_NAME(URM,ICM,param)
db = self.db
alg = self._get_model_name(algname)
self.create_matlab_matrices(force=True)
self._create_model(alg)
@matlab
def _get_rec(self, algname, user, **param):
"""Return a recommendation using the matlab engine"""
#function [recomList] = onLineRecom_ALGORITHM_NAME (userProfile, model,param)
ratings, movies, dimension = self.create_userprofile(user)
alg = self._get_model_name(algname)
self._put('ratings', ratings)
self._put('movies', movies)
self._put('dimension', dimension)
self._run("up = sparse(1, movies, ratings, 1, dimension(1))")
self._run("param = struct()")
for k, v in param.iteritems():
self._run("param." + str(k) + " = " + str(v))
self._run("load('" + os.path.join(self.savepath, alg + '_model') +
"', '" + alg + "_model')")
self._run("[rec] = onLineRecom_" + algname + "(up, " + alg +
"_model, param)")
return self._get("rec")
def get_rec(self, algname, user, max=10, **param):
"""Wrapper aroung the real recommendation getter to set parameters"""
if algname == 'AsySVD':
param = dict(param, userToTest=user)
rec = self._get_rec(algname, user, **param)
rec = numpy.squeeze(rec)
indexes = bottleneck.argpartsort(rec, rec.size - max, axis=0)[-max:]
rec = [(rec[index], index) for index in indexes]
rec.sort(reverse=True)
return rec
@classmethod
def get_algnames(self):
"""Return a list of algorithms in the system"""
algs = list()
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
algs.append(f[12:-2])
algs.sort()
return algs
@classmethod
def _get_model_name(self, algname=None):
for root, dirs, files in os.walk(self.algopath):
for f in files:
if f.startswith('onLineRecom'):
if f[12:-2] == algname:
for mf in files:
if mf.startswith('createModel'):
return mf[12:-2]
return None
@classmethod
def get_models_info(self):
"""Return a dict of algorithms in which there is a model created and the time the model was created"""
algnames = self.get_algnames()
algs = dict()
for root, dirs, files in os.walk(self.savepath):
for f in files:
for algname in algnames:
if f[:-10] in self._get_model_name(
algname) and f[:-10] != '':
algo = f[:-10]
path = os.path.join(root, f)
algs[algname] = datetime.datetime.fromtimestamp(
os.path.getmtime(path))
return algs
@classmethod
def get_matrices_info(cls):
matrices = dict()
for matrice in ALL_MATRICES:
try:
matrices[matrice] = datetime.datetime.fromtimestamp(
os.path.getmtime(
os.path.join(cls.savepath, matrice + '.mat')))
except:
matrices[matrice] = None
return matrices
@classmethod
def get_matrices_path(cls):
matrices = dict()
for matrice in ALL_MATRICES:
try:
matrices[matrice] = os.path.join(cls.savepath,
matrice + '.mat')
except:
matrices[matrice] = None
return matrices
@classmethod
def delete_matrice(cls, matrice):
matrices_path = cls.get_matrices_path()
os.remove(matrices_path[matrice])
def get_contextual_matlab(conf,Wsz):
import platform
if platform.node() != 'g6':
dataPath = '/Users/aarslan/Brown/auto_context'
else:
dataPath = '/home/aarslan/prj/data/auto_context'
print 'computing context features for windows size: ', str(Wsz)
tic = time.time()
session = MatlabSession()
session.run('cd ', dataPath)
session.putvalue('conf',conf)
session.putvalue('Wsz',np.array([Wsz], dtype='float64'))
session.run('B = ctxtFeat(conf, Wsz)')
cf = session.getvalue('B')
session.close()
np.array(cf, dtype='uint8')
print "Context feature for ", str(Wsz), " size windows took ", round(time.time() - tic,2), "seconds"
return cf
def get_contextual_matlab(conf, Wsz):
import platform
if platform.node() != 'g6':
dataPath = '/Users/aarslan/Brown/auto_context'
else:
dataPath = '/home/aarslan/prj/data/auto_context'
print 'computing context features for windows size: ', str(Wsz)
tic = time.time()
session = MatlabSession()
session.run('cd ', dataPath)
session.putvalue('conf', conf)
session.putvalue('Wsz', np.array([Wsz], dtype='float64'))
session.run('B = ctxtFeat(conf, Wsz)')
cf = session.getvalue('B')
session.close()
np.array(cf, dtype='uint8')
print "Context feature for ", str(Wsz), " size windows took ", round(
time.time() - tic, 2), "seconds"
return cf
class BCIConn(object):
"""Connects wit matlab to process the signals and classification"""
MATLAB_STR="matlab -nojvm -nodisplay -logfile ~/tmp/logmat.txt"
INIT="brainz=Brainz(user,session);brainz.init();"
START_TRIAL="brainz.startTrial()"
ADD_DATA="c=brainz.addData(data)"
END_TRIAL="brainz.endTrial()"
def __init__(self):
"""@todo: to be defined """
self.logger=logging.getLogger("logger")
self.session=None
self.lock=threading.Lock()
self.onTrial=False
def start(self,user,session):
""" inits the session with matlab
"""
self.user=user
self.sessionId=session
self.logger.debug("Starting matlab session %s",BCIConn.MATLAB_STR)
self.session = MatlabSession(BCIConn.MATLAB_STR)
self.logger.info("Matlab session started")
self.session.putvalue('user',self.user)
self.session.putvalue('session',self.sessionId)
self.session.run(BCIConn.INIT)
self.logger.debug("Matlab brainz object init with user %i and session %i",self.user,self.sessionId)
def close(self ):
"""Closes the matlab session
"""
self.session.close()
self.logger.info("Matlab session closed")
def startTrial(self):
"""Sends the start trial signal to the matlab object
"""
self.lock.acquire();
self.logger.debug("Sending new trial")
res=self.session.run(BCIConn.START_TRIAL)
self.onTrial=True
print "On trial %s"%self.onTrial
if res!=None:
self.logger.error("Matlab complained %s"%res)
self.lock.release();
def endTrial(self):
"""Sends the start trial signal to the matlab object
"""
self.onTrial=False
self.lock.acquire();
self.logger.debug("Finish trial")
res=self.session.run(BCIConn.END_TRIAL)
if res!=None:
self.logger.error("Matlab complained %s"%res)
self.lock.release();
def append(self, data):
"""sends a matrix of data to matlab to process it
:data: 2-D matrix with the data
:class: the output of the clasification or nil
"""
self.logger.debug("Sending data to matlab")
print "On trial %s"%self.onTrial
# print "MATLAB %s"%threading.currentThread()
if not self.onTrial:
return -1
#self.lock.acquire();
time1=time.time()
data=numpy.array(data,dtype=numpy.float64)
self.session.putvalue('data',data)
res=self.session.run(BCIConn.ADD_DATA)
if res!=None:
self.logger.error("Matlab complained %s"%res)
c=self.session.getvalue('c')
##time.sleep(1)
time2=time.time()
self.logger.debug("done")
#self.logger.debug("Data classified as %i",c)
#self.lock.release();
return int(c)
