def learnModel(self, X):
learner = WRMFRecommender(self.k, self.alpha, self.lmbda, self.maxIterations)
learner.fit(X)
self.U = learner.U
self.V = learner.V
return self.U, self.V
def learnModel(self, X):
learner = WRMFRecommender(self.k, self.alpha, self.lmbda,
self.maxIterations)
learner.fit(X)
self.U = learner.U
self.V = learner.V
return self.U, self.V
def main():
import sys
from mrec import load_sparse_matrix, save_recommender
from mrec.mf.wrmf import WRMFRecommender
file_format = sys.argv[1]
filepath = sys.argv[2]
outfile = sys.argv[3]
# load training set as scipy sparse matrix
train = load_sparse_matrix(file_format, filepath)
model = WRMFRecommender(d=5)
model.fit(train)
save_recommender(model, outfile)
def run_mrec(d=10,num_iters=4,reg=0.02):
#d is dimension of subspace, i.e. groups
import sys
from mrec import load_sparse_matrix, save_recommender
from mrec.sparse import fast_sparse_matrix
from mrec.mf.wrmf import WRMFRecommender
alpha=1.0
start=time.time()
file_format = "csv"
#file shoule be csv, with: row,col,data
#data may just be ones
filepath = PARS['data_dir']+"/reduced_row_col_num_cutoff_1.5.csv"
#filepath = PARS['data_dir']+"test_10_mill.csv"
outfile = make_mrec_outfile(filepath,d,num_iters,reg)
print outfile
print 'reading file: %s'%filepath
# load training set as scipy sparse matrix
print "loading file"
train = load_sparse_matrix(file_format,filepath)
print "loaded file"
print (time.time()-start),"seconds"
print "size:",train.shape
print "creating recommender"
model = WRMFRecommender(d=d,num_iters=num_iters,alpha=alpha,lbda=reg)
print "training on data"
print time.time()-start
model.fit(train)
print "done training"
print time.time()-start
print "saving model"
save_recommender(model,outfile)
print "wrote model to: %s"%outfile
print time.time()-start
return
print "validating"
data,U,V=read_mrec(mrec_file=outfile)
plot_file=outfile.replace('.npz','.png')
multi_thresh(data,model,thresh_list=None,plot_file=plot_file)
run_time=(time.time()-start)/60.0
print "runtime: %0.3f minutes"%run_time
print 'done'
def main():
import sys
from mrec import load_sparse_matrix, save_recommender
from mrec.sparse import fast_sparse_matrix
from mrec.mf.wrmf import WRMFRecommender
file_format = sys.argv[1]
filepath = sys.argv[2]
outfile = sys.argv[3]
# load training set as scipy sparse matrix
train = load_sparse_matrix(file_format,filepath)
model = WRMFRecommender(d=5)
model.fit(train)
save_recommender(model,outfile)
def run_mrec(d=10, num_iters=4, reg=0.02):
#d is dimension of subspace, i.e. groups
import sys
from mrec import load_sparse_matrix, save_recommender
from mrec.sparse import fast_sparse_matrix
from mrec.mf.wrmf import WRMFRecommender
alpha = 1.0
start = time.time()
file_format = "csv"
#file shoule be csv, with: row,col,data
#data may just be ones
filepath = PARS['data_dir'] + "/reduced_row_col_num_cutoff_1.5.csv"
#filepath = PARS['data_dir']+"test_10_mill.csv"
outfile = make_mrec_outfile(filepath, d, num_iters, reg)
print outfile
print 'reading file: %s' % filepath
# load training set as scipy sparse matrix
print "loading file"
train = load_sparse_matrix(file_format, filepath)
print "loaded file"
print(time.time() - start), "seconds"
print "size:", train.shape
print "creating recommender"
model = WRMFRecommender(d=d, num_iters=num_iters, alpha=alpha, lbda=reg)
print "training on data"
print time.time() - start
model.fit(train)
print "done training"
print time.time() - start
print "saving model"
save_recommender(model, outfile)
print "wrote model to: %s" % outfile
print time.time() - start
return
print "validating"
data, U, V = read_mrec(mrec_file=outfile)
plot_file = outfile.replace('.npz', '.png')
multi_thresh(data, model, thresh_list=None, plot_file=plot_file)
run_time = (time.time() - start) / 60.0
print "runtime: %0.3f minutes" % run_time
print 'done'
def main():
import os
import logging
import glob
import subprocess
from optparse import OptionParser
from IPython.parallel import Client
from mrec import load_fast_sparse_matrix, save_recommender
from mrec.item_similarity.slim import SLIM
from mrec.item_similarity.knn import CosineKNNRecommender, DotProductKNNRecommender
from mrec.mf.wrmf import WRMFRecommender
from mrec.mf.warp import WARPMFRecommender
from mrec.mf.warp2 import WARP2MFRecommender
from mrec.popularity import ItemPopularityRecommender
from mrec.parallel.item_similarity import ItemSimilarityRunner
from mrec.parallel.wrmf import WRMFRunner
from mrec.parallel.warp import WARPMFRunner
logging.basicConfig(level=logging.INFO,format='[%(asctime)s] %(levelname)s: %(message)s')
parser = OptionParser()
parser.add_option('-n','--num_engines',dest='num_engines',type='int',default=0,help='number of IPython engines to use')
parser.add_option('--input_format',dest='input_format',help='format of training dataset(s) tsv | csv | mm (matrixmarket) | fsm (fast_sparse_matrix)')
parser.add_option('--train',dest='train',help='glob specifying path(s) to training dataset(s) IMPORTANT: must be in quotes if it includes the * wildcard')
parser.add_option('--outdir',dest='outdir',help='directory for output files')
parser.add_option('--overwrite',dest='overwrite',action='store_true',help='overwrite existing files in outdir')
parser.add_option('--model',dest='model',default='slim',help='type of model to train: slim | knn | wrmf | warp | popularity (default: %default)')
parser.add_option('--max_sims',dest='max_sims',type='int',default=100,help='max similar items to output for each training item (default: %default)')
parser.add_option('--learner',dest='learner',default='sgd',help='underlying learner for SLIM learner: sgd | elasticnet | fs_sgd (default: %default)')
parser.add_option('--l1_reg',dest='l1_reg',type='float',default=0.001,help='l1 regularization constant (default: %default)')
parser.add_option('--l2_reg',dest='l2_reg',type='float',default=0.0001,help='l2 regularization constant (default: %default)')
parser.add_option('--metric',dest='metric',default='cosine',help='metric for knn recommender: cosine | dot (default: %default)')
parser.add_option('--num_factors',dest='num_factors',type='int',default=80,help='number of latent factors (default: %default)')
parser.add_option('--alpha',dest='alpha',type='float',default=1.0,help='wrmf confidence constant (default: %default)')
parser.add_option('--lbda',dest='lbda',type='float',default=0.015,help='wrmf regularization constant (default: %default)')
parser.add_option('--als_iters',dest='als_iters',type='int',default=15,help='number of als iterations (default: %default)')
parser.add_option('--gamma',dest='gamma',type='float',default=0.01,help='warp learning rate (default: %default)')
parser.add_option('--C',dest='C',type='float',default=100.0,help='warp regularization constant (default: %default)')
parser.add_option('--item_feature_format',dest='item_feature_format',help='format of item features tsv | csv | mm (matrixmarket) | npz (numpy arrays)')
parser.add_option('--item_features',dest='item_features',help='path to sparse item features in tsv format (item_id,feature_id,val)')
parser.add_option('--popularity_method',dest='popularity_method',default='count',help='how to compute popularity for baseline recommender: count | sum | avg | thresh (default: %default)')
parser.add_option('--popularity_thresh',dest='popularity_thresh',type='float',default=0,help='ignore scores below this when computing popularity for baseline recommender (default: %default)')
parser.add_option('--packer',dest='packer',default='json',help='packer for IPython.parallel (default: %default)')
parser.add_option('--add_module_paths',dest='add_module_paths',help='optional comma-separated list of paths to append to pythonpath (useful if you need to import uninstalled modules to IPython engines on a cluster)')
(opts,args) = parser.parse_args()
if not opts.input_format or not opts.train or not opts.outdir or not opts.num_engines:
parser.print_help()
raise SystemExit
opts.train = os.path.abspath(os.path.expanduser(opts.train))
opts.outdir = os.path.abspath(os.path.expanduser(opts.outdir))
trainfiles = glob.glob(opts.train)
if opts.model == 'popularity':
# special case, don't need to run in parallel
subprocess.check_call(['mkdir','-p',opts.outdir])
for trainfile in trainfiles:
logging.info('processing {0}...'.format(trainfile))
model = ItemPopularityRecommender(method=opts.popularity_method,thresh=opts.popularity_thresh)
dataset = load_fast_sparse_matrix(opts.input_format,trainfile)
model.fit(dataset)
modelfile = get_modelfile(trainfile,opts.outdir)
save_recommender(model,modelfile)
logging.info('done')
return
# create an ipython client
c = Client(packer=opts.packer)
view = c.load_balanced_view()
if opts.add_module_paths:
c[:].execute('import sys')
for path in opts.add_module_paths.split(','):
logging.info('adding {0} to pythonpath on all engines'.format(path))
c[:].execute("sys.path.append('{0}')".format(path))
if opts.model == 'slim':
if opts.learner == 'fs_sgd':
num_selected_features = 2 * opts.max_sims # preselect this many candidate similar items
model = SLIM(l1_reg=opts.l1_reg,l2_reg=opts.l2_reg,model=opts.learner,num_selected_features=num_selected_features)
else:
model = SLIM(l1_reg=opts.l1_reg,l2_reg=opts.l2_reg,model=opts.learner)
elif opts.model == 'knn':
if opts.metric == 'cosine':
model = CosineKNNRecommender(k=opts.max_sims)
elif opts.metric == 'dot':
model = DotProductKNNRecommender(k=opts.max_sims)
else:
parser.print_help()
raise SystemExit('unknown metric: {0}'.format(opts.metric))
elif opts.model == 'wrmf':
model = WRMFRecommender(d=opts.num_factors,alpha=opts.alpha,lbda=opts.lbda,num_iters=opts.als_iters)
elif opts.model == 'warp':
num_factors_per_engine = max(opts.num_factors/opts.num_engines,1)
if opts.item_features:
model = WARP2MFRecommender(d=num_factors_per_engine,gamma=opts.gamma,C=opts.C)
else:
model = WARPMFRecommender(d=num_factors_per_engine,gamma=opts.gamma,C=opts.C)
else:
parser.print_help()
raise SystemExit('unknown model type: {0}'.format(opts.model))
for trainfile in trainfiles:
logging.info('processing {0}...'.format(trainfile))
modelfile = get_modelfile(trainfile,opts.outdir)
if opts.model == 'wrmf':
runner = WRMFRunner()
factorsdir = get_factorsdir(trainfile,opts.outdir)
runner.run(view,model,opts.input_format,trainfile,opts.num_engines,factorsdir,modelfile)
elif opts.model == 'warp':
runner = WARPMFRunner()
modelsdir = get_modelsdir(trainfile,opts.outdir)
runner.run(view,model,opts.input_format,trainfile,opts.item_feature_format,opts.item_features,opts.num_engines,modelsdir,opts.overwrite,modelfile)
else:
runner = ItemSimilarityRunner()
simsdir = get_simsdir(trainfile,opts.outdir)
simsfile = get_simsfile(trainfile,opts.outdir)
runner.run(view,model,opts.input_format,trainfile,opts.num_engines,simsdir,opts.overwrite,opts.max_sims,simsfile,modelfile)
