def lcmnl_estimate(cmdata,numclasses,csdata,numalts,chosen,maxiter=MAXITER,emtol=EMTOL,\ skipprep=False,csbeta=None,cmbeta=None): loglik = -999999 if csbeta is None: csbeta = [np.random.rand(csdata.shape[1]) for i in range(numclasses)] if not skipprep: cmdata = prep_cm_data(cmdata,numclasses) if cmbeta is None: cmbeta = np.zeros(cmdata.shape[1]) for i in range(maxiter): print "Running iteration %d" % (i+1) print time.ctime() # EXPECTATION print "Running class membership model" cmprobs = mnl.mnl_simulate(cmdata,cmbeta,numclasses,GPU=GPU,returnprobs=1) csprobs = [] for cno in range(numclasses): tmp = mnl.mnl_simulate(csdata,csbeta[cno],numalts,GPU=GPU,returnprobs=1) tmp = np.sum(tmp*chosen,axis=1) # keep only chosen probs csprobs.append(np.reshape(tmp,(-1,1))) csprobs = np.concatenate(csprobs,axis=1) h = csprobs * cmprobs oldloglik = loglik loglik = np.sum(np.log(np.sum(h,axis=1))) print "current csbeta", csbeta print "current cmbeta", cmbeta print "current loglik", loglik, i+1, "\n\n" if abs(loglik-oldloglik) < emtol: break wts = h / np.reshape(np.sum(h,axis=1),(-1,1)) # MAXIMIZATION for cno in range(numclasses): print "Estimating class specific model for class %d" % (cno+1) t1 = time.time() weights=np.reshape(wts[:,cno],(-1,1)) print weights.shape fit, results = mnl.mnl_estimate(csdata,chosen,numalts,GPU=GPU,weights=weights,beta=csbeta[cno]) print "Finished in %fs" % (time.time()-t1) csbeta[cno] = zip(*results)[0] print "Estimating class membership model" t1 = time.time() fit, results = mnl.mnl_estimate(cmdata,None,numclasses,GPU=GPU,weights=wts,lcgrad=True, \ beta=cmbeta,coeffrange=(-1000,1000)) print "Finished in %fs" % (time.time()-t1) cmbeta = zip(*results)[0]
def lcmnl_estimate(cmdata,numclasses,csdata,numalts,chosen,maxiter=MAXITER,emtol=EMTOL): loglik = -999999 beta = [np.ones(csdata.shape[1]) for i in range(numclasses)] #cmbeta = np.ones((cmdata.shape[1]+1)*(numclasses-1)) # +1 if for asc, -1 is base alt cmbeta = np.ones(cmdata.shape[1]) for i in range(maxiter): print "Running iteration %d" % (i+1) # EXPECTATION print "Running class membership model" print "cmbeta", cmbeta cmprobs = mnl.mnl_simulate(cmdata,cmbeta,numclasses,GPU=GPU,returnprobs=1) print "cmprobs", cmprobs csprobs = [] for cno in range(numclasses): print "Running class specific model for class %d" % (cno+1) print "csbeta", beta[cno] tmp = mnl.mnl_simulate(csdata,beta[cno],numalts,GPU=GPU,returnprobs=1) print "csprobs", tmp tmp = np.sum(tmp*chosen,axis=1) # keep only chosen probs csprobs.append(np.reshape(tmp,(-1,1))) csprobs = np.concatenate(csprobs,axis=1) h = csprobs * cmprobs oldloglik = loglik loglik = np.sum(np.log(np.sum(h,axis=1))) print "current loglik", loglik, i+1 print "current beta", beta if abs(loglik-oldloglik) < emtol: break wts = h / np.reshape(np.sum(h,axis=1),(-1,1)) for i in range(wts.shape[1]): print i, pd.Series(wts[:,i]).describe() # MAXIMIZATION print "Estimating class membership model" fit, results = mnl.mnl_estimate(cmdata,None,numclasses,GPU=GPU,weights=wts,lcgrad=True) cmbeta = zip(*results)[0] for cno in range(numclasses): print "Estimating class specific model for class %d" % (cno+1) fit, results = mnl.mnl_estimate(csdata,chosen,numalts,GPU=GPU,weights=wts[:,cno]) beta[cno] = zip(*results)[0]
def lcmnl_estimate(cmdata,numclasses,csdata,numalts,chosen,maxiter=MAXITER,emtol=EMTOL,\ skipprep=False,csbeta=None,cmbeta=None,csfnames=None,cmfnames=None): loglik = -999999 l_0 = None if csbeta is None: csbeta = [np.random.rand(csdata.shape[1]) for i in range(numclasses)] if csfnames is None: csfnames = ['cs%d' % i for i in range(csdata.shape[1])] if cmfnames is None: cmfnames = ['cm%d' % i for i in range(cmdata.shape[1])] if not skipprep: cmdata, cmfnames = prep_cm_data(cmdata, numclasses, cmfnames) if cmbeta is None: cmbeta = np.random.rand(cmdata.shape[1]) * 10.0 - 5.0 results_d = {} for i in range(maxiter): print "Running iteration %d" % (i + 1) print time.ctime() # EXPECTATION def expectation(cmbeta, csbeta): print "Running class membership model" cmprobs = mnl.mnl_simulate(cmdata, cmbeta, numclasses, GPU=GPU, returnprobs=1) csprobs = [] for cno in range(numclasses): tmp = mnl.mnl_simulate(csdata, csbeta[cno], numalts, GPU=GPU, returnprobs=1) tmp = np.sum(tmp * chosen, axis=1) # keep only chosen probs csprobs.append(np.reshape(tmp, (-1, 1))) csprobs = np.concatenate(csprobs, axis=1) h = csprobs * cmprobs loglik = np.sum(np.log(np.sum(h, axis=1))) wts = h / np.reshape(np.sum(h, axis=1), (-1, 1)) return loglik, wts oldloglik = loglik loglik, wts = expectation(cmbeta, csbeta) if l_0 is None: l_0 = loglik print "current cmbeta", cmbeta print "current csbeta", csbeta print "current loglik", loglik, i + 1, "\n\n" if abs(loglik - oldloglik) < emtol: break # MAXIMIZATION for cno in range(numclasses): print "Estimating class specific model for class %d" % (cno + 1) t1 = time.time() weights = np.reshape(wts[:, cno], (-1, 1)) fit, results = mnl.mnl_estimate(csdata, chosen, numalts, GPU=GPU, weights=weights, beta=csbeta[cno]) print "Finished in %fs" % (time.time() - t1) csbeta[cno] = zip(*results)[0] results_d['cs%d' % cno] = results print "Estimating class membership model" t1 = time.time() fit, results = mnl.mnl_estimate(cmdata,None,numclasses,GPU=GPU,weights=wts,lcgrad=True, \ beta=cmbeta,coeffrange=(-1000,1000)) print "Finished in %fs" % (time.time() - t1) cmbeta = zip(*results)[0] results_d['cm'] = results l_1 = loglik l_0, foo = expectation(np.zeros(len(cmbeta)), [np.zeros(len(a)) for a in csbeta]) ll_ratio = 1 - (l_1 / l_0) print "Null Log-liklihood: %f" % l_0 print "Log-liklihood at convergence: %f" % l_1 print "Log-liklihood ratio: %f" % ll_ratio a = [] fnames = [] fnames += cmfnames a += results_d['cm'] for i in range(numclasses): fnames += ['%s cls%d' % (s, i) for s in csfnames] a += results_d['cs%d' % i] print misc.resultstotable(fnames, a) fit = (l_0, l_1, ll_ratio) misc.resultstocsv(fit, fnames, a, "lc-coeff.csv", tblname="Latent Class Model Coefficients") return (l_0, l_1, ll_ratio), results_d
def lcmnl_estimate(cmdata,numclasses,csdata,numalts,chosen,maxiter=MAXITER,emtol=EMTOL,\ skipprep=False,csbeta=None,cmbeta=None,csfnames=None,cmfnames=None): loglik = -999999 l_0 = None if csbeta is None: csbeta = [np.random.rand(csdata.shape[1]) for i in range(numclasses)] if csfnames is None: csfnames = ['cs%d'%i for i in range(csdata.shape[1])] if cmfnames is None: cmfnames = ['cm%d'%i for i in range(cmdata.shape[1])] if not skipprep: cmdata,cmfnames = prep_cm_data(cmdata,numclasses,cmfnames) if cmbeta is None: cmbeta = np.random.rand(cmdata.shape[1])*10.0-5.0 results_d = {} for i in range(maxiter): print "Running iteration %d" % (i+1) print time.ctime() # EXPECTATION def expectation(cmbeta,csbeta): print "Running class membership model" cmprobs = mnl.mnl_simulate(cmdata,cmbeta,numclasses,GPU=GPU,returnprobs=1) csprobs = [] for cno in range(numclasses): tmp = mnl.mnl_simulate(csdata,csbeta[cno],numalts,GPU=GPU,returnprobs=1) tmp = np.sum(tmp*chosen,axis=1) # keep only chosen probs csprobs.append(np.reshape(tmp,(-1,1))) csprobs = np.concatenate(csprobs,axis=1) h = csprobs * cmprobs loglik = np.sum(np.log(np.sum(h,axis=1))) wts = h / np.reshape(np.sum(h,axis=1),(-1,1)) return loglik, wts oldloglik = loglik loglik, wts = expectation(cmbeta,csbeta) if l_0 is None: l_0 = loglik print "current cmbeta", cmbeta print "current csbeta", csbeta print "current loglik", loglik, i+1, "\n\n" if abs(loglik-oldloglik) < emtol: break # MAXIMIZATION for cno in range(numclasses): print "Estimating class specific model for class %d" % (cno+1) t1 = time.time() weights=np.reshape(wts[:,cno],(-1,1)) fit, results = mnl.mnl_estimate(csdata,chosen,numalts,GPU=GPU,weights=weights,beta=csbeta[cno]) print "Finished in %fs" % (time.time()-t1) csbeta[cno] = zip(*results)[0] results_d['cs%d'%cno] = results print "Estimating class membership model" t1 = time.time() fit, results = mnl.mnl_estimate(cmdata,None,numclasses,GPU=GPU,weights=wts,lcgrad=True, \ beta=cmbeta,coeffrange=(-1000,1000)) print "Finished in %fs" % (time.time()-t1) cmbeta = zip(*results)[0] results_d['cm'] = results l_1 = loglik l_0, foo = expectation(np.zeros(len(cmbeta)),[np.zeros(len(a)) for a in csbeta]) ll_ratio = 1-(l_1/l_0) print "Null Log-liklihood: %f" % l_0 print "Log-liklihood at convergence: %f" % l_1 print "Log-liklihood ratio: %f" % ll_ratio a = [] fnames = [] fnames += cmfnames a += results_d['cm'] for i in range(numclasses): fnames += ['%s cls%d'%(s,i) for s in csfnames] a += results_d['cs%d'%i] print misc.resultstotable(fnames,a) fit = (l_0,l_1,ll_ratio) misc.resultstocsv(fit,fnames,a,"lc-coeff.csv",tblname="Latent Class Model Coefficients") return (l_0,l_1,ll_ratio), results_d
def lcmnl_estimate(cmdata,numclasses,csdata,numalts,chosen,maxiter=MAXITER,emtol=EMTOL,\ skipprep=False,csbeta=None,cmbeta=None): loglik = -999999 if csbeta is None: csbeta = [np.random.rand(csdata.shape[1]) for i in range(numclasses)] if not skipprep: cmdata = prep_cm_data(cmdata, numclasses) if cmbeta is None: cmbeta = np.zeros(cmdata.shape[1]) for i in range(maxiter): print "Running iteration %d" % (i + 1) print time.ctime() # EXPECTATION print "Running class membership model" cmprobs = mnl.mnl_simulate(cmdata, cmbeta, numclasses, GPU=GPU, returnprobs=1) csprobs = [] for cno in range(numclasses): tmp = mnl.mnl_simulate(csdata, csbeta[cno], numalts, GPU=GPU, returnprobs=1) tmp = np.sum(tmp * chosen, axis=1) # keep only chosen probs csprobs.append(np.reshape(tmp, (-1, 1))) csprobs = np.concatenate(csprobs, axis=1) h = csprobs * cmprobs oldloglik = loglik loglik = np.sum(np.log(np.sum(h, axis=1))) print "current csbeta", csbeta print "current cmbeta", cmbeta print "current loglik", loglik, i + 1, "\n\n" if abs(loglik - oldloglik) < emtol: break wts = h / np.reshape(np.sum(h, axis=1), (-1, 1)) # MAXIMIZATION for cno in range(numclasses): print "Estimating class specific model for class %d" % (cno + 1) t1 = time.time() weights = np.reshape(wts[:, cno], (-1, 1)) print weights.shape fit, results = mnl.mnl_estimate(csdata, chosen, numalts, GPU=GPU, weights=weights, beta=csbeta[cno]) print "Finished in %fs" % (time.time() - t1) csbeta[cno] = zip(*results)[0] print "Estimating class membership model" t1 = time.time() fit, results = mnl.mnl_estimate(cmdata,None,numclasses,GPU=GPU,weights=wts,lcgrad=True, \ beta=cmbeta,coeffrange=(-1000,1000)) print "Finished in %fs" % (time.time() - t1) cmbeta = zip(*results)[0]