def test_list_gather():
nusers = 3
ndays = 20
nwords = 10
userdayword = scipy.sparse.rand(nwords,
nusers * ndays,
density=0.1,
format="csc")
suwd = billdata.suserdayword(
userdayword, scipy.sparse.csc_matrix(userdayword.transpose()), ndays)
l = suwd.list()
assert len(l) == ndays
def test_list_gather(): nusers = 3 ndays = 20 nwords = 10 userdayword = scipy.sparse.rand( nwords,nusers*ndays,density=0.1,format="csc" ) suwd = billdata.suserdayword( userdayword, scipy.sparse.csc_matrix(userdayword.transpose()), ndays ) l = suwd.list() assert len(l) == ndays
import logging.config
logger.config.fileConfig("logconfig.ini")
# Where the data is coming from and going
user_home = os.environ['HOME']
data_home = "%s/Dropbox/Experiments/twitter_uk_users_MATLAB"%user_home
# load the polls (for the number of days)
p2 = scipy.io.loadmat("%s/Dropbox/TrendMiner/Collaboration/EMNLP_2013/MATLAB_v2/UK_data_for_experiment_PartII.mat"%user_home);
yvalues = p2["polls_vi_cum"][p2["polls_index"]-1,:]
# Load the big dataset
udw = bd.suserdayword(
"%s/user_vsr_for_polls_t.mat"%data_home,
"%s/user_vsr_for_polls.mat"%data_home,
yvalues.shape[0]
)
# extract these days and save
start = 81
delta = 20
end = start + delta
ua,wa = udw.mat(days=(start,end))
matname = "user_vsr_for_polls_day_%d_%d"%(start,end)
outuser = "******"%(data_home,matname)
outword = "%s/%s.mat"%(data_home,matname)
logger.debug("Saving days %d to %d (User Col matrix)"%(start,end))
bd.savesparse(ua,outuser,"%s_t"%matname)
logger.debug("Saving days %d to %d (Word Col matrix)"%(start,end))
bd.savesparse(wa,outword,"%s"%matname)
def experiment(o):
logger.info("Reading initial data")
start = o["start"];ndays = o["ndays"];end = start + ndays
folds = tscv.tsfi(ndays,ntest=o['f_ntest'],nvalidation=o['f_nval'],ntraining=o['f_ntrain'])
tasks = billdata.taskvals(o["task_file"])
ndays_total = tasks.yvalues.shape[0]
if o["user_file_corrected"] is None or not os.path.exists(o["user_file_corrected"]):
logger.info("...Loading and correcting from source")
if "voc_file" in o and not o["word_subsample"] < 1:
logger.info("...Reading vocabulary")
voc = billdata.voc(o["voc_file"]).voc()
# voc = None
else:
voc = None
logger.info("...Reading user days")
user_col, word_col = billdata.suserdayword(
o["user_file"],ndays_total,
nwords=billdata.count_cols_h5(o["word_file"])
).mat(
days=(start,end),
voc=voc
)
if o["user_file_corrected"] is not None:
logger.info("...Saving corrected user_mat")
sio.savemat(o["user_file_corrected"],{"data":user_col.data,"indices":user_col.indices,"indptr":user_col.indptr,"shape":user_col.shape})
else:
logger.info("...Loading corrected user_mat")
# csc_matrix((data, indices, indptr), [shape=(M, N)])
user_col_d = sio.loadmat(o["user_file_corrected"])
user_col = ssp.csc_matrix((user_col_d["data"][:,0],user_col_d["indices"][:,0],user_col_d["indptr"][:,0]),shape=user_col_d["shape"])
logger.info("...User Col read, dimensions: %s"%str(user_col.shape))
logger.info("...Reading task data")
tasks = tasks.mat(days=(start,end),cols=[3,4,5])
logger.info("...Reading tree file")
tree = billdata.tree(o["tree_file"]).spamsobj()
if o["word_subsample"] < 1 or o["user_subsample"] < 1:
user_col=billdata.subsample(user_col,word_subsample=o["word_subsample"],user_subsample=o["user_subsample"],ndays=ndays)
# At this point we've just loaded all the data
# Prepare the optimisation functions
u_lambdas = [float(x) for x in o['u_lambdas_str'].split(",")]
w_lambdas = [float(x) for x in o['w_lambdas_str'].split(",")]
u_lambdas = np.arange(*u_lambdas)
w_lambdas = np.arange(*w_lambdas)
spams_avail = {
"tree":FistaTree(tree,**{
"intercept": True,
"loss":"square",
"regul":"multi-task-tree",
"it0":10,
"lambda2":1000,
"max_it":1000,
"verbose":True
}),
"treecheck":FistaTree(tree,**{
"intercept": True,
"loss":"square",
"regul":"multi-task-tree",
"it0":10,
"max_it":100,
"lambda2":1000,
"verbose":True
}),
"flatcheck":FistaFlat(**{
"intercept": True,
"loss":"square",
"regul":"l1l2",
"it0":50,
"max_it":100,
"verbose":True
}),
"flat":FistaFlat(**{
"intercept": True,
"loss":"square",
"regul":"l1l2",
"it0":50,
"max_it":1000,
"verbose":True
})
}
w_spams = copy.deepcopy(spams_avail[o["w_spams"]])
u_spams = copy.deepcopy(spams_avail[o["u_spams"]])
lambda_set = False
if o["lambda_file"] is not None and os.path.exists(o["lambda_file"]):
logger.info("... loading existing lambda")
lambda_d = sio.loadmat(o["lambda_file"])
w_spams.params["lambda1"] = lambda_d["w_lambda"][0][0]
u_spams.params["lambda1"] = lambda_d["u_lambda"][0][0]
lambda_set = True
# Prepare the learner
learner = BatchBivariateLearner(w_spams,u_spams,bivar_max_it=o["bivar_max_it"])
fold_i = 0
es.exp(os.sep.join([o['exp_out'],"ds:politics_word:l1_user:l1_task:multi"]),fake=False)
# Go through the folds!
for fold in folds:
es.state("fold_%d"%fold_i)
logger.info("Working on fold: %d"%fold_i)
logger.info("... preparing fold parts")
Xparts,Yparts = BatchBivariateLearner.XYparts(fold,user_col,tasks)
if not o["optimise_lambda_once"] or (o["optimise_lambda_once"] and not lambda_set):
logger.debug("... Setting max it to optimisation mode: %d"%o["opt_maxit"])
w_spams.params["max_it"] = o["opt_maxit"]
u_spams.params["max_it"] = o["opt_maxit"]
logger.info("... optimising fold lambda")
ulambda,wlambda = learner.optimise_lambda(
w_lambdas,u_lambdas,Yparts,Xparts,
w_lambda=o["w_lambda"],u_lambda=o["u_lambda"]
)
lambda_set = True
if o["lambda_file"] is not None:
logger.info("... saving optimised lambdas")
sio.savemat(o["lambda_file"],{"w_lambda":wlambda[1],"u_lambda":ulambda[1]})
logger.info("... training fold")
logger.debug("... Setting max it to training mode: %d"%o["train_maxit"])
w_spams.params["max_it"] = o["train_maxit"]
u_spams.params["max_it"] = o["train_maxit"]
learner.process(
Yparts.train_all,Xparts.train_all,
tests={
"test":(Xparts.test,Yparts.test),
"val_it":(Xparts.val_it,Yparts.val_it)
}
)
es.add(locals(),"fold_i","w_lambdas","u_lambdas","fold","Yparts","o")
es.state()["w_spams_params"] = w_spams.params
es.state()["u_spams_params"] = u_spams.params
logger.info("... Saving output")
es.flush()
fold_i += 1
if o["f_maxiter"] is not None and fold_i >= o["f_maxiter"]: break
voc_file = "%s/Dropbox/TrendMiner/Collaboration/EMNLP_2013/MATLAB_v2/voc_matching_v2.mat" % home
start = 81
ndays = 20
end = start + ndays
user_file = os.sep.join([data_home, user_mat_file % (start, end)])
word_file = os.sep.join([data_home, word_mat_file % (start, end)])
folds = tscv.tsfi(ndays, ntest=2)
logger.info("Reading task data")
tasks = billdata.taskvals(task_file).mat(days=(start, end))
# tree = billdata.tree(tree_file).spamsobj()
logger.info("Reading vocabulary")
voc = billdata.voc(voc_file).voc()
user_col, word_col = billdata.suserdayword(user_file, word_file,
ndays).mat(voc=voc)
user_col, word_col = billdata.subsample(user_col,
word_subsample=0.001,
user_subsample=0.001,
ndays=ndays)
# At this point we've just loaded all the data
# Prepare the optimisation functions
u_lambdas = np.arange(0.1, 1, 0.1)
w_lambdas = np.arange(0.1, 2, 0.1)
w_spams = FistaFlat(**{
"intercept": True,
"loss": "square",
"regul": "l1",
"it0": 10,
"max_it": 1000
import logging.config
logger.config.fileConfig("logconfig.ini")
# Where the data is coming from and going
user_home = os.environ['HOME']
data_home = "%s/Dropbox/Experiments/twitter_uk_users_MATLAB" % user_home
# load the polls (for the number of days)
p2 = scipy.io.loadmat(
"%s/Dropbox/TrendMiner/Collaboration/EMNLP_2013/MATLAB_v2/UK_data_for_experiment_PartII.mat"
% user_home)
yvalues = p2["polls_vi_cum"][p2["polls_index"] - 1, :]
# Load the big dataset
udw = bd.suserdayword("%s/user_vsr_for_polls_t.mat" % data_home,
"%s/user_vsr_for_polls.mat" % data_home,
yvalues.shape[0])
# extract these days and save
start = 81
delta = 20
end = start + delta
ua, wa = udw.mat(days=(start, end))
matname = "user_vsr_for_polls_day_%d_%d" % (start, end)
outuser = "******" % (data_home, matname)
outword = "%s/%s.mat" % (data_home, matname)
logger.debug("Saving days %d to %d (User Col matrix)" % (start, end))
bd.savesparse(ua, outuser, "%s_t" % matname)
logger.debug("Saving days %d to %d (Word Col matrix)" % (start, end))
bd.savesparse(wa, outword, "%s" % matname)
tree_file = "%s/Dropbox/TrendMiner/Collaboration/EMNLP_2013/MATLAB_v2/UK_data_for_experiment_PartI.mat"%home
voc_file = "%s/Dropbox/TrendMiner/Collaboration/EMNLP_2013/MATLAB_v2/voc_matching_v2.mat"%home
start = 81;ndays = 20;end = start + ndays
user_file = os.sep.join([data_home,user_mat_file%(start,end)])
word_file = os.sep.join([data_home,word_mat_file%(start,end)])
folds = tscv.tsfi(ndays,ntest=2)
logger.info("Reading task data")
tasks = billdata.taskvals(task_file).mat(days=(start,end))
# tree = billdata.tree(tree_file).spamsobj()
logger.info("Reading vocabulary")
voc = billdata.voc(voc_file).voc()
user_col, word_col = billdata.suserdayword(
user_file,word_file,ndays
).mat(voc=voc)
user_col,word_col=billdata.subsample(user_col,word_subsample=0.001,user_subsample=0.001,ndays=ndays)
# At this point we've just loaded all the data
# Prepare the optimisation functions
u_lambdas = np.arange(0.1,1,0.1)
w_lambdas = np.arange(0.1,2,0.1)
w_spams = FistaFlat(**{
"intercept": True,
"loss":"square",
"regul":"l1",
"it0":10,
"max_it":1000
})
u_spams = FistaFlat(**{