def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training, mapped by task_id
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
@return: trained predictors, mapped by task_id
@rtype: dict<str, SVM>
"""
# init container
svms = {}
# concatenate data
data = PreparedMultitaskData(train_data, shuffle=False)
# create svm
svm = shogun_factory.create_initialized_svm(param, data.examples,
data.labels)
print "starting training procedure"
# train SVM
svm.train()
print "training done"
# use a reference to the same svm several times
for task_id in train_data.keys():
svms[task_id] = svm
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# fetch taxonomy from parameter object
taxonomy = shogun_factory.create_taxonomy(param.taxonomy.data)
# set normalizer
normalizer = MultitaskKernelTreeNormalizer(data.task_vector_names,
data.task_vector_names,
taxonomy)
########################################################
gammas = self.taxonomy_to_gammas(data, taxonomy)
print "gammas before MKL:"
print gammas
########################################################
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
svm = None
num_subk = base_wdk.get_num_subkernels()
print "num subkernels:", num_subk
#print "subkernel weights:", base_wdk.get_subkernel_weights()
self.additional_information["weights_before"] = [
normalizer.get_beta(i) for i in range(num_subk)
]
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
num_threads = 4
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #GLPK) #DIRECT) #NEWTON)#ST_CPLEX)
svm.set_kernel(base_wdk)
svm.set_labels(lab)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
if param.flags["normalize_cost"]:
# normalize cost
norm_c_pos = param.cost / float(
len([l for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
else:
svm.set_C(param.cost, param.cost)
svm.train()
#print "subkernel weights (after):", base_wdk.get_subkernel_weights()
else:
# create SVM (disable unsupported optimizations)
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
print "svm objective:", svm.get_objective()
self.additional_information["weights"] = [
normalizer.get_beta(i) for i in range(num_subk)
]
self.additional_information["gammas"] = self.taxonomy_to_gammas(
data, taxonomy)
print "debug weights:"
print self.additional_information
print ""
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_id in train_data.keys():
svms[task_id] = svm
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
##################################################
# define pockets
##################################################
pockets = [0] * 9
pockets[0] = [1, 5, 6, 7, 8, 31, 32, 33, 34]
pockets[1] = [1, 2, 3, 4, 6, 7, 8, 9, 11, 21, 31]
pockets[2] = [11, 20, 21, 22, 29, 31]
pockets[3] = [8, 30, 31, 32]
pockets[4] = [10, 11, 30]
pockets[5] = [10, 11, 12, 13, 20, 29]
pockets[6] = [10, 12, 20, 22, 26, 27, 28, 29]
pockets[7] = [12, 14, 15, 26]
pockets[8] = [13, 15, 16, 17, 18, 19, 20, 23, 24, 25, 26]
#new_pockets = []
# merge neighboring pockets
#for i in range(8):
# new_pockets.append(list(set(pockets[i]).union(set(pockets[i+1]))))
#pockets = new_pockets
########################################################
print "creating a kernel:"
########################################################
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples)
combined_features = CombinedFeatures()
##################################################
# intra-domain blocks
# intra_block_vec = PairiiVec()
#
# for task_id in data.get_task_ids():
# intra_block_vec.push_back(Pairii(task_id, task_id))
#
#
#
# # create mask-based normalizer
# normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, intra_block_vec)
# kernel = shogun_factory.create_empty_kernel(param)
# kernel.set_normalizer(normalizer)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel)
#
# # append features
# combined_features.append_feature_obj(base_features)
#
# print "------"
#
# ##################################################
# # all blocks
#
#
# all_block_vec = PairiiVec()
#
# for task_id_1 in data.get_task_ids():
# for task_id_2 in data.get_task_ids():
# all_block_vec.push_back(Pairii(task_id_1, task_id_2))
#
#
# # create mask-based normalizer
# normalizer_all = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, all_block_vec)
# kernel_all = shogun_factory.create_empty_kernel(param)
# kernel_all.set_normalizer(normalizer_all)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel_all)
#
# # append features
# combined_features.append_feature_obj(base_features)
##################################################
# add one kernel per similarity position
# init seq handler
pseudoseqs = SequencesHandler()
for pocket in pockets:
print "creating normalizer"
#import pdb
#pdb.set_trace()
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
print "processing pocket", pocket
# set similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
similarity = 0.0
for pseudo_seq_pos in pocket:
similarity += float(
pseudoseqs.get_similarity(task_name_lhs,
task_name_rhs,
pseudo_seq_pos - 1))
# normalize
similarity = similarity / float(len(pocket))
print "pocket %s (%s, %s) = %f" % (
str(pocket), task_name_lhs, task_name_rhs, similarity)
normalizer.set_task_similarity(
data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs), similarity)
print "creating empty kernel"
kernel_pos = shogun_factory.create_empty_kernel(param)
print "setting normalizer"
kernel_pos.set_normalizer(normalizer)
print "appending kernel"
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel_pos)
print "appending features"
# append features
combined_features.append_feature_obj(base_features)
print "done constructing combined kernel"
##################################################
# init combined kernel
# init weights
# combined_kernel.set_subkernel_weights([1.0/2.85]*combined_kernel.get_num_subkernels())
combined_kernel.init(combined_features, combined_features)
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #ST_GLPK) #DIRECT) #NEWTON)#ST_CPLEX) #auto
svm.set_C(param.cost, param.cost)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
# set up SVM
num_threads = 8
svm.io.enable_progress()
#svm.io.set_loglevel(shogun.Classifier.MSG_INFO)
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
#print "WARNING: custom epsilon set"
#svm.set_epsilon(0.05)
# normalize cost
norm_c_pos = param.cost / float(len([l
for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional info
self.additional_information["svm_objective"] = svm.get_objective()
self.additional_information[
"svm num sv"] = svm.get_num_support_vectors()
self.additional_information[
"post_weights"] = combined_kernel.get_subkernel_weights()
print self.additional_information
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), combined_kernel,
svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
assert(param.base_similarity >= 1)
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# set normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
# load data
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_pearson.txt")
f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/All_PseudoSeq_Hamming.txt")
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_euklid.txt")
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_RAxML.txt")
num_lines = int(f.readline().strip())
task_distances = numpy.zeros((num_lines, num_lines))
name_to_id = {}
for (i, line) in enumerate(f):
tokens = line.strip().split("\t")
name = str(tokens[0])
name_to_id[name] = i
entry = numpy.array([v for (j,v) in enumerate(tokens) if j!=0])
assert len(entry)==num_lines, "len_entry %i, num_lines %i" % (len(entry), num_lines)
task_distances[i,:] = entry
# cut relevant submatrix
active_ids = [name_to_id[name] for name in data.get_task_names()]
tmp_distances = task_distances[active_ids, :]
tmp_distances = tmp_distances[:, active_ids]
print "distances ", tmp_distances.shape
# normalize distances
task_distances = task_distances / numpy.max(tmp_distances)
similarities = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
# convert distance to similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
# convert similarity with simple transformation
similarity = param.base_similarity - task_distances[name_to_id[task_name_lhs], name_to_id[task_name_rhs]]
normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity)
# save for later
similarities[data.name_to_id(task_name_lhs),data.name_to_id(task_name_rhs)] = similarity
# set normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
# set up svm
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# normalize cost
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
#self.additional_information["distances"] = distances
self.additional_information["similarities"] = similarities
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, param, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# split for training weak_learners and boosting
(train_weak, train_boosting) = split_data(train_data, 4)
# merge data sets
data = PreparedMultitaskData(train_weak, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
##################################################
# define pockets
##################################################
pockets = [0]*9
pockets[0] = [1, 5, 6, 7, 8, 31, 32, 33, 34]
pockets[1] = [1, 2, 3, 4, 6, 7, 8, 9, 11, 21, 31]
pockets[2] = [11, 20, 21, 22, 29, 31]
pockets[3] = [8, 30, 31, 32]
pockets[4] = [10, 11, 30]
pockets[5] = [10, 11, 12, 13, 20, 29]
pockets[6] = [10, 12, 20, 22, 26, 27, 28, 29]
pockets[7] = [12, 14, 15, 26]
pockets[8] = [13, 15, 16, 17, 18, 19, 20, 23, 24, 25, 26]
pockets = []
for i in xrange(35):
pockets.append([i])
#new_pockets = []
# merge neighboring pockets
#for i in range(8):
# new_pockets.append(list(set(pockets[i]).union(set(pockets[i+1]))))
#pockets = new_pockets
########################################################
print "creating a kernel:"
########################################################
# init seq handler
pseudoseqs = SequencesHandler()
classifiers = []
for pocket in pockets:
print "creating normalizer"
#import pdb
#pdb.set_trace()
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
print "processing pocket", pocket
# set similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
similarity = 0.0
for pseudo_seq_pos in pocket:
similarity += float(pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pseudo_seq_pos-1))
# normalize
similarity = similarity / float(len(pocket))
print "pocket %s (%s, %s) = %f" % (str(pocket), task_name_lhs, task_name_rhs, similarity)
normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity)
print "creating empty kernel"
kernel = shogun_factory.create_kernel(data.examples, param)
print "setting normalizer"
kernel.set_normalizer(normalizer)
print "training SVM for pocket", pocket
svm = self._train_single_svm(param, kernel, lab)
classifiers.append(svm)
print "done obtaining weak learners"
# save additional info
#self.additional_information["svm_objective"] = svm.get_objective()
#self.additional_information["svm num sv"] = svm.get_num_support_vectors()
#self.additional_information["post_weights"] = combined_kernel.get_subkernel_weights()
#print self.additional_information
##################################################
# combine weak learners for each task
##################################################
# set constants
some = 0.9
import cvxmod
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_boosting.keys():
instances = train_boosting[task_name]
N = len(instances)
F = len(pockets)
examples = [inst.example for inst in instances]
labels = [inst.label for inst in instances]
# dim = (F x N)
out = cvxmod.zeros((N,F))
for i in xrange(F):
svm = classifiers[i]
tmp_out = self._predict_weak(svm, examples, data.name_to_id(task_name))
out[:,i] = numpy.sign(tmp_out)
#out[:,i] = tmp_out
#TODO: fix
helper.save("/tmp/out_sparse", (out,labels))
pdb.set_trace()
weights = solve_boosting(out, labels, some, solver="mosek")
svms[task_name] = (data.name_to_id(task_name), svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
for task_id in train_data.keys():
print "task_id:", task_id
# split data for training weak_learners and boosting
(train_weak, train_boosting) = split_data(train_data, 4)
# train on first part of dataset (evaluate on other)
prepared_data_weak = PreparedMultitaskData(train_weak, shuffle=False)
classifiers = self._inner_train(prepared_data_weak, param)
# train on entire dataset
prepared_data_final = PreparedMultitaskData(train_data, shuffle=False)
final_classifiers = self._inner_train(prepared_data_final, param)
print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
print "done training weak learners"
#####################################################
# perform boosting and wrap things up
#####################################################
# wrap up predictors for later use
predictors = {}
for task_name in train_boosting.keys():
instances = train_boosting[task_name]
N = len(instances)
F = len(classifiers)
examples = [inst.example for inst in instances]
labels = [inst.label for inst in instances]
# dim = (F x N)
out = cvxmod.zeros((N,F))
for i in xrange(F):
svm = classifiers[i]
tmp_out = self._predict_weak(svm, examples, prepared_data_weak.name_to_id(task_name), param)
if param.flags["signum"]:
out[:,i] = numpy.sign(tmp_out)
else:
out[:,i] = tmp_out
if param.flags["boosting"] == "ones":
weights = numpy.ones(F)/float(F)
if param.flags["boosting"] == "L1":
weights = solve_boosting(out, labels, param.transform, solver="glpk")
if param.flags["boosting"] == "L2":
weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=False)
if param.flags["boosting"] == "L2_reg":
weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=True)
predictors[task_name] = (final_classifiers, weights, prepared_data_final.name_to_id(task_name), param)
assert prepared_data_final.name_to_id(task_name)==prepared_data_weak.name_to_id(task_name), "name mappings don't match"
#####################################################
# Some sanity checks
#####################################################
# make sure we have the same keys (potentiall in a different order)
sym_diff_keys = set(train_weak.keys()).symmetric_difference(set(predictors.keys()))
assert len(sym_diff_keys)==0, "symmetric difference between keys non-empty: " + str(sym_diff_keys)
return predictors
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# support
support = numpy.linspace(0, 1, 5)
# set normalizer
normalizer = MultitaskKernelPlifNormalizer(support,
data.task_vector_nums)
# fetch taxonomy from parameter object
taxonomy = param.taxonomy.data
taxonomy.plot()
import os
os.system("evince demo.png &")
# compute distances
distances = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
for (i, task_name_lhs) in enumerate(data.get_task_names()):
for (j, task_name_rhs) in enumerate(data.get_task_names()):
distances[i, j] = task_similarities.compute_hop_distance(
taxonomy, task_name_lhs, task_name_rhs)
# normalize distances
distances = distances / numpy.max(distances)
# set distances
for (i, task_name_lhs) in enumerate(data.get_task_names()):
for (j, task_name_rhs) in enumerate(data.get_task_names()):
normalizer.set_task_distance(i, j, distances[i, j])
# assign normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
svm = None
debug_weights = {}
num_subk = base_wdk.get_num_subkernels()
print "num subkernels:", num_subk
#print "subkernel weights:", base_wdk.get_subkernel_weights()
debug_weights["before"] = [
normalizer.get_beta(i) for i in range(num_subk)
]
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
num_threads = 4
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #GLPK) #DIRECT) #NEWTON)#ST_CPLEX)
svm.set_C(param.cost, param.cost)
svm.set_kernel(base_wdk)
svm.set_labels(lab)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
#print "subkernel weights (after):", base_wdk.get_subkernel_weights()
else:
# create SVM (disable unsupported optimizations)
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
print "svm objective:", svm.get_objective()
debug_weights["after"] = [
normalizer.get_beta(i) for i in range(num_subk)
]
# debugging output
print "debug weights (before/after):"
print debug_weights["before"]
print debug_weights["after"]
print ""
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (svm, data.name_to_id(task_name))
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training, mapped by task_id
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
@return: trained predictors, mapped by task_id
@rtype: dict<str, SVM>
"""
# init container
svms = {}
# concatenate data
data = PreparedMultitaskData(train_data, shuffle=False)
# fetch taxonomy
taxonomy = param.taxonomy.data
# compute distances
distances = defaultdict(dict)
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
distances[task_name_lhs][
task_name_rhs] = task_similarities.compute_hop_distance(
taxonomy, task_name_lhs, task_name_rhs)
# use a reference to the same svm several times
for task_name in train_data.keys():
# create svm
svm = shogun_factory.create_initialized_svm(
param, data.examples, data.labels)
print "starting training procedure"
cost = []
max_distance = max(distances[task_name].values())
# set cost
for i in xrange(data.num_examples):
tmp_task_name = data.task_vector_names[i]
distance = distances[task_name][tmp_task_name] / max_distance
base_sim = 1.0
similarity = base_sim - distance
cost.append(similarity * param.cost)
# set cost vector
svm.set_cost(cost)
# train SVM
svm.train()
print "training done"
svms[task_name] = svm
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
assert(param.base_similarity >= 1)
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# create normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
# load hard-coded task-similarity
task_similarity = helper.load("/fml/ag-raetsch/home/cwidmer/svn/projects/alt_splice_code/src/task_sim_tis.bz2")
# set similarity
similarities = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
for (i, task_name_lhs) in enumerate(data.get_task_names()):
#max_value_row = max(task_similarity.get_row(task_name_lhs))
max_value_row = 1.0
for (j, task_name_rhs) in enumerate(data.get_task_names()):
similarity = task_similarity.get_value(task_name_lhs, task_name_rhs) / max_value_row
normalizer.set_task_similarity(i, j, similarity)
similarities[i,j] = similarity
pprint.pprint similarities
# set normalizer
#print "WARNING MTK disabled!!!!!!!!!!!!!!!!!!!!!"
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
# set up svm
param.flags["svm_type"] = "svmlight" #fix svm type
svm = shogun_factory.create_svm(param, base_wdk, lab)
# make sure these parameters are set correctly
#print "WARNING MTK WONT WORK WITH THESE SETTINGS!!!!!!!!!!!!!!!!!!!!!"
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
assert svm.get_linadd_enabled() == False, "linadd should be disabled"
assert svm.get_batch_computation_enabled == False, "batch compute should be disabled"
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
#self.additional_information["distances"] = distances
self.additional_information["similarities"] = similarities
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
#numpy.random.seed(1337)
numpy.random.seed(666)
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_DEBUG)
# set kernel cache
if param.flags.has_key("cache_size"):
combined_kernel.set_cache_size(param.flags["cache_size"])
# create features
base_features = shogun_factory.create_features(data.examples)
combined_features = CombinedFeatures()
########################################################
print "creating a masked kernel for each node:"
########################################################
# fetch taxonomy from parameter object
taxonomy = param.taxonomy.data
# create name to leaf map
nodes = taxonomy.get_all_nodes()
for node in nodes:
print "creating kernel for ", node.name
# fetch sub-tree
active_task_ids = [data.name_to_id(leaf.name) for leaf in node.get_leaves()]
print "masking all entries other than:", active_task_ids
# create mask-based normalizer
normalizer = MultitaskKernelMaskNormalizer(data.task_vector_nums, data.task_vector_nums, active_task_ids)
# normalize trace
if param.flags.has_key("normalize_trace") and param.flags["normalize_trace"]:
norm_factor = len(node.get_leaves()) / len(active_task_ids)
normalizer.set_normalization_constant(norm_factor)
# create kernel
kernel = shogun_factory.create_empty_kernel(param)
kernel.set_normalizer(normalizer)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel)
# append features
combined_features.append_feature_obj(base_features)
print "------"
combined_kernel.init(combined_features, combined_features)
#combined_kernel.precompute_subkernels()
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.flags["mkl_q"] >= 1.0)
if param.flags["mkl_q"] >= 1.0:
# set up MKL
svm = MKLClassification()
# set the "q" in q-norm MKL
svm.set_mkl_norm(param.flags["mkl_q"])
# set interleaved optimization
if param.flags.has_key("interleaved"):
svm.set_interleaved_optimization_enabled(param.flags["interleaved"])
# set solver type
if param.flags.has_key("solver_type") and param.flags["solver_type"]:
if param.flags["solver_type"] == "ST_CPLEX":
svm.set_solver_type(ST_CPLEX)
if param.flags["solver_type"] == "ST_DIRECT":
svm.set_solver_type(ST_DIRECT)
if param.flags["solver_type"] == "ST_NEWTON":
svm.set_solver_type(ST_NEWTON)
if param.flags["solver_type"] == "ST_GLPK":
svm.set_solver_type(ST_GLPK)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create vanilla SVM
svm = SVMLight(param.cost, combined_kernel, lab)
# optimization settings
num_threads = 4
svm.parallel.set_num_threads(num_threads)
if param.flags.has_key("epsilon"):
svm.set_epsilon(param.flags["epsilon"])
# enable output
svm.io.enable_progress()
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
# disable unsupported optimizations (due to special normalizer)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# set cost
if param.flags["normalize_cost"]:
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
else:
svm.set_C(param.cost, param.cost)
# start training
svm.train()
########################################################
print "svm objective:"
print svm.get_objective()
########################################################
# store additional info
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["weights"] = combined_kernel.get_subkernel_weights()
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), len(nodes), combined_kernel, svm)
return svms
def test_data():
##################################################################
# select MSS
##################################################################
mss = expenv.MultiSplitSet.get(379)
##################################################################
# data
##################################################################
# fetch data
instance_set = mss.get_train_data(-1)
# prepare data
data = PreparedMultitaskData(instance_set, shuffle=True)
# set parameters
param = Options()
param.kernel = "WeightedDegreeStringKernel"
param.wdk_degree = 4
param.cost = 1.0
param.transform = 1.0
param.id = 666
param.freeze()
##################################################################
# taxonomy
##################################################################
taxonomy = shogun_factory.create_taxonomy(mss.taxonomy.data)
support = numpy.linspace(0, 100, 4)
distances = [[0, 1, 2, 2], [1, 0, 2, 2], [2, 2, 0, 1], [2, 2, 1, 0]]
# create tree normalizer
tree_normalizer = MultitaskKernelPlifNormalizer(support,
data.task_vector_names)
task_names = data.get_task_names()
FACTOR = 1.0
# init gamma matrix
gammas = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
for t1_name in task_names:
for t2_name in task_names:
similarity = taxonomy.compute_node_similarity(
taxonomy.get_id(t1_name), taxonomy.get_id(t2_name))
gammas[data.name_to_id(t1_name),
data.name_to_id(t2_name)] = similarity
helper.save("/tmp/gammas", gammas)
gammas = gammas * FACTOR
cost = param.cost * numpy.sqrt(FACTOR)
print gammas
##########
# regular normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
for t1_name in task_names:
for t2_name in task_names:
similarity = gammas[data.name_to_id(t1_name),
data.name_to_id(t2_name)]
normalizer.set_task_similarity(data.name_to_id(t1_name),
data.name_to_id(t2_name),
similarity)
##################################################################
# Train SVMs
##################################################################
# create shogun objects
wdk_tree = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
wdk_tree.set_normalizer(tree_normalizer)
wdk_tree.init_normalizer()
print "--->", wdk_tree.get_normalizer().get_name()
svm_tree = SVMLight(cost, wdk_tree, lab)
svm_tree.set_linadd_enabled(False)
svm_tree.set_batch_computation_enabled(False)
svm_tree.train()
del wdk_tree
del tree_normalizer
print "finished training tree-norm SVM:", svm_tree.get_objective()
wdk = shogun_factory.create_kernel(data.examples, param)
wdk.set_normalizer(normalizer)
wdk.init_normalizer()
print "--->", wdk.get_normalizer().get_name()
svm = SVMLight(cost, wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
print "finished training manually set SVM:", svm.get_objective()
alphas_tree = svm_tree.get_alphas()
alphas = svm.get_alphas()
assert (len(alphas_tree) == len(alphas))
for i in xrange(len(alphas)):
assert (abs(alphas_tree[i] - alphas[i]) < 0.0001)
print "success: all alphas are the same"
################################################################## # select MSS ################################################################## mss = expenv.MultiSplitSet.get(379) ################################################################## # data ################################################################## # fetch data instance_set = mss.get_train_data(-1) # prepare data data = PreparedMultitaskData(instance_set, shuffle=True) # set parameters param = Options() param.kernel = "WeightedDegreeStringKernel" param.wdk_degree = 4 param.cost = 1.0 param.transform = 1.0 param.id = 666 param.freeze() ################################################################## # taxonomy ################################################################## taxonomy = shogun_factory.create_taxonomy(mss.taxonomy.data)
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_empty_kernel(param)
lab = shogun_factory.create_labels(data.labels)
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples)
combined_features = CombinedFeatures()
# set normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
# load data
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_pearson.txt")
f = file(
"/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/All_PseudoSeq_Hamming.txt"
)
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_euklid.txt")
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_RAxML.txt")
num_lines = int(f.readline().strip())
task_distances = numpy.zeros((num_lines, num_lines))
name_to_id = {}
for (i, line) in enumerate(f):
tokens = line.strip().split("\t")
name = str(tokens[0])
name_to_id[name] = i
entry = numpy.array([v for (j, v) in enumerate(tokens) if j != 0])
assert len(entry) == num_lines, "len_entry %i, num_lines %i" % (
len(entry), num_lines)
task_distances[i, :] = entry
# cut relevant submatrix
active_ids = [name_to_id[name] for name in data.get_task_names()]
tmp_distances = task_distances[active_ids, :]
tmp_distances = tmp_distances[:, active_ids]
print "distances ", tmp_distances.shape
# normalize distances
task_distances = task_distances / numpy.max(tmp_distances)
similarities = numpy.zeros(
(data.get_num_tasks(), data.get_num_tasks()))
# convert distance to similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
# convert similarity with simple transformation
similarity = param.base_similarity - task_distances[
name_to_id[task_name_lhs], name_to_id[task_name_rhs]]
normalizer.set_task_similarity(data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs),
similarity)
# save for later
similarities[data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs)] = similarity
# set normalizer
base_wdk.set_normalizer(normalizer)
#base_wdk.init_normalizer()
combined_features.append_feature_obj(base_features)
combined_kernel.append_kernel(base_wdk)
##################################################
# intra-domain blocks
intra_block_vec = PairiiVec()
for task_id in data.get_task_ids():
intra_block_vec.push_back(Pairii(task_id, task_id))
# create mask-based normalizer
normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums,
intra_block_vec)
kernel = shogun_factory.create_empty_kernel(param)
kernel.set_normalizer(normalizer)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel)
# append features
combined_features.append_feature_obj(base_features)
# set mixing factor (used if MKL is OFF)
assert (param.base_similarity <= 1)
assert (param.base_similarity >= 0)
combined_kernel.set_subkernel_weights(
[param.base_similarity, 1 - param.base_similarity])
combined_kernel.init(combined_features, combined_features)
svm = None
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #ST_GLPK) #DIRECT) #NEWTON)#ST_CPLEX) #auto
svm.set_C(param.cost, param.cost)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
# set up SVM
num_threads = 8
svm.io.enable_progress()
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
print "WARNING: custom epsilon set"
svm.set_epsilon(0.05)
# normalize cost
norm_c_pos = param.cost / float(len([l
for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
self.additional_information["similarities"] = similarities
self.additional_information[
"post_weights"] = combined_kernel.get_subkernel_weights()
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, combined_kernel, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
import numpy
numpy.random.seed(666)
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_DEBUG)
# set kernel cache
if param.flags.has_key("cache_size"):
combined_kernel.set_cache_size(param.flags["cache_size"])
# create features
base_features = shogun_factory.create_features(data.examples, param)
combined_features = CombinedFeatures()
########################################################
print "creating a masked kernel for possible subset:"
########################################################
power_set_tasks = power_set(data.get_task_ids())
for active_task_ids in power_set_tasks:
print "masking all entries other than:", active_task_ids
# create mask-based normalizer
normalizer = MultitaskKernelMaskNormalizer(data.task_vector_nums,
data.task_vector_nums,
active_task_ids)
# normalize trace
if param.flags.has_key(
"normalize_trace") and param.flags["normalize_trace"]:
norm_factor = len(data.get_task_ids()) / len(active_task_ids)
normalizer.set_normalization_constant(norm_factor)
kernel = shogun_factory.create_empty_kernel(param)
kernel.set_normalizer(normalizer)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel)
# append features
combined_features.append_feature_obj(base_features)
print "------"
combined_kernel.init(combined_features, combined_features)
#combined_kernel.precompute_subkernels()
self.additional_information[
"weights before trainng"] = combined_kernel.get_subkernel_weights(
)
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.flags["mkl_q"] >= 1.0)
if param.flags["mkl_q"] >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.flags["mkl_q"])
# set interleaved optimization
if param.flags.has_key("interleaved"):
svm.set_interleaved_optimization_enabled(
param.flags["interleaved"])
# set solver type
if param.flags.has_key(
"solver_type") and param.flags["solver_type"]:
if param.flags["solver_type"] == "ST_CPLEX":
svm.set_solver_type(ST_CPLEX)
if param.flags["solver_type"] == "ST_DIRECT":
svm.set_solver_type(ST_DIRECT)
if param.flags["solver_type"] == "ST_NEWTON":
svm.set_solver_type(ST_NEWTON)
if param.flags["solver_type"] == "ST_GLPK":
svm.set_solver_type(ST_GLPK)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
svm = SVMLight(param.cost, combined_kernel, lab)
# optimization settings
num_threads = 4
svm.parallel.set_num_threads(num_threads)
if param.flags.has_key("epsilon"):
svm.set_epsilon(param.flags["epsilon"])
# enable output
svm.io.enable_progress()
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
# disable unsupported optimizations (due to special normalizer)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# set cost
if param.flags["normalize_cost"]:
norm_c_pos = param.cost / float(
len([l for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
else:
svm.set_C(param.cost, param.cost)
svm.train()
# prepare mapping
weight_map = {}
weights = combined_kernel.get_subkernel_weights()
for (i, pset) in enumerate(power_set_tasks):
print pset
subset_str = str([data.id_to_name(task_idx) for task_idx in pset])
weight_map[subset_str] = weights[i]
# store additional info
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["weight_map"] = weight_map
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name),
len(power_set_tasks), combined_kernel, svm,
param)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# set normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
########################################################
print "creating a kernel for each node:"
########################################################
# init seq handler
task_kernel = SequencesHandlerRbf(1, param.base_similarity, data.get_task_names(), param.flags["wdk_rbf_on"])
similarities = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
# convert distance to similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
# convert similarity with simple transformation
similarity = task_kernel.get_similarity(task_name_lhs, task_name_rhs)
print similarity
print "similarity (%s,%s)=%f" % (task_name_lhs, task_name_rhs, similarity)
normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity)
# save for later
similarities[data.name_to_id(task_name_lhs),data.name_to_id(task_name_rhs)] = similarity
# set normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
# set up svm
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# normalize cost
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
#self.additional_information["distances"] = distances
self.additional_information["similarities"] = similarities
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, param, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
kernel_matrix = base_wdk.get_kernel_matrix()
lab = shogun_factory.create_labels(data.labels)
# fetch taxonomy from parameter object
taxonomy = param.taxonomy.data
# create name to leaf map
nodes = taxonomy.get_all_nodes()
########################################################
print "creating a kernel for each node:"
########################################################
# assemble combined kernel
from shogun.Kernel import CombinedKernel, CustomKernel
combined_kernel = CombinedKernel()
# indicator to which task each example belongs
task_vector = data.task_vector_names
for node in nodes:
print "creating kernel for ", node.name
# fetch sub-tree
leaf_names = [leaf.name for leaf in node.get_leaves()]
print "masking all entries other than:", leaf_names
# init matrix
kernel_matrix_node = numpy.zeros(kernel_matrix.shape)
# fill matrix for node
for (i, task_lhs) in enumerate(task_vector):
for (j, task_rhs) in enumerate(task_vector):
# only copy values, if both tasks are present in subtree
if task_lhs in leaf_names and task_rhs in leaf_names:
kernel_matrix_node[i,j] = kernel_matrix[i,j]
# create custom kernel
kernel_node = CustomKernel()
kernel_node.set_full_kernel_matrix_from_full(kernel_matrix_node)
# append custom kernel to CombinedKernel
combined_kernel.append_kernel(kernel_node)
print "------"
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
num_threads = 4
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
svm.set_solver_type(ST_GLPK) #DIRECT) #NEWTON)#ST_CPLEX)
svm.set_C(param.cost, param.cost)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
svm.parallel.set_num_threads(num_threads)
#svm.set_linadd_enabled(False)
#svm.set_batch_computation_enabled(False)
svm.train()
print "subkernel weights (after):", combined_kernel.get_subkernel_weights()
else:
# create SVM (disable unsupported optimizations)
svm = SVMLight(param.cost, combined_kernel, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
########################################################
print "svm objective:"
print svm.get_objective()
########################################################
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_id in train_data.keys():
svms[task_id] = svm
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
########################################################
print "creating a kernel for each node:"
########################################################
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples)
combined_features = CombinedFeatures()
##################################################
# intra-domain blocks
# intra_block_vec = PairiiVec()
#
# for task_id in data.get_task_ids():
# intra_block_vec.push_back(Pairii(task_id, task_id))
#
#
#
# # create mask-based normalizer
# normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, intra_block_vec)
# kernel = shogun_factory.create_empty_kernel(param)
# kernel.set_normalizer(normalizer)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel)
#
# # append features
# combined_features.append_feature_obj(base_features)
#
# print "------"
#
# ##################################################
# # all blocks
#
#
# all_block_vec = PairiiVec()
#
# for task_id_1 in data.get_task_ids():
# for task_id_2 in data.get_task_ids():
# all_block_vec.push_back(Pairii(task_id_1, task_id_2))
#
#
# # create mask-based normalizer
# normalizer_all = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, all_block_vec)
# kernel_all = shogun_factory.create_empty_kernel(param)
# kernel_all.set_normalizer(normalizer_all)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel_all)
#
# # append features
# combined_features.append_feature_obj(base_features)
##################################################
# add one kernel per similarity position
# init seq handler
pseudoseqs = SequencesHandler()
pseudoseq_length = pseudoseqs.seq_length
for pos in range(pseudoseq_length):
print "appending kernel for pos %i" % (pos)
print "nums", data.task_vector_nums
pos_block_vec = PairiiVec()
# set similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
similarity = pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pos)
#print "computing similarity for tasks (%s, %s) = %i" % (task_name_lhs, task_name_rhs, similarity)
if similarity == 1:
tmp_pair = Pairii(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs))
pos_block_vec.push_back(tmp_pair)
print "creating normalizer"
normalizer_pos = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, pos_block_vec)
print "creating empty kernel"
kernel_pos = shogun_factory.create_empty_kernel(param)
print "setting normalizer"
kernel_pos.set_normalizer(normalizer_pos)
print "appending kernel"
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel_pos)
print "appending features"
# append features
combined_features.append_feature_obj(base_features)
print "done constructing combined kernel"
##################################################
# init combined kernel
combined_kernel.init(combined_features, combined_features)
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #ST_GLPK) #DIRECT) #NEWTON)#ST_CPLEX) #auto
svm.set_C(param.cost, param.cost)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
# set up SVM
num_threads = 8
svm.io.enable_progress()
#svm.io.set_loglevel(shogun.Classifier.MSG_INFO)
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
print "WARNING: custom epsilon set"
svm.set_epsilon(0.05)
# normalize cost
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional info
self.additional_information["svm_objective"] = svm.get_objective()
self.additional_information["svm num sv"] = svm.get_num_support_vectors()
self.additional_information["mkl weights post-training"] = combined_kernel.get_subkernel_weights()
print self.additional_information
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), combined_kernel, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# dict to save additional information for later analysis
self.additional_information = {}
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
########################################################
print "creating a kernel for each node:"
########################################################
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples, param)
combined_features = CombinedFeatures()
##################################################
# intra-domain blocks (dirac kernel)
intra_block_vec = PairiiVec()
for task_id in data.get_task_ids():
intra_block_vec.push_back(Pairii(task_id, task_id))
# create mask-based normalizer
normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, intra_block_vec)
kernel = shogun_factory.create_empty_kernel(param)
kernel.set_normalizer(normalizer)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel)
# append features
combined_features.append_feature_obj(base_features)
print "------"
##################################################
# all blocks (full kernel matrix)
all_block_vec = PairiiVec()
for task_id_1 in data.get_task_ids():
for task_id_2 in data.get_task_ids():
all_block_vec.push_back(Pairii(task_id_1, task_id_2))
# create mask-based normalizer
normalizer_all = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, all_block_vec)
kernel_all = shogun_factory.create_empty_kernel(param)
kernel_all.set_normalizer(normalizer_all)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel_all)
# append features
combined_features.append_feature_obj(base_features)
##################################################
# hack
# hack_block_vec = PairiiVec()
#
# for task_id_1 in data.get_task_ids():
# for task_id_2 in data.get_task_ids():
# hack_block_vec.push_back(Pairii(task_id_1, task_id_2))
#
# hack_block_vec.push_back(Pairii(data.name_to_id("B_2705"), data.name_to_id("B_4001")))
# other_group = ["B_0702", "B_1501", "B_5801"]
# for task_id_1 in other_group:
# for task_id_2 in other_group:
# hack_block_vec.push_back(Pairii(data.name_to_id(task_id_1), data.name_to_id(task_id_2)))
#
#
#
# # create mask-based normalizer
# normalizer_hack = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, hack_block_vec)
# kernel_hack = shogun_factory.create_empty_kernel(param)
# kernel_hack.set_normalizer(normalizer_hack)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel_hack)
#
# # append features
# combined_features.append_feature_obj(base_features)
##################################################
# init combined kernel
combined_kernel.init(combined_features, combined_features)
#combined_kernel.precompute_subkernels()
self.additional_information["mkl weights before"] = combined_kernel.get_subkernel_weights()
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.flags["mkl_q"] >= 1.0)
if param.flags["mkl_q"] >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.flags["mkl_q"])
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
num_threads = 8
svm.io.enable_progress()
svm.io.set_loglevel(shogun.Classifier.MSG_INFO)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.set_epsilon(0.03)
# set cost
if param.flags["normalize_cost"]:
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
else:
svm.set_C(param.cost, param.cost)
svm.train()
print "subkernel weights (after):", combined_kernel.get_subkernel_weights()
########################################################
print "svm objective:"
print svm.get_objective()
self.additional_information["svm_objective"] = svm.get_objective()
self.additional_information["svm num sv"] = svm.get_num_support_vectors()
self.additional_information["mkl weights post-training"] = combined_kernel.get_subkernel_weights()
########################################################
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), combined_kernel, svm, param)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
##################################################
# define pockets
##################################################
pockets = [0]*9
pockets[0] = [1, 5, 6, 7, 8, 31, 32, 33, 34]
pockets[1] = [1, 2, 3, 4, 6, 7, 8, 9, 11, 21, 31]
pockets[2] = [11, 20, 21, 22, 29, 31]
pockets[3] = [8, 30, 31, 32]
pockets[4] = [10, 11, 30]
pockets[5] = [10, 11, 12, 13, 20, 29]
pockets[6] = [10, 12, 20, 22, 26, 27, 28, 29]
pockets[7] = [12, 14, 15, 26]
pockets[8] = [13, 15, 16, 17, 18, 19, 20, 23, 24, 25, 26]
#new_pockets = []
# merge neighboring pockets
#for i in range(8):
# new_pockets.append(list(set(pockets[i]).union(set(pockets[i+1]))))
#pockets = new_pockets
########################################################
print "creating a kernel:"
########################################################
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples)
combined_features = CombinedFeatures()
##################################################
# intra-domain blocks
# intra_block_vec = PairiiVec()
#
# for task_id in data.get_task_ids():
# intra_block_vec.push_back(Pairii(task_id, task_id))
#
#
#
# # create mask-based normalizer
# normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, intra_block_vec)
# kernel = shogun_factory.create_empty_kernel(param)
# kernel.set_normalizer(normalizer)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel)
#
# # append features
# combined_features.append_feature_obj(base_features)
#
# print "------"
#
# ##################################################
# # all blocks
#
#
# all_block_vec = PairiiVec()
#
# for task_id_1 in data.get_task_ids():
# for task_id_2 in data.get_task_ids():
# all_block_vec.push_back(Pairii(task_id_1, task_id_2))
#
#
# # create mask-based normalizer
# normalizer_all = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, all_block_vec)
# kernel_all = shogun_factory.create_empty_kernel(param)
# kernel_all.set_normalizer(normalizer_all)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel_all)
#
# # append features
# combined_features.append_feature_obj(base_features)
##################################################
# add one kernel per similarity position
# init seq handler
pseudoseqs = SequencesHandler()
for pocket in pockets:
print "creating normalizer"
#import pdb
#pdb.set_trace()
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
print "processing pocket", pocket
# set similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
similarity = 0.0
for pseudo_seq_pos in pocket:
similarity += float(pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pseudo_seq_pos-1))
# normalize
similarity = similarity / float(len(pocket))
print "pocket %s (%s, %s) = %f" % (str(pocket), task_name_lhs, task_name_rhs, similarity)
normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity)
print "creating empty kernel"
kernel_pos = shogun_factory.create_empty_kernel(param)
print "setting normalizer"
kernel_pos.set_normalizer(normalizer)
print "appending kernel"
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel_pos)
print "appending features"
# append features
combined_features.append_feature_obj(base_features)
print "done constructing combined kernel"
##################################################
# init combined kernel
# init weights
# combined_kernel.set_subkernel_weights([1.0/2.85]*combined_kernel.get_num_subkernels())
combined_kernel.init(combined_features, combined_features)
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #ST_GLPK) #DIRECT) #NEWTON)#ST_CPLEX) #auto
svm.set_C(param.cost, param.cost)
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
# set up SVM
num_threads = 8
svm.io.enable_progress()
#svm.io.set_loglevel(shogun.Classifier.MSG_INFO)
svm.io.set_loglevel(shogun.Classifier.MSG_DEBUG)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
#print "WARNING: custom epsilon set"
#svm.set_epsilon(0.05)
# normalize cost
norm_c_pos = param.cost / float(len([l for l in data.labels if l==1]))
norm_c_neg = param.cost / float(len([l for l in data.labels if l==-1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional info
self.additional_information["svm_objective"] = svm.get_objective()
self.additional_information["svm num sv"] = svm.get_num_support_vectors()
self.additional_information["post_weights"] = combined_kernel.get_subkernel_weights()
print self.additional_information
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), combined_kernel, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# dict to save additional information for later analysis
self.additional_information = {}
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
########################################################
print "creating a kernel for each node:"
########################################################
# assemble combined kernel
combined_kernel = CombinedKernel()
combined_kernel.io.set_loglevel(shogun.Kernel.MSG_INFO)
base_features = shogun_factory.create_features(data.examples, param)
combined_features = CombinedFeatures()
##################################################
# intra-domain blocks (dirac kernel)
intra_block_vec = PairiiVec()
for task_id in data.get_task_ids():
intra_block_vec.push_back(Pairii(task_id, task_id))
# create mask-based normalizer
normalizer = MultitaskKernelMaskPairNormalizer(data.task_vector_nums,
intra_block_vec)
kernel = shogun_factory.create_empty_kernel(param)
kernel.set_normalizer(normalizer)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel)
# append features
combined_features.append_feature_obj(base_features)
print "------"
##################################################
# all blocks (full kernel matrix)
all_block_vec = PairiiVec()
for task_id_1 in data.get_task_ids():
for task_id_2 in data.get_task_ids():
all_block_vec.push_back(Pairii(task_id_1, task_id_2))
# create mask-based normalizer
normalizer_all = MultitaskKernelMaskPairNormalizer(
data.task_vector_nums, all_block_vec)
kernel_all = shogun_factory.create_empty_kernel(param)
kernel_all.set_normalizer(normalizer_all)
# append current kernel to CombinedKernel
combined_kernel.append_kernel(kernel_all)
# append features
combined_features.append_feature_obj(base_features)
##################################################
# hack
# hack_block_vec = PairiiVec()
#
# for task_id_1 in data.get_task_ids():
# for task_id_2 in data.get_task_ids():
# hack_block_vec.push_back(Pairii(task_id_1, task_id_2))
#
# hack_block_vec.push_back(Pairii(data.name_to_id("B_2705"), data.name_to_id("B_4001")))
# other_group = ["B_0702", "B_1501", "B_5801"]
# for task_id_1 in other_group:
# for task_id_2 in other_group:
# hack_block_vec.push_back(Pairii(data.name_to_id(task_id_1), data.name_to_id(task_id_2)))
#
#
#
# # create mask-based normalizer
# normalizer_hack = MultitaskKernelMaskPairNormalizer(data.task_vector_nums, hack_block_vec)
# kernel_hack = shogun_factory.create_empty_kernel(param)
# kernel_hack.set_normalizer(normalizer_hack)
#
# # append current kernel to CombinedKernel
# combined_kernel.append_kernel(kernel_hack)
#
# # append features
# combined_features.append_feature_obj(base_features)
##################################################
# init combined kernel
combined_kernel.init(combined_features, combined_features)
#combined_kernel.precompute_subkernels()
self.additional_information[
"mkl weights before"] = combined_kernel.get_subkernel_weights()
print "subkernel weights:", combined_kernel.get_subkernel_weights()
svm = None
print "using MKL:", (param.flags["mkl_q"] >= 1.0)
if param.flags["mkl_q"] >= 1.0:
svm = MKLClassification()
svm.set_mkl_norm(param.flags["mkl_q"])
svm.set_kernel(combined_kernel)
svm.set_labels(lab)
else:
# create SVM (disable unsupported optimizations)
combined_kernel.set_cache_size(500)
svm = SVMLight(param.cost, combined_kernel, lab)
num_threads = 8
svm.io.enable_progress()
svm.io.set_loglevel(shogun.Classifier.MSG_INFO)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.set_epsilon(0.03)
# set cost
if param.flags["normalize_cost"]:
norm_c_pos = param.cost / float(
len([l for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
else:
svm.set_C(param.cost, param.cost)
svm.train()
print "subkernel weights (after):", combined_kernel.get_subkernel_weights(
)
########################################################
print "svm objective:"
print svm.get_objective()
self.additional_information["svm_objective"] = svm.get_objective()
self.additional_information[
"svm num sv"] = svm.get_num_support_vectors()
self.additional_information[
"mkl weights post-training"] = combined_kernel.get_subkernel_weights(
)
########################################################
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (data.name_to_id(task_name), combined_kernel,
svm, param)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
assert (param.base_similarity >= 1)
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# set normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
# load data
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_pearson.txt")
f = file(
"/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/All_PseudoSeq_Hamming.txt"
)
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_PseudoSeq_BlosumEnc_euklid.txt")
#f = file("/fml/ag-raetsch/home/cwidmer/Documents/phd/projects/multitask/data/mhc/MHC_Distanzen/MHC_Distanzen/ALL_RAxML.txt")
num_lines = int(f.readline().strip())
task_distances = numpy.zeros((num_lines, num_lines))
name_to_id = {}
for (i, line) in enumerate(f):
tokens = line.strip().split("\t")
name = str(tokens[0])
name_to_id[name] = i
entry = numpy.array([v for (j, v) in enumerate(tokens) if j != 0])
assert len(entry) == num_lines, "len_entry %i, num_lines %i" % (
len(entry), num_lines)
task_distances[i, :] = entry
# cut relevant submatrix
active_ids = [name_to_id[name] for name in data.get_task_names()]
tmp_distances = task_distances[active_ids, :]
tmp_distances = tmp_distances[:, active_ids]
print "distances ", tmp_distances.shape
# normalize distances
task_distances = task_distances / numpy.max(tmp_distances)
similarities = numpy.zeros(
(data.get_num_tasks(), data.get_num_tasks()))
# convert distance to similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
# convert similarity with simple transformation
similarity = param.base_similarity - task_distances[
name_to_id[task_name_lhs], name_to_id[task_name_rhs]]
normalizer.set_task_similarity(data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs),
similarity)
# save for later
similarities[data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs)] = similarity
# set normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
# set up svm
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# normalize cost
norm_c_pos = param.cost / float(len([l
for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
#self.additional_information["distances"] = distances
self.additional_information["similarities"] = similarities
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, param, svm)
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# support
support = numpy.linspace(0, 1, 5)
# set normalizer
normalizer = MultitaskKernelPlifNormalizer(support, data.task_vector_nums)
# fetch taxonomy from parameter object
taxonomy = param.taxonomy.data
taxonomy.plot()
import os
os.system("evince demo.png &")
# compute distances
distances = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
for (i,task_name_lhs) in enumerate(data.get_task_names()):
for (j, task_name_rhs) in enumerate(data.get_task_names()):
distances[i,j] = task_similarities.compute_hop_distance(taxonomy, task_name_lhs, task_name_rhs)
# normalize distances
distances = distances / numpy.max(distances)
# set distances
for (i,task_name_lhs) in enumerate(data.get_task_names()):
for (j, task_name_rhs) in enumerate(data.get_task_names()):
normalizer.set_task_distance(i, j, distances[i,j])
# assign normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
svm = None
debug_weights = {}
num_subk = base_wdk.get_num_subkernels()
print "num subkernels:", num_subk
#print "subkernel weights:", base_wdk.get_subkernel_weights()
debug_weights["before"] = [normalizer.get_beta(i) for i in range(num_subk)]
print "using MKL:", (param.transform >= 1.0)
if param.transform >= 1.0:
num_threads = 4
svm = MKLClassification()
svm.set_mkl_norm(param.transform)
#svm.set_solver_type(ST_CPLEX) #GLPK) #DIRECT) #NEWTON)#ST_CPLEX)
svm.set_C(param.cost, param.cost)
svm.set_kernel(base_wdk)
svm.set_labels(lab)
svm.parallel.set_num_threads(num_threads)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
#print "subkernel weights (after):", base_wdk.get_subkernel_weights()
else:
# create SVM (disable unsupported optimizations)
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
print "svm objective:", svm.get_objective()
debug_weights["after"] = [normalizer.get_beta(i) for i in range(num_subk)]
# debugging output
print "debug weights (before/after):"
print debug_weights["before"]
print debug_weights["after"]
print ""
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_data.keys():
svms[task_name] = (svm, data.name_to_id(task_name))
return svms
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
for task_id in train_data.keys():
print "task_id:", task_id
# split data for training weak_learners and boosting
(train_weak, train_boosting) = split_data(train_data, 4)
# train on first part of dataset (evaluate on other)
prepared_data_weak = PreparedMultitaskData(train_weak, shuffle=False)
classifiers = self._inner_train(prepared_data_weak, param)
# train on entire dataset
prepared_data_final = PreparedMultitaskData(train_data, shuffle=False)
final_classifiers = self._inner_train(prepared_data_final, param)
print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
print "done training weak learners"
#####################################################
# perform boosting and wrap things up
#####################################################
# wrap up predictors for later use
predictors = {}
for task_name in train_boosting.keys():
instances = train_boosting[task_name]
N = len(instances)
F = len(classifiers)
examples = [inst.example for inst in instances]
labels = [inst.label for inst in instances]
# dim = (F x N)
out = cvxmod.zeros((N,F))
for i in xrange(F):
svm = classifiers[i]
tmp_out = self._predict_weak(svm, examples, prepared_data_weak.name_to_id(task_name))
if param.flags["signum"]:
out[:,i] = numpy.sign(tmp_out)
else:
out[:,i] = tmp_out
if param.flags["boosting"] == "ones":
weights = numpy.ones(F)/float(F)
if param.flags["boosting"] == "L1":
weights = solve_boosting(out, labels, param.transform, solver="glpk")
if param.flags["boosting"] == "L2":
weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=False)
if param.flags["boosting"] == "L2_reg":
weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=True)
predictors[task_name] = (final_classifiers, weights, prepared_data_final.name_to_id(task_name))
assert prepared_data_final.name_to_id(task_name)==prepared_data_weak.name_to_id(task_name), "name mappings don't match"
#####################################################
# Some sanity checks
#####################################################
# make sure we have the same keys (potentiall in a different order)
sym_diff_keys = set(train_weak.keys()).symmetric_difference(set(predictors.keys()))
assert len(sym_diff_keys)==0, "symmetric difference between keys non-empty: " + str(sym_diff_keys)
return predictors
def test_data():
##################################################################
# select MSS
##################################################################
mss = expenv.MultiSplitSet.get(379)
##################################################################
# data
##################################################################
# fetch data
instance_set = mss.get_train_data(-1)
# prepare data
data = PreparedMultitaskData(instance_set, shuffle=True)
# set parameters
param = Options()
param.kernel = "WeightedDegreeStringKernel"
param.wdk_degree = 4
param.cost = 1.0
param.transform = 1.0
param.id = 666
param.freeze()
##################################################################
# taxonomy
##################################################################
taxonomy = shogun_factory.create_taxonomy(mss.taxonomy.data)
support = numpy.linspace(0, 100, 4)
distances = [[0, 1, 2, 2], [1, 0, 2, 2], [2, 2, 0, 1], [2, 2, 1, 0]]
# create tree normalizer
tree_normalizer = MultitaskKernelPlifNormalizer(support, data.task_vector_names)
task_names = data.get_task_names()
FACTOR = 1.0
# init gamma matrix
gammas = numpy.zeros((data.get_num_tasks(), data.get_num_tasks()))
for t1_name in task_names:
for t2_name in task_names:
similarity = taxonomy.compute_node_similarity(taxonomy.get_id(t1_name), taxonomy.get_id(t2_name))
gammas[data.name_to_id(t1_name), data.name_to_id(t2_name)] = similarity
helper.save("/tmp/gammas", gammas)
gammas = gammas * FACTOR
cost = param.cost * numpy.sqrt(FACTOR)
print gammas
##########
# regular normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
for t1_name in task_names:
for t2_name in task_names:
similarity = gammas[data.name_to_id(t1_name), data.name_to_id(t2_name)]
normalizer.set_task_similarity(data.name_to_id(t1_name), data.name_to_id(t2_name), similarity)
##################################################################
# Train SVMs
##################################################################
# create shogun objects
wdk_tree = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
wdk_tree.set_normalizer(tree_normalizer)
wdk_tree.init_normalizer()
print "--->",wdk_tree.get_normalizer().get_name()
svm_tree = SVMLight(cost, wdk_tree, lab)
svm_tree.set_linadd_enabled(False)
svm_tree.set_batch_computation_enabled(False)
svm_tree.train()
del wdk_tree
del tree_normalizer
print "finished training tree-norm SVM:", svm_tree.get_objective()
wdk = shogun_factory.create_kernel(data.examples, param)
wdk.set_normalizer(normalizer)
wdk.init_normalizer()
print "--->",wdk.get_normalizer().get_name()
svm = SVMLight(cost, wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
svm.train()
print "finished training manually set SVM:", svm.get_objective()
alphas_tree = svm_tree.get_alphas()
alphas = svm.get_alphas()
assert(len(alphas_tree)==len(alphas))
for i in xrange(len(alphas)):
assert(abs(alphas_tree[i] - alphas[i]) < 0.0001)
print "success: all alphas are the same"
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# merge data sets
data = PreparedMultitaskData(train_data, shuffle=False)
# create shogun data objects
base_wdk = shogun_factory.create_kernel(data.examples, param)
lab = shogun_factory.create_labels(data.labels)
# set normalizer
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
########################################################
print "creating a kernel for each node:"
########################################################
# init seq handler
task_kernel = SequencesHandlerRbf(1, param.base_similarity,
data.get_task_names(),
param.flags["wdk_rbf_on"])
similarities = numpy.zeros(
(data.get_num_tasks(), data.get_num_tasks()))
# convert distance to similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
# convert similarity with simple transformation
similarity = task_kernel.get_similarity(
task_name_lhs, task_name_rhs)
print similarity
print "similarity (%s,%s)=%f" % (task_name_lhs, task_name_rhs,
similarity)
normalizer.set_task_similarity(data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs),
similarity)
# save for later
similarities[data.name_to_id(task_name_lhs),
data.name_to_id(task_name_rhs)] = similarity
# set normalizer
base_wdk.set_normalizer(normalizer)
base_wdk.init_normalizer()
# set up svm
svm = SVMLight(param.cost, base_wdk, lab)
svm.set_linadd_enabled(False)
svm.set_batch_computation_enabled(False)
# normalize cost
norm_c_pos = param.cost / float(len([l
for l in data.labels if l == 1]))
norm_c_neg = param.cost / float(
len([l for l in data.labels if l == -1]))
svm.set_C(norm_c_neg, norm_c_pos)
# start training
svm.train()
# save additional information
self.additional_information["svm objective"] = svm.get_objective()
self.additional_information["num sv"] = svm.get_num_support_vectors()
#self.additional_information["distances"] = distances
self.additional_information["similarities"] = similarities
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in data.get_task_names():
task_num = data.name_to_id(task_name)
# save svm and task_num
svms[task_name] = (task_num, param, svm)
return svms
# select MSS ################################################################## mss = expenv.MultiSplitSet.get(379) ################################################################## # data ################################################################## # fetch data instance_set = mss.get_train_data(-1) # prepare data data = PreparedMultitaskData(instance_set, shuffle=True) # set parameters param = Options() param.kernel = "WeightedDegreeStringKernel" param.wdk_degree = 4 param.cost = 1.0 param.transform = 1.0 param.id = 666 param.freeze() ################################################################## # taxonomy
def _train(self, train_data, param):
"""
training procedure using training examples and labels
@param train_data: Data relevant to SVM training
@type train_data: dict<str, list<instances> >
@param param: Parameters for the training procedure
@type param: ParameterSvm
"""
# split for training weak_learners and boosting
(train_weak, train_boosting) = split_data(train_data, 4)
# merge data sets
data = PreparedMultitaskData(train_weak, shuffle=True)
# create shogun label
lab = shogun_factory.create_labels(data.labels)
########################################################
print "creating a kernel:"
########################################################
# init seq handler
pseudoseqs = SequencesHandler()
classifiers = []
for pocket in pockets:
print "creating normalizer"
#import pdb
#pdb.set_trace()
normalizer = MultitaskKernelNormalizer(data.task_vector_nums)
print "processing pocket", pocket
# set similarity
for task_name_lhs in data.get_task_names():
for task_name_rhs in data.get_task_names():
similarity = 0.0
for pseudo_seq_pos in pocket:
similarity += float(pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pseudo_seq_pos-1))
# normalize
similarity = similarity / float(len(pocket))
print "pocket %s (%s, %s) = %f" % (str(pocket), task_name_lhs, task_name_rhs, similarity)
normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity)
print "creating empty kernel"
kernel = shogun_factory.create_kernel(data.examples, param)
print "setting normalizer"
kernel.set_normalizer(normalizer)
print "training SVM for pocket", pocket
svm = self._train_single_svm(param, kernel, lab)
classifiers.append(svm)
print "done obtaining weak learners"
# save additional info
#self.additional_information["svm_objective"] = svm.get_objective()
#self.additional_information["svm num sv"] = svm.get_num_support_vectors()
#self.additional_information["post_weights"] = combined_kernel.get_subkernel_weights()
#print self.additional_information
##################################################
# combine weak learners for each task
##################################################
# set constants
some = 0.9
import cvxmod
# wrap up predictors
svms = {}
# use a reference to the same svm several times
for task_name in train_boosting.keys():
instances = train_boosting[task_name]
N = len(instances)
F = len(pockets)
examples = [inst.example for inst in instances]
labels = [inst.label for inst in instances]
# dim = (F x N)
out = cvxmod.zeros((N,F))
for i in xrange(F):
svm = classifiers[i]
tmp_out = self._predict_weak(svm, examples, data.name_to_id(task_name))
out[:,i] = numpy.sign(tmp_out)
#out[:,i] = tmp_out
#TODO: fix
helper.save("/tmp/out_sparse", (out,labels))
pdb.set_trace()
weights = solve_boosting(out, labels, some, solver="mosek")
svms[task_name] = (data.name_to_id(task_name), svm)
return svms