def mkl_multiclass_1(fm_train_real, fm_test_real, label_train_multiclass, C):
kernel = CombinedKernel()
feats_train = CombinedFeatures()
feats_test = CombinedFeatures()
for i in range(-10, 11):
subkfeats_train = RealFeatures(fm_train_real)
subkfeats_test = RealFeatures(fm_test_real)
subkernel = GaussianKernel(pow(2, i + 1))
feats_train.append_feature_obj(subkfeats_train)
feats_test.append_feature_obj(subkfeats_test)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_train)
labels = MulticlassLabels(label_train_multiclass)
mkl = MKLMulticlass(C, kernel, labels)
mkl.set_epsilon(1e-2)
mkl.parallel.set_num_threads(num_threads)
mkl.set_mkl_epsilon(mkl_epsilon)
mkl.set_mkl_norm(1)
mkl.train()
kernel.init(feats_train, feats_test)
out = mkl.apply().get_labels()
return out
def create_combined_kernel(kname, kparam, examples, train_mode, preproc):
"""A wrapper for creating combined kernels.
kname, kparam and examples are lists.
"""
num_kernels = len(kname)
feats['combined'] = CombinedFeatures()
kernel = CombinedKernel()
for kix in xrange(num_kernels):
cur_kname = '%s%d' % (kname[kix],kix)
(cur_feats, cur_preproc) = create_features(kname[kix], examples[kix], kparam[kix], train_mode, preproc)
feats[cur_kname] = cur_feats
cur_kernel = create_kernel(kname[kix], kparam[kix], cur_feats)
kernel.append_kernel(cur_kernel)
return (feats,kernel)
def create_combined_kernel(kname, kparam, examples, train_mode, preproc):
"""A wrapper for creating combined kernels.
kname, kparam and examples are lists.
"""
num_kernels = len(kname)
feats['combined'] = CombinedFeatures()
kernel = CombinedKernel()
for kix in xrange(num_kernels):
cur_kname = '%s%d' % (kname[kix], kix)
(cur_feats, cur_preproc) = create_features(kname[kix], examples[kix],
kparam[kix], train_mode,
preproc)
feats[cur_kname] = cur_feats
cur_kernel = create_kernel(kname[kix], kparam[kix], cur_feats)
kernel.append_kernel(cur_kernel)
return (feats, kernel)
def mkl_multiclass(fm_train_real, fm_test_real, label_train_multiclass, width,
C, epsilon, num_threads, mkl_epsilon, mkl_norm):
from shogun import CombinedFeatures, RealFeatures, MulticlassLabels
from shogun import CombinedKernel, GaussianKernel, LinearKernel, PolyKernel
from shogun import MKLMulticlass
kernel = CombinedKernel()
feats_train = CombinedFeatures()
feats_test = CombinedFeatures()
subkfeats_train = RealFeatures(fm_train_real)
subkfeats_test = RealFeatures(fm_test_real)
subkernel = GaussianKernel(10, width)
feats_train.append_feature_obj(subkfeats_train)
feats_test.append_feature_obj(subkfeats_test)
kernel.append_kernel(subkernel)
subkfeats_train = RealFeatures(fm_train_real)
subkfeats_test = RealFeatures(fm_test_real)
subkernel = LinearKernel()
feats_train.append_feature_obj(subkfeats_train)
feats_test.append_feature_obj(subkfeats_test)
kernel.append_kernel(subkernel)
subkfeats_train = RealFeatures(fm_train_real)
subkfeats_test = RealFeatures(fm_test_real)
subkernel = PolyKernel(10, 2)
feats_train.append_feature_obj(subkfeats_train)
feats_test.append_feature_obj(subkfeats_test)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_train)
labels = MulticlassLabels(label_train_multiclass)
mkl = MKLMulticlass(C, kernel, labels)
mkl.set_epsilon(epsilon)
mkl.parallel.set_num_threads(num_threads)
mkl.set_mkl_epsilon(mkl_epsilon)
mkl.set_mkl_norm(mkl_norm)
mkl.train()
kernel.init(feats_train, feats_test)
out = mkl.apply().get_labels()
return out
def mkl_multiclass (fm_train_real, fm_test_real, label_train_multiclass, width, C, epsilon, num_threads, mkl_epsilon, mkl_norm): from shogun import CombinedFeatures, RealFeatures, MulticlassLabels from shogun import CombinedKernel, GaussianKernel, LinearKernel,PolyKernel from shogun import MKLMulticlass kernel = CombinedKernel() feats_train = CombinedFeatures() feats_test = CombinedFeatures() subkfeats_train = RealFeatures(fm_train_real) subkfeats_test = RealFeatures(fm_test_real) subkernel = GaussianKernel(10, width) feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) subkfeats_train = RealFeatures(fm_train_real) subkfeats_test = RealFeatures(fm_test_real) subkernel = LinearKernel() feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) subkfeats_train = RealFeatures(fm_train_real) subkfeats_test = RealFeatures(fm_test_real) subkernel = PolyKernel(10,2) feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) kernel.init(feats_train, feats_train) labels = MulticlassLabels(label_train_multiclass) mkl = MKLMulticlass(C, kernel, labels) mkl.set_epsilon(epsilon); mkl.parallel.set_num_threads(num_threads) mkl.set_mkl_epsilon(mkl_epsilon) mkl.set_mkl_norm(mkl_norm) mkl.train() kernel.init(feats_train, feats_test) out = mkl.apply().get_labels() return out
def linear_time_mmd_graphical():
# parameters, change to get different results
m=1000 # set to 10000 for a good test result
dim=2
# setting the difference of the first dimension smaller makes a harder test
difference=1
# number of samples taken from null and alternative distribution
num_null_samples=150
# streaming data generator for mean shift distributions
gen_p=MeanShiftDataGenerator(0, dim)
gen_q=MeanShiftDataGenerator(difference, dim)
# use the median kernel selection
# create combined kernel with Gaussian kernels inside (shoguns Gaussian kernel is
# compute median data distance in order to use for Gaussian kernel width
# 0.5*median_distance normally (factor two in Gaussian kernel)
# However, shoguns kernel width is different to usual parametrization
# Therefore 0.5*2*median_distance^2
# Use a subset of data for that, only 200 elements. Median is stable
sigmas=[2**x for x in range(-3,10)]
widths=[x*x*2 for x in sigmas]
print "kernel widths:", widths
combined=CombinedKernel()
for i in range(len(sigmas)):
combined.append_kernel(GaussianKernel(10, widths[i]))
# mmd instance using streaming features, blocksize of 10000
block_size=1000
mmd=LinearTimeMMD(combined, gen_p, gen_q, m, block_size)
# kernel selection instance (this can easily replaced by the other methods for selecting
# single kernels
selection=MMDKernelSelectionOpt(mmd)
# perform kernel selection
kernel=selection.select_kernel()
kernel=GaussianKernel.obtain_from_generic(kernel)
mmd.set_kernel(kernel);
print "selected kernel width:", kernel.get_width()
# sample alternative distribution, stream ensures different samples each run
alt_samples=zeros(num_null_samples)
for i in range(len(alt_samples)):
alt_samples[i]=mmd.compute_statistic()
# sample from null distribution
# bootstrapping, biased statistic
mmd.set_null_approximation_method(PERMUTATION)
mmd.set_num_null_samples(num_null_samples)
null_samples_boot=mmd.sample_null()
# fit normal distribution to null and sample a normal distribution
mmd.set_null_approximation_method(MMD1_GAUSSIAN)
variance=mmd.compute_variance_estimate()
null_samples_gaussian=normal(0,sqrt(variance),num_null_samples)
# to plot data, sample a few examples from stream first
features=gen_p.get_streamed_features(m)
features=features.create_merged_copy(gen_q.get_streamed_features(m))
data=features.get_feature_matrix()
# plot
figure()
# plot data of p and q
subplot(2,3,1)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 4) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 4) ) # reduce number of x-ticks
plot(data[0][0:m], data[1][0:m], 'ro', label='$x$')
plot(data[0][m+1:2*m], data[1][m+1:2*m], 'bo', label='$x$', alpha=0.5)
title('Data, shift in $x_1$='+str(difference)+'\nm='+str(m))
xlabel('$x_1, y_1$')
ylabel('$x_2, y_2$')
# histogram of first data dimension and pdf
subplot(2,3,2)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(data[0], bins=50, alpha=0.5, facecolor='r', normed=True)
hist(data[1], bins=50, alpha=0.5, facecolor='b', normed=True)
xs=linspace(min(data[0])-1,max(data[0])+1, 50)
plot(xs,normpdf( xs, 0, 1), 'r', linewidth=3)
plot(xs,normpdf( xs, difference, 1), 'b', linewidth=3)
xlabel('$x_1, y_1$')
ylabel('$p(x_1), p(y_1)$')
title('Data PDF in $x_1, y_1$')
# compute threshold for test level
alpha=0.05
null_samples_boot.sort()
null_samples_gaussian.sort()
thresh_boot=null_samples_boot[floor(len(null_samples_boot)*(1-alpha))];
thresh_gaussian=null_samples_gaussian[floor(len(null_samples_gaussian)*(1-alpha))];
type_one_error_boot=sum(null_samples_boot<thresh_boot)/float(num_null_samples)
type_one_error_gaussian=sum(null_samples_gaussian<thresh_boot)/float(num_null_samples)
# plot alternative distribution with threshold
subplot(2,3,4)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(alt_samples, 20, normed=True);
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
type_two_error=sum(alt_samples<thresh_boot)/float(num_null_samples)
title('Alternative Dist.\n' + 'Type II error is ' + str(type_two_error))
# compute range for all null distribution histograms
hist_range=[min([min(null_samples_boot), min(null_samples_gaussian)]), max([max(null_samples_boot), max(null_samples_gaussian)])]
# plot null distribution with threshold
subplot(2,3,3)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(null_samples_boot, 20, range=hist_range, normed=True);
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
title('Sampled Null Dist.\n' + 'Type I error is ' + str(type_one_error_boot))
# plot null distribution gaussian
subplot(2,3,5)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(null_samples_gaussian, 20, range=hist_range, normed=True);
axvline(thresh_gaussian, 0, 1, linewidth=2, color='red')
title('Null Dist. Gaussian\nType I error is ' + str(type_one_error_gaussian))
# pull plots a bit apart
subplots_adjust(hspace=0.5)
subplots_adjust(wspace=0.5)
def evaluation_cross_validation_mkl_weight_storage(traindat=traindat, label_traindat=label_traindat):
from shogun import CrossValidation, CrossValidationResult
from shogun import ParameterObserverCV
from shogun import ContingencyTableEvaluation, ACCURACY
from shogun import StratifiedCrossValidationSplitting
from shogun import BinaryLabels
from shogun import RealFeatures, CombinedFeatures
from shogun import GaussianKernel, CombinedKernel
from shogun import LibSVM, MKLClassification
# training data, combined features all on same data
features=RealFeatures(traindat)
comb_features=CombinedFeatures()
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
labels=BinaryLabels(label_traindat)
# kernel, different Gaussians combined
kernel=CombinedKernel()
kernel.append_kernel(GaussianKernel(10, 0.1))
kernel.append_kernel(GaussianKernel(10, 1))
kernel.append_kernel(GaussianKernel(10, 2))
# create mkl using libsvm, due to a mem-bug, interleaved is not possible
svm=MKLClassification(LibSVM());
svm.set_interleaved_optimization_enabled(False);
svm.set_kernel(kernel);
# splitting strategy for 5 fold cross-validation (for classification its better
# to use "StratifiedCrossValidation", but the standard
# "StratifiedCrossValidationSplitting" is also available
splitting_strategy=StratifiedCrossValidationSplitting(labels, 5)
# evaluation method
evaluation_criterium=ContingencyTableEvaluation(ACCURACY)
# cross-validation instance
cross_validation=CrossValidation(svm, comb_features, labels,
splitting_strategy, evaluation_criterium)
cross_validation.set_autolock(False)
# append cross vlaidation output classes
mkl_storage=ParameterObserverCV()
cross_validation.subscribe_to_parameters(mkl_storage)
cross_validation.set_num_runs(3)
# perform cross-validation
result=cross_validation.evaluate()
# print mkl weights
weights = []
for obs_index in range(mkl_storage.get_num_observations()):
obs = mkl_storage.get_observation(obs_index)
for fold_index in range(obs.get_num_folds()):
fold = obs.get_fold(fold_index)
machine = MKLClassification.obtain_from_generic(fold.get_trained_machine())
w = machine.get_kernel().get_subkernel_weights()
weights.append(w)
print("mkl weights during cross--validation")
print(weights)
def evaluation_cross_validation_multiclass_storage(
traindat=traindat, label_traindat=label_traindat):
from shogun import CrossValidation, CrossValidationResult
from shogun import ParameterObserverCV
from shogun import MulticlassAccuracy, F1Measure
from shogun import StratifiedCrossValidationSplitting
from shogun import MulticlassLabels
from shogun import RealFeatures, CombinedFeatures
from shogun import GaussianKernel, CombinedKernel
from shogun import MKLMulticlass
from shogun import Statistics, MSG_DEBUG, Math
from shogun import ROCEvaluation
Math.init_random(1)
# training data, combined features all on same data
features = RealFeatures(traindat)
comb_features = CombinedFeatures()
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
labels = MulticlassLabels(label_traindat)
# kernel, different Gaussians combined
kernel = CombinedKernel()
kernel.append_kernel(GaussianKernel(10, 0.1))
kernel.append_kernel(GaussianKernel(10, 1))
kernel.append_kernel(GaussianKernel(10, 2))
# create mkl using libsvm, due to a mem-bug, interleaved is not possible
svm = MKLMulticlass(1.0, kernel, labels)
svm.set_kernel(kernel)
# splitting strategy for 5 fold cross-validation (for classification its better
# to use "StratifiedCrossValidation", but the standard
# "StratifiedCrossValidationSplitting" is also available
splitting_strategy = StratifiedCrossValidationSplitting(labels, 3)
# evaluation method
evaluation_criterium = MulticlassAccuracy()
# cross-validation instance
cross_validation = CrossValidation(svm, comb_features, labels,
splitting_strategy,
evaluation_criterium)
cross_validation.set_autolock(False)
# append cross validation parameter observer
multiclass_storage = ParameterObserverCV()
cross_validation.subscribe_to_parameters(multiclass_storage)
cross_validation.set_num_runs(3)
# perform cross-validation
result = cross_validation.evaluate()
# get first observation and first fold
obs = multiclass_storage.get_observations()[0]
fold = obs.get_folds_results()[0]
# get fold ROC for first class
eval_ROC = ROCEvaluation()
pred_lab_binary = MulticlassLabels.obtain_from_generic(
fold.get_test_result()).get_binary_for_class(0)
true_lab_binary = MulticlassLabels.obtain_from_generic(
fold.get_test_true_result()).get_binary_for_class(0)
eval_ROC.evaluate(pred_lab_binary, true_lab_binary)
print eval_ROC.get_ROC()
# get fold evaluation result
acc_measure = F1Measure()
print acc_measure.evaluate(pred_lab_binary, true_lab_binary)
def mkl_binclass (fm_train_real=traindat,fm_test_real=testdat,fm_label_twoclass = label_traindat):
##################################
# set up and train
# create some poly train/test matrix
tfeats = RealFeatures(fm_train_real)
tkernel = PolyKernel(10,3)
tkernel.init(tfeats, tfeats)
K_train = tkernel.get_kernel_matrix()
pfeats = RealFeatures(fm_test_real)
tkernel.init(tfeats, pfeats)
K_test = tkernel.get_kernel_matrix()
# create combined train features
feats_train = CombinedFeatures()
feats_train.append_feature_obj(RealFeatures(fm_train_real))
# and corresponding combined kernel
kernel = CombinedKernel()
kernel.append_kernel(CustomKernel(K_train))
kernel.append_kernel(PolyKernel(10,2))
kernel.init(feats_train, feats_train)
# train mkl
labels = BinaryLabels(fm_label_twoclass)
mkl = MKLClassification()
# which norm to use for MKL
mkl.set_mkl_norm(1) #2,3
# set cost (neg, pos)
mkl.set_C(1, 1)
# set kernel and labels
mkl.set_kernel(kernel)
mkl.set_labels(labels)
# train
mkl.train()
#w=kernel.get_subkernel_weights()
#kernel.set_subkernel_weights(w)
##################################
# test
# create combined test features
feats_pred = CombinedFeatures()
feats_pred.append_feature_obj(RealFeatures(fm_test_real))
# and corresponding combined kernel
kernel = CombinedKernel()
kernel.append_kernel(CustomKernel(K_test))
kernel.append_kernel(PolyKernel(10, 2))
kernel.init(feats_train, feats_pred)
# and classify
mkl.set_kernel(kernel)
mkl.apply()
return mkl.apply(),kernel
def combined_kernel(file_type, data_name, operate_type):
if file_type == '4':
X, y = loadFromMat(data_name)
elif file_type == '5':
X, y = loadFromLibsvm(data_name)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=test_size)
if type(X_train) == scipy.sparse.csr.csr_matrix and type(
X_test) == scipy.sparse.csr.csr_matrix:
X_train = X_train.todense()
X_test = X_test.todense()
X_train = X_train.T
X_test = X_test.T
y_train = y_train.reshape(y_train.size, ).astype('float64')
y_test = y_test.reshape(y_test.size, ).astype('float64')
kernel = CombinedKernel()
feats_train = CombinedFeatures()
feats_test = CombinedFeatures()
subkfeats_train = RealFeatures(X_train)
subkfeats_test = RealFeatures(X_test)
for i in range(-10, 11):
subkernel = GaussianKernel(pow(2, i + 1))
feats_train.append_feature_obj(subkfeats_train)
feats_test.append_feature_obj(subkfeats_test)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_train)
tmp_train_csv = NamedTemporaryFile(suffix=data_name + '_combined.csv')
import time
start = time.time()
if operate_type == 'save':
km_train = kernel.get_kernel_matrix()
f = CSVFile(tmp_train_csv.name, "w")
kernel.save(f)
elif operate_type == 'load':
f = CSVFile(tmp_train_csv.name, "r")
kernel.load(f)
end = time.time()
print 'for saving or loading, use time : ' + str(end - start)
labels = MulticlassLabels(y_train)
mkl = MKLMulticlass(C, kernel, labels)
mkl.set_epsilon(epsilon)
mkl.parallel.set_num_threads(num_threads)
mkl.set_mkl_epsilon(mkl_epsilon)
mkl.set_mkl_norm(mkl_norm)
import time
start = time.time()
mkl.train()
end = time.time()
print 'use time : ' + str(end - start)
kernel.init(feats_train, feats_test)
out = mkl.apply().get_labels()
print out.shape
print sum(out == y_test) / float(len(out))
def __init__(self):
self.sensors = list()
self.kernel = CombinedKernel()
self.svs = CombinedFeatures()
self.svm = None
self.window = (+100000, -1000000)
class SignalSensor(object):
"""
A collection of sensors
"""
def __init__(self):
self.sensors = list()
self.kernel = CombinedKernel()
self.svs = CombinedFeatures()
self.svm = None
self.window = (+100000, -1000000)
def from_file(self, file):
sys.stderr.write('loading model file')
l = file.readline();
if l != '%arts version: 1.0\n':
sys.stderr.write("\nfile not an arts definition file\n")
return None
bias = None
alphas = None
num_kernels = None
while l:
# skip comment or empty line
if not (l.startswith('%') or l.startswith('\n')):
if bias is None: bias = parse_float(l, 'b')
if alphas is None: alphas = parse_vector(l, file, 'alphas')
if num_kernels is None: num_kernels = parse_int(l, 'num_kernels')
if num_kernels and bias and alphas is not None:
for i in xrange(num_kernels):
s = Sensor()
(k, f) = s.from_file(file, i + 1)
k.io.enable_progress()
self.window = (min(self.window[0], s.window[0]),
max(self.window[1], s.window[2]))
self.sensors.append(s)
self.kernel.append_kernel(k)
self.svs.append_feature_obj(f)
self.kernel.init(self.svs, self.svs)
self.svm = KernelMachine(self.kernel, alphas,
numpy.arange(len(alphas), dtype=numpy.int32), bias)
self.svm.io.set_target_to_stderr()
self.svm.io.enable_progress()
self.svm.parallel.set_num_threads(self.svm.parallel.get_num_cpus())
sys.stderr.write('done\n')
return
l = file.readline()
sys.stderr.write('error loading model file\n')
def predict(self, seq, chunk_size = int(10e6)):
"""
predicts on whole contig, splits up sequence in chunks of size chunk_size
"""
seq_len = len(seq)
num_chunks = int(numpy.ceil(float(seq_len) / float(chunk_size)))
assert(num_chunks > 0)
sys.stderr.write("number of chunks for contig: %i\n" % (num_chunks))
start = 0
stop = min(chunk_size, seq_len)
out = []
# iterate over chunks
for chunk_idx in range(num_chunks):
sys.stderr.write("processing chunk #%i\n" % (chunk_idx))
assert (start < stop)
chunk = seq[start:stop]
assert(len(self.sensors) > 0)
tf = CombinedFeatures()
for i in xrange(len(self.sensors)):
f = self.sensors[i].get_test_features(chunk, self.window)
tf.append_feature_obj(f)
sys.stderr.write("initialising kernel...")
self.kernel.init(self.svs, tf)
sys.stderr.write("..done\n")
self.svm.set_kernel(self.kernel)
lab_out = self.svm.apply().get_values()
assert(len(lab_out) > 0)
out.extend(lab_out)
# increment chunk
start = stop
stop = min(stop+chunk_size, seq_len)
l = (-self.window[0]) * [-42]
r = self.window[1] * [-42]
# concatenate
ret = l + out + r
assert(len(ret) == len(seq))
return ret
def main(config_module, N_SEED):
C_values = config_module.C_values
norm_values = config_module.norm_values
nested_cv_n_folds = config_module.nested_cv_n_folds
cv_n_folds = config_module.cv_n_folds
precomputed_kernel_files = config_module.precomputed_kernel_files
random.seed(N_SEED)
np.random.seed(N_SEED)
file_npz = np.load("./kernels/" + precomputed_kernel_files[0])
y = file_npz['labels']
y = (y * 2) - 1
y = y.astype('float64')
skf = StratifiedKFold(n_splits=cv_n_folds, shuffle=True, random_state=N_SEED)
cv_test_bac = np.zeros((cv_n_folds,))
cv_test_sens = np.zeros((cv_n_folds,))
cv_test_spec = np.zeros((cv_n_folds,))
cv_error_rate = np.zeros((cv_n_folds,))
kernels = []
print("Loading kernels...")
for precomputed_kernel_file in precomputed_kernel_files:
file_npz = np.load("./kernels/" + precomputed_kernel_file)
kernels.append(file_npz['kernel'])
print("Starting Stratified Cross Validation with ", cv_n_folds, " folds")
for i, (train_index, test_index) in enumerate(skf.split(y, y)):
start_time = time.time()
print("")
print("Fold ", i)
y_train, y_test = y[train_index], y[test_index]
best_C = 1
best_norm = 1
best_performance = 0
for C in C_values:
for norm in norm_values:
fold_prediction = np.zeros((nested_cv_n_folds,))
nested_skf = StratifiedKFold(n_splits=nested_cv_n_folds, shuffle=True, random_state=1)
for j, (train_index2, val_index) in enumerate(nested_skf.split(y_train, y_train)):
y_train2, y_val = y_train[train_index2], y_train[val_index]
# set up
kernel = CombinedKernel()
labels = BinaryLabels(y_train2)
for k in range(len(kernels)):
precomputed_kernel = kernels[k]
x_train2, x_val = precomputed_kernel[train_index[train_index2], :][:,
train_index[train_index2]], precomputed_kernel[train_index[val_index],
:][:, train_index[train_index2]]
##################################
# Kernel Custom
subkernel = CustomKernel(x_train2)
kernel.append_kernel(subkernel)
elasticnet_lambda = 0
# which norm to use for MKL
mkl_norm = norm
mkl_epsilon = 1e-5
# Cost C MKL
C_mkl = 0
# Creating model
mkl = MKLClassification()
mkl.set_elasticnet_lambda(elasticnet_lambda)
mkl.set_C_mkl(C_mkl)
mkl.set_mkl_norm(mkl_norm)
mkl.set_mkl_epsilon(mkl_epsilon)
# set cost (neg, pos)
mkl.set_C(C, C)
# set kernel and labels
mkl.set_kernel(kernel)
mkl.set_labels(labels)
##################################
# Train
mkl.train()
##################################
# Test
kernel = CombinedKernel()
for k in range(len(kernels)):
precomputed_kernel = kernels[k]
x_train2, x_val = precomputed_kernel[train_index[train_index2], :][:,
train_index[train_index2]], precomputed_kernel[train_index[val_index],
:][:, train_index[train_index2]]
##################################
# Kernel Custom
subkernel = CustomKernel(x_val.T)
kernel.append_kernel(subkernel)
# Predicts
mkl.set_kernel(kernel)
prediction = mkl.apply().get_labels()
cm = confusion_matrix(y_val, prediction)
test_bac = np.sum(np.true_divide(np.diagonal(cm), np.sum(cm, axis=1))) / cm.shape[1]
fold_prediction[j] = test_bac
if np.mean(fold_prediction) > best_performance:
best_performance = np.mean(fold_prediction)
best_C = C
best_norm = norm
# set up
kernel = CombinedKernel()
labels = BinaryLabels(y_train)
for j in range(len(kernels)):
precomputed_kernel = kernels[j]
x_train, x_test = precomputed_kernel[train_index, :][:, train_index], precomputed_kernel[test_index, :][:, train_index]
##################################
# Kernel Custom
subkernel = CustomKernel(x_train)
kernel.append_kernel(subkernel)
elasticnet_lambda = 0
# which norm to use for MKL
mkl_norm = best_norm
mkl_epsilon = 1e-5
# Cost C MKL
C_mkl = 0
# Creating model
mkl = MKLClassification()
mkl.set_elasticnet_lambda(elasticnet_lambda)
mkl.set_C_mkl(C_mkl)
mkl.set_mkl_norm(mkl_norm)
mkl.set_mkl_epsilon(mkl_epsilon)
# set cost (neg, pos)
mkl.set_C(best_C, best_C)
# set kernel and labels
mkl.set_kernel(kernel)
mkl.set_labels(labels)
##################################
# Train
mkl.train()
##################################
# Test
kernel = CombinedKernel()
for k in range(len(kernels)):
precomputed_kernel = kernels[k]
x_train, x_test = precomputed_kernel[train_index, :][:, train_index], precomputed_kernel[test_index, :][:, train_index]
##################################
# Kernel Custom
subkernel = CustomKernel(x_test.T)
kernel.append_kernel(subkernel)
# Predicts
mkl.set_kernel(kernel)
prediction = mkl.apply().get_labels()
print("")
print("Confusion matrix")
cm = confusion_matrix(y_test,prediction)
print(cm)
test_bac = np.sum(np.true_divide(np.diagonal(cm), np.sum(cm, axis=1))) / cm.shape[1]
test_sens = np.true_divide(cm[1, 1], np.sum(cm[1, :]))
test_spec = np.true_divide(cm[0, 0], np.sum(cm[0, :]))
error_rate = np.true_divide(cm[0, 1] + cm[1, 0], np.sum(np.sum(cm)))
print("Balanced acc: %.4f " % (test_bac))
print("Sensitivity: %.4f " % (test_sens))
print("Specificity: %.4f " % (test_spec))
print("Error Rate: %.4f " % (error_rate))
cv_test_bac[i] = test_bac
cv_test_sens[i] = test_sens
cv_test_spec[i] = test_spec
cv_error_rate[i] = error_rate
stop_time = time.time()
print("--- %s seconds ---" % (stop_time - start_time))
print("ETA: ", (i-cv_n_folds)*(stop_time - start_time), " seconds")
print("")
print("")
print("Cross-validation balanced acc: %.4f +- %.4f" % (cv_test_bac.mean(), cv_test_bac.std()))
print("Cross-validation Sensitivity: %.4f +- %.4f" % (cv_test_sens.mean(), cv_test_sens.std()))
print("Cross-validation Specificity: %.4f +- %.4f" % (cv_test_spec.mean(), cv_test_spec.std()))
print("Cross-validation Error Rate: %.4f +- %.4f" % (cv_error_rate.mean(), cv_error_rate.std()))
return(cv_test_bac.mean())
def quadratic_time_mmd_graphical():
# parameters, change to get different results
m=100
dim=2
# setting the difference of the first dimension smaller makes a harder test
difference=0.5
# number of samples taken from null and alternative distribution
num_null_samples=500
# streaming data generator for mean shift distributions
gen_p=MeanShiftDataGenerator(0, dim)
gen_q=MeanShiftDataGenerator(difference, dim)
# Stream examples and merge them in order to compute MMD on joint sample
# alternative is to call a different constructor of QuadraticTimeMMD
features=gen_p.get_streamed_features(m)
features=features.create_merged_copy(gen_q.get_streamed_features(m))
# use the median kernel selection
# create combined kernel with Gaussian kernels inside (shoguns Gaussian kernel is
# compute median data distance in order to use for Gaussian kernel width
# 0.5*median_distance normally (factor two in Gaussian kernel)
# However, shoguns kernel width is different to usual parametrization
# Therefore 0.5*2*median_distance^2
# Use a subset of data for that, only 200 elements. Median is stable
sigmas=[2**x for x in range(-3,10)]
widths=[x*x*2 for x in sigmas]
print "kernel widths:", widths
combined=CombinedKernel()
for i in range(len(sigmas)):
combined.append_kernel(GaussianKernel(10, widths[i]))
# create MMD instance, use biased statistic
mmd=QuadraticTimeMMD(combined,features, m)
mmd.set_statistic_type(BIASED)
# kernel selection instance (this can easily replaced by the other methods for selecting
# single kernels
selection=MMDKernelSelectionMax(mmd)
# perform kernel selection
kernel=selection.select_kernel()
kernel=GaussianKernel.obtain_from_generic(kernel)
mmd.set_kernel(kernel);
print "selected kernel width:", kernel.get_width()
# sample alternative distribution (new data each trial)
alt_samples=zeros(num_null_samples)
for i in range(len(alt_samples)):
# Stream examples and merge them in order to replace in MMD
features=gen_p.get_streamed_features(m)
features=features.create_merged_copy(gen_q.get_streamed_features(m))
mmd.set_p_and_q(features)
alt_samples[i]=mmd.compute_statistic()
# sample from null distribution
# bootstrapping, biased statistic
mmd.set_null_approximation_method(PERMUTATION)
mmd.set_statistic_type(BIASED)
mmd.set_num_null_samples(num_null_samples)
null_samples_boot=mmd.sample_null()
# sample from null distribution
# spectrum, biased statistic
if "sample_null_spectrum" in dir(QuadraticTimeMMD):
mmd.set_null_approximation_method(MMD2_SPECTRUM)
mmd.set_statistic_type(BIASED)
null_samples_spectrum=mmd.sample_null_spectrum(num_null_samples, m-10)
# fit gamma distribution, biased statistic
mmd.set_null_approximation_method(MMD2_GAMMA)
mmd.set_statistic_type(BIASED)
gamma_params=mmd.fit_null_gamma()
# sample gamma with parameters
null_samples_gamma=array([gamma(gamma_params[0], gamma_params[1]) for _ in range(num_null_samples)])
# to plot data, sample a few examples from stream first
features=gen_p.get_streamed_features(m)
features=features.create_merged_copy(gen_q.get_streamed_features(m))
data=features.get_feature_matrix()
# plot
figure()
title('Quadratic Time MMD')
# plot data of p and q
subplot(2,3,1)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 4) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 4) ) # reduce number of x-ticks
plot(data[0][0:m], data[1][0:m], 'ro', label='$x$')
plot(data[0][m+1:2*m], data[1][m+1:2*m], 'bo', label='$x$', alpha=0.5)
title('Data, shift in $x_1$='+str(difference)+'\nm='+str(m))
xlabel('$x_1, y_1$')
ylabel('$x_2, y_2$')
# histogram of first data dimension and pdf
subplot(2,3,2)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3 )) # reduce number of x-ticks
hist(data[0], bins=50, alpha=0.5, facecolor='r', normed=True)
hist(data[1], bins=50, alpha=0.5, facecolor='b', normed=True)
xs=linspace(min(data[0])-1,max(data[0])+1, 50)
plot(xs,normpdf( xs, 0, 1), 'r', linewidth=3)
plot(xs,normpdf( xs, difference, 1), 'b', linewidth=3)
xlabel('$x_1, y_1$')
ylabel('$p(x_1), p(y_1)$')
title('Data PDF in $x_1, y_1$')
# compute threshold for test level
alpha=0.05
null_samples_boot.sort()
null_samples_spectrum.sort()
null_samples_gamma.sort()
thresh_boot=null_samples_boot[floor(len(null_samples_boot)*(1-alpha))];
thresh_spectrum=null_samples_spectrum[floor(len(null_samples_spectrum)*(1-alpha))];
thresh_gamma=null_samples_gamma[floor(len(null_samples_gamma)*(1-alpha))];
type_one_error_boot=sum(null_samples_boot<thresh_boot)/float(num_null_samples)
type_one_error_spectrum=sum(null_samples_spectrum<thresh_boot)/float(num_null_samples)
type_one_error_gamma=sum(null_samples_gamma<thresh_boot)/float(num_null_samples)
# plot alternative distribution with threshold
subplot(2,3,4)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(alt_samples, 20, normed=True);
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
type_two_error=sum(alt_samples<thresh_boot)/float(num_null_samples)
title('Alternative Dist.\n' + 'Type II error is ' + str(type_two_error))
# compute range for all null distribution histograms
hist_range=[min([min(null_samples_boot), min(null_samples_spectrum), min(null_samples_gamma)]), max([max(null_samples_boot), max(null_samples_spectrum), max(null_samples_gamma)])]
# plot null distribution with threshold
subplot(2,3,3)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3 )) # reduce number of x-ticks
hist(null_samples_boot, 20, range=hist_range, normed=True);
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
title('Sampled Null Dist.\n' + 'Type I error is ' + str(type_one_error_boot))
grid(True)
# plot null distribution spectrum
subplot(2,3,5)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(null_samples_spectrum, 20, range=hist_range, normed=True);
axvline(thresh_spectrum, 0, 1, linewidth=2, color='red')
title('Null Dist. Spectrum\nType I error is ' + str(type_one_error_spectrum))
# plot null distribution gamma
subplot(2,3,6)
grid(True)
gca().xaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
gca().yaxis.set_major_locator( MaxNLocator(nbins = 3) ) # reduce number of x-ticks
hist(null_samples_gamma, 20, range=hist_range, normed=True);
axvline(thresh_gamma, 0, 1, linewidth=2, color='red')
title('Null Dist. Gamma\nType I error is ' + str(type_one_error_gamma))
# pull plots a bit apart
subplots_adjust(hspace=0.5)
subplots_adjust(wspace=0.5)
def kernel_combined_custom_poly(train_fname=traindat,
test_fname=testdat,
train_label_fname=label_traindat):
from shogun import CombinedFeatures, RealFeatures, BinaryLabels
from shogun import CombinedKernel, PolyKernel, CustomKernel
from shogun import LibSVM, CSVFile
kernel = CombinedKernel()
feats_train = CombinedFeatures()
tfeats = RealFeatures(CSVFile(train_fname))
tkernel = PolyKernel(10, 3)
tkernel.init(tfeats, tfeats)
K = tkernel.get_kernel_matrix()
kernel.append_kernel(CustomKernel(K))
subkfeats_train = RealFeatures(CSVFile(train_fname))
feats_train.append_feature_obj(subkfeats_train)
subkernel = PolyKernel(10, 2)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_train)
labels = BinaryLabels(CSVFile(train_label_fname))
svm = LibSVM(1.0, kernel, labels)
svm.train()
kernel = CombinedKernel()
feats_pred = CombinedFeatures()
pfeats = RealFeatures(CSVFile(test_fname))
tkernel = PolyKernel(10, 3)
tkernel.init(tfeats, pfeats)
K = tkernel.get_kernel_matrix()
kernel.append_kernel(CustomKernel(K))
subkfeats_test = RealFeatures(CSVFile(test_fname))
feats_pred.append_feature_obj(subkfeats_test)
subkernel = PolyKernel(10, 2)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_pred)
svm.set_kernel(kernel)
svm.apply()
km_train = kernel.get_kernel_matrix()
return km_train, kernel
def kernel_combined_custom_poly (train_fname = traindat,test_fname = testdat,train_label_fname=label_traindat):
from shogun import CombinedFeatures, RealFeatures, BinaryLabels
from shogun import CombinedKernel, PolyKernel, CustomKernel
from shogun import LibSVM, CSVFile
kernel = CombinedKernel()
feats_train = CombinedFeatures()
tfeats = RealFeatures(CSVFile(train_fname))
tkernel = PolyKernel(10,3)
tkernel.init(tfeats, tfeats)
K = tkernel.get_kernel_matrix()
kernel.append_kernel(CustomKernel(K))
subkfeats_train = RealFeatures(CSVFile(train_fname))
feats_train.append_feature_obj(subkfeats_train)
subkernel = PolyKernel(10,2)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_train)
labels = BinaryLabels(CSVFile(train_label_fname))
svm = LibSVM(1.0, kernel, labels)
svm.train()
kernel = CombinedKernel()
feats_pred = CombinedFeatures()
pfeats = RealFeatures(CSVFile(test_fname))
tkernel = PolyKernel(10,3)
tkernel.init(tfeats, pfeats)
K = tkernel.get_kernel_matrix()
kernel.append_kernel(CustomKernel(K))
subkfeats_test = RealFeatures(CSVFile(test_fname))
feats_pred.append_feature_obj(subkfeats_test)
subkernel = PolyKernel(10, 2)
kernel.append_kernel(subkernel)
kernel.init(feats_train, feats_pred)
svm.set_kernel(kernel)
svm.apply()
km_train=kernel.get_kernel_matrix()
return km_train,kernel
def serialization_string_kernels(n_data, num_shifts, size):
"""
serialize svm with string kernels
"""
##################################################
# set up toy data and svm
train_xt, train_lt = generate_random_data(n_data)
test_xt, test_lt = generate_random_data(n_data)
feats_train = construct_features(train_xt)
feats_test = construct_features(test_xt)
max_len = len(train_xt[0])
kernel_wdk = WeightedDegreePositionStringKernel(size, 5)
shifts_vector = numpy.ones(max_len, dtype=numpy.int32) * num_shifts
kernel_wdk.set_shifts(shifts_vector)
########
# set up spectrum
use_sign = False
kernel_spec_1 = WeightedCommWordStringKernel(size, use_sign)
kernel_spec_2 = WeightedCommWordStringKernel(size, use_sign)
########
# combined kernel
kernel = CombinedKernel()
kernel.append_kernel(kernel_wdk)
kernel.append_kernel(kernel_spec_1)
kernel.append_kernel(kernel_spec_2)
# init kernel
labels = BinaryLabels(train_lt)
svm = SVMLight(1.0, kernel, labels)
#svm.io.set_loglevel(MSG_DEBUG)
svm.train(feats_train)
##################################################
# serialize to file
fn = "serialized_svm.bz2"
#print("serializing SVM to file", fn)
save(fn, svm)
##################################################
# unserialize and sanity check
#print("unserializing SVM")
svm2 = load(fn)
#print("comparing predictions")
out = svm.apply(feats_test).get_labels()
out2 = svm2.apply(feats_test).get_labels()
# assert outputs are close
for i in range(len(out)):
assert abs(out[i] - out2[i] < 0.000001)
#print("all checks passed.")
return out, out2
def evaluation_cross_validation_multiclass_storage (traindat=traindat, label_traindat=label_traindat):
from shogun import CrossValidation, CrossValidationResult
from shogun import ParameterObserverCV
from shogun import MulticlassAccuracy, F1Measure
from shogun import StratifiedCrossValidationSplitting
from shogun import MulticlassLabels
from shogun import RealFeatures, CombinedFeatures
from shogun import GaussianKernel, CombinedKernel
from shogun import MKLMulticlass
from shogun import Statistics, MSG_DEBUG, Math
from shogun import ROCEvaluation
Math.init_random(1)
# training data, combined features all on same data
features=RealFeatures(traindat)
comb_features=CombinedFeatures()
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
labels=MulticlassLabels(label_traindat)
# kernel, different Gaussians combined
kernel=CombinedKernel()
kernel.append_kernel(GaussianKernel(10, 0.1))
kernel.append_kernel(GaussianKernel(10, 1))
kernel.append_kernel(GaussianKernel(10, 2))
# create mkl using libsvm, due to a mem-bug, interleaved is not possible
svm=MKLMulticlass(1.0,kernel,labels);
svm.set_kernel(kernel);
# splitting strategy for 5 fold cross-validation (for classification its better
# to use "StratifiedCrossValidation", but the standard
# "StratifiedCrossValidationSplitting" is also available
splitting_strategy=StratifiedCrossValidationSplitting(labels, 3)
# evaluation method
evaluation_criterium=MulticlassAccuracy()
# cross-validation instance
cross_validation=CrossValidation(svm, comb_features, labels,
splitting_strategy, evaluation_criterium)
cross_validation.set_autolock(False)
# append cross validation parameter observer
multiclass_storage=ParameterObserverCV()
cross_validation.subscribe_to_parameters(multiclass_storage)
cross_validation.set_num_runs(3)
# perform cross-validation
result=cross_validation.evaluate()
# get first observation and first fold
obs = multiclass_storage.get_observations()[0]
fold = obs.get_folds_results()[0]
# get fold ROC for first class
eval_ROC = ROCEvaluation()
pred_lab_binary = MulticlassLabels.obtain_from_generic(fold.get_test_result()).get_binary_for_class(0)
true_lab_binary = MulticlassLabels.obtain_from_generic(fold.get_test_true_result()).get_binary_for_class(0)
eval_ROC.evaluate(pred_lab_binary, true_lab_binary)
print eval_ROC.get_ROC()
# get fold evaluation result
acc_measure = F1Measure()
print acc_measure.evaluate(pred_lab_binary, true_lab_binary)
def kernel_combined (fm_train_real=traindat,fm_test_real=testdat,fm_train_dna=traindna,fm_test_dna=testdna ): from shogun import CombinedKernel, GaussianKernel, FixedDegreeStringKernel, LocalAlignmentStringKernel from shogun import RealFeatures, StringCharFeatures, CombinedFeatures, DNA kernel=CombinedKernel() feats_train=CombinedFeatures() feats_test=CombinedFeatures() subkfeats_train=RealFeatures(fm_train_real) subkfeats_test=RealFeatures(fm_test_real) subkernel=GaussianKernel(10, 1.1) feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) subkfeats_train=StringCharFeatures(fm_train_dna, DNA) subkfeats_test=StringCharFeatures(fm_test_dna, DNA) degree=3 subkernel=FixedDegreeStringKernel(10, degree) feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) subkfeats_train=StringCharFeatures(fm_train_dna, DNA) subkfeats_test=StringCharFeatures(fm_test_dna, DNA) subkernel=LocalAlignmentStringKernel(10) feats_train.append_feature_obj(subkfeats_train) feats_test.append_feature_obj(subkfeats_test) kernel.append_kernel(subkernel) kernel.init(feats_train, feats_train) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def quadratic_time_mmd_graphical():
# parameters, change to get different results
m = 100
dim = 2
# setting the difference of the first dimension smaller makes a harder test
difference = 0.5
# number of samples taken from null and alternative distribution
num_null_samples = 500
# streaming data generator for mean shift distributions
gen_p = MeanShiftDataGenerator(0, dim)
gen_q = MeanShiftDataGenerator(difference, dim)
# Stream examples and merge them in order to compute MMD on joint sample
# alternative is to call a different constructor of QuadraticTimeMMD
features = gen_p.get_streamed_features(m)
features = features.create_merged_copy(gen_q.get_streamed_features(m))
# use the median kernel selection
# create combined kernel with Gaussian kernels inside (shoguns Gaussian kernel is
# compute median data distance in order to use for Gaussian kernel width
# 0.5*median_distance normally (factor two in Gaussian kernel)
# However, shoguns kernel width is different to usual parametrization
# Therefore 0.5*2*median_distance^2
# Use a subset of data for that, only 200 elements. Median is stable
sigmas = [2**x for x in range(-3, 10)]
widths = [x * x * 2 for x in sigmas]
print "kernel widths:", widths
combined = CombinedKernel()
for i in range(len(sigmas)):
combined.append_kernel(GaussianKernel(10, widths[i]))
# create MMD instance, use biased statistic
mmd = QuadraticTimeMMD(combined, features, m)
mmd.set_statistic_type(BIASED)
# kernel selection instance (this can easily replaced by the other methods for selecting
# single kernels
selection = MMDKernelSelectionMax(mmd)
# perform kernel selection
kernel = selection.select_kernel()
kernel = GaussianKernel.obtain_from_generic(kernel)
mmd.set_kernel(kernel)
print "selected kernel width:", kernel.get_width()
# sample alternative distribution (new data each trial)
alt_samples = zeros(num_null_samples)
for i in range(len(alt_samples)):
# Stream examples and merge them in order to replace in MMD
features = gen_p.get_streamed_features(m)
features = features.create_merged_copy(gen_q.get_streamed_features(m))
mmd.set_p_and_q(features)
alt_samples[i] = mmd.compute_statistic()
# sample from null distribution
# bootstrapping, biased statistic
mmd.set_null_approximation_method(PERMUTATION)
mmd.set_statistic_type(BIASED)
mmd.set_num_null_samples(num_null_samples)
null_samples_boot = mmd.sample_null()
# sample from null distribution
# spectrum, biased statistic
if "sample_null_spectrum" in dir(QuadraticTimeMMD):
mmd.set_null_approximation_method(MMD2_SPECTRUM)
mmd.set_statistic_type(BIASED)
null_samples_spectrum = mmd.sample_null_spectrum(
num_null_samples, m - 10)
# fit gamma distribution, biased statistic
mmd.set_null_approximation_method(MMD2_GAMMA)
mmd.set_statistic_type(BIASED)
gamma_params = mmd.fit_null_gamma()
# sample gamma with parameters
null_samples_gamma = array([
gamma(gamma_params[0], gamma_params[1])
for _ in range(num_null_samples)
])
# to plot data, sample a few examples from stream first
features = gen_p.get_streamed_features(m)
features = features.create_merged_copy(gen_q.get_streamed_features(m))
data = features.get_feature_matrix()
# plot
figure()
title('Quadratic Time MMD')
# plot data of p and q
subplot(2, 3, 1)
grid(True)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=4)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=4)) # reduce number of x-ticks
plot(data[0][0:m], data[1][0:m], 'ro', label='$x$')
plot(data[0][m + 1:2 * m],
data[1][m + 1:2 * m],
'bo',
label='$x$',
alpha=0.5)
title('Data, shift in $x_1$=' + str(difference) + '\nm=' + str(m))
xlabel('$x_1, y_1$')
ylabel('$x_2, y_2$')
# histogram of first data dimension and pdf
subplot(2, 3, 2)
grid(True)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
hist(data[0], bins=50, alpha=0.5, facecolor='r', normed=True)
hist(data[1], bins=50, alpha=0.5, facecolor='b', normed=True)
xs = linspace(min(data[0]) - 1, max(data[0]) + 1, 50)
plot(xs, normpdf(xs, 0, 1), 'r', linewidth=3)
plot(xs, normpdf(xs, difference, 1), 'b', linewidth=3)
xlabel('$x_1, y_1$')
ylabel('$p(x_1), p(y_1)$')
title('Data PDF in $x_1, y_1$')
# compute threshold for test level
alpha = 0.05
null_samples_boot.sort()
null_samples_spectrum.sort()
null_samples_gamma.sort()
thresh_boot = null_samples_boot[floor(
len(null_samples_boot) * (1 - alpha))]
thresh_spectrum = null_samples_spectrum[floor(
len(null_samples_spectrum) * (1 - alpha))]
thresh_gamma = null_samples_gamma[floor(
len(null_samples_gamma) * (1 - alpha))]
type_one_error_boot = sum(
null_samples_boot < thresh_boot) / float(num_null_samples)
type_one_error_spectrum = sum(
null_samples_spectrum < thresh_boot) / float(num_null_samples)
type_one_error_gamma = sum(
null_samples_gamma < thresh_boot) / float(num_null_samples)
# plot alternative distribution with threshold
subplot(2, 3, 4)
grid(True)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
hist(alt_samples, 20, normed=True)
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
type_two_error = sum(alt_samples < thresh_boot) / float(num_null_samples)
title('Alternative Dist.\n' + 'Type II error is ' + str(type_two_error))
# compute range for all null distribution histograms
hist_range = [
min([
min(null_samples_boot),
min(null_samples_spectrum),
min(null_samples_gamma)
]),
max([
max(null_samples_boot),
max(null_samples_spectrum),
max(null_samples_gamma)
])
]
# plot null distribution with threshold
subplot(2, 3, 3)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
hist(null_samples_boot, 20, range=hist_range, normed=True)
axvline(thresh_boot, 0, 1, linewidth=2, color='red')
title('Sampled Null Dist.\n' + 'Type I error is ' +
str(type_one_error_boot))
grid(True)
# plot null distribution spectrum
subplot(2, 3, 5)
grid(True)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
hist(null_samples_spectrum, 20, range=hist_range, normed=True)
axvline(thresh_spectrum, 0, 1, linewidth=2, color='red')
title('Null Dist. Spectrum\nType I error is ' +
str(type_one_error_spectrum))
# plot null distribution gamma
subplot(2, 3, 6)
grid(True)
gca().xaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
gca().yaxis.set_major_locator(
MaxNLocator(nbins=3)) # reduce number of x-ticks
hist(null_samples_gamma, 20, range=hist_range, normed=True)
axvline(thresh_gamma, 0, 1, linewidth=2, color='red')
title('Null Dist. Gamma\nType I error is ' + str(type_one_error_gamma))
# pull plots a bit apart
subplots_adjust(hspace=0.5)
subplots_adjust(wspace=0.5)
def mkl_binclass(fm_train_real=traindat,
fm_test_real=testdat,
fm_label_twoclass=label_traindat):
##################################
# set up and train
# create some poly train/test matrix
tfeats = RealFeatures(fm_train_real)
tkernel = PolyKernel(10, 3)
tkernel.init(tfeats, tfeats)
K_train = tkernel.get_kernel_matrix()
pfeats = RealFeatures(fm_test_real)
tkernel.init(tfeats, pfeats)
K_test = tkernel.get_kernel_matrix()
# create combined train features
feats_train = CombinedFeatures()
feats_train.append_feature_obj(RealFeatures(fm_train_real))
# and corresponding combined kernel
kernel = CombinedKernel()
kernel.append_kernel(CustomKernel(K_train))
kernel.append_kernel(PolyKernel(10, 2))
kernel.init(feats_train, feats_train)
# train mkl
labels = BinaryLabels(fm_label_twoclass)
mkl = MKLClassification()
# which norm to use for MKL
mkl.set_mkl_norm(1) #2,3
# set cost (neg, pos)
mkl.set_C(1, 1)
# set kernel and labels
mkl.set_kernel(kernel)
mkl.set_labels(labels)
# train
mkl.train()
#w=kernel.get_subkernel_weights()
#kernel.set_subkernel_weights(w)
##################################
# test
# create combined test features
feats_pred = CombinedFeatures()
feats_pred.append_feature_obj(RealFeatures(fm_test_real))
# and corresponding combined kernel
kernel = CombinedKernel()
kernel.append_kernel(CustomKernel(K_test))
kernel.append_kernel(PolyKernel(10, 2))
kernel.init(feats_train, feats_pred)
# and classify
mkl.set_kernel(kernel)
mkl.apply()
return mkl.apply(), kernel