def distribution_linearhmm_modular(fm_dna=traindna,
order=3,
gap=0,
reverse=False):
from shogun.Features import StringWordFeatures, StringCharFeatures, DNA
from shogun.Distribution import LinearHMM
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_dna)
feats = StringWordFeatures(charfeat.get_alphabet())
feats.obtain_from_char(charfeat, order - 1, order, gap, reverse)
hmm = LinearHMM(feats)
hmm.train()
hmm.get_transition_probs()
num_examples = feats.get_num_vectors()
num_param = hmm.get_num_model_parameters()
for i in range(num_examples):
for j in range(num_param):
hmm.get_log_derivative(j, i)
out_likelihood = hmm.get_log_likelihood()
out_sample = hmm.get_log_likelihood_sample()
return hmm, out_likelihood, out_sample
def distribution_histogram_modular(fm_dna=traindna,
order=3,
gap=0,
reverse=False):
from shogun.Features import StringWordFeatures, StringCharFeatures, DNA
from shogun.Distribution import Histogram
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_dna)
feats = StringWordFeatures(charfeat.get_alphabet())
feats.obtain_from_char(charfeat, order - 1, order, gap, reverse)
histo = Histogram(feats)
histo.train()
histo.get_histogram()
num_examples = feats.get_num_vectors()
num_param = histo.get_num_model_parameters()
#for i in xrange(num_examples):
# for j in xrange(num_param):
# histo.get_log_derivative(j, i)
out_likelihood = histo.get_log_likelihood()
out_sample = histo.get_log_likelihood_sample()
return histo, out_sample, out_likelihood
def distribution_linearhmm_modular (fm_dna=traindna,order=3,gap=0,reverse=False): from shogun.Features import StringWordFeatures, StringCharFeatures, DNA from shogun.Distribution import LinearHMM charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_dna) feats=StringWordFeatures(charfeat.get_alphabet()) feats.obtain_from_char(charfeat, order-1, order, gap, reverse) hmm=LinearHMM(feats) hmm.train() hmm.get_transition_probs() num_examples=feats.get_num_vectors() num_param=hmm.get_num_model_parameters() for i in range(num_examples): for j in range(num_param): hmm.get_log_derivative(j, i) out_likelihood = hmm.get_log_likelihood() out_sample = hmm.get_log_likelihood_sample() return hmm,out_likelihood ,out_sample
def distribution_hmm_modular(fm_cube, N, M, pseudo, order, gap, reverse, num_examples): from shogun.Features import StringWordFeatures, StringCharFeatures, CUBE from shogun.Distribution import HMM, BW_NORMAL charfeat=StringCharFeatures(CUBE) charfeat.set_features(fm_cube) feats=StringWordFeatures(charfeat.get_alphabet()) feats.obtain_from_char(charfeat, order-1, order, gap, reverse) hmm=HMM(feats, N, M, pseudo) hmm.train() hmm.baum_welch_viterbi_train(BW_NORMAL) num_examples=feats.get_num_vectors() num_param=hmm.get_num_model_parameters() for i in range(num_examples): for j in range(num_param): hmm.get_log_derivative(j, i) best_path=0 best_path_state=0 for i in range(num_examples): best_path+=hmm.best_path(i) for j in range(N): best_path_state+=hmm.get_best_path_state(i, j) lik_example = hmm.get_log_likelihood() lik_sample = hmm.get_log_likelihood_sample() return lik_example, lik_sample, hmm
def linear_hmm (): print 'LinearHMM' from shogun.Features import StringWordFeatures, StringCharFeatures, DNA from shogun.Distribution import LinearHMM order=3 gap=0 reverse=False charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_dna) feats=StringWordFeatures(charfeat.get_alphabet()) feats.obtain_from_char(charfeat, order-1, order, gap, reverse) hmm=LinearHMM(feats) hmm.train() hmm.get_transition_probs() num_examples=feats.get_num_vectors() num_param=hmm.get_num_model_parameters() for i in xrange(num_examples): for j in xrange(num_param): hmm.get_log_derivative(j, i) hmm.get_log_likelihood() hmm.get_log_likelihood_sample()
def distribution_hmm_modular(fm_cube, N, M, pseudo, order, gap, reverse, num_examples): from shogun.Features import StringWordFeatures, StringCharFeatures, CUBE from shogun.Distribution import HMM, BW_NORMAL charfeat=StringCharFeatures(CUBE) charfeat.set_features(fm_cube) feats=StringWordFeatures(charfeat.get_alphabet()) feats.obtain_from_char(charfeat, order-1, order, gap, reverse) hmm=HMM(feats, N, M, pseudo) hmm.train() hmm.baum_welch_viterbi_train(BW_NORMAL) num_examples=feats.get_num_vectors() num_param=hmm.get_num_model_parameters() for i in xrange(num_examples): for j in xrange(num_param): hmm.get_log_derivative(j, i) best_path=0 best_path_state=0 for i in xrange(num_examples): best_path+=hmm.best_path(i) for j in xrange(N): best_path_state+=hmm.get_best_path_state(i, j) lik_example = hmm.get_log_likelihood() lik_sample = hmm.get_log_likelihood_sample() return lik_example, lik_sample, hmm
def histogram (): print 'Histogram' from shogun.Features import StringWordFeatures, StringCharFeatures, DNA from shogun.Distribution import Histogram order=3 gap=0 reverse=False charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_dna) feats=StringWordFeatures(charfeat.get_alphabet()) feats.obtain_from_char(charfeat, order-1, order, gap, reverse) histo=Histogram(feats) histo.train() histo.get_histogram() num_examples=feats.get_num_vectors() num_param=histo.get_num_model_parameters() #for i in xrange(num_examples): # for j in xrange(num_param): # histo.get_log_derivative(j, i) histo.get_log_likelihood() histo.get_log_likelihood_sample()
def kernel_poly_match_word_string_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
degree=2,
inhomogene=True,
order=3,
gap=0,
reverse=False):
from shogun.Kernel import PolyMatchWordStringKernel
from shogun.Features import StringWordFeatures, StringCharFeatures, DNA
charfeat = StringCharFeatures(fm_train_dna, DNA)
feats_train = StringWordFeatures(DNA)
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
charfeat = StringCharFeatures(fm_test_dna, DNA)
feats_test = StringWordFeatures(DNA)
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
kernel = PolyMatchWordStringKernel(feats_train, feats_train, degree,
inhomogene)
km_train = kernel.get_kernel_matrix()
kernel.init(feats_train, feats_test)
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def create_hashed_features_spectrum(param, data):
"""
creates hashed dot features for the spectrum kernel
"""
# extract parameters
order = param["degree_spectrum"]
# fixed parameters
gap = 0
reverse = True
normalize = True
# create features
feats_char = StringCharFeatures(data, DNA)
feats_word = StringWordFeatures(feats_char.get_alphabet())
feats_word.obtain_from_char(feats_char, order - 1, order, gap, reverse)
# create preproc
preproc = SortWordString()
preproc.init(feats_word)
feats_word.add_preproc(preproc)
feats_word.apply_preproc()
# finish
feats = ImplicitWeightedSpecFeatures(feats_word, normalize)
return feats
def kernel_match_word_string_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
degree=3,
scale=1.4,
size_cache=10,
order=3,
gap=0,
reverse=False):
from shogun.Kernel import MatchWordStringKernel, AvgDiagKernelNormalizer
from shogun.Features import StringWordFeatures, StringCharFeatures, DNA
charfeat = StringCharFeatures(fm_train_dna, DNA)
feats_train = StringWordFeatures(DNA)
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
charfeat = StringCharFeatures(fm_test_dna, DNA)
feats_test = StringWordFeatures(DNA)
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
kernel = MatchWordStringKernel(size_cache, degree)
kernel.set_normalizer(AvgDiagKernelNormalizer(scale))
kernel.init(feats_train, feats_train)
km_train = kernel.get_kernel_matrix()
kernel.init(feats_train, feats_test)
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def kernel_salzberg_word_string_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
label_train_dna=label_traindat,
order=3,
gap=0,
reverse=False):
from shogun.Features import StringCharFeatures, StringWordFeatures, DNA, Labels
from shogun.Kernel import SalzbergWordStringKernel
from shogun.Classifier import PluginEstimate
charfeat = StringCharFeatures(fm_train_dna, DNA)
feats_train = StringWordFeatures(charfeat.get_alphabet())
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
charfeat = StringCharFeatures(fm_test_dna, DNA)
feats_test = StringWordFeatures(charfeat.get_alphabet())
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
pie = PluginEstimate()
labels = Labels(label_train_dna)
pie.set_labels(labels)
pie.set_features(feats_train)
pie.train()
kernel = SalzbergWordStringKernel(feats_train, feats_train, pie, labels)
km_train = kernel.get_kernel_matrix()
kernel.init(feats_train, feats_test)
pie.set_features(feats_test)
pie.apply().get_labels()
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def kernel_histogram_word_string_modular (fm_train_dna=traindat,fm_test_dna=testdat,label_train_dna=label_traindat,order=3,gap=0,reverse=False): from shogun.Features import StringCharFeatures, StringWordFeatures, DNA, BinaryLabels from shogun.Kernel import HistogramWordStringKernel from shogun.Classifier import PluginEstimate#, MSG_DEBUG reverse = reverse charfeat=StringCharFeatures(DNA) #charfeat.io.set_loglevel(MSG_DEBUG) charfeat.set_features(fm_train_dna) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_test_dna) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) pie=PluginEstimate() labels=BinaryLabels(label_train_dna) pie.set_labels(labels) pie.set_features(feats_train) pie.train() kernel=HistogramWordStringKernel(feats_train, feats_train, pie) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) pie.set_features(feats_test) pie.apply().get_labels() km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def init_sensor(self, kernel, svs):
f = StringCharFeatures(svs, DNA)
kname = kernel['name']
if kname == 'spectrum':
wf = StringWordFeatures(f.get_alphabet())
wf.obtain_from_char(f, kernel['order'] - 1, kernel['order'], 0,
False)
pre = SortWordString()
pre.init(wf)
wf.add_preprocessor(pre)
wf.apply_preprocessor()
f = wf
k = CommWordStringKernel(0, False)
k.set_use_dict_diagonal_optimization(kernel['order'] < 8)
self.preproc = pre
elif kname == 'wdshift':
k = WeightedDegreePositionStringKernel(0, kernel['order'])
k.set_normalizer(IdentityKernelNormalizer())
k.set_shifts(
kernel['shift'] *
numpy.ones(f.get_max_vector_length(), dtype=numpy.int32))
k.set_position_weights(
1.0 / f.get_max_vector_length() *
numpy.ones(f.get_max_vector_length(), dtype=numpy.float64))
else:
raise "Currently, only wdshift and spectrum kernels supported"
self.kernel = k
self.train_features = f
return (self.kernel, self.train_features)
def init_sensor(self, kernel, svs):
f = StringCharFeatures(svs, DNA)
kname = kernel['name']
if kname == 'spectrum':
wf = StringWordFeatures(f.get_alphabet())
wf.obtain_from_char(f, kernel['order'] - 1, kernel['order'], 0, False)
pre = SortWordString()
pre.init(wf)
wf.add_preprocessor(pre)
wf.apply_preprocessor()
f = wf
k = CommWordStringKernel(0, False)
k.set_use_dict_diagonal_optimization(kernel['order'] < 8)
self.preproc = pre
elif kname == 'wdshift':
k = WeightedDegreePositionStringKernel(0, kernel['order'])
k.set_normalizer(IdentityKernelNormalizer())
k.set_shifts(kernel['shift'] *
numpy.ones(f.get_max_vector_length(), dtype=numpy.int32))
k.set_position_weights(1.0 / f.get_max_vector_length() *
numpy.ones(f.get_max_vector_length(), dtype=numpy.float64))
else:
raise "Currently, only wdshift and spectrum kernels supported"
self.kernel = k
self.train_features = f
return (self.kernel, self.train_features)
def create_hashed_features_spectrum(param, data):
"""
creates hashed dot features for the spectrum kernel
"""
# extract parameters
order = param["degree_spectrum"]
# fixed parameters
gap = 0
reverse = True
normalize = True
# create features
feats_char = StringCharFeatures(data, DNA)
feats_word = StringWordFeatures(feats_char.get_alphabet())
feats_word.obtain_from_char(feats_char, order-1, order, gap, reverse)
# create preproc
preproc = SortWordString()
preproc.init(feats_word)
feats_word.add_preproc(preproc)
feats_word.apply_preproc()
# finish
feats = ImplicitWeightedSpecFeatures(feats_word, normalize)
return feats
def get_test_features(self, seq, window):
start = self.window[0] - window[0]
end = len(seq) - window[1] + self.window[2]
size = self.window[2] - self.window[0] + 1
seq = seq[start:end]
seq = seq.replace("N", "A").replace("R", "A").replace("M", "A")
f = StringCharFeatures([seq], DNA)
if self.preproc:
wf = StringWordFeatures(f.get_alphabet())
o = self.train_features.get_order()
wf.obtain_from_char(f, 0, o, 0, False)
f = wf
f.obtain_by_sliding_window(size, 1, o - 1)
else:
f.obtain_by_sliding_window(size, 1)
return f
def get_test_features(self, seq, window):
start = self.window[0] - window[0]
end = len(seq) - window[1] + self.window[2]
size = self.window[2] - self.window[0] + 1
seq = seq[start:end]
seq = seq.replace("N", "A").replace("R", "A").replace("M", "A")
f = StringCharFeatures([seq], DNA)
if self.preproc:
wf = StringWordFeatures(f.get_alphabet())
o = self.train_features.get_order()
wf.obtain_from_char(f, 0, o, 0, False)
f = wf
f.obtain_by_sliding_window(size, 1, o - 1)
else:
f.obtain_by_sliding_window(size, 1)
return f
def get_spectrum_features(data, order=3, gap=0, reverse=True):
"""
create feature object used by spectrum kernel
"""
charfeat = StringCharFeatures(data, DNA)
feat = StringWordFeatures(charfeat.get_alphabet())
feat.obtain_from_char(charfeat, order - 1, order, gap, reverse)
preproc = SortWordString()
preproc.init(feat)
feat.add_preprocessor(preproc)
feat.apply_preprocessor()
return feat
def kernel_histogram_word_string_modular(
fm_train_dna=traindat, fm_test_dna=testdat, label_train_dna=label_traindat, order=3, gap=0, reverse=False
):
from shogun.Features import StringCharFeatures, StringWordFeatures, DNA, Labels
from shogun.Kernel import HistogramWordStringKernel
from shogun.Classifier import PluginEstimate # , MSG_DEBUG
reverse = reverse
charfeat = StringCharFeatures(DNA)
# charfeat.io.set_loglevel(MSG_DEBUG)
charfeat.set_features(fm_train_dna)
feats_train = StringWordFeatures(charfeat.get_alphabet())
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_test_dna)
feats_test = StringWordFeatures(charfeat.get_alphabet())
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
pie = PluginEstimate()
labels = Labels(label_train_dna)
pie.set_labels(labels)
pie.set_features(feats_train)
pie.train()
kernel = HistogramWordStringKernel(feats_train, feats_train, pie)
km_train = kernel.get_kernel_matrix()
kernel.init(feats_train, feats_test)
pie.set_features(feats_test)
pie.classify().get_labels()
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def kernel_salzberg_word_string_modular (fm_train_dna=traindat,fm_test_dna=testdat,label_train_dna=label_traindat, order=3,gap=0,reverse=False): from shogun.Features import StringCharFeatures, StringWordFeatures, DNA, Labels from shogun.Kernel import SalzbergWordStringKernel from shogun.Classifier import PluginEstimate charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) pie=PluginEstimate() labels=Labels(label_train_dna) pie.set_labels(labels) pie.set_features(feats_train) pie.train() kernel=SalzbergWordStringKernel(feats_train, feats_train, pie, labels) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) pie.set_features(feats_test) pie.apply().get_labels() km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def match_word_string (): print 'MatchWordString' from shogun.Kernel import MatchWordStringKernel, AvgDiagKernelNormalizer from shogun.Features import StringWordFeatures, StringCharFeatures, DNA degree=3 scale=1.4 size_cache=10 order=3 gap=0 reverse=False charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(DNA) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(DNA) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) kernel=MatchWordStringKernel(size_cache, degree) kernel.set_normalizer(AvgDiagKernelNormalizer(scale)) kernel.init(feats_train, feats_train) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix()
def plugin_estimate_salzberg (): print 'PluginEstimate w/ SalzbergWord' from shogun.Features import StringCharFeatures, StringWordFeatures, DNA, Labels from shogun.Kernel import SalzbergWordStringKernel from shogun.Classifier import PluginEstimate order=3 gap=0 reverse=False charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) pie=PluginEstimate() labels=Labels(label_train_dna) pie.set_labels(labels) pie.set_features(feats_train) pie.train() kernel=SalzbergWordStringKernel(feats_train, feats_test, pie, labels) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) pie.set_features(feats_test) pie.classify().get_labels() km_test=kernel.get_kernel_matrix()
def get_spectrum_features(data, order=3, gap=0, reverse=True):
"""
create feature object used by spectrum kernel
"""
charfeat = StringCharFeatures(data, DNA)
feat = StringWordFeatures(charfeat.get_alphabet())
feat.obtain_from_char(charfeat, order-1, order, gap, reverse)
preproc = SortWordString()
preproc.init(feat)
feat.add_preprocessor(preproc)
feat.apply_preprocessor()
return feat
def perform_clustering(mss_id):
import numpy
import expenv
mss = expenv.MultiSplitSet.get(mss_id)
from method_mhc_mkl import SequencesHandler
from shogun.Distance import EuclidianDistance, HammingWordDistance
from shogun.Features import StringCharFeatures, StringWordFeatures, PROTEIN
from shogun.Clustering import Hierarchical
from shogun.PreProc import SortWordString
order = 1
gap = 0
reverse = False
seq_handler = SequencesHandler()
data = [seq_handler.get_seq(ss.dataset.organism) for ss in mss.split_sets]
charfeat = StringCharFeatures(PROTEIN)
charfeat.set_features(data)
feats = StringWordFeatures(charfeat.get_alphabet())
feats.obtain_from_char(charfeat, order - 1, order, gap, reverse)
preproc = SortWordString()
preproc.init(feats)
feats.add_preproc(preproc)
feats.apply_preproc()
use_sign = False
distance = HammingWordDistance(feats, feats, use_sign)
#distance = EuclidianDistance()
merges = 4
hierarchical = Hierarchical(merges, distance)
hierarchical.train()
hierarchical.get_merge_distances()
hierarchical.get_cluster_pairs()
return hierarchical
def perform_clustering(mss_id):
import numpy
import expenv
mss = expenv.MultiSplitSet.get(mss_id)
from method_mhc_mkl import SequencesHandler
from shogun.Distance import EuclidianDistance, HammingWordDistance
from shogun.Features import StringCharFeatures, StringWordFeatures, PROTEIN
from shogun.Clustering import Hierarchical
from shogun.PreProc import SortWordString
order = 1
gap = 0
reverse = False
seq_handler = SequencesHandler()
data = [seq_handler.get_seq(ss.dataset.organism) for ss in mss.split_sets]
charfeat=StringCharFeatures(PROTEIN)
charfeat.set_features(data)
feats=StringWordFeatures(charfeat.get_alphabet())
feats.obtain_from_char(charfeat, order-1, order, gap, reverse)
preproc=SortWordString()
preproc.init(feats)
feats.add_preproc(preproc)
feats.apply_preproc()
use_sign = False
distance = HammingWordDistance(feats, feats, use_sign)
#distance = EuclidianDistance()
merges=4
hierarchical=Hierarchical(merges, distance)
hierarchical.train()
hierarchical.get_merge_distances()
hierarchical.get_cluster_pairs()
return hierarchical
def features_string_word_modular(strings, start, order, gap, rev):
from shogun.Features import StringCharFeatures, StringWordFeatures, RAWBYTE
from numpy import array, uint16
#create string features
cf = StringCharFeatures(strings, RAWBYTE)
wf = StringWordFeatures(RAWBYTE)
wf.obtain_from_char(cf, start, order, gap, rev)
#and output several stats
#print "max string length", wf.get_max_vector_length()
#print "number of strings", wf.get_num_vectors()
#print "length of first string", wf.get_vector_length(0)
#print "string[2]", wf.get_feature_vector(2)
#print "strings", wf.get_features()
#replace string 0
wf.set_feature_vector(array([1, 2, 3, 4, 5], dtype=uint16), 0)
#print "strings", wf.get_features()
return wf.get_features(), wf
def kernel_poly_match_word_string_modular (fm_train_dna=traindat,fm_test_dna=testdat, degree=2,inhomogene=True,order=3,gap=0,reverse=False): from shogun.Kernel import PolyMatchWordStringKernel from shogun.Features import StringWordFeatures, StringCharFeatures, DNA charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(DNA) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(DNA) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) kernel=PolyMatchWordStringKernel(feats_train, feats_train, degree, inhomogene) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def kernel_match_word_string_modular (fm_train_dna=traindat,fm_test_dna=testdat, degree=3,scale=1.4,size_cache=10,order=3,gap=0,reverse=False): from shogun.Kernel import MatchWordStringKernel, AvgDiagKernelNormalizer from shogun.Features import StringWordFeatures, StringCharFeatures, DNA charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(DNA) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(DNA) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) kernel=MatchWordStringKernel(size_cache, degree) kernel.set_normalizer(AvgDiagKernelNormalizer(scale)) kernel.init(feats_train, feats_train) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def features_string_word_modular(strings, start, order, gap, rev):
from shogun.Features import StringCharFeatures, StringWordFeatures, RAWBYTE
from numpy import array, uint16
# create string features
cf = StringCharFeatures(strings, RAWBYTE)
wf = StringWordFeatures(RAWBYTE)
wf.obtain_from_char(cf, start, order, gap, rev)
# and output several stats
# print "max string length", wf.get_max_vector_length()
# print "number of strings", wf.get_num_vectors()
# print "length of first string", wf.get_vector_length(0)
# print "string[2]", wf.get_feature_vector(2)
# print "strings", wf.get_features()
# replace string 0
wf.set_feature_vector(array([1, 2, 3, 4, 5], dtype=uint16), 0)
# print "strings", wf.get_features()
return wf.get_features(), wf
def poly_match_word_string (): print 'PolyMatchWordString' from shogun.Kernel import PolyMatchWordStringKernel from shogun.Features import StringWordFeatures, StringCharFeatures, DNA degree=2 inhomogene=True order=3 gap=0 reverse=False charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(DNA) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(DNA) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) kernel=PolyMatchWordStringKernel(feats_train, feats_train, degree, inhomogene) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix()
def create_features(kname, examples, kparam, train_mode, preproc, seq_source, nuc_con):
"""Converts numpy arrays or sequences into shogun features"""
if kname == 'gauss' or kname == 'linear' or kname == 'poly':
examples = numpy.array(examples)
feats = RealFeatures(examples)
elif kname == 'wd' or kname == 'localalign' or kname == 'localimprove':
if seq_source == 'dna':
examples = non_atcg_convert(examples, nuc_con)
feats = StringCharFeatures(examples, DNA)
elif seq_source == 'protein':
examples = non_aminoacid_converter(examples, nuc_con)
feats = StringCharFeatures(examples, PROTEIN)
else:
sys.stderr.write("Sequence source -"+seq_source+"- is invalid. select [dna|protein]\n")
sys.exit(-1)
elif kname == 'spec' or kname == 'cumspec':
if seq_source == 'dna':
examples = non_atcg_convert(examples, nuc_con)
feats = StringCharFeatures(examples, DNA)
elif seq_source == 'protein':
examples = non_aminoacid_converter(examples, nuc_con)
feats = StringCharFeatures(examples, PROTEIN)
else:
sys.stderr.write("Sequence source -"+seq_source+"- is invalid. select [dna|protein]\n")
sys.exit(-1)
wf = StringUlongFeatures( feats.get_alphabet() )
wf.obtain_from_char(feats, kparam['degree']-1, kparam['degree'], 0, kname=='cumspec')
del feats
if train_mode:
preproc = SortUlongString()
preproc.init(wf)
wf.add_preproc(preproc)
ret = wf.apply_preproc()
#assert(ret)
feats = wf
elif kname == 'spec2' or kname == 'cumspec2':
# spectrum kernel on two sequences
feats = {}
feats['combined'] = CombinedFeatures()
reversed = kname=='cumspec2'
(ex0,ex1) = zip(*examples)
f0 = StringCharFeatures(list(ex0), DNA)
wf = StringWordFeatures(f0.get_alphabet())
wf.obtain_from_char(f0, kparam['degree']-1, kparam['degree'], 0, reversed)
del f0
if train_mode:
preproc = SortWordString()
preproc.init(wf)
wf.add_preprocessor(preproc)
ret = wf.apply_preprocessors()
assert(ret)
feats['combined'].append_feature_obj(wf)
feats['f0'] = wf
f1 = StringCharFeatures(list(ex1), DNA)
wf = StringWordFeatures( f1.get_alphabet() )
wf.obtain_from_char(f1, kparam['degree']-1, kparam['degree'], 0, reversed)
del f1
if train_mode:
preproc = SortWordString()
preproc.init(wf)
wf.add_preproc(preproc)
ret = wf.apply_preproc()
assert(ret)
feats['combined'].append_feature_obj(wf)
feats['f1'] = wf
else:
print 'Unknown kernel %s' % kname
return (feats,preproc)
def fisher (): print "Fisher Kernel" from shogun.Features import StringCharFeatures, StringWordFeatures, FKFeatures, DNA from shogun.Kernel import PolyKernel from shogun.Distribution import HMM, BW_NORMAL N=1 # toy HMM with 1 state M=4 # 4 observations -> DNA pseudo=1e-1 order=1 gap=0 reverse=False kargs=[1, False, True] # train HMM for positive class charfeat=StringCharFeatures(fm_hmm_pos, DNA) hmm_pos_train=StringWordFeatures(charfeat.get_alphabet()) hmm_pos_train.obtain_from_char(charfeat, order-1, order, gap, reverse) pos=HMM(hmm_pos_train, N, M, pseudo) pos.baum_welch_viterbi_train(BW_NORMAL) # train HMM for negative class charfeat=StringCharFeatures(fm_hmm_neg, DNA) hmm_neg_train=StringWordFeatures(charfeat.get_alphabet()) hmm_neg_train.obtain_from_char(charfeat, order-1, order, gap, reverse) neg=HMM(hmm_neg_train, N, M, pseudo) neg.baum_welch_viterbi_train(BW_NORMAL) # Kernel training data charfeat=StringCharFeatures(fm_train_dna, DNA) wordfeats_train=StringWordFeatures(charfeat.get_alphabet()) wordfeats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) # Kernel testing data charfeat=StringCharFeatures(fm_test_dna, DNA) wordfeats_test=StringWordFeatures(charfeat.get_alphabet()) wordfeats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) # get kernel on training data pos.set_observations(wordfeats_train) neg.set_observations(wordfeats_train) feats_train=FKFeatures(10, pos, neg) feats_train.set_opt_a(-1) #estimate prior kernel=PolyKernel(feats_train, feats_train, *kargs) km_train=kernel.get_kernel_matrix() # get kernel on testing data pos_clone=HMM(pos) neg_clone=HMM(neg) pos_clone.set_observations(wordfeats_test) neg_clone.set_observations(wordfeats_test) feats_test=FKFeatures(10, pos_clone, neg_clone) feats_test.set_a(feats_train.get_a()) #use prior from training data kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix()
def preproc_sortwordstring_modular (fm_train_dna=traindna,fm_test_dna=testdna,order=3,gap=0,reverse=False,use_sign=False): from shogun.Kernel import CommWordStringKernel from shogun.Features import StringCharFeatures, StringWordFeatures, DNA from shogun.PreProc import SortWordString charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) preproc=SortWordString() preproc.init(feats_train) feats_train.add_preproc(preproc) feats_train.apply_preproc() charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) feats_test.add_preproc(preproc) feats_test.apply_preproc() kernel=CommWordStringKernel(feats_train, feats_train, use_sign) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def tests_check_commwordkernel_memleak_modular (num, order, gap, reverse): import gc from shogun.Features import Alphabet,StringCharFeatures,StringWordFeatures,DNA from shogun.Preprocessor import SortWordString, MSG_DEBUG from shogun.Kernel import CommWordStringKernel, IdentityKernelNormalizer from numpy import mat POS=[num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT'] NEG=[num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT'] for i in range(10): alpha=Alphabet(DNA) traindat=StringCharFeatures(alpha) traindat.set_features(POS+NEG) trainudat=StringWordFeatures(traindat.get_alphabet()); trainudat.obtain_from_char(traindat, order-1, order, gap, reverse) #trainudat.io.set_loglevel(MSG_DEBUG) pre = SortWordString() #pre.io.set_loglevel(MSG_DEBUG) pre.init(trainudat) trainudat.add_preprocessor(pre) trainudat.apply_preprocessor() spec = CommWordStringKernel(10, False) spec.set_normalizer(IdentityKernelNormalizer()) spec.init(trainudat, trainudat) K=spec.get_kernel_matrix() del POS del NEG del order del gap del reverse return K
def kernel_fisher_modular(fm_train_dna=traindat, fm_test_dna=testdat, label_train_dna=label_traindat, N=1,M=4,pseudo=1e-1,order=1,gap=0,reverse=False, kargs=[1,False,True]): from shogun.Features import StringCharFeatures, StringWordFeatures, FKFeatures, DNA from shogun.Kernel import PolyKernel from shogun.Distribution import HMM, BW_NORMAL#, MSG_DEBUG # train HMM for positive class charfeat=StringCharFeatures(fm_hmm_pos, DNA) #charfeat.io.set_loglevel(MSG_DEBUG) hmm_pos_train=StringWordFeatures(charfeat.get_alphabet()) hmm_pos_train.obtain_from_char(charfeat, order-1, order, gap, reverse) pos=HMM(hmm_pos_train, N, M, pseudo) pos.baum_welch_viterbi_train(BW_NORMAL) # train HMM for negative class charfeat=StringCharFeatures(fm_hmm_neg, DNA) hmm_neg_train=StringWordFeatures(charfeat.get_alphabet()) hmm_neg_train.obtain_from_char(charfeat, order-1, order, gap, reverse) neg=HMM(hmm_neg_train, N, M, pseudo) neg.baum_welch_viterbi_train(BW_NORMAL) # Kernel training data charfeat=StringCharFeatures(fm_train_dna, DNA) wordfeats_train=StringWordFeatures(charfeat.get_alphabet()) wordfeats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) # Kernel testing data charfeat=StringCharFeatures(fm_test_dna, DNA) wordfeats_test=StringWordFeatures(charfeat.get_alphabet()) wordfeats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) # get kernel on training data pos.set_observations(wordfeats_train) neg.set_observations(wordfeats_train) feats_train=FKFeatures(10, pos, neg) feats_train.set_opt_a(-1) #estimate prior kernel=PolyKernel(feats_train, feats_train, *kargs) km_train=kernel.get_kernel_matrix() # get kernel on testing data pos_clone=HMM(pos) neg_clone=HMM(neg) pos_clone.set_observations(wordfeats_test) neg_clone.set_observations(wordfeats_test) feats_test=FKFeatures(10, pos_clone, neg_clone) feats_test.set_a(feats_train.get_a()) #use prior from training data kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def distance_hammingword_modular (fm_train_dna=traindna,fm_test_dna=testdna, fm_test_real=testdat,order=3,gap=0,reverse=False,use_sign=False): from shogun.Features import StringCharFeatures, StringWordFeatures, DNA from shogun.Preprocessor import SortWordString from shogun.Distance import HammingWordDistance charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_train_dna) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) preproc=SortWordString() preproc.init(feats_train) feats_train.add_preprocessor(preproc) feats_train.apply_preprocessor() charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_test_dna) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) feats_test.add_preprocessor(preproc) feats_test.apply_preprocessor() distance=HammingWordDistance(feats_train, feats_train, use_sign) dm_train=distance.get_distance_matrix() distance.init(feats_train, feats_test) dm_test=distance.get_distance_matrix() return distance,dm_train,dm_test
def distance_hammingword_modular(fm_train_dna=traindna,
fm_test_dna=testdna,
fm_test_real=testdat,
order=3,
gap=0,
reverse=False,
use_sign=False):
from shogun.Features import StringCharFeatures, StringWordFeatures, DNA
from shogun.Preprocessor import SortWordString
from shogun.Distance import HammingWordDistance
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_train_dna)
feats_train = StringWordFeatures(charfeat.get_alphabet())
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
preproc = SortWordString()
preproc.init(feats_train)
feats_train.add_preprocessor(preproc)
feats_train.apply_preprocessor()
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_test_dna)
feats_test = StringWordFeatures(charfeat.get_alphabet())
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
feats_test.add_preprocessor(preproc)
feats_test.apply_preprocessor()
distance = HammingWordDistance(feats_train, feats_train, use_sign)
dm_train = distance.get_distance_matrix()
distance.init(feats_train, feats_test)
dm_test = distance.get_distance_matrix()
return distance, dm_train, dm_test
def kernel_fisher_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
label_train_dna=label_traindat,
N=1,
M=4,
pseudo=1e-1,
order=1,
gap=0,
reverse=False,
kargs=[1, False, True]):
from shogun.Features import StringCharFeatures, StringWordFeatures, FKFeatures, DNA
from shogun.Kernel import PolyKernel
from shogun.Distribution import HMM, BW_NORMAL #, MSG_DEBUG
# train HMM for positive class
charfeat = StringCharFeatures(fm_hmm_pos, DNA)
#charfeat.io.set_loglevel(MSG_DEBUG)
hmm_pos_train = StringWordFeatures(charfeat.get_alphabet())
hmm_pos_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
pos = HMM(hmm_pos_train, N, M, pseudo)
pos.baum_welch_viterbi_train(BW_NORMAL)
# train HMM for negative class
charfeat = StringCharFeatures(fm_hmm_neg, DNA)
hmm_neg_train = StringWordFeatures(charfeat.get_alphabet())
hmm_neg_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
neg = HMM(hmm_neg_train, N, M, pseudo)
neg.baum_welch_viterbi_train(BW_NORMAL)
# Kernel training data
charfeat = StringCharFeatures(fm_train_dna, DNA)
wordfeats_train = StringWordFeatures(charfeat.get_alphabet())
wordfeats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
# Kernel testing data
charfeat = StringCharFeatures(fm_test_dna, DNA)
wordfeats_test = StringWordFeatures(charfeat.get_alphabet())
wordfeats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
# get kernel on training data
pos.set_observations(wordfeats_train)
neg.set_observations(wordfeats_train)
feats_train = FKFeatures(10, pos, neg)
feats_train.set_opt_a(-1) #estimate prior
kernel = PolyKernel(feats_train, feats_train, *kargs)
km_train = kernel.get_kernel_matrix()
# get kernel on testing data
pos_clone = HMM(pos)
neg_clone = HMM(neg)
pos_clone.set_observations(wordfeats_test)
neg_clone.set_observations(wordfeats_test)
feats_test = FKFeatures(10, pos_clone, neg_clone)
feats_test.set_a(feats_train.get_a()) #use prior from training data
kernel.init(feats_train, feats_test)
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def kernel_top_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
label_train_dna=label_traindat,
pseudo=1e-1,
order=1,
gap=0,
reverse=False,
kargs=[1, False, True]):
from shogun.Features import StringCharFeatures, StringWordFeatures, TOPFeatures, DNA
from shogun.Kernel import PolyKernel
from shogun.Distribution import HMM, BW_NORMAL
N = 1 # toy HMM with 1 state
M = 4 # 4 observations -> DNA
# train HMM for positive class
charfeat = StringCharFeatures(fm_hmm_pos, DNA)
hmm_pos_train = StringWordFeatures(charfeat.get_alphabet())
hmm_pos_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
pos = HMM(hmm_pos_train, N, M, pseudo)
pos.baum_welch_viterbi_train(BW_NORMAL)
# train HMM for negative class
charfeat = StringCharFeatures(fm_hmm_neg, DNA)
hmm_neg_train = StringWordFeatures(charfeat.get_alphabet())
hmm_neg_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
neg = HMM(hmm_neg_train, N, M, pseudo)
neg.baum_welch_viterbi_train(BW_NORMAL)
# Kernel training data
charfeat = StringCharFeatures(fm_train_dna, DNA)
wordfeats_train = StringWordFeatures(charfeat.get_alphabet())
wordfeats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
# Kernel testing data
charfeat = StringCharFeatures(fm_test_dna, DNA)
wordfeats_test = StringWordFeatures(charfeat.get_alphabet())
wordfeats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
# get kernel on training data
pos.set_observations(wordfeats_train)
neg.set_observations(wordfeats_train)
feats_train = TOPFeatures(10, pos, neg, False, False)
kernel = PolyKernel(feats_train, feats_train, *kargs)
km_train = kernel.get_kernel_matrix()
# get kernel on testing data
pos_clone = HMM(pos)
neg_clone = HMM(neg)
pos_clone.set_observations(wordfeats_test)
neg_clone.set_observations(wordfeats_test)
feats_test = TOPFeatures(10, pos_clone, neg_clone, False, False)
kernel.init(feats_train, feats_test)
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
def manhattan_word_distance (): print 'ManhattanWordDistance' from shogun.Features import StringCharFeatures, StringWordFeatures, DNA from shogun.PreProc import SortWordString from shogun.Distance import ManhattanWordDistance order=3 gap=0 reverse=False charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_train_dna) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) preproc=SortWordString() preproc.init(feats_train) feats_train.add_preproc(preproc) feats_train.apply_preproc() charfeat=StringCharFeatures(DNA) charfeat.set_features(fm_test_dna) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) feats_test.add_preproc(preproc) feats_test.apply_preproc() distance=ManhattanWordDistance(feats_train, feats_train) dm_train=distance.get_distance_matrix() distance.init(feats_train, feats_test) dm_test=distance.get_distance_matrix()
def kernel_weighted_comm_word_string_modular (fm_train_dna=traindat,fm_test_dna=testdat,order=3,gap=0,reverse=True ): from shogun.Kernel import WeightedCommWordStringKernel from shogun.Features import StringWordFeatures, StringCharFeatures, DNA from shogun.Preprocessor import SortWordString charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) preproc=SortWordString() preproc.init(feats_train) feats_train.add_preprocessor(preproc) feats_train.apply_preprocessor() charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) feats_test.add_preprocessor(preproc) feats_test.apply_preprocessor() use_sign=False kernel=WeightedCommWordStringKernel(feats_train, feats_train, use_sign) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def sort_word_string (): print 'CommWordString' from shogun.Kernel import CommWordStringKernel from shogun.Features import StringCharFeatures, StringWordFeatures, DNA from shogun.PreProc import SortWordString order=3 gap=0 reverse=False charfeat=StringCharFeatures(fm_train_dna, DNA) feats_train=StringWordFeatures(charfeat.get_alphabet()) feats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) preproc=SortWordString() preproc.init(feats_train) feats_train.add_preproc(preproc) feats_train.apply_preproc() charfeat=StringCharFeatures(fm_test_dna, DNA) feats_test=StringWordFeatures(charfeat.get_alphabet()) feats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) feats_test.add_preproc(preproc) feats_test.apply_preproc() use_sign=False kernel=CommWordStringKernel(feats_train, feats_train, use_sign) km_train=kernel.get_kernel_matrix() kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix()
def tests_check_commwordkernel_memleak_modular(num, order, gap, reverse): import gc from shogun.Features import Alphabet,StringCharFeatures,StringWordFeatures,DNA from shogun.Preprocessor import SortWordString, MSG_DEBUG from shogun.Kernel import CommWordStringKernel, IdentityKernelNormalizer from numpy import mat POS=[num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT'] NEG=[num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'TTGT', num*'TTGT', num*'TTGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT',num*'ACGT', num*'ACGT', num*'ACGT'] for i in xrange(10): alpha=Alphabet(DNA) traindat=StringCharFeatures(alpha) traindat.set_features(POS+NEG) trainudat=StringWordFeatures(traindat.get_alphabet()); trainudat.obtain_from_char(traindat, order-1, order, gap, reverse) #trainudat.io.set_loglevel(MSG_DEBUG) pre = SortWordString() #pre.io.set_loglevel(MSG_DEBUG) pre.init(trainudat) trainudat.add_preproc(pre) trainudat.apply_preproc() spec = CommWordStringKernel(10, False) spec.set_normalizer(IdentityKernelNormalizer()) spec.init(trainudat, trainudat) K=spec.get_kernel_matrix() del POS del NEG del order del gap del reverse return K
def kernel_top_modular(fm_train_dna=traindat,fm_test_dna=testdat,label_train_dna=label_traindat,pseudo=1e-1, order=1,gap=0,reverse=False,kargs=[1, False, True]): from shogun.Features import StringCharFeatures, StringWordFeatures, TOPFeatures, DNA from shogun.Kernel import PolyKernel from shogun.Distribution import HMM, BW_NORMAL N=1 # toy HMM with 1 state M=4 # 4 observations -> DNA # train HMM for positive class charfeat=StringCharFeatures(fm_hmm_pos, DNA) hmm_pos_train=StringWordFeatures(charfeat.get_alphabet()) hmm_pos_train.obtain_from_char(charfeat, order-1, order, gap, reverse) pos=HMM(hmm_pos_train, N, M, pseudo) pos.baum_welch_viterbi_train(BW_NORMAL) # train HMM for negative class charfeat=StringCharFeatures(fm_hmm_neg, DNA) hmm_neg_train=StringWordFeatures(charfeat.get_alphabet()) hmm_neg_train.obtain_from_char(charfeat, order-1, order, gap, reverse) neg=HMM(hmm_neg_train, N, M, pseudo) neg.baum_welch_viterbi_train(BW_NORMAL) # Kernel training data charfeat=StringCharFeatures(fm_train_dna, DNA) wordfeats_train=StringWordFeatures(charfeat.get_alphabet()) wordfeats_train.obtain_from_char(charfeat, order-1, order, gap, reverse) # Kernel testing data charfeat=StringCharFeatures(fm_test_dna, DNA) wordfeats_test=StringWordFeatures(charfeat.get_alphabet()) wordfeats_test.obtain_from_char(charfeat, order-1, order, gap, reverse) # get kernel on training data pos.set_observations(wordfeats_train) neg.set_observations(wordfeats_train) feats_train=TOPFeatures(10, pos, neg, False, False) kernel=PolyKernel(feats_train, feats_train, *kargs) km_train=kernel.get_kernel_matrix() # get kernel on testing data pos_clone=HMM(pos) neg_clone=HMM(neg) pos_clone.set_observations(wordfeats_test) neg_clone.set_observations(wordfeats_test) feats_test=TOPFeatures(10, pos_clone, neg_clone, False, False) kernel.init(feats_train, feats_test) km_test=kernel.get_kernel_matrix() return km_train,km_test,kernel
def kernel_comm_word_string_modular(fm_train_dna=traindat,
fm_test_dna=testdat,
order=3,
gap=0,
reverse=False,
use_sign=False):
from shogun.Kernel import CommWordStringKernel
from shogun.Features import StringWordFeatures, StringCharFeatures, DNA
from shogun.Preprocessor import SortWordString
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_train_dna)
feats_train = StringWordFeatures(charfeat.get_alphabet())
feats_train.obtain_from_char(charfeat, order - 1, order, gap, reverse)
preproc = SortWordString()
preproc.init(feats_train)
feats_train.add_preproc(preproc)
feats_train.apply_preproc()
charfeat = StringCharFeatures(DNA)
charfeat.set_features(fm_test_dna)
feats_test = StringWordFeatures(charfeat.get_alphabet())
feats_test.obtain_from_char(charfeat, order - 1, order, gap, reverse)
feats_test.add_preproc(preproc)
feats_test.apply_preproc()
kernel = CommWordStringKernel(feats_train, feats_train, use_sign)
km_train = kernel.get_kernel_matrix()
kernel.init(feats_train, feats_test)
km_test = kernel.get_kernel_matrix()
return km_train, km_test, kernel
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT',100*'ACGT', 100*'ACGT',
100*'ACGT',100*'ACGT', 100*'ACGT', 100*'ACGT']
order=7
gap=0
reverse=False
for i in xrange(10):
alpha=Alphabet(DNA)
traindat=StringCharFeatures(alpha)
traindat.set_features(POS+NEG)
trainudat=StringWordFeatures(traindat.get_alphabet());
trainudat.obtain_from_char(traindat, order-1, order, gap, reverse)
#trainudat.io.set_loglevel(MSG_DEBUG)
pre = SortWordString()
#pre.io.set_loglevel(MSG_DEBUG)
pre.init(trainudat)
trainudat.add_preproc(pre)
trainudat.apply_preproc()
spec = CommWordStringKernel(10, False)
spec.set_normalizer(IdentityKernelNormalizer())
spec.init(trainudat, trainudat)
K=mat(spec.get_kernel_matrix())
del POS
del NEG
del order