def main(args):
"""
EM only with shift
templates are generated based on the configuration script
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
num_shifts=config_d['EMTRAINING']['num_shifts']
num_models = len(args.models)
assert num_models == len(args.data)
models = []
backgrounds = []
template_log_inv_probs = []
template_log_odds = []
for i in xrange(num_models):
models.append(np.load(args.models[i]))
backgrounds.append(np.load(args.bgds[i]))
template_log_inv_probs.append(np.log(1-models[i]))
template_log_odds.append(np.log(models[i]) - template_log_inv_probs[i])
data = []
out_likelihoods = []
for i in xrange(num_models):
data.append(np.load(args.data[i]))
likelihoods = []
for j in xrange(num_models):
bgd_sums = prep_bgd_sums(data[i],backgrounds[j])
likelihoods.append(
compute_likelihood(
data[i],
template_log_odds[j],
template_log_inv_probs[j],
bgd_sums,
config_d['INFERENCE']['start_time'],
config_d['INFERENCE']['num_shifts']))
out_likelihoods.append(np.array(likelihoods))
confusion_matrix = np.zeros((num_models,num_models))
for data_id in xrange(num_models):
for model_id in xrange(num_models):
confusion_matrix[data_id,
model_id] = np.sum(out_likelihoods[data_id].argmax(0) == model_id)
num_data = confusion_matrix.sum()
error_rate = (num_data - np.diag(confusion_matrix).sum())/num_data
print "error_rate = %f" % error_rate
np.save('%sconfusion_matrix.npy' % args.out,confusion_matrix)
def main(args):
"""
Get the start and end frame associated to each example
and then extract the example and save into a matrix
we make the assumption that all the examples have the exact
same length as the first
"""
config_d = configParserWrapper.load_settings(open(args.c, 'r'))
uttid_dict = dict(
tuple((uttid, k) for k, uttid in enumerate(
open(args.uttids, 'r').read().strip().split('\n'))))
examples = []
examples_S = []
output_info = []
for line_id, line in enumerate(open(args.transcript, 'r')):
split_line = line.strip().split()
fpath = split_line[0]
uttid = '_'.join(fpath.strip('.wav').split('/')[-3:])
split_line = split_line[1:]
frame_starts = np.array(tuple(int(k) for k in split_line[::3]),
dtype=int)
frame_ends = np.array(tuple(int(k) for k in split_line[1::3]),
dtype=int)
example_length = int((frame_ends - frame_starts).max())
sr, x = wavfile.read(fpath)
x = x.astype(float) / 2**15
x = x / np.std(x)
S, sample_mapping, sample_to_frames = gtrd.get_spectrogram_use_config(
x, config_d['SPECTROGRAM'], return_sample_mapping=True)
X = gtrd.get_edge_features_use_config(S.T, config_d['EDGES'])
S = np.vstack((S[:example_length][::-1], S, S[::-1][:example_length]))
X = np.vstack((X[:example_length][::-1], X, X[::-1][:example_length]))
frame_starts += example_length
frame_ends += example_length
for i, s_e in enumerate(itertools.izip(frame_starts, frame_ends)):
s, e = s_e
examples.append(X[s:e])
examples_S.append(S[s:e])
output_info.append(
(uttid_dict[uttid], i, s - example_length, e - example_length))
if line_id % 100 == 0:
print line_id, uttid, line
examples = np.array(examples, dtype=np.uint8)
examples_S = np.array(examples_S)
np.save(args.output, examples)
if args.output_info is not None:
np.save(args.output_info, np.array(output_info, dtype=int))
if args.output_spec is not None:
np.save(args.output_spec, examples_S)
def main(args):
"""
Get the start and end frame associated to each example
and then extract the example and save into a matrix
we make the assumption that all the examples have the exact
same length as the first
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
uttid_dict = dict(tuple( (uttid,k) for k,uttid in enumerate(open(args.uttids,'r').read().strip().split('\n'))))
examples = []
examples_S = []
output_info = []
for line_id, line in enumerate(open(args.transcript,'r')):
split_line = line.strip().split()
fpath = split_line[0]
uttid = '_'.join(fpath.strip('.wav').split('/')[-3:])
split_line = split_line[1:]
frame_starts = np.array(tuple(int(k) for k in split_line[::3]), dtype=int)
frame_ends = np.array( tuple( int(k) for k in split_line[1::3]),dtype=int)
example_length = int((frame_ends - frame_starts).max())
sr, x = wavfile.read(fpath)
S,sample_mapping, sample_to_frames = gtrd.get_spectrogram_use_config(x,config_d['SPECTROGRAM'],return_sample_mapping=True)
X = gtrd.get_edge_features_use_config(S.T,config_d['EDGES'])
S = np.vstack( (S[:example_length][::-1],
S,
S[::-1][:example_length]))
X = np.vstack( (X[:example_length][::-1],
X,
X[::-1][:example_length]))
frame_starts += example_length
frame_ends += example_length
for i, s_e in enumerate(itertools.izip(frame_starts,frame_ends)):
s,e = s_e
examples.append(X[s:e])
examples_S.append(S[s:e])
output_info.append( (uttid_dict[uttid],i,s-example_length,
e-example_length))
if line_id % 100 == 0:
print line_id, uttid, line
examples = np.array(examples,dtype=np.uint8)
examples_S = np.array(examples_S)
np.save(args.output, examples)
if args.output_info is not None:
np.save(args.output_info,np.array(output_info,dtype=int))
if args.output_spec is not None:
np.save(args.output_spec,examples_S)
def main(args):
"""
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
fls = open(args.f,'r').read().strip().split('\n')
avg_bgd = AverageBackground()
for fl in fls:
sr, x = wavfile.read(fl)
X = compute_edge_features(x,config_d)
avg_bgd.add_frames(X)
np.save(args.o,avg_bgd.E)
def main(args):
config_d = configParserWrapper.load_settings(open(args.c,'r'))
# load in the data
X = np.load("%s_E.npy" % args.f)
M = np.array([ tuple(int(e) for e in l.split()) for l in open("%s_meta.txt" % args.f,'r')])
bgd = np.clip(np.load(args.b),
config_d['TEMPLATE_TRAINING']['bgd_min_prob'],
config_d['TEMPLATE_TRAINING']['bgd_max_prob']).ravel()
# prepare the background portion of the log likelihood ratio filter
bgd_inv_log = np.log(1-bgd)
bgd_log = np.log(bgd)
bgd_norm_term = np.sum(bgd_inv_log)
bgd_log_odds = bgd_log - bgd_inv_log
# construct the model portion of the log likelihood ratio filter
models = np.zeros(X.shape)
for example_id, example_stats in enumerate(M):
start,end,word_id = example_stats
for edge_id in xrange(X.shape[-1]):
models[start:end,:,edge_id] = kde_bernoulli_probs(
X[start:end,:,edge_id],
config_d['TEMPLATE_TRAINING']['bandwidth'])
models = np.clip(models,
config_d['TEMPLATE_TRAINING']['min_probability'],
1-config_d['TEMPLATE_TRAINING']['min_probability'])
models = models.reshape(*((len(models),np.prod(models.shape[1:]))))
models_inv_log = np.log(1-models)
model_norm_terms = models_inv_log.sum(1)
model_log_odds = np.log(models) - models_inv_log
log_ratio_filter = model_log_odds - bgd_log_odds
log_ratio_const = model_norm_terms - bgd_norm_term
bgd_response = (log_ratio_filter * bgd).sum(1) + log_ratio_const
np.save('%s_filter.npy' % args.o,log_ratio_filter)
np.save('%s_const.npy' % args.o,log_ratio_const)
np.save('%s_bgd_response.npy' % args.o,bgd_response)
np.save('%s_meta.npy' %args.o,M)
def main(args):
"""
Visualize the parts and where they are active on
a given utterance
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
part_means = np.load(args.part_edge_means)
spec_means = np.load(args.spec_means)
log_part_means = np.log(part_means)
log_part_inv_means = np.log(1-part_means)
log_part_odds = log_part_means - log_part_inv_means
part_shape = log_part_odds.shape[1:]
log_part_odds = log_part_odds.reshape(log_part_odds.shape[0],
np.prod(part_shape)).T
constants = log_part_inv_means.sum(-1).sum(-1).sum(-1)
sr, x = wavfile.read(args.wavfile)
S,sample_mapping, sample_to_frames = gtrd.get_spectrogram_use_config(x,config_d['SPECTROGRAM'],return_sample_mapping=True)
E = gtrd.get_edge_features_use_config(S.T,config_d['EDGES'])
X = code_spread_parts(E,
log_part_odds,
constants,part_shape,part_shape[:-1],
count_threshold=20,likelihood_threshold=-300/200. * np.prod(part_shape))
for part_id in xrange(X.shape[-1]):
for i in xrange(int((E.shape[0] - 200)/50.)):
P_chunk = X[i*50:i*50+200,:,part_id]
S_chunk = S[i*50:i*50+200]
savename = '%s_%d_%d.png' % (args.viz_save_prefix,
part_id,
i)
plot_spec_chunk_binary_features(S_chunk,P_chunk,savename,spec_means[part_id])
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
config_d = configParserWrapper.load_settings(open(args.config, 'r'))
true_examples = []
false_examples = []
mean = 0
total = 0
num_less_than_eq = np.zeros(20)
all_X_patches = []
all_S_patches = []
for phn_id, (fl_edge, fl_spec) in enumerate(
itertools.izip(args.data, args.data_spec)):
if len(all_X_patches) > 100000: break
print phn_id
X = np.load(fl_edge)
X_shape = X.shape[1:]
S = np.load(fl_spec)
if args.do_exp_weighted_divergence:
S *= np.exp(S)
X_patches = []
S_patches = []
for i in xrange(len(X)):
if len(X_patches) > 1000: break
p_edge, p_spec = get_maximal_patches(
X[i], S[i], patch_radius=args.patch_radius)
X_patches.extend(p_edge)
S_patches.extend(p_spec)
num_new_patches = len(X_patches)
print phn_id, num_new_patches
all_X_patches.extend(X_patches)
all_S_patches.extend(S_patches)
X = np.array(all_X_patches)
S = np.array(all_S_patches)
data_shape = X.shape[1:]
X = X.reshape(X.shape[0], np.prod(data_shape))
bmm = bernoullimm.BernoulliMM(n_components=args.n_components,
n_init=20,
n_iter=500,
random_state=0,
verbose=args.v,
tol=1e-6)
bmm.fit(X)
# check above 30
use_means = bmm.predict_proba(X).sum(0) > 30
print use_means.sum()
try:
np.save(
args.save_parts,
bmm.means_.reshape(*((bmm.n_components, ) +
data_shape))[use_means])
except:
import pdb
pdb.set_trace()
S_shape = S.shape[1:]
S_clusters = bmm.cluster_underlying_data(
S.reshape(len(S), np.prod(S_shape)),
X).reshape(*((bmm.n_components, ) + S_shape))[use_means]
np.save(args.spec_save_parts, S_clusters)
ncols = int(np.sqrt(args.n_components))
nrows = int(np.ceil(args.n_components / ncols))
if args.viz_spec_parts is not None:
plt.close('all')
fig = plt.figure(1, (6, 6))
grid = ImageGrid(
fig,
111, # similar to subplot(111)
nrows_ncols=(nrows, ncols), # creates 2x2 grid of axes
axes_pad=0.001, # pad between axes in inch.
)
for i in xrange(S_clusters.shape[0]):
try:
grid[i].imshow(S_clusters[i],
cmap=cm.binary,
interpolation='nearest')
grid[i].spines['bottom'].set_color('red')
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
except:
import pdb
pdb.set_trace()
for i in xrange(S_clusters.shape[0], nrows * ncols):
try:
grid[i].spines['bottom'].set_color('red')
except:
import pdb
pdb.set_trace()
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
plt.savefig('%s' % args.viz_spec_parts, bbox_inches='tight')
def main(args):
"""
"""
config_d = configParserWrapper.load_settings(open(args.config, 'r'))
y = []
for fpath_id, fpath in enumerate(args.input_data_matrices):
X0 = np.load(fpath)
X0_shape = X0.shape[1:]
y.extend(fpath_id * np.ones(len(X0)))
y = np.array(y)
# add in two coordinates for the length
if args.input_lengths is not None:
X = np.zeros((len(y), np.prod(X0_shape) + 2), dtype=float)
else:
X = np.zeros((len(y), np.prod(X0_shape)), dtype=float)
cur_idx = 0
for fpath_id, fpath in enumerate(args.input_data_matrices):
X0 = np.load(fpath)
X[cur_idx:cur_idx + len(X0), :np.prod(X0_shape)] = X0.reshape(
len(X0), np.prod(X0_shape))
cur_idx += len(X0)
if args.input_lengths is not None:
train_ls = []
for fpath in args.input_lengths:
ls = np.loadtxt(fpath, dtype=int)
train_ls.extend(ls[:, 2])
train_ls = np.log(np.tile(np.array(train_ls), (2, 1)).T)
train_ls[:, 1] *= train_ls[:, 1]
X[:, -2:] = train_ls
X_test = []
y_test = []
for fpath_id, fpath in enumerate(args.input_test_data_matrices):
X0 = np.load(fpath).astype(np.float)
X0_shape = X0.shape[1:]
X_test.extend(X0.reshape(len(X0), np.prod(X0_shape)))
y_test.extend(fpath_id * np.ones(len(X0)))
X_test = np.array(X_test)
y_test = np.array(y_test)
if args.input_test_lengths is not None:
test_ls = []
for fpath in args.input_test_lengths:
ls = np.loadtxt(fpath, dtype=int)
test_ls.extend(ls[:, 2])
test_ls = np.log(np.tile(np.array(test_ls), (2, 1)).T)
test_ls[:, 1] *= test_ls[:, 1]
X_test = np.hstack((X_test, test_ls))
if args.use_weights is not None:
weights = np.load(args.use_weights)
weights = weights.reshape(weights.size)
X *= weights
X_test *= weights
if args.do_kernel_dist_features:
X_test = np.exp(-cdist(X_test, X, 'euclidean'))
X = np.exp(-squareform(pdist(X, 'euclidean')))
penalty_names = config_d['SVM']['penalty_list'][::2]
penalty_values = tuple(
float(k) for k in config_d['SVM']['penalty_list'][1::2])
dev_results = ()
exp_descriptions = ()
exp_description_id = 0
for penalty_name, penalty_value in itertools.izip(penalty_names,
penalty_values):
if args.v:
print '%s %s' % ('linear', penalty_name)
if config_d['SVM']['kernel'] == 'linear':
clf = svm.LinearSVC(C=penalty_value, loss='l1')
clf.fit(X, y)
elif config_d['SVM']['kernel'] == 'polynomial':
import pdb
pdb.set_trace()
np.save(
'%s_%s_%s_coef.npy' %
(args.output_fls_prefix, config_d['SVM']['kernel'], penalty_name),
clf.coef_[0])
np.save(
'%s_%s_%s_intercept.npy' %
(args.output_fls_prefix, config_d['SVM']['kernel'], penalty_name),
clf.intercept_)
y_test_hat = clf.predict(X_test)
exp_descriptions += ((config_d['SVM']['kernel'], penalty_name), )
dev_results += (
(
exp_description_id,
np.abs(y_test_hat - y_test).sum() / len(y_test), # error rate
np.abs(y_test_hat[y_test == 0] - y_test[y_test == 0]).sum() /
len(y_test[y_test == 0]), # mistakes by class 0
np.abs(y_test_hat[y_test == 1] - y_test[y_test == 1]).sum() /
len(y_test[y_test == 1]) # mistakes by class 1
), )
if args.v:
print '\t'.join(tuple(str(k) for k in dev_results[-1]))
exp_description_id += 1
open('%s_exp_descriptions' % args.output_fls_prefix, 'w').write('\n'.join(
tuple('%d %s' % (k, ' '.join(d))
for k, d in enumerate(exp_descriptions))))
np.save('%s_dev_results.npy' % args.output_fls_prefix,
np.array(dev_results))
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
config_d = configParserWrapper.load_settings(open(args.config,'r'))
true_examples = []
false_examples = []
mean = 0
total = 0
num_less_than_eq = np.zeros(20)
all_X_patches = []
all_S_patches = []
for phn_id, (fl_edge, fl_spec) in enumerate(itertools.izip(args.data,args.data_spec)):
if len(all_X_patches) > 100000: break
print phn_id
X = np.load(fl_edge)
X_shape = X.shape[1:]
S = np.load(fl_spec)
if args.do_exp_weighted_divergence:
S *= np.exp(S)
X_patches = []
S_patches = []
for i in xrange(len(X)):
if len(X_patches) > 1000: break
p_edge,p_spec = get_maximal_patches(X[i],S[i],patch_radius=args.patch_radius)
X_patches.extend(p_edge)
S_patches.extend(p_spec)
num_new_patches = len(X_patches)
print phn_id, num_new_patches
all_X_patches.extend(X_patches)
all_S_patches.extend(S_patches)
X = np.array(all_X_patches)
S = np.array(all_S_patches)
data_shape = X.shape[1:]
X = X.reshape(X.shape[0],np.prod(data_shape))
bmm = bernoullimm.BernoulliMM(n_components=args.n_components,
n_init= 20,
n_iter= 500,
random_state=0,
verbose=args.v, tol=1e-6)
bmm.fit(X)
# check above 30
use_means = bmm.predict_proba(X).sum(0) > 30
print use_means.sum()
try:
np.save(args.save_parts,bmm.means_.reshape(*( (bmm.n_components,)+data_shape))[use_means])
except:
import pdb; pdb.set_trace()
S_shape = S.shape[1:]
S_clusters = bmm.cluster_underlying_data(S.reshape(len(S),np.prod(S_shape)),X).reshape(
*( (bmm.n_components,) + S_shape))[use_means]
np.save(args.spec_save_parts,S_clusters)
ncols = int(np.sqrt(args.n_components))
nrows = int(np.ceil(args.n_components/ncols))
if args.viz_spec_parts is not None:
plt.close('all')
fig = plt.figure(1, (6, 6))
grid = ImageGrid(fig, 111, # similar to subplot(111)
nrows_ncols = (nrows,ncols ), # creates 2x2 grid of axes
axes_pad=0.001, # pad between axes in inch.
)
for i in xrange(S_clusters.shape[0]):
try:
grid[i].imshow(S_clusters[i],cmap=cm.binary,interpolation='nearest')
grid[i].spines['bottom'].set_color('red')
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
except:
import pdb; pdb.set_trace()
for i in xrange(S_clusters.shape[0],nrows*ncols):
try:
grid[i].spines['bottom'].set_color('red')
except: import pdb; pdb.set_trace()
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
plt.savefig('%s' % args.viz_spec_parts
,bbox_inches='tight')
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
log_odds, constants, template_phn_ids, template_component_ids = get_bernoulli_templates(args.templates)
true_examples = []
false_examples = []
for phn_id, fl in enumerate(args.data):
X = np.load(fl)
X_shape = X.shape[1:]
X = X.reshape(X.shape[0],np.prod(X_shape))
bernoulli_scores = np.dot(X,log_odds) + constants
sorted_bernoulli_phn_ids = template_phn_ids[np.argsort(-bernoulli_scores,1).ravel()].reshape(bernoulli_scores.shape)
sorted_bernoulli_component_ids = template_component_ids[np.argsort(-bernoulli_scores,1).ravel()].reshape(bernoulli_scores.shape)
component_indices = np.argmax((sorted_bernoulli_component_ids == args.component_id) * (sorted_bernoulli_phn_ids == args.label),1)
best_component_indices = np.argmax(sorted_bernoulli_phn_ids==args.label,1)
if phn_id == args.label:
# consider a true example if it is in the top 6
# and the component is the right component
true_examples.extend(
X[ (component_indices==best_component_indices) *
(component_indices < 10)])
else:
# we find the index for the true phone
true_phn_idx = np.argmin(((sorted_bernoulli_phn_ids != phn_id).astype(int)*log_odds.shape[1] + 1 ) * (1+np.arange(log_odds.shape[1])),1)
false_examples.extend(
X[ # (component_indices < true_phn_idx)*
(component_indices < 6) ])
if min(len(true_examples),len(false_examples)) < 20:
print "len(true_examples)=%d,len(false_examples)=%d" % (len(true_examples),len(false_examples))
print "So no svm trained for label %d component %d" % (args.label,args.component_id)
return
y = np.array(len(true_examples) * [0] + len(false_examples)*[1])
X = np.array(true_examples + false_examples)
del true_examples
del false_examples
config_d = configParserWrapper.load_settings(open(args.config,'r'))
penalty_names = config_d['SVM']['penalty_list'][::2]
penalty_values = tuple( float(k) for k in config_d['SVM']['penalty_list'][1::2])
for penalty_name, penalty_value in itertools.izip(penalty_names,penalty_values):
if args.v:
print '%s %s' % ('linear', penalty_name)
if config_d['SVM']['kernel'] == 'linear':
clf = svm.LinearSVC(C=penalty_value,
loss='l1')
clf.fit(X,y)
else:
import pdb; pdb.set_trace()
coef = clf.coef_.reshape(clf.coef_.size)
y_hat = np.dot(X,coef) + clf.intercept_[0]
print np.sum((y_hat>0).astype(int) == y.astype(int))/len(y)
np.save('%s_%s_%s_coef.npy' % (args.out_svm_prefix,config_d['SVM']['kernel'],
penalty_name),
coef)
np.save('%s_%s_%s_intercept.npy' % (args.out_svm_prefix,config_d['SVM']['kernel'],
penalty_name),
clf.intercept_)
def main(args):
"""
get all the data matrices and process the data
"""
config_d = configParserWrapper.load_settings(open(args.config,'r'))
phones = np.loadtxt(args.phones,dtype=str)
max_n_classifiers = len(phones) * args.ncomponents
classifier_id = 0
print "ncomponents = %d" % args.ncomponents
for phone_id, phone in enumerate(phones):
if args.v:
print "Working on phone %s which has id %d" % (phone,phone_id)
print "classifier_id = %d" % classifier_id
X = np.load('%s/%s_%s' % ( args.data_prefix,
phone,
args.data_suffix))
if phone_id == 0:
avgs = np.zeros((max_n_classifiers,
) + X.shape[1:])
counts = np.zeros(max_n_classifiers
)
# will keep track of which average belongs to which
# phone and mixture component--this allows us to
# drop mixture components if they are potentially
# not helping
weights = np.zeros(max_n_classifiers,dtype=float)
meta = np.zeros((max_n_classifiers
,2),dtype=int)
if args.ncomponents == 1:
avgs[phone_id] = X.mean(0)
counts[phone_id] = X.shape[0]
weights[phone_id] = 1
meta[phone_id,0] = phone_id
meta[phone_id,1] = 0
classifier_id += 1
else:
bmm = bernoullimm.BernoulliMM(n_components=args.ncomponents,
n_init= config_d['EM']['n_init'],
n_iter= config_d['EM']['n_iter'],
tol=config_d['EM']['tol'],
random_state=config_d['EM']['random_seed'],
verbose=args.v)
bmm.fit(X)
responsibilities = bmm.predict_proba(X)
component_counts = responsibilities.sum(0)
no_use_components = component_counts < config_d['EM']['min_data_count']
while no_use_components.sum() > 0:
n_use_components = len(component_counts) -1
bad_component = np.argmin(component_counts)
use_components = np.ones(len(component_counts),
dtype=bool)
use_components[bad_component] = False
bmm.means_ = bmm.means_[use_components]
bmm.weights_ = bmm.weights_[use_components]
bmm.weights_ /= bmm.weights_.sum()
bmm.n_components = n_use_components
bmm.log_odds_, bmm.log_inv_mean_sums_ = bernoullimm._compute_log_odds_inv_means_sums(bmm.means_)
bmm.n_iter = 1
bmm.init_params=''
bmm.fit(X)
responsibilities = bmm.predict_proba(X)
component_counts = responsibilities.sum(0)
no_use_components = component_counts < config_d['EM']['min_data_count']
print component_counts
n_use_components = bmm.n_components
cur_means = bmm.means_.reshape(
*((n_use_components,)
+ avgs.shape[1:]))
cur_counts = component_counts
cur_weights = bmm.weights_
avgs[classifier_id:classifier_id+
n_use_components] = cur_means
counts[classifier_id:
classifier_id + n_use_components] = cur_counts
weights[classifier_id:
classifier_id + n_use_components] = cur_weights
meta[classifier_id:classifier_id+n_use_components,0] = phone_id
meta[classifier_id:classifier_id+n_use_components,1] = np.arange(n_use_components)
# make sure we move forward in the vector
classifier_id += n_use_components
print "Total of %d models" % classifier_id
np.save('%s/avgs_%s' % (args.out_prefix, args.out_suffix),
avgs[:classifier_id])
np.save('%s/counts_%s' % (args.out_prefix, args.out_suffix),
counts[:classifier_id])
np.save('%s/weights_%s' % (args.out_prefix, args.out_suffix),
weights[:classifier_id])
np.save('%s/meta_%s' % (args.out_prefix, args.out_suffix),
meta[:classifier_id])
def main(args):
"""
"""
config_d = configParserWrapper.load_settings(open(args.config,'r'))
y = []
for fpath_id,fpath in enumerate(args.input_data_matrices):
X0 = np.load(fpath)
X0_shape = X0.shape[1:]
y.extend(fpath_id * np.ones(len(X0)))
y = np.array(y)
# add in two coordinates for the length
if args.input_lengths is not None:
X = np.zeros((len(y),np.prod(X0_shape)+2),dtype=float)
else:
X = np.zeros((len(y),np.prod(X0_shape)),dtype=float)
cur_idx = 0
for fpath_id,fpath in enumerate(args.input_data_matrices):
X0 = np.load(fpath)
X[cur_idx:cur_idx+len(X0),:np.prod(X0_shape)] = X0.reshape(len(X0),
np.prod(X0_shape))
cur_idx += len(X0)
if args.input_lengths is not None:
train_ls = []
for fpath in args.input_lengths:
ls = np.loadtxt(fpath,dtype=int)
train_ls.extend(ls[:,2])
train_ls = np.log(np.tile(np.array(train_ls),
(2,1)).T)
train_ls[:,1] *= train_ls[:,1]
X[:,-2:] = train_ls
X_test = []
y_test = []
for fpath_id,fpath in enumerate(args.input_test_data_matrices):
X0 = np.load(fpath).astype(np.float)
X0_shape = X0.shape[1:]
X_test.extend(X0.reshape(len(X0),
np.prod(X0_shape)))
y_test.extend(fpath_id * np.ones(len(X0)))
X_test = np.array(X_test)
y_test = np.array(y_test)
if args.input_test_lengths is not None:
test_ls = []
for fpath in args.input_test_lengths:
ls = np.loadtxt(fpath,dtype=int)
test_ls.extend(ls[:,2])
test_ls = np.log(np.tile(np.array(test_ls),
(2,1)).T)
test_ls[:,1] *= test_ls[:,1]
X_test = np.hstack((X_test,test_ls))
if args.use_weights is not None:
weights = np.load(args.use_weights)
weights = weights.reshape(weights.size)
X *= weights
X_test *= weights
if args.do_kernel_dist_features:
X_test = np.exp(-cdist(X_test,X, 'euclidean'))
X = np.exp(-squareform(pdist(X, 'euclidean')))
penalty_names = config_d['SVM']['penalty_list'][::2]
penalty_values = tuple( float(k) for k in config_d['SVM']['penalty_list'][1::2])
dev_results = ()
exp_descriptions = ()
exp_description_id = 0
for penalty_name, penalty_value in itertools.izip(penalty_names,penalty_values):
if args.v:
print '%s %s' % ('linear', penalty_name)
if config_d['SVM']['kernel'] == 'linear':
clf = svm.LinearSVC(C=penalty_value,
loss='l1')
clf.fit(X,y)
elif config_d['SVM']['kernel'] == 'polynomial':
import pdb; pdb.set_trace()
np.save('%s_%s_%s_coef.npy' % (args.output_fls_prefix,config_d['SVM']['kernel'],
penalty_name),
clf.coef_[0])
np.save('%s_%s_%s_intercept.npy' % (args.output_fls_prefix,config_d['SVM']['kernel'],
penalty_name),
clf.intercept_)
y_test_hat = clf.predict(X_test)
exp_descriptions += ((config_d['SVM']['kernel'],penalty_name),)
dev_results += ( (exp_description_id,
np.abs(y_test_hat-y_test).sum()/len(y_test), # error rate
np.abs(y_test_hat[y_test==0]-y_test[y_test==0]).sum()/len(y_test[y_test==0]), # mistakes by class 0
np.abs(y_test_hat[y_test==1]-y_test[y_test==1]).sum()/len(y_test[y_test==1]) # mistakes by class 1
),)
if args.v:
print '\t'.join(tuple( str(k) for k in dev_results[-1]))
exp_description_id +=1
open('%s_exp_descriptions' % args.output_fls_prefix,
'w').write('\n'.join(tuple(
'%d %s' % (k,
' '.join(d))
for k,d in enumerate(exp_descriptions))))
np.save('%s_dev_results.npy' % args.output_fls_prefix,
np.array(dev_results))
def main(args):
"""
Generate the training vectors with a random start time
base everything on the config file
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
X = np.zeros((config_d['TRAINDATA']['num_training'],
config_d['TRAINDATA']['vector_length'],
config_d['TRAINDATA']['num_features']),
dtype=np.uint8)
template_lengths = config_d['TRAINDATA']['template_lengths']
single_template_length = type(template_lengths) == int
max_template_length = np.max(template_lengths)
if config_d['TRAINDATA']['random_seed'] is not None:
np.random.seed(config_d['TRAINDATA']['random_seed'])
# uniformly sample the start times
start_times = (np.random.rand(config_d['TRAINDATA']['num_training']) * config_d['TRAINDATA']['num_shifts']).astype(int)
true_templates = np.zeros(
(config_d['TRAINDATA']['num_classes'],
max_template_length,config_d['TRAINDATA']['num_features']))
# generate true templates
if config_d['TRAINDATA']['template_type'] == 'uniform':
true_templates += 1
true_templates *= config_d['TRAINDATA']['background_means']
feature_block_size = config_d['TRAINDATA']['num_features']/config_d['TRAINDATA']['num_classes']
for class_id in xrange(config_d['TRAINDATA']['num_classes']):
if single_template_length:
cur_template_length = template_lengths
else:
cur_template_length = template_lengths[class_id]
feature_block_start = class_id*feature_block_size
feature_block_end = min((class_id+1)*feature_block_size,
config_d['TRAINDATA']['num_features'])
num_block_features = feature_block_end - feature_block_start
true_templates[class_id,:cur_template_length,
feature_block_start:
feature_block_end
] = config_d['TRAINDATA']['template_means']
if cur_template_length < true_templates.shape[1]:
true_templates[class_id,cur_template_length:,
feature_block_start:
feature_block_end
] = config_d['TRAINDATA']['background_means']
elif config_d['TRAINDATA']['template_type'] == 'quadratic_curves':
for i in xrange(len(true_templates)):
if single_template_length:
cur_template_length = template_lengths
else:
cur_template_length = template_lengths[class_id]
for j in xrange(config_d['TRAINDATA']['num_curves']):
true_templates[i,:cur_template_length] = np.maximum(
true_templates[i],
quadratic_curves_template(
cur_template_length,
config_d['TRAINDATA']['num_features'],
config_d['TRAINDATA']['template_means'],
config_d['TRAINDATA']['background_means'],
config_d['TRAINDATA']['template_blur_sigma']))
if cur_template_length < true_templates.shape[1]:
true_templates[i,cur_template_length:
] = config_d['TRAINDATA']['background_means']
else:
print "no option corresponding to main.config"
import pdb; pdb.set_trace()
class_ids = np.random.randint(
config_d['TRAINDATA']['num_classes'],
size=config_d['TRAINDATA']['num_training']).astype(int)
for i,x in enumerate(X):
x[:] = (np.random.rand(config_d['TRAINDATA']['vector_length'],
config_d['TRAINDATA']['num_features']) < config_d['TRAINDATA']['background_means']).astype(np.uint8)
if single_template_length:
cur_template_length = template_lengths
else:
cur_template_length = template_lengths[class_id]
x[start_times[i]:start_times[i]+cur_template_length] = (np.random.rand(cur_template_length,
config_d['TRAINDATA']['num_features']) < true_templates[class_ids[i],:cur_template_length]).astype(np.uint8)
if args.visualize_data is not None:
for i,x in enumerate(X):
plt.close('all')
plt.imshow(
x.T,
origin='lower',
interpolation='nearest',
cmap='bone')
plt.savefig("%s_%d.png" % (args.visualize_data,
i),
bbox_inches='tight')
if args.visualize_templates is not None:
for c_id, c_template in enumerate(true_templates):
plt.close('all')
plt.imshow(c_template.T,
cmap='bone',
interpolation='nearest',
origin='lower',vmin=0,vmax=1)
plt.axis('off')
plt.savefig('%stemplates_%d.png' % (args.visualize_templates,
c_id),
bbox_inches='tight')
#import pdb; pdb.set_trace()
np.save('%s_X.npy' % args.o,X)
np.save('%s_start_times.npy' % args.o,start_times)
np.save('%s_class_ids.npy' % args.o,class_ids)
np.save('%s_true_templates.npy' % args.o, true_templates)
def main(args):
"""
Get the start and end frame associated to each example
and then extract the example and save into a matrix
we make the assumption that all the examples have the exact
same length as the first
"""
phones = np.loadtxt(args.phones,dtype='str')
config_d = configParserWrapper.load_settings(open(args.c,'r'))
# uttid_dict = dict(tuple( (uttid,k) for k,uttid in enumerate(open(args.uttids,'r').read().strip().split('\n'))))
examples = []
examples_sfa = []
get_dict = lambda x: dict(l.split() for l in open(x,'r').read().strip().split('\n'))
transcript_dict = get_dict(args.transcripts)
output_dict = get_dict(args.outputs)
output_waves_dict = get_dict(args.output_waves)
# we assume that Q should have at least length 2 for this
# code to work
htemp, dhtemp, ddhtemp, tttemp = fb.hermite_window(
config_d['BPF']['winsize']-1,
config_d['BPF']['order'],
config_d['BPF']['half_time_support'])
h = np.zeros((htemp.shape[0],
htemp.shape[1]+1))
h[:,:-1] = htemp
dh = np.zeros((dhtemp.shape[0],
dhtemp.shape[1]+1))
dh[:,:-1] = dhtemp
tt = (2*tttemp[-1] -tttemp[-2])*np.ones(tttemp.shape[0]+1)
tt[:-1] = tttemp
oversampling=config_d['BPF']['oversampling']
T_s=config_d['OBJECT']['t_s']
gsigma = config_d['BPF']['gsigma']
gfilter= fb.get_gauss_filter(config_d['BPF']['gt'],
config_d['BPF']['gf'],
gsigma)
run_transform = lambda x: binary_phase_features(
preemphasis(process_wav(x),preemph=config_d['BPF']['preemph']),
config_d['BPF']['sample_rate'],
config_d['BPF']['freq_cutoff'],
config_d['BPF']['winsize'],
config_d['BPF']['nfft'],
oversampling,
h,
dh,
tt,
gfilter,
gsigma,
config_d['BPF']['fthresh'],
config_d['BPF']['othresh'],
spread_length=config_d['BPF']['spread_length'],
return_midpoints=True)
for phone_id, phone in enumerate(phones):
trans_list = transcript_list(transcript_dict[phone])
Ss, TFs, example_length = extract_examples(trans_list, run_transform, T_s)
if phone_id > 0:
assert old_example_length == example_length
else:
old_example_length = example_length
print output_dict[phone], output_waves_dict[phone], transcript_dict[phone]
np.save(output_dict[phone],TFs)
np.save(output_waves_dict[phone],Ss)
print phone, example_length
def main(args):
"""
Get the start and end frame associated to each example
and then extract the example and save into a matrix
we make the assumption that all the examples have the exact
same length as the first
"""
phones = np.loadtxt(args.phones, dtype='str')
config_d = configParserWrapper.load_settings(open(args.c, 'r'))
# uttid_dict = dict(tuple( (uttid,k) for k,uttid in enumerate(open(args.uttids,'r').read().strip().split('\n'))))
examples = []
examples_sfa = []
get_dict = lambda x: dict(l.split()
for l in open(x, 'r').read().strip().split('\n'))
transcript_dict = get_dict(args.transcripts)
output_dict = get_dict(args.outputs)
output_waves_dict = get_dict(args.output_waves)
# we assume that Q should have at least length 2 for this
# code to work
htemp, dhtemp, ddhtemp, tttemp = fb.hermite_window(
config_d['BPF']['winsize'] - 1, config_d['BPF']['order'],
config_d['BPF']['half_time_support'])
h = np.zeros((htemp.shape[0], htemp.shape[1] + 1))
h[:, :-1] = htemp
dh = np.zeros((dhtemp.shape[0], dhtemp.shape[1] + 1))
dh[:, :-1] = dhtemp
tt = (2 * tttemp[-1] - tttemp[-2]) * np.ones(tttemp.shape[0] + 1)
tt[:-1] = tttemp
oversampling = config_d['BPF']['oversampling']
T_s = config_d['OBJECT']['t_s']
gsigma = config_d['BPF']['gsigma']
gfilter = fb.get_gauss_filter(config_d['BPF']['gt'], config_d['BPF']['gf'],
gsigma)
run_transform = lambda x: binary_phase_features(
preemphasis(process_wav(x), preemph=config_d['BPF']['preemph']),
config_d['BPF']['sample_rate'],
config_d['BPF']['freq_cutoff'],
config_d['BPF']['winsize'],
config_d['BPF']['nfft'],
oversampling,
h,
dh,
tt,
gfilter,
gsigma,
config_d['BPF']['fthresh'],
config_d['BPF']['othresh'],
spread_length=config_d['BPF']['spread_length'],
return_midpoints=True)
for phone_id, phone in enumerate(phones):
trans_list = transcript_list(transcript_dict[phone])
Ss, TFs, example_length = extract_examples(trans_list, run_transform,
T_s)
if phone_id > 0:
assert old_example_length == example_length
else:
old_example_length = example_length
print output_dict[phone], output_waves_dict[phone], transcript_dict[
phone]
np.save(output_dict[phone], TFs)
np.save(output_waves_dict[phone], Ss)
print phone, example_length
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
log_odds, constants, template_phn_ids, template_component_ids = get_bernoulli_templates(
args.templates)
labels = np.load(args.labels)
components = np.load(args.components)
predicted_labels = np.load(args.predicted_labels)
predicted_components = np.load(args.predicted_components)
true_predicted_label_ranks = np.argmax(
predicted_labels == np.lib.stride_tricks.as_strided(
labels,
shape=(len(labels), predicted_labels.shape[1]),
strides=(8, 0)), 1).astype(int)
true_in_top_predictions = np.max(
predicted_labels == np.lib.stride_tricks.as_strided(
labels,
shape=(len(labels), predicted_labels.shape[1]),
strides=(8, 0)), 1)
true_predicted_label_ranks[
-true_in_top_predictions] = 1 + true_predicted_label_ranks.max()
true_examples = []
false_examples = []
for phn_id, fl in enumerate(args.data):
X = np.load(fl)
X_shape = X.shape[1:]
X = X.reshape(X.shape[0], np.prod(X_shape))
cur_labels = labels[labels == phn_id]
cur_components = components[labels == phn_id]
if phn_id == args.label:
true_examples.extend(X[(cur_components == args.component_id) *
true_in_top_predictions[labels == phn_id]])
else:
# we find all of th e
false_predicted_label_rank = \
get_false_predicted_label_ranks(
args.label,args.component_id,
true_predicted_label_ranks[labels==phn_id],
predicted_labels[labels==phn_id],
predicted_components[labels==phn_id])
false_examples.extend(
X[false_predicted_label_rank > true_predicted_label_ranks[
labels == phn_id]])
if min(len(true_examples), len(false_examples)) < 20:
print "len(true_examples)=%d,len(false_examples)=%d" % (
len(true_examples), len(false_examples))
print "So no svm trained for label %d component %d" % (
args.label, args.component_id)
return
y = np.array(len(true_examples) * [0] + len(false_examples) * [1])
X = np.array(true_examples + false_examples)
del true_examples
del false_examples
config_d = configParserWrapper.load_settings(open(args.config, 'r'))
penalty_names = config_d['SVM']['penalty_list'][::2]
penalty_values = tuple(
float(k) for k in config_d['SVM']['penalty_list'][1::2])
for penalty_name, penalty_value in itertools.izip(penalty_names,
penalty_values):
if args.v:
print '%s %s' % ('linear', penalty_name)
if config_d['SVM']['kernel'] == 'linear':
clf = svm.LinearSVC(C=penalty_value, loss='l1')
clf.fit(X, y)
else:
import pdb
pdb.set_trace()
coef = clf.coef_.reshape(clf.coef_.size)
y_hat = np.dot(X, coef) + clf.intercept_[0]
print np.sum((y_hat > 0).astype(int) == y.astype(int)) / len(y)
np.save(
'%s_%s_%s_coef.npy' %
(args.out_svm_prefix, config_d['SVM']['kernel'], penalty_name),
coef)
np.save(
'%s_%s_%s_intercept.npy' %
(args.out_svm_prefix, config_d['SVM']['kernel'], penalty_name),
clf.intercept_)
def main(args):
"""
EM only with shift
templates are generated based on the configuration script
"""
config_d = configParserWrapper.load_settings(open(args.c,'r'))
if config_d['EMTRAINING']['random_seed'] is not None:
np.random.seed(config_d['EMTRAINING']['random_seed'])
num_shifts=config_d['EMTRAINING']['num_shifts']
template_length=config_d['EMTRAINING']['template_length']
start_time=config_d['EMTRAINING']['start_time']
num_classes=config_d['EMTRAINING']['num_classes']
min_prob=config_d['EMTRAINING']['min_prob']
class_shift_min_prob = config_d['EMTRAINING']['class_shift_min_prob']
X = np.load(args.i).astype(np.uint8)
num_data = X.shape[0]
X_shape = X.shape[2:]
if len(X_shape) > 1:
X = X.reshape(X.shape[0],X.shape[1],
np.prod(X_shape))
# just a random initialization
init_class_ids = np.random.randint(num_classes,size=num_data)
posteriors = np.zeros((X.shape[0],num_classes,
num_shifts))
probs = posteriors.copy()
if config_d['EMTRAINING'].has_key('initialization') and config_d['EMTRAINING']['initialization']=='random_class_uniform_shift':
for i, class_id in enumerate(init_class_ids):
posteriors[i,class_id,:] = 1./(num_shifts)
else:
print 'shift spike at %d' % ((num_shifts-1)/2)
for i, class_id in enumerate(init_class_ids):
posteriors[i,class_id,(num_shifts-1)/2] = 1.
# initialize the class and shift probs
class_shift_probs = 1./(num_classes * num_shifts) * np.ones((num_classes,num_shifts))
class_shift_mask = np.ones((num_classes,
num_shifts),dtype=bool)
template = np.zeros((num_classes,
template_length,
X.shape[2]))
if config_d['EMTRAINING']['initialization']=='template':
template[:] = np.load(args.init_templates)
else:
bernoullishiftonly_em_fast.compute_template(X,posteriors,template,start_time)
template = np.clip(template,min_prob,1-min_prob)
background = np.zeros(X.shape[2])
bernoullishiftonly_em_fast.compute_background(X,
posteriors,
background,
start_time,
template_length)
background = np.clip(background,min_prob,1-min_prob)
log_template_inv_prob = np.log(1-template)
log_template_odds = np.log(template) - log_template_inv_prob
bgd_sums = prep_bgd_sums(X,background)
likelihood = np.sum(compute_likelihood_posteriors(X,class_shift_probs,
posteriors,
probs,
log_template_odds,
log_template_inv_prob,
bgd_sums,
start_time, class_shift_min_prob,
class_shift_type=config_d['EMTRAINING']['class_shift_type'],
min_shift_sigma=config_d['EMTRAINING']['min_shift_sigma']))
not_converged = True
iteration = 0
while not_converged:
prev_template0 = template[0]
print "iter= %d\tlikelihood = %g" % (iteration,likelihood)
iteration += 1
template[:] = 0
bernoullishiftonly_em_fast.compute_template(X,posteriors,template,start_time)
# import pdb; pdb.set_trace()
background[:] = 0
bernoullishiftonly_em_fast.compute_background(X,
posteriors,
background,
start_time,
template_length)
template = np.clip(template,min_prob,1-min_prob)
background = np.clip(background,min_prob,1-min_prob)
# plt.close('all');
# for i in xrange(num_classes):
# plt.subplot(num_classes,1,i+1)
# plt.imshow(template[i].T,origin='lower',cmap='bone',
# interpolation='nearest')
# plt.show()
log_template_inv_prob = np.log(1-template)
log_template_odds = np.log(template) - log_template_inv_prob
bgd_sums = prep_bgd_sums(X,background)
new_likelihood = np.sum(compute_likelihood_posteriors(X,class_shift_probs,
posteriors,
probs,
log_template_odds,
log_template_inv_prob,
bgd_sums,start_time, class_shift_min_prob,
class_shift_type=config_d['EMTRAINING']['class_shift_type'],
min_shift_sigma=config_d['EMTRAINING']['min_shift_sigma']))
print (likelihood-new_likelihood)/likelihood, config_d['EMTRAINING']['tolerance']
not_converged = ((likelihood-new_likelihood)/likelihood > config_d['EMTRAINING']['tolerance'] or iteration < 10) and iteration < 1000
likelihood = new_likelihood
if args.out_templates is None:
np.save('%stemplates.npy' % args.o, template)
else:
np.save(args.out_templates, template)
if args.out_backgrounds is None:
np.save('%sbackground.npy' % args.o, background)
else:
np.save(args.out_backgrounds, background)
if args.out_posteriors is None:
np.save('%sposteriors.npy' % args.o,posteriors)
else:
np.save(args.out_posteriors, posteriors)
if args.out_class_shift_probs is None:
np.save('%sclass_shift_probs.npy' % args.o,class_shift_probs)
else:
np.save(args.out_class_shift_probs,
class_shift_probs)
if args.visualize_templates:
for c_id, c_template in enumerate(template):
cur_template = c_template.reshape(
*(
(c_template.shape[0],)
+ X_shape ))
for i in xrange(cur_template.shape[-1]):
plt.close('all')
plt.imshow(cur_template[:,:,i].T,
cmap='hot',
interpolation='nearest',
origin='lower',vmin=0,vmax=1)
plt.axis('off')
plt.savefig('%stemplates_%d_%d.png' % (args.o,
c_id,i),
bbox_inches='tight')
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
log_odds, constants, template_phn_ids, template_component_ids = get_bernoulli_templates(args.templates)
labels = np.load(args.labels)
components = np.load(args.components)
predicted_labels = np.load(args.predicted_labels)
predicted_components = np.load(args.predicted_components)
true_predicted_label_ranks = np.argmax(
predicted_labels
== np.lib.stride_tricks.as_strided(
labels,
shape=(len(labels),
predicted_labels.shape[1]),
strides=(8,0)),1).astype(int)
true_in_top_predictions = np.max(predicted_labels
== np.lib.stride_tricks.as_strided(
labels,
shape=(len(labels),
predicted_labels.shape[1]),
strides=(8,0)),1)
true_predicted_label_ranks[-true_in_top_predictions] = 1+true_predicted_label_ranks.max()
true_examples = []
false_examples = []
for phn_id, fl in enumerate(args.data):
X = np.load(fl)
X_shape = X.shape[1:]
X = X.reshape(X.shape[0],np.prod(X_shape))
cur_labels = labels[labels==phn_id]
cur_components = components[labels==phn_id]
if phn_id == args.label:
true_examples.extend(X[(cur_components == args.component_id) * true_in_top_predictions[labels==phn_id]])
else:
# we find all of th e
false_predicted_label_rank = \
get_false_predicted_label_ranks(
args.label,args.component_id,
true_predicted_label_ranks[labels==phn_id],
predicted_labels[labels==phn_id],
predicted_components[labels==phn_id])
false_examples.extend(X[false_predicted_label_rank > true_predicted_label_ranks[labels==phn_id]])
if min(len(true_examples),len(false_examples)) < 20:
print "len(true_examples)=%d,len(false_examples)=%d" % (len(true_examples),len(false_examples))
print "So no svm trained for label %d component %d" % (args.label,args.component_id)
return
y = np.array(len(true_examples) * [0] + len(false_examples)*[1])
X = np.array(true_examples + false_examples)
del true_examples
del false_examples
config_d = configParserWrapper.load_settings(open(args.config,'r'))
penalty_names = config_d['SVM']['penalty_list'][::2]
penalty_values = tuple( float(k) for k in config_d['SVM']['penalty_list'][1::2])
for penalty_name, penalty_value in itertools.izip(penalty_names,penalty_values):
if args.v:
print '%s %s' % ('linear', penalty_name)
if config_d['SVM']['kernel'] == 'linear':
clf = svm.LinearSVC(C=penalty_value,
loss='l1')
clf.fit(X,y)
else:
import pdb; pdb.set_trace()
coef = clf.coef_.reshape(clf.coef_.size)
y_hat = np.dot(X,coef) + clf.intercept_[0]
print np.sum((y_hat>0).astype(int) == y.astype(int))/len(y)
np.save('%s_%s_%s_coef.npy' % (args.out_svm_prefix,config_d['SVM']['kernel'],
penalty_name),
coef)
np.save('%s_%s_%s_intercept.npy' % (args.out_svm_prefix,config_d['SVM']['kernel'],
penalty_name),
clf.intercept_)
def main(args):
"""
For each label and component constructed a positive and negative
training set and train a linear SVM to separate them
"""
config_d = configParserWrapper.load_settings(open(args.config,'r'))
true_examples = []
false_examples = []
mean = 0
total = 0
num_less_than_eq = np.zeros(20)
fls = np.loadtxt(args.fls_txt, dtype=str)
all_X_patches = []
all_S_patches = []
htemp, dhtemp, ddhtemp, tttemp = fb.hermite_window(
args.winsize,
args.num_tapers,
args.win_half_time_support)
run_transform = lambda x, winlength : esp.get_spectrogram_features(x,
16000,
winlength,
80,
2**(int(np.ceil(np.log2(winlength)))),
4000,
7,
)
X_patches = []
S_patches = []
for fl_id, fl_path in enumerate(fls):
if len(X_patches) > 100000: break
S = run_transform(wavfile.read(fl_path)[1], args.winsize)
# spectrogram(,
# 16000,
# 3200,
# args.winsize,
# 2**int(np.ceil(np.log2(args.winsize))),
# 2,
# htemp)
if args.do_exp_weighted_divergence:
Sold = S.copy()
S *=np.exp(S)
X = get_edge_features_use_config(S.T,config_d['EDGES'])
cur_X_patches, cur_S_patches = get_maximal_patches(X,S,patch_radius=2)
X_patches.extend(cur_X_patches)
S_patches.extend(cur_S_patches)
num_new_patches = len(X_patches)
X = np.array(X_patches)
S = np.array(S_patches)
data_shape = X.shape[1:]
X = X.reshape(X.shape[0],np.prod(data_shape))
bmm = bernoullimm.BernoulliMM(n_components=args.n_components,
n_init= 50,
n_iter= 500,
random_state=0,
verbose=args.v, tol=1e-6)
bmm.fit(X)
# check above 30
use_means = bmm.predict_proba(X).sum(0) > 30
print use_means.sum()
try:
np.save(args.save_parts,bmm.means_.reshape(*( (bmm.n_components,)+data_shape))[use_means])
except:
import pdb; pdb.set_trace()
S_shape = S.shape[1:]
import pdb; pdb.set_trace()
S_clusters = bmm.cluster_underlying_data(S.reshape(len(S),np.prod(S_shape)),X).reshape(
*( (bmm.n_components,) + S_shape))[use_means]
np.save(args.spec_save_parts,S_clusters)
ncols = int(np.sqrt(args.n_components))
nrows = int(np.ceil(args.n_components/ncols))
if args.viz_spec_parts is not None:
plt.close('all')
fig = plt.figure(1, (6, 6))
grid = ImageGrid(fig, 111, # similar to subplot(111)
nrows_ncols = (nrows,ncols ), # creates 2x2 grid of axes
axes_pad=0.001, # pad between axes in inch.
)
for i in xrange(S_clusters.shape[0]):
try:
grid[i].imshow(S_clusters[i],cmap=cm.binary,interpolation='nearest')
grid[i].spines['bottom'].set_color('red')
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
except:
import pdb; pdb.set_trace()
for i in xrange(S_clusters.shape[0],nrows*ncols):
try:
grid[i].spines['bottom'].set_color('red')
except: import pdb; pdb.set_trace()
grid[i].spines['top'].set_color('red')
grid[i].spines['left'].set_color('red')
grid[i].spines['right'].set_color('red')
for a in grid[i].axis.values():
a.toggle(all=False)
plt.savefig('%s' % args.viz_spec_parts
,bbox_inches='tight')