def score_with_lm1():
for video_num, indices in test_set.sentences_index.items():
# visual_model_guesses = df_probs.iloc[indices,:].idxmax(axis=1)
ngram_indices = []
for sentence_idx, word_idx in enumerate(indices):
if ngram_indices:
ngram_prefix = df_probs.iloc[ngram_indices, :].idxmax(
axis=1).tolist()
row = df_probs.iloc[word_idx, :]
for col_idx, col in enumerate(row.iteritems()):
word = re.sub("\d+$", "", col[0])
ngram = ngram_prefix + [word]
visual_model_log_l = col[1]
ngram_log_l = lm.log_p(ngram)
updated_likelihood = lm_factor * (1 / len(
ngram_prefix)) * ngram_log_l + visual_model_log_l
df_probs.iloc[word_idx, col_idx] = updated_likelihood
print(
"word {} ngram {} ngram_L={} visual_model_L={} updated L = {}"
.format(word, ngram, ngram_log_l, visual_model_log_l,
updated_likelihood))
ngram_indices.append(word_idx)
if len(ngram_indices) > 2:
ngram_indices.pop(0)
guesses = df_probs.idxmax(axis=1).tolist()
print("guesses len {}, test set wordlist len {}".format(
len(guesses), len(test_set.wordlist)))
# print("{}".format(guesses))
show_errors(guesses, test_set)
def ensemble_guess(features_probs=None, features_weights=None, test_set=None):
if features_probs is None or features_weights is None:
l = pickle.load(open("data/feature_models_data.pkl", "rb"))
features_probs = l["features_probs"]
features_weights = l["features_weights"]
print("feature models data loaded")
if test_set is None:
test_set = asl.build_test(features_ground)
features_weights = dict(features_weights)
ensemble_probabilities = [
merge_seq_dicts(ground, norm, polar, delta, custom, features_weights)
for (ground, norm, polar, delta,
custom) in zip(features_probs["ground"], features_probs["norm"],
features_probs["polar"], features_probs["delta"],
features_probs["custom"])
]
pguesses = guess_by_combination(test_set, ensemble_probabilities)
guesses = [w for (w, p) in pguesses]
# guesses = []
# for word_probs in ensemble_probabilities:
# bestguess = max(word_probs, key=lambda i: word_probs[i])
# guesses.append(bestguess)
# print('Ensembled Guess')
show_errors(guesses, test_set)
def run_test(name, features, model_selector):
print("running ", name)
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
show_errors(guesses, test_set)
def customRec(asl):
# autoreload for automatically reloading changes made in my_model_selectors and my_recognizer
from my_model_selectors import SelectorConstant
from my_model_selectors import SelectorBIC
from my_model_selectors import SelectorDIC
from my_model_selectors import SelectorCV
features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly']
features_polar = ['polar-rr', 'polar-rtheta', 'polar-lr', 'polar-ltheta']
features_norm = ['norm-rx', 'norm-ry', 'norm-lx', 'norm-ly']
features_delta = ['delta-rx', 'delta-ry', 'delta-lx', 'delta-ly']
features_custom = [
'grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly', 'delta-rx', 'delta-ry',
'delta-lx', 'delta-ly'
]
def train_all_words(features, model_selector):
training = asl.build_training(
features
) # Experiment here with different feature sets defined in part 1
sequences = training.get_all_sequences()
Xlengths = training.get_all_Xlengths()
model_dict = {}
for word in training.words:
model = model_selector(sequences, Xlengths, word,
n_constant=3).select()
model_dict[word] = model
return model_dict
models = train_all_words(features_ground, SelectorConstant)
print("Number of word models returned = {}".format(len(models)))
from my_recognizer import recognize
from asl_utils import show_errors
import timeit
features_list = [
features_ground, features_norm, features_polar, features_delta,
features_custom
]
model_selector_list = [
SelectorConstant, SelectorBIC, SelectorDIC, SelectorCV
]
#model_selector_list = [SelectorBIC]
for features in features_list:
for model_selector in model_selector_list:
print("\n", features, model_selector.__name__)
start = timeit.default_timer()
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
end = timeit.default_timer() - start
print("Training and test took {} seconds".format(end))
show_errors(guesses, test_set)
return
def guess_with_cv_polar_and_slm():
training = asl.build_training(features_polar)
models_cv_polar = train_all_words2(training, SelectorCV)
test_set = asl.build_test(features_polar)
probabilities, guesses = recognize(models_cv_polar, test_set)
print("With cv_polar models")
show_errors(guesses, test_set)
pguesses = guess_by_combination(test_set, probabilities)
guesses = [w for (w, p) in pguesses]
print("With cv_polar models and language model")
show_errors(guesses, test_set)
def top_iterative():
guesses = []
# guess sentence-wise
for video_num in test_set.sentences_index:
first = test_set.sentences_index[video_num][0]
top_first_words = sorted(
[(gm_log_p + LM_SCALE * lm_log_p([clean_word(word)]), word)
for word, gm_log_p in gm[first].items()],
reverse=True)[:5]
top_sentences = [
build_sentence(log_p, w, test_set.sentences_index[video_num][1:])
for log_p, w in top_first_words
]
_, best = max(top_sentences)
guesses.extend(best)
show_errors(guesses, test_set)
def iterative():
guesses = []
for video_num in test_set.sentences_index:
guess = []
lm_guess = []
# iteratively extend the sentence getting the max gesture model prob and language model prob
for i in test_set.sentences_index[video_num]:
log_p, word = max([
(gm_log_p + LM_SCALE * lm_log_p(lm_guess + [clean_word(word)]),
word) for word, gm_log_p in gm[i].items()
])
guess.append(word)
lm_guess.append(clean_word(word))
guesses.extend(guess)
show_errors(guesses, test_set)
def grid_search_models():
model_selectors = {
"CV": SelectorCV,
"BIC": SelectorBIC,
"DIC": SelectorDIC
}
features_dict = {
"GROUND": features_ground,
"NORM": features_norm,
"POLAR": features_polar,
"DELTA": features_delta,
"CUSTOM": features_norm
}
wers = {}
grid = {}
for feat_name, features in features_dict.items():
test_set = asl.build_test(features)
words_data = asl.build_training(features)
for sel_name, selector in model_selectors.items():
models = train_all_words2(words_data, selector)
probabilities, guesses = recognize(models, test_set)
key = sel_name + "-" + feat_name
print(key)
wer = show_errors(guesses, test_set)
wers[key] = wer
grid[key] = (wer, models)
s = [(k, wers[k]) for k in sorted(wers, key=wers.get, reverse=False)]
print(s)
pickle.dump(grid, open("data/grid_models.pkl", "wb"))
def combinations():
guesses = []
for video_num in test_set.sentences_index:
# get the top 3 candidate words from our gesture model (for each word in the sentence)
top_words = [
sorted(gm[i].items(), key=lambda item: item[1], reverse=True)[:3]
for i in test_set.sentences_index[video_num]
]
sentences = []
# for each permutation of words, calculate the log_p
for permutations in itertools.product(*top_words):
sentence = [word for word, _ in permutations]
clean_sentence = [clean_word(word) for word, _ in permutations]
log_p = sum([gm_log_p for _, gm_log_p in permutations
]) + LM_SCALE * lm_log_p(clean_sentence)
sentences.append((log_p, sentence))
_, best = max(sentences)
print(best)
guesses.extend(best)
show_errors(guesses, test_set)
def ensemble_models():
features_data = [
(name, f, s)
for (name, f,
s) in [("ground", features_ground,
SelectorBIC), ("norm", features_norm, SelectorDIC),
("polar", features_polar,
SelectorBIC), (
"delta", features_delta,
SelectorCV), ("custom", features_custom, SelectorDIC)]
]
features_probs = {}
features_success_rates = {}
test_set = None
for name, features, selector in features_data:
test_set = asl.build_test(features)
words_data = asl.build_training(features)
models = train_all_words2(words_data, selector)
#probabilities is a list of dictionaries, each dictionary gives each possible word prob for the sequence [{word:prob}]
probabilities, guesses = recognize(models, test_set)
#todo use the feature relative wer as its proportion in the ensemble output
features_probs[name] = probabilities
print("{} features:".format(name))
wer = show_errors(guesses, test_set)
features_success_rates[name] = 1 - wer
sm = sum(features_success_rates.values())
features_weights = [(k, v / sm)
for (k, v) in features_success_rates.items()]
pickle.dump(
{
"features_probs": features_probs,
"features_weights": features_weights
}, open("data/feature_models_data.pkl", "wb"))
ensemble_guess(features_probs, features_weights, test_set)
def train(features):
model_selector = SelectorBIC # change as needed
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
print("Features:", features)
print("Model Selector:", model_selector)
show_errors(guesses, test_set)
model_selector = SelectorDIC # change as needed
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
print("Features:", features)
print("Model Selector:", model_selector)
show_errors(guesses, test_set)
model_selector = SelectorCV # change as needed
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
print("Features:", features)
print("Model Selector:", model_selector)
show_errors(guesses, test_set)
print("Number of test set sentences: {}".format(
len(test_set.sentences_index)))
# TODO implement the recognize method in my_recognizer
from my_recognizer import recognize
from asl_utils import show_errors
# TODO Choose a feature set and model selector
features = features_ground # change as needed
model_selector = SelectorDIC # change as needed
# TODO Recognize the test set and display the result with the show_errors method
models = train_all_words(features, model_selector)
test_set = asl.build_test(features)
probabilities, guesses = recognize(models, test_set)
show_errors(guesses, test_set)
# In[ ]:
# TODO Choose a feature set and model selector
# TODO Recognize the test set and display the result with the show_errors method
# In[ ]:
# TODO Choose a feature set and model selector
# TODO Recognize the test set and display the result with the show_errors method
# **Question 3:** Summarize the error results from three combinations of features and model selectors. What was the "best" combination and why? What additional information might we use to improve our WER? For more insight on improving WER, take a look at the introduction to Part 4.
#
# **Answer 3:**
model = model_selector(sequences, Xlengths, word).select()
model_dict[word] = model
return model_dict
start = timeit.default_timer()
features = features_norm_delta # change as needed
model_selector = SelectorBIC # change as needed
models = train_all_words(features, model_selector)
print('words trained, created {} models'.format(len(models)))
test_set = asl.build_test(features)
print('test set built')
probabilities, guesses = recognize(models, test_set)
one_gram_guesses = OneGram().guess_words(probabilities)
two_gram_guesses = TwoGram().guess_words(test_set, probabilities)
three_gram_guesses = ThreeGram().guess_words(test_set, probabilities)
print('recognizer results:')
show_errors(guesses, test_set) # WER 51.7 / 52.8
print('unigram results:')
show_errors(one_gram_guesses, test_set) # WER 56.2 / 58.4
print('bigram results:')
show_errors(two_gram_guesses, test_set) # WER 42.1 / 46.6
print('trigram results:')
show_errors(three_gram_guesses, test_set) # WER 39.3 / 41.6
end = timeit.default_timer() - start
print('time: {}'.format(end))
if __name__ == "__main__":
asl = AslDb()
add_features1(asl)
for feats in feature_sets:
print("===============================================")
print(" FEATURES")
print(" {}".format(feats))
print("===============================================")
for sel in selectors:
print(" Selector: {}".format(sel))
start_t = time.time()
print("Start time: {}".format(start_t))
print("--------------------------")
try:
models = train_all_words(feats, sel)
test_set = asl.build_test(feats)
probabilities, guesses = recognize(models, test_set)
print(show_errors(guesses, test_set))
end_t = time.time()
total_t = end_t - start_t
print("End time: {}".format(end_t))
print("Total time in seconds: {}".format(total_t))
except Exception as exc:
print(traceback.format_exc())
print(exc)
print("There was some kind of problem with the specified selector")
print("--------------------------")
def run_my_recognizer(asl, features):
from my_model_selectors import SelectorConstant
from my_model_selectors import SelectorCV
from my_model_selectors import SelectorBIC
from my_model_selectors import SelectorDIC
from my_recognizer import recognize
from asl_utils import show_errors
def train_all_words(features, model_selector, index):
training = asl.build_training(features)
print("Available Training words - words: ", training.words)
print("Quantity of Training words - num_items: ", training.num_items)
print("Chosen Training words: All words")
# Get every fifth element since may be combination of
# more than one Feature Set (where each Feature Set contains four elements)
features_abbrev = features[::4]
chosen_abbrev_list = []
for i, abbrev in enumerate(features_abbrev):
chosen_abbrev_list.append(abbrev.split('-')[0])
chosen = ', '.join(map(str, chosen_abbrev_list))
print("Chosen Features: ", chosen)
# print("Chosen Features: ", features[index].split('-')[0])
print("Chosen Model Selector: ", model_selector.__name__)
sequences = training.get_all_sequences()
# Note:
#
# training.get_all_Xlengths()
#
# This gets the entire db of words in the form of a
# dictionary of (X, lengths) tuples, and returns two lists
# for each word:
# - where X is a numpy array of feature lists
# (concatenation of all the sequences of frames with the
# values of the features i.e. grnd-rx, grnd-lx, ..., over time).
# The first sequence of frames is 14 frames long, and
# second is 18 frames long in the example, but only two from each
# are shown. A sequence of frames corresponds to a succession of
# movements corresponding to a word
# - where lengths is a list of lengths of sequences within X
# (length of the sequences concatenated in the first list)
#
# i.e. {'FRANK': (
# array(
# [[ 87, 225],[ 87, 225], ...
# [ 87, 225, 62, 127], [ 87, 225, 65, 128]]
# ),
# [14, 18]
# ),
# }
#
# The first list
Xlengths = training.get_all_Xlengths()
words_to_train = training.words
model_dict = {}
timeframes = []
states = []
for word in words_to_train:
start = timeit.default_timer()
model = model_selector(sequences, Xlengths, word,
n_constant=3).select()
model_dict[word] = model
end = timeit.default_timer() - start
if model is not None:
timeframes.append(end)
states.append(model.n_components)
print(
"Training complete for {} with {} states with time {} seconds"
.format(word, model.n_components, end))
else:
print("Training failed for {}".format(word))
if timeframes:
print("Average Timeframe: ", np.mean(timeframes))
if states:
print("Average States used: ", np.mean(states))
return model_dict
# features_ground # Difference between hand and nose locations
# features_norm # Fix for different height and arm lengths
# features_polar # Fix discontinuity in signing area to prevent interfere with results
# features_delta # Speed and direction of hands
# features_rescaled # Faster and interpretation and analysis of plotted data easier
# All features
features_all = features[0] + features[1] + features[2] + features[
3] + features[4]
# All features without norm and polar (i.e. without fixes)
features_without_norm_and_polar = features[0] + features[3] + features[4]
# All features without rescaled
features_without_rescaled = features[0] + features[1] + features[
2] + features[3]
# All features without delta
features_without_delta = features[0] + features[1] + features[
2] + features[4]
# CHOSEN COMBOS - Choose 3 OFF feature set and model selector combos
# (16+ combinations all together)
# Note: Ensure same length of of chosen_features and chosen_model_selectors
chosen_features = [
features_all, features_all, features_all, features_all,
features_without_norm_and_polar, features_without_norm_and_polar,
features_without_norm_and_polar, features_without_norm_and_polar,
features_without_rescaled, features_without_rescaled,
features_without_rescaled, features_without_rescaled,
features_without_delta, features_without_delta, features_without_delta,
features_without_delta, features[0], features[0], features[0],
features[0], features[1], features[1], features[1], features[1],
features[2], features[2], features[2], features[2], features[3],
features[3], features[3], features[3], features[4], features[4],
features[4], features[4]
]
chosen_model_selectors = [
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC,
SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC
]
# Iterate through the 3 OFF interesting combos chosen.
# Recognize the test set and display the result with the
# show_errors method
for index, chosen_feature in enumerate(chosen_features):
models = train_all_words(chosen_feature, chosen_model_selectors[index],
index)
print("Number of word models returned = {}".format(len(models)))
test_set = asl.build_test(chosen_feature)
print("Number of test set items: {}".format(test_set.num_items))
print("Number of test set sentences: {}".format(
len(test_set.sentences_index)))
probabilities, guesses = recognize(models, test_set)
# Show the Word Error Rate (WER) and sentence differences in tabular form
try:
result = show_errors(guesses, test_set)
except Exception as e:
print("Error showing errors: ", e)
print("Finished processing combo... Trying others if any exist.")
def show_errors_for(self, guesses, selector_names, test_set):
for (guess, selector_name) in zip(guesses, selector_names):
print("Selector: {}".format(selector_name))
show_errors(guess, test_set)
