def setUp(self): print('Start test setup') asl = AslDb() self.training = asl.build_training(FEATURES) self.sequences = self.training.get_all_sequences() self.xlengths = self.training.get_all_Xlengths() print('End test setup')
class TestRecognize(TestCase): def setUp(self): self.asl = AslDb() self.training_set = self.asl.build_training(FEATURES) self.test_set = self.asl.build_test(FEATURES) self.models = train_all_words(self.training_set, SelectorConstant) def test_recognize_probabilities_interface(self): probs, _ = recognize(self.models, self.test_set) self.assertEqual( len(probs), self.test_set.num_items, "Number of test items in probabilities list incorrect.") self.assertIn( 'FRANK', probs[0], "Dictionary of probabilities does not contain correct keys") self.assertIn( 'CHICKEN', probs[-1], "Dictionary of probabilities does not contain correct keys") def test_recognize_guesses_interface(self): _, guesses = recognize(self.models, self.test_set) self.assertEqual(len(guesses), self.test_set.num_items, "Number of test items in guesses list incorrect.") self.assertIsInstance(guesses[0], str, "The guesses are not strings") self.assertIsInstance(guesses[-1], str, "The guesses are not strings")
def setUp(self): self.asl = AslDb() # initializes the database self.asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame self.asl.df['grnd-ry'] = self.asl.df['right-y'] - self.asl.df['nose-y'] self.asl.df['grnd-rx'] = self.asl.df['right-x'] - self.asl.df['nose-x'] self.asl.df['grnd-ly'] = self.asl.df['left-y'] - self.asl.df['nose-y'] self.asl.df['grnd-lx'] = self.asl.df['left-x'] - self.asl.df['nose-x'] self.features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] self.words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] self.training = self.asl.build_training( self.features_ground ) # Experiment here with different feature sets defined in part 1
class MyDICTest(unittest.TestCase): def setUp(self): self.asl = AslDb() # initializes the database self.asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame self.asl.df['grnd-ry'] = self.asl.df['right-y'] - self.asl.df['nose-y'] self.asl.df['grnd-rx'] = self.asl.df['right-x'] - self.asl.df['nose-x'] self.asl.df['grnd-ly'] = self.asl.df['left-y'] - self.asl.df['nose-y'] self.asl.df['grnd-lx'] = self.asl.df['left-x'] - self.asl.df['nose-x'] self.features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] self.words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] self.training = self.asl.build_training( self.features_ground ) # Experiment here with different feature sets defined in part 1 def test_something(self): sequences = self.training.get_all_sequences() Xlengths = self.training.get_all_Xlengths() for word in self.words_to_train: start = timeit.default_timer() model = my_model_selectors.SelectorDIC(sequences, Xlengths, word, min_n_components=2, max_n_components=15, random_state=14).select() end = timeit.default_timer() - start if model is not None: print( "Training complete for {} with {} states with time {} seconds" .format(word, model.n_components, end)) else: print("Training failed for {}".format(word))
class TestRecognize(TestCase): def setUp(self): self.asl = AslDb() self.training_set = self.asl.build_training(FEATURES) self.test_set = self.asl.build_test(FEATURES) self.models = train_all_words(self.training_set, SelectorConstant) def test_recognize_probabilities_interface(self): probs, _ = recognize(self.models, self.test_set) self.assertEqual(len(probs), self.test_set.num_items, "Number of test items in probabilities list incorrect.") self.assertIn('FRANK', probs[0], "Dictionary of probabilities does not contain correct keys") self.assertIn('CHICKEN', probs[-1], "Dictionary of probabilities does not contain correct keys") def test_recognize_guesses_interface(self): _, guesses = recognize(self.models, self.test_set) self.assertEqual(len(guesses), self.test_set.num_items, "Number of test items in guesses list incorrect.") self.assertIsInstance(guesses[0], str, "The guesses are not strings") self.assertIsInstance(guesses[-1], str, "The guesses are not strings")
def initialise_asl_db(features): asl = AslDb() add_features_ground(asl) add_features_norm(asl, features[1]) add_features_polar(asl, features[0], features[2]) add_features_delta(asl, features[3]) add_features_rescaling(asl, features[0], features[4]) # Display first five rows (pandas data frame) of ASL database, indexed by video and frame # asl.df.head() return asl
def setUp(self): asl = AslDb() self.training = asl.build_training(FEATURES) self.sequences = self.training.get_all_sequences() self.xlengths = self.training.get_all_Xlengths()
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] training = asl.build_training(features_ground) df_means = asl.df.groupby('speaker').mean() asl.df['left-x-mean'] = asl.df['speaker'].map(df_means['left-x']) df_std = asl.df.groupby('speaker').std() def z_score(col): Xmean = asl.df['speaker'].map(df_means[col]) Xstd = asl.df['speaker'].map(df_std[col]) return (asl.df[col] - Xmean) / Xstd # x: array([-0.891, 0.742, 1.153, 1.663]) # y: array([ 1.153, 1.663, -0.891, 0.742]) cols = ['right-x', 'right-y', 'left-x', 'left-y'] features_norm = ['norm-rx', 'norm-ry', 'norm-lx', 'norm-ly'] for f_norm, col in zip(features_norm, cols): asl.df[f_norm] = z_score(col)
import numpy as np import pandas as pd from asl_data import AslDb from my_model_selectors import SelectorCV from my_model_selectors import SelectorBIC from my_model_selectors import SelectorDIC asl = AslDb() # initializes the database asl.df.ix[98, 1] # look at the data available for an individual frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] from asl_utils import test_features_tryit # TODO add df columns for 'grnd-rx', 'grnd-ly', 'grnd-lx' representing differences between hand and nose locations # grnd-rx asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] # grnd-ly asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] # grnd-lx asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] # test the code # test_features_tryit(asl) # collect the features into a list features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly']
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() # initializes the database asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] features_ground = ['grnd-rx','grnd-ry','grnd-lx','grnd-ly'] df_means = asl.df.groupby('speaker').mean() df_std = asl.df.groupby('speaker').std() features_norm = ['norm-rx', 'norm-ry', 'norm-lx','norm-ly'] eps = 1e-20 asl.df['norm-lx'] = (asl.df['left-x'] - asl.df['speaker'].map(df_means['left-x'])) / (asl.df['speaker'].map(df_std['left-x']) + eps) asl.df['norm-ly'] = (asl.df['left-y'] - asl.df['speaker'].map(df_means['left-y'])) / (asl.df['speaker'].map(df_std['left-y']) + eps) asl.df['norm-rx'] = (asl.df['right-x'] - asl.df['speaker'].map(df_means['right-x'])) / (asl.df['speaker'].map(df_std['right-x']) + eps) asl.df['norm-ry'] = (asl.df['right-y'] - asl.df['speaker'].map(df_means['right-y'])) / (asl.df['speaker'].map(df_std['right-y']) + eps) features_polar = ['polar-rr', 'polar-rtheta', 'polar-lr', 'polar-ltheta'] def get_theta(y, x): # np.arctan2 expects the first argument to be the y coordinate, but the instructions say to swap the # x and y coordinates.
def init(): asl = AslDb() #dimensions hand = ['right', 'left'] side = ['r', 'l'] cartesian = ['x', 'y'] polar = ['r', 'theta'] #rename the raw data for consistency raw_names = { h + '-' + c: 'raw-' + h[0] + c for h in hand for c in cartesian } asl.df = asl.df.rename(columns=raw_names) cartesian_features = ['grnd', 'norm', 'delta'] features = { k: [k + '-' + h[0] + c for h in hand for c in cartesian] for k in cartesian_features } features['polar'] = ['polar' + '-' + s + c for s in side for c in polar] #derive the features for f in features['grnd']: asl.df[f] = asl.df['raw' + f[-3:]] - asl.df['nose-' + f[-1:]] df_means = asl.df.groupby('speaker').mean() df_std = asl.df.groupby('speaker').std() for f in features['norm']: ref = 'raw' + f[-3:] asl.df[f] = (asl.df[ref] - asl.df['speaker'].map( df_means[ref])) / asl.df['speaker'].map(df_std[ref]) for f in features['delta']: ref = 'grnd' + f[-3:] asl.df[f] = (asl.df[ref].diff()).fillna(0) ref = 'grnd' asl.df['polar-rtheta'] = (np.arctan2(asl.df[ref + '-rx'], asl.df[ref + '-ry'])) asl.df['polar-ltheta'] = (np.arctan2(asl.df[ref + '-lx'], asl.df[ref + '-ly'])) asl.df['polar-rr'] = np.sqrt(asl.df[ref + '-rx']**2 + asl.df[ref + '-ry']**2) asl.df['polar-lr'] = np.sqrt(asl.df[ref + '-lx']**2 + asl.df[ref + '-ly']**2) training = {k: asl.build_training(v) for k, v in features.items()} xlens = training['grnd'].get_all_Xlengths() lens_stats = [(k, len(v[1]), min(v[1]), sum(v[1]) / len(v[1]), max(v[1]), max(v[1]) - min(v[1])) for k, v in xlens.items()] words_stats = pd.DataFrame.from_records( lens_stats, columns=['word', 'count', 'min', 'avg', 'max', 'range']).set_index('word') words_stats['spread'] = words_stats['range'] / words_stats['avg'] #include all words min_len = 0 words = words_stats[words_stats['min'] > min_len].sort_values( by='count', ascending=False).index.tolist() samples = dict() for f in features: samples[f] = {k: get_word(training, features, f, k) for k in words} threshold = 1e-9 separated = { k: ([s for s in v if min(s.std()) < threshold], [s for s in v if min(s.std()) > threshold]) for k, v in samples['norm'].items() } separated_stats = pd.DataFrame.from_records( {k: (len(v[0]), len(v[1])) for k, v in separated.items()}).T.rename(columns={ 0: 'single', 1: 'double' }) return asl, features, training, samples, words_stats.join( separated_stats), separated
'BOOK').select() self.assertGreaterEqual(model.n_components, 2) #def test_select_bic_interface(self): # model = SelectorBIC(self.sequences, self.xlengths, 'FRANK').select() # self.assertGreaterEqual(model.n_components, 2) # model = SelectorBIC(self.sequences, self.xlengths, 'VEGETABLE').select() # self.assertGreaterEqual(model.n_components, 2) def test_select_cv_interface(self): model = SelectorCV(self.sequences, self.xlengths, 'JOHN').select() self.assertGreaterEqual(model.n_components, 2) model = SelectorCV(self.sequences, self.xlengths, 'CHICKEN').select() self.assertGreaterEqual(model.n_components, 2) #def test_select_dic_interface(self): # model = SelectorDIC(self.sequences, self.xlengths, 'MARY').select() # self.assertGreaterEqual(model.n_components, 2) # model = SelectorDIC(self.sequences, self.xlengths, 'TOY').select() # self.assertGreaterEqual(model.n_components, 2) if __name__ == '__main__': asl = AslDb() training = asl.build_training(FEATURES) sequences = training.get_all_sequences() xlengths = training.get_all_Xlengths() #SelectorCV(sequences, xlengths, 'JOHN').select() SelectorDIC(sequences, xlengths, 'FRANK').select()
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] #words_to_train = ['FISH'] words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] import timeit from my_model_selectors import SelectorCV, SelectorBIC, SelectorDIC training = asl.build_training(features_ground) sequences = training.get_all_sequences() Xlengths = training.get_all_Xlengths() for word in words_to_train: start = timeit.default_timer() #model = SelectorCV(sequences, Xlengths, word, # min_n_components=2, max_n_components=15, random_state = 14).select() # model = SelectorBIC(sequences, Xlengths, word, # min_n_components=2, max_n_components=15, random_state = 14).select() model = SelectorDIC(sequences, Xlengths, word,
from asl_utils import test_features_tryit import warnings from hmmlearn.hmm import GaussianHMM import math from matplotlib import (cm, pyplot as plt, mlab) from my_model_selectors import SelectorConstant from my_model_selectors import SelectorCV from sklearn.model_selection import KFold import timeit asl = AslDb() # initializes the database #print(asl.df.head()) # displays the first five rows of the asl database, indexed by video and frame #print(asl.df.ix[98,1]) # look at the data available for an individual frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] # TODO add df columns for 'grnd-rx', 'grnd-ly', 'grnd-lx' representing differences between hand and nose locations asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] #print(asl.df.head()) # the new feature 'grnd-ry' is now in the frames dictionary # test the code #test_features_tryit(asl) # collect the features into a list features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly']
def train_all_words(features, model_selector): training = asl.build_training( features ) # Experiment here with different feature sets defined in part 1 sequences = training.get_all_sequences() Xlengths = training.get_all_Xlengths() model_dict = {} for word in training.words: model = model_selector(sequences, Xlengths, word, n_constant=3).select() model_dict[word] = model return model_dict asl = AslDb() # initializes the database df_means = asl.df.groupby('speaker').mean() df_std = asl.df.groupby('speaker').std() features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] features_norm = ['norm-rx', 'norm-ry', 'norm-lx', 'norm-ly'] features_polar = ['polar-rr', 'polar-rtheta', 'polar-lr', 'polar-ltheta'] features_delta = ['delta-rx', 'delta-ry', 'delta-lx', 'delta-ly'] # c.log # features_custom = ['norm-delta-rx', 'norm-delta-ry', 'norm-delta-lx', 'norm-delta-ly'] # d.log features_custom1 = [ 'norm-grnd-rx', 'norm-grnd-ry', 'norm-grnd-lx', 'norm-grnd-ly' ] # g.log features_custom2 = [ 'delta-grnd-rx', 'delta-grnd-ry', 'delta-grnd-lx', 'delta-grnd-ly'
# -*- coding: utf-8 -*- """ Created on Mon Oct 15 22:10:03 2018 @author: WIN """ import numpy as np import pandas as pd from asl_data import AslDb from asl_utils import test_features_tryit asl = AslDb() # initializes the database s = asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame asl.df.iloc[99, 1] # look at the data available for an individual frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df.head() # the new feature 'grnd-ry' is now in the frames dictionary # TODO add df columns for 'grnd-rx', 'grnd-ly', 'grnd-lx' representing differences # between hand and nose locations asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] # test the code test_features_tryit(asl)
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() # initializes the database asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame asl.df.ix[98, 2] # look at the data available for an individual frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df.head() # the new feature 'grnd-ry' is now in the frames dictionary from asl_utils import test_features_tryit asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] test_features_tryit(asl) features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] #show a single set of features for a given (video, frame) tuple [asl.df.ix[98, 1][v] for v in features_ground] training = asl.build_training(features_ground) print("Training words: {}".format(training.words)) training.get_word_Xlengths('CHOCOLATE') df_means = asl.df.groupby('speaker').mean() df_means
training = asl.build_training(features) # Experiment here with different feature sets defined in part 1 sequences = training.get_all_sequences() xlengths = training.get_all_Xlengths() model_dict = {} for word in training.words: model = model_selector(sequences, xlengths, word, n_constant=3).select() model_dict[word] = model return model_dict if __name__ == "__main__": asl = AslDb() add_features1(asl) for feats in feature_sets: print("===============================================") print(" FEATURES") print(" {}".format(feats)) print("===============================================") for sel in selectors: print(" Selector: {}".format(sel)) start_t = time.time() print("Start time: {}".format(start_t)) print("--------------------------") try: models = train_all_words(feats, sel) test_set = asl.build_test(feats) probabilities, guesses = recognize(models, test_set)
# <a id='part1_tutorial'></a> # ## PART 1: Data # # ### Features Tutorial # ##### Load the initial database # A data handler designed for this database is provided in the student codebase as the `AslDb` class in the `asl_data` module. This handler creates the initial [pandas](http://pandas.pydata.org/pandas-docs/stable/) dataframe from the corpus of data included in the `data` directory as well as dictionaries suitable for extracting data in a format friendly to the [hmmlearn](https://hmmlearn.readthedocs.io/en/latest/) library. We'll use those to create models in Part 2. # # To start, let's set up the initial database and select an example set of features for the training set. At the end of Part 1, you will create additional feature sets for experimentation. # In[2]: import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() # initializes the database print(asl.df.shape) asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame # In[3]: asl.df.ix[98, 1] # look at the data available for an individual frame # The frame represented by video 98, frame 1 is shown here: # ![Video 98](http://www-i6.informatik.rwth-aachen.de/~dreuw/database/rwth-boston-104/overview/images/orig/098-start.jpg) # ##### Feature selection for training the model # The objective of feature selection when training a model is to choose the most relevant variables while keeping the model as simple as possible, thus reducing training time. We can use the raw features already provided or derive our own and add columns to the pandas dataframe `asl.df` for selection. As an example, in the next cell a feature named `'grnd-ry'` is added. This feature is the difference between the right-hand y value and the nose y value, which serves as the "ground" right y value. # In[4]:
from asl_utils import show_errors def train_all_words(features, model_selector): training = asl.build_training(features) # Experiment here with different feature sets defined in part 1 sequences = training.get_all_sequences() Xlengths = training.get_all_Xlengths() model_dict = {} for word in training.words: model = model_selector(sequences, Xlengths, word, n_constant=3, verbose=False).select() model_dict[word]=model return model_dict asl = AslDb() # initializes the database asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] df_means = asl.df.groupby('speaker').mean() asl.df['right-x-mean'] = asl.df['speaker'].map(df_means['right-x']) asl.df['right-y-mean'] = asl.df['speaker'].map(df_means['right-y']) asl.df['left-x-mean'] = asl.df['speaker'].map(df_means['left-x']) asl.df['left-y-mean'] = asl.df['speaker'].map(df_means['left-y']) df_std = asl.df.groupby('speaker').std() asl.df['right-x-std'] = asl.df['speaker'].map(df_std['right-x'])
def build_asl_db(): asl = AslDb() # ground features asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] # polar features asl.df['polar-rr'] = np.sqrt(asl.df['grnd-rx'] ** 2 + asl.df['grnd-ry'] ** 2) asl.df['polar-rtheta'] = np.arctan2(asl.df['grnd-rx'], asl.df['grnd-ry']) asl.df['polar-lr'] = np.sqrt(asl.df['grnd-lx'] ** 2 + asl.df['grnd-ly'] ** 2) asl.df['polar-ltheta'] = np.arctan2(asl.df['grnd-lx'], asl.df['grnd-ly']) refs = ['right-x', 'right-y', 'left-x', 'left-y'] features_delta = ['delta-rx', 'delta-ry', 'delta-lx', 'delta-ly'] for ref, feat in zip(refs, features_delta): asl.df[feat] = asl.df[ref].diff() asl.df[feat].fillna(0., inplace=True) # compute mean/std df_means = asl.df.groupby('speaker').mean() df_stds = asl.df.groupby('speaker').std() # normalized features refs = ['right-x', 'right-y', 'left-x', 'left-y'] features_norm = ['norm-rx', 'norm-ry', 'norm-lx', 'norm-ly'] for ref, feat in zip(refs, features_norm): asl.df[feat] = (asl.df[ref] - asl.df['speaker'].map(df_means[ref])) / asl.df['speaker'].map(df_stds[ref]) refs = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] features_norm = ['norm-grx', 'norm-gry', 'norm-glx', 'norm-gly'] for ref, feat in zip(refs, features_norm): asl.df[feat] = (asl.df[ref] - asl.df['speaker'].map(df_means[ref])) / asl.df['speaker'].map(df_stds[ref]) refs = ['polar-rr', 'polar-lr'] polars_norm = ['norm-prr', 'norm-plr'] for ref, feat in zip(refs, polars_norm): asl.df[feat] = (asl.df[ref] - asl.df['speaker'].map(df_means[ref])) / asl.df['speaker'].map(df_stds[ref]) return asl
@author: Markus """ import timeit import numpy as np import pandas as pd from asl_data import AslDb from my_model_selectors import (SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC) from asl_utils import (combine_sequences, show_errors) import math from matplotlib import (cm, pyplot as plt, mlab) from my_recognizer import recognize from my_ngrams import (OneGram, TwoGram, ThreeGram) asl = AslDb() # initializes the database #print('asl head: {}'.format(asl.df.head())) # asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] #asl.df.head() # ## collect the features into a list features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] # #show a single set of features for a given (video, frame) tuple #[asl.df.ix[98,1][v] for v in features_ground] #training = asl.build_training(features_ground) #print("Training words: {}".format(training.words))
def setUp(self): self.asl = AslDb() self.training_set = self.asl.build_training(FEATURES) self.test_set = self.asl.build_test(FEATURES) self.models = train_all_words(self.training_set, SelectorConstant)
import numpy as np import pandas as pd import timeit from asl_data import AslDb from my_model_selectors import SelectorConstant, SelectorCV, SelectorBIC, SelectorDIC from my_recognizer import recognize from asl_utils import show_errors asl = AslDb() def run_experiment(asl): run_features = ['custom', 'custom-delta-norm-polar'] selectors = {"Constant" : SelectorConstant, "CV": SelectorCV, "BIC" : SelectorBIC, "DIC" : SelectorDIC} #selectors = {"DIC" : SelectorDIC} features = load_features(asl) print("Feature, Selector, Training Time, Recognize Time, WER") for feature in run_features: for selector_name, selector in selectors.items(): experiment(asl, feature, features[feature], selector_name,selector) def experiment(asl, feature_name, feature, selector_name, selector): start = timeit.default_timer() models = train_all_words(asl, feature, selector) t_time = timeit.default_timer()-start test_set = asl.build_test(feature) probabilities, guesses = recognize(models, test_set) r_time = timeit.default_timer()-start show_results(feature_name, selector_name, t_time, r_time, guesses, test_set) def show_results(feature_name, selector_name, training_time, recognize_time, guesses, test_set):
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() # initializes the database # print(asl.df.head()) words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] import timeit features_ground = ['grnd-rx','grnd-ry','grnd-lx','grnd-ly'] asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] df_means = asl.df.groupby('speaker').mean() asl.df['left-x-mean']= asl.df['speaker'].map(df_means['left-x']) from asl_utils import test_std_tryit df_std = asl.df.groupby('speaker').std() cols = ['right-x', 'right-y', 'left-x', 'left-y'] features_norm = ['norm-rx', 'norm-ry', 'norm-lx','norm-ly'] for i, feature in enumerate(features_norm): col = cols[i] means = asl.df['speaker'].map(df_means[col]) stds = asl.df['speaker'].map(df_std[col])
import numpy as np import pandas as pd from asl_data import AslDb # PART 1: Data asl = AslDb() # initializes the database asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame asl.df.ix[98, 1] # look at the data available for an individual frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df.head() # the new feature 'grnd-ry' is now in the frames dictionary from asl_utils import test_features_tryit # TODO add df columns for 'grnd-rx', 'grnd-ly', 'grnd-lx' representing differences between hand and nose locations # test the code test_features_tryit(asl) # collect the features into a list features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly'] #show a single set of features for a given (video, frame) tuple [asl.df.ix[98, 1][v] for v in features_ground] training = asl.build_training(features_ground) print("Training words: {}".format(training.words)) training.get_word_Xlengths('CHOCOLATE') df_means = asl.df.groupby('speaker').mean() df_means asl.df['left-x-mean'] = asl.df['speaker'].map(df_means['left-x'])
from my_model_selectors import SelectorCV, SelectorBIC, SelectorDIC import numpy as np import pandas as pd from asl_data import AslDb import timeit asl = AslDb() # initializes the database asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] features_ground = ['grnd-rx','grnd-ry','grnd-lx','grnd-ly'] words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] training = asl.build_training(features_ground) # Experiment here with different feature sets defined in part 1 sequences = training.get_all_sequences() Xlengths = training.get_all_Xlengths() for word in words_to_train: #try: start = timeit.default_timer() model = SelectorDIC(sequences, Xlengths, word, min_n_components=2, max_n_components=15, random_state=14).select() end = timeit.default_timer()-start #except ValueError as ve: # model=None # print('caught an valueerror for word {}'.format(word)) if model is not None: print("Training complete for {} with {} states with time {} seconds (BIC {})" .format(word, model.n_components, end, getattr(model, 'dic', None))) else: print("Training failed for {}".format(word))
from asl_data import AslDb import numpy as np asl = AslDb() # initializes the database asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] features_ground = ['grnd-rx','grnd-ry','grnd-lx','grnd-ly'] # training = asl.build_training(features_ground) # print("Training words: {}".format(training.words)) # training.get_word_Xlengths('CHOCOLATE') df_means = asl.df.groupby('speaker').mean() asl.df['left-x-mean']= asl.df['speaker'].map(df_means['left-x']) df_std = asl.df.groupby('speaker').std() asl.df['norm-rx'] = (asl.df['right-x'] - asl.df['speaker'].map(df_means['right-x']))/asl.df['speaker'].map(df_std['right-x']) asl.df['norm-ry'] = (asl.df['right-y'] - asl.df['speaker'].map(df_means['right-y']))/asl.df['speaker'].map(df_std['right-y']) asl.df['norm-lx'] = (asl.df['left-x'] - asl.df['speaker'].map(df_means['left-x']))/asl.df['speaker'].map(df_std['left-x']) asl.df['norm-ly'] = (asl.df['left-y'] - asl.df['speaker'].map(df_means['left-y']))/asl.df['speaker'].map(df_std['left-y']) features_norm = ['norm-rx', 'norm-ry', 'norm-lx','norm-ly'] asl.df['polar-rr'] = ((asl.df['grnd-rx'])**2+(asl.df['grnd-ry'])**2)**0.5 asl.df['polar-rtheta'] = np.arctan2([asl.df['grnd-rx']],[asl.df['grnd-ry']])[0] asl.df['polar-lr'] = ((asl.df['grnd-lx'])**2+(asl.df['grnd-ly'])**2)**0.5 asl.df['polar-ltheta'] = np.arctan2([asl.df['grnd-lx']],[asl.df['grnd-ly']])[0]
import numpy as np import pandas as pd from asl_data import AslDb asl = AslDb() # initializes the database asl.df.head() # displays the first five rows of the asl database, indexed by video and frame
# # ### Features Tutorial # ##### Load the initial database # A data handler designed for this database is provided in the student codebase as the `AslDb` class in the `asl_data` module. This handler creates the initial [pandas](http://pandas.pydata.org/pandas-docs/stable/) dataframe from the corpus of data included in the `data` directory as well as dictionaries suitable for extracting data in a format friendly to the [hmmlearn](https://hmmlearn.readthedocs.io/en/latest/) library. We'll use those to create models in Part 2. # # To start, let's set up the initial database and select an example set of features for the training set. At the end of Part 1, you will create additional feature sets for experimentation. # In[46]: import numpy as np import pandas as pd from asl_data import AslDb if __name__ == "__main__": asl = AslDb() # initializes the database asl.df.head( ) # displays the first five rows of the asl database, indexed by video and frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df.head() # the new feature 'grnd-ry' is now in the frames dictionary # ##### Try it! # In[47]: asl.df.ix[98, 1] # look at the data available for an individual frame # The frame represented by video 98, frame 1 is shown here: # ![Video 98](http://www-i6.informatik.rwth-aachen.de/~dreuw/database/rwth-boston-104/overview/images/orig/098-start.jpg) # ##### Feature selection for training the model
import math import statistics import warnings import numpy as np from hmmlearn.hmm import GaussianHMM from sklearn.model_selection import KFold from asl_utils import combine_sequences from asl_data import AslDb asl = AslDb() # initializes the database asl.df.head() # displays the first five rows of the asl database, indexed by video and frame asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y'] asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x'] asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y'] asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x'] # collect the features into a list features_ground = ['grnd-rx','grnd-ry','grnd-lx','grnd-ly'] #show a single set of features for a given (video, frame) tuple from my_model_selectors import SelectorConstant from my_model_selectors import SelectorCV words_to_train = ['FISH', 'BOOK', 'VEGETABLE', 'FUTURE', 'JOHN'] #words_to_train = ['CHICKEN'] import timeit # training = asl.build_training(features_ground) # Experiment here with different feature sets defined in part 1 # sequences = training.get_all_sequences()