def __init__(self):
self.preprocessor = Preprocessor()
self.feature_extractor = FeatureExtractor()
self.crf_analyzer = CRFAnalyzer()
self.sentiment_analyzer = SentimentAnalyzer()
print("\nAll module instantiated and ready to go....\n")
def __init__(self, dataset_cfg, feature_params, collect_wavs=False, verbose=False):
# parent init
super().__init__(dataset_cfg, feature_params, collect_wavs=collect_wavs, verbose=verbose)
# feature extractor
self.feature_extractor = FeatureExtractor(feature_params=self.feature_params)
# short vars
self.N = self.feature_extractor.N
self.hop = self.feature_extractor.hop
# create plot plaths if not already exists
create_folder(list(self.plot_paths.values()))
# recreate
if self.dataset_cfg['recreate'] or not check_folders_existance(self.wav_folders, empty_check=True):
# delete old data
delete_files_in_path(self.wav_folders, file_ext=self.dataset_cfg['file_ext'])
# create folder wav folders
create_folder(self.wav_folders)
# create sets (specific to dataset)
self.create_sets()
# get audio files from sets
self.get_audiofiles()
self.get_annotation_files()
def __init__(self,
classifier,
mic_params,
is_audio_record=False,
root_path='./'):
# arguments
self.classifier = classifier
self.mic_params = mic_params
self.is_audio_record = is_audio_record
self.root_path = root_path
# plot path
self.plot_path = self.root_path + self.mic_params['plot_path']
# create folder for plot path
create_folder([self.plot_path])
# shortcuts
self.feature_params = classifier.feature_params
# feature extractor
self.feature_extractor = FeatureExtractor(self.feature_params)
# windowing params
self.N, self.hop = self.feature_extractor.N, self.feature_extractor.hop
# queue
self.q = queue.Queue()
# collector
self.collector = Collector(
N=self.N,
hop=self.hop,
frame_size=self.feature_params['frame_size'],
update_size=self.mic_params['update_size'],
frames_post=self.mic_params['frames_post'],
is_audio_record=self.is_audio_record)
# device
self.device = sd.default.device[0] if not self.mic_params[
'select_device'] else self.mic_params['device']
# determine downsample
self.downsample = self.mic_params['fs_device'] // self.feature_params[
'fs']
# get input devices
self.input_dev_dict = self.extract_devices()
# show devices
print("\ndevice list: \n", sd.query_devices())
print("\ninput devs: ", self.input_dev_dict.keys())
# stream
self.stream = None
# change device flag
self.change_device_flag = False
def audio_set_wavs(cfg):
"""
audio set wavs
"""
# plot path
plot_path = '../docu/thesis/5_exp/figs/'
# audio sets
a1 = AudioDataset(cfg['datasets']['speech_commands'],
cfg['feature_params'],
root_path='../')
a2 = AudioDataset(cfg['datasets']['my_recordings'],
cfg['feature_params'],
root_path='../')
# feature extractor
feature_extractor = FeatureExtractor(cfg['feature_params'])
# get audio files
a1.get_audiofiles()
# random seed
np.random.seed(1234)
r = np.random.randint(low=0, high=150, size=len(a1.set_audio_files[1]))
wav_grid = []
# process wavs
for wav in sorted([
label_wavs[r[i]]
for i, label_wavs in enumerate(a1.set_audio_files[1])
]):
# info
print("wav: ", wav)
# get raw
x, _ = a1.wav_pre_processing(wav)
# extract feature vectors [m x l]
_, bon_pos = feature_extractor.extract_mfcc(x,
reduce_to_best_onset=False)
# append to wav grid
wav_grid.append((librosa.util.normalize(x),
re.sub(r'[0-9]+-', '',
wav.split('/')[-1].split('.')[0]), bon_pos))
# plot wav grid
plot_wav_grid(wav_grid,
feature_params=a1.feature_params,
grid_size=(6, 5),
plot_path=plot_path,
name='wav_grid_c30',
show_plot=True)
def __init__(self,classes,fs,channels,train_dir,duration):
self.classes = classes
f = FeatureExtractor(fs,duration,channels,train_dir,classes)
self.feature_matrix = f.build_feature_vector()
self.len_train = len(self.feature_matrix[self.classes[0]])
self.testdata = None
self.traindata = None
self.alldata = None
self.trainer = None
self.fnet = None
def __init__(self,
classifier,
feature_params,
mic_params,
is_audio_record=False):
# arguments
self.classifier = classifier
self.feature_params = feature_params
self.mic_params = mic_params
self.is_audio_record = is_audio_record
# windowing params
self.N, self.hop = int(
feature_params['N_s'] * feature_params['fs']), int(
feature_params['hop_s'] * feature_params['fs'])
# queue
self.q = queue.Queue()
# collector
self.collector = Collector(
N=self.N,
hop=self.hop,
frame_size=self.feature_params['frame_size'],
update_size=self.mic_params['update_size'],
frames_post=self.mic_params['frames_post'],
is_audio_record=self.is_audio_record)
# feature extractor
self.feature_extractor = FeatureExtractor(
self.feature_params['fs'],
N=self.N,
hop=self.hop,
n_filter_bands=self.feature_params['n_filter_bands'],
n_ceps_coeff=self.feature_params['n_ceps_coeff'],
frame_size=self.feature_params['frame_size'])
# select microphone yourself (usually not necessary)
if mic_params['select_device']:
sd.default.device = self.mic_params['device']
# determine downsample
self.downsample = self.mic_params['fs_device'] // self.feature_params[
'fs']
# show devices
print("\ndevice list: \n", sd.query_devices())
# setup stream sounddevice
self.stream = sd.InputStream(samplerate=self.mic_params['fs_device'],
blocksize=int(self.hop * self.downsample),
channels=self.mic_params['channels'],
callback=self.callback_mic)
def time_measurements(x, u, feature_params):
"""
time measurements
"""
# create feature extractor
feature_extractor = FeatureExtractor(feature_params)
# n measurements
delta_time_list = []
for i in range(100):
# measure extraction time - start
start_time = time.time()
# time: 0.030081419944763182
#y = calc_mfcc39(x, fs, N=400, hop=160, n_filter_bands=32, n_ceps_coeff=12, use_librosa=False)
# time: 0.009309711456298829
#y = calc_mfcc39(x, fs, N=400, hop=160, n_filter_bands=32, n_ceps_coeff=12, use_librosa=True)
# time: 0.00014737367630004883
#y = (custom_dct(np.log(u), n_filter_bands).T)
# time: 6.929159164428711e-05
#y = scipy.fftpack.dct(np.log(u), type=2, n=n_filter_bands, axis=1, norm=None, overwrite_x=False).T
# time: 0.00418839693069458 *** winner
y, _ = feature_extractor.extract_mfcc(x)
# time: 0.015525884628295898
#y, _ = feature_extractor.extract_mfcc39_slow(x)
# time: 0.011266257762908936s
#y = custom_stft(x, N=N, hop=hop, norm=True)
# time: 0.0005800390243530274s
#y = 2 / N * librosa.stft(x, n_fft=N, hop_length=hop, win_length=N, window='hann', center=True, dtype=None, pad_mode='reflect')
# time: 0.00044193744659423826s
#_, _, y = scipy.signal.stft(x, fs=1.0, window='hann', nperseg=N, noverlap=N-hop, nfft=N, detrend=False, return_onesided=True, boundary='zeros', padded=False, axis=- 1)
# result of measured time diff
delta_time_list.append(time.time() - start_time)
# data shpae
print("y: ", y.shape)
# times
print("delta_time: ", np.mean(delta_time_list))
def train_classifier(self):
"""
Train classifier and save to disk
:return:
"""
feature_extractor = FeatureExtractor.tfidf(ngram_range=(1, 2),
max_df=0.5,
use_idf=False,
sublinear_tf=True)
clf = SVC(kernel='linear',
C=100,
gamma=0.01,
decision_function_shape='ovo',
probability=True)
counts, targets = feature_extractor.extract_features_from_csv
print('start training...')
clf.fit(counts, targets) # train the classifier
print('training finished. start dumping model...')
# save model and classifier to disk
joblib.dump(clf, self.resource_path + 'booking_classifier.pkl')
joblib.dump(feature_extractor,
self.resource_path + 'booking_features.pkl')
self.load_model()
def mfcc_stuff(cfg):
"""
for dct, filter bands, etc
"""
# plot path
plot_path = '../docu/thesis/3_signal/figs/'
# init feature extractor
feature_extractor = FeatureExtractor(cfg['feature_params'])
# plot dct
plot_dct(custom_dct_matrix(cfg['feature_params']['n_filter_bands']),
plot_path=plot_path,
name='signal_mfcc_dct',
show_plot=False)
plot_dct(custom_dct_matrix(cfg['feature_params']['n_filter_bands']),
plot_path=plot_path,
context='dct-div',
name='signal_mfcc_dct-div',
show_plot=False)
# mel scale
plot_mel_scale(plot_path=plot_path,
name='signal_mfcc_mel_scale',
show_plot=False)
# plot mel bands
plot_mel_band_weights(feature_extractor.w_f,
feature_extractor.w_mel,
feature_extractor.f,
feature_extractor.m,
plot_path=plot_path,
name='signal_mfcc_weights',
show_plot=True)
class ProcessController:
def __init__(self):
self.preprocessor = Preprocessor()
self.feature_extractor = FeatureExtractor()
self.crf_analyzer = CRFAnalyzer()
self.sentiment_analyzer = SentimentAnalyzer()
print("\nAll module instantiated and ready to go....\n")
def start_process(self, sentence):
data = dict()
preprocess_result, headword_data, spacy_parsing_result = self.preprocessor.start_preprocess(
sentence)
features = [
self.feature_extractor.extract_features(preprocess_result,
headword_data)
]
crf_result = self.crf_analyzer.analyze(preprocess_result, features)
self.sentiment_analyzer.create_graph_data(spacy_parsing_result)
sentiment_analyzer_result = self.sentiment_analyzer.analyze(crf_result)
data["sentence"] = sentence
data["result"] = sentiment_analyzer_result
return data
def fit(ml_algorithm, train_files, train_labels, name):
train_features, _ = FeatureExtractor.get_features(train_files, [])
try:
algorithm_type = ALGORITHMS[ml_algorithm]
except KeyError:
print 'Algorithm type not valid!'
return
mla = algorithm_type()
mla.fit(train_features, train_labels, name)
def get_probs(ml_algorithm, train_files, test_files, name):
_, test_features = FeatureExtractor.get_features(train_files, test_files)
try:
algorithm_type = ALGORITHMS[ml_algorithm]
except KeyError:
print 'Algorithm type not valid!'
return
mla = algorithm_type()
prob_ind, highest_score = mla.predict(test_features, name)
return prob_ind, highest_score
def train_and_predict(ml_algorithm, train_files, train_labels, test_files, test_labels):
train_features, test_features = FeatureExtractor.get_features(train_files, test_files)
try:
algorithm_type = ALGORITHMS[ml_algorithm]
except KeyError:
print 'Algorithm type not valid!'
return
mla = algorithm_type()
score, indices, probs = mla.fit_and_predict(train_features, test_features, train_labels, test_labels)
print 'Score: {}'.format(score)
return indices, probs
def train_kfold(ml_algorithm, source_code, labels, ast_nodes=None):
source_code = np.array(source_code)
labels = np.array(labels)
#ast_nodes = np.array(ast_nodes)
try:
algorithm_type = ALGORITHMS[ml_algorithm]
except KeyError:
print 'Algorithm type not valid!'
return
start = time.time()
n_trees = [400]
for nt in n_trees:
mla = algorithm_type(n_trees=nt)
print 'nt: {}'.format(nt)
k = 10
code_per_author = 10
accuracy = 0
for i in range(k):
test_indices = []
it = code_per_author / k
for j in range(it):
test_indices.extend(np.array(range(i*it+j, len(source_code), code_per_author)))
train_indices = []
for sci in range(len(source_code)):
if sci not in test_indices:
train_indices.append(sci)
train_features, test_features = FeatureExtractor.get_features(source_code[train_indices],
source_code[test_indices])
score = mla.fit_and_predict(train_features, test_features, labels[train_indices], labels[test_indices])
print 'Score after {}th fold is {}'.format(i, score)
accuracy += score
print 'Final score: {}'.format(accuracy / float(k))
end = time.time() - start
print 'Execution time: {}'.format(end)
print '-----------------------------------------------'
def prep(n, labelcol):
"""
Load the data
"""
f = pf.open(n)
data = f[1].data
names = f[1].columns.names
f.close()
try:
labels = data.field(labelcol)
except:
labels = np.zeros(data.field(0).size) - 99
ylim = np.inf
featurenames = ['cmodelmag', 'psffwhm', 'petror50', 'petror90']
targetnames = ['psfmag', 'cmodelmag']
filters = ['u', 'g', 'r', 'i', 'z']
x = FeatureExtractor(data, featurenames, filters, color_band='r', scale_kind=None,
mag_range=None)
data = data[x.idx]
labels = labels[x.idx]
y = FeatureExtractor(data, targetnames, filters, color_band=None, scale_kind=None,
mag_range=None)
# taylor to target, set for psf - model
y.features[:, :5] = y.features[:, :5] - y.features[:, 5:10]
y.features[:, 5:10] = np.sqrt(y.features[:, 10:15] ** 2. + y.features[:, 15:20] ** 2.)
y.features = y.features[:, :10]
# restrict y range
ylim = 10.
ind = y.features[:, 2] < ylim
x.features = x.features[ind]
y.features = y.features[ind]
labels = labels[ind]
y.Ndata = y.features.shape[0]
return x, y, labels
def __init__(self, *args, **kwargs):
BaseEntity.__init__(self, *args, **kwargs)
self.rotation_parametrization = rotation_parametrizations[
self.args.rotation_parametrization]
self._create_parameter_info_table()
if self.z_up:
self._vertical_axis = "z"
self._coordinate_up = 2
else:
self._vertical_axis = "y"
self._coordinate_up = 1
self._max_angular_step = self.args.max_angular_step
self._normalized_constrainers = self._create_constrainers()
self._unnormalized_constrainers = self._create_constrainers()
self.modified_root_vertical_orientation = None
self._last_root_vertical_orientation = None
self._rotation_interpolators = {}
self._enable_friction = self.args.friction
if hasattr(self.args, "enable_features") and self.args.enable_features:
self.feature_extractor = FeatureExtractor(self._coordinate_up)
self._bvh_joint_name_for_feature = self._get_bvh_joint_names_for_features_from_args()
def predict_equation(self, image: Image) -> str:
"""Given an image predicts the LaTeX equation for that image.
:param image: An image.
:returns: The predicted LaTeX equation for the image.
"""
feature_vectors = []
image = image.convert('L')
while image is not None:
image = imhelp.remove_border_and_padding(
image,
black_threshold=0,
white_threshold=fe.DEFAULT_WHITE_THRESHOLD,
rem_black=True,
rem_white=True)
left_im, image = imhelp.extract_leftmost_symbol(image)
fv = FeatureExtractor.extract_features_from_image(left_im)
feature_vectors.append(fv)
feature_matrix = fe.FeatureExtractor.get_feature_matrix(
feature_vectors)
return ' '.join(self.predict(feature_matrix))
def write_predictions(self):
"""Output the predictions to a text file."""
res = FeatureExtractor("test").run()
model = torch.load("model.pt")
test = namedtuple("res",
["lsr", "feats", "scores"])(lsr=res.lsr.reshape(
-1, 2048),
feats=res.feats,
scores=res.scores)
dev_ = data_utils.TensorDataset(*[
torch.tensor(getattr(test, i)).float()
for i in ["lsr", "feats", "scores"]
])
with torch.no_grad():
preds = model.forward(*dev_.tensors[:2]).cpu().numpy()
np.set_printoptions(suppress=True)
np.savetxt("predictions.txt",
preds.astype(float),
delimiter="\n",
fmt="%f")
print("Predictions saved to predictions.txt")
def run_fetc():
if "-exp" in sys.argv:
exp_pos = sys.argv.index("-exp")
param_file = sys.argv[exp_pos+1]
else:
param_file = "experiment.yaml"
exp = read_parameters(param_file=param_file)
ovw = parse_commandline(sys.argv)
overwrite_params(exp, ovw)
update_parameters(exp)
#print exp
if exp['steps']['extract_features']:
fe = FeatureExtractor.create(params=exp)
fe.run()
if exp['steps']['aggregate_features']:
fa = FeatureAggregator.create(params=exp)
fa.run()
if exp['steps']['train']:
t = ModelTrainer.create(params=exp)
t.run()
if exp['steps']['test']:
t = ModelTester.create(params=exp)
t.run()
if exp['steps']['evaluate']:
t = Evaluator.create(params=exp)
t.run()
def run_fetc():
if "-exp" in sys.argv:
exp_pos = sys.argv.index("-exp")
param_file = sys.argv[exp_pos + 1]
else:
param_file = "experiment.yaml"
exp = read_parameters(param_file=param_file)
ovw = parse_commandline(sys.argv)
overwrite_params(exp, ovw)
update_parameters(exp)
#print exp
if exp['steps']['extract_features']:
fe = FeatureExtractor.create(params=exp)
fe.run()
if exp['steps']['aggregate_features']:
fa = FeatureAggregator.create(params=exp)
fa.run()
if exp['steps']['train']:
t = ModelTrainer.create(params=exp)
t.run()
if exp['steps']['test']:
t = ModelTester.create(params=exp)
t.run()
if exp['steps']['evaluate']:
t = Evaluator.create(params=exp)
t.run()
vocab[i] = vocab[i].strip()
def build_model(n_word):
global num_class
model = Sequential()
model.add(Flatten(input_shape=(n_word, 500)))
model.add(Dropout(0.1))
model.add(Dense(num_class, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='sgd')
return model
# model = load_model('model/bengio_sgd_2.h5')
# print(model.summary())
fe = FeatureExtractor(5)
fe.set_w2v(w2v_pathname, 500, keep_alive=True)
for epoch in range(9, 11):
model = load_model('model/bengio_sgd_{}.h5'.format(epoch - 1))
for i in range(1, 1001):
filename = 'data/batch/bengio/6/{}.txt'.format(i)
print(filename)
with open(filename) as file:
data = file.readlines()
word_seq = []
label = []
for j in range(len(data)):
data[j] = data[j].strip()
splitted = data[j].split(';')
word_seq.append(splitted[:-1])
def classify(bow=False,
plot=False,
multinomial_nb=False,
bernoulli_nb=False,
knn=False,
support_vm=False,
svm_sgd=False,
decision_tree=False,
random_forest=False,
persist=False,
logistic_regression=False):
"""
Validate the classifier against unseen resources using k-fold cross validation
"""
if multinomial_nb:
clf_title = 'Multinomial NB'
if bow:
vectorizer_title = 'Bag-of-Words'
counts, targets = FeatureExtractor.bow(
max_df=0.25, ngram_range=(1, 3)).extract_features_from_csv
clf = MultinomialNB(alpha=1e-05)
else:
vectorizer_title = 'TF-IDF'
counts, targets = FeatureExtractor.tfidf(
analyzer='char',
max_df=0.5,
ngram_range=(1, 4),
norm='l1',
sublinear_tf=False,
use_idf=True).extract_features_from_csv
clf = MultinomialNB(alpha=1e-07)
elif bernoulli_nb:
clf_title = 'Bernoulli NB'
if bow:
vectorizer_title = 'Bag-of-Words'
counts, targets = FeatureExtractor.bow(
max_df=0.25, ngram_range=(1, 3)).extract_features_from_csv
clf = BernoulliNB(alpha=1e-05)
else:
vectorizer_title = 'TF-IDF'
counts, targets = FeatureExtractor.tfidf(
analyzer='word',
max_df=0.25,
ngram_range=(1, 3),
norm='l1',
sublinear_tf=True,
use_idf=True).extract_features_from_csv
clf = BernoulliNB(alpha=1e-05)
elif knn:
clf_title = 'K-Nearest-Neighbour'
if bow:
vectorizer_title = 'Bag-of-Words'
counts, targets = FeatureExtractor.bow(
max_df=0.5, ngram_range=(1, 1)).extract_features_from_csv
clf = KNeighborsClassifier(weights='distance',
n_neighbors=2,
leaf_size=20,
algorithm='auto')
else:
vectorizer_title = 'TF-IDF'
counts, targets = FeatureExtractor.tfidf(
analyzer='word',
max_df=0.5,
ngram_range=(1, 1),
norm='l1',
sublinear_tf=True,
use_idf=True).extract_features_from_csv
clf = KNeighborsClassifier(weights='distance',
n_neighbors=2,
leaf_size=20,
algorithm='auto')
elif support_vm:
clf_title = 'Support Vector Machine'
if bow:
vectorizer_title = 'Bag-of-Words'
counts, targets = FeatureExtractor.tfidf(
max_df=0.5, ngram_range=(1, 2)).extract_features_from_csv
clf = SVC(kernel='sigmoid',
C=100,
gamma=0.01,
decision_function_shape='ovo',
probability=True)
else:
vectorizer_title = 'TF-IDF'
counts, targets = FeatureExtractor.tfidf(
analyzer='char',
max_df=1.0,
ngram_range=(1, 4),
norm='l2',
use_idf=False,
sublinear_tf=True).extract_features_from_csv
clf = SVC(kernel='sigmoid',
C=10,
gamma=1.4,
decision_function_shape='ovo',
probability=True)
elif svm_sgd:
clf_title = 'SVM (SGD)'
if bow:
vectorizer_title = 'Bag-of-Words'
counts, targets = FeatureExtractor.bow(
max_df=0.25, ngram_range=(1, 4)).extract_features_from_csv
target_ints = []
for target in targets:
target_ints.append(category_names_reverse.index(target))
class_weights = get_classweight(targets)
clf = SGDClassifier(loss='squared_hinge',
penalty='l1',
alpha=1e-05,
max_iter=100,
tol=0.2,
class_weight=class_weights)
else:
vectorizer_title = 'TF-IDF'
counts, targets = FeatureExtractor.tfidf(
ngram_range=(1, 4),
max_df=0.25,
use_idf=False,
sublinear_tf=True).extract_features_from_csv
target_ints = []
for target in targets:
target_ints.append(category_names_reverse.index(target))
class_weights = get_classweight(targets)
clf = SGDClassifier(loss='hinge',
penalty='l1',
alpha=1e-05,
max_iter=100,
tol=0.2,
class_weight=class_weights)
else:
print('Please provide a classifer algorithm')
return
clf.fit(counts, targets)
if persist:
joblib.dump(clf, clf_title + '.pkl')
# scores
ac_scores = []
f1_scores = []
prec_scores = []
rec_scores = []
confusion = np.array([[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]])
cv = list(
StratifiedKFold(n_splits=15, random_state=1).split(counts, targets))
for k, (train_indices, test_indices) in enumerate(cv):
train_text = counts[train_indices]
train_y = targets[train_indices]
test_text = counts[test_indices]
test_y = targets[test_indices]
clf.fit(train_text, train_y)
predictions = clf.predict(test_text)
confusion += confusion_matrix(test_y, predictions)
ac_scores.append(accuracy_score(test_y, predictions))
f1_scores.append(f1_score(test_y, predictions, average="macro"))
prec_scores.append(
precision_score(test_y, predictions, average="macro"))
rec_scores.append(recall_score(test_y, predictions, average="macro"))
print("---------------------- \nResults for ", clf_title, " with ",
vectorizer_title, ":")
print("K-Folds Accuracy-score: ", sum(ac_scores) / len(ac_scores))
print("K-Folds F1-score: ", sum(f1_scores) / len(f1_scores))
print("K-Folds Precision-score: ", sum(prec_scores) / len(prec_scores))
print("K-Folds Recall-score: ", sum(rec_scores) / len(rec_scores))
print("CV accuracy : %.3f +/- %.3f" %
(np.mean(ac_scores), np.std(ac_scores)))
labels = [
'Barent.', 'Finanzen', 'Freiz.&Lifes.', 'Lebensh.', 'Mob.&Verk.',
'Versich.', 'Wohn.&Haus.'
]
if plot:
Plotter.plot_and_show_confusion_matrix(confusion,
labels,
normalize=True,
title=clf_title,
save=True)
from feature_extraction import FeatureExtractor
from keras.models import load_model
from keras import backend as K
import tensorflow as tf
from keras.preprocessing.text import text_to_word_sequence
import json
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.5
K.tensorflow_backend.set_session(tf.Session(config=config))
ta = BasicTextAugmentation()
vocab = list_from_file('resource/vocabulary.txt')
w2v_pathname = 'resource/w2v_path.txt'
fe = FeatureExtractor(5)
fe.set_w2v(w2v_pathname, 500, keep_alive=True)
class BengioAugmenter():
def __init__(self, model_filename):
self.gram = 5
self.model = load_model(model_filename)
def prob(self, sequences):
for i in range(len(sequences)):
if sequences[i] not in vocab:
if sequences[i].isdigit():
sequences[i] = '<NUM>'
else:
sequences[i] = '<UNK>'
feature = sequences[:-1]
concat = Concatenate()([pooled_conv_dropped_a, pooled_conv_dropped_b])
output = TimeDistributed(Dense(units = 1,
activation = 'sigmoid',
))(concat)
model = Model(my_input, output)
model.compile(loss='categorical_hinge',
optimizer = 'adagrad',
metrics = ['accuracy'])
return model
# print(model.summary())
fe = FeatureExtractor(6)
fe.set_w2v(w2v_pathname, 500, keep_alive=True)
for epoch in range(2, 6):
model = load_model('model/cnw100_{}.h5'.format(epoch-1))
for i in range(1,1001):
true_filename = 'data/batch/bengio/6/{}.txt'.format(i)
false_filename = 'data/batch/cnw/6/{}.txt'.format(i)
with open(true_filename) as file:
true_data = file.readlines()
with open(false_filename) as file:
false_data = file.readlines()
true_word_seq = []
false_word_seq = []
def __init__(self):
self.face_detector = FaceDetector()
self.preprocessor = Preprocessor()
self.extractor = FeatureExtractor()
class Mic():
"""
Mic class
"""
def __init__(self,
classifier,
feature_params,
mic_params,
is_audio_record=False):
# arguments
self.classifier = classifier
self.feature_params = feature_params
self.mic_params = mic_params
self.is_audio_record = is_audio_record
# windowing params
self.N, self.hop = int(
feature_params['N_s'] * feature_params['fs']), int(
feature_params['hop_s'] * feature_params['fs'])
# queue
self.q = queue.Queue()
# collector
self.collector = Collector(
N=self.N,
hop=self.hop,
frame_size=self.feature_params['frame_size'],
update_size=self.mic_params['update_size'],
frames_post=self.mic_params['frames_post'],
is_audio_record=self.is_audio_record)
# feature extractor
self.feature_extractor = FeatureExtractor(
self.feature_params['fs'],
N=self.N,
hop=self.hop,
n_filter_bands=self.feature_params['n_filter_bands'],
n_ceps_coeff=self.feature_params['n_ceps_coeff'],
frame_size=self.feature_params['frame_size'])
# select microphone yourself (usually not necessary)
if mic_params['select_device']:
sd.default.device = self.mic_params['device']
# determine downsample
self.downsample = self.mic_params['fs_device'] // self.feature_params[
'fs']
# show devices
print("\ndevice list: \n", sd.query_devices())
# setup stream sounddevice
self.stream = sd.InputStream(samplerate=self.mic_params['fs_device'],
blocksize=int(self.hop * self.downsample),
channels=self.mic_params['channels'],
callback=self.callback_mic)
def callback_mic(self, indata, frames, time, status):
"""
Input Stream Callback
"""
if status:
print(status)
#self.q.put(indata[:, 0].copy())
# add to queue with primitive downsampling
self.q.put(indata[::self.downsample, 0].copy())
def clear_mic_queue(self):
"""
clear the queue after classification
"""
# process data
for i in range(self.q.qsize()):
# get chunk
x = self.q.get()
# onset and energy archiv
e, _ = onset_energy_level(x, alpha=self.mic_params['energy_thres'])
# update collector
self.collector.x_all = np.append(self.collector.x_all, x)
self.collector.e_all = np.append(self.collector.e_all, e)
def read_mic_data(self):
"""
reads the input from the queue
"""
# init
x_collect = np.empty(shape=(0), dtype=np.float32)
e_collect = np.empty(shape=(0), dtype=np.float32)
# onset flag
is_onset = False
# process data
if self.q.qsize():
for i in range(self.q.qsize()):
# get data
x = self.q.get()
# append chunk
x_collect = np.append(x_collect, x.copy())
# append energy level
e_collect = np.append(e_collect, 1)
# detect onset
e_onset, is_onset = onset_energy_level(
x_collect, alpha=self.mic_params['energy_thres'])
# collection update
self.collector.update_collect(x_collect.copy(),
e=e_collect.copy() * e_onset,
on=is_onset)
return is_onset
def update_read_command(self):
"""
update mic
"""
# read chunk
is_onset = self.read_mic_data()
# onset was detected
if is_onset:
# start collection of items
self.collector.start_collecting()
# collection is full
if self.collector.is_full():
# read out collection
x_onset = self.collector.read_collection()
# extract features
mfcc_bon, bon_pos = self.feature_extractor.extract_mfcc39(x_onset)
# classify collection
y_hat, label = self.classifier.classify_sample(mfcc_bon)
# plot
plot_mfcc_profile(
x_onset[bon_pos *
self.hop:(bon_pos +
self.feature_params['frame_size']) *
self.hop],
self.feature_params['fs'],
self.N,
self.hop,
mfcc_bon,
frame_size=self.feature_params['frame_size'],
plot_path=self.mic_params['plot_path'],
name='collect-{}_label-{}'.format(
self.collector.collection_counter, label),
enable_plot=self.mic_params['enable_plot'])
# clear read queue
self.clear_mic_queue()
return y_hat
return None
def stop_mic_condition(self, time_duration):
"""
stop mic if time duration is exceeded (memory issue in recording)
"""
return (self.collector.x_all.shape[0] >=
(time_duration *
self.feature_params['fs'])) and self.is_audio_record
def save_audio_file(self):
"""
saves collection to audio file
"""
# has not recorded audio
if not self.is_audio_record:
print("***you did not set the record flag!")
return
import soundfile
# save audio
soundfile.write('{}out_audio.wav'.format(self.mic_params['plot_path']),
self.collector.x_all,
self.feature_params['fs'],
subtype=None,
endian=None,
format=None,
closefd=True)
class Mic():
"""
Mic class
"""
def __init__(self,
classifier,
mic_params,
is_audio_record=False,
root_path='./'):
# arguments
self.classifier = classifier
self.mic_params = mic_params
self.is_audio_record = is_audio_record
self.root_path = root_path
# plot path
self.plot_path = self.root_path + self.mic_params['plot_path']
# create folder for plot path
create_folder([self.plot_path])
# shortcuts
self.feature_params = classifier.feature_params
# feature extractor
self.feature_extractor = FeatureExtractor(self.feature_params)
# windowing params
self.N, self.hop = self.feature_extractor.N, self.feature_extractor.hop
# queue
self.q = queue.Queue()
# collector
self.collector = Collector(
N=self.N,
hop=self.hop,
frame_size=self.feature_params['frame_size'],
update_size=self.mic_params['update_size'],
frames_post=self.mic_params['frames_post'],
is_audio_record=self.is_audio_record)
# device
self.device = sd.default.device[0] if not self.mic_params[
'select_device'] else self.mic_params['device']
# determine downsample
self.downsample = self.mic_params['fs_device'] // self.feature_params[
'fs']
# get input devices
self.input_dev_dict = self.extract_devices()
# show devices
print("\ndevice list: \n", sd.query_devices())
print("\ninput devs: ", self.input_dev_dict.keys())
# stream
self.stream = None
# change device flag
self.change_device_flag = False
def load_user_settings(self, user_setting_file):
"""
load user settings like device and energy threshold
"""
# load user settings
user_settings = yaml.safe_load(open(
user_setting_file)) if os.path.isfile(user_setting_file) else {}
# update mic params
self.mic_params.update(user_settings)
# device
self.device = sd.default.device[0] if not self.mic_params[
'select_device'] else self.mic_params['device']
def init_stream(self):
"""
init stream
"""
self.stream = sd.InputStream(device=self.device,
samplerate=self.mic_params['fs_device'],
blocksize=int(self.hop * self.downsample),
channels=self.mic_params['channels'],
callback=self.callback_mic)
self.change_device_flag = False
def change_device(self, device):
"""
change to device
"""
self.change_device_flag = True
self.device = device
def extract_devices(self):
"""
extract only input devices
"""
return {
i: dev
for i, dev in enumerate(sd.query_devices())
if dev['max_input_channels']
}
def callback_mic(self, indata, frames, time, status):
"""
Input Stream Callback
"""
# debug
if status: print(status)
#self.q.put(indata[:, 0].copy())
# add to queue with primitive downsampling
self.q.put(indata[::self.downsample, 0].copy())
def clear_mic_queue(self):
"""
clear the queue after classification
"""
# process data
for i in range(self.q.qsize()):
# get chunk
x = self.q.get()
# onset and energy archiv
e, _ = self.onset_energy_level(
x, alpha=self.mic_params['energy_thresh'])
# update collector
self.collector.x_all = np.append(self.collector.x_all, x)
self.collector.e_all = np.append(self.collector.e_all, e)
def read_mic_data(self):
"""
reads the input from the queue
"""
# init
x_collect = np.empty(shape=(0), dtype=np.float32)
e_collect = np.empty(shape=(0), dtype=np.float32)
# onset flag
is_onset = False
# process data
if self.q.qsize():
for i in range(self.q.qsize()):
# get data
x = self.q.get()
# append chunk
x_collect = np.append(x_collect, x.copy())
# append energy level
e_collect = np.append(e_collect, 1)
# detect onset
e_onset, is_onset = self.onset_energy_level(
x_collect, alpha=self.mic_params['energy_thresh'])
# collection update
self.collector.update_collect(x_collect.copy(),
e=e_collect.copy() * e_onset,
on=is_onset)
return is_onset
def onset_energy_level(self, x, alpha=0.01):
"""
onset detection with energy level
x: [n x c]
n: samples
c: channels
"""
e = x.T @ x / len(x)
return e, e > alpha
def update_read_command(self):
"""
update mic
"""
# read chunk
is_onset = self.read_mic_data()
# onset was detected
if is_onset:
# start collection of items
self.collector.start_collecting()
# collection is full
if self.collector.is_full():
# read out collection
x_onset = self.collector.read_collection()
# extract features
mfcc_bon, bon_pos = self.feature_extractor.extract_mfcc(x_onset)
# classify collection
y_hat, label = self.classifier.classify(mfcc_bon)
# plot
plot_mfcc_profile(
x_onset[bon_pos *
self.hop:(bon_pos +
self.feature_params['frame_size']) *
self.hop],
self.feature_params['fs'],
self.N,
self.hop,
mfcc_bon,
frame_size=self.feature_params['frame_size'],
plot_path=self.plot_path,
name='collect-{}_label-{}'.format(
self.collector.collection_counter, label),
enable_plot=self.mic_params['enable_plot'])
# clear read queue
self.clear_mic_queue()
return label
return None
def stop_mic_condition(self, time_duration):
"""
stop mic if time duration is exceeded (memory issue in recording)
"""
return (self.collector.x_all.shape[0] >=
(time_duration *
self.feature_params['fs'])) and self.is_audio_record
def save_audio_file(self):
"""
saves collection to audio file
"""
# has not recorded audio
if not self.is_audio_record:
print("***you did not set the record flag!")
return
import soundfile
# save audio
soundfile.write('{}out_audio.wav'.format(self.plot_path),
self.collector.x_all,
self.feature_params['fs'],
subtype=None,
endian=None,
format=None,
closefd=True)
sys.path.append("../")
from common import create_folder
from feature_extraction import FeatureExtractor
# plot path
#plot_path = './ignore/plots/fe/'
# create folder
#create_folder([plot_path])
# yaml config file
cfg = yaml.safe_load(open("../config.yaml"))
# init feature extractor
feature_extractor = FeatureExtractor(cfg['feature_params'])
# --
# params
fs = 16000
N = 400
hop = 160
n_filter_bands = 16
n_ceps_coeff = 12
# --
# test signal
# generate test signal
x = some_test_signal(fs, t=1, save_to_file=False)
def run_benchmarks(training_data_dir: str, results_dir: str):
"""
Runs benchmark tests and records observations.
:param training_data_dir: The directory which contains the training
grid files and labels.
:param results_dir: The directory into which results should be
written. The results are recorded in a file
named `benchmark.csv` with each model's MCC,
training time, and prediction time recorded.
The method will also produce confusion
matrices for each model benchmarked for
debugging.
"""
if os.path.isdir(results_dir):
shutil.rmtree(results_dir)
os.makedirs(results_dir, exist_ok=True)
training_pdf = os.path.join(training_data_dir, "training_data.pdf")
labels_file = os.path.join(training_data_dir,
"training_labels.txt")
conf_mat_figsize = (14, 14)
if not os.path.isfile(training_pdf):
raise IOError("Training input PDF not found.")
if not os.path.isfile(labels_file):
raise IOError("Training labels file not found.")
training_file = fe.TrainingFileSpec(
training_pdf, fe.DEFAULT_GRID_SPEC,
labels_file, dpi=96, white_threshold=fe.DEFAULT_WHITE_THRESHOLD)
angles = np.arange(-5, 5, 1)
feature_vectors, feature_labels = \
FeatureExtractor.extract_labelled_feature_vectors(training_file,
rotations=angles)
train_ft, test_ft, train_lb, test_lb = train_test_split(feature_vectors,
feature_labels,
test_size=0.33,
random_state=81)
all_labels = [lbl for lbl in np.unique(feature_labels)]
# We still store our benchmarks here
bench = pd.DataFrame(columns=['Model', 'MCC', 'TestTime', 'PredTime'])
models = []
# 1 hidden layer with various sizes
for i in [26, 52, 80, 100]:
models.append(MLPLearningModel(name="MLP (%i)" % i,
hidden_layer_sizes=(i,),
max_iter=1000))
# 2 hidden layers with various sizes
for i in [10, 26, 100]:
for j in [10, 26, 100]:
models.append(MLPLearningModel(name="MLP (%i, %i)" % (i, j),
hidden_layer_sizes=(i, j,),
max_iter=1000))
# KNN models with various k values
for k in [1, 5, 10, 15, 30]:
models.append(KNNLearningModel(name="KNN (k=%i)" % k, k=k))
for model in models:
begin = time.time()
model.train(train_ft, train_lb)
test_time = time.time() - begin
begin = time.time()
predictions = model.predict(test_ft)
pred_time = time.time() - begin
mcc = matthews_corrcoef(y_pred=predictions, y_true=test_lb)
bench = bench.append({'Model': model.name,
'MCC': mcc,
'TestTime': test_time,
'PredTime': pred_time}, ignore_index=True)
conf_mat = confusion_matrix(y_pred=predictions, y_true=test_lb,
normalize='true')
plot_title = "Confusion matrix for %s" % model.name
fig = plothelp.plot_confusion_matrix(data=conf_mat,
title=plot_title,
figsize=conf_mat_figsize,
dpi=120,
labels=all_labels)
plot_file_name = model.name.replace(" ", "_").replace(",", "_").\
replace("(", "_").replace(")", "_").replace("=", "_") + ".png"
fig.savefig(os.path.join(results_dir, plot_file_name))
pyplot.close(fig)
bench.to_csv(
path_or_buf=os.path.join(results_dir, "benchmark.csv"),
header=True, index=False)
class Entity(BaseEntity):
@staticmethod
def add_parser_arguments(parser):
parser.add_argument("--rotation-parametrization", "-r",
choices=["vectors", "quaternion"])
parser.add_argument("--naive-quaternion-interpolation", action="store_true")
parser.add_argument("--max-angular-step", type=float, default=1.)
parser.add_argument("--translate", action="store_true")
parser.add_argument("--translation-weight", type=float, default=1.)
parser.add_argument("--friction", action="store_true")
parser.add_argument("--confinement", action="store_true")
parser.add_argument("--confinement-rate", type=float, default=0.03)
parser.add_argument("--pose-scale", type=float, default=.5)
parser.add_argument("--random-slide", type=float, default=0.0)
parser.add_argument("--circle-slide", action="store_true")
parser.add_argument("--left-foot")
parser.add_argument("--left-hand")
parser.add_argument("--left-forearm")
parser.add_argument("--left-shoulder")
parser.add_argument("--left-knee")
parser.add_argument("--left-hip")
parser.add_argument("--right-foot")
parser.add_argument("--right-hand")
parser.add_argument("--right-forearm")
parser.add_argument("--right-shoulder")
parser.add_argument("--right-knee")
parser.add_argument("--right-hip")
parser.add_argument("--torso")
parser.add_argument("--neck")
parser.add_argument("--head")
def __init__(self, *args, **kwargs):
BaseEntity.__init__(self, *args, **kwargs)
self.rotation_parametrization = rotation_parametrizations[
self.args.rotation_parametrization]
self._create_parameter_info_table()
if self.z_up:
self._vertical_axis = "z"
self._coordinate_up = 2
else:
self._vertical_axis = "y"
self._coordinate_up = 1
self._max_angular_step = self.args.max_angular_step
self._normalized_constrainers = self._create_constrainers()
self._unnormalized_constrainers = self._create_constrainers()
self.modified_root_vertical_orientation = None
self._last_root_vertical_orientation = None
self._rotation_interpolators = {}
self._enable_friction = self.args.friction
if hasattr(self.args, "enable_features") and self.args.enable_features:
self.feature_extractor = FeatureExtractor(self._coordinate_up)
self._bvh_joint_name_for_feature = self._get_bvh_joint_names_for_features_from_args()
def _get_bvh_joint_names_for_features_from_args(self):
result = {}
for joint_name in self.feature_extractor.INPUT_JOINTS:
bvh_joint_name = getattr(self.args, joint_name)
if bvh_joint_name is None:
raise Exception("please specify BVH joint for %s" % joint_name)
result[joint_name] = bvh_joint_name
return result
def _create_constrainers(self):
return Constrainers(
self._coordinate_up,
enable_friction=(self.args.friction and not (
hasattr(self.args, "show_all_feature_matches") and self.args.show_all_feature_matches)),
enable_floor=self.floor,
enable_confinement=self.args.confinement,
confinement_rate=self.args.confinement_rate,
enable_random_slide=(self.args.random_slide > 0),
random_slide=self.args.random_slide,
enable_circle_slide=self.args.circle_slide)
def reset_constrainers(self):
self._normalized_constrainers.reset()
self._unnormalized_constrainers.reset()
def _create_parameter_info_table(self):
self._parameter_info = []
root_joint_definition = self.bvh_reader.get_hierarchy().get_root_joint_definition()
self._extend_parameter_info_table_recurse(root_joint_definition)
def _extend_parameter_info_table_recurse(self, joint_definition):
if not joint_definition.has_parent and self.args.translate:
self._parameter_info.extend(
[{"category": "translate", "component": "X"},
{"category": "translate", "component": "Y"},
{"category": "translate", "component": "Z"}])
if joint_definition.has_rotation and not joint_definition.has_static_rotation:
for n in range(self.rotation_parametrization.num_parameters):
self._parameter_info.append({"category": joint_definition.name, "component": str(n)})
for child_definition in joint_definition.child_definitions:
self._extend_parameter_info_table_recurse(child_definition)
def parameter_info(self, index):
return self._parameter_info[index]
def get_value_length(self):
return len(self._parameter_info)
def get_value(self):
self.bvh_reader.set_pose_from_time(self.pose, self._t * self.args.bvh_speed)
return self._joint_to_parameters(self.pose.get_root_joint())
def get_value_from_frame(self, frame, **kwargs):
self.bvh_reader.set_pose_from_frame(self.pose, frame, **kwargs)
return self._joint_to_parameters(self.pose.get_root_joint())
def get_random_value(self):
self.bvh_reader.set_pose_from_time(self.pose,
random.uniform(0, self.bvh_reader.get_duration()))
return self._joint_to_parameters(self.pose.get_root_joint())
def get_duration(self):
return self.bvh_reader.get_duration() / self.args.bvh_speed
def _joint_to_parameters(self, root_joint):
parameters = []
self._add_joint_parameters_recurse(root_joint, parameters)
return parameters
def _add_joint_parameters_recurse(self, joint, parameters):
if not joint.definition.has_parent and self.args.translate:
self._add_joint_translation_parameters(joint, parameters)
if joint.definition.has_rotation and not joint.definition.has_static_rotation:
self._add_joint_rotation_parameters(joint, parameters)
for child in joint.children:
self._add_joint_parameters_recurse(child, parameters)
def _add_joint_translation_parameters(self, joint, parameters):
vertex = joint.get_vertex()
normalized_vector = self.bvh_reader.normalize_vector(vertex)
weighted_vector = self.args.translation_weight * normalized_vector
parameters.extend(weighted_vector)
def _add_joint_rotation_parameters(self, joint, parameters):
rotation_parameters = self.rotation_parametrization.rotation_to_parameters(
joint.rotation)
parameters.extend(rotation_parameters)
def process_input(self, parameters):
return self._parameters_to_scaled_normalized_vertices(parameters)
def process_output(self, parameters):
return self._constrained_output_vertices(parameters)
def process_io_blend(self, parameters):
return self._constrained_output_vertices(parameters)
def _constrained_output_vertices(self, parameters):
vertices = self._parameters_to_scaled_normalized_vertices(parameters)
vertices = self._normalized_constrainers.constrain(vertices)
return vertices
def _parameters_to_scaled_normalized_vertices(self, parameters):
self._set_pose_from_parameters(parameters)
root_joint = self.pose.get_root_joint()
vertices = root_joint.get_vertices()
normalized_vertices = [
self.bvh_reader.normalize_vector_without_translation(vertex) * self.args.pose_scale
for vertex in vertices]
return normalized_vertices
def _set_pose_from_parameters(self, parameters):
self._parameters_to_joint_recurse(parameters, self.pose.get_root_joint())
self.bvh_reader.get_hierarchy().update_pose_world_positions(self.pose)
def _parameters_to_joint_recurse(self, parameters, joint, parameter_index=0):
if not joint.definition.has_parent and self.args.translate:
parameter_index = self._parameters_to_joint_translation(parameters, joint, parameter_index)
if joint.definition.has_rotation and not joint.definition.has_static_rotation:
parameter_index = self._parameters_to_joint_rotation(parameters, joint, parameter_index)
for child in joint.children:
parameter_index = self._parameters_to_joint_recurse(parameters, child, parameter_index)
return parameter_index
def _parameters_to_joint_translation(self, parameters, joint, parameter_index):
weighted_vector = parameters[parameter_index:parameter_index+3]
parameter_index += 3
if self.args.translation_weight == 0:
joint.translation = [0, 0, 0]
else:
normalized_vector = numpy.array(weighted_vector) / self.args.translation_weight
joint.translation = self.bvh_reader.skeleton_scale_vector(normalized_vector)
return parameter_index
def _parameters_to_joint_rotation(self, parameters, joint, parameter_index):
rotation_parameters = parameters[
parameter_index:parameter_index+
self.rotation_parametrization.num_parameters]
parameter_index += self.rotation_parametrization.num_parameters
radians = self.rotation_parametrization.parameters_to_rotation(
rotation_parameters, joint.definition.axes)
if joint.parent is None:
radians = self._process_root_orientation(joint, radians)
joint.angles = radians
return parameter_index
def _process_root_orientation(self, root_joint, radians):
return self._process_vertical_axis(radians, root_joint.definition.axes)
def _process_vertical_axis(self, euler_angles, axes):
if axes[1] == self._vertical_axis:
return self._process_vertical_orientation_as_first_axis(euler_angles)
else:
if self._vertical_axis == "y":
axes_with_vertical_first = "ryxz"
elif self._vertical_axis == "z":
axes_with_vertical_first = "rzxy"
euler_angles_vertical_axis_first = euler_from_quaternion(
quaternion_from_euler(*euler_angles, axes=axes),
axes=axes_with_vertical_first)
euler_angles_vertical_axis_first_reoriented = \
self._process_vertical_orientation_as_first_axis(
euler_angles_vertical_axis_first)
return euler_from_quaternion(
quaternion_from_euler(
*euler_angles_vertical_axis_first_reoriented,
axes=axes_with_vertical_first),
axes=axes)
def _process_vertical_orientation_as_first_axis(self, euler_angles):
euler_angles = list(euler_angles)
self._last_root_vertical_orientation = euler_angles[0]
if self.modified_root_vertical_orientation is not None:
euler_angles[0] = self.modified_root_vertical_orientation
return euler_angles
def get_last_root_vertical_orientation(self):
return self._last_root_vertical_orientation
def parameters_to_processed_pose(self, parameters, output_pose):
self._set_pose_from_parameters(parameters)
root_joint = self.pose.get_root_joint()
vertices = root_joint.get_vertices()
vertices = self._unnormalized_constrainers.constrain(vertices)
self.bvh_reader.get_hierarchy().set_pose_vertices(
output_pose, vertices, not ASSUME_NO_TRANSLATIONAL_OFFSETS_IN_NON_ROOT)
def extract_features(self, pose):
positions = [
pose.get_joint(self._bvh_joint_name_for_feature[joint_name]).worldpos
for joint_name in self.feature_extractor.INPUT_JOINTS]
return self.feature_extractor.extract_features(*positions)
def interpolate(self, parameters1, parameters2, amount):
result = []
self._interpolate_recurse(
parameters1, parameters2, amount, self.pose.get_root_joint(), result)
return result
def _interpolate_recurse(self, parameters1, parameters2, amount, joint, result, parameter_index=0):
if not joint.definition.has_parent and self.args.translate:
interpolated_translation, parameter_index = self._interpolate_translation(
parameters1, parameters2, amount, joint, parameter_index)
result += interpolated_translation
if joint.definition.has_rotation and not joint.definition.has_static_rotation:
interpolated_rotation, parameter_index = self._interpolate_rotation(
parameters1, parameters2, amount, joint, parameter_index)
result += interpolated_rotation
for child in joint.children:
parameter_index = self._interpolate_recurse(
parameters1, parameters2, amount, child, result, parameter_index)
return parameter_index
def _interpolate_translation(self, parameters1, parameters2, amount, joint, parameter_index):
vector1 = parameters1[parameter_index:parameter_index+3]
vector2 = parameters2[parameter_index:parameter_index+3]
parameter_index += 3
result = list(numpy.array(vector2) * amount + numpy.array(vector1) * (1-amount))
return result, parameter_index
def _interpolate_rotation(self, parameters1, parameters2, amount, joint, parameter_index):
rotation_params1 = numpy.array(parameters1[
parameter_index:parameter_index + self.rotation_parametrization.num_parameters])
rotation_params2 = numpy.array(parameters2[
parameter_index:parameter_index + self.rotation_parametrization.num_parameters])
parameter_index += self.rotation_parametrization.num_parameters
interpolator = self._get_rotation_interpolator(joint, rotation_params1, rotation_params2)
try:
result = list(interpolator.interpolate(rotation_params1, rotation_params2, amount))
except ZeroNormedQuaternion as exception:
print "WARNING: %s (joint: %s)" % (exception, joint.definition.name)
result = rotation_params1
return result, parameter_index
def _get_rotation_interpolator(self, joint, r1, r2):
if self.rotation_parametrization == EulerToQuaternion:
return self._get_quaternion_interpolator(joint, r1, r2)
else:
return linear_interpolator
def _get_quaternion_interpolator(self, joint, r1, r2):
if self.args.naive_quaternion_interpolation:
return static_quaternions_interpolator
else:
joint_index = joint.definition.index
if joint_index not in self._rotation_interpolators:
interpolator = DynamicQuaternionsInterpolator(joint.definition.name)
interpolator.set_max_angular_step(self._max_angular_step)
self._rotation_interpolators[joint_index] = interpolator
return self._rotation_interpolators[joint_index]
def set_friction(self, enable_friction):
self._enable_friction = enable_friction
self._normalized_constrainers.set_friction(enable_friction)
self._unnormalized_constrainers.set_friction(enable_friction)
def get_friction(self):
return self._enable_friction
def set_confinement(self, enable_confinement):
self._normalized_constrainers.set_confinement(enable_confinement)
self._unnormalized_constrainers.set_confinement(enable_confinement)
def set_confinement_rate(self, rate):
self._normalized_constrainers.set_confinement_rate(rate)
self._unnormalized_constrainers.set_confinement_rate(rate)
def set_confinement_target_position(self, target_position):
self._normalized_constrainers.set_confinement_target_position(target_position)
self._unnormalized_constrainers.set_confinement_target_position(target_position)
def set_max_angular_step(self, max_angular_step):
self._max_angular_step = max_angular_step
for interpolator in self._rotation_interpolators.values():
interpolator.set_max_angular_step(max_angular_step)
def showcase_wavs(cfg,
raw_plot=True,
spec_plot=True,
mfcc_plot=True,
show_plot=False):
"""
showcase wavs
"""
# plot path
plot_path = '../docu/thesis/3_signal/figs/'
# change params
feature_params = cfg['feature_params'].copy()
feature_params['n_ceps_coeff'] = 32
feature_params['norm_features'] = True
# init feature extractor
feature_extractor = FeatureExtractor(feature_params)
# wav, anno dir
wav_dir, anno_dir = '../ignore/my_recordings/showcase_wavs/', '../ignore/my_recordings/showcase_wavs/annotation/'
# analyze some wavs
for wav, anno in zip(glob(wav_dir + '*.wav'),
glob(anno_dir + '*.TextGrid')):
# info
print("\nwav: ", wav), print("anno: ", anno)
# load file
x, _ = librosa.load(wav, sr=feature_params['fs'])
# raw waveform
if raw_plot:
plot_waveform(x,
feature_params['fs'],
anno_file=anno,
hop=feature_extractor.hop,
plot_path=plot_path,
name='signal_raw_' +
wav.split('/')[-1].split('.')[0] + '_my',
show_plot=show_plot)
# spectogram
if spec_plot:
plot_spec_profile(x,
feature_extractor.calc_spectogram(x).T,
feature_params['fs'],
feature_extractor.N,
feature_extractor.hop,
anno_file=anno,
plot_path=plot_path,
title=wav.split('/')[-1].split('.')[0] + '_my',
name='signal_spec-lin_' +
wav.split('/')[-1].split('.')[0] + '_my',
show_plot=show_plot)
plot_spec_profile(x,
feature_extractor.calc_spectogram(x).T,
feature_params['fs'],
feature_extractor.N,
feature_extractor.hop,
log_scale=True,
anno_file=anno,
plot_path=plot_path,
title=wav.split('/')[-1].split('.')[0] + '_my',
name='signal_spec-log_' +
wav.split('/')[-1].split('.')[0] + '_my',
show_plot=show_plot)
# mfcc
if mfcc_plot:
mfcc, bon_pos = feature_extractor.extract_mfcc(
x, reduce_to_best_onset=False)
plot_mfcc_profile(x,
cfg['feature_params']['fs'],
feature_extractor.N,
feature_extractor.hop,
mfcc,
anno_file=anno,
sep_features=True,
bon_pos=bon_pos,
frame_size=cfg['feature_params']['frame_size'],
plot_path=plot_path,
name='signal_mfcc_' +
wav.split('/')[-1].split('.')[0] + '_my',
close_plot=False,
show_plot=show_plot)
mlp_model = MLPLearningModel(name="default",
hidden_layer_sizes=(80, ), max_iter=500)
training_pdf = args.symbols_file
labels_file = args.labels_file
if args.reset and os.path.isdir(state_dir):
shutil.rmtree(state_dir)
if not os.path.isfile(training_pdf):
raise IOError("Symbols file %s not found." % training_pdf)
if not os.path.isfile(labels_file):
raise IOError("Labels file %s not found." % labels_file)
training_file = fe.TrainingFileSpec(
training_pdf, fe.DEFAULT_GRID_SPEC,
labels_file, dpi=96, white_threshold=fe.DEFAULT_WHITE_THRESHOLD)
feature_matrix, feature_labels = \
FeatureExtractor.extract_labelled_feature_vectors(
training_file, rotations=np.zeros(1))
if not os.path.isdir(state_dir):
mlp_model.train(feature_matrix, feature_labels)
mlp_model.dump(state_dir)
print("Model trained. Program finished.")
else:
mlp_model.load(state_dir)
mlp_model.improve(feature_matrix, feature_labels)
mlp_model.dump(state_dir)
print("Model improved. Program finished.")
def run(self):
"""Run whole data loading, feature extraction, model training and regressing pipeline."""
if self.mode == "extract":
print("Extracting features")
train = FeatureExtractor("train").run()
dev = FeatureExtractor("dev").run()
print("Saving features")
np.save("saved_features/train_lsr", train.lsr)
np.save("saved_features/train_nlp", train.feats)
np.save("saved_features/train_scores", train.scores)
np.save("saved_features/dev_lsr", dev.lsr)
np.save("saved_features/dev_nlp", dev.feats)
np.save("saved_features/dev_scores", dev.scores)
else: # Load saved extracted features
print("Loading saved features")
split = False if self.full_data else True
train, dev = load_features(split=split, nt=True)
if self.params["upsample"]:
train = self.upsample(train)
train_loader = create_loader(train, self.params["batch_size_train"])
dev_loader = create_loader(dev, validate=True)
# We set a random seed to ensure that results are reproducible.
# Also set a cuda GPU if available
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
GPU = True
else:
GPU = False
device_idx = 0
if GPU:
device = torch.device(
"cuda:" +
str(device_idx) if torch.cuda.is_available() else "cpu")
else:
device = torch.device("cpu")
print(f"Running on {device}")
if self.bUseConv:
model = RecursiveNN(
ModelBlock,
self.params["conv_dict"],
self.params["conv_ffnn_dict"],
BASELINE_dim=self.params["NBaseline"],
)
else:
model = RecursiveNN_Linear(
in_features=2048,
N1=self.params["N1"],
N2=self.params["N2"],
out_features=self.params["out_features"],
dropout=self.params["dropout"],
leaky_relu=self.params["leaky_relu"],
)
model = model.to(device)
weights_initialiser = True
if weights_initialiser:
model.apply(weights_init)
params_net = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("Total number of parameters in Model is: {}".format(params_net))
print(model)
optimizer = optim.Adam(model.parameters(), lr=self.params["lr"])
scheduler = optim.lr_scheduler.StepLR(
optimizer,
step_size=self.params["step_size"],
gamma=self.params["gamma"])
date_string = (str(datetime.datetime.now())[:16].replace(":",
"-").replace(
" ", "-"))
writer = SummaryWriter(logdir + date_string)
print("Running model")
for epoch in range(self.params["epochs"]):
train_model(
model,
train_loader,
optimizer,
epoch,
log_interval=1000,
scheduler=scheduler,
writer=writer,
)
test_loss = test_model(model, dev_loader, epoch, writer=writer)
torch.save(model, "model.pt")
self.model = model
class SpeechCommandsDataset(AudioDataset):
"""
Speech Commands Dataset extraction and set creation
"""
def __init__(self, dataset_cfg, feature_params, collect_wavs=False, verbose=False):
# parent init
super().__init__(dataset_cfg, feature_params, collect_wavs=collect_wavs, verbose=verbose)
# feature extractor
self.feature_extractor = FeatureExtractor(feature_params=self.feature_params)
# short vars
self.N = self.feature_extractor.N
self.hop = self.feature_extractor.hop
# create plot plaths if not already exists
create_folder(list(self.plot_paths.values()))
# recreate
if self.dataset_cfg['recreate'] or not check_folders_existance(self.wav_folders, empty_check=True):
# delete old data
delete_files_in_path(self.wav_folders, file_ext=self.dataset_cfg['file_ext'])
# create folder wav folders
create_folder(self.wav_folders)
# create sets (specific to dataset)
self.create_sets()
# get audio files from sets
self.get_audiofiles()
self.get_annotation_files()
def create_sets(self):
"""
copy wav files from dataset path to wav folders with splitting
"""
# get all class directories except the ones starting with _
class_dirs = glob(self.dataset_path + '[!_]*/')
# run through all class directories
for class_dir in class_dirs:
# extract label
label = class_dir.split('/')[-2]
# get all .wav files
wavs = glob(class_dir + '*' + self.dataset_cfg['file_ext'])
# calculate split numbers in train, test, eval and split position
n_split = (len(wavs) * np.array(self.dataset_cfg['split_percs'])).astype(int)
n_split_pos = np.cumsum(n_split)
# print some info
print("label: [{}]\tn_split: [{}]\ttotal:[{}]".format(label, n_split, np.sum(n_split)))
# actual path
p = 0
# shuffle
if self.dataset_cfg['shuffle_wavs']: np.random.shuffle(wavs)
# run through each path
for i, wav in enumerate(wavs):
# split in new path
if i >= n_split_pos[p]: p += 1
# stop if out of range (happens at rounding errors)
if p >= len(self.wav_folders): break
# wav name
wav_name = wav.split('/')[-1].split('.')[0]
# copy files to folder
copyfile(wav, self.wav_folders[p] + label + str(i) + '--' + wav_name + self.dataset_cfg['file_ext'])
def extract_features(self):
"""
extract mfcc features and save them
"""
print("\n--feature extraction:")
# create folder structure
create_folder(self.feature_folders)
for i, (set_name, wavs, annos) in enumerate(zip(self.set_names, self.set_audio_files, self.set_annotation_files)):
print("{}) extract set: {} with label num: {}".format(i, set_name, len(wavs)))
# examples with splits
n_examples = int(self.dataset_cfg['n_examples'] * self.dataset_cfg['split_percs'][i])
# extract data
x, y, t, index = self.extract_mfcc_data(wavs=wavs, annos=annos, n_examples=n_examples, set_name=set_name) if self.feature_params['use_mfcc_features'] else self.extract_raw_data(wavs=wavs, annos=annos, n_examples=n_examples, set_name=set_name)
# add noise if requested
if self.dataset_cfg['add_noise'] and self.feature_params['use_mfcc_features']: x, y, index = self.add_noise_to_dataset(x, y, index, n_examples)
# print label stats
self.label_stats(y)
# save mfcc data file
np.savez(self.feature_files[i], x=x, y=y, t=t, index=index, params=self.feature_params)
print("--save data to: ", self.feature_files[i])
def extract_mfcc_data(self, wavs, annos, n_examples, set_name=None):
"""
extract mfcc data from wav-files
wavs must be in a 2D-array [[wavs_class1], [wavs_class2]] so that n_examples will work properly
"""
# mfcc_data: [n x m x l], labels and index
mfcc_data, label_data, index_data = np.empty(shape=(0, self.channel_size, self.feature_size, self.frame_size), dtype=np.float64), [], []
# extract class wavs
for class_wavs, class_annos in zip(wavs, annos):
# class annotation file names extraction
class_annos_file_names = [l + i for f, i, l in [self.file_naming_extraction(a, file_ext='.TextGrid') for a in class_annos]]
# number of class examples
num_class_examples = 0
# run through each example in class wavs
for wav in class_wavs:
# extract file namings
file_name, file_index, label = self.file_naming_extraction(wav, file_ext=self.dataset_cfg['file_ext'])
# get annotation if available
anno = None
if label + file_index in class_annos_file_names: anno = class_annos[class_annos_file_names.index(label + file_index)]
# load and pre-process audio
x, wav_is_useless = self.wav_pre_processing(wav)
if wav_is_useless: continue
# print some info
if self.verbose: print("wav: [{}] with label: [{}], samples=[{}], time=[{}]s".format(wav, label, len(x), len(x) / self.feature_params['fs']))
# extract feature vectors [m x l]
mfcc, bon_pos = self.feature_extractor.extract_mfcc(x, reduce_to_best_onset=False)
# collect wavs
if self.collect_wavs: self.pre_wavs.append((librosa.util.normalize(x), label + str(file_index) + '_' + set_name, bon_pos))
# plot mfcc features
plot_mfcc_profile(x, self.feature_params['fs'], self.feature_extractor.N, self.feature_extractor.hop, mfcc, anno_file=anno, onsets=None, bon_pos=bon_pos, mient=None, minreg=None, frame_size=self.frame_size, plot_path=self.plot_paths['mfcc'], name=label + str(file_index) + '_' + set_name, enable_plot=self.dataset_cfg['enable_plot'])
# damaged file check
if self.dataset_cfg['filter_damaged_files']:
# handle damaged files
if self.detect_damaged_file(mfcc, wav): continue
# add to mfcc_data container
mfcc_data = np.vstack((mfcc_data, mfcc[np.newaxis, :, :, bon_pos:bon_pos+self.frame_size]))
label_data.append(label)
index_data.append(label + file_index)
# update number of examples per class
num_class_examples += 1
# stop if desired examples are reached
if num_class_examples >= n_examples: break
return mfcc_data, label_data, None, index_data
def extract_raw_data(self, wavs, annos, n_examples, set_name=None):
"""
raw data extraction
"""
# raw data: [n x m], labels and index
raw_data, label_data, target_data, index_data = np.empty(shape=(0, self.channel_size, self.raw_frame_size), dtype=np.float64), [], np.empty(shape=(0, self.raw_frame_size), dtype=np.int64), []
# extract class wavs
for class_wavs, class_annos in zip(wavs, annos):
# class annotation file names extraction
class_annos_file_names = [l + i for f, i, l in [self.file_naming_extraction(a, file_ext='.TextGrid') for a in class_annos]]
# number of class examples
num_class_examples = 0
# run through each example in class wavs
for wav in class_wavs:
# extract file namings
file_name, file_index, label = self.file_naming_extraction(wav, file_ext=self.dataset_cfg['file_ext'])
# get annotation if available
anno = class_annos[class_annos_file_names.index(label + file_index)] if label + file_index in class_annos_file_names else None
# load and pre-process audio
x, wav_is_useless = self.wav_pre_processing(wav)
if wav_is_useless: continue
# print some info
if self.verbose: print("wav: [{}] with label: [{}], samples=[{}], time=[{}]s".format(wav, label, len(x), len(x) / self.feature_params['fs']))
# extract raw samples from region of energy
raw, bon_pos = self.feature_extractor.get_best_raw_samples(x)
# add dither and do normalization
raw = self.wav_post_processing(raw)
# quantize data
t = self.feature_extractor.quantize(raw)
# plot waveform
if self.dataset_cfg['enable_plot']: plot_waveform(x, self.feature_params['fs'], bon_samples=[bon_pos, bon_pos+self.raw_frame_size], title=label + file_index, plot_path=self.plot_paths['waveform'], name=label + file_index, show_plot=False, close_plot=True)
# collect wavs
if self.collect_wavs: self.pre_wavs.append((librosa.util.normalize(x), label + str(file_index) + '_' + set_name, bon_pos / self.hop))
# add to mfcc_data container
raw_data = np.vstack((raw_data, raw[np.newaxis, :]))
target_data = np.vstack((target_data, t))
label_data.append(label)
index_data.append(label + file_index)
# update number of examples per class
num_class_examples += 1
# stop if desired examples are reached
if num_class_examples >= n_examples: break
return raw_data, label_data, target_data, index_data
def detect_damaged_file(self, mfcc, wav):
"""
detect if file is damaged
"""
# energy calc
#e = np.einsum('ij,ji->j', mfcc, mfcc.T)
#e = e / np.max(e)
# calculate damaged score of energy deltas
if mfcc.shape[1] == 39: z_est, z_lim = np.sum(np.abs(mfcc[0, 37:39, :])), 60
#if mfcc.shape[0] == 39: z_est = np.sum(mfcc[37:39, :] @ mfcc[37:39, :].T)
#else: z_est = np.sum(np.abs(np.diff(mfcc[-1, :])))
#else: z_est = np.diff(mfcc[-1, :]) @ np.diff(mfcc[-1, :]).T
#else: z_est = np.sum(mfcc[0, :])
#else: z_est = np.diff(mfcc[0, :]) @ np.diff(mfcc[0, :]).T
#else: z_est = np.abs(np.diff(mfcc[0, :])) @ mfcc[0, :-1].T
#else: z_est = np.sum(np.diff(mfcc, axis=1) @ np.diff(mfcc, axis=1).T)
#else: z_est = np.sum(e)
#else: z_est = np.diff(e) @ np.diff(e).T
else: z_est, z_lim = mfcc[0, 0, :-1] @ np.abs(np.diff(mfcc[0, 0, :])).T, 3.5
# add score to list
self.damaged_score_list.append(z_est)
# damaged file
is_damaged = z_est > z_lim
# add to damaged file list
if is_damaged: self.damaged_file_list.append((wav, z_est))
# return score and damaged indicator
return is_damaged
data = f[1].data
names = f[1].columns.names
f.close()
print "\n\n\nThese are the column names:\n%s\n\n" % names
Ng = np.where(data.field("type") == 3)[0].size
Ns = np.where(data.field("type") == 6)[0].size
print "\nSDSS says there are %d galaxies and %d stars.\n\n" % (Ng, Ns)
ylim = np.inf
featurenames = ["cmodelmag", "psffwhm", "petror50", "petror90"]
targetnames = ["psfmag", "cmodelmag"]
filters = ["u", "g", "r", "i", "z"]
x = FeatureExtractor(data, featurenames, filters, color_band="r", scale_kind=None, mag_range=None)
data = data[x.idx]
y = FeatureExtractor(data, targetnames, filters, color_band=None, scale_kind=None, mag_range=None)
# taylor to target, set for psf - model
y.features[:, :5] = y.features[:, :5] - y.features[:, 5:10]
y.features[:, 5:10] = np.sqrt(y.features[:, 10:15] ** 2.0 + y.features[:, 15:20] ** 2.0)
y.features = y.features[:, :10]
# restrict x range
xlim = (19.5, 20.5)
ind = (x.features[:, 2] > xlim[0]) & (x.features[:, 2] < xlim[1])
x.features = x.features[ind]
y.features = y.features[ind]
y.Ndata = y.features.shape[0]
def main():
parser = get_parser()
args = parser.parse_args()
feature_extractor = FeatureExtractor()
if args.pipeline_type == "analysis":
text_preprocessor = TextPreProcessor(
stop_words_file_path=args.stopwords_file_path)
analyser = DataAnalyser(input_file=args.input_file_path,
text_preprocessor=text_preprocessor)
analyser.get_data_distribution(plot_bar=args.plot_bar)
analyser.get_word_weights(word_thresh=args.word_thresh)
if args.word_cloud:
analyser.generate_word_cloud()
elif args.pipeline_type == "model_selection":
text_preprocessor = TextPreProcessor(
stop_words_file_path=args.stopwords_file_path)
training_data_df = load_training_data(args.train_file_path)
training_data_df["sentence"] = training_data_df["sentence"].map(
text_preprocessor.process)
features = feature_extractor.get_features_for_training(
training_data_df["sentence"], args.vectorizer)
labels = training_data_df["class"]
apply_cross_validation(
features=features,
labels=labels,
k_folds=args.kfolds,
use_svm=args.use_svm,
use_naive_bayes=args.use_naive_bayes,
use_random_forest=args.use_random_forest,
use_logistic_regression=args.use_logistic_regression,
use_xgboost=args.use_xgboost,
use_gradient_boosting=args.use_gradient_boosting,
plot_cv_graph=True,
)
elif args.pipeline_type == "training":
trainer = Trainer(
train_file_path=args.train_file_path,
val_file_path=args.val_file_path,
stop_words_file_path=args.stopwords_file_path,
model_name=args.best_model,
feature_extractor=feature_extractor,
)
training_data_df = load_training_data(args.train_file_path)
trainer.train(
training_data_df,
split_test_size=args.split_size,
vectorizer_name=args.vectorizer,
get_classification_report=args.get_classification_report,
get_confusion_matrix=args.get_confusion_matrix,
)
validation_data_df = load_validation_data(args.val_file_path)
trainer.validate(validation_data_df, vectorizer_name=args.vectorizer)
if args.model_check_point_path:
trainer.save_trained_model(args.model_check_point_path)
elif args.pipeline_type == "prediction":
if not args.stopwords_file_path:
predictor = Predictor()
else:
predictor = Predictor(stop_words_file=args.stopwords_file_path)
if args.input_file_path:
predictor.predict_csv(args.input_file_path, args.output_file_path,
args.model_path)
if args.test_input:
model, vectorizer = predictor.unpickle_the_model(args.model_path)
predictor.predict(args.test_input, model, vectorizer)
