def __init__(self): self.preprocessor = Preprocessor() self.feature_extractor = FeatureExtractor() self.crf_analyzer = CRFAnalyzer() self.sentiment_analyzer = SentimentAnalyzer() print("\nAll module instantiated and ready to go....\n")
def __init__(self, dataset_cfg, feature_params, collect_wavs=False, verbose=False): # parent init super().__init__(dataset_cfg, feature_params, collect_wavs=collect_wavs, verbose=verbose) # feature extractor self.feature_extractor = FeatureExtractor(feature_params=self.feature_params) # short vars self.N = self.feature_extractor.N self.hop = self.feature_extractor.hop # create plot plaths if not already exists create_folder(list(self.plot_paths.values())) # recreate if self.dataset_cfg['recreate'] or not check_folders_existance(self.wav_folders, empty_check=True): # delete old data delete_files_in_path(self.wav_folders, file_ext=self.dataset_cfg['file_ext']) # create folder wav folders create_folder(self.wav_folders) # create sets (specific to dataset) self.create_sets() # get audio files from sets self.get_audiofiles() self.get_annotation_files()
def __init__(self, classifier, mic_params, is_audio_record=False, root_path='./'): # arguments self.classifier = classifier self.mic_params = mic_params self.is_audio_record = is_audio_record self.root_path = root_path # plot path self.plot_path = self.root_path + self.mic_params['plot_path'] # create folder for plot path create_folder([self.plot_path]) # shortcuts self.feature_params = classifier.feature_params # feature extractor self.feature_extractor = FeatureExtractor(self.feature_params) # windowing params self.N, self.hop = self.feature_extractor.N, self.feature_extractor.hop # queue self.q = queue.Queue() # collector self.collector = Collector( N=self.N, hop=self.hop, frame_size=self.feature_params['frame_size'], update_size=self.mic_params['update_size'], frames_post=self.mic_params['frames_post'], is_audio_record=self.is_audio_record) # device self.device = sd.default.device[0] if not self.mic_params[ 'select_device'] else self.mic_params['device'] # determine downsample self.downsample = self.mic_params['fs_device'] // self.feature_params[ 'fs'] # get input devices self.input_dev_dict = self.extract_devices() # show devices print("\ndevice list: \n", sd.query_devices()) print("\ninput devs: ", self.input_dev_dict.keys()) # stream self.stream = None # change device flag self.change_device_flag = False
def audio_set_wavs(cfg): """ audio set wavs """ # plot path plot_path = '../docu/thesis/5_exp/figs/' # audio sets a1 = AudioDataset(cfg['datasets']['speech_commands'], cfg['feature_params'], root_path='../') a2 = AudioDataset(cfg['datasets']['my_recordings'], cfg['feature_params'], root_path='../') # feature extractor feature_extractor = FeatureExtractor(cfg['feature_params']) # get audio files a1.get_audiofiles() # random seed np.random.seed(1234) r = np.random.randint(low=0, high=150, size=len(a1.set_audio_files[1])) wav_grid = [] # process wavs for wav in sorted([ label_wavs[r[i]] for i, label_wavs in enumerate(a1.set_audio_files[1]) ]): # info print("wav: ", wav) # get raw x, _ = a1.wav_pre_processing(wav) # extract feature vectors [m x l] _, bon_pos = feature_extractor.extract_mfcc(x, reduce_to_best_onset=False) # append to wav grid wav_grid.append((librosa.util.normalize(x), re.sub(r'[0-9]+-', '', wav.split('/')[-1].split('.')[0]), bon_pos)) # plot wav grid plot_wav_grid(wav_grid, feature_params=a1.feature_params, grid_size=(6, 5), plot_path=plot_path, name='wav_grid_c30', show_plot=True)
def __init__(self,classes,fs,channels,train_dir,duration): self.classes = classes f = FeatureExtractor(fs,duration,channels,train_dir,classes) self.feature_matrix = f.build_feature_vector() self.len_train = len(self.feature_matrix[self.classes[0]]) self.testdata = None self.traindata = None self.alldata = None self.trainer = None self.fnet = None
def __init__(self, classifier, feature_params, mic_params, is_audio_record=False): # arguments self.classifier = classifier self.feature_params = feature_params self.mic_params = mic_params self.is_audio_record = is_audio_record # windowing params self.N, self.hop = int( feature_params['N_s'] * feature_params['fs']), int( feature_params['hop_s'] * feature_params['fs']) # queue self.q = queue.Queue() # collector self.collector = Collector( N=self.N, hop=self.hop, frame_size=self.feature_params['frame_size'], update_size=self.mic_params['update_size'], frames_post=self.mic_params['frames_post'], is_audio_record=self.is_audio_record) # feature extractor self.feature_extractor = FeatureExtractor( self.feature_params['fs'], N=self.N, hop=self.hop, n_filter_bands=self.feature_params['n_filter_bands'], n_ceps_coeff=self.feature_params['n_ceps_coeff'], frame_size=self.feature_params['frame_size']) # select microphone yourself (usually not necessary) if mic_params['select_device']: sd.default.device = self.mic_params['device'] # determine downsample self.downsample = self.mic_params['fs_device'] // self.feature_params[ 'fs'] # show devices print("\ndevice list: \n", sd.query_devices()) # setup stream sounddevice self.stream = sd.InputStream(samplerate=self.mic_params['fs_device'], blocksize=int(self.hop * self.downsample), channels=self.mic_params['channels'], callback=self.callback_mic)
def time_measurements(x, u, feature_params): """ time measurements """ # create feature extractor feature_extractor = FeatureExtractor(feature_params) # n measurements delta_time_list = [] for i in range(100): # measure extraction time - start start_time = time.time() # time: 0.030081419944763182 #y = calc_mfcc39(x, fs, N=400, hop=160, n_filter_bands=32, n_ceps_coeff=12, use_librosa=False) # time: 0.009309711456298829 #y = calc_mfcc39(x, fs, N=400, hop=160, n_filter_bands=32, n_ceps_coeff=12, use_librosa=True) # time: 0.00014737367630004883 #y = (custom_dct(np.log(u), n_filter_bands).T) # time: 6.929159164428711e-05 #y = scipy.fftpack.dct(np.log(u), type=2, n=n_filter_bands, axis=1, norm=None, overwrite_x=False).T # time: 0.00418839693069458 *** winner y, _ = feature_extractor.extract_mfcc(x) # time: 0.015525884628295898 #y, _ = feature_extractor.extract_mfcc39_slow(x) # time: 0.011266257762908936s #y = custom_stft(x, N=N, hop=hop, norm=True) # time: 0.0005800390243530274s #y = 2 / N * librosa.stft(x, n_fft=N, hop_length=hop, win_length=N, window='hann', center=True, dtype=None, pad_mode='reflect') # time: 0.00044193744659423826s #_, _, y = scipy.signal.stft(x, fs=1.0, window='hann', nperseg=N, noverlap=N-hop, nfft=N, detrend=False, return_onesided=True, boundary='zeros', padded=False, axis=- 1) # result of measured time diff delta_time_list.append(time.time() - start_time) # data shpae print("y: ", y.shape) # times print("delta_time: ", np.mean(delta_time_list))
def train_classifier(self): """ Train classifier and save to disk :return: """ feature_extractor = FeatureExtractor.tfidf(ngram_range=(1, 2), max_df=0.5, use_idf=False, sublinear_tf=True) clf = SVC(kernel='linear', C=100, gamma=0.01, decision_function_shape='ovo', probability=True) counts, targets = feature_extractor.extract_features_from_csv print('start training...') clf.fit(counts, targets) # train the classifier print('training finished. start dumping model...') # save model and classifier to disk joblib.dump(clf, self.resource_path + 'booking_classifier.pkl') joblib.dump(feature_extractor, self.resource_path + 'booking_features.pkl') self.load_model()
def mfcc_stuff(cfg): """ for dct, filter bands, etc """ # plot path plot_path = '../docu/thesis/3_signal/figs/' # init feature extractor feature_extractor = FeatureExtractor(cfg['feature_params']) # plot dct plot_dct(custom_dct_matrix(cfg['feature_params']['n_filter_bands']), plot_path=plot_path, name='signal_mfcc_dct', show_plot=False) plot_dct(custom_dct_matrix(cfg['feature_params']['n_filter_bands']), plot_path=plot_path, context='dct-div', name='signal_mfcc_dct-div', show_plot=False) # mel scale plot_mel_scale(plot_path=plot_path, name='signal_mfcc_mel_scale', show_plot=False) # plot mel bands plot_mel_band_weights(feature_extractor.w_f, feature_extractor.w_mel, feature_extractor.f, feature_extractor.m, plot_path=plot_path, name='signal_mfcc_weights', show_plot=True)
class ProcessController: def __init__(self): self.preprocessor = Preprocessor() self.feature_extractor = FeatureExtractor() self.crf_analyzer = CRFAnalyzer() self.sentiment_analyzer = SentimentAnalyzer() print("\nAll module instantiated and ready to go....\n") def start_process(self, sentence): data = dict() preprocess_result, headword_data, spacy_parsing_result = self.preprocessor.start_preprocess( sentence) features = [ self.feature_extractor.extract_features(preprocess_result, headword_data) ] crf_result = self.crf_analyzer.analyze(preprocess_result, features) self.sentiment_analyzer.create_graph_data(spacy_parsing_result) sentiment_analyzer_result = self.sentiment_analyzer.analyze(crf_result) data["sentence"] = sentence data["result"] = sentiment_analyzer_result return data
def fit(ml_algorithm, train_files, train_labels, name): train_features, _ = FeatureExtractor.get_features(train_files, []) try: algorithm_type = ALGORITHMS[ml_algorithm] except KeyError: print 'Algorithm type not valid!' return mla = algorithm_type() mla.fit(train_features, train_labels, name)
def get_probs(ml_algorithm, train_files, test_files, name): _, test_features = FeatureExtractor.get_features(train_files, test_files) try: algorithm_type = ALGORITHMS[ml_algorithm] except KeyError: print 'Algorithm type not valid!' return mla = algorithm_type() prob_ind, highest_score = mla.predict(test_features, name) return prob_ind, highest_score
def train_and_predict(ml_algorithm, train_files, train_labels, test_files, test_labels): train_features, test_features = FeatureExtractor.get_features(train_files, test_files) try: algorithm_type = ALGORITHMS[ml_algorithm] except KeyError: print 'Algorithm type not valid!' return mla = algorithm_type() score, indices, probs = mla.fit_and_predict(train_features, test_features, train_labels, test_labels) print 'Score: {}'.format(score) return indices, probs
def train_kfold(ml_algorithm, source_code, labels, ast_nodes=None): source_code = np.array(source_code) labels = np.array(labels) #ast_nodes = np.array(ast_nodes) try: algorithm_type = ALGORITHMS[ml_algorithm] except KeyError: print 'Algorithm type not valid!' return start = time.time() n_trees = [400] for nt in n_trees: mla = algorithm_type(n_trees=nt) print 'nt: {}'.format(nt) k = 10 code_per_author = 10 accuracy = 0 for i in range(k): test_indices = [] it = code_per_author / k for j in range(it): test_indices.extend(np.array(range(i*it+j, len(source_code), code_per_author))) train_indices = [] for sci in range(len(source_code)): if sci not in test_indices: train_indices.append(sci) train_features, test_features = FeatureExtractor.get_features(source_code[train_indices], source_code[test_indices]) score = mla.fit_and_predict(train_features, test_features, labels[train_indices], labels[test_indices]) print 'Score after {}th fold is {}'.format(i, score) accuracy += score print 'Final score: {}'.format(accuracy / float(k)) end = time.time() - start print 'Execution time: {}'.format(end) print '-----------------------------------------------'
def prep(n, labelcol): """ Load the data """ f = pf.open(n) data = f[1].data names = f[1].columns.names f.close() try: labels = data.field(labelcol) except: labels = np.zeros(data.field(0).size) - 99 ylim = np.inf featurenames = ['cmodelmag', 'psffwhm', 'petror50', 'petror90'] targetnames = ['psfmag', 'cmodelmag'] filters = ['u', 'g', 'r', 'i', 'z'] x = FeatureExtractor(data, featurenames, filters, color_band='r', scale_kind=None, mag_range=None) data = data[x.idx] labels = labels[x.idx] y = FeatureExtractor(data, targetnames, filters, color_band=None, scale_kind=None, mag_range=None) # taylor to target, set for psf - model y.features[:, :5] = y.features[:, :5] - y.features[:, 5:10] y.features[:, 5:10] = np.sqrt(y.features[:, 10:15] ** 2. + y.features[:, 15:20] ** 2.) y.features = y.features[:, :10] # restrict y range ylim = 10. ind = y.features[:, 2] < ylim x.features = x.features[ind] y.features = y.features[ind] labels = labels[ind] y.Ndata = y.features.shape[0] return x, y, labels
def __init__(self, *args, **kwargs): BaseEntity.__init__(self, *args, **kwargs) self.rotation_parametrization = rotation_parametrizations[ self.args.rotation_parametrization] self._create_parameter_info_table() if self.z_up: self._vertical_axis = "z" self._coordinate_up = 2 else: self._vertical_axis = "y" self._coordinate_up = 1 self._max_angular_step = self.args.max_angular_step self._normalized_constrainers = self._create_constrainers() self._unnormalized_constrainers = self._create_constrainers() self.modified_root_vertical_orientation = None self._last_root_vertical_orientation = None self._rotation_interpolators = {} self._enable_friction = self.args.friction if hasattr(self.args, "enable_features") and self.args.enable_features: self.feature_extractor = FeatureExtractor(self._coordinate_up) self._bvh_joint_name_for_feature = self._get_bvh_joint_names_for_features_from_args()
def predict_equation(self, image: Image) -> str: """Given an image predicts the LaTeX equation for that image. :param image: An image. :returns: The predicted LaTeX equation for the image. """ feature_vectors = [] image = image.convert('L') while image is not None: image = imhelp.remove_border_and_padding( image, black_threshold=0, white_threshold=fe.DEFAULT_WHITE_THRESHOLD, rem_black=True, rem_white=True) left_im, image = imhelp.extract_leftmost_symbol(image) fv = FeatureExtractor.extract_features_from_image(left_im) feature_vectors.append(fv) feature_matrix = fe.FeatureExtractor.get_feature_matrix( feature_vectors) return ' '.join(self.predict(feature_matrix))
def write_predictions(self): """Output the predictions to a text file.""" res = FeatureExtractor("test").run() model = torch.load("model.pt") test = namedtuple("res", ["lsr", "feats", "scores"])(lsr=res.lsr.reshape( -1, 2048), feats=res.feats, scores=res.scores) dev_ = data_utils.TensorDataset(*[ torch.tensor(getattr(test, i)).float() for i in ["lsr", "feats", "scores"] ]) with torch.no_grad(): preds = model.forward(*dev_.tensors[:2]).cpu().numpy() np.set_printoptions(suppress=True) np.savetxt("predictions.txt", preds.astype(float), delimiter="\n", fmt="%f") print("Predictions saved to predictions.txt")
def run_fetc(): if "-exp" in sys.argv: exp_pos = sys.argv.index("-exp") param_file = sys.argv[exp_pos+1] else: param_file = "experiment.yaml" exp = read_parameters(param_file=param_file) ovw = parse_commandline(sys.argv) overwrite_params(exp, ovw) update_parameters(exp) #print exp if exp['steps']['extract_features']: fe = FeatureExtractor.create(params=exp) fe.run() if exp['steps']['aggregate_features']: fa = FeatureAggregator.create(params=exp) fa.run() if exp['steps']['train']: t = ModelTrainer.create(params=exp) t.run() if exp['steps']['test']: t = ModelTester.create(params=exp) t.run() if exp['steps']['evaluate']: t = Evaluator.create(params=exp) t.run()
def run_fetc(): if "-exp" in sys.argv: exp_pos = sys.argv.index("-exp") param_file = sys.argv[exp_pos + 1] else: param_file = "experiment.yaml" exp = read_parameters(param_file=param_file) ovw = parse_commandline(sys.argv) overwrite_params(exp, ovw) update_parameters(exp) #print exp if exp['steps']['extract_features']: fe = FeatureExtractor.create(params=exp) fe.run() if exp['steps']['aggregate_features']: fa = FeatureAggregator.create(params=exp) fa.run() if exp['steps']['train']: t = ModelTrainer.create(params=exp) t.run() if exp['steps']['test']: t = ModelTester.create(params=exp) t.run() if exp['steps']['evaluate']: t = Evaluator.create(params=exp) t.run()
vocab[i] = vocab[i].strip() def build_model(n_word): global num_class model = Sequential() model.add(Flatten(input_shape=(n_word, 500))) model.add(Dropout(0.1)) model.add(Dense(num_class, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd') return model # model = load_model('model/bengio_sgd_2.h5') # print(model.summary()) fe = FeatureExtractor(5) fe.set_w2v(w2v_pathname, 500, keep_alive=True) for epoch in range(9, 11): model = load_model('model/bengio_sgd_{}.h5'.format(epoch - 1)) for i in range(1, 1001): filename = 'data/batch/bengio/6/{}.txt'.format(i) print(filename) with open(filename) as file: data = file.readlines() word_seq = [] label = [] for j in range(len(data)): data[j] = data[j].strip() splitted = data[j].split(';') word_seq.append(splitted[:-1])
def classify(bow=False, plot=False, multinomial_nb=False, bernoulli_nb=False, knn=False, support_vm=False, svm_sgd=False, decision_tree=False, random_forest=False, persist=False, logistic_regression=False): """ Validate the classifier against unseen resources using k-fold cross validation """ if multinomial_nb: clf_title = 'Multinomial NB' if bow: vectorizer_title = 'Bag-of-Words' counts, targets = FeatureExtractor.bow( max_df=0.25, ngram_range=(1, 3)).extract_features_from_csv clf = MultinomialNB(alpha=1e-05) else: vectorizer_title = 'TF-IDF' counts, targets = FeatureExtractor.tfidf( analyzer='char', max_df=0.5, ngram_range=(1, 4), norm='l1', sublinear_tf=False, use_idf=True).extract_features_from_csv clf = MultinomialNB(alpha=1e-07) elif bernoulli_nb: clf_title = 'Bernoulli NB' if bow: vectorizer_title = 'Bag-of-Words' counts, targets = FeatureExtractor.bow( max_df=0.25, ngram_range=(1, 3)).extract_features_from_csv clf = BernoulliNB(alpha=1e-05) else: vectorizer_title = 'TF-IDF' counts, targets = FeatureExtractor.tfidf( analyzer='word', max_df=0.25, ngram_range=(1, 3), norm='l1', sublinear_tf=True, use_idf=True).extract_features_from_csv clf = BernoulliNB(alpha=1e-05) elif knn: clf_title = 'K-Nearest-Neighbour' if bow: vectorizer_title = 'Bag-of-Words' counts, targets = FeatureExtractor.bow( max_df=0.5, ngram_range=(1, 1)).extract_features_from_csv clf = KNeighborsClassifier(weights='distance', n_neighbors=2, leaf_size=20, algorithm='auto') else: vectorizer_title = 'TF-IDF' counts, targets = FeatureExtractor.tfidf( analyzer='word', max_df=0.5, ngram_range=(1, 1), norm='l1', sublinear_tf=True, use_idf=True).extract_features_from_csv clf = KNeighborsClassifier(weights='distance', n_neighbors=2, leaf_size=20, algorithm='auto') elif support_vm: clf_title = 'Support Vector Machine' if bow: vectorizer_title = 'Bag-of-Words' counts, targets = FeatureExtractor.tfidf( max_df=0.5, ngram_range=(1, 2)).extract_features_from_csv clf = SVC(kernel='sigmoid', C=100, gamma=0.01, decision_function_shape='ovo', probability=True) else: vectorizer_title = 'TF-IDF' counts, targets = FeatureExtractor.tfidf( analyzer='char', max_df=1.0, ngram_range=(1, 4), norm='l2', use_idf=False, sublinear_tf=True).extract_features_from_csv clf = SVC(kernel='sigmoid', C=10, gamma=1.4, decision_function_shape='ovo', probability=True) elif svm_sgd: clf_title = 'SVM (SGD)' if bow: vectorizer_title = 'Bag-of-Words' counts, targets = FeatureExtractor.bow( max_df=0.25, ngram_range=(1, 4)).extract_features_from_csv target_ints = [] for target in targets: target_ints.append(category_names_reverse.index(target)) class_weights = get_classweight(targets) clf = SGDClassifier(loss='squared_hinge', penalty='l1', alpha=1e-05, max_iter=100, tol=0.2, class_weight=class_weights) else: vectorizer_title = 'TF-IDF' counts, targets = FeatureExtractor.tfidf( ngram_range=(1, 4), max_df=0.25, use_idf=False, sublinear_tf=True).extract_features_from_csv target_ints = [] for target in targets: target_ints.append(category_names_reverse.index(target)) class_weights = get_classweight(targets) clf = SGDClassifier(loss='hinge', penalty='l1', alpha=1e-05, max_iter=100, tol=0.2, class_weight=class_weights) else: print('Please provide a classifer algorithm') return clf.fit(counts, targets) if persist: joblib.dump(clf, clf_title + '.pkl') # scores ac_scores = [] f1_scores = [] prec_scores = [] rec_scores = [] confusion = np.array([[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]]) cv = list( StratifiedKFold(n_splits=15, random_state=1).split(counts, targets)) for k, (train_indices, test_indices) in enumerate(cv): train_text = counts[train_indices] train_y = targets[train_indices] test_text = counts[test_indices] test_y = targets[test_indices] clf.fit(train_text, train_y) predictions = clf.predict(test_text) confusion += confusion_matrix(test_y, predictions) ac_scores.append(accuracy_score(test_y, predictions)) f1_scores.append(f1_score(test_y, predictions, average="macro")) prec_scores.append( precision_score(test_y, predictions, average="macro")) rec_scores.append(recall_score(test_y, predictions, average="macro")) print("---------------------- \nResults for ", clf_title, " with ", vectorizer_title, ":") print("K-Folds Accuracy-score: ", sum(ac_scores) / len(ac_scores)) print("K-Folds F1-score: ", sum(f1_scores) / len(f1_scores)) print("K-Folds Precision-score: ", sum(prec_scores) / len(prec_scores)) print("K-Folds Recall-score: ", sum(rec_scores) / len(rec_scores)) print("CV accuracy : %.3f +/- %.3f" % (np.mean(ac_scores), np.std(ac_scores))) labels = [ 'Barent.', 'Finanzen', 'Freiz.&Lifes.', 'Lebensh.', 'Mob.&Verk.', 'Versich.', 'Wohn.&Haus.' ] if plot: Plotter.plot_and_show_confusion_matrix(confusion, labels, normalize=True, title=clf_title, save=True)
from feature_extraction import FeatureExtractor from keras.models import load_model from keras import backend as K import tensorflow as tf from keras.preprocessing.text import text_to_word_sequence import json config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.5 K.tensorflow_backend.set_session(tf.Session(config=config)) ta = BasicTextAugmentation() vocab = list_from_file('resource/vocabulary.txt') w2v_pathname = 'resource/w2v_path.txt' fe = FeatureExtractor(5) fe.set_w2v(w2v_pathname, 500, keep_alive=True) class BengioAugmenter(): def __init__(self, model_filename): self.gram = 5 self.model = load_model(model_filename) def prob(self, sequences): for i in range(len(sequences)): if sequences[i] not in vocab: if sequences[i].isdigit(): sequences[i] = '<NUM>' else: sequences[i] = '<UNK>' feature = sequences[:-1]
concat = Concatenate()([pooled_conv_dropped_a, pooled_conv_dropped_b]) output = TimeDistributed(Dense(units = 1, activation = 'sigmoid', ))(concat) model = Model(my_input, output) model.compile(loss='categorical_hinge', optimizer = 'adagrad', metrics = ['accuracy']) return model # print(model.summary()) fe = FeatureExtractor(6) fe.set_w2v(w2v_pathname, 500, keep_alive=True) for epoch in range(2, 6): model = load_model('model/cnw100_{}.h5'.format(epoch-1)) for i in range(1,1001): true_filename = 'data/batch/bengio/6/{}.txt'.format(i) false_filename = 'data/batch/cnw/6/{}.txt'.format(i) with open(true_filename) as file: true_data = file.readlines() with open(false_filename) as file: false_data = file.readlines() true_word_seq = [] false_word_seq = []
def __init__(self): self.face_detector = FaceDetector() self.preprocessor = Preprocessor() self.extractor = FeatureExtractor()
class Mic(): """ Mic class """ def __init__(self, classifier, feature_params, mic_params, is_audio_record=False): # arguments self.classifier = classifier self.feature_params = feature_params self.mic_params = mic_params self.is_audio_record = is_audio_record # windowing params self.N, self.hop = int( feature_params['N_s'] * feature_params['fs']), int( feature_params['hop_s'] * feature_params['fs']) # queue self.q = queue.Queue() # collector self.collector = Collector( N=self.N, hop=self.hop, frame_size=self.feature_params['frame_size'], update_size=self.mic_params['update_size'], frames_post=self.mic_params['frames_post'], is_audio_record=self.is_audio_record) # feature extractor self.feature_extractor = FeatureExtractor( self.feature_params['fs'], N=self.N, hop=self.hop, n_filter_bands=self.feature_params['n_filter_bands'], n_ceps_coeff=self.feature_params['n_ceps_coeff'], frame_size=self.feature_params['frame_size']) # select microphone yourself (usually not necessary) if mic_params['select_device']: sd.default.device = self.mic_params['device'] # determine downsample self.downsample = self.mic_params['fs_device'] // self.feature_params[ 'fs'] # show devices print("\ndevice list: \n", sd.query_devices()) # setup stream sounddevice self.stream = sd.InputStream(samplerate=self.mic_params['fs_device'], blocksize=int(self.hop * self.downsample), channels=self.mic_params['channels'], callback=self.callback_mic) def callback_mic(self, indata, frames, time, status): """ Input Stream Callback """ if status: print(status) #self.q.put(indata[:, 0].copy()) # add to queue with primitive downsampling self.q.put(indata[::self.downsample, 0].copy()) def clear_mic_queue(self): """ clear the queue after classification """ # process data for i in range(self.q.qsize()): # get chunk x = self.q.get() # onset and energy archiv e, _ = onset_energy_level(x, alpha=self.mic_params['energy_thres']) # update collector self.collector.x_all = np.append(self.collector.x_all, x) self.collector.e_all = np.append(self.collector.e_all, e) def read_mic_data(self): """ reads the input from the queue """ # init x_collect = np.empty(shape=(0), dtype=np.float32) e_collect = np.empty(shape=(0), dtype=np.float32) # onset flag is_onset = False # process data if self.q.qsize(): for i in range(self.q.qsize()): # get data x = self.q.get() # append chunk x_collect = np.append(x_collect, x.copy()) # append energy level e_collect = np.append(e_collect, 1) # detect onset e_onset, is_onset = onset_energy_level( x_collect, alpha=self.mic_params['energy_thres']) # collection update self.collector.update_collect(x_collect.copy(), e=e_collect.copy() * e_onset, on=is_onset) return is_onset def update_read_command(self): """ update mic """ # read chunk is_onset = self.read_mic_data() # onset was detected if is_onset: # start collection of items self.collector.start_collecting() # collection is full if self.collector.is_full(): # read out collection x_onset = self.collector.read_collection() # extract features mfcc_bon, bon_pos = self.feature_extractor.extract_mfcc39(x_onset) # classify collection y_hat, label = self.classifier.classify_sample(mfcc_bon) # plot plot_mfcc_profile( x_onset[bon_pos * self.hop:(bon_pos + self.feature_params['frame_size']) * self.hop], self.feature_params['fs'], self.N, self.hop, mfcc_bon, frame_size=self.feature_params['frame_size'], plot_path=self.mic_params['plot_path'], name='collect-{}_label-{}'.format( self.collector.collection_counter, label), enable_plot=self.mic_params['enable_plot']) # clear read queue self.clear_mic_queue() return y_hat return None def stop_mic_condition(self, time_duration): """ stop mic if time duration is exceeded (memory issue in recording) """ return (self.collector.x_all.shape[0] >= (time_duration * self.feature_params['fs'])) and self.is_audio_record def save_audio_file(self): """ saves collection to audio file """ # has not recorded audio if not self.is_audio_record: print("***you did not set the record flag!") return import soundfile # save audio soundfile.write('{}out_audio.wav'.format(self.mic_params['plot_path']), self.collector.x_all, self.feature_params['fs'], subtype=None, endian=None, format=None, closefd=True)
class Mic(): """ Mic class """ def __init__(self, classifier, mic_params, is_audio_record=False, root_path='./'): # arguments self.classifier = classifier self.mic_params = mic_params self.is_audio_record = is_audio_record self.root_path = root_path # plot path self.plot_path = self.root_path + self.mic_params['plot_path'] # create folder for plot path create_folder([self.plot_path]) # shortcuts self.feature_params = classifier.feature_params # feature extractor self.feature_extractor = FeatureExtractor(self.feature_params) # windowing params self.N, self.hop = self.feature_extractor.N, self.feature_extractor.hop # queue self.q = queue.Queue() # collector self.collector = Collector( N=self.N, hop=self.hop, frame_size=self.feature_params['frame_size'], update_size=self.mic_params['update_size'], frames_post=self.mic_params['frames_post'], is_audio_record=self.is_audio_record) # device self.device = sd.default.device[0] if not self.mic_params[ 'select_device'] else self.mic_params['device'] # determine downsample self.downsample = self.mic_params['fs_device'] // self.feature_params[ 'fs'] # get input devices self.input_dev_dict = self.extract_devices() # show devices print("\ndevice list: \n", sd.query_devices()) print("\ninput devs: ", self.input_dev_dict.keys()) # stream self.stream = None # change device flag self.change_device_flag = False def load_user_settings(self, user_setting_file): """ load user settings like device and energy threshold """ # load user settings user_settings = yaml.safe_load(open( user_setting_file)) if os.path.isfile(user_setting_file) else {} # update mic params self.mic_params.update(user_settings) # device self.device = sd.default.device[0] if not self.mic_params[ 'select_device'] else self.mic_params['device'] def init_stream(self): """ init stream """ self.stream = sd.InputStream(device=self.device, samplerate=self.mic_params['fs_device'], blocksize=int(self.hop * self.downsample), channels=self.mic_params['channels'], callback=self.callback_mic) self.change_device_flag = False def change_device(self, device): """ change to device """ self.change_device_flag = True self.device = device def extract_devices(self): """ extract only input devices """ return { i: dev for i, dev in enumerate(sd.query_devices()) if dev['max_input_channels'] } def callback_mic(self, indata, frames, time, status): """ Input Stream Callback """ # debug if status: print(status) #self.q.put(indata[:, 0].copy()) # add to queue with primitive downsampling self.q.put(indata[::self.downsample, 0].copy()) def clear_mic_queue(self): """ clear the queue after classification """ # process data for i in range(self.q.qsize()): # get chunk x = self.q.get() # onset and energy archiv e, _ = self.onset_energy_level( x, alpha=self.mic_params['energy_thresh']) # update collector self.collector.x_all = np.append(self.collector.x_all, x) self.collector.e_all = np.append(self.collector.e_all, e) def read_mic_data(self): """ reads the input from the queue """ # init x_collect = np.empty(shape=(0), dtype=np.float32) e_collect = np.empty(shape=(0), dtype=np.float32) # onset flag is_onset = False # process data if self.q.qsize(): for i in range(self.q.qsize()): # get data x = self.q.get() # append chunk x_collect = np.append(x_collect, x.copy()) # append energy level e_collect = np.append(e_collect, 1) # detect onset e_onset, is_onset = self.onset_energy_level( x_collect, alpha=self.mic_params['energy_thresh']) # collection update self.collector.update_collect(x_collect.copy(), e=e_collect.copy() * e_onset, on=is_onset) return is_onset def onset_energy_level(self, x, alpha=0.01): """ onset detection with energy level x: [n x c] n: samples c: channels """ e = x.T @ x / len(x) return e, e > alpha def update_read_command(self): """ update mic """ # read chunk is_onset = self.read_mic_data() # onset was detected if is_onset: # start collection of items self.collector.start_collecting() # collection is full if self.collector.is_full(): # read out collection x_onset = self.collector.read_collection() # extract features mfcc_bon, bon_pos = self.feature_extractor.extract_mfcc(x_onset) # classify collection y_hat, label = self.classifier.classify(mfcc_bon) # plot plot_mfcc_profile( x_onset[bon_pos * self.hop:(bon_pos + self.feature_params['frame_size']) * self.hop], self.feature_params['fs'], self.N, self.hop, mfcc_bon, frame_size=self.feature_params['frame_size'], plot_path=self.plot_path, name='collect-{}_label-{}'.format( self.collector.collection_counter, label), enable_plot=self.mic_params['enable_plot']) # clear read queue self.clear_mic_queue() return label return None def stop_mic_condition(self, time_duration): """ stop mic if time duration is exceeded (memory issue in recording) """ return (self.collector.x_all.shape[0] >= (time_duration * self.feature_params['fs'])) and self.is_audio_record def save_audio_file(self): """ saves collection to audio file """ # has not recorded audio if not self.is_audio_record: print("***you did not set the record flag!") return import soundfile # save audio soundfile.write('{}out_audio.wav'.format(self.plot_path), self.collector.x_all, self.feature_params['fs'], subtype=None, endian=None, format=None, closefd=True)
sys.path.append("../") from common import create_folder from feature_extraction import FeatureExtractor # plot path #plot_path = './ignore/plots/fe/' # create folder #create_folder([plot_path]) # yaml config file cfg = yaml.safe_load(open("../config.yaml")) # init feature extractor feature_extractor = FeatureExtractor(cfg['feature_params']) # -- # params fs = 16000 N = 400 hop = 160 n_filter_bands = 16 n_ceps_coeff = 12 # -- # test signal # generate test signal x = some_test_signal(fs, t=1, save_to_file=False)
def run_benchmarks(training_data_dir: str, results_dir: str): """ Runs benchmark tests and records observations. :param training_data_dir: The directory which contains the training grid files and labels. :param results_dir: The directory into which results should be written. The results are recorded in a file named `benchmark.csv` with each model's MCC, training time, and prediction time recorded. The method will also produce confusion matrices for each model benchmarked for debugging. """ if os.path.isdir(results_dir): shutil.rmtree(results_dir) os.makedirs(results_dir, exist_ok=True) training_pdf = os.path.join(training_data_dir, "training_data.pdf") labels_file = os.path.join(training_data_dir, "training_labels.txt") conf_mat_figsize = (14, 14) if not os.path.isfile(training_pdf): raise IOError("Training input PDF not found.") if not os.path.isfile(labels_file): raise IOError("Training labels file not found.") training_file = fe.TrainingFileSpec( training_pdf, fe.DEFAULT_GRID_SPEC, labels_file, dpi=96, white_threshold=fe.DEFAULT_WHITE_THRESHOLD) angles = np.arange(-5, 5, 1) feature_vectors, feature_labels = \ FeatureExtractor.extract_labelled_feature_vectors(training_file, rotations=angles) train_ft, test_ft, train_lb, test_lb = train_test_split(feature_vectors, feature_labels, test_size=0.33, random_state=81) all_labels = [lbl for lbl in np.unique(feature_labels)] # We still store our benchmarks here bench = pd.DataFrame(columns=['Model', 'MCC', 'TestTime', 'PredTime']) models = [] # 1 hidden layer with various sizes for i in [26, 52, 80, 100]: models.append(MLPLearningModel(name="MLP (%i)" % i, hidden_layer_sizes=(i,), max_iter=1000)) # 2 hidden layers with various sizes for i in [10, 26, 100]: for j in [10, 26, 100]: models.append(MLPLearningModel(name="MLP (%i, %i)" % (i, j), hidden_layer_sizes=(i, j,), max_iter=1000)) # KNN models with various k values for k in [1, 5, 10, 15, 30]: models.append(KNNLearningModel(name="KNN (k=%i)" % k, k=k)) for model in models: begin = time.time() model.train(train_ft, train_lb) test_time = time.time() - begin begin = time.time() predictions = model.predict(test_ft) pred_time = time.time() - begin mcc = matthews_corrcoef(y_pred=predictions, y_true=test_lb) bench = bench.append({'Model': model.name, 'MCC': mcc, 'TestTime': test_time, 'PredTime': pred_time}, ignore_index=True) conf_mat = confusion_matrix(y_pred=predictions, y_true=test_lb, normalize='true') plot_title = "Confusion matrix for %s" % model.name fig = plothelp.plot_confusion_matrix(data=conf_mat, title=plot_title, figsize=conf_mat_figsize, dpi=120, labels=all_labels) plot_file_name = model.name.replace(" ", "_").replace(",", "_").\ replace("(", "_").replace(")", "_").replace("=", "_") + ".png" fig.savefig(os.path.join(results_dir, plot_file_name)) pyplot.close(fig) bench.to_csv( path_or_buf=os.path.join(results_dir, "benchmark.csv"), header=True, index=False)
class Entity(BaseEntity): @staticmethod def add_parser_arguments(parser): parser.add_argument("--rotation-parametrization", "-r", choices=["vectors", "quaternion"]) parser.add_argument("--naive-quaternion-interpolation", action="store_true") parser.add_argument("--max-angular-step", type=float, default=1.) parser.add_argument("--translate", action="store_true") parser.add_argument("--translation-weight", type=float, default=1.) parser.add_argument("--friction", action="store_true") parser.add_argument("--confinement", action="store_true") parser.add_argument("--confinement-rate", type=float, default=0.03) parser.add_argument("--pose-scale", type=float, default=.5) parser.add_argument("--random-slide", type=float, default=0.0) parser.add_argument("--circle-slide", action="store_true") parser.add_argument("--left-foot") parser.add_argument("--left-hand") parser.add_argument("--left-forearm") parser.add_argument("--left-shoulder") parser.add_argument("--left-knee") parser.add_argument("--left-hip") parser.add_argument("--right-foot") parser.add_argument("--right-hand") parser.add_argument("--right-forearm") parser.add_argument("--right-shoulder") parser.add_argument("--right-knee") parser.add_argument("--right-hip") parser.add_argument("--torso") parser.add_argument("--neck") parser.add_argument("--head") def __init__(self, *args, **kwargs): BaseEntity.__init__(self, *args, **kwargs) self.rotation_parametrization = rotation_parametrizations[ self.args.rotation_parametrization] self._create_parameter_info_table() if self.z_up: self._vertical_axis = "z" self._coordinate_up = 2 else: self._vertical_axis = "y" self._coordinate_up = 1 self._max_angular_step = self.args.max_angular_step self._normalized_constrainers = self._create_constrainers() self._unnormalized_constrainers = self._create_constrainers() self.modified_root_vertical_orientation = None self._last_root_vertical_orientation = None self._rotation_interpolators = {} self._enable_friction = self.args.friction if hasattr(self.args, "enable_features") and self.args.enable_features: self.feature_extractor = FeatureExtractor(self._coordinate_up) self._bvh_joint_name_for_feature = self._get_bvh_joint_names_for_features_from_args() def _get_bvh_joint_names_for_features_from_args(self): result = {} for joint_name in self.feature_extractor.INPUT_JOINTS: bvh_joint_name = getattr(self.args, joint_name) if bvh_joint_name is None: raise Exception("please specify BVH joint for %s" % joint_name) result[joint_name] = bvh_joint_name return result def _create_constrainers(self): return Constrainers( self._coordinate_up, enable_friction=(self.args.friction and not ( hasattr(self.args, "show_all_feature_matches") and self.args.show_all_feature_matches)), enable_floor=self.floor, enable_confinement=self.args.confinement, confinement_rate=self.args.confinement_rate, enable_random_slide=(self.args.random_slide > 0), random_slide=self.args.random_slide, enable_circle_slide=self.args.circle_slide) def reset_constrainers(self): self._normalized_constrainers.reset() self._unnormalized_constrainers.reset() def _create_parameter_info_table(self): self._parameter_info = [] root_joint_definition = self.bvh_reader.get_hierarchy().get_root_joint_definition() self._extend_parameter_info_table_recurse(root_joint_definition) def _extend_parameter_info_table_recurse(self, joint_definition): if not joint_definition.has_parent and self.args.translate: self._parameter_info.extend( [{"category": "translate", "component": "X"}, {"category": "translate", "component": "Y"}, {"category": "translate", "component": "Z"}]) if joint_definition.has_rotation and not joint_definition.has_static_rotation: for n in range(self.rotation_parametrization.num_parameters): self._parameter_info.append({"category": joint_definition.name, "component": str(n)}) for child_definition in joint_definition.child_definitions: self._extend_parameter_info_table_recurse(child_definition) def parameter_info(self, index): return self._parameter_info[index] def get_value_length(self): return len(self._parameter_info) def get_value(self): self.bvh_reader.set_pose_from_time(self.pose, self._t * self.args.bvh_speed) return self._joint_to_parameters(self.pose.get_root_joint()) def get_value_from_frame(self, frame, **kwargs): self.bvh_reader.set_pose_from_frame(self.pose, frame, **kwargs) return self._joint_to_parameters(self.pose.get_root_joint()) def get_random_value(self): self.bvh_reader.set_pose_from_time(self.pose, random.uniform(0, self.bvh_reader.get_duration())) return self._joint_to_parameters(self.pose.get_root_joint()) def get_duration(self): return self.bvh_reader.get_duration() / self.args.bvh_speed def _joint_to_parameters(self, root_joint): parameters = [] self._add_joint_parameters_recurse(root_joint, parameters) return parameters def _add_joint_parameters_recurse(self, joint, parameters): if not joint.definition.has_parent and self.args.translate: self._add_joint_translation_parameters(joint, parameters) if joint.definition.has_rotation and not joint.definition.has_static_rotation: self._add_joint_rotation_parameters(joint, parameters) for child in joint.children: self._add_joint_parameters_recurse(child, parameters) def _add_joint_translation_parameters(self, joint, parameters): vertex = joint.get_vertex() normalized_vector = self.bvh_reader.normalize_vector(vertex) weighted_vector = self.args.translation_weight * normalized_vector parameters.extend(weighted_vector) def _add_joint_rotation_parameters(self, joint, parameters): rotation_parameters = self.rotation_parametrization.rotation_to_parameters( joint.rotation) parameters.extend(rotation_parameters) def process_input(self, parameters): return self._parameters_to_scaled_normalized_vertices(parameters) def process_output(self, parameters): return self._constrained_output_vertices(parameters) def process_io_blend(self, parameters): return self._constrained_output_vertices(parameters) def _constrained_output_vertices(self, parameters): vertices = self._parameters_to_scaled_normalized_vertices(parameters) vertices = self._normalized_constrainers.constrain(vertices) return vertices def _parameters_to_scaled_normalized_vertices(self, parameters): self._set_pose_from_parameters(parameters) root_joint = self.pose.get_root_joint() vertices = root_joint.get_vertices() normalized_vertices = [ self.bvh_reader.normalize_vector_without_translation(vertex) * self.args.pose_scale for vertex in vertices] return normalized_vertices def _set_pose_from_parameters(self, parameters): self._parameters_to_joint_recurse(parameters, self.pose.get_root_joint()) self.bvh_reader.get_hierarchy().update_pose_world_positions(self.pose) def _parameters_to_joint_recurse(self, parameters, joint, parameter_index=0): if not joint.definition.has_parent and self.args.translate: parameter_index = self._parameters_to_joint_translation(parameters, joint, parameter_index) if joint.definition.has_rotation and not joint.definition.has_static_rotation: parameter_index = self._parameters_to_joint_rotation(parameters, joint, parameter_index) for child in joint.children: parameter_index = self._parameters_to_joint_recurse(parameters, child, parameter_index) return parameter_index def _parameters_to_joint_translation(self, parameters, joint, parameter_index): weighted_vector = parameters[parameter_index:parameter_index+3] parameter_index += 3 if self.args.translation_weight == 0: joint.translation = [0, 0, 0] else: normalized_vector = numpy.array(weighted_vector) / self.args.translation_weight joint.translation = self.bvh_reader.skeleton_scale_vector(normalized_vector) return parameter_index def _parameters_to_joint_rotation(self, parameters, joint, parameter_index): rotation_parameters = parameters[ parameter_index:parameter_index+ self.rotation_parametrization.num_parameters] parameter_index += self.rotation_parametrization.num_parameters radians = self.rotation_parametrization.parameters_to_rotation( rotation_parameters, joint.definition.axes) if joint.parent is None: radians = self._process_root_orientation(joint, radians) joint.angles = radians return parameter_index def _process_root_orientation(self, root_joint, radians): return self._process_vertical_axis(radians, root_joint.definition.axes) def _process_vertical_axis(self, euler_angles, axes): if axes[1] == self._vertical_axis: return self._process_vertical_orientation_as_first_axis(euler_angles) else: if self._vertical_axis == "y": axes_with_vertical_first = "ryxz" elif self._vertical_axis == "z": axes_with_vertical_first = "rzxy" euler_angles_vertical_axis_first = euler_from_quaternion( quaternion_from_euler(*euler_angles, axes=axes), axes=axes_with_vertical_first) euler_angles_vertical_axis_first_reoriented = \ self._process_vertical_orientation_as_first_axis( euler_angles_vertical_axis_first) return euler_from_quaternion( quaternion_from_euler( *euler_angles_vertical_axis_first_reoriented, axes=axes_with_vertical_first), axes=axes) def _process_vertical_orientation_as_first_axis(self, euler_angles): euler_angles = list(euler_angles) self._last_root_vertical_orientation = euler_angles[0] if self.modified_root_vertical_orientation is not None: euler_angles[0] = self.modified_root_vertical_orientation return euler_angles def get_last_root_vertical_orientation(self): return self._last_root_vertical_orientation def parameters_to_processed_pose(self, parameters, output_pose): self._set_pose_from_parameters(parameters) root_joint = self.pose.get_root_joint() vertices = root_joint.get_vertices() vertices = self._unnormalized_constrainers.constrain(vertices) self.bvh_reader.get_hierarchy().set_pose_vertices( output_pose, vertices, not ASSUME_NO_TRANSLATIONAL_OFFSETS_IN_NON_ROOT) def extract_features(self, pose): positions = [ pose.get_joint(self._bvh_joint_name_for_feature[joint_name]).worldpos for joint_name in self.feature_extractor.INPUT_JOINTS] return self.feature_extractor.extract_features(*positions) def interpolate(self, parameters1, parameters2, amount): result = [] self._interpolate_recurse( parameters1, parameters2, amount, self.pose.get_root_joint(), result) return result def _interpolate_recurse(self, parameters1, parameters2, amount, joint, result, parameter_index=0): if not joint.definition.has_parent and self.args.translate: interpolated_translation, parameter_index = self._interpolate_translation( parameters1, parameters2, amount, joint, parameter_index) result += interpolated_translation if joint.definition.has_rotation and not joint.definition.has_static_rotation: interpolated_rotation, parameter_index = self._interpolate_rotation( parameters1, parameters2, amount, joint, parameter_index) result += interpolated_rotation for child in joint.children: parameter_index = self._interpolate_recurse( parameters1, parameters2, amount, child, result, parameter_index) return parameter_index def _interpolate_translation(self, parameters1, parameters2, amount, joint, parameter_index): vector1 = parameters1[parameter_index:parameter_index+3] vector2 = parameters2[parameter_index:parameter_index+3] parameter_index += 3 result = list(numpy.array(vector2) * amount + numpy.array(vector1) * (1-amount)) return result, parameter_index def _interpolate_rotation(self, parameters1, parameters2, amount, joint, parameter_index): rotation_params1 = numpy.array(parameters1[ parameter_index:parameter_index + self.rotation_parametrization.num_parameters]) rotation_params2 = numpy.array(parameters2[ parameter_index:parameter_index + self.rotation_parametrization.num_parameters]) parameter_index += self.rotation_parametrization.num_parameters interpolator = self._get_rotation_interpolator(joint, rotation_params1, rotation_params2) try: result = list(interpolator.interpolate(rotation_params1, rotation_params2, amount)) except ZeroNormedQuaternion as exception: print "WARNING: %s (joint: %s)" % (exception, joint.definition.name) result = rotation_params1 return result, parameter_index def _get_rotation_interpolator(self, joint, r1, r2): if self.rotation_parametrization == EulerToQuaternion: return self._get_quaternion_interpolator(joint, r1, r2) else: return linear_interpolator def _get_quaternion_interpolator(self, joint, r1, r2): if self.args.naive_quaternion_interpolation: return static_quaternions_interpolator else: joint_index = joint.definition.index if joint_index not in self._rotation_interpolators: interpolator = DynamicQuaternionsInterpolator(joint.definition.name) interpolator.set_max_angular_step(self._max_angular_step) self._rotation_interpolators[joint_index] = interpolator return self._rotation_interpolators[joint_index] def set_friction(self, enable_friction): self._enable_friction = enable_friction self._normalized_constrainers.set_friction(enable_friction) self._unnormalized_constrainers.set_friction(enable_friction) def get_friction(self): return self._enable_friction def set_confinement(self, enable_confinement): self._normalized_constrainers.set_confinement(enable_confinement) self._unnormalized_constrainers.set_confinement(enable_confinement) def set_confinement_rate(self, rate): self._normalized_constrainers.set_confinement_rate(rate) self._unnormalized_constrainers.set_confinement_rate(rate) def set_confinement_target_position(self, target_position): self._normalized_constrainers.set_confinement_target_position(target_position) self._unnormalized_constrainers.set_confinement_target_position(target_position) def set_max_angular_step(self, max_angular_step): self._max_angular_step = max_angular_step for interpolator in self._rotation_interpolators.values(): interpolator.set_max_angular_step(max_angular_step)
def showcase_wavs(cfg, raw_plot=True, spec_plot=True, mfcc_plot=True, show_plot=False): """ showcase wavs """ # plot path plot_path = '../docu/thesis/3_signal/figs/' # change params feature_params = cfg['feature_params'].copy() feature_params['n_ceps_coeff'] = 32 feature_params['norm_features'] = True # init feature extractor feature_extractor = FeatureExtractor(feature_params) # wav, anno dir wav_dir, anno_dir = '../ignore/my_recordings/showcase_wavs/', '../ignore/my_recordings/showcase_wavs/annotation/' # analyze some wavs for wav, anno in zip(glob(wav_dir + '*.wav'), glob(anno_dir + '*.TextGrid')): # info print("\nwav: ", wav), print("anno: ", anno) # load file x, _ = librosa.load(wav, sr=feature_params['fs']) # raw waveform if raw_plot: plot_waveform(x, feature_params['fs'], anno_file=anno, hop=feature_extractor.hop, plot_path=plot_path, name='signal_raw_' + wav.split('/')[-1].split('.')[0] + '_my', show_plot=show_plot) # spectogram if spec_plot: plot_spec_profile(x, feature_extractor.calc_spectogram(x).T, feature_params['fs'], feature_extractor.N, feature_extractor.hop, anno_file=anno, plot_path=plot_path, title=wav.split('/')[-1].split('.')[0] + '_my', name='signal_spec-lin_' + wav.split('/')[-1].split('.')[0] + '_my', show_plot=show_plot) plot_spec_profile(x, feature_extractor.calc_spectogram(x).T, feature_params['fs'], feature_extractor.N, feature_extractor.hop, log_scale=True, anno_file=anno, plot_path=plot_path, title=wav.split('/')[-1].split('.')[0] + '_my', name='signal_spec-log_' + wav.split('/')[-1].split('.')[0] + '_my', show_plot=show_plot) # mfcc if mfcc_plot: mfcc, bon_pos = feature_extractor.extract_mfcc( x, reduce_to_best_onset=False) plot_mfcc_profile(x, cfg['feature_params']['fs'], feature_extractor.N, feature_extractor.hop, mfcc, anno_file=anno, sep_features=True, bon_pos=bon_pos, frame_size=cfg['feature_params']['frame_size'], plot_path=plot_path, name='signal_mfcc_' + wav.split('/')[-1].split('.')[0] + '_my', close_plot=False, show_plot=show_plot)
mlp_model = MLPLearningModel(name="default", hidden_layer_sizes=(80, ), max_iter=500) training_pdf = args.symbols_file labels_file = args.labels_file if args.reset and os.path.isdir(state_dir): shutil.rmtree(state_dir) if not os.path.isfile(training_pdf): raise IOError("Symbols file %s not found." % training_pdf) if not os.path.isfile(labels_file): raise IOError("Labels file %s not found." % labels_file) training_file = fe.TrainingFileSpec( training_pdf, fe.DEFAULT_GRID_SPEC, labels_file, dpi=96, white_threshold=fe.DEFAULT_WHITE_THRESHOLD) feature_matrix, feature_labels = \ FeatureExtractor.extract_labelled_feature_vectors( training_file, rotations=np.zeros(1)) if not os.path.isdir(state_dir): mlp_model.train(feature_matrix, feature_labels) mlp_model.dump(state_dir) print("Model trained. Program finished.") else: mlp_model.load(state_dir) mlp_model.improve(feature_matrix, feature_labels) mlp_model.dump(state_dir) print("Model improved. Program finished.")
def run(self): """Run whole data loading, feature extraction, model training and regressing pipeline.""" if self.mode == "extract": print("Extracting features") train = FeatureExtractor("train").run() dev = FeatureExtractor("dev").run() print("Saving features") np.save("saved_features/train_lsr", train.lsr) np.save("saved_features/train_nlp", train.feats) np.save("saved_features/train_scores", train.scores) np.save("saved_features/dev_lsr", dev.lsr) np.save("saved_features/dev_nlp", dev.feats) np.save("saved_features/dev_scores", dev.scores) else: # Load saved extracted features print("Loading saved features") split = False if self.full_data else True train, dev = load_features(split=split, nt=True) if self.params["upsample"]: train = self.upsample(train) train_loader = create_loader(train, self.params["batch_size_train"]) dev_loader = create_loader(dev, validate=True) # We set a random seed to ensure that results are reproducible. # Also set a cuda GPU if available if torch.cuda.is_available(): torch.backends.cudnn.deterministic = True GPU = True else: GPU = False device_idx = 0 if GPU: device = torch.device( "cuda:" + str(device_idx) if torch.cuda.is_available() else "cpu") else: device = torch.device("cpu") print(f"Running on {device}") if self.bUseConv: model = RecursiveNN( ModelBlock, self.params["conv_dict"], self.params["conv_ffnn_dict"], BASELINE_dim=self.params["NBaseline"], ) else: model = RecursiveNN_Linear( in_features=2048, N1=self.params["N1"], N2=self.params["N2"], out_features=self.params["out_features"], dropout=self.params["dropout"], leaky_relu=self.params["leaky_relu"], ) model = model.to(device) weights_initialiser = True if weights_initialiser: model.apply(weights_init) params_net = sum(p.numel() for p in model.parameters() if p.requires_grad) print("Total number of parameters in Model is: {}".format(params_net)) print(model) optimizer = optim.Adam(model.parameters(), lr=self.params["lr"]) scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=self.params["step_size"], gamma=self.params["gamma"]) date_string = (str(datetime.datetime.now())[:16].replace(":", "-").replace( " ", "-")) writer = SummaryWriter(logdir + date_string) print("Running model") for epoch in range(self.params["epochs"]): train_model( model, train_loader, optimizer, epoch, log_interval=1000, scheduler=scheduler, writer=writer, ) test_loss = test_model(model, dev_loader, epoch, writer=writer) torch.save(model, "model.pt") self.model = model
class SpeechCommandsDataset(AudioDataset): """ Speech Commands Dataset extraction and set creation """ def __init__(self, dataset_cfg, feature_params, collect_wavs=False, verbose=False): # parent init super().__init__(dataset_cfg, feature_params, collect_wavs=collect_wavs, verbose=verbose) # feature extractor self.feature_extractor = FeatureExtractor(feature_params=self.feature_params) # short vars self.N = self.feature_extractor.N self.hop = self.feature_extractor.hop # create plot plaths if not already exists create_folder(list(self.plot_paths.values())) # recreate if self.dataset_cfg['recreate'] or not check_folders_existance(self.wav_folders, empty_check=True): # delete old data delete_files_in_path(self.wav_folders, file_ext=self.dataset_cfg['file_ext']) # create folder wav folders create_folder(self.wav_folders) # create sets (specific to dataset) self.create_sets() # get audio files from sets self.get_audiofiles() self.get_annotation_files() def create_sets(self): """ copy wav files from dataset path to wav folders with splitting """ # get all class directories except the ones starting with _ class_dirs = glob(self.dataset_path + '[!_]*/') # run through all class directories for class_dir in class_dirs: # extract label label = class_dir.split('/')[-2] # get all .wav files wavs = glob(class_dir + '*' + self.dataset_cfg['file_ext']) # calculate split numbers in train, test, eval and split position n_split = (len(wavs) * np.array(self.dataset_cfg['split_percs'])).astype(int) n_split_pos = np.cumsum(n_split) # print some info print("label: [{}]\tn_split: [{}]\ttotal:[{}]".format(label, n_split, np.sum(n_split))) # actual path p = 0 # shuffle if self.dataset_cfg['shuffle_wavs']: np.random.shuffle(wavs) # run through each path for i, wav in enumerate(wavs): # split in new path if i >= n_split_pos[p]: p += 1 # stop if out of range (happens at rounding errors) if p >= len(self.wav_folders): break # wav name wav_name = wav.split('/')[-1].split('.')[0] # copy files to folder copyfile(wav, self.wav_folders[p] + label + str(i) + '--' + wav_name + self.dataset_cfg['file_ext']) def extract_features(self): """ extract mfcc features and save them """ print("\n--feature extraction:") # create folder structure create_folder(self.feature_folders) for i, (set_name, wavs, annos) in enumerate(zip(self.set_names, self.set_audio_files, self.set_annotation_files)): print("{}) extract set: {} with label num: {}".format(i, set_name, len(wavs))) # examples with splits n_examples = int(self.dataset_cfg['n_examples'] * self.dataset_cfg['split_percs'][i]) # extract data x, y, t, index = self.extract_mfcc_data(wavs=wavs, annos=annos, n_examples=n_examples, set_name=set_name) if self.feature_params['use_mfcc_features'] else self.extract_raw_data(wavs=wavs, annos=annos, n_examples=n_examples, set_name=set_name) # add noise if requested if self.dataset_cfg['add_noise'] and self.feature_params['use_mfcc_features']: x, y, index = self.add_noise_to_dataset(x, y, index, n_examples) # print label stats self.label_stats(y) # save mfcc data file np.savez(self.feature_files[i], x=x, y=y, t=t, index=index, params=self.feature_params) print("--save data to: ", self.feature_files[i]) def extract_mfcc_data(self, wavs, annos, n_examples, set_name=None): """ extract mfcc data from wav-files wavs must be in a 2D-array [[wavs_class1], [wavs_class2]] so that n_examples will work properly """ # mfcc_data: [n x m x l], labels and index mfcc_data, label_data, index_data = np.empty(shape=(0, self.channel_size, self.feature_size, self.frame_size), dtype=np.float64), [], [] # extract class wavs for class_wavs, class_annos in zip(wavs, annos): # class annotation file names extraction class_annos_file_names = [l + i for f, i, l in [self.file_naming_extraction(a, file_ext='.TextGrid') for a in class_annos]] # number of class examples num_class_examples = 0 # run through each example in class wavs for wav in class_wavs: # extract file namings file_name, file_index, label = self.file_naming_extraction(wav, file_ext=self.dataset_cfg['file_ext']) # get annotation if available anno = None if label + file_index in class_annos_file_names: anno = class_annos[class_annos_file_names.index(label + file_index)] # load and pre-process audio x, wav_is_useless = self.wav_pre_processing(wav) if wav_is_useless: continue # print some info if self.verbose: print("wav: [{}] with label: [{}], samples=[{}], time=[{}]s".format(wav, label, len(x), len(x) / self.feature_params['fs'])) # extract feature vectors [m x l] mfcc, bon_pos = self.feature_extractor.extract_mfcc(x, reduce_to_best_onset=False) # collect wavs if self.collect_wavs: self.pre_wavs.append((librosa.util.normalize(x), label + str(file_index) + '_' + set_name, bon_pos)) # plot mfcc features plot_mfcc_profile(x, self.feature_params['fs'], self.feature_extractor.N, self.feature_extractor.hop, mfcc, anno_file=anno, onsets=None, bon_pos=bon_pos, mient=None, minreg=None, frame_size=self.frame_size, plot_path=self.plot_paths['mfcc'], name=label + str(file_index) + '_' + set_name, enable_plot=self.dataset_cfg['enable_plot']) # damaged file check if self.dataset_cfg['filter_damaged_files']: # handle damaged files if self.detect_damaged_file(mfcc, wav): continue # add to mfcc_data container mfcc_data = np.vstack((mfcc_data, mfcc[np.newaxis, :, :, bon_pos:bon_pos+self.frame_size])) label_data.append(label) index_data.append(label + file_index) # update number of examples per class num_class_examples += 1 # stop if desired examples are reached if num_class_examples >= n_examples: break return mfcc_data, label_data, None, index_data def extract_raw_data(self, wavs, annos, n_examples, set_name=None): """ raw data extraction """ # raw data: [n x m], labels and index raw_data, label_data, target_data, index_data = np.empty(shape=(0, self.channel_size, self.raw_frame_size), dtype=np.float64), [], np.empty(shape=(0, self.raw_frame_size), dtype=np.int64), [] # extract class wavs for class_wavs, class_annos in zip(wavs, annos): # class annotation file names extraction class_annos_file_names = [l + i for f, i, l in [self.file_naming_extraction(a, file_ext='.TextGrid') for a in class_annos]] # number of class examples num_class_examples = 0 # run through each example in class wavs for wav in class_wavs: # extract file namings file_name, file_index, label = self.file_naming_extraction(wav, file_ext=self.dataset_cfg['file_ext']) # get annotation if available anno = class_annos[class_annos_file_names.index(label + file_index)] if label + file_index in class_annos_file_names else None # load and pre-process audio x, wav_is_useless = self.wav_pre_processing(wav) if wav_is_useless: continue # print some info if self.verbose: print("wav: [{}] with label: [{}], samples=[{}], time=[{}]s".format(wav, label, len(x), len(x) / self.feature_params['fs'])) # extract raw samples from region of energy raw, bon_pos = self.feature_extractor.get_best_raw_samples(x) # add dither and do normalization raw = self.wav_post_processing(raw) # quantize data t = self.feature_extractor.quantize(raw) # plot waveform if self.dataset_cfg['enable_plot']: plot_waveform(x, self.feature_params['fs'], bon_samples=[bon_pos, bon_pos+self.raw_frame_size], title=label + file_index, plot_path=self.plot_paths['waveform'], name=label + file_index, show_plot=False, close_plot=True) # collect wavs if self.collect_wavs: self.pre_wavs.append((librosa.util.normalize(x), label + str(file_index) + '_' + set_name, bon_pos / self.hop)) # add to mfcc_data container raw_data = np.vstack((raw_data, raw[np.newaxis, :])) target_data = np.vstack((target_data, t)) label_data.append(label) index_data.append(label + file_index) # update number of examples per class num_class_examples += 1 # stop if desired examples are reached if num_class_examples >= n_examples: break return raw_data, label_data, target_data, index_data def detect_damaged_file(self, mfcc, wav): """ detect if file is damaged """ # energy calc #e = np.einsum('ij,ji->j', mfcc, mfcc.T) #e = e / np.max(e) # calculate damaged score of energy deltas if mfcc.shape[1] == 39: z_est, z_lim = np.sum(np.abs(mfcc[0, 37:39, :])), 60 #if mfcc.shape[0] == 39: z_est = np.sum(mfcc[37:39, :] @ mfcc[37:39, :].T) #else: z_est = np.sum(np.abs(np.diff(mfcc[-1, :]))) #else: z_est = np.diff(mfcc[-1, :]) @ np.diff(mfcc[-1, :]).T #else: z_est = np.sum(mfcc[0, :]) #else: z_est = np.diff(mfcc[0, :]) @ np.diff(mfcc[0, :]).T #else: z_est = np.abs(np.diff(mfcc[0, :])) @ mfcc[0, :-1].T #else: z_est = np.sum(np.diff(mfcc, axis=1) @ np.diff(mfcc, axis=1).T) #else: z_est = np.sum(e) #else: z_est = np.diff(e) @ np.diff(e).T else: z_est, z_lim = mfcc[0, 0, :-1] @ np.abs(np.diff(mfcc[0, 0, :])).T, 3.5 # add score to list self.damaged_score_list.append(z_est) # damaged file is_damaged = z_est > z_lim # add to damaged file list if is_damaged: self.damaged_file_list.append((wav, z_est)) # return score and damaged indicator return is_damaged
data = f[1].data names = f[1].columns.names f.close() print "\n\n\nThese are the column names:\n%s\n\n" % names Ng = np.where(data.field("type") == 3)[0].size Ns = np.where(data.field("type") == 6)[0].size print "\nSDSS says there are %d galaxies and %d stars.\n\n" % (Ng, Ns) ylim = np.inf featurenames = ["cmodelmag", "psffwhm", "petror50", "petror90"] targetnames = ["psfmag", "cmodelmag"] filters = ["u", "g", "r", "i", "z"] x = FeatureExtractor(data, featurenames, filters, color_band="r", scale_kind=None, mag_range=None) data = data[x.idx] y = FeatureExtractor(data, targetnames, filters, color_band=None, scale_kind=None, mag_range=None) # taylor to target, set for psf - model y.features[:, :5] = y.features[:, :5] - y.features[:, 5:10] y.features[:, 5:10] = np.sqrt(y.features[:, 10:15] ** 2.0 + y.features[:, 15:20] ** 2.0) y.features = y.features[:, :10] # restrict x range xlim = (19.5, 20.5) ind = (x.features[:, 2] > xlim[0]) & (x.features[:, 2] < xlim[1]) x.features = x.features[ind] y.features = y.features[ind] y.Ndata = y.features.shape[0]
def main(): parser = get_parser() args = parser.parse_args() feature_extractor = FeatureExtractor() if args.pipeline_type == "analysis": text_preprocessor = TextPreProcessor( stop_words_file_path=args.stopwords_file_path) analyser = DataAnalyser(input_file=args.input_file_path, text_preprocessor=text_preprocessor) analyser.get_data_distribution(plot_bar=args.plot_bar) analyser.get_word_weights(word_thresh=args.word_thresh) if args.word_cloud: analyser.generate_word_cloud() elif args.pipeline_type == "model_selection": text_preprocessor = TextPreProcessor( stop_words_file_path=args.stopwords_file_path) training_data_df = load_training_data(args.train_file_path) training_data_df["sentence"] = training_data_df["sentence"].map( text_preprocessor.process) features = feature_extractor.get_features_for_training( training_data_df["sentence"], args.vectorizer) labels = training_data_df["class"] apply_cross_validation( features=features, labels=labels, k_folds=args.kfolds, use_svm=args.use_svm, use_naive_bayes=args.use_naive_bayes, use_random_forest=args.use_random_forest, use_logistic_regression=args.use_logistic_regression, use_xgboost=args.use_xgboost, use_gradient_boosting=args.use_gradient_boosting, plot_cv_graph=True, ) elif args.pipeline_type == "training": trainer = Trainer( train_file_path=args.train_file_path, val_file_path=args.val_file_path, stop_words_file_path=args.stopwords_file_path, model_name=args.best_model, feature_extractor=feature_extractor, ) training_data_df = load_training_data(args.train_file_path) trainer.train( training_data_df, split_test_size=args.split_size, vectorizer_name=args.vectorizer, get_classification_report=args.get_classification_report, get_confusion_matrix=args.get_confusion_matrix, ) validation_data_df = load_validation_data(args.val_file_path) trainer.validate(validation_data_df, vectorizer_name=args.vectorizer) if args.model_check_point_path: trainer.save_trained_model(args.model_check_point_path) elif args.pipeline_type == "prediction": if not args.stopwords_file_path: predictor = Predictor() else: predictor = Predictor(stop_words_file=args.stopwords_file_path) if args.input_file_path: predictor.predict_csv(args.input_file_path, args.output_file_path, args.model_path) if args.test_input: model, vectorizer = predictor.unpickle_the_model(args.model_path) predictor.predict(args.test_input, model, vectorizer)