def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] y_val = inputs['phonemes'] drops_forw_states_val = inputs['drops_forw_states'] drops_forw_cells_val = inputs['drops_forw_cells'] drops_forw_igates_val = inputs['drops_forw_igates'] drops_back_states_val = inputs['drops_back_states'] drops_back_cells_val = inputs['drops_back_cells'] drops_back_igates_val = inputs['drops_back_igates'] y_mask_val = inputs['phonemes_mask'] y_hat = self.prediction_func( x_val, x_mask_val, drops_forw_states_val, drops_forw_cells_val, drops_forw_igates_val, drops_back_states_val, drops_back_cells_val, drops_back_igates_val) y_predict = numpy.argmax(y_hat, axis=2) for batch in xrange(inputs['features'].shape[1]): y_val_cur = y_val[:sum(y_mask_val[:, batch]), batch] predicted = y_predict[:sum(x_mask_val[:, batch]), batch] predicted = ctc_strip(predicted) predictions = [ self.phoneme_dict[phone_ind] for phone_ind in predicted if self.phoneme_dict[phone_ind] not in self.black_list ] targets = [ self.phoneme_dict[phone_ind] for phone_ind in y_val_cur if self.phoneme_dict[phone_ind] not in self.black_list ] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' % i) loss += Evaluation.wer([predictions], [targets]) num_examples += 1 print '.. found sequence example:', ' '.join(predictions) print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' per = loss.sum() / num_examples return {'per': per}
def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] y_val = inputs['phonemes'] y_mask_val = inputs['phonemes_mask'] for batch_ind in xrange(inputs['features'].shape[1]): if x_val.ndim == 2: input_beam = numpy.tile(x_val[:, batch_ind][:, None], (1, self.beam_size)) else: input_beam = numpy.tile(x_val[:, batch_ind, :][:, None, :], (1, self.beam_size, 1)) input_mask_beam = numpy.tile(x_mask_val[:, batch_ind][:, None], (1, self.beam_size)) predictions, _ = self.beam_search.search( {self.x: input_beam, self.x_mask: input_mask_beam}, self.eol_symbol, 100) predictions = [self.phoneme_dict[phone_ind] for phone_ind in predictions[0] if self.phoneme_dict[phone_ind] not in self.black_list][1:-1] targets = y_val[:sum(y_mask_val[:, batch_ind]), batch_ind] targets = [self.phoneme_dict[phone_ind] for phone_ind in targets if self.phoneme_dict[phone_ind] not in self.black_list][1:-1] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' %i) loss += Evaluation.wer([predictions], [targets]) num_examples += 1 print '.. found sequence example:', ' '.join(predictions) print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' per = loss.sum() / num_examples return {'per': per}
def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] #transpose y_val = inputs['phonemes'] y_mask_val = inputs['phonemes_mask'] y_hat = self.prediction_func(x_val, x_mask_val) y_predict = numpy.argmax(y_hat, axis=2) for batch in xrange(inputs['features'].shape[0]): y_val_cur = y_val[:sum(y_mask_val[:, batch]), batch] predicted = y_predict[:sum(x_mask_val[:, batch]), batch] predicted = ctc_strip(predicted) predictions = [self.phoneme_dict[phone_ind] for phone_ind in predicted if self.phoneme_dict[phone_ind] not in self.black_list] targets = [self.phoneme_dict[phone_ind] for phone_ind in y_val_cur if self.phoneme_dict[phone_ind] not in self.black_list] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' %i) loss += Evaluation.wer([predictions], [targets]) num_examples += 1 print '.. found sequence example:', ' '.join(predictions) print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' per = loss.sum() / num_examples return {'per': per}
def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] y_val = inputs['triphones'] y_mask_val = inputs['triphones_mask'] y_hat = self.prediction_func(x_val, x_mask_val) y_predict = numpy.argmax(y_hat, axis=2) for batch in xrange(inputs['features'].shape[1]): y_val_cur = y_val[:sum(y_mask_val[:, batch]), batch] predicted = y_predict[:sum(x_mask_val[:, batch]), batch] predictions = [str(trihone) for triphone in predicted] targets = [str(triphone) for triphone in y_val_cur] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' % i) loss += (sum(numpy.not_equal(y_val_cur, predicted))).astype( 'float32') /\ len(predicted) num_examples += 1 #print '.. found sequence example:', ' '.join(predictions) #print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' fer = loss / num_examples return {'fer': fer}
def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] y_val = inputs['triphones'] y_mask_val = inputs['triphones_mask'] y_hat = self.prediction_func(x_val, x_mask_val) y_predict = numpy.argmax(y_hat, axis=2) for batch in xrange(inputs['features'].shape[1]): y_val_cur = y_val[:sum(y_mask_val[:, batch]), batch] predicted = y_predict[:sum(x_mask_val[:, batch]), batch] predictions = [str(trihone) for triphone in predicted] targets = [str(triphone) for triphone in y_val_cur] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' %i) loss += (sum(numpy.not_equal(y_val_cur, predicted))).astype( 'float32') /\ len(predicted) num_examples += 1 #print '.. found sequence example:', ' '.join(predictions) #print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' fer = loss / num_examples return {'fer': fer}
def verify_groupby(*args, **kwargs): if 'n' in kwargs: if 'm' not in kwargs: raise ValueError('got n without m') pickle = True n = kwargs.pop('n') m = kwargs.pop('m') elif 'm' in kwargs: raise ValueError('got m without n') else: pickle = False n = m = None if 'pickle_outer' in kwargs: pickle_outer = kwargs.pop('pickle_outer') else: pickle_outer = None reference = itertools.groupby(*args, **kwargs) actual = groupby(*args, **kwargs) outer_iters = 0 while True: if outer_iters == pickle_outer: actual = cPickle.loads(cPickle.dumps(actual)) try: ref_key, ref_grouper = next(reference) except StopIteration: check_stops(actual) break try: actual_key, actual_grouper = next(actual) except StopIteration: assert False, "prematurely exhausted; expected {}".format(ref_key) if pickle: this_n = n[0] n = n[1:] this_m = m[0] m = m[1:] verify_pickle(partial(_identity, actual_grouper), partial(_identity, ref_grouper), this_n, this_m) else: verify_same(partial(_identity, actual_grouper), partial(_identity, ref_grouper), None) outer_iters += 1
def verify_groupby(*args, **kwargs): if 'n' in kwargs: if 'm' not in kwargs: raise ValueError('got n without m') pickle = True n = kwargs.pop('n') m = kwargs.pop('m') elif 'm' in kwargs: raise ValueError('got m without n') else: pickle = False n = m = None if 'pickle_outer' in kwargs: pickle_outer = kwargs.pop('pickle_outer') else: pickle_outer = None reference = itertools.groupby(*args, **kwargs) actual = groupby(*args, **kwargs) outer_iters = 0 while True: if outer_iters == pickle_outer: actual = cPickle.loads(cPickle.dumps(actual)) try: ref_key, ref_grouper = next(reference) except StopIteration: check_stops(actual) break try: actual_key, actual_grouper = next(actual) except StopIteration: assert False, "prematurely exhausted; expected {}".format(ref_key) if pickle: this_n = n[0] n = n[1:] this_m = m[0] m = m[1:] verify_pickle(partial(_identity, actual_grouper), partial(_identity, ref_grouper), this_n, this_m) else: verify_same(partial(_identity, actual_grouper), partial(_identity, ref_grouper), None) outer_iters += 1
def network_evaluation(predict_fn, data_stream, phoneme_dict, black_list): ##################### # get data iterator # ##################### data_iterator = data_stream.get_epoch_iterator() ######################### # set evaluation result # ######################### total_ctc = 0. total_per = 0. total_sample = 0. total_batch = 0. ############## # evaluation # ############## # for each batch iteration for i, data in enumerate(data_iterator): # get input data input_data = data[0] input_mask = data[1] # get target data target_data = data[2] target_mask = data[3] # get prediction data predict_output = predict_fn(input_data, input_mask, target_data, target_mask) predict_idx = predict_output[0] predict_ctc_cost = predict_output[1] pred_cost_per_char = predict_output[2] # add up ctc cost total_ctc += predict_ctc_cost total_batch += 1. # for each data, per evaluation for j in range(input_data.shape[0]): cur_target_data = target_data[j, :numpy.sum(target_mask[j])] cur_predict_data = predict_idx[j, :numpy.sum(input_mask[j])] cur_predict_data = ctc_strip(cur_predict_data) cur_target_phoneme = [phoneme_dict[phone_ind] for phone_ind in cur_target_data if phoneme_dict[phone_ind] not in black_list] cur_predict_phoneme = [phoneme_dict[phone_ind] for phone_ind in cur_predict_data if phoneme_dict[phone_ind] not in black_list] targets = [x[0] for x in groupby(cur_target_phoneme)] predictions = [x[0] for x in groupby(cur_predict_phoneme)] total_per += Evaluation.wer([predictions], [targets]) total_sample += 1 total_ctc = total_ctc/total_batch total_per = total_per.sum()/total_sample return total_ctc, total_per
def evaluate(self, data_stream, train=False, file_pred=None, file_targets=None): loss = 0. num_examples = 0 iterator = data_stream.get_epoch_iterator() if train: print 'Train evaluation started' i = 0 for inputs in iterator: inputs = dict(zip(data_stream.sources, inputs)) x_mask_val = inputs['features_mask'] x_val = inputs['features'] y_val = inputs['phonemes'] y_mask_val = inputs['phonemes_mask'] for batch_ind in xrange(inputs['features'].shape[1]): if x_val.ndim == 2: input_beam = numpy.tile(x_val[:, batch_ind][:, None], (1, self.beam_size)) else: input_beam = numpy.tile(x_val[:, batch_ind, :][:, None, :], (1, self.beam_size, 1)) input_mask_beam = numpy.tile(x_mask_val[:, batch_ind][:, None], (1, self.beam_size)) predictions, _ = self.beam_search.search( { self.x: input_beam, self.x_mask: input_mask_beam }, self.eol_symbol, 100) predictions = [ self.phoneme_dict[phone_ind] for phone_ind in predictions[0] if self.phoneme_dict[phone_ind] not in self.black_list ][1:-1] targets = y_val[:sum(y_mask_val[:, batch_ind]), batch_ind] targets = [ self.phoneme_dict[phone_ind] for phone_ind in targets if self.phoneme_dict[phone_ind] not in self.black_list ][1:-1] predictions = [x[0] for x in groupby(predictions)] targets = [x[0] for x in groupby(targets)] i += 1 if file_pred: file_pred.write(' '.join(predictions) + '(%d)\n' % i) if file_targets: file_targets.write(' '.join(targets) + '(%d)\n' % i) loss += Evaluation.wer([predictions], [targets]) num_examples += 1 print '.. found sequence example:', ' '.join(predictions) print '.. real output was: ', ' '.join(targets) if train: break if train: print 'Train evaluation finished' per = loss.sum() / num_examples return {'per': per}