def __init__(self, conf): '''TextProcessor constructor Args: conf: processor configuration as a configparser ''' #create the normalizer self.normalizer = normalizer_factory.factory( conf.get('processor', 'normalizer')) self.alphabet = conf.get('processor', 'alphabet').strip().split(' ') self.alphabet = [c if c != '\\;' else ';' for c in self.alphabet] print 'in text processor' print conf.sections() print conf.items('processor') #initialize the metadata self.max_length = 0 self.sequence_length_histogram = np.zeros(0, dtype=np.int32) if conf.get('processor', 'nonesymbol') != 'None': self.nonesymbol = conf.get('processor', 'nonesymbol') else: self.nonesymbol = '' super(TextProcessor, self).__init__(conf)
def __init__(self, conf): '''TextProcessor constructor Args: conf: processor configuration as a configparser ''' #create the normalizer self.normalizer = normalizer_factory.factory( conf.get('processor', 'normalizer')) self.alphabet = conf.get('processor', 'alphabet').split(' ') #initialize the metadata self.max_length = 0 self.sequence_length_histogram = np.zeros(0, dtype=np.int32) if conf.has_option('processor', 'nonesymbol'): self.nonesymbol = conf.get('processor', 'nonesymbol') else: self.nonesymbol = '' super(TextProcessor, self).__init__(conf)
this file will do the dataprep for lm training''' import os from six.moves import configparser from nabu.processing.target_normalizers import normalizer_factory #pointer to the confif file database_cfg_file = 'config/lm_databases/TIMIT.conf' #read the database config file database_cfg = configparser.ConfigParser() database_cfg.read(database_cfg_file) database_cfg = dict(database_cfg.items('database')) #create the text normalizer normalizer = normalizer_factory.factory(database_cfg['normalizer']) print '------- normalizing training text -----------' sourcefiles = database_cfg['train_data'].split(' ') if not os.path.isdir(database_cfg['train_dir']): os.makedirs(database_cfg['train_dir']) target_fid = open(os.path.join(database_cfg['train_dir'], 'text'), 'w') max_num_chars = 0 numlines = 0 #read the textfiles line by line, normalize and write in target file for sourcefile in sourcefiles: with open(sourcefile) as fid: for line in fid.readlines(): normalized = normalizer(line.strip()) max_num_chars = max(max_num_chars, len(normalized.split(' ')))