def InitData(self): """ initialize dataflow as the mode :return: initialized dataflow """ config = self._config # speech data parsed_cfg = configparser.ConfigParser() parsed_cfg.read(os.path.join(config.train_data_expdir, 'database.conf')) name = parsed_cfg.sections()[0] conf = dict(parsed_cfg.items(name)) proc_cfg = configparser.ConfigParser() proc_cfg.read(os.path.join(config.train_data_expdir, 'processor.cfg')) self.processor = processor_factory.factory( proc_cfg.get('processor', 'processor'))(proc_cfg) datafile = conf['datafiles'].split(' ')[0] #self._dataflow = cycle(open(datafile).readlines()) # text data text_parsed_cfg = configparser.ConfigParser() text_parsed_cfg.read( os.path.join(config.train_text_data_expdir, 'database.conf')) text_data_name = text_parsed_cfg.sections()[0] text_conf = dict(text_parsed_cfg.items(text_data_name)) text_proc_cfg = configparser.ConfigParser() text_proc_cfg.read( os.path.join(config.train_text_data_expdir, 'processor.cfg')) self.text_processor = processor_factory.factory( text_proc_cfg.get('processor', 'processor'))(text_proc_cfg) text_datafile = text_conf['datafiles'].split(' ')[0] self._dataflow = (cycle(open(datafile).readlines()), cycle(open(text_datafile).readlines())) self._alphabet = dict( text_proc_cfg.items('processor'))['alphabet'].split(' ') print(self._alphabet) self._lookup = dict() for i in range(len(self._alphabet)): self._lookup[self._alphabet[i]] = i print(self._lookup) #print(len(open(datafile).readlines())) return self._dataflow
def __init__(self, dataconf, segment_lengths=['full']): '''DataReader constructor Args: dataconf: the database configuration segment_lengths: A list containing the desired lengths of segments. Possibly multiple segment lengths ''' if len(segment_lengths) > 1: print( 'Warning: Not yet implemented __call__ correctly for multiple' 'segments. The returned utt_info, does not contain the _part sufix and' 'processed returns only 1 processed') self.segment_lengths = segment_lengths #read the processor config proc_cfg_file = dataconf['processor_config'] parsed_proc_cfg = configparser.ConfigParser() parsed_proc_cfg.read(proc_cfg_file) proc_cfg = dict(parsed_proc_cfg.items('processor')) #create a processor self.processor = processor_factory.factory(proc_cfg['processor'])( proc_cfg, self.segment_lengths) #get the datafiles lines datafile = dataconf[ 'datafiles'] #TODO: for the moment expecting only 1 file, but this also makes sense? if datafile[-3:] == '.gz': open_fn = gzip.open else: open_fn = open f = open(datafile) self.datafile_lines = f.readlines()
def main(expdir): '''main function''' #read the data conf file parsed_cfg = configparser.ConfigParser() parsed_cfg.read(os.path.join(expdir, 'database.cfg')) #loop over the sections in the data config name = parsed_cfg.sections()[0] #read the section conf = dict(parsed_cfg.items(name)) #read the processor config proc_cfg = configparser.ConfigParser() proc_cfg.read(os.path.join(expdir, 'processor.cfg')) #create a processor processor = processor_factory.factory( proc_cfg.get('processor', 'processor'))(proc_cfg) #create a writer writer = tfwriter_factory.factory(conf['type'])(conf['dir']) #loop over the data files for datafile in conf['datafiles'].split(' '): if datafile[-3:] == '.gz': open_fn = gzip.open else: open_fn = open count = 0 #loop over the lines in the datafile for line in open_fn(datafile): #split the name and the data line splitline = line.strip().split(' ') name = splitline[0] dataline = ' '.join(splitline[1:]) print dataline try: #process the dataline processed = processor(dataline) #write the processed data to disk if processed is not None: writer.write(processed, name) except: count += 1 print count print count #write the metadata to file processor.write_metadata(conf['dir'])
def __init__(self, dataconfs, segment_lengths=['full']): """DataReader constructor Args: dataconfs: the database configuration segment_lengths: A list containing the desired lengths of segments. Possibly multiple segment lengths """ if len(segment_lengths) > 1: print( 'Warning: Not yet implemented __call__ correctly for multiple segments. The returned utt_info, does not ' \ 'contain the _part sufix and processed returns only 1 processed') self.segment_lengths = segment_lengths self.processors = [] self.start_index_set = [0] self.datafile_lines = [] for dataconf in dataconfs: # read the processor config proc_cfg_file = dataconf['processor_config'] if not os.path.isfile(proc_cfg_file): raise BaseException('%s does not exist' % proc_cfg_file) parsed_proc_cfg = configparser.ConfigParser() parsed_proc_cfg.read(proc_cfg_file) proc_cfg = dict(parsed_proc_cfg.items('processor')) # create a processor self.processors.append( processor_factory.factory(proc_cfg['processor'])( proc_cfg, self.segment_lengths)) # get the datafiles lines datafile = dataconf[ 'datafiles'] # TODO: for the moment expecting only 1 file, but this also makes sense? if datafile[-3:] == '.gz': open_fn = gzip.open else: open_fn = open f = open_fn(datafile) datalines = f.readlines() self.start_index_set.append(self.start_index_set[-1] + len(datalines)) self.datafile_lines.extend(datalines)
def main(expdir): '''main function''' #read the data conf file parsed_cfg = configparser.ConfigParser() parsed_cfg.read(os.path.join(expdir, 'database.cfg')) #loop over the sections in the data config name = parsed_cfg.sections()[0] #read the section conf = dict(parsed_cfg.items(name)) #the length of the segments. Possibly multiple segment lengths if 'segment_lengths' in conf: segment_lengths = conf['segment_lengths'].split(' ') else: segment_lengths = ['full'] if not os.path.exists(conf['store_dir']): os.makedirs(conf['store_dir']) else: print '%s already exists, skipping this section' % conf['store_dir'] return #read the processor config parsed_proc_cfg = configparser.ConfigParser() parsed_proc_cfg.read(os.path.join(expdir, 'processor.cfg')) proc_cfg = dict(parsed_proc_cfg.items('processor')) #create a processor processor = processor_factory.factory(proc_cfg['processor'])(proc_cfg, segment_lengths) #create the writers writers = dict() for seg_length in segment_lengths: writer_store_dir = os.path.join(conf['store_dir'],seg_length) writers[seg_length] = tfwriter_factory.factory(conf['writer_style'])(writer_store_dir) #before looping over the data, allow the processor to access the data (e.g. #for global mean and variance calculation) (or should this be done in init?) processor.pre_loop(conf) #loop over the data files for datafile in conf['datafiles'].split(' '): if datafile[-3:] == '.gz': open_fn = gzip.open else: open_fn = open #loop over the lines in the datafile for line in open_fn(datafile): #split the name and the data line splitline = line.strip().split(' ') utt_name = splitline[0] dataline = ' '.join(splitline[1:]) #process the dataline processed, _ = processor(dataline) #write the processed data to disk for seg_length in segment_lengths: for i,proc_seg in enumerate(processed[seg_length]): seg_utt_name = utt_name + '_part %d' %i writers[seg_length].write(proc_seg, seg_utt_name) #after looping over the data, allow the processor to access the data processor.post_loop(conf) #write the metadata to file processor.write_metadata(conf['store_dir'])
def main(expdir): """main function""" # read the data conf file parsed_cfg = configparser.ConfigParser() parsed_cfg.read(os.path.join(expdir, 'database.cfg')) # loop over the sections in the data config name = parsed_cfg.sections()[0] # read the section conf = dict(parsed_cfg.items(name)) # the length of the segments. Possibly multiple segment lengths if 'segment_lengths' in conf: segment_lengths = conf['segment_lengths'].split(' ') else: segment_lengths = ['full'] if conf['store_dir'] == '/esat/spchtemp/scratch/jzegers/dataforTF/sreMix_segmented_DANet_recs/singlefeatures_hamming_scipy/train_150k': start_ind = 106370 start_ind_per_segment_length = {'500': 239577, 'full': 106370} segment_lengths_still_to_process = segment_lengths else: start_ind = 0 start_ind_per_segment_length = { seg_len: 0 for seg_len in segment_lengths } if not os.path.exists(conf['store_dir']): os.makedirs(conf['store_dir']) segment_lengths_still_to_process = segment_lengths # copy config files to store_dir for archive purposes shutil.copyfile(os.path.join(expdir, 'database.cfg'), os.path.join(conf['store_dir'], 'database.cfg')) shutil.copyfile(os.path.join(expdir, 'processor.cfg'), os.path.join(conf['store_dir'], 'processor.cfg')) else: tmp = os.listdir(conf['store_dir']) if all([seg_len in tmp for seg_len in segment_lengths]): print('%s already exists, skipping this section' % conf['store_dir']) return else: segment_lengths_still_to_process = [ seg_len for seg_len in segment_lengths if seg_len not in tmp ] # read the processor config parsed_proc_cfg = configparser.ConfigParser() parsed_proc_cfg.read(os.path.join(expdir, 'processor.cfg')) proc_cfg = dict(parsed_proc_cfg.items('processor')) # create a processor processor = processor_factory.factory(proc_cfg['processor'])( proc_cfg, segment_lengths_still_to_process) # create the writers writers = dict() for seg_length in segment_lengths_still_to_process: writer_store_dir = os.path.join(conf['store_dir'], seg_length) writers[seg_length] = tfwriter_factory.factory(conf['writer_style'])( writer_store_dir, start_ind=start_ind_per_segment_length[seg_length]) # before looping over the data, allow the processor to access the data (e.g. # for global mean and variance calculation) (or should this be done in init?) processor.pre_loop(conf) # loop over the data files for datafile in conf['datafiles'].split(' '): if datafile[-3:] == '.gz': open_fn = gzip.open else: open_fn = open # loop over the lines in the datafile ind = 0 for line in open_fn(datafile): print(ind) if ind < start_ind: ind += 1 continue # split the name and the data line splitline = line.strip().split(' ') utt_name = splitline[0] dataline = ' '.join(splitline[1:]) # process the dataline processed, _ = processor(dataline) # write the processed data to disk for seg_length in segment_lengths_still_to_process: for i, proc_seg in enumerate(processed[seg_length]): seg_utt_name = utt_name + '_part %d' % i writers[seg_length].write(proc_seg, seg_utt_name) ind += 1 # after looping over the data, allow the processor to access the data processor.post_loop(conf) # write the metadata to file processor.write_metadata(conf['store_dir'])