def dump_data(self, out_fn, out_featdim): file_reader = read_dataset(self.data_spec, pad_zeros=True) #taking only one reader out_options = self.data_spec.copy() out_options['base_path'] = self.export_path #updating the base_path out_options['featdim'] = out_featdim out_options['writer_type'] = "NP" file_writer = write_dataset(out_options) batch_size = file_reader.batch_size while not file_reader.is_finish(): for batch_index in xrange(file_reader.nBatches): s_idx = batch_index * batch_size e_idx = s_idx + batch_size data = out_fn(file_reader.feat[s_idx:e_idx]) label = file_reader.label[s_idx:e_idx] if ((batch_index == file_reader.nBatches - 1) and (not file_reader.num_pad_frames == 0)): data = data[:-file_reader.num_pad_frames] label = label[:-file_reader.num_pad_frames] file_writer.write_data(data, label) file_reader.read_next_partition_data(pad_zeros=True) logger.debug('NP Dataexporter : data is exported to %s' % self.export_path)
def dump_data(self,out_fn,out_featdim): filepath = self.data_spec['base_path'] + os.sep + self.data_spec['filename'] copy_path = create_folder_structure_if_not_exists(self.export_path + os.sep + self.data_spec['filename']) shutil.copy(filepath,copy_path); #copies the file directly self.filehandle = open(filepath,'rb') line = self.filehandle.readline(); # reading file header header = line.split(); num_classes = int(header[1]); for idx in xrange(num_classes): level1_filename = self.filehandle.readline().strip(); level1_filepath = self.data_spec['base_path'] + os.sep + level1_filename #filename of individual classes copy_path = create_folder_structure_if_not_exists(self.export_path + os.sep + level1_filename) shutil.copy(level1_filepath,copy_path); #copies the index file directly from the source directly self.level1FileHandle = open(level1_filepath,'rb'); level2_filepath = self.level1FileHandle.readline().strip(); while level2_filepath.__len__()!= 0: in_child_options = self.data_spec.copy(); in_child_options['filename'] = level2_filepath #filename of individual classes in_child_options['reader_type'] = "TD" in_child_options['label'] = idx; file_reader = read_dataset(in_child_options,pad_zeros=True) #taking only one reader out_child_options = in_child_options.copy(); out_child_options['base_path'] = self.export_path; #updating the base_path out_child_options['featdim'] = out_featdim; out_child_options['writer_type'] = "TD" file_writer = write_dataset(out_child_options); batch_size=file_reader.batch_size while not file_reader.is_finish(): for batch_index in xrange(file_reader.nBatches): s_idx = batch_index * batch_size; e_idx = s_idx + batch_size data = out_fn(file_reader.feat[s_idx:e_idx]) label = file_reader.label[s_idx:e_idx]; if ((batch_index == file_reader.nBatches-1) and (not file_reader.num_pad_frames == 0)) : data=data[:-file_reader.num_pad_frames] label = label[:-file_reader.num_pad_frames] file_writer.write_data(data,label); file_reader.read_next_partition_data(pad_zeros=True); level2_filepath = self.level1FileHandle.readline().strip(); logger.debug('T2 Dataexporter : data is exported to %s' % self.export_path);
def dump_data(self, out_fn, out_featdim): filepath = self.data_spec['base_path'] + os.sep + self.data_spec[ 'filename'] copy_path = create_folder_structure_if_not_exists( self.export_path + os.sep + self.data_spec['filename']) shutil.copy(filepath, copy_path) #copies the file directly self.filehandle = open(filepath, 'rb') line = self.filehandle.readline() # reading file header header = line.split() num_classes = int(header[1]) for idx in xrange(num_classes): in_child_options = self.data_spec.copy() in_child_options['filename'] = self.filehandle.readline().strip( ) #filename of individual classes in_child_options['reader_type'] = "TD" in_child_options['label'] = idx file_reader = read_dataset(in_child_options, pad_zeros=True) #taking only one reader out_child_options = in_child_options.copy() out_child_options['base_path'] = self.export_path out_child_options['featdim'] = out_featdim out_child_options['writer_type'] = "TD" file_writer = write_dataset(out_child_options) batch_size = file_reader.batch_size while (not file_reader.is_finish()): for batch_index in xrange(file_reader.nBatches): s_idx = batch_index * batch_size e_idx = s_idx + batch_size data = out_fn(file_reader.feat[s_idx:e_idx]) label = file_reader.label[s_idx:e_idx] if ((batch_index == file_reader.nBatches - 1) and (not file_reader.num_pad_frames == 0)): data = data[:-file_reader.num_pad_frames] label = label[:-file_reader.num_pad_frames] file_writer.write_data(data, label) file_reader.read_next_partition_data(pad_zeros=True) logger.debug('T1 Dataexporter : data is exported to %s' % self.export_path)
def dump_data(self,out_fn,out_featdim): file_reader = read_dataset(self.data_spec,pad_zeros=True) #taking only one reader out_options = self.data_spec.copy(); out_options['base_path'] = self.export_path; #updating the base_path out_options['featdim'] = out_featdim; out_options['writer_type'] = "NP" file_writer = write_dataset(out_options); batch_size=file_reader.batch_size while not file_reader.is_finish(): for batch_index in xrange(file_reader.nBatches): s_idx = batch_index * batch_size; e_idx = s_idx + batch_size data = out_fn(file_reader.feat[s_idx:e_idx]) label = file_reader.label[s_idx:e_idx]; if ((batch_index == file_reader.nBatches-1) and (not file_reader.num_pad_frames == 0)) : data=data[:-file_reader.num_pad_frames] label = label[:-file_reader.num_pad_frames] file_writer.write_data(data,label); file_reader.read_next_partition_data(pad_zeros=True); logger.debug('NP Dataexporter : data is exported to %s' % self.export_path);