def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True): data = numpy.fromfile(self.filehandle,dtype=self.dtype,count=self.frames_per_partition); cur_frame_num = data.__len__(); if cur_frame_num > 0: self.cur_frame_num = cur_frame_num self.feat = numpy.asarray(data['d'], dtype = theano.config.floatX) self.label = numpy.asarray(data['l'], dtype = theano.config.floatX) self.partition_num = self.partition_num + 1 if pad_zeros: self.pad_zeros(); if not self.options['keep_flatten'] : #reshape the vector if needed logger.debug('NP Filereader : Reshape input...') shape = [self.cur_frame_num]; shape.extend(self.header['input_shape']); self.feat = self.feat.reshape(shape); self.feat = dimshuffle(self.feat,self.options['dim_shuffle']) if makeShared: self.make_partition_shared(); self.nBatches = self.cur_frame_num/self.batch_size; logger.debug('T1 Filereader : from file %s, %d partition has %d frames [%d Batches]', self.filepath,self.partition_num-1,self.cur_frame_num,self.nBatches); else: self.finished = True; logger.debug('TD Filereader : NO more frames to read from %s',self.filepath)
def read_next_partition_data(self, already_read=0, pad_zeros=False, makeShared=True): data = numpy.fromfile(self.filehandle, dtype=self.dtype, count=self.frames_per_partition) cur_frame_num = data.__len__() if cur_frame_num > 0: self.cur_frame_num = cur_frame_num self.feat = numpy.asarray(data['d'], dtype=theano.config.floatX) self.label = numpy.asarray(data['l'], dtype=theano.config.floatX) self.partition_num = self.partition_num + 1 if pad_zeros: self.pad_zeros() if not self.options['keep_flatten']: #reshape the vector if needed logger.debug('NP Filereader : Reshape input...') shape = [self.cur_frame_num] shape.extend(self.header['input_shape']) self.feat = self.feat.reshape(shape) self.feat = dimshuffle(self.feat, self.options['dim_shuffle']) if makeShared: self.make_partition_shared() self.nBatches = self.cur_frame_num / self.batch_size logger.debug( 'T1 Filereader : from file %s, %d partition has %d frames [%d Batches]', self.filepath, self.partition_num - 1, self.cur_frame_num, self.nBatches) else: self.finished = True logger.debug('TD Filereader : NO more frames to read from %s', self.filepath)
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True): if self.finished: raise EOFError('Reader has reached EOF'); cur_frame_num = 0 feat = [] label = [] while ((cur_frame_num < self.frames_per_partition-already_read) or self.frames_per_partition==float('inf')): if not self.filehandles[self.last_class_idx].finished: #if TD is not finshed. # Read Next part. logger.debug("T1: loading %s ",self.filehandles[self.last_class_idx].filepath) self.filehandles[self.last_class_idx].read_next_partition_data( already_read=self.frames_per_partition-self.frames_per_class, makeShared=False) if self.filehandles[self.last_class_idx].finished: pass else: feat = numpy.append(feat,self.filehandles[self.last_class_idx].feat) label = numpy.append(label,self.filehandles[self.last_class_idx].label) cur_frame_num += self.filehandles[self.last_class_idx].cur_frame_num self.last_class_idx = (self.last_class_idx + 1) % self.classes if all([x.finished for x in self.filehandles]): break; if cur_frame_num > 0: self.feat = feat; self.label = label; self.cur_frame_num = cur_frame_num; if pad_zeros: self.pad_zeros(); self.partition_num = self.partition_num + 1 self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim]) self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX) if not self.options['keep_flatten'] : #reshape the vector if needed logger.debug('T1 Filereader : Reshape input...') shape = [self.cur_frame_num]; shape.extend(self.options['input_shape']); self.feat = self.feat.reshape(shape); self.feat = dimshuffle(self.feat,self.options['dim_shuffle']) if makeShared: self.make_partition_shared(); self.nBatches = self.cur_frame_num/self.batch_size; logger.debug('T1 Filereader : from file %s, %d partition has %d frames [%d Batches]', self.filepath,self.partition_num-1,self.cur_frame_num,self.nBatches); else: self.finished = True logger.debug('TD Filereader : NO more frames to read from %s',self.filepath)
def read_next_partition_data(self, already_read=0, pad_zeros=False, makeShared=True): if self.finished: raise EOFError('Reader has reached EOF') #self.feat=[] fvalues = [] cur_frame_num = 0 while ((cur_frame_num < self.frames_per_partition - already_read) or self.frames_per_partition == float('inf')): values = self.filehandle.readline().split() if values.__len__( ) == 0: #No more values available in the data file break fvalues = numpy.append(fvalues, [float(value) for value in values]) cur_frame_num += 1 if cur_frame_num > 0: self.cur_frame_num = cur_frame_num self.feat = fvalues if pad_zeros: self.pad_zeros() self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim]) # convert float64 to floatX self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX) self.label = numpy.asarray([self.lbl] * self.cur_frame_num, dtype=theano.config.floatX) self.partition_num = self.partition_num + 1 logger.debug('TD Filereader : %d frames read from %s', self.cur_frame_num, self.filepath) if not self.options['keep_flatten']: #reshape the vector if needed logger.debug('TD Filereader : Reshape input...') shape = [self.cur_frame_num] shape.extend(self.options['input_shape']) self.feat = self.feat.reshape(shape) self.feat = dimshuffle(self.feat, self.options['dim_shuffle']) if makeShared: self.make_partition_shared() self.nBatches = self.cur_frame_num / self.batch_size else: logger.debug('TD Filereader : NO more frames to read from %s', self.filepath) self.finished = True
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True): if self.finished: raise EOFError('Reader has reached EOF'); #self.feat=[] fvalues = [] cur_frame_num = 0 while ((cur_frame_num < self.frames_per_partition-already_read) or self.frames_per_partition==float('inf')): values = self.filehandle.readline().split() if values.__len__()==0: #No more values available in the data file break; fvalues=numpy.append(fvalues,[float(value) for value in values]); cur_frame_num += 1 if cur_frame_num > 0: self.cur_frame_num = cur_frame_num; self.feat = fvalues; if pad_zeros: self.pad_zeros(); self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim]) # convert float64 to floatX self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX) self.label = numpy.asarray([self.lbl]*self.cur_frame_num , dtype = theano.config.floatX) self.partition_num = self.partition_num + 1 logger.debug('TD Filereader : %d frames read from %s',self.cur_frame_num,self.filepath) if not self.options['keep_flatten'] : #reshape the vector if needed logger.debug('TD Filereader : Reshape input...') shape = [self.cur_frame_num]; shape.extend(self.options['input_shape']); self.feat = self.feat.reshape(shape); self.feat = dimshuffle(self.feat,self.options['dim_shuffle']) if makeShared: self.make_partition_shared(); self.nBatches = self.cur_frame_num/self.batch_size; else: logger.debug('TD Filereader : NO more frames to read from %s',self.filepath) self.finished = True;
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True): cur_frame_num = 0 feat = numpy.array([],dtype=theano.config.floatX) label = [] none_cnt = 0 while cur_frame_num < self.frames_per_partition and none_cnt < self.classes : if self.childhandles[self.last_class_idx] is None: #if the child handle is not initialized data_file = self.filenames[self.last_class_idx][self.fileIndex[self.last_class_idx]]; ##Get Next Filename in last Class. if data_file != None: #if Next Filename == NULL; self.fileIndex[self.last_class_idx] = self.fileIndex[self.last_class_idx] + 1; child_options = self.options.copy() child_options['filename']=data_file child_options['label']= self.last_class_idx child_options['keep_flatten'] = True data_file = child_options['filename'] self.childhandles[self.last_class_idx] = TDFileReader(data_file,child_options) if not self.childhandles[self.last_class_idx] is None: none_cnt = 0 self.childhandles[self.last_class_idx].read_next_partition_data( already_read=self.frames_per_partition-self.frames_per_class, makeShared=False) if not self.childhandles[self.last_class_idx].finished: feat = numpy.append(feat,self.childhandles[self.last_class_idx].feat) label = numpy.append(label,self.childhandles[self.last_class_idx].label) cur_frame_num += self.childhandles[self.last_class_idx].cur_frame_num if self.childhandles[self.last_class_idx].is_finish(): self.childhandles[self.last_class_idx] = None else: none_cnt+=1 self.last_class_idx = (self.last_class_idx + 1) % self.classes if cur_frame_num > 0: self.feat = feat; self.label = label; self.cur_frame_num = cur_frame_num; if pad_zeros: #padding zeros self.pad_zeros(); logger.debug('T2 Filereader : from file %s, %d partition has %d frames', self.filepath,self.partition_num,self.cur_frame_num); self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim]) self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX) self.partition_num = self.partition_num + 1 if not self.options['keep_flatten'] : #reshape the vector if needed logger.debug('T2 Filereader : Reshape input...') shape = [self.cur_frame_num]; shape.extend(self.options['input_shape']); self.feat = self.feat.reshape(shape); self.feat = dimshuffle(self.feat,self.options['dim_shuffle']) if makeShared: self.make_partition_shared(); self.nBatches = self.cur_frame_num/self.batch_size; else: self.finished = True
def read_next_partition_data(self, already_read=0, pad_zeros=False, makeShared=True): cur_frame_num = 0 feat = numpy.array([], dtype=theano.config.floatX) label = [] none_cnt = 0 while cur_frame_num < self.frames_per_partition and none_cnt < self.classes: if self.childhandles[self.last_class_idx] is None: #if the child handle is not initialized data_file = self.filenames[self.last_class_idx][self.fileIndex[ self.last_class_idx]] ##Get Next Filename in last Class. if data_file != None: #if Next Filename == NULL; self.fileIndex[self.last_class_idx] = self.fileIndex[ self.last_class_idx] + 1 child_options = self.options.copy() child_options['filename'] = data_file child_options['label'] = self.last_class_idx child_options['keep_flatten'] = True data_file = child_options['filename'] self.childhandles[self.last_class_idx] = TDFileReader( data_file, child_options) if not self.childhandles[self.last_class_idx] is None: none_cnt = 0 self.childhandles[ self.last_class_idx].read_next_partition_data( already_read=self.frames_per_partition - self.frames_per_class, makeShared=False) if not self.childhandles[self.last_class_idx].finished: feat = numpy.append( feat, self.childhandles[self.last_class_idx].feat) label = numpy.append( label, self.childhandles[self.last_class_idx].label) cur_frame_num += self.childhandles[ self.last_class_idx].cur_frame_num if self.childhandles[self.last_class_idx].is_finish(): self.childhandles[self.last_class_idx] = None else: none_cnt += 1 self.last_class_idx = (self.last_class_idx + 1) % self.classes if cur_frame_num > 0: self.feat = feat self.label = label self.cur_frame_num = cur_frame_num if pad_zeros: #padding zeros self.pad_zeros() logger.debug( 'T2 Filereader : from file %s, %d partition has %d frames', self.filepath, self.partition_num, self.cur_frame_num) self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim]) self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX) self.partition_num = self.partition_num + 1 if not self.options['keep_flatten']: #reshape the vector if needed logger.debug('T2 Filereader : Reshape input...') shape = [self.cur_frame_num] shape.extend(self.options['input_shape']) self.feat = self.feat.reshape(shape) self.feat = dimshuffle(self.feat, self.options['dim_shuffle']) if makeShared: self.make_partition_shared() self.nBatches = self.cur_frame_num / self.batch_size else: self.finished = True
def read_next_partition_data(self, already_read=0, pad_zeros=False, makeShared=True): if self.finished: raise EOFError('Reader has reached EOF') cur_frame_num = 0 feat = [] label = [] while ((cur_frame_num < self.frames_per_partition - already_read) or self.frames_per_partition == float('inf')): if not self.filehandles[self.last_class_idx].finished: #if TD is not finshed. # Read Next part. logger.debug("T1: loading %s ", self.filehandles[self.last_class_idx].filepath) self.filehandles[self.last_class_idx].read_next_partition_data( already_read=self.frames_per_partition - self.frames_per_class, makeShared=False) if self.filehandles[self.last_class_idx].finished: pass else: feat = numpy.append( feat, self.filehandles[self.last_class_idx].feat) label = numpy.append( label, self.filehandles[self.last_class_idx].label) cur_frame_num += self.filehandles[ self.last_class_idx].cur_frame_num self.last_class_idx = (self.last_class_idx + 1) % self.classes if all([x.finished for x in self.filehandles]): break if cur_frame_num > 0: self.feat = feat self.label = label self.cur_frame_num = cur_frame_num if pad_zeros: self.pad_zeros() self.partition_num = self.partition_num + 1 self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim]) self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX) if not self.options['keep_flatten']: #reshape the vector if needed logger.debug('T1 Filereader : Reshape input...') shape = [self.cur_frame_num] shape.extend(self.options['input_shape']) self.feat = self.feat.reshape(shape) self.feat = dimshuffle(self.feat, self.options['dim_shuffle']) if makeShared: self.make_partition_shared() self.nBatches = self.cur_frame_num / self.batch_size logger.debug( 'T1 Filereader : from file %s, %d partition has %d frames [%d Batches]', self.filepath, self.partition_num - 1, self.cur_frame_num, self.nBatches) else: self.finished = True logger.debug('TD Filereader : NO more frames to read from %s', self.filepath)