def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True):
data = numpy.fromfile(self.filehandle,dtype=self.dtype,count=self.frames_per_partition);
cur_frame_num = data.__len__();
if cur_frame_num > 0:
self.cur_frame_num = cur_frame_num
self.feat = numpy.asarray(data['d'], dtype = theano.config.floatX)
self.label = numpy.asarray(data['l'], dtype = theano.config.floatX)
self.partition_num = self.partition_num + 1
if pad_zeros:
self.pad_zeros();
if not self.options['keep_flatten'] : #reshape the vector if needed
logger.debug('NP Filereader : Reshape input...')
shape = [self.cur_frame_num];
shape.extend(self.header['input_shape']);
self.feat = self.feat.reshape(shape);
self.feat = dimshuffle(self.feat,self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared();
self.nBatches = self.cur_frame_num/self.batch_size;
logger.debug('T1 Filereader : from file %s, %d partition has %d frames [%d Batches]',
self.filepath,self.partition_num-1,self.cur_frame_num,self.nBatches);
else:
self.finished = True;
logger.debug('TD Filereader : NO more frames to read from %s',self.filepath)
def read_next_partition_data(self,
already_read=0,
pad_zeros=False,
makeShared=True):
data = numpy.fromfile(self.filehandle,
dtype=self.dtype,
count=self.frames_per_partition)
cur_frame_num = data.__len__()
if cur_frame_num > 0:
self.cur_frame_num = cur_frame_num
self.feat = numpy.asarray(data['d'], dtype=theano.config.floatX)
self.label = numpy.asarray(data['l'], dtype=theano.config.floatX)
self.partition_num = self.partition_num + 1
if pad_zeros:
self.pad_zeros()
if not self.options['keep_flatten']: #reshape the vector if needed
logger.debug('NP Filereader : Reshape input...')
shape = [self.cur_frame_num]
shape.extend(self.header['input_shape'])
self.feat = self.feat.reshape(shape)
self.feat = dimshuffle(self.feat, self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared()
self.nBatches = self.cur_frame_num / self.batch_size
logger.debug(
'T1 Filereader : from file %s, %d partition has %d frames [%d Batches]',
self.filepath, self.partition_num - 1, self.cur_frame_num,
self.nBatches)
else:
self.finished = True
logger.debug('TD Filereader : NO more frames to read from %s',
self.filepath)
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True):
if self.finished:
raise EOFError('Reader has reached EOF');
cur_frame_num = 0
feat = []
label = []
while ((cur_frame_num < self.frames_per_partition-already_read) or self.frames_per_partition==float('inf')):
if not self.filehandles[self.last_class_idx].finished:
#if TD is not finshed.
# Read Next part.
logger.debug("T1: loading %s ",self.filehandles[self.last_class_idx].filepath)
self.filehandles[self.last_class_idx].read_next_partition_data(
already_read=self.frames_per_partition-self.frames_per_class,
makeShared=False)
if self.filehandles[self.last_class_idx].finished:
pass
else:
feat = numpy.append(feat,self.filehandles[self.last_class_idx].feat)
label = numpy.append(label,self.filehandles[self.last_class_idx].label)
cur_frame_num += self.filehandles[self.last_class_idx].cur_frame_num
self.last_class_idx = (self.last_class_idx + 1) % self.classes
if all([x.finished for x in self.filehandles]):
break;
if cur_frame_num > 0:
self.feat = feat;
self.label = label;
self.cur_frame_num = cur_frame_num;
if pad_zeros:
self.pad_zeros();
self.partition_num = self.partition_num + 1
self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim])
self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX)
if not self.options['keep_flatten'] : #reshape the vector if needed
logger.debug('T1 Filereader : Reshape input...')
shape = [self.cur_frame_num];
shape.extend(self.options['input_shape']);
self.feat = self.feat.reshape(shape);
self.feat = dimshuffle(self.feat,self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared();
self.nBatches = self.cur_frame_num/self.batch_size;
logger.debug('T1 Filereader : from file %s, %d partition has %d frames [%d Batches]',
self.filepath,self.partition_num-1,self.cur_frame_num,self.nBatches);
else:
self.finished = True
logger.debug('TD Filereader : NO more frames to read from %s',self.filepath)
def read_next_partition_data(self,
already_read=0,
pad_zeros=False,
makeShared=True):
if self.finished:
raise EOFError('Reader has reached EOF')
#self.feat=[]
fvalues = []
cur_frame_num = 0
while ((cur_frame_num < self.frames_per_partition - already_read)
or self.frames_per_partition == float('inf')):
values = self.filehandle.readline().split()
if values.__len__(
) == 0: #No more values available in the data file
break
fvalues = numpy.append(fvalues, [float(value) for value in values])
cur_frame_num += 1
if cur_frame_num > 0:
self.cur_frame_num = cur_frame_num
self.feat = fvalues
if pad_zeros:
self.pad_zeros()
self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim])
# convert float64 to floatX
self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX)
self.label = numpy.asarray([self.lbl] * self.cur_frame_num,
dtype=theano.config.floatX)
self.partition_num = self.partition_num + 1
logger.debug('TD Filereader : %d frames read from %s',
self.cur_frame_num, self.filepath)
if not self.options['keep_flatten']: #reshape the vector if needed
logger.debug('TD Filereader : Reshape input...')
shape = [self.cur_frame_num]
shape.extend(self.options['input_shape'])
self.feat = self.feat.reshape(shape)
self.feat = dimshuffle(self.feat, self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared()
self.nBatches = self.cur_frame_num / self.batch_size
else:
logger.debug('TD Filereader : NO more frames to read from %s',
self.filepath)
self.finished = True
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True):
if self.finished:
raise EOFError('Reader has reached EOF');
#self.feat=[]
fvalues = []
cur_frame_num = 0
while ((cur_frame_num < self.frames_per_partition-already_read) or self.frames_per_partition==float('inf')):
values = self.filehandle.readline().split()
if values.__len__()==0: #No more values available in the data file
break;
fvalues=numpy.append(fvalues,[float(value) for value in values]);
cur_frame_num += 1
if cur_frame_num > 0:
self.cur_frame_num = cur_frame_num;
self.feat = fvalues;
if pad_zeros:
self.pad_zeros();
self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim])
# convert float64 to floatX
self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX)
self.label = numpy.asarray([self.lbl]*self.cur_frame_num , dtype = theano.config.floatX)
self.partition_num = self.partition_num + 1
logger.debug('TD Filereader : %d frames read from %s',self.cur_frame_num,self.filepath)
if not self.options['keep_flatten'] : #reshape the vector if needed
logger.debug('TD Filereader : Reshape input...')
shape = [self.cur_frame_num];
shape.extend(self.options['input_shape']);
self.feat = self.feat.reshape(shape);
self.feat = dimshuffle(self.feat,self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared();
self.nBatches = self.cur_frame_num/self.batch_size;
else:
logger.debug('TD Filereader : NO more frames to read from %s',self.filepath)
self.finished = True;
def read_next_partition_data(self,already_read=0,pad_zeros=False,makeShared=True):
cur_frame_num = 0
feat = numpy.array([],dtype=theano.config.floatX)
label = []
none_cnt = 0
while cur_frame_num < self.frames_per_partition and none_cnt < self.classes :
if self.childhandles[self.last_class_idx] is None:
#if the child handle is not initialized
data_file = self.filenames[self.last_class_idx][self.fileIndex[self.last_class_idx]];
##Get Next Filename in last Class.
if data_file != None:
#if Next Filename == NULL;
self.fileIndex[self.last_class_idx] = self.fileIndex[self.last_class_idx] + 1;
child_options = self.options.copy()
child_options['filename']=data_file
child_options['label']= self.last_class_idx
child_options['keep_flatten'] = True
data_file = child_options['filename']
self.childhandles[self.last_class_idx] = TDFileReader(data_file,child_options)
if not self.childhandles[self.last_class_idx] is None:
none_cnt = 0
self.childhandles[self.last_class_idx].read_next_partition_data(
already_read=self.frames_per_partition-self.frames_per_class,
makeShared=False)
if not self.childhandles[self.last_class_idx].finished:
feat = numpy.append(feat,self.childhandles[self.last_class_idx].feat)
label = numpy.append(label,self.childhandles[self.last_class_idx].label)
cur_frame_num += self.childhandles[self.last_class_idx].cur_frame_num
if self.childhandles[self.last_class_idx].is_finish():
self.childhandles[self.last_class_idx] = None
else:
none_cnt+=1
self.last_class_idx = (self.last_class_idx + 1) % self.classes
if cur_frame_num > 0:
self.feat = feat;
self.label = label;
self.cur_frame_num = cur_frame_num;
if pad_zeros: #padding zeros
self.pad_zeros();
logger.debug('T2 Filereader : from file %s, %d partition has %d frames',
self.filepath,self.partition_num,self.cur_frame_num);
self.feat = self.feat.reshape([self.cur_frame_num,self.feat_dim])
self.feat = numpy.asarray(self.feat, dtype = theano.config.floatX)
self.partition_num = self.partition_num + 1
if not self.options['keep_flatten'] : #reshape the vector if needed
logger.debug('T2 Filereader : Reshape input...')
shape = [self.cur_frame_num];
shape.extend(self.options['input_shape']);
self.feat = self.feat.reshape(shape);
self.feat = dimshuffle(self.feat,self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared();
self.nBatches = self.cur_frame_num/self.batch_size;
else:
self.finished = True
def read_next_partition_data(self,
already_read=0,
pad_zeros=False,
makeShared=True):
cur_frame_num = 0
feat = numpy.array([], dtype=theano.config.floatX)
label = []
none_cnt = 0
while cur_frame_num < self.frames_per_partition and none_cnt < self.classes:
if self.childhandles[self.last_class_idx] is None:
#if the child handle is not initialized
data_file = self.filenames[self.last_class_idx][self.fileIndex[
self.last_class_idx]]
##Get Next Filename in last Class.
if data_file != None:
#if Next Filename == NULL;
self.fileIndex[self.last_class_idx] = self.fileIndex[
self.last_class_idx] + 1
child_options = self.options.copy()
child_options['filename'] = data_file
child_options['label'] = self.last_class_idx
child_options['keep_flatten'] = True
data_file = child_options['filename']
self.childhandles[self.last_class_idx] = TDFileReader(
data_file, child_options)
if not self.childhandles[self.last_class_idx] is None:
none_cnt = 0
self.childhandles[
self.last_class_idx].read_next_partition_data(
already_read=self.frames_per_partition -
self.frames_per_class,
makeShared=False)
if not self.childhandles[self.last_class_idx].finished:
feat = numpy.append(
feat, self.childhandles[self.last_class_idx].feat)
label = numpy.append(
label, self.childhandles[self.last_class_idx].label)
cur_frame_num += self.childhandles[
self.last_class_idx].cur_frame_num
if self.childhandles[self.last_class_idx].is_finish():
self.childhandles[self.last_class_idx] = None
else:
none_cnt += 1
self.last_class_idx = (self.last_class_idx + 1) % self.classes
if cur_frame_num > 0:
self.feat = feat
self.label = label
self.cur_frame_num = cur_frame_num
if pad_zeros: #padding zeros
self.pad_zeros()
logger.debug(
'T2 Filereader : from file %s, %d partition has %d frames',
self.filepath, self.partition_num, self.cur_frame_num)
self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim])
self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX)
self.partition_num = self.partition_num + 1
if not self.options['keep_flatten']: #reshape the vector if needed
logger.debug('T2 Filereader : Reshape input...')
shape = [self.cur_frame_num]
shape.extend(self.options['input_shape'])
self.feat = self.feat.reshape(shape)
self.feat = dimshuffle(self.feat, self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared()
self.nBatches = self.cur_frame_num / self.batch_size
else:
self.finished = True
def read_next_partition_data(self,
already_read=0,
pad_zeros=False,
makeShared=True):
if self.finished:
raise EOFError('Reader has reached EOF')
cur_frame_num = 0
feat = []
label = []
while ((cur_frame_num < self.frames_per_partition - already_read)
or self.frames_per_partition == float('inf')):
if not self.filehandles[self.last_class_idx].finished:
#if TD is not finshed.
# Read Next part.
logger.debug("T1: loading %s ",
self.filehandles[self.last_class_idx].filepath)
self.filehandles[self.last_class_idx].read_next_partition_data(
already_read=self.frames_per_partition -
self.frames_per_class,
makeShared=False)
if self.filehandles[self.last_class_idx].finished:
pass
else:
feat = numpy.append(
feat, self.filehandles[self.last_class_idx].feat)
label = numpy.append(
label, self.filehandles[self.last_class_idx].label)
cur_frame_num += self.filehandles[
self.last_class_idx].cur_frame_num
self.last_class_idx = (self.last_class_idx + 1) % self.classes
if all([x.finished for x in self.filehandles]):
break
if cur_frame_num > 0:
self.feat = feat
self.label = label
self.cur_frame_num = cur_frame_num
if pad_zeros:
self.pad_zeros()
self.partition_num = self.partition_num + 1
self.feat = self.feat.reshape([self.cur_frame_num, self.feat_dim])
self.feat = numpy.asarray(self.feat, dtype=theano.config.floatX)
if not self.options['keep_flatten']: #reshape the vector if needed
logger.debug('T1 Filereader : Reshape input...')
shape = [self.cur_frame_num]
shape.extend(self.options['input_shape'])
self.feat = self.feat.reshape(shape)
self.feat = dimshuffle(self.feat, self.options['dim_shuffle'])
if makeShared:
self.make_partition_shared()
self.nBatches = self.cur_frame_num / self.batch_size
logger.debug(
'T1 Filereader : from file %s, %d partition has %d frames [%d Batches]',
self.filepath, self.partition_num - 1, self.cur_frame_num,
self.nBatches)
else:
self.finished = True
logger.debug('TD Filereader : NO more frames to read from %s',
self.filepath)
