def read(self):
remainder=None
base=None
import lg_dpl_toolbox.dpl.TimeSource as TimeSource
timesource=TimeSource.CompoundTimeGenerator(self.timesource)
for f in self.framestream:
if timesource.isDone:
break
if remainder==None:
remainder=copy.deepcopy(f)
else:
remainder.append(f)
t=getattr(remainder,remainder._timevarname)
#print t.shape
if t.shape[0]==0:
continue
requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=t[-1]))
if requestedtimes.size<2:
continue
lastTime=requestedtimes[-1]
retarr=remainder
remainder=hau.trimTimeInterval(retarr,lastTime,datetime(2200,1,1,0,0,0))
retarr.trimTimeInterval(base,lastTime)
retarr.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans)
print 'range',base,lastTime,'returning:',retarr,'remainder',remainder
yield retarr
base=lastTime
if remainder!=None and timesource.end_time!=None:
requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=timesource.end_time,inclusive=True))
remainder.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans)
if getattr(remainder,remainder._timevarname).shape[0]>0:
print 'range',base,timesource.end_time,'returning:',remainder
yield remainder
def read(self):
""" main read generator
"""
import hsrl.data_stream.processing_utilities as pu
params=self.params
firsttimeever=None
intervalTime=None
intervalEnd=None
rawsrc=iter(self.rawsrc)
#if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])):
# params['timeres']=None #pure native
end_time_datetime=params['finalTime']
#timemodoffset=time_mod(params['realStartTime'],params['timeres'])
noframe='noframe'
fullrange=False #if this is true, it will pad the start with any missing times.
remainder=None
cdf_to_hsrl = None
preprocess_ave = None
requested_times=None
instrument=self.hsrl_instrument
intcount=0
rs_mem = None
#rs=None
timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None
for calv in self.cal_narr:
if intervalTime is None:
firsttimeever=calv['chunk_start_time']
intervalTime=calv['chunk_start_time']
intervalEnd=intervalTime
chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset)
#print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres']
if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None:
if params['block_when_out_of_data']:
if 'timeres' not in params or params['timeres'] is None:
sleep(calv['rs_constants']['integration_time'])
else:
sleep(params['timeres'].total_seconds())
else:
yield None #this is done to get out of here, and not get stuck in a tight loop
continue
while intervalTime<chunk_end_to_use:
integration_time = calv['rs_constants']['integration_time']
doPresample=True
#END init section
if intervalEnd>chunk_end_to_use:
print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime
break
else:
intervalEnd=chunk_end_to_use
#print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime']
print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem
if True:#requested_times==None or requested_times.shape[0]>0:
try:
try:
while rawsrc is not None:
if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
break
tmp=rawsrc.next()
if hasattr(tmp,'rs_raw'):
if rs_mem is not None:
rs_mem.append(tmp.rs_raw)
else:
rs_mem=copy.deepcopy(tmp.rs_raw)
if rs_mem is not None and rs_mem.times.shape>0:
break
else:
rs_mem=None
except StopIteration:
print 'Raw HSRL stream is ended'
rawsrc=None
if rs_mem is None or rs_mem.times.size==0:
rs_mem=None
elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0]
break
else:
intervalEnd=rs_mem.times[-1]
print 'read in raw frame to mean',rs_mem,remainder
if rawsrc is None:
intervalEnd=chunk_end_to_use
print 'trimmed ',rs_mem
if timesource is not None:
if timesource.isDone:
break
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime)
timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime)))
print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd
elif 'timeres' in params and params['timeres'] is not None:
tmp=intervalTime
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
timevals=[]
timevals.append(tmp)
while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats.
#print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds()
tmp+=params['timeres']
timevals.append(tmp)
#intervalEnd=tmp
intcount+=len(timevals)
if usePrebinnedTimes:
intcount-=1
print 'Now %i intervals' % (intcount)
timevals=hau.T_Array(timevals)
else:
print 'Using Native timing'
timevals=None
print ' new window is ', intervalTime, ' to ' , intervalEnd
requested_times=timevals
requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime]
if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None:
#if rawsrc is not None:
print "not enough time to process"
continue
elif rawsrc is None and rs_mem is None and remainder is None:
#chunk_end_to_use=intervalTime
#continue
#print ''
break
rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd
,params['min_alt'], params['max_alt'], requested_chunk_times
, rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal']
, None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control
, self.compute_stats,remainder=remainder)
rs_mem=None
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0:
rs_chunk=None
if rs_chunk is None and rawsrc is None:
break
#print rs_chunk
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0:
if fullrange and requested_chunk_times is not None:
v=hau.Time_Z_Group(like=rs_chunk.rs_mean)
v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]])
if v.times.size>0:
rs_chunk.rs_mean.prepend(v)
rs_chunk.calv=calv
yield rs_chunk
intervalTime=intervalEnd
except Exception, e:
print 'Exception occured in update_cal_and_process'
print 'Exception = ',e
print traceback.format_exc()
if isinstance(e,(MemoryError,)):
print 'Please Adjust Your Parameters to be more Server-friendly and try again'
raise
if not isinstance(e,(AttributeError,)):
raise