def read(self): remainder=None base=None import lg_dpl_toolbox.dpl.TimeSource as TimeSource timesource=TimeSource.CompoundTimeGenerator(self.timesource) for f in self.framestream: if timesource.isDone: break if remainder==None: remainder=copy.deepcopy(f) else: remainder.append(f) t=getattr(remainder,remainder._timevarname) #print t.shape if t.shape[0]==0: continue requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=t[-1])) if requestedtimes.size<2: continue lastTime=requestedtimes[-1] retarr=remainder remainder=hau.trimTimeInterval(retarr,lastTime,datetime(2200,1,1,0,0,0)) retarr.trimTimeInterval(base,lastTime) retarr.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans) print 'range',base,lastTime,'returning:',retarr,'remainder',remainder yield retarr base=lastTime if remainder!=None and timesource.end_time!=None: requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=timesource.end_time,inclusive=True)) remainder.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans) if getattr(remainder,remainder._timevarname).shape[0]>0: print 'range',base,timesource.end_time,'returning:',remainder yield remainder
def read(self): """ main read generator """ import hsrl.data_stream.processing_utilities as pu params=self.params firsttimeever=None intervalTime=None intervalEnd=None rawsrc=iter(self.rawsrc) #if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])): # params['timeres']=None #pure native end_time_datetime=params['finalTime'] #timemodoffset=time_mod(params['realStartTime'],params['timeres']) noframe='noframe' fullrange=False #if this is true, it will pad the start with any missing times. remainder=None cdf_to_hsrl = None preprocess_ave = None requested_times=None instrument=self.hsrl_instrument intcount=0 rs_mem = None #rs=None timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None for calv in self.cal_narr: if intervalTime is None: firsttimeever=calv['chunk_start_time'] intervalTime=calv['chunk_start_time'] intervalEnd=intervalTime chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset) #print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres'] if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None: if params['block_when_out_of_data']: if 'timeres' not in params or params['timeres'] is None: sleep(calv['rs_constants']['integration_time']) else: sleep(params['timeres'].total_seconds()) else: yield None #this is done to get out of here, and not get stuck in a tight loop continue while intervalTime<chunk_end_to_use: integration_time = calv['rs_constants']['integration_time'] doPresample=True #END init section if intervalEnd>chunk_end_to_use: print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime break else: intervalEnd=chunk_end_to_use #print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime'] print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem if True:#requested_times==None or requested_times.shape[0]>0: try: try: while rawsrc is not None: if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): break tmp=rawsrc.next() if hasattr(tmp,'rs_raw'): if rs_mem is not None: rs_mem.append(tmp.rs_raw) else: rs_mem=copy.deepcopy(tmp.rs_raw) if rs_mem is not None and rs_mem.times.shape>0: break else: rs_mem=None except StopIteration: print 'Raw HSRL stream is ended' rawsrc=None if rs_mem is None or rs_mem.times.size==0: rs_mem=None elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0] break else: intervalEnd=rs_mem.times[-1] print 'read in raw frame to mean',rs_mem,remainder if rawsrc is None: intervalEnd=chunk_end_to_use print 'trimmed ',rs_mem if timesource is not None: if timesource.isDone: break useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime) timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime))) print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd elif 'timeres' in params and params['timeres'] is not None: tmp=intervalTime useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet timevals=[] timevals.append(tmp) while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats. #print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds() tmp+=params['timeres'] timevals.append(tmp) #intervalEnd=tmp intcount+=len(timevals) if usePrebinnedTimes: intcount-=1 print 'Now %i intervals' % (intcount) timevals=hau.T_Array(timevals) else: print 'Using Native timing' timevals=None print ' new window is ', intervalTime, ' to ' , intervalEnd requested_times=timevals requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime] if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None: #if rawsrc is not None: print "not enough time to process" continue elif rawsrc is None and rs_mem is None and remainder is None: #chunk_end_to_use=intervalTime #continue #print '' break rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd ,params['min_alt'], params['max_alt'], requested_chunk_times , rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal'] , None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control , self.compute_stats,remainder=remainder) rs_mem=None if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0: rs_chunk=None if rs_chunk is None and rawsrc is None: break #print rs_chunk if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0: if fullrange and requested_chunk_times is not None: v=hau.Time_Z_Group(like=rs_chunk.rs_mean) v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]]) if v.times.size>0: rs_chunk.rs_mean.prepend(v) rs_chunk.calv=calv yield rs_chunk intervalTime=intervalEnd except Exception, e: print 'Exception occured in update_cal_and_process' print 'Exception = ',e print traceback.format_exc() if isinstance(e,(MemoryError,)): print 'Please Adjust Your Parameters to be more Server-friendly and try again' raise if not isinstance(e,(AttributeError,)): raise