def process(self):
#calsource=time_frame.TimeTrickle(self.cals,timename='chunk_start_time',endtimename='chunk_end_time')
rif = self.rif
pickle = self.pickle
lasttime = None
calsource = time_frame.TimeTrickle(self.cals,
timename='chunk_start_time',
endtimename='chunk_end_time')
for orig in self.datasource:
if self.sourcename is None:
highf = orig
else:
if isinstance(orig, dict) and self.sourcename in orig:
highf = orig[self.sourcename]
elif hasattr(orig, self.sourcename):
highf = getattr(orig, self.sourcename)
else:
print "No Raman Inv to process. can't find subframe " + self.sourcename
highf = None
if highf is not None and highf.times.size == 0:
highf = None
if highf is None:
print "No Raman Inv to process. Raman Merge is empty"
continue
print "Raman Merge to INV running"
lasttime = highf.times[0]
caldictionary = calsource(lasttime)
pname = ('frame', 'cal frame')
p1 = []
p1.append(pickle.dumps(highf))
p1.append(pickle.dumps(caldictionary))
ret = rif.process_raman(caldictionary['rs_constants'], highf,
self.raman_process_control,
caldictionary['rs_cal'],
caldictionary['Cxx'],
self.raman_corr_adjusts)
p2 = []
p2.append(pickle.dumps(highf))
p2.append(pickle.dumps(caldictionary))
for i in range(len(p1)):
if p1[i] != p2[i]:
print 'Detected modification of ' + pname[i] + '. FAIL'
raise RuntimeError('Modification of Raman ' + pname[i])
#hau.verifyAllNew(ramancal=caldictionary,sounding=sounding,ramanmergehigh=highf,ramanmerge=lowf,ramaninv=ret)
if ret is None:
print "Raman INV is none"
import time
time.sleep(5)
if self.destination is not None:
if isinstance(orig, dict):
orig[self.destination] = ret
else:
setattr(orig, self.destination, ret)
yield orig
else:
yield ret
def __init__(self, narrator):
import lg_dpl_toolbox.filters.time_frame as dt
self.dt = dt
self.narr = narrator
self.source = dt.TimeTrickle(self.narr,
timename='times',
getClosest=True)
def process(self):
fr = None
flags = None
olda = None
qasource = tf.TimeTrickle(self.qasource, 'time')
altitudes = hau.Z_Array(self.qaparser.altitudeAxis)
for f in self.timealtsource:
#FIXME include angles
if not isinstance(f, dict):
f = vars(f)
t = f[self.timename]
a = self.constantAltitude
if fr is None or ((not qasource.atEnd) and t >= qasource.nextTime
): #if need an update to the qa record
#print 'Getting qa source for time',t
fr = qasource(t)
flags = None
if 'range_flags' in fr and fr[
'range_flags'] is not None: #if there is a range dependence, and a potentially non-constant altitude
if self.altname is not None and self.altname in f:
a = f[self.altname]
if a is None:
raise RuntimeError(
'Need platform altitude to merge in range-dependant qa Flags'
)
if olda is None or a != olda:
flags = None
olda = a
if flags is None: #was cleared either because new flags from the qa file, or new altitude from the stream
if 'range_flags' in fr and fr['range_flags'] is not None:
flags = self.qaparser.mergeVectors(fr['flags'],
fr['range_flags'], a)
else:
flags = fr['flags']
flags = self.qaparser.translateToEnumeration(flags)
flags = hau.TZ_Array(flags.reshape([1] + list(flags.shape)),
dtype='int32',
summode='and')
ret = hau.Time_Z_Group(timevarname='times', altname='altitudes')
setattr(ret, 'times', hau.T_Array([t]))
setattr(ret, 'delta_t', hau.T_Array([f['width'].total_seconds()]))
setattr(ret, 'altitudes', copy.copy(altitudes))
setattr(ret, 'start', f['start'])
setattr(ret, 'width', f['width'])
if self.splitFields:
for f, idx in self.qaparser.flagbits.items():
setattr(
ret, 'qa_' + f,
hau.TZ_Array((flags / (10**idx)) % 10,
dtype='int32',
summode='and'))
else:
setattr(ret, 'qaflags', flags)
yield ret
def process(self):
""" main read generator
"""
profiles = None
calsource = None if self.calsource is None else time_frame.TimeTrickle(
self.calsource,
timename='chunk_start_time',
endtimename='chunk_end_time')
for f in self.framestream:
if f is None:
continue
origf = f
if self.srcscopename is not None:
print 'Updating profile for scope ', self.srcscopename
if hasattr(f, self.srcscopename):
f = getattr(f, self.srcscopename)
else:
f = None
if f is None or f.times.size == 0:
continue
lasttime = f.times[0]
profparams = {}
if calsource is not None:
calframe = calsource(lasttime).copy()
else:
calframe = dict() #rs_constants=self.raman_constants_first)
profparams['calframe'] = calframe
profiles = self.updateProfiles(profiles,
f,
constants=calframe.pop(
'rs_constants', None),
**profparams)
if not self.onlyFinal:
ret = copy.deepcopy(profiles)
if self.subscopename is not None:
tmp = copy.copy(origf)
setattr(tmp, self.subscopename, ret)
ret = tmp
yield ret
elif self.providesRunning:
if profiles is not None: #this is safe only if we are top level
self.setProvidesUsing(profiles)
print 'Profiles code shortcircuiting to get provides out'
self.doingShortCircuit()
yield None
if self.onlyFinal and profiles is not None:
yield profiles
def process(self):
#calsource=time_frame.TimeTrickle(self.cals,timename='chunk_start_time',endtimename='chunk_end_time')
rif = self.rif
pickle = self.pickle
lasttime = None
calsource = time_frame.TimeTrickle(self.cals,
timename='chunk_start_time',
endtimename='chunk_end_time')
for f in self.datasource:
orig = f
if self.sourcename is not None:
if isinstance(f, dict) and self.sourcename in f:
f = f[self.sourcename]
elif hasattr(f, self.sourcename):
f = getattr(f, self.sourcename)
else:
print "No Raman Ranged to process. can't find subframe " + self.sourcename
continue
if f is None or f.times.size == 0:
print "No Raman Ranged to process. Raman Merge is empty"
continue
print "Raman Merge to Ranged running"
lasttime = f.times[0]
caldictionary = calsource(lasttime)
pname = ('frame', 'cal frame')
p1 = []
p1.append(pickle.dumps(f))
p1.append(pickle.dumps(caldictionary))
ret = rif.process_ranged_raman(caldictionary['rs_constants'], f,
self.raman_process_control,
caldictionary['rs_cal'],
self.raman_corr_adjusts)
p2 = []
p2.append(pickle.dumps(f))
p2.append(pickle.dumps(caldictionary))
for i in range(len(p1)):
if p1[i] != p2[i]:
print 'Detected modification of ' + pname[i] + '. FAIL'
raise RuntimeError('Modification of Raman Ranged ' +
pname[i])
hau.verifyAllNew(ramancal=caldictionary,
ramanmerge=f,
ramanrange=ret)
if ret is None:
print "Raman Range is none"
import time
time.sleep(5)
elif hasattr(f, 'alt'): #platform height from sealevel
setattr(ret, '_altitudevarname', 'altitudes')
setattr(ret, "altitudes", hau.Z_Array(f.heights + f.alt))
if self.destination is not None:
if isinstance(orig, dict):
orig[self.destination] = ret
else:
setattr(orig, self.destination, ret)
yield orig
else:
yield ret
def read(self):
"""generator function.
"""
soundingsource = time_frame.TimeTrickle(self.sounding_source,
timename='sample_time',
getClosest=True)
chunk_start_time = None
chunk_end_time = None
sounding = None
priorsounding = None
priorconstants = None
priorcal = None
#print 'beginning const iteration using ',self.consts
for calframe in self.tables:
#print 'const frame is ',constframe
rs_constants = calframe['rs_constants']
rs_cal = calframe['rs_cal']
nextcaltime = calframe['chunk_end_time']
if chunk_start_time == None:
chunk_start_time = calframe['chunk_start_time']
for sounding, chunk_end_time in soundingsource.trickleGenerator(
chunk_start_time, nextcaltime):
pos = ''
if 'installation' in rs_constants and rs_constants[
'installation'] != 'ground':
pos = ' ; %fN %fE' % (sounding.latitude,
sounding.longitude)
print 'sounding= "%s" [ %s%s ]' % (
sounding.station_id[:],
sounding.times.strftime('%d-%b-%y %H:%M'), pos)
if chunk_end_time <= chunk_start_time:
chunk_end_time = nextcaltime
if chunk_end_time == chunk_start_time:
print 'Ignored. chunk is length', (chunk_end_time -
chunk_start_time)
print 'times are', chunk_start_time, chunk_end_time, nextcaltime
break
try:
if priorsounding is not sounding or priorconstants is not rs_constants or priorcal is not rs_cal:
rs_Cxx = rif.raman_quick_cal(rs_constants, rs_cal,
sounding)
priorsounding = sounding
priorconstants = rs_constants
priorcal = rs_cal
except RuntimeError:
if chunk_start_time > datetime.utcnow():
print 'Repeating last calibration. Requested interval extends beyond now (and known calibrations)'
else:
print 'calibration error for an interval in non-future space.'
raise #reraise this.
if chunk_end_time <= chunk_start_time:
print 'WARNING dpl_calibration_narr trying to use 0-length window. source is behind', chunk_end_time, chunk_start_time, nextcaltime
chunk_end_time = nextcaltime
if chunk_end_time <= chunk_start_time:
break
#chunk_end_time=use_end_time
#break
retdict = calframe.copy()
retdict.update(
dict(chunk_start_time=chunk_start_time,
chunk_end_time=chunk_end_time,
sounding=sounding,
Cxx=rs_Cxx))
print 'Yielding calframe ', chunk_start_time, chunk_end_time, sounding.times
yield retdict
chunk_start_time = chunk_end_time
return
def read(self):
"""generator function.
"""
soundingsource = time_frame.TimeTrickle(self.sounding_source,
timename='sample_time',
getClosest=True)
chunk_start_time = None
chunk_end_time = None
sounding = None
priorsounding = None
priorconstants = None
priorcal = None
#print 'beginning const iteration using ',self.consts
for calframe in self.tables:
#print 'const frame is ',constframe
rs_constants = calframe['rs_constants']
rs_cal = calframe['rs_cal']
nextcaltime = calframe['chunk_end_time']
if chunk_start_time == None:
chunk_start_time = calframe['chunk_start_time']
for sounding, chunk_end_time in soundingsource.trickleGenerator(
chunk_start_time, nextcaltime):
pos = ''
if 'installation' in rs_constants and rs_constants[
'installation'] != 'ground':
pos = ' ; %fN %fE' % (sounding.latitude,
sounding.longitude)
print 'sounding= "%s" [ %s%s ]' % (
sounding.station_id[:],
sounding.times.strftime('%d-%b-%y %H:%M'), pos)
if chunk_end_time <= chunk_start_time:
chunk_end_time = nextcaltime
if chunk_end_time == chunk_start_time:
print 'Ignored. chunk is length', (chunk_end_time -
chunk_start_time)
print 'times are', chunk_start_time, chunk_end_time, nextcaltime
break
try:
if priorsounding is not sounding or priorconstants is not rs_constants or priorcal is not rs_cal:
rs_Cxx = cu.update_Cxx(rs_constants, rs_cal, sounding,
self.process_control,
self.corr_adjusts)
if rs_Cxx is None:
continue
priorsounding = sounding
priorconstants = rs_constants
priorcal = rs_cal
except RuntimeError:
if chunk_start_time > datetime.utcnow():
print 'Repeating last calibration. Requested interval extends beyond now (and known calibrations)'
else:
print 'calibration error for an interval in non-future space.'
raise #reraise this.
if chunk_end_time <= chunk_start_time:
warnings.warn(
"dpl_calibration_narr trying to use 0-length window. source is behind"
)
print "behind times", chunk_end_time, chunk_start_time, nextcaltime
chunk_end_time = nextcaltime
if chunk_end_time <= chunk_start_time:
break
#chunk_end_time=use_end_time
#break
#sounding=soundingsource(chunk_start_time)
#setattr(self,'hsrl_constants',rs_constants)#FIXME exposing these in this way, when they are dynamic parts of the framestream is Bad Bad Bad
setattr(
self, 'sounding', sounding
) # should only be init parameters that can be exposed this way
setattr(self, 'Cxx', rs_Cxx)
retdict = calframe.copy()
retdict.update(
dict(chunk_start_time=chunk_start_time,
chunk_end_time=chunk_end_time,
sounding=sounding,
rs_Cxx=rs_Cxx))
print 'Yielding calframe ', chunk_start_time, chunk_end_time, sounding.times
yield retdict
chunk_start_time = chunk_end_time
return