def flush(self, raw=None):
if raw is not None:
self.Rem.append(raw)
raw = self.Rem
self.Rem = hau.Time_Z_Group(like=raw)
if self.post_operator != None:
return self.post_operator.flush(raw)
return raw
def triage(self,*fr):
timeaxis=None
for fram in fr:
if (timeaxis==None or timeaxis.size==0) and fram!=None and hasattr(fram,fram._timevarname):
timeaxis=getattr(fram,fram._timevarname)
if timeaxis is None:
return None
return hau.Time_Z_Group(timeaxis.copy(),timevarname='times',altname='heights')
def ms_sums(diameter, beta, depol, multiple_scatter_parameters, ms_obj):
''' ms_sums(diameter,beta,depol,multiple_scatter_parameters,ms_obj):
diameter = particle diameter profile or profiles
beta = extinction or backscatter cross section profile or profiles
depol = depolarization profile or profiles
returns:
ms_obj = summed profiles
" .diameter_ice
" .diameter_water
" .beta_ice
" .beta_water
" .n_samples_ice
" .n_samples_water '''
is_ice = np.zeros_like(beta)
is_water = np.zeros_like(beta)
is_ice[depol >= multiple_scatter_parameters['h2o_depol_threshold']] = 1.0
is_water[depol < multiple_scatter_parameters['h2o_depol_threshold']] = 1.0
#diameter_ice = diameter * is_ice
#diameter_water = diameter * is_water
beta_ice = beta * is_ice
beta_water = beta * is_water
#compute diameter * beta for ice and water components
if not diameter == None:
diameter_ice = diameter * beta * is_ice
diameter_water = diameter * beta * is_water
#if diameter is not supplied from lidar-radar retrieval get it from constants in multiple_scatter_parameters
else:
diameter_ice = hau.Z_Array(
np.ones_like(beta) *
multiple_scatter_parameters['mode_diameter_ice'] * beta * is_ice)
diameter_water = hau.Z_Array(
np.ones_like(beta) *
multiple_scatter_parameters['mode_diameter_water'] * beta *
is_water)
if ms_obj == None:
ms_obj = hau.Time_Z_Group()
setattr(ms_obj, 'beta_ice', hau.Z_Array(nansum(beta_ice, 0)))
setattr(ms_obj, 'beta_water', hau.Z_Array(nansum(beta_water, 0)))
setattr(ms_obj, 'diameter_ice', hau.Z_Array(nansum(diameter_ice, 0)))
setattr(ms_obj, 'diameter_water',
hau.Z_Array(nansum(diameter_water, 0)))
setattr(ms_obj, 'n_samples_ice',
hau.Z_Array(sum(~np.isnan(beta_ice) * is_ice, 0)))
setattr(ms_obj, 'n_samples_water',
hau.Z_Array(sum(~np.isnan(beta_water) * is_water, 0)))
else:
ms_obj.beta_ice += nansum(beta_ice, 0)
ms_obj.beta_water += nansum(beta_water, 0)
ms_obj.diameter_ice += nansum(diameter_ice, 0)
ms_obj.diameter_water += nansum(diameter_water, 0)
ms_obj.n_samples_ice += sum(~np.isnan(beta) * is_ice, 0)
ms_obj.n_samples_water += sum(~np.isnan(beta) * is_water, 0)
return ms_obj
def triage(self,hsrl,radar):
timeaxis=None
if (timeaxis==None or timeaxis.size==0) and hsrl!=None and hasattr(hsrl,hsrl._timevarname):
timeaxis=getattr(hsrl,hsrl._timevarname)
if (timeaxis==None or timeaxis.size==0) and radar!=None and hasattr(radar,radar._timevarname):
timeaxis=getattr(radar,radar._timevarname)
if timeaxis!=None:
return hau.Time_Z_Group(timeaxis.copy(),timevarname='times',altname='heights')
return None
def combine(self):
tracks = OrderedDict()
for k in self.sechost.keys():
tracks[k] = {}
tracks[k]['iter'] = iter(self.sechost[k])
tracks[k]['store'] = []
if True: #try:
for pf in self.prim:
ret = OrderedDict()
ret[self.primaryname] = pf
lastTime = self.__getLastTime(pf)
frameCount = self.__getCount(pf)
for k in tracks.keys():
if k in self.countMatch:
frames, timerange = self.__eatFrameByCount(
k, tracks[k], frameCount)
else:
frames, timerange = self.__eatFrameByTime(
k, tracks[k], lastTime)
if timerange != None and len(frames) > 0 and hasattr(
frames[0], 'append'):
fms = frames
frames = copy.deepcopy(fms[0])
for x in range(1, len(fms)):
if fms[x] is self.noFramePlaceholder:
if k in self.countMatch:
VERBOSE(
'extending', k,
'FIXME this should have been done by the resampler'
)
frames.append(
hau.Time_Z_Group(
like=frames,
times=self.__getTime(
pf,
self.__getCount(frames),
asArray=True)))
else:
pass #by time, or no extend
else:
frames.append(fms[x])
setattr(frames, 'start', timerange['start'])
setattr(frames, 'width', timerange['width'])
if not frames is self.noFramePlaceholder:
ret[k] = frames
if self.retclass == None:
retv = ret
elif self.classdictparm:
d = self.classparms.copy()
d[self.classdictparm] = ret
retv = self.retclass(**d)
else:
retv = self.retclass(**self.classparms)
for k, i in ret.items():
setattr(retv, k, i)
#print 'merged is',retv
yield retv
def convert(self, profile, posframe):
sounding = hau.Time_Z_Group()
if 'cache' in self.name:
sounding.sounding_type = 'virtual'
sounding.sounding_id = 'Cached Forecast'
sounding.station_id = 'Cached Forecast'
elif 'virtual' in self.name:
sounding.sounding_type = 'virtual'
sounding.sounding_id = 'NWP Virt'
sounding.station_id = 'NWP Virt'
else:
sounding.sounding_type = 'model'
sounding.sounding_id = 'Static GRIB'
sounding.station_id = 'Static GRIB'
sounding.latitude = profile['lat'][0]
sounding.longitude = profile['lon'][0]
sounding.times = datetime(1970, 1, 1, 0, 0, 0) + timedelta(
seconds=profile['base_time']) + timedelta(
seconds=profile['time_offset'][0])
if posframe is None:
sounding.sample_latitude = sounding.latitude
sounding.sample_longitude = sounding.longitude
sounding.sample_time = sounding.times
else:
sounding.sample_latitude = posframe['latitude']
sounding.sample_longitude = posframe['longitude']
sounding.sample_time = posframe['start']
minlen = profile['alt'].size
for k, v in profile.items():
if hasattr(v, 'shape'):
if profile['alt'].size != v.size:
print 'WARNING SOUNDING ATTRIBUTE ' + k + ' has a length', v.size, 'while alts are', profile[
'alt'].size
profile[k] = v[:profile['alt'].size]
if minlen > v.size:
minlen = v.size
if minlen < profile['alt'].size:
print "TRIMMING TO SHORTENED SOUNDING ATTRIBUTE of length", minlen
for k, v in profile.items():
if hasattr(v, 'shape'):
profile[k] = v[:minlen]
sounding.top = numpy.max(profile['alt'])
sounding.bot = numpy.min(profile['alt'])
sounding.altitudes = hau.Z_Array(profile['alt'])
if 'tkel' in profile:
sounding.temps = hau.Z_Array(profile['tkel'])
else:
sounding.temps = hau.Z_Array(profile['tdry']) + 273.15
sounding.pressures = hau.Z_Array(profile['pres'])
sounding.dew_points = hau.Z_Array(
self.su.cal_dew_point(hau.Z_Array(profile['rh']), sounding.temps))
sounding.frost_points = hau.Z_Array(
self.su.cal_frost_point(sounding.dew_points))
return sounding
def process(self):
fr = None
flags = None
olda = None
qasource = tf.TimeTrickle(self.qasource, 'time')
altitudes = hau.Z_Array(self.qaparser.altitudeAxis)
for f in self.timealtsource:
#FIXME include angles
if not isinstance(f, dict):
f = vars(f)
t = f[self.timename]
a = self.constantAltitude
if fr is None or ((not qasource.atEnd) and t >= qasource.nextTime
): #if need an update to the qa record
#print 'Getting qa source for time',t
fr = qasource(t)
flags = None
if 'range_flags' in fr and fr[
'range_flags'] is not None: #if there is a range dependence, and a potentially non-constant altitude
if self.altname is not None and self.altname in f:
a = f[self.altname]
if a is None:
raise RuntimeError(
'Need platform altitude to merge in range-dependant qa Flags'
)
if olda is None or a != olda:
flags = None
olda = a
if flags is None: #was cleared either because new flags from the qa file, or new altitude from the stream
if 'range_flags' in fr and fr['range_flags'] is not None:
flags = self.qaparser.mergeVectors(fr['flags'],
fr['range_flags'], a)
else:
flags = fr['flags']
flags = self.qaparser.translateToEnumeration(flags)
flags = hau.TZ_Array(flags.reshape([1] + list(flags.shape)),
dtype='int32',
summode='and')
ret = hau.Time_Z_Group(timevarname='times', altname='altitudes')
setattr(ret, 'times', hau.T_Array([t]))
setattr(ret, 'delta_t', hau.T_Array([f['width'].total_seconds()]))
setattr(ret, 'altitudes', copy.copy(altitudes))
setattr(ret, 'start', f['start'])
setattr(ret, 'width', f['width'])
if self.splitFields:
for f, idx in self.qaparser.flagbits.items():
setattr(
ret, 'qa_' + f,
hau.TZ_Array((flags / (10**idx)) % 10,
dtype='int32',
summode='and'))
else:
setattr(ret, 'qaflags', flags)
yield ret
def process_hsrl_raman_profile(hsrl_profile,
raman_profile,
parameters,
entire_frame=None):
print
print 'entering process_hsrl_raman_profile in cooperative-------------------'
rs = hau.Time_Z_Group(like=hsrl_profile)
rs.hsrl_profile = copy.deepcopy(hsrl_profile)
rs.raman_profile = copy.deepcopy(raman_profile)
print 'leaving process_hsrl_raman_profile'
return rs
def triage(self, hsrl, raman):
timeaxis = None
if (timeaxis == None or timeaxis.size
== 0) and hsrl != None and hasattr(hsrl, hsrl._timevarname):
timeaxis = getattr(hsrl, hsrl._timevarname)
if (timeaxis == None or timeaxis.size
== 0) and raman != None and hasattr(raman, raman._timevarname):
timeaxis = getattr(raman, raman._timevarname)
if timeaxis != None:
return hau.Time_Z_Group(timeaxis.copy(),
timevarname='times',
altname='altitudes')
return None
def triage(self, hsrl, particle, rs):
timeaxis = None
if (timeaxis == None or timeaxis.size
== 0) and hsrl != None and hasattr(hsrl, hsrl._timevarname):
timeaxis = getattr(hsrl, hsrl._timevarname)
if (timeaxis == None
or timeaxis.size == 0) and particle != None and hasattr(
particle, particle._timevarname):
timeaxis = getattr(particle, particle._timevarname)
if timeaxis == None:
return None
return hau.Time_Z_Group(timeaxis.copy(),
timevarname='times',
altname='heights')
def process_hsrl_radar(process_parameters=None,rs_inv=None,rs_mean=None,rs_mwacr=None,rs_kazrge=None,**kwargs):
#create timez group and add heights
rs_cooperative=hau.Time_Z_Group(rs_inv.times.copy(),timevarname='times',altname='heights')
setattr(rs_cooperative,'heights',rs_inv.msl_altitudes.copy())
print 'called process_hsrl_radar'
print kwargs
print rs_mwacr
print rs_kazrge
#raise RuntimeError('RUNNING HSRLRADAR')
return rs_cooperative
def render(self):
subframes = None
for frame in self.framestream:
if frame != None:
if self.limit_frame_to != None or self.breakup_nesting:
if subframes is None:
if self.limit_frame_to is not None:
subframes = [self.limit_frame_to]
else:
subframes = []
omit = None
include = None
if hasattr(frame, 'rs_mean') or hasattr(
frame,
'rs_inv') or not hasattr(frame, 'rs_raw'):
omit = 'raw'
else:
include = 'raw'
for x in vars(frame).keys():
if x.startswith('_'):
continue
if omit is not None and omit in x:
continue
if include is not None and include not in x:
continue
subframes.append(x)
for k in subframes:
if hasattr(frame, k):
if self.flat_frame:
print 'Showing images for HSRL flat frame ' + k
self.renderframe(getattr(frame, k), frame)
else:
import lg_base.core.array_utils as hau
sendframe = hau.Time_Z_Group(like=frame)
setattr(sendframe, k, getattr(frame, k))
print 'Showing images for HSRL subframe ' + k
if k == 'rs_raw':
setattr(sendframe, 'rs_mean',
getattr(frame, k))
elif k == 'rs_mean':
setattr(sendframe, 'rs_raw',
getattr(frame, k))
self.renderframe(sendframe, frame)
else:
self.renderframe(frame, frame)
self.figcontainer.shownew()
yield frame
def accumulate_raman_profiles(consts,
rs_mean,
qc_mask,
old_profiles,
process_control=None,
rs_cal=None,
Cxx=None,
corr_adjusts=None):
indices = np.arange(rs_mean.times.shape[0])
if len(indices) == 0:
return old_profiles
if qc_mask is not None and processing_defaults.get_value(
'averaged_profiles', 'apply_mask'):
#make mask array with NaN's for array elements where bit[0] of qc_mask==0
#all other elements of mask = 1
mask = (np.bitwise_and(qc_mask, 1)).astype(
'float') #mask is float to allow use of NaN values
mask[mask == 0] = np.NaN
print 'qc_mask applied to time averaged profiles vs altitude'
else:
#set mask == 1
mask = None
#for sh in channel_shorthand.keys():
#
# if hasattr(rs_mean,sh):
# mask = np.ones_like(getattr(rs_mean,sh))
# break
#if mask is None:
# mask = np.ones((rs_mean.times.shape[0],0))
print 'qc_mask has not been applied to time averaged profiles'
#energies=('transmitted_1064_energy','transmitted_energy')
profiles = hau.Time_Z_Group(can_append=False, altname='altitudes')
profiles.hist = hau.Time_Z_Group()
ft = None
tc = 0
#tv=0
lt = None
if old_profiles is not None:
#total_seeded_shots=total_seeded_shots+profiles.hist.total_seeded_shots
ft = old_profiles.hist.ft
tc = old_profiles.hist.tc
#tv=old_profiles.hist.tv
lt = old_profiles.hist.lt
if len(indices) > 0:
if ft is None:
ft = rs_mean.times[indices][0]
lt = rs_mean.times[indices][-1] + timedelta(
seconds=rs_mean.delta_t[indices][-1]
if not np.isnan(rs_mean.delta_t[indices][-1]) else 0)
for x in indices:
if rs_mean.times[x] is not None:
tc = tc + 1
#tv=tv+(rs_mean.times[x]-ft).total_seconds()
if tc > 0:
profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ])
profiles.start = ft
profiles.width = lt - ft
profiles.delta_t = hau.T_Array([profiles.width.total_seconds()])
else:
profiles.times = hau.T_Array([])
profiles.start = ft
profiles.width = timedelta(seconds=0)
profiles.delta_t = hau.T_Array([])
profiles.start_time = ft
profiles.end_time = lt
profiles.hist.ft = ft
profiles.hist.tc = tc
#profiles.hist.tv=tv
profiles.hist.lt = lt
#profiles.hist.total_seeded_shots=total_seeded_shots
if rs_mean is not None and hasattr(rs_mean, 'altitudes'):
profiles.altitudes = rs_mean.altitudes.copy()
#elif hasattr(old_profiles,'altitudes'):
# profiles.altitudes=old_profiles.altitudes
elif hasattr(old_profiles, 'heights'):
profiles.heights = old_profiles.heights
accumulate(profiles,
old_profiles,
rs_mean,
indices,
'shots',
pref='mean_',
filler=hau.T_Array([0]))
total_shots = profiles.mean_shots
profiles.shots = total_shots.copy()
print 'Total shots for profile =', total_shots
for e in energies:
accumulate(profiles, old_profiles, rs_mean, indices, e, total_shots)
# create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke
for chan in channel_shorthand.keys():
accumulate(profiles, old_profiles, rs_mean, indices, chan, total_shots,
mask)
#compute inverted profiles from mean count profiles
if Cxx is not None and hasattr(profiles, 'elastic_counts'):
import raman.core.raman_processing_utilities as rpu
profiles.inv = rpu.process_raman(consts, profiles, process_control,
rs_cal, Cxx, corr_adjusts)
import lidar.sg_extinction as lsge
filter_params = lsge.filter_setup(profiles.inv.altitudes,
process_control, consts)
sg_ext = lsge.sg_extinction(filter_params)
profiles.inv.extinction,profiles.inv.extinction_aerosol,profiles.inv.p180 \
= sg_ext(profiles.times,profiles.delta_t,profiles.nitrogen_counts ,profiles.inv.beta_a_backscat,profiles.inv.integ_backscat,beta_r=Cxx.beta_r_355)
elif hasattr(old_profiles, 'inv'):
profiles.inv = old_profiles.inv
return profiles
def __init__(self, ntime_ave, post_operator=None):
self.ntime_ave = ntime_ave
self.post_operator = post_operator
self.Rem = hau.Time_Z_Group()
def read(self):
""" main read generator
"""
import hsrl.data_stream.processing_utilities as pu
params=self.params
firsttimeever=None
intervalTime=None
intervalEnd=None
rawsrc=iter(self.rawsrc)
#if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])):
# params['timeres']=None #pure native
end_time_datetime=params['finalTime']
#timemodoffset=time_mod(params['realStartTime'],params['timeres'])
noframe='noframe'
fullrange=False #if this is true, it will pad the start with any missing times.
remainder=None
cdf_to_hsrl = None
preprocess_ave = None
requested_times=None
instrument=self.hsrl_instrument
intcount=0
rs_mem = None
#rs=None
timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None
for calv in self.cal_narr:
if intervalTime is None:
firsttimeever=calv['chunk_start_time']
intervalTime=calv['chunk_start_time']
intervalEnd=intervalTime
chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset)
#print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres']
if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None:
if params['block_when_out_of_data']:
if 'timeres' not in params or params['timeres'] is None:
sleep(calv['rs_constants']['integration_time'])
else:
sleep(params['timeres'].total_seconds())
else:
yield None #this is done to get out of here, and not get stuck in a tight loop
continue
while intervalTime<chunk_end_to_use:
integration_time = calv['rs_constants']['integration_time']
doPresample=True
#END init section
if intervalEnd>chunk_end_to_use:
print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime
break
else:
intervalEnd=chunk_end_to_use
#print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime']
print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem
if True:#requested_times==None or requested_times.shape[0]>0:
try:
try:
while rawsrc is not None:
if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
break
tmp=rawsrc.next()
if hasattr(tmp,'rs_raw'):
if rs_mem is not None:
rs_mem.append(tmp.rs_raw)
else:
rs_mem=copy.deepcopy(tmp.rs_raw)
if rs_mem is not None and rs_mem.times.shape>0:
break
else:
rs_mem=None
except StopIteration:
print 'Raw HSRL stream is ended'
rawsrc=None
if rs_mem is None or rs_mem.times.size==0:
rs_mem=None
elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0]
break
else:
intervalEnd=rs_mem.times[-1]
print 'read in raw frame to mean',rs_mem,remainder
if rawsrc is None:
intervalEnd=chunk_end_to_use
print 'trimmed ',rs_mem
if timesource is not None:
if timesource.isDone:
break
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime)
timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime)))
print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd
elif 'timeres' in params and params['timeres'] is not None:
tmp=intervalTime
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
timevals=[]
timevals.append(tmp)
while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats.
#print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds()
tmp+=params['timeres']
timevals.append(tmp)
#intervalEnd=tmp
intcount+=len(timevals)
if usePrebinnedTimes:
intcount-=1
print 'Now %i intervals' % (intcount)
timevals=hau.T_Array(timevals)
else:
print 'Using Native timing'
timevals=None
print ' new window is ', intervalTime, ' to ' , intervalEnd
requested_times=timevals
requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime]
if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None:
#if rawsrc is not None:
print "not enough time to process"
continue
elif rawsrc is None and rs_mem is None and remainder is None:
#chunk_end_to_use=intervalTime
#continue
#print ''
break
rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd
,params['min_alt'], params['max_alt'], requested_chunk_times
, rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal']
, None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control
, self.compute_stats,remainder=remainder)
rs_mem=None
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0:
rs_chunk=None
if rs_chunk is None and rawsrc is None:
break
#print rs_chunk
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0:
if fullrange and requested_chunk_times is not None:
v=hau.Time_Z_Group(like=rs_chunk.rs_mean)
v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]])
if v.times.size>0:
rs_chunk.rs_mean.prepend(v)
rs_chunk.calv=calv
yield rs_chunk
intervalTime=intervalEnd
except Exception, e:
print 'Exception occured in update_cal_and_process'
print 'Exception = ',e
print traceback.format_exc()
if isinstance(e,(MemoryError,)):
print 'Please Adjust Your Parameters to be more Server-friendly and try again'
raise
if not isinstance(e,(AttributeError,)):
raise
def generate_ave_profiles(rs_mean,
qc_mask,
rs_Cxx,
rs_constants,
processing_defaults,
sel_telescope_dir,
corr_adjusts,
old_profiles=None):
"""create average of profiles dependent on telescope pointing direction
, telescope pointing may be 'all', 'zenith', or 'nadir' """
try:
[ntimes, nalts] = rs_mean.molecular_counts.shape
except AttributeError:
#raise RuntimeError, \
# "molecular counts missing"
ntimes = 0
nalts = 500
#if rs_mean.times.shape[0] == 0:
#raise RuntimeError, \
# "times missing"
# return old_profiles
if qc_mask is not None and processing_defaults.get_value(
'averaged_profiles', 'apply_mask'):
#make mask array with NaN's for array elements where bit[0] of qc_mask==0
#all other elements of mask = 1
mask = (np.bitwise_and(qc_mask, 1)).astype('float')
#mask is float to allow use of NaN values
mask[mask == 0] = np.NaN
print 'qc_mask applied to time averaged profiles vs altitude'
else:
#set mask == 1
mask = None
print 'qc_mask has not been applied to time averaged profiles'
# most of the time we need to generate some/all of these profiles,
# because users are plotting them
# if this is too time-consuming, we'll generate just the ones necessary for the
# user selected plots.
indices = np.arange(rs_mean.times.shape[0])
indices = (indices >= 0) #boolean now
if ntimes == 0 and len(indices) == 0:
return old_profiles
#average only those profiles occuring when locked to the i2 line
#and not in cal_scan mode
#op_mode = rs.rs_raw.op_mode.astype(int)
#not_cal_scan = (~op_mode[:] & 32)/32
#locked_to_i2 = ((op_mode[:] & 4)/4)
#bits = np.zeros((len(rs.rs_raw.times),len(bit_value)+1))
#opmode=rs.rs_raw.op_mode.astype('uint32')
#for i in range(len(bit_value)):
# bits[:,i]=(i+1)*(opmode[:] & bit_value[i])/bit_value[i]
#this dictionary maps the long name to the shorthand used with dark_counts variables
# the keys are used here as a list of all possible channels too
channel_shorthand = dict(molecular_counts='mol',
combined_hi_counts='c_hi',
combined_lo_counts='c_lo',
combined_wfov_counts='c_wfov',
combined_1064_counts='combined_1064',
molecular_wfov_counts='m_wfov',
molecular_i2a_counts='mol_i2a',
cross_pol_counts='c_pol',
combined_counts='comb')
#,combined_1064_counts='combined_1064'
#select desired telescope pointing direction for profiles
if (rs_constants['installation'] == 'ground'
or rs_constants['installation'] == 'shipborne'
or sel_telescope_dir == 'all'):
print 'Selecting all telescope pointing directions for profiles'
if processing_defaults is not None and processing_defaults.get_value(
'averaged_profiles', 'apply_mask'):
indices[rs_mean.i2_locked <= 0.99] = False
elif sel_telescope_dir == 'zenith':
#if telescope pointing exists, limit to selected pointing direction
if rs_mean.telescope_pointing.shape[0]:
print 'Selecting only zenith pointing data for profiles'
indices[rs_mean.telescope_pointing != 1] = False
rs_mean.telescope_pointing = rs_mean.telescope_pointing[indices]
else:
print 'Warning--using all shots--no telescope pointing direction in data file'
elif sel_telescope_dir == 'nadir':
#if telescope pointing exists, limit to selected pointing direction
if rs_mean.telescope_pointing.shape[0]:
print 'Selecting only nadir pointing data for profiles'
indices[rs_mean.telescope_pointing != 0] = False
else:
print 'Warning--using all shots--no telescope pointing direction in data file'
else:
raise RuntimeError, \
"Unrecognized value '%s' for telescope pointing dir--valid(all, zenith,nadir)" \
% (sel_telescope_dir)
#this is meant to filter out chunks from rs_mean that are calibration intervals
if hasattr(rs_mean, 'op_mode'):
indices[np.bitwise_and(rs_mean.op_mode, 16) != 0] = False
indices = np.arange(indices.shape[0])[indices] #back to indexes
profiles = hau.Time_Z_Group(can_append=False)
profiles.hist = hau.Time_Z_Group()
ft = None
tc = 0
#tv=0
lt = None
if old_profiles is not None:
#total_seeded_shots=total_seeded_shots+profiles.hist.total_seeded_shots
ft = old_profiles.hist.ft
tc = old_profiles.hist.tc
#tv=old_profiles.hist.tv
lt = old_profiles.hist.lt
if len(indices) > 0:
if ft is None:
ft = rs_mean.times[indices][0]
lt = rs_mean.times[indices][-1] + timedelta(
seconds=rs_mean.delta_t[indices][-1]
if not np.isnan(rs_mean.delta_t[indices][-1]) else 0)
for x in indices:
if rs_mean.times[x] is not None:
tc = tc + 1
#tv=tv+(rs_mean.times[x]-ft).total_seconds()
if tc > 0:
profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ])
profiles.start = ft
profiles.width = lt - ft
profiles.delta_t = hau.T_Array([profiles.width.total_seconds()])
else:
profiles.times = hau.T_Array([])
profiles.start = ft
profiles.width = timedelta(seconds=0)
profiles.delta_t = hau.T_Array([])
profiles.start_time = ft
profiles.end_time = lt
profiles.hist.ft = ft
profiles.hist.tc = tc
#profiles.hist.tv=tv
profiles.hist.lt = lt
#profiles.hist.total_seeded_shots=total_seeded_shots
if rs_mean is not None and hasattr(rs_mean, 'msl_altitudes'):
profiles.msl_altitudes = rs_mean.msl_altitudes
elif hasattr(old_profiles, 'msl_altitudes'):
profiles.msl_altitudes = old_profiles.msl_altitudes
if rs_mean is not None and hasattr(rs_mean, 'geo_corr'):
profiles.geo_corr = rs_mean.geo_corr
elif hasattr(old_profiles, 'geo_corr'):
profiles.geo_corr = old_profiles.geo_corr
if rs_constants['installation'] == 'airborne' and len(indices) > 0:
if hasattr(rs_mean, 'GPS_MSL_Alt'):
profiles.min_GPS_MSL_Alt = np.min(rs_mean.GPS_MSL_Alt[indices])
profiles.mean_GPS_MSL_Alt = nanmean(rs_mean.GPS_MSL_Alt[indices])
profiles.max_GPS_MSL_Alt = np.max(rs_mean.GPS_MSL_Alt[indices])
profiles.telescope_pointing = np.zeros(1)
if hasattr(rs_mean, 'telescope_pointing'):
if (rs_mean.telescope_pointing > .95).all():
profiles.telescope_pointing[0] = 1
elif (rs_mean.telescope_pointing < .05).all():
profiles.telescope_pointing[0] = 0
else:
profiles.telescope_pointing[0] = np.NaN
accumulate(profiles,
old_profiles,
rs_mean,
indices,
'seeded_shots',
pref='mean_',
filler=hau.T_Array([0]))
total_seeded_shots = profiles.mean_seeded_shots
profiles.seeded_shots = total_seeded_shots.copy()
print 'Total seeded shots for profile =', total_seeded_shots
accumulate(profiles, old_profiles, rs_mean, indices,
'transmitted_1064_energy', total_seeded_shots)
accumulate(profiles, old_profiles, rs_mean, indices, 'transmitted_energy',
total_seeded_shots)
# create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke
for chan in channel_shorthand.keys():
accumulate(profiles, old_profiles, rs_mean, indices, chan,
total_seeded_shots, mask)
#compute inverted profiles from mean count profiles
if rs_Cxx is not None and hasattr(profiles, 'molecular_counts'):
profiles.inv = cu.hsrl_inversion(profiles, rs_Cxx, rs_constants,
corr_adjusts, processing_defaults)
elif hasattr(old_profiles, 'inv'):
profiles.inv = old_profiles.inv
#adds raw_color_ratio to profiles.inv
if hasattr(profiles,'inv') and hasattr(profiles,'combined_counts') \
and hasattr(profiles,'combined_1064_counts'):
if 0:
print 'profiles'
print dir(profiles)
print 'inv'
print dir(profiles.inv)
import matplotlib.pylab as plt
plt.figure(3333)
plt.plot(
profiles.combined_counts[0, :], profiles.inv.msl_altitudes,
'r', profiles.cross_pol_counts[0, :],
profiles.inv.msl_altitudes, 'g',
profiles.combined_counts[0, :] +
rs_constants['combined_to_cross_pol_gain_ratio'] *
profiles.cross_pol_counts[0, :], 'c',
profiles.combined_1064_counts[0, :],
profiles.inv.msl_altitudes, 'k')
plt.grid(True)
ax = plt.gca()
ax.set_xscale('log')
profiles.inv.raw_color_ratio = cu.compute_raw_color_ratio(
profiles, rs_Cxx, rs_constants, corr_adjusts)
if 0:
plt.figure(3334)
plt.plot(profiles.inv.raw_color_ratio[0, :],
profiles.inv.msl_altitudes, 'c')
plt.grid(True)
ax = plt.gca()
ax.set_xscale('log')
#generate klett profiles if requested
if processing_defaults is not None and processing_defaults.get_value(
'klett', 'enable'):
ref_altitude = processing_defaults.get_value('klett',
'ref_altitude') * 1000.0
if ref_altitude < profiles.inv.msl_altitudes[0] \
or ref_altitude > profiles.inv.msl_altitudes[-1] :
print
print
print 'WARNING---klett ref altitutde=', ref_altitude, ' is not in requested altitudes'
print 'no klett profile retrieval attempted '
print
else:
if hasattr(profiles, 'combined_1064_counts'):
profiles.inv.beta_a_1064_backscat_klett = lu.compute_klett_backscatter(
profiles.combined_1064_counts,
profiles.inv.beta_r_backscat / 16.0,
profiles.inv.msl_altitudes,
processing_defaults.get_value('klett', 'lidar_ratio_532'),
ref_altitude)
if hasattr(profiles, 'combined_counts'):
profiles.inv.beta_a_532_backscat_klett = lu.compute_klett_backscatter(
profiles.combined_counts, profiles.inv.beta_r_backscat,
profiles.msl_altitudes,
processing_defaults.get_value('klett', 'lidar_ratio_532'),
ref_altitude)
for chan in channel_shorthand.keys():
accumulate(profiles, old_profiles, rs_mean, indices, 'var_raw_' + chan,
total_seeded_shots, mask)
accumulate(profiles, old_profiles, rs_mean, indices, 'raw_' + chan,
total_seeded_shots, mask)
if processing_defaults is not None and processing_defaults.get_value(
'compute_stats', 'enable'):
#kludge
pf = hau.Time_Z_Group()
for chan in channel_shorthand.keys():
if hasattr(profiles, 'sum_var_raw_' + chan):
setattr(pf, chan, getattr(profiles, 'sum_' + chan))
setattr(pf, 'var_raw_' + chan,
getattr(profiles, 'sum_var_raw_' + chan))
if rs_Cxx is not None and hasattr(profiles, 'inv'):
pu.compute_photon_statistics(pf, profiles.inv, rs_Cxx,
rs_constants)
if rs_Cxx is not None and hasattr(profiles, 'inv'):
[profiles.inv.optical_depth, profiles.inv.optical_depth_aerosol
, profiles.inv.mol_norm_index,profiles.inv.mol_ref_aod] = \
lu.compute_optical_depth(profiles.inv.Nm \
,profiles.inv.beta_r_backscat\
,profiles.msl_altitudes\
,processing_defaults\
,rs_constants
,profiles.inv.telescope_pointing if hasattr(profiles.inv,'telescope_pointing') else None)
#add 1064 aerosol backscatter and color ratio to profiles
if hasattr(profiles, 'combined_counts') and hasattr(
profiles, 'combined_1064_counts'):
cu.compute_1064_aerosol_backscatter(profiles, profiles.inv,
processing_defaults,
rs_constants, corr_adjusts)
cu.compute_color_ratio(profiles.inv)
if processing_defaults.get_value('extinction_processing',
'filter_type') == 'savitzky_golay':
od_threshhold = processing_defaults.get_value(
'extinction_processing', 'od_threshhold')
z_window_width = processing_defaults.get_value(
'extinction_processing', 'alt_window_length')
order = processing_defaults.get_value('extinction_processing',
'polynomial_order')
min_filter_alt = processing_defaults.get_value(
'extinction_processing', 'min_alt')
if min_filter_alt < profiles.msl_altitudes[0]:
min_filter_alt = profiles.msl_altitudes[0]
adaptive = processing_defaults.get_value('extinction_processing',
'adaptive')
t_window_width = 0.0
if profiles.inv.times.size == 0:
pass
elif hasattr(rs_mean, 'telescope_pointing'):
profiles.inv = filtered_extinction(
profiles.inv, profiles.msl_altitudes, min_filter_alt,
od_threshhold, t_window_width, z_window_width, order,
adaptive, rs_mean.telescope_pointing)
else:
profiles.inv = filtered_extinction(
profiles.inv, profiles.msl_altitudes, min_filter_alt,
od_threshhold, t_window_width, z_window_width, order,
adaptive)
else:
bin_delta = processing_defaults.get_value('extinction_processing',
'bin_delta')
bin_delta = int(bin_delta)
pts_to_ave = processing_defaults.get_value('extinction_processing',
'ext_pts_to_ave')
if hasattr(rs_mean, 'telescope_pointing'):
profiles.inv = pu.compute_extinction(
profiles.inv, profiles.msl_altitudes, bin_delta,
pts_to_ave, rs_mean.telescope_pointing)
else:
profiles.inv = pu.compute_extinction(profiles.inv,
profiles.msl_altitudes,
bin_delta, pts_to_ave)
# raw profiles--ave photons/bin/laser_pulse
#for chan in channel_shorthand.keys():
# accumulate(profiles,old_profiles,rs_raw,raw_indices,chan,raw_total_seeded_shots,pref='raw_')
if 0:
import matplotlib.pylab as plt
plt.figure(898989)
plt.plot(np.nanmean(rs_mean.combined_1064_counts,
0), rs_mean.msl_altitudes, 'k',
np.nanmean(rs_mean.combined_counts, 0), rs_mean.msl_altitudes,
'r')
ax = plt.gca()
ax.set_xscale('log')
# dark corrected raw profiles
for chan, shorthand in channel_shorthand.items():
dcchan = shorthand + '_dark_counts'
correc = 'dc_' + chan
source = 'raw_' + chan
if accumulate(profiles,
old_profiles,
rs_mean,
indices,
dcchan,
total_seeded_shots,
extravars=[correc]):
if hasattr(profiles, source):
#print 'Applying dark count from raw frame to raw counts*******'
setattr(profiles, correc,
getattr(profiles, source) - getattr(profiles, dcchan))
elif hasattr(old_profiles, correc):
setattr(profiles, correc, getattr(old_profiles, correc))
print 'Copying corrected counts because source doesn\'t exist???? WARNING ***'
if hasattr(old_profiles, source):
setattr(profiles, source, getattr(old_profiles, source))
else:
raise RuntimeError(
'Something is wrong with input channels. BUG!')
if processing_defaults is not None:
profiles.mol_norm_alt = processing_defaults.get_value(
'mol_norm_alt', 'meters')
return profiles
def select_raqms_profile(soundings,
request_time,
requested_altitudes,
offset=0):
"""selects sounding prior to request_time from soundings -- the sounding
is returned in a Time_Z_Group as Z_arrays"""
if soundings is None or soundings.times.size == 0:
raise RuntimeError, "select_faqms_profile: No soundings for %s " %\
request_time
import atmospheric_profiles.soundings.sounding_utilities as su
sounding = hau.Time_Z_Group()
sounding.altitudes = hau.Z_Array(requested_altitudes)
max_alt = requested_altitudes[-1]
max_bin = len(requested_altitudes)
index = sum(soundings.times <= request_time) - 1 + offset
if index < 0 or index >= len(soundings.times):
return None
#initialize variables for inclusion in T_Z_Group
sounding.temps = hau.Z_Array(np.zeros((max_bin)))
sounding.dew_points = hau.Z_Array(np.zeros(max_bin))
sounding.frost_points = hau.Z_Array(np.zeros(max_bin))
sounding.pressures = hau.TZ_Array(np.zeros(max_bin))
sounding.ext_total = hau.TZ_Array(np.zeros(max_bin))
sounding.ext_salt = hau.TZ_Array(np.zeros(max_bin))
sounding.wind_spd = hau.TZ_Array(np.zeros(max_bin))
sounding.wind_dir = hau.TZ_Array(np.zeros(max_bin))
#sounding.times is a single time at this point, however it will later be included
#in a list of all the soundings used in this processing request. In order that it
#be treated properly it must be defined as a T_Array
sounding.times = hau.T_Array([soundings.times[index]])
sounding.latitude = hau.T_Array([soundings.latitude[index]])
sounding.longitude = hau.T_Array([soundings.longitude[index]])
#sounding.times=hau.T_Array([soundings.times[index]])
#interpolate model levels to lidar bin altitudes
#temp=interpolate.splrep(soundings.model_level_alts[index,-1::-1] \
# ,soundings.temperatures[index,-1::-1])
#sounding.temps=interpolate.splev(sounding.altitudes,temp,der=0)
temp=interpolate.splrep(soundings.model_level_alts[index,-1::-1] \
,soundings.pressures[index,-1::-1])
sounding.pressures = interpolate.splev(sounding.altitudes, temp, der=0)
sounding.temps=np.interp(sounding.altitudes \
,soundings.model_level_alts[index,-1::-1] \
,soundings.temperatures[index,-1::-1])
#calculate dew point at model levels for selected profile
dew_pts=su.cal_dew_point(soundings.relative_humidity[index,:] \
,soundings.temperatures[index,:])
frost_pts = su.cal_frost_point(dew_pts)
#calculate wind speed and direction from u and v
u_vel = soundings.u_vel[index, -1::-1]
v_vel = soundings.v_vel[index, -1::-1]
wind_spd = np.sqrt(u_vel**2 + v_vel**2)
wind_dir = np.arctan(v_vel / u_vel) * 180.0 / np.pi
for i in range(len(u_vel)):
if (u_vel[i] < 0 and v_vel[i]) < 0:
wind_dir[i] = 180.0 - wind_dir[i]
elif (u_vel[i] > 0 and v_vel[i]) > 0:
wind_dir[i] = 180.0 + wind_dir[i]
elif u_vel[i] < 0:
wind_dir[i] = 270.0 - wind_dir[i]
else:
wind_dir[i] = np.nan
#interpolate to lidar bin altitudes
sounding.frost_points=np.interp(sounding.altitudes \
,soundings.model_level_alts[index,-1::-1],frost_pts[-1::-1])
sounding.dew_points=np.interp(sounding.altitudes \
,soundings.model_level_alts[index,-1::-1],dew_pts[-1::-1])
sounding.ext_total=np.interp(sounding.altitudes\
,soundings.model_level_alts[index,-1::-1]\
,soundings.ext_total[index,-1::-1])
sounding.ext_salt=np.interp(sounding.altitudes\
,soundings.model_level_alts[index,-1::-1]\
,soundings.ext_salt[index,-1::-1])
sounding.ext_dust=np.interp(sounding.altitudes\
,soundings.model_level_alts[index,-1::-1]\
,soundings.ext_dust[index,-1::-1])
sounding.wind_dir = np.interp(sounding.altitudes \
,soundings.model_level_alts[index,-1::-1],wind_dir)
sounding.wind_spd = np.interp(sounding.altitudes \
,soundings.model_level_alts[index,-1::-1],wind_spd)
sounding.top = sounding.altitudes[-1]
sounding.bot = sounding.altitudes[0]
#plt.figure(1)
#plt.plot(temperatures,altitudes,dew_points,altitudes)
#plt.figure(2)
#plt.plot(ext_total,altitudes)
#plt.show()
return sounding
def quick_cal( i2_scan, Cam, Cam_i2a,sounding, wavelength, method_string
, i2_scan_corr, i2a_scan_corr):
"""quick_cal_stream(i2_scan, Cam, sounding, wavelength, method_string, i2_scan_corr)
A function which computes hsrl calibration coefficients.
at the altitudes specified (in meters) within 'alt_vector'
using a precomputed iodine scan file(i2_scan_file).
i2_scan(:,:) = input containing i2 scan info
i2_scan(:,0) = freq (GHz)
-------(:,1) = combined channel scan
-------(:,2) = molecular channel scan
-------(:,3) = theoretical i2 transmission
-------(:,4) = measured i2/combined
if bagohsrl with argon buffered i2 cell
-------(:,5) = molecular i2a/combined
-------(:,6) = molecular i2a channel scan
Cam = aerosol in molecular coefficent
Cam_i2a = aerosol in argon-buffered molecular channel (will = None when not present)
sounding = return structure from read_sounding_file.py contains temp profile
wavelength = laser wavelength (nm)
method_string = molecular line shape ''maxwellian','tenti_s6','wirtschas'
i2_scan_corr = adjustment factor for i2 scan that adjusts
i2 molecular channel gain relative to combined channel gain
i2a_scan_corr = adjustment factor for i2a scan that adjusts
i2a molecular channel gain relative to combined channel gain
rs = return structure containing calibration coefficents
calibration values are returned at altitudes
rs_sounding.alititudes[:]
1 = particulate in combined channel
rs.Cmc[i] = molecular in combined channel
rs.Cmm[i] = molecular in molecular channel
rs.Cam = particulate in molecular channel
rs.beta_r[i] = Rayleigh scattering cross section (1/m)"""
rs = hau.Time_Z_Group() # calibration structure
# i2_scan=cal_vec.i2_scan
i2_scan=i2_scan.copy()
# if selected use theory*combined as synthetic mol scan
print method_string
if method_string.find('i2 theory') >= 0:
i2_scan[:, 2] = i2_scan[:, 4] * i2_scan[:, 1]
# trim i2 scan to +-4 GHz about line center
#i2_scan = i2_scan[abs(i2_scan[:, 0]) <= 4, :]
# rescale i2 molecular component of i2 scan if required
if i2_scan_corr != 1.0:
i2_scan[:, 2] = i2_scan[:, 2] * i2_scan_corr
# rescale i2a molecular component of i2 scan if required
if i2a_scan_corr != 1.0 and i2_scan.shape[1]>6:
i2_scan[:, 6] = i2_scan[:, 6] * i2a_scan_corr
if 0:
import matplotlib.pylab as plt
plt.figure(444443)
plt.plot(i2_scan[:,0],i2_scan[:,1:3],i2_scan[:,0],i2_scan[:,2]/i2_scan[:,1],'k')
plt.xlabel('freq GHz')
plt.grid(True)
# trim i2 scan to +-4 GHz about line center
i2_scan = i2_scan[abs(i2_scan[:, 0]) <= 4, :]
if 0:
import matplotlib.pylab as plt
plt.figure(444444)
plt.plot(i2_scan[:,0],i2_scan[:,1:3])
plt.xlabel('freq GHz')
plt.grid(True)
# compute Rayleigh scattering cross section
# see R Holz thesis for this equation giving the Rayleigh scatter cross section.
# beta=3.78e-6*press/temp at a wavelength of 532 nm
# then rescale to actual wavelength
nalts = sounding.altitudes.shape[0]
if not (nalts==sounding.pressures.shape[0] and nalts==sounding.temps.shape[0]):
print "ERROR: SOMETHIGN BAD ABOUT SOUNDING AT TIME ",sounding.times
return None
rs.beta_r = hau.Z_Array(np.zeros( nalts ))
rs.beta_r[:nalts] = 3.78e-6 * sounding.pressures[:nalts] / sounding.temps[:nalts]
rs.beta_r = rs.beta_r * (532.0 / wavelength) ** 4
# spectral width of molecular scattering
m_bar = 28.97 * 1.65978e-27 # average mass of an air molecule
sigma_0 = 1 / (wavelength * 1e-9) # number in 1/meters
kb = 1.38044e-23 # Boltzmans constant J/(K deg)
c = 3e8 # speed of light in m/s
sigma = i2_scan[:, 0] * 1e9 / c # wavenumber vector
rs.Cmm = hau.Z_Array(np.zeros( nalts ))
rs.Cmc = hau.Z_Array(np.zeros( nalts ))
sample_altitudes = np.zeros(nalts)
#for bagohsrl with argon buffered i2 cell
if len(i2_scan[0,:]) > 6:
rs.Cmm_i2a = hau.Z_Array(np.zeros( nalts ))
rs.Cam_i2a = Cam_i2a
rs.Cam = Cam
print 'RBS computed with '+method_string+ ' spectrum'
spectrum_time = datetime.utcnow()
dz = sounding.altitudes[2]-sounding.altitudes[1]
delta_i = np.int(np.ceil(300.0/dz))
nk=int(nalts/delta_i)
if delta_i>1 and nk<2:# if interpolation is to happen, but not enough to interpolate, force it to the edge
nk=2
delta_i=nalts-1
sample_altitudes = np.zeros(nk)
rs.msl_altitudes = sounding.altitudes.copy()
i=0
k=0
while k < len(sample_altitudes):
if not np.isfinite(sounding.temps[i]) or not np.isfinite(sounding.pressures[i]):
i=i+delta_i
k=k+1
continue
if method_string.find('maxwellian') >= 0:
norm = m_bar * c ** 2 / (8 * sigma_0 ** 2 * kb
* sounding.temps[ i])
spectrum = np.exp(-norm * sigma ** 2)
elif method_string.find('tenti_s6') >= 0:
from tenti_s6 import tenti_s6
spectrum = tenti_s6(wavelength * 1e-9,sounding.temps[i],
sounding.pressures[ i],
i2_scan[:, 0] * 1e9)
elif method_string.find('witschas') >= 0:
spectrum = witschas_spectrum(sounding.temps[i],
sounding.pressures[ i], wavelength * 1e-9,
i2_scan[:, 0] * 1e9)
spectrum = spectrum / sum(spectrum)
sample_altitudes[k] = sounding.altitudes[i]
rs.Cmc[ k] = sum(spectrum * i2_scan[:, 1])
rs.Cmm[ k] = sum(spectrum * i2_scan[:, 2])
if i2_scan.shape[1]>6:
rs.Cmm_i2a[ k] = sum(spectrum * i2_scan[:, 6])
i = i + delta_i
k = k + 1
# if Cxx computed at less than full altitude resolution
if delta_i >1:
rs.Cmc = np.interp(sounding.altitudes,sample_altitudes[0:k-1]
,rs.Cmc[0:k-1])
rs.Cmm = np.interp(sounding.altitudes,sample_altitudes[0:k-1]
,rs.Cmm[0:k-1])
if hasattr(rs,'Cmm_i2a'):
rs.Cmm_i2a = np.interp(sounding.altitudes,sample_altitudes[0:k-1]
,rs.Cmm_i2a[0:k-1])
print method_string, 'computed for ',k-1,' altitudes in '\
, (datetime.utcnow() - spectrum_time).total_seconds(),' seconds'
plots = 0
if plots:
import matplotlib.pyplot as plt
plt.figure(600)
plt.plot(i2_scan[:, 0], spectrum[:])
fig = plt.grid(True)
plt.xlabel('Frequency (GHz)')
plt.ylabel('Intensity')
# ax=gca()
# ax.set_yscale('log')
plt.figure(601)
plt.plot(rs.Cmm,sounding.altitudes/1000.0,'b'
,rs.Cmm_i2a,sounding.altitudes/1000.0,'r')
plt.grid(True)
plt.xlabel('Cmm, Cmm_i2a')
plt.ylabel('Altitude')
plt.show()
# add sounding identification to return structure
rs.sounding_id = sounding.station_id
rs.sounding_time = sounding.times
return rs
def hsrl_inversion(r_msl, rs_Cxx, rs_constants,corr_adjusts,process_defaults):
"""hsrl_inversion(range_processed_returns,calibration_structure
,system_constants)
Invert hsrl raw count data into separate molecular and particulate profiles
and compute backcatter cross-section and depolarization from the profiles
returned structure rs always returns:
times = times of records
delta_t = time seperation between records
msl_altitudes = bin altitudes in meters
seeded shots = total number of seeded laser shots in profile
beta_r_backscat = Rayleigh scattering cross section from sounding
Na = number of aerosol photon counts
Nm = total number of molecular counts(including i2a if present)
Nm_i2 = number of molecular counts in i2 channel
Na = number of particulate counts
Ncp = Ncp photon counts
linear_depol = fractional particulate depolarization
beta_a_backscat = particulate aerosol backscatter cross-section
beta_a_backscat_par = par polarization component of backscat cross-section
beta_a_backscat_perp= perp polarization component of backscat cross-section
integrated_backscat = cumsum of backscatter cross section in altitude
if i2a channel exists the following are added to rs:
Nm_i2a = number of molecular photons derived from i2a channel
Nm = rs.Nm_2i + rs.Nm_i2a
Ncp_i2a
beta_a_backscat_par_i2a/Nm
beta_a_backscat_perp_i2a
if these exist in input file they are added to rs:
telescope_pointing
GPS_MSL_Alt
circular_depol
"""
rs = hau.Time_Z_Group(like=r_msl)
# r_msl.molecular_counts is hau.TZ_Array (2D)
nalts = r_msl.molecular_counts.shape[1]
if rs_Cxx.beta_r.shape[0] < nalts:
print 'hsrl_inversion(): size too small on calibration arrays : rs_Cxx.beta_r = %d vs nalts = %d. padding cal with last value' % \
(rs_Cxx.beta_r.shape[0], nalts)
#assert(rs_Cxx.beta_r.shape[0] != nalts)
os=rs_Cxx.beta_r.shape[0]
for k,v in vars(rs_Cxx).items():
if hasattr(v,'size') and v.size==os:
ns=list(v.shape)
ns[0]=nalts
tmp=np.zeros(ns,dtype=v.dtype)
tmp[:]=v[-1]
tmp[:os]=v
setattr(rs_Cxx,k,tmp)
elif rs_Cxx.beta_r.shape[0] > nalts:
print 'WARNING hsrl_inversion(): size larger on calibration arrays. may be an error : rs_Cxx.beta_r = %d vs nalts = %d' % \
(rs_Cxx.beta_r.shape[0], nalts)
rs.times = r_msl.times.copy()
rs.delta_t = r_msl.delta_t.copy()
rs.msl_altitudes = r_msl.msl_altitudes.copy()
rs.seeded_shots=r_msl.seeded_shots.copy()
if hasattr(r_msl,'telescope_pointing'):
rs.telescope_pointing=r_msl.telescope_pointing.copy()
# Rayleigh backscatter cross section profile
rs.beta_r_backscat = hau.Z_Array(np.zeros(nalts))
rs.beta_r_backscat[:nalts] = rs_Cxx.beta_r[:nalts] * 3.0 / (8.0 * np.pi)
#for normal i2 channel
kk = 1.0 / (rs_Cxx.Cmm[:nalts] - rs_Cxx.Cmc[:nalts] * rs_Cxx.Cam)
rs.Na = hau.TZ_Array( kk[:nalts] * (rs_Cxx.Cmm[:nalts] * r_msl.combined_counts\
- rs_Cxx.Cmc[ : nalts] * r_msl.molecular_counts) )
rs.Nm = hau.TZ_Array( kk[:nalts] * (r_msl.molecular_counts \
- rs_Cxx.Cam * r_msl.combined_counts) )
rs.Nm_i2 = rs.Nm.copy()
#if data includes an i2a channel we generation a seperate inversion--bagohsrl only
#systems with i2a channel record linear depolarization
if hasattr(r_msl,'molecular_i2a_counts') and hasattr(rs_Cxx,'Cmm_i2a'):
kk = 1.0 / (rs_Cxx.Cmm_i2a[:nalts] - rs_Cxx.Cmc[:nalts] * rs_Cxx.Cam_i2a)
rs.Na_i2a = hau.TZ_Array( kk[:nalts] * (rs_Cxx.Cmm_i2a[:nalts]
* r_msl.combined_counts - rs_Cxx.Cmc[ : nalts] * r_msl.molecular_i2a_counts) )
rs.Nm_i2a = hau.TZ_Array( kk[:nalts] * (r_msl.molecular_i2a_counts \
- rs_Cxx.Cam_i2a * r_msl.combined_counts) )
rs.Nm = (rs.Nm_i2 + rs.Nm_i2a)/2.0
#bagohsrl is only hsrl with i2a channel--it measures linear depolarization
rs.Ncp = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']
* r_msl.combined_counts) - rs.Nm_i2 * 0.0035 / (1.0 - 0.0035)
rs.Ncp_i2a = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']
* r_msl.combined_counts) - rs.Nm_i2a * 0.0035 / (1.0 - 0.0035)
if not rs_constants.has_key('no_depol_channel') or rs_constants['no_depol_channel']==0 :
rs.linear_depol_i2a = rs.Ncp_i2a / rs.Na_i2a
#compute the linear depolarization as the average of normal and i2a values
#note these two componets should be almost identical
rs.linear_depol = (rs.Ncp + rs.Ncp_i2a)/(rs.Na_i2a + rs.Na)
#when backscatter is small linear_depol can become indeterminate--bound values
rs.linear_depol[rs.linear_depol < 0.0] = 0.0
rs.linear_depol[rs.linear_depol > 0.6] = 0.6
else:
rs.linear_depol = np.zeros_like(rs.Nm)
rs.linear_depol_i2a = np.zeros_like(rs.Nm)
rs.beta_a_backscat_perp_i2a=rs.Na_i2a/rs.Nm_i2a \
*rs.linear_depol_i2a*rs.beta_r_backscat
rs.beta_a_backscat_perp = rs.Na/rs.Nm_i2 \
*rs.linear_depol * rs.beta_r_backscat
rs.beta_a_backscat_par_i2a = rs.Na_i2a / rs.Nm_i2a * rs.beta_r_backscat
rs.beta_a_backscat_par = rs.Na / rs.Nm_i2 * rs.beta_r_backscat
rs.beta_a_backscat = 0.5 *(rs.beta_a_backscat_par +rs.beta_a_backscat_perp \
+ rs.beta_a_backscat_par_i2a + rs.beta_a_backscat_perp_i2a)
if rs_constants.has_key('no_i2_channel') and rs_constants['no_i2_channel']==1:
print
print 'WARNING--I2 channel is not being used in calculations'
print "calvals has 'no_i2_channel'== 1"
print
rs.linear_depol = rs.Ncp_i2a / rs.Na_i2a
rs.beta_a_backscat = rs.beta_a_backscat_par_i2a + rs.beta_a_backscat_perp_i2a
if not process_defaults.enabled('depolarization_is_aerosol_only'):
#user wants bulk depolarization aerosol combined with molecular
#recompute depolarization
print 'computing bulk depolarization--aerosol and molecular combined'
rs.linear_depol = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts\
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']\
* r_msl.combined_counts) / r_msl.combined_counts
#compute circular polarization from linear--good only for vertical pointing systems.
rs.circular_depol = 2 * rs.linear_depol / (1 - rs.linear_depol)
elif hasattr(rs_Cxx,'Cmm_i2a') or rs_constants.has_key('polarization_is_linear') \
and rs_constants['polarization_is_linear']==1:
if hasattr(rs_Cxx,'Cmm_i2a'):
print
print 'hsrl_inversion(): WARNING i2a counts found, but no calibration'
print 'computing without i2a channel'
print
rs.Ncp = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']
* r_msl.combined_counts) - rs.Nm_i2 * 0.0035 / (1.0 - 0.0035)
rs.linear_depol = rs.Ncp/rs.Na
#when backscatter is small linear_depol can become indeterminate--bound values
rs.linear_depol[rs.linear_depol < 0.0] = 0.0
rs.linear_depol[rs.linear_depol > 0.6] = 0.6
if not process_defaults.enabled('depolarization_is_aerosol_only'):
#user wants bulk depolarization aerosol combined with molecular
#recompute depolarization
rs.linear_depol = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts\
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']\
* r_msl.combined_counts) / r_msl.combined_counts
#compute circular from linear--good only for vertical pointing system
rs.circular_depol = 2*rs.linear_depol /(1 - rs.linear_depol)
rs.beta_a_backscat_perp = rs.Na/rs.Nm_i2 \
*rs.linear_depol * rs.beta_r_backscat
rs.beta_a_backscat_par = rs.Na / rs.Nm * rs.beta_r_backscat
rs.beta_a_backscat = rs.beta_a_backscat_par +rs.beta_a_backscat_perp
else: #instrument with no i2a channel and measures circular polarization
rs.Ncp = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts\
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']\
* r_msl.combined_counts) - rs.Nm_i2 * 0.0074 / (1.0 - 0.0074)
rs.circular_depol = rs.Ncp / rs.Na
#when Na becomes small, circular_depol may become indeterminate
# (and don't fault on Nans)
#rs.circular_depol = np.nan_to_num(rs.circular_depol)
rs.circular_depol[rs.circular_depol < 0.0] = 0.0
rs.circular_depol[rs.circular_depol > 3.0] = 3.0
rs.beta_a_backscat_perp=rs.Na/rs.Nm_i2*rs.circular_depol*rs.beta_r_backscat
rs.beta_a_backscat_par = rs.Na / rs.Nm_i2 * rs.beta_r_backscat
rs.beta_a_backscat = rs.beta_a_backscat_par + rs.beta_a_backscat_perp
if not process_defaults.enabled('depolarization_is_aerosol_only'):
#user wants bulk depolarization containing both aerosol and molecular
print 'computing bulk depolarization--air and aerosol together'
rs.circular_depol = rs_constants['combined_to_cross_pol_gain_ratio'] \
* (r_msl.cross_pol_counts\
- rs_constants['polarization_cross_talk']*corr_adjusts['pol_x_talk']\
* r_msl.combined_counts) / r_msl.combined_counts
rs.linear_depol = rs.circular_depol / (2
+ rs.circular_depol)
#compute integrated backscatter cross section
rs.integrated_backscat = rs.beta_a_backscat.copy()
rs.integrated_backscat[np.isnan(rs.integrated_backscat)] = 0.0
da=rs.msl_altitudes.copy()
da[:-1]=da[1:]-da[:-1]
da[-1]=da[-2]
tda=hau.TZ_Array(np.transpose(da[:,np.newaxis]*np.ones((1,rs.times.size))))
rs.integrated_backscat = np.cumsum(rs.integrated_backscat,1) \
*tda
if hasattr(r_msl,'GPS_MSL_Alt'):
rs.GPS_MSL_Alt = r_msl.GPS_MSL_Alt
if hasattr(r_msl,'combined_1064_counts'):
rs.combined_1064_counts = r_msl.combined_1064_counts\
* rs_constants['IR_combined_hi_gain_ratio']
if hasattr(r_msl,'wfov_extinction_corr'):
#transf wfov_extinction correction to output structure
rs.wfov_extinction_corr = r_msl.wfov_extinction_corr
return rs
def process_spheroid_particle(rs_inv,
rs_radar,
particle_parameters,
lambda_radar,
entire_frame,
sounding=None,
size_dist=None):
"""
process_spheroid_particle(rs_inv,rs_radar,particle_parameters,lambda_radar,entire_frame,
sounding=None,p180_water=None,size_dist=None):
generate and return the particle measurements based on a given hsrl inverted data,
radar (and its lambda), and particle parameters dictionary
"""
#create timez group and add heights
rs_particle = hau.Time_Z_Group(rs_inv.times.copy(),
timevarname='times',
altname='heights')
setattr(rs_particle, 'heights', rs_inv.msl_altitudes.copy())
setattr(rs_particle, 'delta_t', rs_inv.delta_t.copy())
#remove points where lidar signal is noise dominated by setting to
#very small value.
#clipped_beta_a_back=rs_inv.beta_a_backscat.copy()
#if 0: #if hasattr(rs_inv,'std_beta_a_backscat'):
# clipped_beta_a_back[clipped_beta_a_back<(2*rs_inv.std_beta_a_backscat)]=-np.inf
#else:
# print 'No std_beta_a_backscat statistics to filter particle measurements'
#clipped_beta_a_back[np.logical_not(np.isfinite(rs_inv.beta_a_backscat))]=-np.inf;
rs_particle.q_backscatter = np.NaN * np.zeros_like(rs_inv.beta_a_backscat)
rs_particle.phase = np.zeros_like(rs_inv.beta_a_backscat)
rs_particle.phase[
rs_inv.linear_depol > particle_parameters['h2o_depol_threshold']] = 1
rs_particle.phase[np.isnan(rs_inv.beta_a_backscat)] = np.NaN
#set aspect ratio parameter for ice filled bins
rs_particle.zeta = np.ones(rs_inv.beta_a_backscat.shape)
rs_particle.zeta[rs_inv.linear_depol > particle_parameters['h2o_depol_threshold']] \
= particle_parameters['zeta']
print 'Extinction cross section for particle size calculations derived from ' \
,particle_parameters['ext_source']
print 'Extinction due nonprecipitating aerosols = '\
,particle_parameters['background_aerosol_bs'],'1/(m sr)'
#store the mask field with the particle info
rs_particle.qc_mask = rs_inv.qc_mask.copy()
clipped_beta_a_backscat = rs_inv.beta_a_backscat.copy()
copy_radar_backscatter = rs_radar.Backscatter.copy()
#clipped_beta_a_backscat = copy_beta_a.copy()
clipped_beta_a_backscat = clipped_beta_a_backscat \
- particle_parameters['background_aerosol_bs']
clipped_beta_a_backscat[clipped_beta_a_backscat < 0] = np.NaN
#create an empty mode_diameter array
rs_particle.mode_diameter = np.zeros_like(rs_inv.beta_a_backscat)
#create an empty array for extinction--used only for particle calculations
#bs_ratio_to_dmode will return extinction cross section in clipped_beta_a
clipped_beta_a = np.NaN * np.zeros_like(rs_inv.beta_a_backscat)
#water
#compute mode diameter, extinction cross section, and backscatter efficeincy
#from radar and lidar backscatter cross sections using mie theory and assumed
#size distribution to predict mode diameter and q_backscatter for points
#identified as water.
if particle_parameters['radar_model'] == "Mie":
rs_particle.mode_diameter, clipped_beta_a, rs_particle.q_backscatter\
,rs_particle.dstar \
= size_dist.dmode_from_radar_lidar_mie(copy_radar_backscatter\
,clipped_beta_a_backscat)
else:
#use only Rayliegh approx solution--particle_parameter['radar_model']=="Rayleigh"
#mode diameter is computed for all points assuming everything is water
#subsequent calculation will replace ice phase points.
if particle_parameters['ext_source'] == 'ext':
clipped_beta_a = rs_inv.extinction_aerosol.copy()
elif particle_parameters['ext_source'] == 'bs/p180':
clipped_beta_a = clipped_beta_a_backscat / particle_parameters[
'p180_water']
else:
print 'particle_parameters=', particle_parameters[
'ext_source'], ' not supported'
print 'in spheroid_particle_processing'
print j
clipped_beta_a[np.isnan(clipped_beta_a_backscat)] = np.NaN
phase = np.zeros_like(rs_inv.beta_a_backscat)
zeta = np.ones_like(rs_inv.beta_a_backscat)
rs_particle.mode_diameter = size_dist.dmode_from_lidar_radar_rayleigh(
rs_particle.mode_diameter, clipped_beta_a, copy_radar_backscatter,
zeta, phase)
#ice
#compute extinction cross section for ice points using backscatter phase function
clipped_beta_a[rs_particle.phase==1] = \
clipped_beta_a_backscat[rs_particle.phase==1]/particle_parameters['p180_ice']
zeta = np.zeros_like(clipped_beta_a)
zeta[rs_particle.phase == 1] = particle_parameters['zeta']
#derive mode_diameter directly from radar backscatter and lidar extinction
#cross sections for parts of image populated by ice
rs_particle.mode_diameter[rs_particle.phase==1] = size_dist.dmode_from_lidar_radar_rayleigh(\
rs_particle.mode_diameter[rs_particle.phase==1] \
,clipped_beta_a[rs_particle.phase==1],copy_radar_backscatter[rs_particle.phase==1]\
,zeta[rs_particle.phase==1],rs_particle.phase[rs_particle.phase==1])
#creates effective_diameter_prime array from mode diameter
rs_particle.effective_diameter_prime = \
size_dist.deff_prime(rs_particle.mode_diameter,rs_particle.phase,zeta)
rs_particle.effective_diameter = size_dist.eff_diameter(\
rs_particle.mode_diameter,rs_particle.phase)
rs_particle.mean_diameter = size_dist.mean_diameter(\
rs_particle.mode_diameter,rs_particle.phase)
#compute liquid water content for bins with phase == 0
#bins with phase > 0 will return with NaN's
if particle_parameters['radar_model'] == "Mie":
rs_particle.LWC = su.liquid_water_content_mie(
rs_particle.effective_diameter, clipped_beta_a,
rs_particle.q_backscatter)
rs_particle.p180_extinction = rs_inv.beta_a_backscat / rs_particle.q_backscatter
else:
if particle_parameters['ext_source'] == 'bs/p180':
rs_particle.extinction_aerosol = rs_inv.beta_a_backscat / particle_parameters[
'p180_water']
clipped_beta_a = rs_particle.extinction_aerosol.copy()
else:
clipped_beta_a = rs_inv.extinction_aerosol.copy()
clipped_beta_a[np.isnan(clipped_beta_a_backscat)] = np.NaN
rs_particle.LWC = np.NaN * np.zeros_like(
rs_particle.effective_diameter)
su.liquid_water_content_ext_approx(rs_particle.LWC,
rs_particle.effective_diameter,
clipped_beta_a, rs_particle.phase)
rs_particle.p180_extinction = rs_inv.beta_a_backscat / particle_parameters[
'p180_water']
rs_particle.extinction_aerosol = rs_inv.extinction_aerosol.copy()
#compute ice water water content for bins with phase > 0 (kg/m^3)
#return in LWC array bins with phase > 0
su.ice_water_content(rs_particle.LWC, rs_particle.effective_diameter,
clipped_beta_a, rs_particle.phase)
if hasattr(rs_radar, 'vertically_averaged_doppler'):
rs_radar.raw_MeanDopplerVelocity = rs_radar.MeanDopplerVelocity.copy()
motion_correction = np.transpose(rs_radar.vertically_averaged_doppler\
*np.transpose(np.ones_like(rs_radar.MeanDopplerVelocity)))
rs_radar.MeanDopplerVelocity -= motion_correction
if sounding != None:
s_time = datetime.utcnow()
rs_particle.rw_fall_velocity,rs_particle.mw_fall_velocity \
,rs_particle.model_spectral_width,rs_particle.nw_fall_velocity\
= su.weighted_fall_velocity(
rs_particle.mode_diameter
,particle_parameters
,rs_particle.zeta
,sounding.temps
,sounding.pressures
,rs_particle.phase,size_dist)
print 'time for fall_velocity = ', datetime.utcnow() - s_time
# compute precip rate (m/s) #rain_rate = 1/density
# (m^3/kg) * LWC (kg/m^3) * fall_velocity (m/s) #using
# Doppler velocity rs_particle.hsrl_radar_dv_precip_rate =
# 0.001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity
#using raw Doppler to give precip rate in m/s
rs_particle.hsrl_radar_dv_precip_rate = 0.001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity
#remove points with Doppler folding
rs_particle.hsrl_radar_dv_precip_rate[
rs_radar.MeanDopplerVelocity < -2.0] = np.NaN
if sounding != None:
#using modeled mass weighted velocity and dividing by the density of water,
# 1000 kg/m^3, to give precip_rate in m/s
rs_particle.hsrl_radar_precip_rate = 0.001 * rs_particle.LWC * rs_particle.mw_fall_velocity
#retype all these fields to a proper TZ_Array
#for f in ['effective_diameter_prime']:
for f in vars(rs_particle).keys():
v = getattr(rs_particle, f)
if isinstance(v, hau.Z_Array):
continue #properly typed. continue
elif isinstance(v, np.ndarray):
if len(v.shape) == 2:
setattr(rs_particle, f, hau.TZ_Array(v))
elif len(v.shape) == 1:
print '1 Dimensional Variable ' + f + ' will be changed to a T_Array. FIXME!!!!'
setattr(rs_particle, f, hau.T_Array(v))
else:
raise RuntimeError(
"I don't know what to type particle array " + f +
" with dimensions " + repr(v.shape))
else:
pass #not an array type. should be safe to ignore
"""
#compute fitting error of computed radar weighted fall velocity
#to measured Doppler veleocity.
temp = rs_radar.Backscatter.copy()
temp[np.isnan(rs_radar.Backscatter)]=0.0
temp[rs_inv.msl_altitudes>400]=0.0
fitting_error = np.sqrt(nanmean((rs_particle.rw_fall_velocity[temp >1e-9] \
- rs_radar.MeanDopplerVelocity[temp >1e-9])**2))
print
print rs_radar.times[0],' ---> ' ,rs_radar.times[-1]
print 'fitting_error (m/s)= ',fitting_error
print
"""
'rs_particle--spher'
print dir(rs_particle)
return rs_particle
def accumulate_raman_inverted_profiles(consts,
rs_mean,
qc_mask,
old_profiles,
process_control=None,
corr_adjusts=None):
indices = np.arange(rs_mean.times.shape[0])
if len(indices) == 0:
return old_profiles
if qc_mask is not None and processing_defaults.get_value(
'averaged_profiles', 'apply_mask'):
#make mask array with NaN's for array elements where bit[0] of qc_mask==0
#all other elements of mask = 1
mask = (np.bitwise_and(qc_mask, 1)).astype(
'float') #mask is float to allow use of NaN values
mask[mask == 0] = np.NaN
print 'qc_mask applied to time averaged profiles vs altitude'
else:
#set mask == 1
mask = None
#for sh in channel_shorthand.keys():
#
# if hasattr(rs_mean,sh):
# mask = np.ones_like(getattr(rs_mean,sh))
# break
#if mask is None:
# mask = np.ones((rs_mean.times.shape[0],0))
print 'qc_mask has not been applied to time averaged profiles'
#energies=('transmitted_1064_energy','transmitted_energy')
profiles = hau.Time_Z_Group(can_append=False, altname='altitudes')
profiles.hist = hau.Time_Z_Group()
ft = None
tc = 0
#tv=0
lt = None
if old_profiles is not None:
ft = old_profiles.hist.ft
tc = old_profiles.hist.tc
#tv=old_profiles.hist.tv
lt = old_profiles.hist.lt
if len(indices) > 0:
if ft is None:
ft = rs_mean.times[indices][0]
lt = rs_mean.times[indices][-1] + timedelta(
seconds=rs_mean.delta_t[indices][-1]
if not np.isnan(rs_mean.delta_t[indices][-1]) else 0)
for x in indices:
if rs_mean.times[x] is not None:
tc = tc + 1
#tv=tv+(rs_mean.times[x]-ft).total_seconds()
if tc > 0:
profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ])
profiles.start = ft
profiles.width = lt - ft
profiles.delta_t = hau.T_Array([profiles.width.total_seconds()])
else:
profiles.times = hau.T_Array([])
profiles.start = ft
profiles.width = timedelta(seconds=0)
profiles.delta_t = hau.T_Array([])
profiles.start_time = ft
profiles.end_time = lt
profiles.hist.ft = ft
profiles.hist.tc = tc
#profiles.hist.tv=tv
profiles.hist.lt = lt
if rs_mean is not None and hasattr(rs_mean, 'altitudes'):
profiles.altitudes = rs_mean.altitudes.copy()
elif hasattr(old_profiles, 'altitudes'):
profiles.altitudes = old_profiles.altitudes
#FIXME need to accumulate the inverted products here, so here's a hack
interval = hau.Time_Z_Group()
interval.intervals = hau.T_Array(np.ones(rs_mean.times.shape))
accumulate(profiles, old_profiles, interval, indices, 'intervals')
interval_count = profiles.intervals
print 'Total intervals for profile =', interval_count
for k, v in vars(rs_mean).items():
if k.startswith('_') or k in ('times', 'start', 'width', 'delta_t'):
continue
if not isinstance(v, hau.T_Array):
continue
if isinstance(v, hau.TZ_Array):
continue
accumulate(profiles, old_profiles, rs_mean, indices, k, interval_count)
# create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke
for k, v in vars(rs_mean).items():
if k.startswith('_'):
continue
if not isinstance(v, hau.TZ_Array):
continue
if len(v.shape) != 2:
continue
accumulate(profiles, old_profiles, rs_mean, indices, k, interval_count,
mask)
return profiles
def read(self, *args, **kwargs):
if self.val!=None:
yield self.val
return
if len(args):
print 'Unused args = ',args
if len(kwargs):
print "Unused kwargs = ",kwargs
ret=hau.Time_Z_Group()
delattr(ret,'times')
data=self.data
if self.group is not None:
for g in self.group.split('/'):
data=data.groups[g]
if 'base_time' in data.variables:
epochtime=datetime.datetime(1970,1,1,0,0,0)
basetime=epochtime+datetime.timedelta(seconds=long(data.variables['base_time'].getValue()))
print 'basetime is ',basetime
if 'time_coverage_start' in data.variables and 'time_coverage_end' in data.variables:
basetime=datetime.datetime.strptime(''.join(data.variables['time_coverage_start'][:20]), '%Y-%m-%dT%H:%M:%SZ')
for v in self.xlateTable:
vari=data.variables[v]
parts=self.xlateTable[v].split('.')
finalpart=parts[-1]
parts=parts[0:(len(parts)-1)]
base=ret
for i in parts:
if not hasattr(base,i):
setattr(base,i,hau.Time_Z_Group())
delattr(getattr(base,i),'times')
base=getattr(base,i)
idx=[slice(None) for x in range(len(vari.shape))]
if len(idx)==0:
newval=vari.getValue()
elif len(idx)==1:
newval=numpy.array(vari[idx[0]])
else:
newval=numpy.array(vari[tuple(idx)])
if 'dpl_py_type' in vari.ncattrs():
dpltype=vari.dpl_py_type[:]
if dpltype=='matplotlib_num2date':
newval=numpy.array([(date2num(basetime+datetime.timedelta(seconds=float(d))) if d<1e35 else float('nan')) for d in newval])
if dpltype=='python_datetime':
newval=numpy.array([(basetime+datetime.timedelta(seconds=float(d)) if d<1e35 else None) for d in newval])
setattr(base,finalpart,newval)
addpartsbylength=dict()
fillins=(('times',),
('delta_t',),
('msl_altitudes','heights'))
for ka,f in vars(ret).items():
for ks in fillins:
for k in ks:
if hasattr(f,k):
t=getattr(f,k)
if t.size==0:
continue
for nk in ks:
if nk not in addpartsbylength:
addpartsbylength[nk]=dict()
if t.size not in addpartsbylength[nk]:
addpartsbylength[nk][t.size]=t
#print 'available parts:',addpartsbylength
for k,f in vars(ret).items():
dim_preferredsizes=[[],[]]
try:
for ak,av in vars(f).items():
if hasattr(av,'shape') and len(av.shape)>=2:
for x in range(2):
dim_preferredsizes[x].append(av.shape[x])
except TypeError:
#print 'no parts for ',k
continue
#print 'sizes for',k,dim_preferredsizes
if len(dim_preferredsizes[0])==0 or len(dim_preferredsizes[1])==0:
continue
dim_preferred=[]
for x in range(2):
dim_preferred.append(int(numpy.median(dim_preferredsizes[x])))
#print 'sizes are',dim_preferred,'median of ',dim_preferredsizes
for nk,kv in addpartsbylength.items(): #TOTAL HACK
x=None if not hasattr(f,nk) else getattr(f,nk)
if x is None or not hasattr(x,'shape') or x.size==0:
try:
if nk in ('times','delta_t'):
dimpreferredidx=0
else:
dimpreferredidx=1
count=dim_preferred[dimpreferredidx]
if count in kv:
setattr(f,nk,kv[count])
#else:
# print nk,'doesnt exist for length',count
except AttributeError:
pass
self.val=ret
yield ret #this means this operates as an iterator that runs once
def read_raqms_file(instrument, start_time):
"""read raqms file between start and end time
instrument - e.g. 'gvhsrl'
start_time - datetime object
"""
raqms = hau.Time_Z_Group(altname='model_level_alts')
filename = find_raqms_filename(instrument, start_time)
if not filename:
return None
nc = Dataset(filename, 'r')
times = getAll(nc, 'time')
aircraft_alts = getAll(nc, 'alt')
pressures = getAll(nc, 'pressure')
temperatures = getAll(nc, 'temperature')
model_level_alts = getAll(nc, 'altitude')
relative_humidity = getAll(nc, 'rh')
latitude = getAll(nc, 'lat')
longitude = getAll(nc, 'lon')
u_vel = getAll(nc, 'uvel')
v_vel = getAll(nc, 'vvel')
ext_total = getAll(nc, 'ext_tot')
ext_dust = getAll(nc, 'ext_dust')
ext_salt = getAll(nc, 'ext_salt')
base_time = datetime(start_time.year, start_time.month, start_time.day, 0,
0, 0)
#np.fix(start_time)
#time=times.astype('float64')
#convert raqms seconds from start of day to python datetimes
#times=base_time + time/(3600.0*24.0)
times = hau.T_Array(
[base_time + timedelta(seconds=float(x)) for x in times])
assert (times.size > 0)
selectedMask = (times > start_time)
for i, x in enumerate(selectedMask):
fi = i
if x:
if i > 0:
selectedMask[i - 1] = True
break
selectedMask[-1] = True
selectedTimes = np.arange(times.size)[selectedMask]
raqms.latitude = hau.T_Array(latitude[selectedTimes])
raqms.longitude = hau.T_Array(longitude[selectedTimes])
raqms.pressures = hau.TZ_Array(pressures[selectedTimes, :])
raqms.temperatures = hau.TZ_Array(temperatures[selectedTimes, :])
raqms.ext_total = hau.TZ_Array(ext_total[selectedTimes, :])
raqms.ext_dust = hau.TZ_Array(ext_dust[selectedTimes, :])
raqms.ext_salt = hau.TZ_Array(ext_salt[selectedTimes, :])
raqms.relative_humidity = hau.TZ_Array(relative_humidity[selectedTimes, :])
raqms.u_vel = hau.TZ_Array(u_vel[selectedTimes, :])
raqms.v_vel = hau.TZ_Array(v_vel[selectedTimes, :])
raqms.model_level_alts = hau.TZ_Array(model_level_alts[selectedTimes, :] *
1000.0)
raqms.times = times[selectedTimes]
return raqms
def get_data_at_index(struct,t_index,z_index):
"""get_data_at_point(struct,t_index,z_index):
extract values of variables at a given array index"""
rs_init = struct['rs_init'].__dict__
rs_inv = struct['rs_inv'].__dict__
rs_mmcr = struct['rs_mmcr'].__dict__
rs_spheroid_particle = struct['rs_spheroid_particle'].__dict__
sounding = rs_init['sounding']
ptv = hau.Time_Z_Group()
particle_parameters = rs_spheroid_particle['particle_parameters']
ptv.beta_ext_radar = np.NaN * np.zeros((1,1))
ptv.beta_ext_lidar = np.NaN * np.zeros((1,1))
ptv.beta_a_backscat = np.NaN * np.zeros((1,1))
ptv.radar_spectral_width = np.NaN * np.zeros((1,1))
ptv.MeanDopplerVelocity = np.NaN * np.zeros((1,1))
ptv.zeta = np.NaN * np.zeros((1,1))
ptv.phase = np.NaN * np.zeros((1,1))
ptv.temps = np.NaN * np.zeros(1)
ptv.pressures = np.NaN * np.zeros(1)
ptv.eff_diameter_prime\
= np.NaN * np.zeros((1,1))
ptv.dmode = np.NaN * np.zeros((1,1))
ptv.effective_diameter = np.NaN * np.zeros((1,1))
ptv.mean_diameter = np.NaN * np.zeros((1,1))
ptv.mean_mass_diameter = np.NaN * np.zeros((1,1))
ptv.LWC_ext = np.NaN * np.zeros((1,1))
ptv.LWC_p180 = np.NaN * np.zeros((1,1))
ptv.hsrl_radar_precip_rate=np.NaN * np.zeros((1,1))
ptv.rw_fall_velocity = np.NaN * np.zeros((1,1))
ptv.mw_fall_velocity = np.NaN * np.zeros((1,1))
ptv.model_spectral_width\
= np.NaN * np.zeros((1,1))
ptv.beta_ext_lidar[0,0] = rs_inv['beta_a'][t_index,z_index]
ptv.beta_a_backscat[0,0] = rs_inv['beta_a_backscat'][t_index,z_index]
ptv.beta_ext_radar[0,0] \
= rs_mmcr['Backscatter'][t_index,z_index] * 8 * np.pi/3.0
ptv.radar_spectral_width[0,0] = rs_mmcr['SpectralWidth'][t_index,z_index]
ptv.MeanDopplerVelocity[0,0] \
= rs_mmcr['MeanDopplerVelocity'][t_index,z_index]
ptv.zeta[0,0] = rs_spheroid_particle['zeta'][t_index,z_index]
ptv.phase[0,0] = rs_spheroid_particle['phase'][t_index,z_index]
ptv.eff_diameter_prime[0,0] \
= rs_spheroid_particle['effective_diameter_prime'][t_index,z_index]
ptv.temps[0] = sounding.temps[z_index]
ptv.pressures[0] = sounding.pressures[z_index]
ptv.dmode[0,0] = rs_spheroid_particle['mode_diameter'][t_index,z_index]
ptv.effective_diameter[0,0] = rs_spheroid_particle['effective_diameter'][t_index,z_index]
ptv.mean_diameter[0,0] = rs_spheroid_particle['mean_diameter'][t_index,z_index]
ptv.mean_mass_diameter[0,0] =rs_spheroid_particle['mean_mass_diameter'][t_index,z_index]
ptv.LWC_ext[0,0] = rs_spheroid_particle['LWC'][t_index,z_index]
ptv.hsrl_radar_precip_rate[0,0]=rs_spheroid_particle['hsrl_radar_precip_rate'][t_index,z_index]
ptv.rw_fall_velocity[0,0] = rs_spheroid_particle['rw_fall_velocity'][t_index,z_index]
ptv.mw_fall_velocity[0,0] = rs_spheroid_particle['mw_fall_velocity'][t_index,z_index]
ptv.model_spectral_width[0,0]= rs_spheroid_particle['model_spectral_width'][t_index,z_index]
return ptv,particle_parameters
def process_mass_dimension_particle(rs_inv, rs_radar, particle_parameters,
lambda_radar, entire_frame):
"""
generate and return the particle measurements based on a given hsrl inverted data, radar (and its lambda), and particle parameters dictionary
"""
ParticleParameters = namedtuple('ParticleParameters',
','.join(particle_parameters.keys()))
pparm = ParticleParameters(
**particle_parameters
) #pparm is a structure of the particle parameters, instead of a dictionary 'particle_parameters'
#create timez group and add heights
rs_particle = hau.Time_Z_Group(rs_inv.times.copy(),
timevarname='times',
altname='heights')
setattr(rs_particle, 'heights', rs_inv.msl_altitudes.copy())
#remove points where lidar signal is noise dominated by setting to
#very small value.
clipped_beta_a_back = rs_inv.beta_a_backscat.copy()
if hasattr(rs_inv, 'std_beta_a_backscat'):
clipped_beta_a_back[clipped_beta_a_back < (
2 * rs_inv.std_beta_a_backscat)] = -np.inf
else:
print 'No std_beta_a_backscat statistics to filter particle measurements'
clipped_beta_a_back[np.logical_not(np.isfinite(
rs_inv.beta_a_backscat))] = -np.inf
used_beta_a = None
if hasattr(rs_inv, 'beta_a'):
used_beta_a = rs_inv.beta_a.copy()
#mask beta_a?
rs_particle.effective_diameter_prime,used_beta_a = \
lred.lidar_radar_eff_diameter_prime(
beta_a_backscat=clipped_beta_a_back
,radar_backscat=rs_radar.Backscatter
,depol=rs_inv.linear_depol
,h2o_depol_threshold=particle_parameters['h2o_depol_threshold']
,beta_a=used_beta_a
,p180_ice=particle_parameters['p180_ice']
,lambda_radar=lambda_radar)
rs_particle.effective_diameter,rs_particle.num_particles,rs_particle.LWC,rs_particle.mean_diameter=\
lred.d_eff_from_d_eff_prime(
rs_particle.effective_diameter_prime
,clipped_beta_a_back
,rs_inv.linear_depol
,used_beta_a
,pparm)
#rs_particle.hsrl_radar_rain_rate = 3600 * 10* .0001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity
#convert to mks units LWC kg/m^3, Doppler m/s, rain_rate m/s
rs_particle.hsrl_radar_rain_rate = 0.001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity
#retype all these fields to a proper TZ_Array
for f in [
'hsrl_radar_rain_rate', 'effective_diameter_prime',
'effective_diameter', 'num_particles', 'LWC', 'mean_diameter'
]:
if hasattr(rs_particle, f):
setattr(rs_particle, f, hau.TZ_Array(getattr(rs_particle, f)))
return rs_particle
def read_sounding_file(instrument, sounding_type, id, start_time,
requested_altitudes):
""" read_sounding_file([instrument,sounding_type,id,start_time,alt_res,max_alt)
returns arrays rs.temps(sounding#, rs.dew_points(sounding#,alt_index),
rs.wdir(sounding#,alt_index), rs.wspd(sounding#,alt_index) along with several
scalers with sounding info instrument (e.g. 'ahsrl','gvhsrl','mf2hsrl','nshsrl')
sounding_type may be radiosonde station id, model identifier, or other instrument
sounding_type (e.g. 'NOAA','ARM',.......
sounding id (for sounding_type=NOAA, this a 3-letter e.g. 'MSN')
start_time first time for which the sounding is needed, provided as matplot time
requested_altitudes is a vector of altitudes at which sounding values are requested (m)
returns temp_sounding(object) with all soundings from the current file after the starting time
returns the last time at which this sounding can be used as rs.expire_time"""
import lg_dpl_toolbox.core.archival as hru
if sounding_type[:].find('NOAA raob') >= 0:
rs = hau.Time_Z_Group()
time_struct = start_time
dir_path = hru.get_path_to_data(instrument, start_time)
filename = dir_path + '/' + '%4i' % time_struct.year + '/' \
+ '%02i' % time_struct.month + '/sondes.' + id[:] + '.nc'
print 'sounding file--', filename
if not os.path.exists(filename):
return None
#raise RuntimeError, 'sounding file %s does not exist' \
# % filename
nc = Dataset(filename, 'r')
times = getAll(nc, 'synTime')
# combine mandatory and sig height measurements
heights = np.hstack((getAll(nc, 'htMan'), getAll(nc, 'htSigT')))
epoch = datetime(1970, 1, 1, 0, 0, 0)
t_mask = times < 1e36
for i in range(len(times)):
t_mask[i] = times[i] < 1e36 and any(heights[i, :] < 1e36)
times = times[t_mask]
times = [epoch + timedelta(seconds=soff) for soff in times[:]]
# select times, one prior to start time --> last profile in file
#indices = np.arange(len(times))
#start_index = max(indices[times <= start_time])
#rs.times = zeros(len(times) - start_index)
rs.times = np.zeros(len(times))
#rs.times = times[start_index:]
# rs.times = hau.T_Array( rs.times )
rs.times = times[:]
wmosta = getAll(nc, 'wmoStat') # wmo station number
stalong = getAll(nc, 'staLon')
stalat = getAll(nc, 'staLat')
# combine mandatory and sig height measurements
temps = np.hstack((getAll(nc, 'tpMan'), getAll(nc, 'tpSigT')))
pressures = np.hstack((getAll(nc, 'prMan'), getAll(nc, 'prSigT')))
dew_points = np.hstack((getAll(nc, 'tdMan'), getAll(nc, 'tdSigT')))
wind_dir = np.hstack((getAll(nc, 'wdMan'), getAll(nc, 'wdSigT')))
wind_spd = np.hstack((getAll(nc, 'wsMan'), getAll(nc, 'wsSigT')))
heights = heights[t_mask, :]
temps = temps[t_mask, :]
pressures = pressures[t_mask, :]
dew_points = dew_points[t_mask, :]
wind_dir = wind_dir[t_mask, :]
wind_spd = wind_spd[t_mask, :]
[n_soundings, n_heights] = temps.shape
# defined standard atmosphere climatology for use above highest reported level
# climate=temp_sounding()
climate = hau.Time_Z_Group()
climate.altitudes = np.zeros((n_soundings, 9))
climate.temps = np.zeros((n_soundings, 9))
climate.pressures = np.zeros((n_soundings, 9))
climate.dew_pt = np.zeros((n_soundings, 9))
climate.wind_spd = np.zeros((n_soundings, 9))
climate.wind_dir = np.zeros((n_soundings, 9))
# find the highest valid point in each sounding
rs.top = np.zeros((n_soundings, ))
rs.bot = np.zeros((n_soundings, ))
# climate.altitudes[0,:]=array([10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000])
for i in range(n_soundings):
mask = heights[i, :] <= 50000
if np.any(mask == True):
rs.top[i] = max(heights[i, mask])
rs.bot[i] = min(heights[i, temps[i, :] != 99999])
else:
rs.top[i] = 0.0
rs.bot[i] = 0.0
rs.top = hau.T_Array(rs.top)
rs.bot = hau.T_Array(rs.bot)
climate.altitudes[i, :] = np.array([
10000,
15000,
20000,
25000,
30000,
35000,
40000,
45000,
50000,
])
climate.temps[i, :] = np.array([
223.1,
216,
216,
221,
226,
237,
251,
265,
270,
])
climate.pressures[i, :] = np.array([
264.3,
120.45,
54.75,
25.11,
11.71,
5.58,
2.77,
1.43,
0.759,
])
climate.dew_pt[i, :] = np.NaN
# don't use climatology lower than 2km above highest valid measurement
climate.altitudes[climate.altitudes <= rs.top[i] + 2000] = \
9e36
climate.temps[climate.altitudes <= rs.top[i] + 2000] = 9e36
climate.pressures[climate.altitudes <= rs.top[i] + 2000] = \
9e36
# stack the climatology on top of the observations
heights = np.hstack((heights, climate.altitudes))
temps = np.hstack((temps, climate.temps))
pressures = np.hstack((pressures, climate.pressures))
dew_points = np.hstack((dew_points, climate.dew_pt))
wind_dir = np.hstack((wind_dir, climate.wind_dir))
wind_spd = np.hstack((wind_spd, climate.wind_spd))
#print heights.shape
heights_unsorted = heights.copy()
temps_unsorted = temps.copy()
pressures_unsorted = pressures.copy()
dew_points_unsorted = dew_points.copy()
wind_dir_unsorted = wind_dir.copy()
wind_spd_unsorted = wind_spd.copy()
for i in range(heights_unsorted.shape[0]):
indices = np.argsort(heights_unsorted[i, :])
heights[i, :] = heights_unsorted[i, indices]
temps[i, :] = temps_unsorted[i, indices]
pressures[i, :] = pressures_unsorted[i, indices]
dew_points[i, :] = dew_points_unsorted[i, indices]
wind_dir[i, :] = wind_dir_unsorted[i, indices]
wind_spd[i, :] = wind_spd_unsorted[i, indices]
# sort combined file by height and select times of interest
if 0:
indices = heights.argsort(axis=1)
index_a = np.transpose(
np.transpose(np.ones(heights.shape, dtype=int)) *
np.arange(heights.shape[0]))
heights = heights[index_a, indices]
temps = temps[index_a, indices]
pressures = pressures[index_a, indices]
dew_points = dew_points[index_a, indices]
wind_dir = wind_dir[index_a, indices]
wind_spd = wind_spd[index_a, indices]
pressures[heights > 1e5] = np.NaN
temps[heights > 1e5] = np.NaN
dew_points[heights > 1e5] = np.NaN
# interpolate to altitude resolution requested
# and remove missing data points
max_alt = requested_altitudes[-1]
max_bin = requested_altitudes.shape[0]
alts = requested_altitudes
n_soundings = len(rs.times)
#max_bin = round(max_alt / float(alt_res)) + 1
# create sounding arrays as hau class items
rs.altitudes = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
rs.temps = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
rs.dew_points = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
rs.pressures = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
rs.wind_spd = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
rs.wind_dir = hau.TZ_Array(np.zeros((n_soundings, max_bin)))
#rs.times =hau.T_Array(np.zeros(n_soundings))
rs.times = hau.T_Array(times)
#rs.expire_time =hau.T_Array(np.zeros(n_soundings))
rs.stalat = hau.T_Array(np.zeros(n_soundings))
rs.stalong = hau.T_Array(np.zeros(n_soundings))
rs.wmosta = hau.T_Array(np.zeros(n_soundings))
rs.stalat[:] = stalat[0]
rs.stalong[:] = stalong[0]
rs.wmosta[:] = wmosta[0]
rs.station_id = id
# interpolate to lidar altitude scale
for i in range(n_soundings):
rs.altitudes[i, :] = alts
k = i
rs.temps[i, :] = np.interp(alts, heights[k, temps[k, :] != 99999],
temps[k, temps[k, :] != 99999])
rs.dew_points[i, :] = np.interp(
alts, heights[k, dew_points[k, :] != 99999],
dew_points[k, dew_points[k, :] != 99999])
#now using spline fit to pressures
#should use hydrostatic equation to interpolate
press1 = pressures[k, pressures[k, :] != 99999]
h1 = heights[k, pressures[k, :] != 99999]
temp = interpolate.splrep(h1[~np.isnan(press1)],
press1[~np.isnan(press1)])
rs.pressures[i, :] = interpolate.splev(alts, temp, der=0)
rs.wind_spd[i, :] = np.interp(alts,
heights[k, wind_spd[k, :] != 99999],
wind_spd[k, wind_spd[k, :] != 99999])
rs.wind_dir[i, :] = np.interp(alts,
heights[k, wind_dir[k, :] != 99999],
wind_dir[k, wind_dir[k, :] != 99999])
#if i < n_soundings - 1:
# #rs.expire_time[i] = (rs.times[i] + rs.times[i + 1])
# rs.expire_time = rs.times[i] + timedelta(seconds=(rs.times[i+1] - rs.times[i]).total_seconds() / 2.0) + timedelta(seconds=60*30) # add 1/2 hour to point to next sounding
#else:
# #rs.expire_time[i] = rs.times[i] + 0.25 + 1 / 48.0 # new sonde profile expected in 6 1/2 hrs
# rs.expire_time = rs.times[i] + timedelta(days=0.25, seconds= 30*60) # new sonde profile expected in 6 1/2 hrs
# convert dew point depression to dew point temp
rs.dew_points = rs.temps - rs.dew_points
plots = 0 # FIXME
if plots == 1:
import matplotlib.pylab as plt
plt.figure(801)
plt.plot(temps[0, :], heights[0, :] / 1000, 'r', dew_points[0, :],
heights[0, :] / 1000)
fig = plt.grid(True)
plt.xlabel('deg-K ')
plt.ylabel('Altitude MSL (km)')
plt.title('Temperature, dew point')
plt.figure(802)
#set_printoptions(threshold=np.NaN)
plt.plot(rs.temps[0, :], rs.altitudes[0, :] / 1000, 'r',
rs.dew_points[0, :], rs.altitudes[0, :] / 1000)
fig = plt.grid(True)
plt.xlabel('deg-K ')
plt.ylabel('Altitude MSL (km)')
plt.title('Temperature, dew point')
plt.figure(803)
plt.plot(pressures[0, :], heights[0, :] / 1000, 'r')
fig = plt.grid(True)
ax = plt.gca()
#ax.set_xscale('log')
plt.xlabel('deg-K ')
plt.ylabel('Altitude MSL (km)')
plt.title('Pressures')
plt.figure(804)
bin_vec = range(len(heights[0, :]))
heights[heights > 1000] = np.NaN
plt.plot(heights[0, :] / 1000, bin_vec, 'r')
fig = plt.grid(True)
ax = plt.gca()
#ax.set_xscale('log')
plt.xlabel('altitudes')
plt.ylabel('bins')
plt.title('Heights')
plt.show()
raise RuntimeError('deliberate abort')
else:
print ' '
print 'ERROR**************unknown sounding source************'
print ' '
rs = []
return rs
def __call__(self,rs_inv,rs_particle,calvals):#update to process,and return a completed frame
rs_multiple_scattering=hau.Time_Z_Group(rs_inv.times.copy(),timevarname='times',altname='msl_altitudes')
N1 = 2
N2 = self.multiple_scatter_parameters['highest_order']
start_alt = self.multiple_scatter_parameters['lowest_altitude']
p180_water = self.multiple_scatter_parameters['p180_water']
p180_ice = self.multiple_scatter_parameters['p180_ice']
h2o_depol_threshold = self.multiple_scatter_parameters['h2o_depol_threshold']
p180_ice = self.multiple_scatter_parameters['p180_ice']
p180_water = self.multiple_scatter_parameters['p180_water']
second_wavelength =self.multiple_scatter_parameters['second_wavelength']
wavelength = calvals['wavelength']*1e-9
#assert(rs_particle!=None or self.multiple_scatter_parameters['particle_info_source'] == 'constant')
if 1: #self.multiple_scatter_parameters['processing_mode'] == '1d':
if self.multiple_scatter_parameters['particle_info_source'] == 'constant':
#in this case the mode diameter will be reset to ice or water values from multiple_scattering.json file
#depending on h2o_depol_threshold
mode_diameter = None
else:
#use lidar-radar retrieved particle sizes
#print 'particle'
#print dir(rs_particle)
mode_diameter = rs_particle.mode_diameter.copy()
#accumulates sums for 1-d average multiple scatter profile
self.ms_obj = msu.ms_sums(mode_diameter,rs_inv.beta_a_backscat
,rs_inv.linear_depol,self.multiple_scatter_parameters,self.ms_obj)
self.ms_obj.beta_water[self.ms_obj.beta_water < 0] = 0.0
self.ms_obj.beta_ice[self.ms_obj.beta_ice < 0] = 0.0
beta_total = self.ms_obj.beta_water +self.ms_obj.beta_ice
total_samples = self.ms_obj.n_samples_water +self.ms_obj.n_samples_ice
#when no ice or water data points are present averages must be zero
#compute weighted averages of beta, diameter when ice and water are present
#ms_obj.beta_water and ms_obj.beta_ice have the sum of beta_backscatter for ice and water
#self.ms_obj.n_samples_water[self.ms_obj.n_samples_water == 0] = np.infty
#self.ms_obj.n_samples_ice[self.ms_obj.n_samples_ice == 0] = np.infty
#compute ave beta_extinction profile from sum beta_backscat profiles
extinction_profile = (self.ms_obj.beta_water/p180_water + self.ms_obj.beta_ice/p180_ice)/total_samples
diameter = (self.ms_obj.diameter_ice + self.ms_obj.diameter_water)/beta_total
#convert altitudes into range
ranges = rs_inv.msl_altitudes.copy()
zenith_angle = np.abs(calvals['telescope_roll_angle_offset'])*np.pi/180.0
ranges = ranges/np.cos(zenith_angle)
start_range = start_alt/np.cos(zenith_angle)
end_range = rs_inv.msl_altitudes[-1]/np.cos(zenith_angle)
if start_range >= end_range:
raise RuntimeError(' start altitude'+str(np.int(start_range))+ ' is above highest data point')
ms_ratios_profile = msu.msinteg(N1,N2,start_range \
,end_range,self.multiple_scatter_parameters['step_size'], extinction_profile, diameter
,ranges,wavelength,self.multiple_scatter_parameters,calvals)
rs_multiple_scattering.ranges = ms_ratios_profile[:,0]
rs_multiple_scattering.ms_ratios_profile = ms_ratios_profile
rs_multiple_scattering.extinction_profile = hau.Z_Array(extinction_profile[np.newaxis,:])
rs_multiple_scattering.weighted_diameter = hau.Z_Array(diameter[np.newaxis,:])
rs_multiple_scattering.msl_altitudes = rs_inv.msl_altitudes.copy()
rs_multiple_scattering.wavelength = wavelength
#compute multiple scattering for a second wavelength if it is provided
#assume no change is extinction cross section
if second_wavelength:
ms_ratios_profile_2 = msu.msinteg(N1,N2,start_range \
,end_range,self.multiple_scatter_parameters['step_size'], extinction_profile, diameter
,ranges,second_wavelength,self.multiple_scatter_parameters,calvals)
rs_multiple_scattering.ms_ratios_profile_2 = ms_ratios_profile_2
rs_multiple_scattering.second_wavelength = second_wavelength
if self.multiple_scatter_parameters['processing_mode'] == '2d': #do multiple scatter calculation for all profiles in frame
print 'begining 2d multiple scatter processing'
#estimate extinction based on backscatter phase function
beta = rs_inv.beta_a_backscat.copy()
beta = beta/p180_water
beta[rs_inv.linear_depol>self.multiple_scatter_parameters['h2o_depol_threshold']] \
= beta[rs_inv.linear_depol>self.multiple_scatter_parameters['h2o_depol_threshold']]*p180_water/p180_ice
beta[beta < 0]=0.0
if self.multiple_scatter_parameters['particle_info_source'] == 'constant':
mode_diameter = np.ones_like(rs_inv.beta_a_backscat) \
* self.multiple_scatter_parameters['mode_diameter_water']
mode_diameter[rs_inv.linear_depol > self.multiple_scatter_parameters['h2o_depol_threshold']] \
= self.multiple_scatter_parameters['mode_diameter_ice']
#convert altitudes into ranges
ranges = rs_inv.msl_altitudes.copy()
zenith_angle = np.abs(calvals['telescope_roll_angle_offset'])*np.pi/180.0
ranges = ranges/np.cos(zenith_angle)
start_range = start_alt/np.cos(zenith_angle)
end_range = rs_inv.msl_altitudes[-1]/np.cos(zenith_angle)
for i in range(rs_inv.beta_a_backscat.shape[0]):
print 'Computing multiple scattering for ' ,rs_inv.times[i]
if self.multiple_scatter_parameters['particle_info_source'] == 'constant':
ratios = msu.msinteg(N1,N2,start_range \
,end_range,self.multiple_scatter_parameters['step_size'],beta[i,:],mode_diameter[i,:]
,ranges,wavelength,self.multiple_scatter_parameters,calvals)
else: #get mode diameter from lidar_radar measured values
ratios = msu.msinteg(N1,N2,start_range \
,end_range,wavelength,self.multiple_scatter_parameters['step_size']
,beta[i,:],rs_particle.mode_diameter[i,:]
,ranges,self.multiple_scatter_parameters,calvals)
#load values into output array
if not hasattr(rs_multiple_scattering,'ms_ratio_total'):
rs_multiple_scattering.ms_ratio_total = np.zeros((beta.shape[0],ratios.shape[0]))
rs_multiple_scattering.ms_ratio_total[i,:] =nansum(ratios[:,2:],1)
rs_multiple_scattering.msl_altitudes = rs_inv.msl_altitudes.copy()
rs_multiple_scattering.ms_ratio_total =hau.TZ_Array(rs_multiple_scattering.ms_ratio_total)
#rs_multiple_scattering.start_altitude = start_alt
return rs_multiple_scattering
def profile(self, request_time, request_lat, request_long, offset=0):
"""returns a profile of temperature,pressure, dew_point, frost point at
time, lat, and long extracted from the sounding_archive. If request_lat and
request_long are empty they are ignored
request_time = python datetime for reqested sounding
request_lat = requested latitude for sounding--ignored if []
request_lon = requested longitude for sounding--ignored if []"""
if self.soundings == None:
return None
print 'sounding_type= ', self.sounding_type, request_time
if self.sounding_type == 'NOAA raob':
temp_sounding = hau.Time_Z_Group()
#find correct sounding profile out of archive file
sounding = hau.selectByTime(self.soundings,
request_time,
offset=offset)
if sounding is None:
return None
sounding.sounding_type = self.sounding_type
sounding.sounding_id = sounding.station_id
sounding.latitude = sounding.stalat
sounding.longitude = sounding.stalong
sounding.frost_points = cal_frost_point(sounding.dew_points)
temp_sounding.type = self.sounding_type
temp_sounding.times = sounding.times
temp_sounding.altitudes = hau.Z_Array(sounding.altitudes)
temp_sounding.temps = hau.TZ_Array(sounding.temps[np.newaxis, :])
temp_sounding.pressures = hau.TZ_Array(
sounding.pressures[np.newaxis, :])
temp_sounding.dew_pts = hau.TZ_Array(
sounding.dew_points[np.newaxis, :])
temp_sounding.frost_pts = hau.TZ_Array(
sounding.frost_points[np.newaxis, :])
temp_sounding.wind_dir = hau.TZ_Array(
sounding.wind_dir[np.newaxis, :])
temp_sounding.wind_spd = hau.TZ_Array(
sounding.wind_spd[np.newaxis, :])
temp_sounding.wind_spd = hau.TZ_Array(
sounding.wind_spd[np.newaxis, :])
temp_sounding.station_id = sounding.station_id
temp_sounding.top = sounding.top
sounding.times = sounding.times[0]
#sounding.expire_time=sounding.expire_time[0]
elif self.sounding_type == "time curtain" \
and self.sounding_id == "raqms":
temp_sounding = hau.Time_Z_Group()
sounding = raqms.select_raqms_profile(self.soundings,request_time \
,self.requested_altitudes,offset=offset)
# ,self.max_alt,self.alt_res)
if sounding == None:
return None
sounding.station_id = self.sounding_id
temp_sounding.type = self.sounding_type
temp_sounding.times = sounding.times
temp_sounding.latitude = hau.T_Array(sounding.latitude)
temp_sounding.longitude = hau.T_Array(sounding.longitude)
temp_sounding.altitudes = hau.Z_Array(sounding.altitudes)
temp_sounding.temps = hau.TZ_Array(sounding.temps[np.newaxis, :])
temp_sounding.pressures = hau.TZ_Array(
sounding.pressures[np.newaxis, :])
temp_sounding.dew_pts = hau.TZ_Array(
sounding.dew_points[np.newaxis, :])
temp_sounding.frost_pts = hau.TZ_Array(
sounding.frost_points[np.newaxis, :])
temp_sounding.wind_dir = hau.TZ_Array(
sounding.wind_dir[np.newaxis, :])
temp_sounding.wind_spd = hau.TZ_Array(
sounding.wind_spd[np.newaxis, :])
temp_sounding.ext_total = hau.TZ_Array(
sounding.ext_total[np.newaxis, :])
temp_sounding.ext_salt = hau.TZ_Array(
sounding.ext_salt[np.newaxis, :])
temp_sounding.ext_dust = hau.TZ_Array(
sounding.ext_dust[np.newaxis, :])
temp_sounding.wind_spd = hau.TZ_Array(
sounding.wind_spd[np.newaxis, :])
temp_sounding.station_id = sounding.station_id
temp_sounding.top = sounding.top
sounding.times = sounding.times[0]
#set up time to read in new sounding (in datetime)
#sounding.expire_time = sounding.times+timedelta(seconds=5*60)
#expire time can not be less then request time--raqms file must be missing soundings
#set expire_time 5 minutes after request time
#if sounding.expire_time <= request_time:
# print "****Warning----missing raqms sounding at ",request_time
# print " using sounding from ",sounding.times
# sounding.expire_time = request_time + timedelta(seconds=5*60)
#remove any negative pressure values
temp_sounding.pressures[temp_sounding.pressures < 0] = 0.0
#remove NaN's from pressure and temperature values
temp_sounding.pressures[np.isnan(temp_sounding.pressures)] = 0.0
temp_sounding.temps[np.isnan(temp_sounding.temps)] = 1.0
return sounding
def raman_inversion(mean, consts, Cxx, corr_adjusts, process_control):
"""raman_inversion(mean,consts,Cxx,corr_adjusts,process_control)
"""
inv = hau.Time_Z_Group(like=mean)
inv.beta_r_355 = Cxx.beta_r_355 #FIXME
inv.beta_r_387 = Cxx.beta_r_387
beta_r_n2 = Cxx.beta_r_387
inv.times = mean.times.copy()
inv.delta_t = mean.delta_t.copy()
inv.altitudes = mean.altitudes.copy()
#add backscatter ratios to inv.
adj = consts['elastic_to_n2_gain_ratio']
wfov_adj = consts['wfov_elastic_to_n2_gain_ratio']
inv.aerosol_backscatter_ratio = (mean.elastic_counts - mean.nitrogen_counts)\
/mean.nitrogen_counts
inv.aerosol_backscatter_ratio_low = (wfov_adj * mean.elastic_counts_low - mean.nitrogen_counts_low)\
/mean.nitrogen_counts_low
inv.aerosol_backscatter_ratio_high = (adj * mean.elastic_counts_high - mean.nitrogen_counts_high)\
/ mean.nitrogen_counts_high
inv.beta_a_backscat_low = 3 * inv.aerosol_backscatter_ratio_low * Cxx.beta_r_355 / (
8.0 * np.pi)
inv.beta_a_backscat = 3 * inv.aerosol_backscatter_ratio * Cxx.beta_r_355 / (
8.0 * np.pi)
inv.integrated_backscatter = lu.integrated_backscatter(
inv.beta_a_backscat, inv.altitudes)
print
print
print 'check depol adjustment in raman_inversion***********************************************'
print
print
inv.linear_depol = mean.depolarization_counts_high / (
mean.elastic_counts_high - mean.nitrogen_counts_high)
if 0:
import matplotlib.pylab as plt
bin_vec = np.arange(inv.altitudes.shape[0])
plt.figure(5555)
plt.plot(np.nanmean(inv.aerosol_backscatter_ratio_low, 0), bin_vec,
'r', np.nanmean(inv.aerosol_backscatter_ratio_high,
0), bin_vec, 'm',
np.nanmean(inv.aerosol_backscatter_ratio, 0), bin_vec, 'k')
plt.grid(True)
#new code for raman extinction
#filter_params needs to be called only when one of the calling parameters changes
filter_params = lsge.filter_setup(inv.altitudes, process_control, consts,
mean.delta_t[0])
#compute range integrated backscatter cross section
temp = inv.beta_a_backscat.copy()
temp[np.isnan(temp)] = 0.0
inv.integ_backscat = np.cumsum(temp, 1) * filter_params['dz']
#for key in filter_params:
# print key ,' = ',filter_params[key]
inv.extinction = np.NaN * np.zeros_like(mean.nitrogen_counts)
ntimes = len(mean.nitrogen_counts[:, 0])
inv.extinction_aerosol = np.NaN * np.ones_like(mean.nitrogen_counts)
inv.p180 = np.NaN * np.ones_like(mean.nitrogen_counts)
if ntimes == 1: #dont care
alreadyfiltered = hasattr(mean, 'filtered_nitrogen')
Nm = mean.filtered_nitrogen if alreadyfiltered else mean.nitrogen_counts
elif hasattr(mean, 'filtered_nitrogen') and not (
mean.nitrogen_counts == mean.filtered_nitrogen).all():
alreadyfiltered = True
Nm = mean.filtered_nitrogen
print "@@@@@@@@@ SKIPPING ROLLING FILTER. Already done"
else:
print '@@@@@ Trying bad ROLLING FILTER. Already done'
alreadyfiltered = False
Nm = mean.nitrogen_counts
if not alreadyfiltered:
try:
half_slice = filter_params['t_window_pts'] / 2
except:
half_slice = 1
sg_ext = lsge.sg_extinction(filter_params)
if ntimes == 1: #for profile call
sl = np.arange(1)
inv.extinction[0,:],inv.extinction_aerosol[0,:],inv.p180[0,:] \
= sg_ext(mean.times[sl],mean.delta_t[sl],Nm[sl,:]
,inv.beta_a_backscat[0,:],inv.integ_backscat[0,:],beta_r=Cxx.beta_r_355)
else:
#needs to add edge times with padded intervals
state = []
if alreadyfiltered:
fullrange = range(ntimes)
else:
fullrange = range(half_slice, ntimes - half_slice - 1)
for i in fullrange:
if alreadyfiltered:
sl = np.arange(i, i + 1) #already filtered
else:
sl = np.arange(i - half_slice, i + half_slice + 1)
inv.extinction[i,:],inv.extinction_aerosol[i,:],inv.p180[i,:] \
= sg_ext(mean.times[sl],mean.delta_t[sl],Nm[sl,:]
,inv.beta_a_backscat[i,:],inv.integ_backscat[i,:],beta_r=Cxx.beta_r_355,state=state)
#end of new code for Raman exticntion
if 0:
import matplotlib.pylab as plt
plt.ion()
plt.figure(3002)
plt.plot(np.nanmean(inv.extinction, 0), inv.altitudes / 1000.0)
ax = plt.gca()
ax.set_xscale('log')
plt.xlabel('extinction (1/m)')
plt.ylabel('altitude (km)')
plt.grid(True)
return inv
class dpl_hsrl_narr(dplkit.role.narrator.aNarrator):
""" DPL HSRL Narrator Object. should only be created by dpl_hsrl object
:param params: parameters dictionary
:param cal_narr: calibration framestream narration object
:param timesource: time axis generation source (could be synthetic or from another stream)
:param rawsrc: raw data source. if not provided, will create a lot of it here
:param lib: raw hsrl reading library object only used if rawsrc is not given
:param zoo: raw hsrl zookeeper object only used if rawsrc is not given
exposed attributes:
- hsrl_cal_stream (calibration stream, can be used for parallel stream collection)
exposed field type in chain:
- hsrl.dpl.calibration.dpl_calibration_narr
"""
#def get_process_control(self):
# return self.cal_narr.get_process_control()
@property
def hsrl_cal_stream(self):
return self.cal_narr
def __init__(self,params,cal_narr,timesource,rawsrc=None,compute_stats=0):
super(dpl_hsrl_narr,self).__init__(None,cal_narr) #FIXME replace url with some manner of unique path
#import dpl_calibration
#self.dpl_calibration=dpl_calibration
#self.provides=libr.provides
self.compute_stats=compute_stats
self.rawsrc=rawsrc
self.params=params
self.timesource=timesource
self.cal_narr=cal_narr
def __repr__(self):
return 'DPL HSRL Framestream Narrator (%s)' % (self.params)
def read(self):
""" main read generator
"""
import hsrl.data_stream.processing_utilities as pu
params=self.params
firsttimeever=None
intervalTime=None
intervalEnd=None
rawsrc=iter(self.rawsrc)
#if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])):
# params['timeres']=None #pure native
end_time_datetime=params['finalTime']
#timemodoffset=time_mod(params['realStartTime'],params['timeres'])
noframe='noframe'
fullrange=False #if this is true, it will pad the start with any missing times.
remainder=None
cdf_to_hsrl = None
preprocess_ave = None
requested_times=None
instrument=self.hsrl_instrument
intcount=0
rs_mem = None
#rs=None
timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None
for calv in self.cal_narr:
if intervalTime is None:
firsttimeever=calv['chunk_start_time']
intervalTime=calv['chunk_start_time']
intervalEnd=intervalTime
chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset)
#print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres']
if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None:
if params['block_when_out_of_data']:
if 'timeres' not in params or params['timeres'] is None:
sleep(calv['rs_constants']['integration_time'])
else:
sleep(params['timeres'].total_seconds())
else:
yield None #this is done to get out of here, and not get stuck in a tight loop
continue
while intervalTime<chunk_end_to_use:
integration_time = calv['rs_constants']['integration_time']
doPresample=True
#END init section
if intervalEnd>chunk_end_to_use:
print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime
break
else:
intervalEnd=chunk_end_to_use
#print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime']
print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem
if True:#requested_times==None or requested_times.shape[0]>0:
try:
try:
while rawsrc is not None:
if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
break
tmp=rawsrc.next()
if hasattr(tmp,'rs_raw'):
if rs_mem is not None:
rs_mem.append(tmp.rs_raw)
else:
rs_mem=copy.deepcopy(tmp.rs_raw)
if rs_mem is not None and rs_mem.times.shape>0:
break
else:
rs_mem=None
except StopIteration:
print 'Raw HSRL stream is ended'
rawsrc=None
if rs_mem is None or rs_mem.times.size==0:
rs_mem=None
elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime):
print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0]
break
else:
intervalEnd=rs_mem.times[-1]
print 'read in raw frame to mean',rs_mem,remainder
if rawsrc is None:
intervalEnd=chunk_end_to_use
print 'trimmed ',rs_mem
if timesource is not None:
if timesource.isDone:
break
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime)
timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime)))
print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd
elif 'timeres' in params and params['timeres'] is not None:
tmp=intervalTime
useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap
usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet
timevals=[]
timevals.append(tmp)
while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats.
#print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds()
tmp+=params['timeres']
timevals.append(tmp)
#intervalEnd=tmp
intcount+=len(timevals)
if usePrebinnedTimes:
intcount-=1
print 'Now %i intervals' % (intcount)
timevals=hau.T_Array(timevals)
else:
print 'Using Native timing'
timevals=None
print ' new window is ', intervalTime, ' to ' , intervalEnd
requested_times=timevals
requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime]
if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None:
#if rawsrc is not None:
print "not enough time to process"
continue
elif rawsrc is None and rs_mem is None and remainder is None:
#chunk_end_to_use=intervalTime
#continue
#print ''
break
rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd
,params['min_alt'], params['max_alt'], requested_chunk_times
, rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal']
, None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control
, self.compute_stats,remainder=remainder)
rs_mem=None
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0:
rs_chunk=None
if rs_chunk is None and rawsrc is None:
break
#print rs_chunk
if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0:
if fullrange and requested_chunk_times is not None:
v=hau.Time_Z_Group(like=rs_chunk.rs_mean)
v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]])
if v.times.size>0:
rs_chunk.rs_mean.prepend(v)
rs_chunk.calv=calv
yield rs_chunk
intervalTime=intervalEnd
except Exception, e:
print 'Exception occured in update_cal_and_process'
print 'Exception = ',e
print traceback.format_exc()
if isinstance(e,(MemoryError,)):
print 'Please Adjust Your Parameters to be more Server-friendly and try again'
raise
if not isinstance(e,(AttributeError,)):
raise
assert(remainder is None or remainder.times.size==0)
if fullrange and end_time_datetime is not None and timesource is not None and (not timesource.isDone or requested_times is not None) and firsttimeever!=intervalTime:#either timesource indicates it wasn't run completely or requested times wasn't cleared
requested_times=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=end_time_datetime))
if requested_times is not None and len(requested_times)>1:
print 'NO DATA to end from ',intervalTime,' to ',end_time_datetime #FIXME IS THIS USED? JPG 20160504
print "times to use are ",requested_times[:-1]
rs= hau.Time_Z_Group()
rs.rs_mean=hau.Time_Z_Group()
rs.rs_mean.times=hau.T_Array(requested_times[:-1]).copy() #causes the time axis to be stored, but all others may be implied MISSING
setattr(rs.rs_mean,'delta_t',hau.T_Array(np.zeros(rs.rs_mean.times.shape)))
yield rs
