def read(self): remainder=None base=None import lg_dpl_toolbox.dpl.TimeSource as TimeSource timesource=TimeSource.CompoundTimeGenerator(self.timesource) for f in self.framestream: if timesource.isDone: break if remainder==None: remainder=copy.deepcopy(f) else: remainder.append(f) t=getattr(remainder,remainder._timevarname) #print t.shape if t.shape[0]==0: continue requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=t[-1])) if requestedtimes.size<2: continue lastTime=requestedtimes[-1] retarr=remainder remainder=hau.trimTimeInterval(retarr,lastTime,datetime(2200,1,1,0,0,0)) retarr.trimTimeInterval(base,lastTime) retarr.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans) print 'range',base,lastTime,'returning:',retarr,'remainder',remainder yield retarr base=lastTime if remainder!=None and timesource.end_time!=None: requestedtimes=hau.T_Array(timesource.getBinsFor(starttime=base,endtime=timesource.end_time,inclusive=True)) remainder.hereGoneBinTimes(requestedtimes,allow_nans=self.allow_nans) if getattr(remainder,remainder._timevarname).shape[0]>0: print 'range',base,timesource.end_time,'returning:',remainder yield remainder
def process(self): fr = None flags = None olda = None qasource = tf.TimeTrickle(self.qasource, 'time') altitudes = hau.Z_Array(self.qaparser.altitudeAxis) for f in self.timealtsource: #FIXME include angles if not isinstance(f, dict): f = vars(f) t = f[self.timename] a = self.constantAltitude if fr is None or ((not qasource.atEnd) and t >= qasource.nextTime ): #if need an update to the qa record #print 'Getting qa source for time',t fr = qasource(t) flags = None if 'range_flags' in fr and fr[ 'range_flags'] is not None: #if there is a range dependence, and a potentially non-constant altitude if self.altname is not None and self.altname in f: a = f[self.altname] if a is None: raise RuntimeError( 'Need platform altitude to merge in range-dependant qa Flags' ) if olda is None or a != olda: flags = None olda = a if flags is None: #was cleared either because new flags from the qa file, or new altitude from the stream if 'range_flags' in fr and fr['range_flags'] is not None: flags = self.qaparser.mergeVectors(fr['flags'], fr['range_flags'], a) else: flags = fr['flags'] flags = self.qaparser.translateToEnumeration(flags) flags = hau.TZ_Array(flags.reshape([1] + list(flags.shape)), dtype='int32', summode='and') ret = hau.Time_Z_Group(timevarname='times', altname='altitudes') setattr(ret, 'times', hau.T_Array([t])) setattr(ret, 'delta_t', hau.T_Array([f['width'].total_seconds()])) setattr(ret, 'altitudes', copy.copy(altitudes)) setattr(ret, 'start', f['start']) setattr(ret, 'width', f['width']) if self.splitFields: for f, idx in self.qaparser.flagbits.items(): setattr( ret, 'qa_' + f, hau.TZ_Array((flags / (10**idx)) % 10, dtype='int32', summode='and')) else: setattr(ret, 'qaflags', flags) yield ret
def process(self): priorFrame = None priorFrameD = None averageDiff = 0.0 for f in self.s: assert (f is not None) f = copy.copy(f) _f = f if not isinstance(_f, dict): _f = vars(_f) tf = _f[self.timefieldname] dtf = hau.T_Array(numpy.zeros(tf.shape), summode='sum') _f[self.dtfieldname] = dtf for i in range(tf.size - 1): dtf[i] = (tf[i + 1] - tf[i]).total_seconds() if priorFrame is not None: if priorFrameD[self.dtfieldname].size > 0 and tf.size > 0: priorFrameD[self.dtfieldname][-1] = ( tf[0] - priorFrameD[self.timefieldname][-1]).total_seconds() yield priorFrame priorFrame = f priorFrameD = _f if priorFrameD[self.dtfieldname].size > 1: priorFrameD[self.dtfieldname][-1] = priorFrameD[ self.dtfieldname][-2] if priorFrame is not None: yield priorFrame
def process(self): for f in self.s: f = copy.copy(f) _f = f if not isinstance(f, dict): _f = vars(f) #print _f.keys() _f[self.timefieldname] = hau.T_Array( [_f['start']], summode='first' ) #+timedelta(seconds=_f['width'].total_seconds()/2.0)]) _f[self.dtfieldname] = hau.T_Array([ _f['width'].total_seconds() if _f['width'] != None and _f['width'].total_seconds() > 0.0 else numpy.NaN ], summode='sum') yield f
def dark_count_correction_from_signal(chan_sel_dict, rs, start_index, end_index): """ dark_count_correction_from_signal(apply_to,rs,start_index,end_index) finds nanmean of signal in altitude bins between start and end indices for each profile and subtracts this value for each profile Also add a vector T_Array vector of dark counts for each profile. chan_sel_dict = dict(chanel_names = "dark_count_names" ......) correct channel_names with corresponding dark_counts rs = structure containing variables to be corrected start_index = array index for start of dark average end_index = array index for end of dark count average """ #for field in apply_to: for channel_name, dark_count_name in chan_sel_dict.iteritems(): #if hasattr(rs,field): if hasattr(rs, channel_name): temp_d = getattr(rs, channel_name).copy() ones_array = np.transpose(np.ones_like(temp_d)) dark_corr = hau.T_Array( np.nanmean(temp_d[:, start_index:end_index], 1)) dark_corr_array = np.transpose(ones_array * dark_corr) temp_d -= dark_corr_array #write corrected counts back to rs setattr(rs, channel_name, temp_d) #print 'field entering',channel_name #add dark count field to rs setattr(rs, dark_count_name, dark_corr) return rs
def realdocall(self, myslice, templaterec): """ call the filter with the given slices as input :param myslice: list of frames in the window :param templaterec: record in the window to base the return off of if this filter was given a varlist, this will call the filter for each individual variable, with a 2D array as the content. the return of each call will replace the content of each var in templaterec and returned if this filter was not given a varlist, the list of frames passed to this function will instead be passed to a single call of the filter. will return what the filter returns :returns: simple frame result """ if not self.inMiddle(myslice, templaterec): print 'FAILED TO ALIGN WINDOW: len ', len( myslice), ' index ', self.indexOf(myslice, templaterec) b = hau.T_Array(np.ones((1, 6))) c = len(myslice) x = self.indexOf(myslice, templaterec) b[0, 0] = c b[0, 1] = x b[0, 2] = self.delta_t(myslice).total_seconds() b[0, 3] = self.delta_t(myslice[:x] if x > 0 else []).total_seconds() b[0, 4] = self.delta_t(myslice[x:(x + 1)]).total_seconds() b[0, 5] = self.delta_t(myslice[( x + 1):] if x < (c - 1) else []).total_seconds() # print 'rolling window stats',b if self.varlist == None: kwcargs = self.kwcargs if self.middleframeparametername is not None: kwcargs = kwcargs.copy() kwcargs[self.middleframeparametername] = copy.deepcopy( templaterec) ret = self.callable(myslice, *self.cargs, **kwcargs) else: ret = copy.deepcopy(templaterec) for i, v in enumerate(self.varlist): try: av = deepattribute(myslice[0], v) except KeyError: print 'Warning: configured key', v, 'not found in frame' raise continue av = copy.deepcopy(av) for f in myslice[1:]: av.append(deepattribute(f, v)) setdeepattribute( ret, v, self.callable(av, *self.cargs, **self.kwcargs)) _ret = ret if not isinstance(ret, dict): _ret = vars(ret) if b is not None and self.windowinfo_perframe is not None: if self.windowinfo_perframe not in _ret: _ret[self.windowinfo_perframe] = b else: _ret[self.windowinfo_perframe] = np.concatenate( (_ret[self.windowinfo_perframe], b), 1) return ret
def addCommonAttributes(x, attrs): for attr, dest in attrrep.items(): if attr not in attrs: continue v = attrs[attr][:].split() val = hau.T_Array(np.ones(x.times.shape) * float(v[0])) if len(v) > 1: if v[1] in ('(GHz)', 'GHz'): val *= 1e9 setattr(x, dest or attr, val)
def radar_masking(rs_radar, processing_defaults, instrument): """radar_masking(rs_radar,processing_defaults) rs_radar = structure generated by radar processing stream processing_defaults = config info from 'radar_processing_defaults.json' qc_radar_mask, bit[0] = longical and of all other bits qc_radar_mask, bit[1] = cleared if radar refectivity fall below SNR threshhold """ assert (rs_radar != None) if not hasattr(rs_radar, 'Backscatter'): return rs_radar threshhold = processing_defaults.get_value('radar_SNR_mask', 'threshhold') #radar mask ok when sig_to_noise greater than threshhold in dB rs_radar = copy.copy(rs_radar) #fix me this has been changed to a single bit because the interpolator is #not aware of summode rs_radar.qc_radar_mask = \ hau.TZ_Array(np.ones(rs_radar.Backscatter.shape),summode='or',dtype='uint16') #rs_radar.qc_radar_mask[:,:]=65535 #mask = np.ones_like(rs_radar.Backscatter) #mask[rs_radar.SignalToNoiseRatio < threshhold] = 0 #mask = ~(3*mask.astype('uint16')) #rs_radar.qc_radar_mask &= mask rs_radar.qc_radar_mask[rs_radar.SignalToNoiseRatio < threshhold] = 0 rs_radar.qc_radar_mask[np.isnan(rs_radar.SignalToNoiseRatio)] = 0 print print 'instrument', instrument if (instrument == 'magkazrge' or 'magmwacr') \ and processing_defaults.enabled('ship_motion_correction'): temp = rs_radar.MeanDopplerVelocity.copy() temp[rs_radar.qc_radar_mask == 0] = np.NaN rs_radar.vertically_averaged_doppler = hau.T_Array(nanmean(temp, 1)) return rs_radar
def read_sounding_file(instrument, sounding_type, id, start_time, requested_altitudes): """ read_sounding_file([instrument,sounding_type,id,start_time,alt_res,max_alt) returns arrays rs.temps(sounding#, rs.dew_points(sounding#,alt_index), rs.wdir(sounding#,alt_index), rs.wspd(sounding#,alt_index) along with several scalers with sounding info instrument (e.g. 'ahsrl','gvhsrl','mf2hsrl','nshsrl') sounding_type may be radiosonde station id, model identifier, or other instrument sounding_type (e.g. 'NOAA','ARM',....... sounding id (for sounding_type=NOAA, this a 3-letter e.g. 'MSN') start_time first time for which the sounding is needed, provided as matplot time requested_altitudes is a vector of altitudes at which sounding values are requested (m) returns temp_sounding(object) with all soundings from the current file after the starting time returns the last time at which this sounding can be used as rs.expire_time""" import lg_dpl_toolbox.core.archival as hru if sounding_type[:].find('NOAA raob') >= 0: rs = hau.Time_Z_Group() time_struct = start_time dir_path = hru.get_path_to_data(instrument, start_time) filename = dir_path + '/' + '%4i' % time_struct.year + '/' \ + '%02i' % time_struct.month + '/sondes.' + id[:] + '.nc' print 'sounding file--', filename if not os.path.exists(filename): return None #raise RuntimeError, 'sounding file %s does not exist' \ # % filename nc = Dataset(filename, 'r') times = getAll(nc, 'synTime') # combine mandatory and sig height measurements heights = np.hstack((getAll(nc, 'htMan'), getAll(nc, 'htSigT'))) epoch = datetime(1970, 1, 1, 0, 0, 0) t_mask = times < 1e36 for i in range(len(times)): t_mask[i] = times[i] < 1e36 and any(heights[i, :] < 1e36) times = times[t_mask] times = [epoch + timedelta(seconds=soff) for soff in times[:]] # select times, one prior to start time --> last profile in file #indices = np.arange(len(times)) #start_index = max(indices[times <= start_time]) #rs.times = zeros(len(times) - start_index) rs.times = np.zeros(len(times)) #rs.times = times[start_index:] # rs.times = hau.T_Array( rs.times ) rs.times = times[:] wmosta = getAll(nc, 'wmoStat') # wmo station number stalong = getAll(nc, 'staLon') stalat = getAll(nc, 'staLat') # combine mandatory and sig height measurements temps = np.hstack((getAll(nc, 'tpMan'), getAll(nc, 'tpSigT'))) pressures = np.hstack((getAll(nc, 'prMan'), getAll(nc, 'prSigT'))) dew_points = np.hstack((getAll(nc, 'tdMan'), getAll(nc, 'tdSigT'))) wind_dir = np.hstack((getAll(nc, 'wdMan'), getAll(nc, 'wdSigT'))) wind_spd = np.hstack((getAll(nc, 'wsMan'), getAll(nc, 'wsSigT'))) heights = heights[t_mask, :] temps = temps[t_mask, :] pressures = pressures[t_mask, :] dew_points = dew_points[t_mask, :] wind_dir = wind_dir[t_mask, :] wind_spd = wind_spd[t_mask, :] [n_soundings, n_heights] = temps.shape # defined standard atmosphere climatology for use above highest reported level # climate=temp_sounding() climate = hau.Time_Z_Group() climate.altitudes = np.zeros((n_soundings, 9)) climate.temps = np.zeros((n_soundings, 9)) climate.pressures = np.zeros((n_soundings, 9)) climate.dew_pt = np.zeros((n_soundings, 9)) climate.wind_spd = np.zeros((n_soundings, 9)) climate.wind_dir = np.zeros((n_soundings, 9)) # find the highest valid point in each sounding rs.top = np.zeros((n_soundings, )) rs.bot = np.zeros((n_soundings, )) # climate.altitudes[0,:]=array([10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000]) for i in range(n_soundings): mask = heights[i, :] <= 50000 if np.any(mask == True): rs.top[i] = max(heights[i, mask]) rs.bot[i] = min(heights[i, temps[i, :] != 99999]) else: rs.top[i] = 0.0 rs.bot[i] = 0.0 rs.top = hau.T_Array(rs.top) rs.bot = hau.T_Array(rs.bot) climate.altitudes[i, :] = np.array([ 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, ]) climate.temps[i, :] = np.array([ 223.1, 216, 216, 221, 226, 237, 251, 265, 270, ]) climate.pressures[i, :] = np.array([ 264.3, 120.45, 54.75, 25.11, 11.71, 5.58, 2.77, 1.43, 0.759, ]) climate.dew_pt[i, :] = np.NaN # don't use climatology lower than 2km above highest valid measurement climate.altitudes[climate.altitudes <= rs.top[i] + 2000] = \ 9e36 climate.temps[climate.altitudes <= rs.top[i] + 2000] = 9e36 climate.pressures[climate.altitudes <= rs.top[i] + 2000] = \ 9e36 # stack the climatology on top of the observations heights = np.hstack((heights, climate.altitudes)) temps = np.hstack((temps, climate.temps)) pressures = np.hstack((pressures, climate.pressures)) dew_points = np.hstack((dew_points, climate.dew_pt)) wind_dir = np.hstack((wind_dir, climate.wind_dir)) wind_spd = np.hstack((wind_spd, climate.wind_spd)) #print heights.shape heights_unsorted = heights.copy() temps_unsorted = temps.copy() pressures_unsorted = pressures.copy() dew_points_unsorted = dew_points.copy() wind_dir_unsorted = wind_dir.copy() wind_spd_unsorted = wind_spd.copy() for i in range(heights_unsorted.shape[0]): indices = np.argsort(heights_unsorted[i, :]) heights[i, :] = heights_unsorted[i, indices] temps[i, :] = temps_unsorted[i, indices] pressures[i, :] = pressures_unsorted[i, indices] dew_points[i, :] = dew_points_unsorted[i, indices] wind_dir[i, :] = wind_dir_unsorted[i, indices] wind_spd[i, :] = wind_spd_unsorted[i, indices] # sort combined file by height and select times of interest if 0: indices = heights.argsort(axis=1) index_a = np.transpose( np.transpose(np.ones(heights.shape, dtype=int)) * np.arange(heights.shape[0])) heights = heights[index_a, indices] temps = temps[index_a, indices] pressures = pressures[index_a, indices] dew_points = dew_points[index_a, indices] wind_dir = wind_dir[index_a, indices] wind_spd = wind_spd[index_a, indices] pressures[heights > 1e5] = np.NaN temps[heights > 1e5] = np.NaN dew_points[heights > 1e5] = np.NaN # interpolate to altitude resolution requested # and remove missing data points max_alt = requested_altitudes[-1] max_bin = requested_altitudes.shape[0] alts = requested_altitudes n_soundings = len(rs.times) #max_bin = round(max_alt / float(alt_res)) + 1 # create sounding arrays as hau class items rs.altitudes = hau.TZ_Array(np.zeros((n_soundings, max_bin))) rs.temps = hau.TZ_Array(np.zeros((n_soundings, max_bin))) rs.dew_points = hau.TZ_Array(np.zeros((n_soundings, max_bin))) rs.pressures = hau.TZ_Array(np.zeros((n_soundings, max_bin))) rs.wind_spd = hau.TZ_Array(np.zeros((n_soundings, max_bin))) rs.wind_dir = hau.TZ_Array(np.zeros((n_soundings, max_bin))) #rs.times =hau.T_Array(np.zeros(n_soundings)) rs.times = hau.T_Array(times) #rs.expire_time =hau.T_Array(np.zeros(n_soundings)) rs.stalat = hau.T_Array(np.zeros(n_soundings)) rs.stalong = hau.T_Array(np.zeros(n_soundings)) rs.wmosta = hau.T_Array(np.zeros(n_soundings)) rs.stalat[:] = stalat[0] rs.stalong[:] = stalong[0] rs.wmosta[:] = wmosta[0] rs.station_id = id # interpolate to lidar altitude scale for i in range(n_soundings): rs.altitudes[i, :] = alts k = i rs.temps[i, :] = np.interp(alts, heights[k, temps[k, :] != 99999], temps[k, temps[k, :] != 99999]) rs.dew_points[i, :] = np.interp( alts, heights[k, dew_points[k, :] != 99999], dew_points[k, dew_points[k, :] != 99999]) #now using spline fit to pressures #should use hydrostatic equation to interpolate press1 = pressures[k, pressures[k, :] != 99999] h1 = heights[k, pressures[k, :] != 99999] temp = interpolate.splrep(h1[~np.isnan(press1)], press1[~np.isnan(press1)]) rs.pressures[i, :] = interpolate.splev(alts, temp, der=0) rs.wind_spd[i, :] = np.interp(alts, heights[k, wind_spd[k, :] != 99999], wind_spd[k, wind_spd[k, :] != 99999]) rs.wind_dir[i, :] = np.interp(alts, heights[k, wind_dir[k, :] != 99999], wind_dir[k, wind_dir[k, :] != 99999]) #if i < n_soundings - 1: # #rs.expire_time[i] = (rs.times[i] + rs.times[i + 1]) # rs.expire_time = rs.times[i] + timedelta(seconds=(rs.times[i+1] - rs.times[i]).total_seconds() / 2.0) + timedelta(seconds=60*30) # add 1/2 hour to point to next sounding #else: # #rs.expire_time[i] = rs.times[i] + 0.25 + 1 / 48.0 # new sonde profile expected in 6 1/2 hrs # rs.expire_time = rs.times[i] + timedelta(days=0.25, seconds= 30*60) # new sonde profile expected in 6 1/2 hrs # convert dew point depression to dew point temp rs.dew_points = rs.temps - rs.dew_points plots = 0 # FIXME if plots == 1: import matplotlib.pylab as plt plt.figure(801) plt.plot(temps[0, :], heights[0, :] / 1000, 'r', dew_points[0, :], heights[0, :] / 1000) fig = plt.grid(True) plt.xlabel('deg-K ') plt.ylabel('Altitude MSL (km)') plt.title('Temperature, dew point') plt.figure(802) #set_printoptions(threshold=np.NaN) plt.plot(rs.temps[0, :], rs.altitudes[0, :] / 1000, 'r', rs.dew_points[0, :], rs.altitudes[0, :] / 1000) fig = plt.grid(True) plt.xlabel('deg-K ') plt.ylabel('Altitude MSL (km)') plt.title('Temperature, dew point') plt.figure(803) plt.plot(pressures[0, :], heights[0, :] / 1000, 'r') fig = plt.grid(True) ax = plt.gca() #ax.set_xscale('log') plt.xlabel('deg-K ') plt.ylabel('Altitude MSL (km)') plt.title('Pressures') plt.figure(804) bin_vec = range(len(heights[0, :])) heights[heights > 1000] = np.NaN plt.plot(heights[0, :] / 1000, bin_vec, 'r') fig = plt.grid(True) ax = plt.gca() #ax.set_xscale('log') plt.xlabel('altitudes') plt.ylabel('bins') plt.title('Heights') plt.show() raise RuntimeError('deliberate abort') else: print ' ' print 'ERROR**************unknown sounding source************' print ' ' rs = [] return rs
def read_raqms_file(instrument, start_time): """read raqms file between start and end time instrument - e.g. 'gvhsrl' start_time - datetime object """ raqms = hau.Time_Z_Group(altname='model_level_alts') filename = find_raqms_filename(instrument, start_time) if not filename: return None nc = Dataset(filename, 'r') times = getAll(nc, 'time') aircraft_alts = getAll(nc, 'alt') pressures = getAll(nc, 'pressure') temperatures = getAll(nc, 'temperature') model_level_alts = getAll(nc, 'altitude') relative_humidity = getAll(nc, 'rh') latitude = getAll(nc, 'lat') longitude = getAll(nc, 'lon') u_vel = getAll(nc, 'uvel') v_vel = getAll(nc, 'vvel') ext_total = getAll(nc, 'ext_tot') ext_dust = getAll(nc, 'ext_dust') ext_salt = getAll(nc, 'ext_salt') base_time = datetime(start_time.year, start_time.month, start_time.day, 0, 0, 0) #np.fix(start_time) #time=times.astype('float64') #convert raqms seconds from start of day to python datetimes #times=base_time + time/(3600.0*24.0) times = hau.T_Array( [base_time + timedelta(seconds=float(x)) for x in times]) assert (times.size > 0) selectedMask = (times > start_time) for i, x in enumerate(selectedMask): fi = i if x: if i > 0: selectedMask[i - 1] = True break selectedMask[-1] = True selectedTimes = np.arange(times.size)[selectedMask] raqms.latitude = hau.T_Array(latitude[selectedTimes]) raqms.longitude = hau.T_Array(longitude[selectedTimes]) raqms.pressures = hau.TZ_Array(pressures[selectedTimes, :]) raqms.temperatures = hau.TZ_Array(temperatures[selectedTimes, :]) raqms.ext_total = hau.TZ_Array(ext_total[selectedTimes, :]) raqms.ext_dust = hau.TZ_Array(ext_dust[selectedTimes, :]) raqms.ext_salt = hau.TZ_Array(ext_salt[selectedTimes, :]) raqms.relative_humidity = hau.TZ_Array(relative_humidity[selectedTimes, :]) raqms.u_vel = hau.TZ_Array(u_vel[selectedTimes, :]) raqms.v_vel = hau.TZ_Array(v_vel[selectedTimes, :]) raqms.model_level_alts = hau.TZ_Array(model_level_alts[selectedTimes, :] * 1000.0) raqms.times = times[selectedTimes] return raqms
def select_raqms_profile(soundings, request_time, requested_altitudes, offset=0): """selects sounding prior to request_time from soundings -- the sounding is returned in a Time_Z_Group as Z_arrays""" if soundings is None or soundings.times.size == 0: raise RuntimeError, "select_faqms_profile: No soundings for %s " %\ request_time import atmospheric_profiles.soundings.sounding_utilities as su sounding = hau.Time_Z_Group() sounding.altitudes = hau.Z_Array(requested_altitudes) max_alt = requested_altitudes[-1] max_bin = len(requested_altitudes) index = sum(soundings.times <= request_time) - 1 + offset if index < 0 or index >= len(soundings.times): return None #initialize variables for inclusion in T_Z_Group sounding.temps = hau.Z_Array(np.zeros((max_bin))) sounding.dew_points = hau.Z_Array(np.zeros(max_bin)) sounding.frost_points = hau.Z_Array(np.zeros(max_bin)) sounding.pressures = hau.TZ_Array(np.zeros(max_bin)) sounding.ext_total = hau.TZ_Array(np.zeros(max_bin)) sounding.ext_salt = hau.TZ_Array(np.zeros(max_bin)) sounding.wind_spd = hau.TZ_Array(np.zeros(max_bin)) sounding.wind_dir = hau.TZ_Array(np.zeros(max_bin)) #sounding.times is a single time at this point, however it will later be included #in a list of all the soundings used in this processing request. In order that it #be treated properly it must be defined as a T_Array sounding.times = hau.T_Array([soundings.times[index]]) sounding.latitude = hau.T_Array([soundings.latitude[index]]) sounding.longitude = hau.T_Array([soundings.longitude[index]]) #sounding.times=hau.T_Array([soundings.times[index]]) #interpolate model levels to lidar bin altitudes #temp=interpolate.splrep(soundings.model_level_alts[index,-1::-1] \ # ,soundings.temperatures[index,-1::-1]) #sounding.temps=interpolate.splev(sounding.altitudes,temp,der=0) temp=interpolate.splrep(soundings.model_level_alts[index,-1::-1] \ ,soundings.pressures[index,-1::-1]) sounding.pressures = interpolate.splev(sounding.altitudes, temp, der=0) sounding.temps=np.interp(sounding.altitudes \ ,soundings.model_level_alts[index,-1::-1] \ ,soundings.temperatures[index,-1::-1]) #calculate dew point at model levels for selected profile dew_pts=su.cal_dew_point(soundings.relative_humidity[index,:] \ ,soundings.temperatures[index,:]) frost_pts = su.cal_frost_point(dew_pts) #calculate wind speed and direction from u and v u_vel = soundings.u_vel[index, -1::-1] v_vel = soundings.v_vel[index, -1::-1] wind_spd = np.sqrt(u_vel**2 + v_vel**2) wind_dir = np.arctan(v_vel / u_vel) * 180.0 / np.pi for i in range(len(u_vel)): if (u_vel[i] < 0 and v_vel[i]) < 0: wind_dir[i] = 180.0 - wind_dir[i] elif (u_vel[i] > 0 and v_vel[i]) > 0: wind_dir[i] = 180.0 + wind_dir[i] elif u_vel[i] < 0: wind_dir[i] = 270.0 - wind_dir[i] else: wind_dir[i] = np.nan #interpolate to lidar bin altitudes sounding.frost_points=np.interp(sounding.altitudes \ ,soundings.model_level_alts[index,-1::-1],frost_pts[-1::-1]) sounding.dew_points=np.interp(sounding.altitudes \ ,soundings.model_level_alts[index,-1::-1],dew_pts[-1::-1]) sounding.ext_total=np.interp(sounding.altitudes\ ,soundings.model_level_alts[index,-1::-1]\ ,soundings.ext_total[index,-1::-1]) sounding.ext_salt=np.interp(sounding.altitudes\ ,soundings.model_level_alts[index,-1::-1]\ ,soundings.ext_salt[index,-1::-1]) sounding.ext_dust=np.interp(sounding.altitudes\ ,soundings.model_level_alts[index,-1::-1]\ ,soundings.ext_dust[index,-1::-1]) sounding.wind_dir = np.interp(sounding.altitudes \ ,soundings.model_level_alts[index,-1::-1],wind_dir) sounding.wind_spd = np.interp(sounding.altitudes \ ,soundings.model_level_alts[index,-1::-1],wind_spd) sounding.top = sounding.altitudes[-1] sounding.bot = sounding.altitudes[0] #plt.figure(1) #plt.plot(temperatures,altitudes,dew_points,altitudes) #plt.figure(2) #plt.plot(ext_total,altitudes) #plt.show() return sounding
def read(self): """ main read generator """ import hsrl.data_stream.processing_utilities as pu params=self.params firsttimeever=None intervalTime=None intervalEnd=None rawsrc=iter(self.rawsrc) #if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])): # params['timeres']=None #pure native end_time_datetime=params['finalTime'] #timemodoffset=time_mod(params['realStartTime'],params['timeres']) noframe='noframe' fullrange=False #if this is true, it will pad the start with any missing times. remainder=None cdf_to_hsrl = None preprocess_ave = None requested_times=None instrument=self.hsrl_instrument intcount=0 rs_mem = None #rs=None timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None for calv in self.cal_narr: if intervalTime is None: firsttimeever=calv['chunk_start_time'] intervalTime=calv['chunk_start_time'] intervalEnd=intervalTime chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset) #print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres'] if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None: if params['block_when_out_of_data']: if 'timeres' not in params or params['timeres'] is None: sleep(calv['rs_constants']['integration_time']) else: sleep(params['timeres'].total_seconds()) else: yield None #this is done to get out of here, and not get stuck in a tight loop continue while intervalTime<chunk_end_to_use: integration_time = calv['rs_constants']['integration_time'] doPresample=True #END init section if intervalEnd>chunk_end_to_use: print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime break else: intervalEnd=chunk_end_to_use #print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime'] print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem if True:#requested_times==None or requested_times.shape[0]>0: try: try: while rawsrc is not None: if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): break tmp=rawsrc.next() if hasattr(tmp,'rs_raw'): if rs_mem is not None: rs_mem.append(tmp.rs_raw) else: rs_mem=copy.deepcopy(tmp.rs_raw) if rs_mem is not None and rs_mem.times.shape>0: break else: rs_mem=None except StopIteration: print 'Raw HSRL stream is ended' rawsrc=None if rs_mem is None or rs_mem.times.size==0: rs_mem=None elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0] break else: intervalEnd=rs_mem.times[-1] print 'read in raw frame to mean',rs_mem,remainder if rawsrc is None: intervalEnd=chunk_end_to_use print 'trimmed ',rs_mem if timesource is not None: if timesource.isDone: break useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime) timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime))) print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd elif 'timeres' in params and params['timeres'] is not None: tmp=intervalTime useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet timevals=[] timevals.append(tmp) while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats. #print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds() tmp+=params['timeres'] timevals.append(tmp) #intervalEnd=tmp intcount+=len(timevals) if usePrebinnedTimes: intcount-=1 print 'Now %i intervals' % (intcount) timevals=hau.T_Array(timevals) else: print 'Using Native timing' timevals=None print ' new window is ', intervalTime, ' to ' , intervalEnd requested_times=timevals requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime] if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None: #if rawsrc is not None: print "not enough time to process" continue elif rawsrc is None and rs_mem is None and remainder is None: #chunk_end_to_use=intervalTime #continue #print '' break rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd ,params['min_alt'], params['max_alt'], requested_chunk_times , rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal'] , None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control , self.compute_stats,remainder=remainder) rs_mem=None if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0: rs_chunk=None if rs_chunk is None and rawsrc is None: break #print rs_chunk if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0: if fullrange and requested_chunk_times is not None: v=hau.Time_Z_Group(like=rs_chunk.rs_mean) v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]]) if v.times.size>0: rs_chunk.rs_mean.prepend(v) rs_chunk.calv=calv yield rs_chunk intervalTime=intervalEnd except Exception, e: print 'Exception occured in update_cal_and_process' print 'Exception = ',e print traceback.format_exc() if isinstance(e,(MemoryError,)): print 'Please Adjust Your Parameters to be more Server-friendly and try again' raise if not isinstance(e,(AttributeError,)): raise
def __call__(self, raw, cdf_attr): """input_translator convert raw netcdf variables into form used by the hsrl processing code and preforms pileup correction on photon counts""" if hasattr( raw, 'wfov_counts') and self.constants['wfov_type'] == 'molecular': raw.molecular_wfov_counts = raw.wfov_counts.copy() elif hasattr(raw, 'wfov_counts'): raw.combined_wfov_hi_counts = raw.wfov_counts.copy() if hasattr(raw, 'op_mode'): #extract i2 lock bit from operating mode #this will allow testing of bit even after averaging raw.i2_locked = (raw.op_mode[:].astype(int) & 4) / 4 if hasattr(raw, 'seeded_shots'): setattr( raw, 'delta_t', raw.seeded_shots[:, 0] / float(self.constants['laser_rep_rate'])) else: setattr(raw, 'delta_t', np.zeros([0])) #for i in np.arange(raw.times.size): # raw.times[i]-=timedelta(seconds=raw.delta_t[i]) if hasattr(raw, 'transmitted_energy'): # convert to mJ per preaveraged accumulation interval raw.transmitted_energy[:] = raw.transmitted_energy \ *self.constants['transmitted_energy_monitor'][0]\ +self.constants['transmitted_energy_monitor'][1]\ *raw.seeded_shots[:,0] #compute tranmitted power setattr(raw, 'transmitted_power', raw.transmitted_energy / raw.delta_t) if hasattr(raw, 'transmitted_1064_energy'): # convert to mJ per preaveraged accumulation interval raw.transmitted_1064_energy[:] = raw.transmitted_1064_energy \ *self.constants['transmitted_1064_energy_monitor'][0]\ +self.constants['transmitted_1064_energy_monitor'][1]\ *raw.seeded_shots[:,0] #compute tranmitted 1064 power setattr(raw, 'transmitted_1064_power', raw.transmitted_1064_energy / raw.delta_t) if hasattr(raw, 'filtered_energy'): if raw.filtered_energy.dtype == 'int32': raw.nonfiltered_energy = raw.nonfiltered_energy.astype( 'float64') raw.filtered_energy = raw.filtered_energy.astype('float64') if len(raw.filtered_energy.shape) == 1: raw.filtered_energy = raw.filtered_energy[:, np.newaxis] raw.nonfiltered_energy = raw.nonfiltered_energy[:, np.newaxis] raw.filtered_energy[raw.filtered_energy > 1e10] = np.NaN raw.nonfiltered_energy[raw.nonfiltered_energy > 1e10] = np.NaN if hasattr(raw, 'builduptime') and raw.builduptime.size > 0: raw.qswitch_buildup_time = raw.builduptime[:, 0] raw.min_qswitch_buildup_time = raw.builduptime[:, 1] raw.max_qswitch_buildup_time = raw.builduptime[:, 2] if hasattr(raw, 'superseedlasercontrollog'): raw.superseedlasercontrollog[ raw.superseedlasercontrollog > 1e10] = np.NaN if hasattr(raw,'energyRatioLockPoint') \ and raw.energyRatioLockPoint.size>0: if len(raw.energyRatioLockPoint.shape) == 2: raw.filtered_lockpoint = raw.energyRatioLockPoint[:, 0] raw.nonfiltered_lockpoint = raw.energyRatioLockPoint[:, 1] else: clarray = hau.T_Array(np.ones([raw.filtered_energy.shape[0]])) raw.filtered_lockpoint = clarray * raw.energyRatioLockPoint[0] raw.nonfiltered_lockpoint = clarray * raw.energyRatioLockPoint[ 1] if hasattr(raw, 'raw_analog_interferometertemperature'): thermistor_cal = self.constants['interferometer_temp_cal'] R = np.abs(raw.raw_analog_interferometertemperature / 0.000250) raw.interferometer_temp = 1/(thermistor_cal[0] + thermistor_cal[1] \ * np.log(R) + thermistor_cal[2] * np.log(R)** 3) - 273.15 ntemps = len(raw.interferometer_temp) if 0: #ntemps > 5: #do eleven point median filter temps = np.zeros((ntemps, 11)) temps[0:ntemps - 5, 0] = raw.interferometer_temp[5:] temps[0:ntemps - 4, 1] = raw.interferometer_temp[4:] temps[0:ntemps - 3, 2] = raw.interferometer_temp[3:] temps[0:ntemps - 2, 3] = raw.interferometer_temp[2:] temps[0:ntemps - 1, 4] = raw.interferometer_temp[1:] temps[:, 5] = raw.interferometer_temp temps[1:, 6] = raw.interferometer_temp[:ntemps - 1] temps[2:, 7] = raw.interferometer_temp[:ntemps - 2] temps[3:, 8] = raw.interferometer_temp[:ntemps - 3] temps[4:, 9] = raw.interferometer_temp[:ntemps - 4] temps[5:, 10] = raw.interferometer_temp[:ntemps - 5] raw.interferometer_temp = hau.T_Array(np.median(temps, 1)) else: raw.interferometer_temp = hau.T_Array(raw.interferometer_temp) if hasattr(raw, 'raw_analog_etalontemperature'): # convert etalon thermistor voltage to themistor resistance # T(degC) =1/( a + b(Ln R) + cLn R)^3)-273.15 #(Steinhart Hart equation) # Where: # a = 0.000862448 # b = 0.000258456 # c = 0.000000142 # and # R = (Volts ADC Reading)/(0.000250 amps) thermistor_cal = self.constants['interferometer_temp_cal'] R = np.abs(raw.raw_analog_etalontemperature / 0.000250) raw.etalon_temp = (hau.T_Array( np.array( (1.0 / (thermistor_cal[0] + thermistor_cal[1] * np.log(R) + thermistor_cal[2] * np.log(R)**3) - 273.15), dtype=np.float32, ndmin=1))) if hasattr(raw, 'raw_analog_coolanttemperature'): # convert coolant thermistor voltage to themistor resistance # T(degC) =1/( a + b(Ln R) + cLn R)^3)-273.15 #(Steinhart Hart equation) # Where: # a = 0.000862448 # b = 0.000258456 # c = 0.000000142 # and # R = (Volts ADC Reading)/(0.000250 amps) thermistor_cal = self.constants['interferometer_temp_cal'] R = np.abs(raw.raw_analog_coolanttemperature / 0.000250) raw.coolant_temperature = \ (hau.T_Array(np.array((1.0 / (thermistor_cal[0] + thermistor_cal[1] * np.log(R) + thermistor_cal[2] * np.log(R)** 3) - 273.15),dtype=np.float32,ndmin=1))) if hasattr(raw, 'telescope_pointing'): if not hasattr(raw, 'telescope_locked'): setattr(raw, 'telescope_locked', np.ones_like(raw.telescope_pointing)) raw.telescope_pointing = raw.telescope_pointing.astype('float64') raw.telescope_pointing[raw.telescope_locked == 0] = .5 #roll component of telescope mounting angle in degrees measured relative #to platform (zero degrees = vertical) #roll angle is + in clockwise direction if not hasattr(raw, 'telescope_roll_angle_offset'): setattr(raw, 'telescope_roll_angle_offset', np.ones_like(raw.telescope_pointing)) raw.telescope_roll_angle_offset[:] = self.constants[ 'telescope_roll_angle_offset'] raw.telescope_roll_angle_offset[raw.telescope_pointing == 0] = \ 180.0 - self.constants['telescope_roll_angle_offset'] if hasattr(raw, 'raw_analog_telescope_temperature'): # convert coolant thermistor voltage to themistor resistance # T(degC) =1/( a + b(Ln R) + cLn R)^3)-273.15 #(Steinhart Hart equation) # Where: # a = 0.000862448 # b = 0.000258456 # c = 0.000000142 # and # R = (Volts ADC Reading)/(0.000250 amps) thermistor_cal = self.constants['interferometer_temp_cal'] R = np.abs(raw.raw_analog_telescope_temperature / 0.000250) raw.telescope_temperature = \ (hau.T_Array(np.array((1.0 / (thermistor_cal[0] + thermistor_cal[1] * np.log(R) + thermistor_cal[2] * np.log(R)** 3) - 273.15),dtype=np.float32,ndmin=1))) if hasattr(raw,'OutgoingBeamPosition_centermass')\ and raw.OutgoingBeamPosition_centermass.size > 0 : raw.cg_xs = raw.OutgoingBeamPosition_centermass[:, 0] raw.cg_ys = raw.OutgoingBeamPosition_centermass[:, 1] if hasattr(raw,'OutgoingBeamPosition2_centermass')\ and raw.OutgoingBeamPosition2_centermass.size > 0 : raw.cg_xs2 = raw.OutgoingBeamPosition2_centermass[:, 0] raw.cg_ys2 = raw.OutgoingBeamPosition2_centermass[:, 1] if hasattr(raw,'interferometer_intensity') \ and raw.interferometer_intensity.size > 0: interf_peak = \ self.constants['interferometer_spectral_peak'] phase_to_freq = \ self.constants['interferometer_phase_to_freq'] npixels = self.constants['interferometer_fft_npixels'] xform = np.fft.rfft(raw.interferometer_intensity[:, :npixels], axis=1) tmp = np.concatenate( ([self.unwrap_firstangle], np.angle(xform[:, interf_peak]))) newlast = tmp[-1] tmp = np.unwrap(tmp) tmp = (self.unwrap_firstangle_atmagnitude - tmp[0]) + tmp if np.isfinite(tmp[-1]): self.unwrap_firstangle_atmagnitude = tmp[-1] self.unwrap_firstangle = newlast raw.interf_freq = tmp[1:] raw.interf_freq = hau.T_Array(-raw.interf_freq * phase_to_freq[0]) #compute temperature compensated interferometer freq if 0 and hasattr(raw,'interferometer_temp') \ and hasattr(raw,'interf_freq')\ and self.constants.has_key('interf_temp_coef'): raw.tcomp_interf_freq = raw.interf_freq \ - (raw.interferometer_temp-raw.interferometer_temp[0])\ * self.constants['interf_temp_coef']*1e9 for imagetime in ('interferometer_snapshot_time', 'outgoingbeamalignment_snapshot_time', 'overhead_snapshot_time', 'snowscope_snapshot_time'): if hasattr(raw, imagetime): setattr( raw, imagetime, hru.convert_to_python_times( getattr(raw, imagetime)[np.newaxis, :])) #replace missing values witn NaN's if hasattr(raw, 'seedvoltage'): raw.seedvoltage[raw.seedvoltage > 100] = np.NaN if hasattr(raw, 'latitude'): raw.latitude[raw.latitude > 100] = np.NaN if hasattr(raw, 'longitude'): raw.longitude[raw.longitude > 200] = np.NaN if hasattr(raw, 'laserpowervalues') and raw.laserpowervalues.size > 0: raw.laser_current = raw.laserpowervalues[:, 0] raw.laser_voltage = raw.laserpowervalues[:, 1] if raw.laserpowervalues.shape[1] > 2: raw.laser_current_setpoint = raw.laserpowervalues[:, 2] raw.laser_diode_temp = raw.laserpowervalues[:, 3] raw.laser_diode_temp_setpoint = raw.laserpowervalues[:, 4] if raw.laserpowervalues.shape[1] > 6: raw.ktp_temp = raw.laserpowervalues[:, 5] raw.ktp_temp_setpoint = raw.laserpowervalues[:, 6] #remove spikes from tcs records for fiel in ('tcsopticstop_', 'tcsoptics_', 'tcstelescope_', 'thermal1_', 'thermal2_', 'tcsaft_', 'tcsfore_'): for f in vars(raw).keys(): if f.startswith(fiel): v = getattr(raw, f) v[v > 1000] = np.NaN if hasattr(raw,'one_wire_temperatures') \ and raw.one_wire_temperatures.size >0 : #raw.one_wire_attrib = cdf_attr['one_wire_temperatures'] raw.one_wire_attrib = [] [ntime, ntemps] = raw.one_wire_temperatures.shape for i in range(ntemps): string = 'field' + str(i) + '_name' try: raw.one_wire_attrib.append( cdf_attr['one_wire_temperatures'][string]) except KeyError: print "Couldn't find attribute for ", string raw.one_wire_attrib.append(None) #remove spikes of 1e37 that appear in temperatues raw.one_wire_temperatures[raw.one_wire_temperatures>1000.]\ =np.NaN if hasattr(raw, 'RemoveLongI2Cell'): servo_range = cdf_attr['RemoveLongI2Cell']['range'] raw.i2_cell_out = np.abs(raw.RemoveLongI2Cell-servo_range[1]) \ > np.abs(raw.RemoveLongI2Cell-servo_range[0]) if hasattr(raw, 'RemoveLongI2ArCell'): servo_range = cdf_attr['RemoveLongI2ArCell']['range'] raw.i2a_cell_out = np.abs(raw.RemoveLongI2ArCell-servo_range[1]) \ > np.abs(raw.RemoveLongI2ArCell-servo_range[0]) if hasattr(raw, 'shot_count'): if raw.shot_count.size > 0: raw.shot_count = raw.shot_count[:, 0] else: raw.shot_count = raw.shot_count.reshape([0]) if hasattr(raw, 'seeded_shots'): if raw.seeded_shots.size > 0: raw.seeded_shots = raw.seeded_shots[:, 0] else: raw.seeded_shots = raw.seeded_shots.reshape([0]) #extract average dark counts from profiles and add dark counts to raw #dark count extracted from 'first_bins' or 'last_bins' as specified in constants #pu.extract_dark_count(raw,self.constants) #moved to after PILEUP 20140805 #extract cal pulse from light scattered as laser pulse exits system #and place in raw #pu.extract_cal_pulse(raw,self.constants) if hasattr(raw, 'l3cavityvoltage') and raw.l3cavityvoltage.size > 0: raw.piezo_voltage_ave = raw.l3cavityvoltage[:, 0] raw.piezo_voltage_min = raw.l3cavityvoltage[:, 1] raw.piezo_voltage_max = raw.l3cavityvoltage[:, 2] if hasattr(raw, 'l3locking_stats' ) and 'l3slope_to_frequency' in self.constants: raw.l3frequency_offset = raw.l3locking_stats.copy() for x in range(0, 3): raw.l3frequency_offset[:, x] = np.polyval( self.constants['l3slope_to_frequency'], raw.l3locking_stats[:, x]) if hasattr(raw, 'GPS_MSL_Alt'): #replace and spikes in a altitude with base altitude #this allows the code to run but produces garbage data if np.any(raw.GPS_MSL_Alt > 20000.0): raw.GPS_MSL_Alt[raw.GPS_MSL_Alt > 20000.0] = \ self.constants['lidar_altitude'] if hasattr(raw, 'roll_angle'): if anynan(raw.roll_angle): raw.roll_angle[np.isnan(raw.roll_angle)] = 0.0 if anynan(raw.pitch_angle): raw.pitch_angle[np.isnan(raw.pitch_angle)] = 0.0 if hasattr(raw, 'opticalbenchairpressure'): #convert psi to mb #print 'pre--opticalbenchairpressure ',raw.opticalbenchairpressure.shape raw.opticalbenchairpressure=hau.T_Array(np.array((raw.opticalbenchairpressure\ *self.constants['optical_bench_air_pressure_cal']),ndmin=1,dtype=np.float32)) #print 'optical_bench_air_pressure.size',raw.opticalbenchairpressure.size,raw.times.size if hasattr(raw, 'chillertemperature') and raw.chillertemperature.size > 0: raw.chiller_temp = raw.chillertemperature[:, 0] raw.chiller_setpt = raw.chillertemperature[:, 1] if hasattr(raw, 'etalon_pressure'): raw.etalon_pressure = raw.etalon_pressure * self.constants[ 'etalon_pressure'] if hasattr(raw, 'qw_rotation_angle'): #convert gv quarter wave plate rotation angle from radians to deg raw.qw_rotation_angle = raw.qw_rotation_angle * 180.0 / np.pi if hasattr(raw, 'GPS_MSL_Alt') or self.constants['installation'] in ( 'airborne', 'shipborne'): #do quality check on aircraft GPS and INS data pu.gps_quality_check(raw, self.constants) if hasattr(raw, 'molecular_counts'): for k, v in vars(raw).items(): if '_counts' in k: if raw.molecular_counts.shape[1] != v.shape[1]: print 'raw field ', k, ' is messed up. size difference' tmp = copy.deepcopy(raw.molecular_counts) minidx = min(tmp.shape[1], v.shape[1]) tmp[:, :] = 0 tmp[:, :minidx] = v[:, :minidx] setattr(raw, k, tmp) #do pileup correction on signals before any averaging is done pu.pileup_correction(raw, self.constants, self.corr_adjusts) #extract average dark counts from profiles and add dark counts to raw #dark count extracted from 'first_bins' or 'last_bins' as specified in constants if hasattr(raw, 'molecular_counts'): pu.extract_dark_count( raw, self.constants) #relocated from above 20140805 #extract cal pulse from light scattered as laser pulse exits system #and place in raw pu.extract_cal_pulse(raw, self.constants) if 0: import hsrl.simulation.rb_simulation as sim #rescale for new energy sim.rb_simulation(raw, self.constants) #redo dark count pu.extract_dark_count(raw, self.constants)
def generate_ave_profiles(rs_mean, qc_mask, rs_Cxx, rs_constants, processing_defaults, sel_telescope_dir, corr_adjusts, old_profiles=None): """create average of profiles dependent on telescope pointing direction , telescope pointing may be 'all', 'zenith', or 'nadir' """ try: [ntimes, nalts] = rs_mean.molecular_counts.shape except AttributeError: #raise RuntimeError, \ # "molecular counts missing" ntimes = 0 nalts = 500 #if rs_mean.times.shape[0] == 0: #raise RuntimeError, \ # "times missing" # return old_profiles if qc_mask is not None and processing_defaults.get_value( 'averaged_profiles', 'apply_mask'): #make mask array with NaN's for array elements where bit[0] of qc_mask==0 #all other elements of mask = 1 mask = (np.bitwise_and(qc_mask, 1)).astype('float') #mask is float to allow use of NaN values mask[mask == 0] = np.NaN print 'qc_mask applied to time averaged profiles vs altitude' else: #set mask == 1 mask = None print 'qc_mask has not been applied to time averaged profiles' # most of the time we need to generate some/all of these profiles, # because users are plotting them # if this is too time-consuming, we'll generate just the ones necessary for the # user selected plots. indices = np.arange(rs_mean.times.shape[0]) indices = (indices >= 0) #boolean now if ntimes == 0 and len(indices) == 0: return old_profiles #average only those profiles occuring when locked to the i2 line #and not in cal_scan mode #op_mode = rs.rs_raw.op_mode.astype(int) #not_cal_scan = (~op_mode[:] & 32)/32 #locked_to_i2 = ((op_mode[:] & 4)/4) #bits = np.zeros((len(rs.rs_raw.times),len(bit_value)+1)) #opmode=rs.rs_raw.op_mode.astype('uint32') #for i in range(len(bit_value)): # bits[:,i]=(i+1)*(opmode[:] & bit_value[i])/bit_value[i] #this dictionary maps the long name to the shorthand used with dark_counts variables # the keys are used here as a list of all possible channels too channel_shorthand = dict(molecular_counts='mol', combined_hi_counts='c_hi', combined_lo_counts='c_lo', combined_wfov_counts='c_wfov', combined_1064_counts='combined_1064', molecular_wfov_counts='m_wfov', molecular_i2a_counts='mol_i2a', cross_pol_counts='c_pol', combined_counts='comb') #,combined_1064_counts='combined_1064' #select desired telescope pointing direction for profiles if (rs_constants['installation'] == 'ground' or rs_constants['installation'] == 'shipborne' or sel_telescope_dir == 'all'): print 'Selecting all telescope pointing directions for profiles' if processing_defaults is not None and processing_defaults.get_value( 'averaged_profiles', 'apply_mask'): indices[rs_mean.i2_locked <= 0.99] = False elif sel_telescope_dir == 'zenith': #if telescope pointing exists, limit to selected pointing direction if rs_mean.telescope_pointing.shape[0]: print 'Selecting only zenith pointing data for profiles' indices[rs_mean.telescope_pointing != 1] = False rs_mean.telescope_pointing = rs_mean.telescope_pointing[indices] else: print 'Warning--using all shots--no telescope pointing direction in data file' elif sel_telescope_dir == 'nadir': #if telescope pointing exists, limit to selected pointing direction if rs_mean.telescope_pointing.shape[0]: print 'Selecting only nadir pointing data for profiles' indices[rs_mean.telescope_pointing != 0] = False else: print 'Warning--using all shots--no telescope pointing direction in data file' else: raise RuntimeError, \ "Unrecognized value '%s' for telescope pointing dir--valid(all, zenith,nadir)" \ % (sel_telescope_dir) #this is meant to filter out chunks from rs_mean that are calibration intervals if hasattr(rs_mean, 'op_mode'): indices[np.bitwise_and(rs_mean.op_mode, 16) != 0] = False indices = np.arange(indices.shape[0])[indices] #back to indexes profiles = hau.Time_Z_Group(can_append=False) profiles.hist = hau.Time_Z_Group() ft = None tc = 0 #tv=0 lt = None if old_profiles is not None: #total_seeded_shots=total_seeded_shots+profiles.hist.total_seeded_shots ft = old_profiles.hist.ft tc = old_profiles.hist.tc #tv=old_profiles.hist.tv lt = old_profiles.hist.lt if len(indices) > 0: if ft is None: ft = rs_mean.times[indices][0] lt = rs_mean.times[indices][-1] + timedelta( seconds=rs_mean.delta_t[indices][-1] if not np.isnan(rs_mean.delta_t[indices][-1]) else 0) for x in indices: if rs_mean.times[x] is not None: tc = tc + 1 #tv=tv+(rs_mean.times[x]-ft).total_seconds() if tc > 0: profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ]) profiles.start = ft profiles.width = lt - ft profiles.delta_t = hau.T_Array([profiles.width.total_seconds()]) else: profiles.times = hau.T_Array([]) profiles.start = ft profiles.width = timedelta(seconds=0) profiles.delta_t = hau.T_Array([]) profiles.start_time = ft profiles.end_time = lt profiles.hist.ft = ft profiles.hist.tc = tc #profiles.hist.tv=tv profiles.hist.lt = lt #profiles.hist.total_seeded_shots=total_seeded_shots if rs_mean is not None and hasattr(rs_mean, 'msl_altitudes'): profiles.msl_altitudes = rs_mean.msl_altitudes elif hasattr(old_profiles, 'msl_altitudes'): profiles.msl_altitudes = old_profiles.msl_altitudes if rs_mean is not None and hasattr(rs_mean, 'geo_corr'): profiles.geo_corr = rs_mean.geo_corr elif hasattr(old_profiles, 'geo_corr'): profiles.geo_corr = old_profiles.geo_corr if rs_constants['installation'] == 'airborne' and len(indices) > 0: if hasattr(rs_mean, 'GPS_MSL_Alt'): profiles.min_GPS_MSL_Alt = np.min(rs_mean.GPS_MSL_Alt[indices]) profiles.mean_GPS_MSL_Alt = nanmean(rs_mean.GPS_MSL_Alt[indices]) profiles.max_GPS_MSL_Alt = np.max(rs_mean.GPS_MSL_Alt[indices]) profiles.telescope_pointing = np.zeros(1) if hasattr(rs_mean, 'telescope_pointing'): if (rs_mean.telescope_pointing > .95).all(): profiles.telescope_pointing[0] = 1 elif (rs_mean.telescope_pointing < .05).all(): profiles.telescope_pointing[0] = 0 else: profiles.telescope_pointing[0] = np.NaN accumulate(profiles, old_profiles, rs_mean, indices, 'seeded_shots', pref='mean_', filler=hau.T_Array([0])) total_seeded_shots = profiles.mean_seeded_shots profiles.seeded_shots = total_seeded_shots.copy() print 'Total seeded shots for profile =', total_seeded_shots accumulate(profiles, old_profiles, rs_mean, indices, 'transmitted_1064_energy', total_seeded_shots) accumulate(profiles, old_profiles, rs_mean, indices, 'transmitted_energy', total_seeded_shots) # create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke for chan in channel_shorthand.keys(): accumulate(profiles, old_profiles, rs_mean, indices, chan, total_seeded_shots, mask) #compute inverted profiles from mean count profiles if rs_Cxx is not None and hasattr(profiles, 'molecular_counts'): profiles.inv = cu.hsrl_inversion(profiles, rs_Cxx, rs_constants, corr_adjusts, processing_defaults) elif hasattr(old_profiles, 'inv'): profiles.inv = old_profiles.inv #adds raw_color_ratio to profiles.inv if hasattr(profiles,'inv') and hasattr(profiles,'combined_counts') \ and hasattr(profiles,'combined_1064_counts'): if 0: print 'profiles' print dir(profiles) print 'inv' print dir(profiles.inv) import matplotlib.pylab as plt plt.figure(3333) plt.plot( profiles.combined_counts[0, :], profiles.inv.msl_altitudes, 'r', profiles.cross_pol_counts[0, :], profiles.inv.msl_altitudes, 'g', profiles.combined_counts[0, :] + rs_constants['combined_to_cross_pol_gain_ratio'] * profiles.cross_pol_counts[0, :], 'c', profiles.combined_1064_counts[0, :], profiles.inv.msl_altitudes, 'k') plt.grid(True) ax = plt.gca() ax.set_xscale('log') profiles.inv.raw_color_ratio = cu.compute_raw_color_ratio( profiles, rs_Cxx, rs_constants, corr_adjusts) if 0: plt.figure(3334) plt.plot(profiles.inv.raw_color_ratio[0, :], profiles.inv.msl_altitudes, 'c') plt.grid(True) ax = plt.gca() ax.set_xscale('log') #generate klett profiles if requested if processing_defaults is not None and processing_defaults.get_value( 'klett', 'enable'): ref_altitude = processing_defaults.get_value('klett', 'ref_altitude') * 1000.0 if ref_altitude < profiles.inv.msl_altitudes[0] \ or ref_altitude > profiles.inv.msl_altitudes[-1] : print print print 'WARNING---klett ref altitutde=', ref_altitude, ' is not in requested altitudes' print 'no klett profile retrieval attempted ' print else: if hasattr(profiles, 'combined_1064_counts'): profiles.inv.beta_a_1064_backscat_klett = lu.compute_klett_backscatter( profiles.combined_1064_counts, profiles.inv.beta_r_backscat / 16.0, profiles.inv.msl_altitudes, processing_defaults.get_value('klett', 'lidar_ratio_532'), ref_altitude) if hasattr(profiles, 'combined_counts'): profiles.inv.beta_a_532_backscat_klett = lu.compute_klett_backscatter( profiles.combined_counts, profiles.inv.beta_r_backscat, profiles.msl_altitudes, processing_defaults.get_value('klett', 'lidar_ratio_532'), ref_altitude) for chan in channel_shorthand.keys(): accumulate(profiles, old_profiles, rs_mean, indices, 'var_raw_' + chan, total_seeded_shots, mask) accumulate(profiles, old_profiles, rs_mean, indices, 'raw_' + chan, total_seeded_shots, mask) if processing_defaults is not None and processing_defaults.get_value( 'compute_stats', 'enable'): #kludge pf = hau.Time_Z_Group() for chan in channel_shorthand.keys(): if hasattr(profiles, 'sum_var_raw_' + chan): setattr(pf, chan, getattr(profiles, 'sum_' + chan)) setattr(pf, 'var_raw_' + chan, getattr(profiles, 'sum_var_raw_' + chan)) if rs_Cxx is not None and hasattr(profiles, 'inv'): pu.compute_photon_statistics(pf, profiles.inv, rs_Cxx, rs_constants) if rs_Cxx is not None and hasattr(profiles, 'inv'): [profiles.inv.optical_depth, profiles.inv.optical_depth_aerosol , profiles.inv.mol_norm_index,profiles.inv.mol_ref_aod] = \ lu.compute_optical_depth(profiles.inv.Nm \ ,profiles.inv.beta_r_backscat\ ,profiles.msl_altitudes\ ,processing_defaults\ ,rs_constants ,profiles.inv.telescope_pointing if hasattr(profiles.inv,'telescope_pointing') else None) #add 1064 aerosol backscatter and color ratio to profiles if hasattr(profiles, 'combined_counts') and hasattr( profiles, 'combined_1064_counts'): cu.compute_1064_aerosol_backscatter(profiles, profiles.inv, processing_defaults, rs_constants, corr_adjusts) cu.compute_color_ratio(profiles.inv) if processing_defaults.get_value('extinction_processing', 'filter_type') == 'savitzky_golay': od_threshhold = processing_defaults.get_value( 'extinction_processing', 'od_threshhold') z_window_width = processing_defaults.get_value( 'extinction_processing', 'alt_window_length') order = processing_defaults.get_value('extinction_processing', 'polynomial_order') min_filter_alt = processing_defaults.get_value( 'extinction_processing', 'min_alt') if min_filter_alt < profiles.msl_altitudes[0]: min_filter_alt = profiles.msl_altitudes[0] adaptive = processing_defaults.get_value('extinction_processing', 'adaptive') t_window_width = 0.0 if profiles.inv.times.size == 0: pass elif hasattr(rs_mean, 'telescope_pointing'): profiles.inv = filtered_extinction( profiles.inv, profiles.msl_altitudes, min_filter_alt, od_threshhold, t_window_width, z_window_width, order, adaptive, rs_mean.telescope_pointing) else: profiles.inv = filtered_extinction( profiles.inv, profiles.msl_altitudes, min_filter_alt, od_threshhold, t_window_width, z_window_width, order, adaptive) else: bin_delta = processing_defaults.get_value('extinction_processing', 'bin_delta') bin_delta = int(bin_delta) pts_to_ave = processing_defaults.get_value('extinction_processing', 'ext_pts_to_ave') if hasattr(rs_mean, 'telescope_pointing'): profiles.inv = pu.compute_extinction( profiles.inv, profiles.msl_altitudes, bin_delta, pts_to_ave, rs_mean.telescope_pointing) else: profiles.inv = pu.compute_extinction(profiles.inv, profiles.msl_altitudes, bin_delta, pts_to_ave) # raw profiles--ave photons/bin/laser_pulse #for chan in channel_shorthand.keys(): # accumulate(profiles,old_profiles,rs_raw,raw_indices,chan,raw_total_seeded_shots,pref='raw_') if 0: import matplotlib.pylab as plt plt.figure(898989) plt.plot(np.nanmean(rs_mean.combined_1064_counts, 0), rs_mean.msl_altitudes, 'k', np.nanmean(rs_mean.combined_counts, 0), rs_mean.msl_altitudes, 'r') ax = plt.gca() ax.set_xscale('log') # dark corrected raw profiles for chan, shorthand in channel_shorthand.items(): dcchan = shorthand + '_dark_counts' correc = 'dc_' + chan source = 'raw_' + chan if accumulate(profiles, old_profiles, rs_mean, indices, dcchan, total_seeded_shots, extravars=[correc]): if hasattr(profiles, source): #print 'Applying dark count from raw frame to raw counts*******' setattr(profiles, correc, getattr(profiles, source) - getattr(profiles, dcchan)) elif hasattr(old_profiles, correc): setattr(profiles, correc, getattr(old_profiles, correc)) print 'Copying corrected counts because source doesn\'t exist???? WARNING ***' if hasattr(old_profiles, source): setattr(profiles, source, getattr(old_profiles, source)) else: raise RuntimeError( 'Something is wrong with input channels. BUG!') if processing_defaults is not None: profiles.mol_norm_alt = processing_defaults.get_value( 'mol_norm_alt', 'meters') return profiles
def compute_optical_depth(Nm, beta_r_backscat, msl_altitudes, processing_defaults, constants, telescope_pointing=None): """uses Nm and beta_r_backscat to compute the optical depth profile with the optical depth profile set to zero at an alititude given by 'mol_norm_alt' which must provided in meters. returns: od = total optical depth od_aerosol = aerosol optical depth mol_norm_index = bin number at which optical depth is normalized to zero mol_ref_aod = estimated optical depth at altitudes[mol_norm_index]""" #mol_od_between_lidar_and_norm_alt = 0.0 mol_norm_alt = processing_defaults.get_value('mol_norm_alt', 'meters') mol_norm_index = len(msl_altitudes[msl_altitudes <= mol_norm_alt]) if processing_defaults.enabled('molecular_smooth'): window_offset = np.float( processing_defaults.get_value('molecular_smooth', 'window_length')) / 2.0 else: window_offset = 0.0 if ('installation' not in constants or constants['installation'] == 'ground' or constants['installation'] == 'shipborne')\ and mol_norm_alt < (constants['lidar_altitude']+150.0 + window_offset): mol_norm_alt = constants['lidar_altitude'] + 150. + window_offset mol_norm_index = len(msl_altitudes[msl_altitudes <= mol_norm_alt]) lidar_alt_index = len( msl_altitudes[msl_altitudes <= constants['lidar_altitude']]) print 'requested molecular norm altitude too low-- reset at ', mol_norm_alt, ' m' processing_defaults.set_value('mol_norm_alt', 'meters', mol_norm_alt) od = hau.TZ_Array(np.NaN * np.zeros_like(Nm)) indices = np.arange(len(msl_altitudes)) if processing_defaults.enabled('molecular_smooth'): window = processing_defaults.get_value('molecular_smooth', 'window_length') if mol_norm_alt < constants['lidar_altitude'] + 150 + window / 2.0: mol_norm_alt = constants['lidar_altitude'] + 150 + window / 2.0 mol_norm_index = len(msl_altitudes[msl_altitudes <= mol_norm_alt]) print ' ' print 'Warning:******requested mol_normalization_altitude not within acquired data' print ' setting normalization altitude = ', msl_altitudes[ mol_norm_index] print ' ' if mol_norm_index >= len(msl_altitudes): mol_norm_index = len(msl_altitudes) - 1 print ' ' print 'Warning:******requested normalization altitude higher than requested range' print ' setting normalization altitude =', msl_altitudes[ mol_norm_index] if mol_norm_index <= Nm.shape[1]: if 0: print print print 'alts', msl_altitudes.shape print 'beta_r', beta_r_backscat.shape print 'Nm', Nm.shape print 'norm_alt', msl_altitudes[mol_norm_index] print import matplotlib.pylab as plt plt.figure(2000) plt.plot( msl_altitudes, np.nanmean(Nm, 0) * beta_r_backscat[mol_norm_index] / np.nanmean(Nm[:, mol_norm_index], 0), msl_altitudes, beta_r_backscat) plt.ylabel('altitude') ax = plt.gca() ax.set_yscale('log') plt.grid(True) time_vec = np.ones_like(Nm[:, 0]) bin_vec = np.ones_like(beta_r_backscat) beta_r_array = (time_vec[:, np.newaxis] * beta_r_backscat[np.newaxis, :]) Nm_norm_array = Nm[:, mol_norm_index] Nm_norm_array = Nm_norm_array[:, np.newaxis] * bin_vec[np.newaxis, :] prelog_od = Nm[:, :] / ( beta_r_array * Nm_norm_array) * beta_r_backscat[mol_norm_index] prelog_od[np.isnan(prelog_od)] = 0 prelog_od[prelog_od <= 0.0] = np.NaN od = -0.5 * np.log(prelog_od) if telescope_pointing is not None: tmpod = od od = od.copy() od[:, :] = np.NAN od[telescope_pointing < 0.1, :] = -1.0 * tmpod[ telescope_pointing < 0.1, :] od[telescope_pointing > 0.9, :] = tmpod[ telescope_pointing > 0.9, :] dz = msl_altitudes[1] - msl_altitudes[0] mol_od = 8.0 * np.pi * np.cumsum(beta_r_backscat) * dz / 3.0 mol_od = mol_od - mol_od[mol_norm_index] mol_od_array = time_vec[:, np.newaxis] * mol_od[np.newaxis, :] od_aerosol = od - mol_od_array else: od[:, :] = np.NaN od_aerosol[:, :] = od.copy() print ' ' print '*******requested od_normalization altitude not within acquired data' print ' ' #compute optical depth below mol_norm_index #extrapolate from just above mol_norm_index to estimate unmeasured optical depth if dz >= 100: n_bins = 1 else: n_bins = int(100.0 / dz + 1) norm_alt_range = msl_altitudes[mol_norm_index] - constants['lidar_altitude'] mol_ref_aod = (od[:, mol_norm_index + n_bins] - od[:, mol_norm_index]) * (norm_alt_range / (dz * n_bins)) mol_ref_aod -= (mol_od_array[:,mol_norm_index + n_bins] - mol_od_array[:,mol_norm_index])\ *norm_alt_range/(dz*n_bins) mol_ref_aod = hau.T_Array(mol_ref_aod) od = hau.TZ_Array(od) od_aerosol = hau.TZ_Array(od_aerosol) return (od, od_aerosol, mol_norm_index, mol_ref_aod)
def process_spheroid_particle(rs_inv, rs_radar, particle_parameters, lambda_radar, entire_frame, sounding=None, size_dist=None): """ process_spheroid_particle(rs_inv,rs_radar,particle_parameters,lambda_radar,entire_frame, sounding=None,p180_water=None,size_dist=None): generate and return the particle measurements based on a given hsrl inverted data, radar (and its lambda), and particle parameters dictionary """ #create timez group and add heights rs_particle = hau.Time_Z_Group(rs_inv.times.copy(), timevarname='times', altname='heights') setattr(rs_particle, 'heights', rs_inv.msl_altitudes.copy()) setattr(rs_particle, 'delta_t', rs_inv.delta_t.copy()) #remove points where lidar signal is noise dominated by setting to #very small value. #clipped_beta_a_back=rs_inv.beta_a_backscat.copy() #if 0: #if hasattr(rs_inv,'std_beta_a_backscat'): # clipped_beta_a_back[clipped_beta_a_back<(2*rs_inv.std_beta_a_backscat)]=-np.inf #else: # print 'No std_beta_a_backscat statistics to filter particle measurements' #clipped_beta_a_back[np.logical_not(np.isfinite(rs_inv.beta_a_backscat))]=-np.inf; rs_particle.q_backscatter = np.NaN * np.zeros_like(rs_inv.beta_a_backscat) rs_particle.phase = np.zeros_like(rs_inv.beta_a_backscat) rs_particle.phase[ rs_inv.linear_depol > particle_parameters['h2o_depol_threshold']] = 1 rs_particle.phase[np.isnan(rs_inv.beta_a_backscat)] = np.NaN #set aspect ratio parameter for ice filled bins rs_particle.zeta = np.ones(rs_inv.beta_a_backscat.shape) rs_particle.zeta[rs_inv.linear_depol > particle_parameters['h2o_depol_threshold']] \ = particle_parameters['zeta'] print 'Extinction cross section for particle size calculations derived from ' \ ,particle_parameters['ext_source'] print 'Extinction due nonprecipitating aerosols = '\ ,particle_parameters['background_aerosol_bs'],'1/(m sr)' #store the mask field with the particle info rs_particle.qc_mask = rs_inv.qc_mask.copy() clipped_beta_a_backscat = rs_inv.beta_a_backscat.copy() copy_radar_backscatter = rs_radar.Backscatter.copy() #clipped_beta_a_backscat = copy_beta_a.copy() clipped_beta_a_backscat = clipped_beta_a_backscat \ - particle_parameters['background_aerosol_bs'] clipped_beta_a_backscat[clipped_beta_a_backscat < 0] = np.NaN #create an empty mode_diameter array rs_particle.mode_diameter = np.zeros_like(rs_inv.beta_a_backscat) #create an empty array for extinction--used only for particle calculations #bs_ratio_to_dmode will return extinction cross section in clipped_beta_a clipped_beta_a = np.NaN * np.zeros_like(rs_inv.beta_a_backscat) #water #compute mode diameter, extinction cross section, and backscatter efficeincy #from radar and lidar backscatter cross sections using mie theory and assumed #size distribution to predict mode diameter and q_backscatter for points #identified as water. if particle_parameters['radar_model'] == "Mie": rs_particle.mode_diameter, clipped_beta_a, rs_particle.q_backscatter\ ,rs_particle.dstar \ = size_dist.dmode_from_radar_lidar_mie(copy_radar_backscatter\ ,clipped_beta_a_backscat) else: #use only Rayliegh approx solution--particle_parameter['radar_model']=="Rayleigh" #mode diameter is computed for all points assuming everything is water #subsequent calculation will replace ice phase points. if particle_parameters['ext_source'] == 'ext': clipped_beta_a = rs_inv.extinction_aerosol.copy() elif particle_parameters['ext_source'] == 'bs/p180': clipped_beta_a = clipped_beta_a_backscat / particle_parameters[ 'p180_water'] else: print 'particle_parameters=', particle_parameters[ 'ext_source'], ' not supported' print 'in spheroid_particle_processing' print j clipped_beta_a[np.isnan(clipped_beta_a_backscat)] = np.NaN phase = np.zeros_like(rs_inv.beta_a_backscat) zeta = np.ones_like(rs_inv.beta_a_backscat) rs_particle.mode_diameter = size_dist.dmode_from_lidar_radar_rayleigh( rs_particle.mode_diameter, clipped_beta_a, copy_radar_backscatter, zeta, phase) #ice #compute extinction cross section for ice points using backscatter phase function clipped_beta_a[rs_particle.phase==1] = \ clipped_beta_a_backscat[rs_particle.phase==1]/particle_parameters['p180_ice'] zeta = np.zeros_like(clipped_beta_a) zeta[rs_particle.phase == 1] = particle_parameters['zeta'] #derive mode_diameter directly from radar backscatter and lidar extinction #cross sections for parts of image populated by ice rs_particle.mode_diameter[rs_particle.phase==1] = size_dist.dmode_from_lidar_radar_rayleigh(\ rs_particle.mode_diameter[rs_particle.phase==1] \ ,clipped_beta_a[rs_particle.phase==1],copy_radar_backscatter[rs_particle.phase==1]\ ,zeta[rs_particle.phase==1],rs_particle.phase[rs_particle.phase==1]) #creates effective_diameter_prime array from mode diameter rs_particle.effective_diameter_prime = \ size_dist.deff_prime(rs_particle.mode_diameter,rs_particle.phase,zeta) rs_particle.effective_diameter = size_dist.eff_diameter(\ rs_particle.mode_diameter,rs_particle.phase) rs_particle.mean_diameter = size_dist.mean_diameter(\ rs_particle.mode_diameter,rs_particle.phase) #compute liquid water content for bins with phase == 0 #bins with phase > 0 will return with NaN's if particle_parameters['radar_model'] == "Mie": rs_particle.LWC = su.liquid_water_content_mie( rs_particle.effective_diameter, clipped_beta_a, rs_particle.q_backscatter) rs_particle.p180_extinction = rs_inv.beta_a_backscat / rs_particle.q_backscatter else: if particle_parameters['ext_source'] == 'bs/p180': rs_particle.extinction_aerosol = rs_inv.beta_a_backscat / particle_parameters[ 'p180_water'] clipped_beta_a = rs_particle.extinction_aerosol.copy() else: clipped_beta_a = rs_inv.extinction_aerosol.copy() clipped_beta_a[np.isnan(clipped_beta_a_backscat)] = np.NaN rs_particle.LWC = np.NaN * np.zeros_like( rs_particle.effective_diameter) su.liquid_water_content_ext_approx(rs_particle.LWC, rs_particle.effective_diameter, clipped_beta_a, rs_particle.phase) rs_particle.p180_extinction = rs_inv.beta_a_backscat / particle_parameters[ 'p180_water'] rs_particle.extinction_aerosol = rs_inv.extinction_aerosol.copy() #compute ice water water content for bins with phase > 0 (kg/m^3) #return in LWC array bins with phase > 0 su.ice_water_content(rs_particle.LWC, rs_particle.effective_diameter, clipped_beta_a, rs_particle.phase) if hasattr(rs_radar, 'vertically_averaged_doppler'): rs_radar.raw_MeanDopplerVelocity = rs_radar.MeanDopplerVelocity.copy() motion_correction = np.transpose(rs_radar.vertically_averaged_doppler\ *np.transpose(np.ones_like(rs_radar.MeanDopplerVelocity))) rs_radar.MeanDopplerVelocity -= motion_correction if sounding != None: s_time = datetime.utcnow() rs_particle.rw_fall_velocity,rs_particle.mw_fall_velocity \ ,rs_particle.model_spectral_width,rs_particle.nw_fall_velocity\ = su.weighted_fall_velocity( rs_particle.mode_diameter ,particle_parameters ,rs_particle.zeta ,sounding.temps ,sounding.pressures ,rs_particle.phase,size_dist) print 'time for fall_velocity = ', datetime.utcnow() - s_time # compute precip rate (m/s) #rain_rate = 1/density # (m^3/kg) * LWC (kg/m^3) * fall_velocity (m/s) #using # Doppler velocity rs_particle.hsrl_radar_dv_precip_rate = # 0.001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity #using raw Doppler to give precip rate in m/s rs_particle.hsrl_radar_dv_precip_rate = 0.001 * rs_particle.LWC * rs_radar.MeanDopplerVelocity #remove points with Doppler folding rs_particle.hsrl_radar_dv_precip_rate[ rs_radar.MeanDopplerVelocity < -2.0] = np.NaN if sounding != None: #using modeled mass weighted velocity and dividing by the density of water, # 1000 kg/m^3, to give precip_rate in m/s rs_particle.hsrl_radar_precip_rate = 0.001 * rs_particle.LWC * rs_particle.mw_fall_velocity #retype all these fields to a proper TZ_Array #for f in ['effective_diameter_prime']: for f in vars(rs_particle).keys(): v = getattr(rs_particle, f) if isinstance(v, hau.Z_Array): continue #properly typed. continue elif isinstance(v, np.ndarray): if len(v.shape) == 2: setattr(rs_particle, f, hau.TZ_Array(v)) elif len(v.shape) == 1: print '1 Dimensional Variable ' + f + ' will be changed to a T_Array. FIXME!!!!' setattr(rs_particle, f, hau.T_Array(v)) else: raise RuntimeError( "I don't know what to type particle array " + f + " with dimensions " + repr(v.shape)) else: pass #not an array type. should be safe to ignore """ #compute fitting error of computed radar weighted fall velocity #to measured Doppler veleocity. temp = rs_radar.Backscatter.copy() temp[np.isnan(rs_radar.Backscatter)]=0.0 temp[rs_inv.msl_altitudes>400]=0.0 fitting_error = np.sqrt(nanmean((rs_particle.rw_fall_velocity[temp >1e-9] \ - rs_radar.MeanDopplerVelocity[temp >1e-9])**2)) print print rs_radar.times[0],' ---> ' ,rs_radar.times[-1] print 'fitting_error (m/s)= ',fitting_error print """ 'rs_particle--spher' print dir(rs_particle) return rs_particle
def profile(self, request_time, request_lat, request_long, offset=0): """returns a profile of temperature,pressure, dew_point, frost point at time, lat, and long extracted from the sounding_archive. If request_lat and request_long are empty they are ignored request_time = python datetime for reqested sounding request_lat = requested latitude for sounding--ignored if [] request_lon = requested longitude for sounding--ignored if []""" if self.soundings == None: return None print 'sounding_type= ', self.sounding_type, request_time if self.sounding_type == 'NOAA raob': temp_sounding = hau.Time_Z_Group() #find correct sounding profile out of archive file sounding = hau.selectByTime(self.soundings, request_time, offset=offset) if sounding is None: return None sounding.sounding_type = self.sounding_type sounding.sounding_id = sounding.station_id sounding.latitude = sounding.stalat sounding.longitude = sounding.stalong sounding.frost_points = cal_frost_point(sounding.dew_points) temp_sounding.type = self.sounding_type temp_sounding.times = sounding.times temp_sounding.altitudes = hau.Z_Array(sounding.altitudes) temp_sounding.temps = hau.TZ_Array(sounding.temps[np.newaxis, :]) temp_sounding.pressures = hau.TZ_Array( sounding.pressures[np.newaxis, :]) temp_sounding.dew_pts = hau.TZ_Array( sounding.dew_points[np.newaxis, :]) temp_sounding.frost_pts = hau.TZ_Array( sounding.frost_points[np.newaxis, :]) temp_sounding.wind_dir = hau.TZ_Array( sounding.wind_dir[np.newaxis, :]) temp_sounding.wind_spd = hau.TZ_Array( sounding.wind_spd[np.newaxis, :]) temp_sounding.wind_spd = hau.TZ_Array( sounding.wind_spd[np.newaxis, :]) temp_sounding.station_id = sounding.station_id temp_sounding.top = sounding.top sounding.times = sounding.times[0] #sounding.expire_time=sounding.expire_time[0] elif self.sounding_type == "time curtain" \ and self.sounding_id == "raqms": temp_sounding = hau.Time_Z_Group() sounding = raqms.select_raqms_profile(self.soundings,request_time \ ,self.requested_altitudes,offset=offset) # ,self.max_alt,self.alt_res) if sounding == None: return None sounding.station_id = self.sounding_id temp_sounding.type = self.sounding_type temp_sounding.times = sounding.times temp_sounding.latitude = hau.T_Array(sounding.latitude) temp_sounding.longitude = hau.T_Array(sounding.longitude) temp_sounding.altitudes = hau.Z_Array(sounding.altitudes) temp_sounding.temps = hau.TZ_Array(sounding.temps[np.newaxis, :]) temp_sounding.pressures = hau.TZ_Array( sounding.pressures[np.newaxis, :]) temp_sounding.dew_pts = hau.TZ_Array( sounding.dew_points[np.newaxis, :]) temp_sounding.frost_pts = hau.TZ_Array( sounding.frost_points[np.newaxis, :]) temp_sounding.wind_dir = hau.TZ_Array( sounding.wind_dir[np.newaxis, :]) temp_sounding.wind_spd = hau.TZ_Array( sounding.wind_spd[np.newaxis, :]) temp_sounding.ext_total = hau.TZ_Array( sounding.ext_total[np.newaxis, :]) temp_sounding.ext_salt = hau.TZ_Array( sounding.ext_salt[np.newaxis, :]) temp_sounding.ext_dust = hau.TZ_Array( sounding.ext_dust[np.newaxis, :]) temp_sounding.wind_spd = hau.TZ_Array( sounding.wind_spd[np.newaxis, :]) temp_sounding.station_id = sounding.station_id temp_sounding.top = sounding.top sounding.times = sounding.times[0] #set up time to read in new sounding (in datetime) #sounding.expire_time = sounding.times+timedelta(seconds=5*60) #expire time can not be less then request time--raqms file must be missing soundings #set expire_time 5 minutes after request time #if sounding.expire_time <= request_time: # print "****Warning----missing raqms sounding at ",request_time # print " using sounding from ",sounding.times # sounding.expire_time = request_time + timedelta(seconds=5*60) #remove any negative pressure values temp_sounding.pressures[temp_sounding.pressures < 0] = 0.0 #remove NaN's from pressure and temperature values temp_sounding.pressures[np.isnan(temp_sounding.pressures)] = 0.0 temp_sounding.temps[np.isnan(temp_sounding.temps)] = 1.0 return sounding
class dpl_hsrl_narr(dplkit.role.narrator.aNarrator): """ DPL HSRL Narrator Object. should only be created by dpl_hsrl object :param params: parameters dictionary :param cal_narr: calibration framestream narration object :param timesource: time axis generation source (could be synthetic or from another stream) :param rawsrc: raw data source. if not provided, will create a lot of it here :param lib: raw hsrl reading library object only used if rawsrc is not given :param zoo: raw hsrl zookeeper object only used if rawsrc is not given exposed attributes: - hsrl_cal_stream (calibration stream, can be used for parallel stream collection) exposed field type in chain: - hsrl.dpl.calibration.dpl_calibration_narr """ #def get_process_control(self): # return self.cal_narr.get_process_control() @property def hsrl_cal_stream(self): return self.cal_narr def __init__(self,params,cal_narr,timesource,rawsrc=None,compute_stats=0): super(dpl_hsrl_narr,self).__init__(None,cal_narr) #FIXME replace url with some manner of unique path #import dpl_calibration #self.dpl_calibration=dpl_calibration #self.provides=libr.provides self.compute_stats=compute_stats self.rawsrc=rawsrc self.params=params self.timesource=timesource self.cal_narr=cal_narr def __repr__(self): return 'DPL HSRL Framestream Narrator (%s)' % (self.params) def read(self): """ main read generator """ import hsrl.data_stream.processing_utilities as pu params=self.params firsttimeever=None intervalTime=None intervalEnd=None rawsrc=iter(self.rawsrc) #if params['timeres']!=None and params['timeres']<datetime.timedelta(seconds=float(self.cal_narr.hsrl_constants['integration_time'])): # params['timeres']=None #pure native end_time_datetime=params['finalTime'] #timemodoffset=time_mod(params['realStartTime'],params['timeres']) noframe='noframe' fullrange=False #if this is true, it will pad the start with any missing times. remainder=None cdf_to_hsrl = None preprocess_ave = None requested_times=None instrument=self.hsrl_instrument intcount=0 rs_mem = None #rs=None timesource=TimeSource.CompoundTimeGenerator(self.timesource) if self.timesource is not None else None for calv in self.cal_narr: if intervalTime is None: firsttimeever=calv['chunk_start_time'] intervalTime=calv['chunk_start_time'] intervalEnd=intervalTime chunk_end_to_use=calv['chunk_end_time']#-time_mod(calv['chunk_end_time'],params['timeres'],timemodoffset) #print 'old end',calv['chunk_end_time'],'vs new end',chunk_end_to_use,'mod base',params['timeres'] if calv['chunk_end_time']==calv['chunk_start_time'] and end_time_datetime is None: if params['block_when_out_of_data']: if 'timeres' not in params or params['timeres'] is None: sleep(calv['rs_constants']['integration_time']) else: sleep(params['timeres'].total_seconds()) else: yield None #this is done to get out of here, and not get stuck in a tight loop continue while intervalTime<chunk_end_to_use: integration_time = calv['rs_constants']['integration_time'] doPresample=True #END init section if intervalEnd>chunk_end_to_use: print 'Breaking calibration on endtime. proc ',intervalEnd,chunk_end_to_use,end_time_datetime break else: intervalEnd=chunk_end_to_use #print ' Absolute window is ', actualStartTime, ' to ' , params['finalTime'] print ' prior window was ', intervalTime, ' to ' , intervalEnd, 'terminating at ',chunk_end_to_use,rs_mem if True:#requested_times==None or requested_times.shape[0]>0: try: try: while rawsrc is not None: if rs_mem is not None and rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): break tmp=rawsrc.next() if hasattr(tmp,'rs_raw'): if rs_mem is not None: rs_mem.append(tmp.rs_raw) else: rs_mem=copy.deepcopy(tmp.rs_raw) if rs_mem is not None and rs_mem.times.shape>0: break else: rs_mem=None except StopIteration: print 'Raw HSRL stream is ended' rawsrc=None if rs_mem is None or rs_mem.times.size==0: rs_mem=None elif rs_mem.times[0]>=chunk_end_to_use and (end_time_datetime is None or chunk_end_to_use<end_time_datetime): print 'HSRL RAW skipping to next cal because of times',intervalTime,chunk_end_to_use,end_time_datetime,rs_mem.times[0] break else: intervalEnd=rs_mem.times[-1] print 'read in raw frame to mean',rs_mem,remainder if rawsrc is None: intervalEnd=chunk_end_to_use print 'trimmed ',rs_mem if timesource is not None: if timesource.isDone: break useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet inclusive=rawsrc is None and (end_time_datetime!=None and intervalEnd>=end_time_datetime) timevals=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=intervalEnd,inclusive=inclusive))#,inclusive=(end_time_datetime!=None and intervalEnd>=end_time_datetime))) print 'Now %i intervals %s' % (timevals.size-1, "INC" if inclusive else "NOINC"),intervalTime,intervalEnd elif 'timeres' in params and params['timeres'] is not None: tmp=intervalTime useMungedTimes=False #this is in case this code will need to start shifting bins (which assumes resolutions, and implies start and end of intervales, rather than explicitly to avoid overlap or underlap usePrebinnedTimes=True #this goes in the other direction of munged times to say provided times are timebin borders, and the last time is the end of the last, not included, and thus expected to be the first bin on the next window. thats the fully explicit way to describe the bins in code, but standards in describing bins to the user (a single time when the bin spans a range) is not defined yet timevals=[] timevals.append(tmp) while tmp<intervalEnd:# using python datetimes for making the axis is much much more precise than matplotlib floats. #print tmp, ' = ' , du.date2num(tmp) , ' = ' , (tmp-self.actualStartTime).total_seconds() tmp+=params['timeres'] timevals.append(tmp) #intervalEnd=tmp intcount+=len(timevals) if usePrebinnedTimes: intcount-=1 print 'Now %i intervals' % (intcount) timevals=hau.T_Array(timevals) else: print 'Using Native timing' timevals=None print ' new window is ', intervalTime, ' to ' , intervalEnd requested_times=timevals requested_chunk_times= requested_times#requested_times[requested_times >=intervalTime] if requested_chunk_times is not None and len(requested_chunk_times)<2 and rawsrc is not None: #if rawsrc is not None: print "not enough time to process" continue elif rawsrc is None and rs_mem is None and remainder is None: #chunk_end_to_use=intervalTime #continue #print '' break rs_chunk,remainder = pu.process_data( instrument, intervalTime, intervalEnd ,params['min_alt'], params['max_alt'], requested_chunk_times , rs_mem, calv['rs_Cxx'], calv['rs_constants'], calv['rs_cal'] , None , self.cal_narr.hsrl_corr_adjusts, self.cal_narr.hsrl_process_control , self.compute_stats,remainder=remainder) rs_mem=None if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size==0: rs_chunk=None if rs_chunk is None and rawsrc is None: break #print rs_chunk if rs_chunk is not None and hasattr(rs_chunk,'rs_mean') and rs_chunk.rs_mean is not None and rs_chunk.rs_mean.times.size>0: if fullrange and requested_chunk_times is not None: v=hau.Time_Z_Group(like=rs_chunk.rs_mean) v.times=hau.T_Array(requested_chunk_times[requested_chunk_times<rs_chunk.rs_mean.times[0]]) if v.times.size>0: rs_chunk.rs_mean.prepend(v) rs_chunk.calv=calv yield rs_chunk intervalTime=intervalEnd except Exception, e: print 'Exception occured in update_cal_and_process' print 'Exception = ',e print traceback.format_exc() if isinstance(e,(MemoryError,)): print 'Please Adjust Your Parameters to be more Server-friendly and try again' raise if not isinstance(e,(AttributeError,)): raise assert(remainder is None or remainder.times.size==0) if fullrange and end_time_datetime is not None and timesource is not None and (not timesource.isDone or requested_times is not None) and firsttimeever!=intervalTime:#either timesource indicates it wasn't run completely or requested times wasn't cleared requested_times=hau.T_Array(timesource.getBinsFor(starttime=intervalTime,endtime=end_time_datetime)) if requested_times is not None and len(requested_times)>1: print 'NO DATA to end from ',intervalTime,' to ',end_time_datetime #FIXME IS THIS USED? JPG 20160504 print "times to use are ",requested_times[:-1] rs= hau.Time_Z_Group() rs.rs_mean=hau.Time_Z_Group() rs.rs_mean.times=hau.T_Array(requested_times[:-1]).copy() #causes the time axis to be stored, but all others may be implied MISSING setattr(rs.rs_mean,'delta_t',hau.T_Array(np.zeros(rs.rs_mean.times.shape))) yield rs
def show_pars(instrument,display_defaults,rs,usetimes,usealts,figs): #toplevel if usealts==None and hasattr(rs,'heights'): usealts=rs.heights if usetimes==None and hasattr(rs,'times'): usetimes=rs.times #delta_t=np.zeros(rs.times.shape) #make vector of time differences between data points delta_t = hau.T_Array(np.array([(rs.times[x+1]-rs.times[x]).total_seconds() for x in range(rs.times.size-1)]+[0.0])) if delta_t.size>1: delta_t[-1]=nanmean(delta_t[:-2]) if figs==None: figs=gt.figurelist() #start here if instrument.find('pars2S1') >= 0: color ='r' instrment = 'Pars2S1' else: color = 'b' instrument = 'Pars2S2' if display_defaults.enabled('parsivel_precip_rate'): if hasattr(rs,'precip_rate'): gt.plot_vs_time('parsivel_precip_rate' ,instrument ,usetimes ,[rs.precip_rate] ,[color] ,[2] ,[] ,'' ,'Parsivel precip rate (mm/hr)' ,[] ,'Parsivel precip rate vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel precip rate' else: print print 'No Parsivel precip rate image--precip rate field not found' if display_defaults.enabled('parsivel_accumulated_precip'): if hasattr(rs,'precip_rate'): #precip rate in units of mm/hr--convert seconds to hours temp = (rs.precip_rate * delta_t/3600.0).copy() temp[np.isnan(temp)]=0.0 accumulated_precip = np.cumsum(temp,0) gt.plot_vs_time('parsivel_accumulated_precip' ,instrument ,usetimes ,[accumulated_precip] ,[color] ,[2] ,[] ,'' ,'Parsivel precip (mm)' ,[] ,'Parsivel precip vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel precip' else: print print 'No Parsivel precip image--precip_rate field not found' if display_defaults.enabled('parsivel_median_volume_diameter'): if hasattr(rs,'median_volume_diameter'): temp = rs.median_volume_diameter.copy() temp[temp <0] = np.NaN gt.plot_vs_time('parsivel_median_volume_diameter' ,instrument ,usetimes ,[temp] ,[color] ,[2] ,None ,'' ,'median volume diameter' ,'mm' ,'parsivel median vol. dia. vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel median volume diameter' else: print print 'No Parsivel median volume diameter plot--field not found' if display_defaults.enabled('parsivel_liquid_water_content'): if hasattr(rs,'liquid_water_content'): gt.plot_vs_time('parsivel_liquid_water_content' ,instrument ,usetimes ,[rs.liquid_water_content * 1e-3] #convert from mm^3/m^3 to gr/m^3 ,[color] ,[2] ,None ,'' ,'Liquid water content' ,'mm' ,'Parsivel LWC vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel liquid water content' else: print print 'No Parsivel liquid water content plot--field not found' if display_defaults.enabled('parsivel_radar_reflectivity'): if hasattr(rs,'equivalent_radar_reflectivity'): temp = rs.equivalent_radar_reflectivity.copy() temp[temp<-50] = np.NaN gt.plot_vs_time('parsivel_radar_reflectiviy' ,instrument ,usetimes ,[temp] ,[color] ,[2] ,None ,'' ,'Radar reflectivity' ,'dBZ' ,'Parsivel radar reflectivity vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel radar reflectivity' else: print print 'No Parsivel equivalent radar reflectivity--field not found' if 0: print 'particle size' print rs.particle_size.shape print rs.particle_size print 'particle width' print rs.class_size_width.shape print rs.class_size_width print 'fall velocity' print rs.raw_fall_velocity.shape print rs.raw_fall_velocity if 0: #display_defaults.enabled('parsivel_fall_velocity'): if hasattr(rs,'raw_fall_velocity'): gt.plot_vs_time('parsivel_fall_velocity' ,instrument ,usetimes ,[rs.raw_fall_velocity] ,[color] ,[2] ,None ,'' ,'Raw fall velocity' ,'m/s' ,'Parsivel fall velocity vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel raw fall velocity' else: print print 'No Parsivel raw fall velocity plot--field not found' if 0: # display_defaults.enabled('parsivelfall_velocity_vs_time'): if hasattr(rs,'raw_spectrum'): fall_velocity = rs.raw_spectrum.copy() fall_velocity[fall_velocity < -9000.0] = 0.0 #sum over particle size fall_velocity = np.sum gt.plot_vs_time('parsivel_fall_velocity_vs_time' ,instrument ,usetimes ,[fall_velocity] ,[color] ,[2] ,[] ,'' ,'Parsivel fall velocity (m/s)' ,[] ,'Parsivel fall velocity vs time' ,False #FIXME ,display_defaults ,figs) print 'rendering Parsivel raw fall velocity' else: print print 'No Parsivel raw fall velocity plot--field not found' if display_defaults.enabled('parsivel_size_spectrum'): if hasattr(rs,'raw_spectrum'): raw_spectrum = rs.raw_spectrum.copy() raw_spectrum[raw_spectrum < -9000] = 0.0 #sum over time raw_spectrum = np.sum(raw_spectrum,0) #sum over velocities raw_spectrum = np.sum(raw_spectrum,1) np.set_printoptions(threshold=np.NaN) gt.plot_xy('parsivel_size_spectrum' #plot name ,'Parsivel' ,rs.times ,[rs.particle_size[:19]] ,[raw_spectrum[:19]] ,[color] ,[] ,[] ,['-'] ,[2] ,[] ,'upper right' ,'particle size ' ,'mm' ,'number' ,None ,'Raw spectrum' ,'' #text_str ,None #'text_position_x ,None #text_position_y ,None #text_angle ,display_defaults ,figs) else: print print 'Parsivel raw spectum plot not plotted--variable not found' if display_defaults.enabled('parsivel_fall_velocity_spectrum'): if hasattr(rs,'raw_spectrum'): raw_spectrum = rs.raw_spectrum.copy() raw_spectrum[raw_spectrum < -9000] = 0.0 #sum over time raw_spectrum = np.sum(raw_spectrum,0) #sum over sizes raw_spectrum = np.sum(raw_spectrum,1) np.set_printoptions(threshold=np.NaN) gt.plot_xy('parsivel_fall_velocity_spectrum' #plot name ,'Parsivel' ,rs.times ,[rs.raw_fall_velocity] ,[raw_spectrum] ,[color] ,[] ,[] ,['-'] ,[2] ,[] ,'upper right' ,'fall velocity' ,'m/s' ,'number' ,None ,'fall spectrum' ,'' #text_str ,None #'text_position_x ,None #text_position_y ,None #text_angle ,display_defaults ,figs) else: print print 'Parsivel fall velocity spectum not plotted--variable not found' if display_defaults.enabled('parsivel_number_detected_particles_vs_time'): if hasattr(rs,'number_detected_particles'): print 'rendering parsivel number_detected_particles_vs_time' gt.plot_vs_time('parsivel_number_detected_particles_vs_time' ,instrument ,usetimes ,[rs.number_detected_particles] ,[color] ,[2] ,[] ,'' ,'Number of particles' ,None ,'Parsivel number of detected particles vs time' ,False #FIXME ,display_defaults ,figs) else: print print 'Pareivel number_detected_particles not plotted--variable not found' if display_defaults.enabled('parsivel_number_density'): if hasattr(rs,'number_density_drops'): number_density = rs.number_density_drops.copy() number_density[number_density < -9000] = 0.0 #sum over time number_density = np.mean(number_density,0) #np.set_printoptions(threshold=np.NaN) gt.plot_xy('parsivel_number_density' ,instrument ,rs.times ,[rs.particle_size[:19]] ,[number_density[:19]] ,[color] ,[] ,[] ,['-'] ,[2] ,[] ,'upper right' ,'particle size ' ,'mm' ,'number density' ,'1/(m^3 *mm)' ,'Number density' ,'' #text_str ,None #'text_position_x ,None #text_position_y ,None #text_angle ,display_defaults ,figs) else: print print 'Parsivel number density not plotted--variable not found'
def accumulate_raman_inverted_profiles(consts, rs_mean, qc_mask, old_profiles, process_control=None, corr_adjusts=None): indices = np.arange(rs_mean.times.shape[0]) if len(indices) == 0: return old_profiles if qc_mask is not None and processing_defaults.get_value( 'averaged_profiles', 'apply_mask'): #make mask array with NaN's for array elements where bit[0] of qc_mask==0 #all other elements of mask = 1 mask = (np.bitwise_and(qc_mask, 1)).astype( 'float') #mask is float to allow use of NaN values mask[mask == 0] = np.NaN print 'qc_mask applied to time averaged profiles vs altitude' else: #set mask == 1 mask = None #for sh in channel_shorthand.keys(): # # if hasattr(rs_mean,sh): # mask = np.ones_like(getattr(rs_mean,sh)) # break #if mask is None: # mask = np.ones((rs_mean.times.shape[0],0)) print 'qc_mask has not been applied to time averaged profiles' #energies=('transmitted_1064_energy','transmitted_energy') profiles = hau.Time_Z_Group(can_append=False, altname='altitudes') profiles.hist = hau.Time_Z_Group() ft = None tc = 0 #tv=0 lt = None if old_profiles is not None: ft = old_profiles.hist.ft tc = old_profiles.hist.tc #tv=old_profiles.hist.tv lt = old_profiles.hist.lt if len(indices) > 0: if ft is None: ft = rs_mean.times[indices][0] lt = rs_mean.times[indices][-1] + timedelta( seconds=rs_mean.delta_t[indices][-1] if not np.isnan(rs_mean.delta_t[indices][-1]) else 0) for x in indices: if rs_mean.times[x] is not None: tc = tc + 1 #tv=tv+(rs_mean.times[x]-ft).total_seconds() if tc > 0: profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ]) profiles.start = ft profiles.width = lt - ft profiles.delta_t = hau.T_Array([profiles.width.total_seconds()]) else: profiles.times = hau.T_Array([]) profiles.start = ft profiles.width = timedelta(seconds=0) profiles.delta_t = hau.T_Array([]) profiles.start_time = ft profiles.end_time = lt profiles.hist.ft = ft profiles.hist.tc = tc #profiles.hist.tv=tv profiles.hist.lt = lt if rs_mean is not None and hasattr(rs_mean, 'altitudes'): profiles.altitudes = rs_mean.altitudes.copy() elif hasattr(old_profiles, 'altitudes'): profiles.altitudes = old_profiles.altitudes #FIXME need to accumulate the inverted products here, so here's a hack interval = hau.Time_Z_Group() interval.intervals = hau.T_Array(np.ones(rs_mean.times.shape)) accumulate(profiles, old_profiles, interval, indices, 'intervals') interval_count = profiles.intervals print 'Total intervals for profile =', interval_count for k, v in vars(rs_mean).items(): if k.startswith('_') or k in ('times', 'start', 'width', 'delta_t'): continue if not isinstance(v, hau.T_Array): continue if isinstance(v, hau.TZ_Array): continue accumulate(profiles, old_profiles, rs_mean, indices, k, interval_count) # create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke for k, v in vars(rs_mean).items(): if k.startswith('_'): continue if not isinstance(v, hau.TZ_Array): continue if len(v.shape) != 2: continue accumulate(profiles, old_profiles, rs_mean, indices, k, interval_count, mask) return profiles
def accumulate_raman_profiles(consts, rs_mean, qc_mask, old_profiles, process_control=None, rs_cal=None, Cxx=None, corr_adjusts=None): indices = np.arange(rs_mean.times.shape[0]) if len(indices) == 0: return old_profiles if qc_mask is not None and processing_defaults.get_value( 'averaged_profiles', 'apply_mask'): #make mask array with NaN's for array elements where bit[0] of qc_mask==0 #all other elements of mask = 1 mask = (np.bitwise_and(qc_mask, 1)).astype( 'float') #mask is float to allow use of NaN values mask[mask == 0] = np.NaN print 'qc_mask applied to time averaged profiles vs altitude' else: #set mask == 1 mask = None #for sh in channel_shorthand.keys(): # # if hasattr(rs_mean,sh): # mask = np.ones_like(getattr(rs_mean,sh)) # break #if mask is None: # mask = np.ones((rs_mean.times.shape[0],0)) print 'qc_mask has not been applied to time averaged profiles' #energies=('transmitted_1064_energy','transmitted_energy') profiles = hau.Time_Z_Group(can_append=False, altname='altitudes') profiles.hist = hau.Time_Z_Group() ft = None tc = 0 #tv=0 lt = None if old_profiles is not None: #total_seeded_shots=total_seeded_shots+profiles.hist.total_seeded_shots ft = old_profiles.hist.ft tc = old_profiles.hist.tc #tv=old_profiles.hist.tv lt = old_profiles.hist.lt if len(indices) > 0: if ft is None: ft = rs_mean.times[indices][0] lt = rs_mean.times[indices][-1] + timedelta( seconds=rs_mean.delta_t[indices][-1] if not np.isnan(rs_mean.delta_t[indices][-1]) else 0) for x in indices: if rs_mean.times[x] is not None: tc = tc + 1 #tv=tv+(rs_mean.times[x]-ft).total_seconds() if tc > 0: profiles.times = hau.T_Array([ft]) #+timedelta(seconds=tv/tc), ]) profiles.start = ft profiles.width = lt - ft profiles.delta_t = hau.T_Array([profiles.width.total_seconds()]) else: profiles.times = hau.T_Array([]) profiles.start = ft profiles.width = timedelta(seconds=0) profiles.delta_t = hau.T_Array([]) profiles.start_time = ft profiles.end_time = lt profiles.hist.ft = ft profiles.hist.tc = tc #profiles.hist.tv=tv profiles.hist.lt = lt #profiles.hist.total_seeded_shots=total_seeded_shots if rs_mean is not None and hasattr(rs_mean, 'altitudes'): profiles.altitudes = rs_mean.altitudes.copy() #elif hasattr(old_profiles,'altitudes'): # profiles.altitudes=old_profiles.altitudes elif hasattr(old_profiles, 'heights'): profiles.heights = old_profiles.heights accumulate(profiles, old_profiles, rs_mean, indices, 'shots', pref='mean_', filler=hau.T_Array([0])) total_shots = profiles.mean_shots profiles.shots = total_shots.copy() print 'Total shots for profile =', total_shots for e in energies: accumulate(profiles, old_profiles, rs_mean, indices, e, total_shots) # create TZ_Array with time dimension of '1', so hsrl_inversion doesn't choke for chan in channel_shorthand.keys(): accumulate(profiles, old_profiles, rs_mean, indices, chan, total_shots, mask) #compute inverted profiles from mean count profiles if Cxx is not None and hasattr(profiles, 'elastic_counts'): import raman.core.raman_processing_utilities as rpu profiles.inv = rpu.process_raman(consts, profiles, process_control, rs_cal, Cxx, corr_adjusts) import lidar.sg_extinction as lsge filter_params = lsge.filter_setup(profiles.inv.altitudes, process_control, consts) sg_ext = lsge.sg_extinction(filter_params) profiles.inv.extinction,profiles.inv.extinction_aerosol,profiles.inv.p180 \ = sg_ext(profiles.times,profiles.delta_t,profiles.nitrogen_counts ,profiles.inv.beta_a_backscat,profiles.inv.integ_backscat,beta_r=Cxx.beta_r_355) elif hasattr(old_profiles, 'inv'): profiles.inv = old_profiles.inv return profiles
def HSRLPolConfig(x, thQWP): """ outputs the polarization configuration vectors of all 4 HSRL channels for a provided x - a dictionary containing all polarization calibration terms thQWP - a range of QWP angles in radians returns [Si,DxV,DtV,DmV] """ thQWP = thQWP + x['qwp_rot_offset'] DxV = np.zeros((len(thQWP), 4)) # N x 4 DtV = np.zeros((len(thQWP), 4)) # N x 4 DmV = np.zeros((len(thQWP), 4)) # N x 4 Si = np.zeros((4, len(thQWP))) # 4 x N Stx = np.array((1, cos(2 * x['tx_elip']) * cos(2 * x['tx_rot']), cos(2 * x['tx_elip']) * sin(2 * x['tx_rot']), sin(2 * x['tx_elip']))) # 4 x 1 Dx = np.array((1, x['cross_rx_diat'] * cos(2 * x['cross_rx_rot']) * cos(2 * x['cross_rx_elip']), x['cross_rx_diat'] * sin(2 * x['cross_rx_rot']) * cos(2 * x['cross_rx_elip']), x['cross_rx_diat'] * sin(2 * x['cross_rx_elip']))) # 1 x 4 Dm = np.array((1, x['mol_rx_diat'] * cos(2 * x['mol_rx_rot']) * cos(2 * x['mol_rx_elip']), x['mol_rx_diat'] * sin(2 * x['mol_rx_rot']) * cos(2 * x['mol_rx_elip']), x['mol_rx_diat'] * sin(2 * x['mol_rx_elip']))) # 1 x 4 Dt = np.array((1, x['total_rx_diat'] * cos(2 * x['total_rx_rot']) * cos(2 * x['total_rx_elip']), x['total_rx_diat'] * sin(2 * x['total_rx_rot']) * cos(2 * x['total_rx_elip']), x['total_rx_diat'] * sin(2 * x['total_rx_elip']))) # 1 x 4 Dl = np.array( (1, x['lowgain_rx_diat'] * cos(2 * x['lowgain_rx_rot']) * cos(2 * x['lowgain_rx_elip']), x['lowgain_rx_diat'] * sin(2 * x['lowgain_rx_rot']) * cos(2 * x['lowgain_rx_elip']), x['lowgain_rx_diat'] * sin(2 * x['lowgain_rx_elip']))) # Matt: Changed to Matrix Multiplication Mir1 = np.dot(RMueller(x['mirror_rot']), np.dot( VWP(0, x['mirror_phase']), np.dot(DiatU(x['mirror_diat'] * np.array((1, 0, 0))), RMueller(-x['mirror_rot'])))) # 4 x 4 Mir2 = RevMueller(Mir1) # 4 x 4 # Matt: Changed to Matrix Multiplication Twin1 = np.dot(RMueller(x['window_diat_rot']),np.dot(DiatU(x['window_diat']*np.array((1,0,0))) , \ np.dot(RMueller(-x['window_diat_rot']),VWP(x['window_phase_rot'],x['window_phase']) ) ) ) # 4 x 4 Twin2 = RevMueller(Twin1) # 4 x 4 for ai in range(len(thQWP)): QWP1 = VWP( -thQWP[ai] + x['qwp_rot_0'], x['qwp_phase_0'] + x['qwp_phase_2'] * np.cos(-2 * (thQWP[ai] - x['qwp_rot_0'])) + x['qwp_phase_1'] * np.cos(-thQWP[ai] + x['qwp_rot_1'])) # 4 x 4 QWP2 = RevMueller(QWP1) # 4 x 4 Si[:, ai] = np.dot(np.dot(np.dot(Twin1, Mir1), QWP1), Stx) DxV[ai, :] = np.dot(np.dot(np.dot(Dx, QWP2), Mir2), Twin2) DtV[ai, :] = np.dot(np.dot(np.dot(Dt, QWP2), Mir2), Twin2) DmV[ai, :] = np.dot(np.dot(np.dot(Dm, QWP2), Mir2), Twin2) # convert to T_Array, so we can process properly Si = hau.T_Array(Si) DxV = hau.T_Array(DxV) DtV = hau.T_Array(DtV) DmV = hau.T_Array(DmV) out = { 'Si': Si, 'Dm': Dm, 'Dx': DxV, 'Dt': DtV, 'DmV': DmV, \ 'QWP1' : QWP1, 'QWP2': QWP2, 'Mir1': Mir1, 'Mir2': Mir2, 'thQWP': thQWP, 'Twin1' : Twin1, 'Twin2' : Twin2, 'Dt': Dt, 'Stx': Stx } # return out return [Si, DxV, DtV, DmV]