def __init__(self,
instrument,
start_time_datetime,
end_time_datetime,
timeres_timedelta=None,
maxtimeslice_timedelta=None,
min_alt_m=None,
max_alt_m=None,
altres_m=None,
process_defaults=None,
data_request=None):
print '**** dpl_rti is deprecated. use dpl_hsrl'
self.dpl = dpl_hsrl(instrument=instrument,
maxtimeslice_timedelta=maxtimeslice_timedelta,
process_defaults=process_defaults,
data_request=data_request)
self.timeslice = maxtimeslice_timedelta
self.parms = {
'start_time_datetime': start_time_datetime,
'end_time_datetime': end_time_datetime,
'timeres_timedelta': timeres_timedelta,
'min_alt_m': min_alt_m,
'max_alt_m': max_alt_m,
'altres_m': altres_m
}
import lg_base.core.array_utils as hau #import T_Array,Z_Array,TZ_Array,Time_Z_Group
self.rs_static = hau.rs_xfer()
#self.rs_static.processing_defaults = self.dpl.get_processing_defaults(process_defaults)[0]
return
def __init__(self, instrument, startTime, endTime,
window=5*60 ,
process_control = None, minalt_km=0,
maxalt_km = 15):
self.instrument = instrument
self.startTime = startTime
self.endTime = endTime
self.currentStart = startTime
self.window = timedelta(seconds=window)
self.process_control = process_control
self.minalt_km = minalt_km
self.maxalt_km = maxalt_km
frame_substruct.SubstructBrancher.multiprocessable=False
self.dplobj=dpl_hsrl(instrument=self.instrument,
process_control=self.process_control)
def showL3Shift(inst, start, end, cal):
from hsrl.dpl.dpl_hsrl import dpl_hsrl
dpl = dpl_hsrl(inst, filetype='data')
r = dpl(start_time_datetime=start,
end_time_datetime=end,
min_alt_m=0,
max_alt_m=5000,
forimage=False,
raw_only=True,
with_profiles=False)
myvars = ('times', 'interf_freq', 'filtered_energy', 'nonfiltered_energy',
'l3cavityvoltage', 'l3locking_stats', 'superseedlasercontrollog',
'molecular_cal_pulse')
content = dict()
for x in r:
for v in myvars:
if not hasattr(x.rs_raw, v):
print 'no ', v
if v not in content:
content[v] = getattr(x.rs_raw, v).copy()
else:
content[v].append(getattr(x.rs_raw, v))
if len(content.keys()) == 0:
return None
interfoffset = content['interf_freq']
seedoffset = content['superseedlasercontrollog'][:, 7] * r.hsrl_constants[
'seedlaser_temp_to_freq']
ratio = content['filtered_energy'][:, 0] / content['nonfiltered_energy'][:,
0]
l3slope = content['l3locking_stats'][:, 0].copy()
#plt.subplot(411);plt.plot(content['times'],ratio)
plt.subplot(412)
plt.plot(content['times'], seedoffset, 'b', content['times'], interfoffset,
'r')
plt.subplot(413)
plt.plot(content['times'], l3slope, 'b')
ret = np.polyval(cal, l3slope)
plt.subplot(414)
plt.plot(content['times'], ret, 'g')
#plt.subplot(414);plt.plot(seedoffset,l3slope,seedfit,l3slope)
#plt.show()
return ret
def write_cfradial(output_filename,
start_dt,
minutes,
timeres_s=5,
altres_m=60,
maxtimeslice_td=datetime.timedelta(seconds=30 * 60),
instrument='gvhsrl',
min_alt_m=0,
max_alt_m=5000,
store_calibrations=False):
""" writes HSRL data in CfRadial netcdf format
output_filename = where to write the data
start_td = = datetime.datetime object first time to retrieve.
minutes = how many minutes to process
timeres_s = time resolution in seconds (native would be 2.5)
altres_m = altitude resolution in meters
maxtimeslice_timedelta = datetime.timedelta object for amount of data processed (safe is 1 or 2 hours)
instrument = hsrl id string (eg. 'ahsrl','gvhsrl','nshsrl','mf2hsrl').
min_alt_m = minimum altitude in meters to display
max_alt_m = maximum altitude in meters to display.
"""
cdl = locate_file('hsrl_cfradial.cdl', forModule=lgtb)
print 'CDL = ', cdl
timeres_td = datetime.timedelta(seconds=timeres_s)
netcdf = Dataset(output_filename, 'w', clobber=True)
delta = datetime.timedelta(minutes=minutes)
timeres_delta = datetime.timedelta(seconds=timeres_s)
end_dt = start_dt + delta
gen = dpl_hsrl(instrument)
if store_calibrations: # to store calibrations, newer actors are needed, as well as the precall methods (FIXME better design)
import maestro.netcdf_precall as npc
args = []
kwargs = dict(output=netcdf,
template=cdl,
usecfradial=True,
basetime=start_dt)
x = npc.addConstantsToParms(npc.addCalibrationsToNetCDF())
hsrlnar = gen(start_dt,
end_dt,
timeres_timedelta=timeres_delta,
min_alt_m=min_alt_m,
max_alt_m=max_alt_m,
altres_m=altres_m)
x(hsrlnar, args, kwargs)
nar = artists.dpl_netcdf_artist(hsrlnar, *args, **kwargs)
#framestream,template,outputfilename=None,format=None,usecfradial=None,selected_bindings=None,output=None,forModule=None,withUnlimited=None,basetime=None,addAttributes={}):
for x in nar:
pass
else:
v = None
try:
# store each lidar record
for tzg in gen(start_dt,
end_dt,
timeres_timedelta=timeres_delta,
min_alt_m=min_alt_m,
max_alt_m=max_alt_m,
altres_m=altres_m):
if v == None:
v = cfr.DplCreateCfradial(cdl, netcdf, tzg)
v.append_data(tzg)
v.close()
except RuntimeError, msg:
print msg
traceback.print_exc()
print 'write_cfradial: could not process data for %s starting at %s' % \
(instrument, start_dt.strftime('%Y-%m-%d %H:%M:%S'))
def __init__(self, inst):
from hsrl.dpl.dpl_hsrl import dpl_hsrl
self.dpl = dpl_hsrl(inst, filetype='calibration')
def makeArchiveImages(instrument,datetimestart,range_km=15,full24hour=None,filename=None,reprocess=False,attenuated=False,frappe=False,ir1064=False,ismf2ship=False,completeframe=None,*args,**kwargs):
import lg_dpl_toolbox.filters.substruct as frame_substruct
frame_substruct.SubstructBrancher.multiprocessable=False
from lg_dpl_toolbox.dpl.dpl_read_templatenetcdf import dpl_read_templatenetcdf
from lg_dpl_toolbox.filters.time_frame import FrameCachedConcatenate
from hsrl.dpl.dpl_hsrl import dpl_hsrl
from radar.dpl.dpl_radar import dpl_radar
from lg_dpl_toolbox.dpl.dpl_create_templatenetcdf import dpl_create_templatenetcdf
import hsrl.dpl.dpl_artists as hsrl_artists
import radar.dpl.dpl_artists as radar_artists
import raman.dpl.dpl_artists as raman_artists
import lg_dpl_toolbox.core.archival as hru
import hsrl.graphics.hsrl_display as du
if filename!=None:
useFile=True
else:
useFile=False #True
if not useFile or not os.access(filename,os.F_OK):
reprocess=True
realend=None
#process_control=None
if reprocess:
print datetime
#instrument='ahsrl'
if useFile:
n=Dataset(filename,'w',clobber=True)
n.instrument=instrument
print datetime
realstart=datetimestart
if full24hour==None:
realstart=datetimestart.replace(hour=0 if datetimestart.hour<12 else 12,minute=0,second=0,microsecond=0)
elif frappe:
realstart=datetimestart.replace(hour=0,minute=0,second=0,microsecond=0)
elif ismf2ship:
realend=realstart
realstart=realstart-timedelta(days=2.0)
else:
realstart=realstart-timedelta(days=1.0)
if 'realstart' in kwargs:
realstart=kwargs['realstart']
if realend is None:
realend=realstart+timedelta(days=.5 if full24hour==None else 1.0)
if 'realend' in kwargs:
realend=kwargs['realend']
isHsrl=False
isRadar=False
isRaman=False
instrumentprefix=None
if instrument.endswith('hsrl'):
dpl=dpl_hsrl(instrument=instrument,filetype='data')
dpl.hsrl_process_control.set_value('quality_assurance','enable',False)
#dpl.hsrl_process_control.set_value('extinction_processing','enable',False)
dpl.hsrl_process_control.set_value('wfov_corr','enable',False)
gen=dpl(start_time_datetime=realstart,end_time_datetime=realend,min_alt_m=0,max_alt_m=range_km*1000,with_profiles=False)
isHsrl=True
#process_control=gen.hsrl_process_control
hsrlinstrument=instrument
if os.getenv('COMPLETEFRAME',completeframe)==None:
import lg_dpl_toolbox.filters.substruct as frame_substruct
dropcontent=['rs_raw','rs_mean']
gen=frame_substruct.DropFrameContent(gen,dropcontent)
elif ('kazr' in instrument) or ('mwacr' in instrument) or instrument=='mmcr':
dpl=dpl_radar(instrument=instrument)
gen=dpl(start_time_datetime=realstart,end_time_datetime=realend,min_alt_m=0,max_alt_m=range_km*1000,forimage=True,allow_nans=True)
hsrlinstrument=dpl.instrumentbase
isRadar=True
if 'mwacr' in instrument:
instrumentprefix='mwacr'
else:
instrumentprefix='radar'
#merge=picnicsession.PicnicProgressNarrator(dplc,getLastOf('start'), searchparms['start_time_datetime'],searchparms['end_time_datetime'],session)
#hasProgress=True
elif instrument.startswith('rlprof'):
from raman.dpl.raman_dpl import dpl_raman
import lg_dpl_toolbox.filters.time_frame as time_slicing
dpl=dpl_raman('bagohsrl',instrument.replace('rlprof',''))
gen=dpl(start_time_datetime=realstart,end_time_datetime=realend,min_alt_m=0,max_alt_m=range_km*1000,forimage=True,allow_nans=True,inclusive=True)
import functools
import lg_base.core.array_utils as hau
from dplkit.simple.blender import TimeInterpolatedMerge
import lg_dpl_toolbox.filters.substruct as frame_substruct
import lg_dpl_toolbox.dpl.TimeSource as TimeSource
import lg_base.core.canvas_info as ci
gen=time_slicing.TimeGinsu(gen,timefield='times',dtfield=None)
forimage=ci.load_canvas_info()['canvas_pixels']
timesource=TimeSource.TimeGenerator(realstart,realend,time_step_count=forimage['x'])
gen=TimeInterpolatedMerge(timesource,[gen], allow_nans=True)
gen=TimeSource.AddAppendableTime(gen,'times','delta_t')
gen=frame_substruct.Retyper(gen,functools.partial(hau.Time_Z_Group,timevarname='times',altname='altitudes'))
hsrlinstrument='bagohsrl'
instrumentprefix=instrument
isRaman=True
if reprocess and useFile:
v=None
for i in gen:
if v==None:
v=dpl_create_templatenetcdf(locate_file('hsrl_nomenclature.cdl'),n,i)
v.appendtemplatedata(i)
#raise TypeError
#break
n.sync()
n.close()
#fn='outtest.nc'
if useFile:
v=dpl_read_templatenetcdf(filename)
instrument=v.raw_netcdf().instrument[:]
else:
v=gen
defaultjson='archive_plots.json'
if ismf2ship:
defaultjson='mf2ship_plots.json'
(disp,conf)=du.get_display_defaults(os.getenv("PLOTSJSON",defaultjson))
v=FrameCachedConcatenate(v)
import lg_dpl_toolbox.dpl.TimeSource as TimeSource
import lg_dpl_toolbox.filters.fill as fill
import lg_base.core.canvas_info as ci
forimage=ci.load_canvas_info()['canvas_pixels']
ts=TimeSource.TimeGenerator(realstart,realend,time_step_count=forimage['x'])
v=fill.FillIn(v,[np.array([x['start'] for x in ts])],ignoreGroups=ignoreSomeGroups)
rs=None
for n in v:
if rs==None:
rs=n
else:
rs.append(n)
if isHsrl:
if attenuated:
bsfigname='atten_backscat_image'
bsprefix='attbscat'
disp.set_value(bsfigname,'enable',1)
disp.set_value('backscat_image','enable',0)
disp.set_value('linear_depol_image','figure',bsfigname)
#disp.set_value('circular_depol_image','figure',bsfigname)
else:
bsfigname='backscat_image'
bsprefix='bscat'
if ir1064 or (hasattr(rs,'rs_inv') and hasattr(rs.rs_inv,'color_ratio')):
disp.set_value('raw_color_ratio_image','enable',1)
disp.set_value('color_ratio_image','enable',1)
#if hasattr(rs.rs_inv,'circular_depol'):
# disp.set_value('linear_depol_image','enable',0) #this modification works because the artist doesn't render until after the yield
# field='circular_depol'
#else:
#disp.set_value('circular_depol_image','enable',0)
field='linear_depol'
if isHsrl:
#if process_control==None:
# print 'loading process control from json'
# process_control=jc.json_config(locate_file('process_control.json'),'process_defaults')
v=hsrl_artists.dpl_images_artist(v,instrument=instrument,max_alt=None,display_defaults=disp)
elif isRadar:
v=radar_artists.dpl_radar_images_artist(framestream=v,instrument=v.radarType,display_defaults=disp,subframe=None)
elif isRaman:
v=raman_artists.dpl_raman_images_artist(framestream=v,display_defaults=disp)
for n in v:
pass #run once more with cached value and actual artists
usedpi=90
#rs=Rti('ahsrl',stime.strftime('%d-%b-%y %H:%M'),dtime.total_seconds()/(60*60),minalt_km,maxalt_km,.5,'archive_plots.json')
imtime=realstart
imetime=realend
imtimestr=imtime.strftime('%e-%b-%Y ')
file_timetag=imtime.strftime("_%Y%m%dT%H%M_")+imetime.strftime("%H%M_")+("%i" % range_km)
if not full24hour:
imtimestr+= ('AM' if imtime.hour<12 else 'PM' )
file_timetag+="_am" if imtime.hour<12 else "_pm"
filelocation=hru.get_path_to_data(hsrlinstrument,None)
print filelocation
filelocation=os.path.join(filelocation,imtime.strftime("%Y/%m/%d/images/"))
try:
os.makedirs(filelocation)
except OSError:
pass
print filelocation
if os.getenv("DEBUG",None):
filelocation='.'
print 'DEBUG is on. storing in current directory'
figs=v.figs
extensionsList=('.png','.jpg','.gif','.jpeg')
preferredFormat='jpg'
preferredExtension='.'+preferredFormat
if isHsrl:
#figs=du.show_images(instrument,rs,None,{},process_control,disp,None,None,None)
#for x in figs:
# fig = figs.figure(x)
# fig.canvas.draw()
print figs
f=figs.figure(bsfigname)
#f.set_size_inches(f.get_size_inches(),forward=True)
if frappe:
frappetag=imtime.strftime('upperair.UW_HSRL.%Y%m%d%H%M.BAO_UWRTV_')
hiresfile=os.path.join('.',frappetag+'backscatter_%ikm%s' %(range_km,preferredExtension))
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
else:
hiresfile=os.path.join(filelocation,bsprefix+"_depol" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
if disp.enabled('raw_color_ratio_image'):
f=figs.figure('color_ratio_image')
#f.set_size_inches(f.get_size_inches(),forward=True)
hiresfileir=os.path.join(filelocation,"ratioIR" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfileir,extensionsList)
f.savefig(hiresfileir,format=preferredFormat,bbox_inches='tight')
figs.close()
if not full24hour and hasattr(rs,'rs_inv'):
scale = [float(disp.get_value(bsfigname,'lo_color_lmt')),
float(disp.get_value(bsfigname,'hi_color_lmt'))]
depol=100*getattr(rs.rs_inv,field)
depolscale = [float(disp.get_value(field+'_image', 'lo_color_lmt')),
float(disp.get_value(field+'_image', 'hi_color_lmt'))]
if attenuated:
backscat=rs.rs_inv.atten_beta_a_backscat
elif hasattr(rs.rs_inv,'beta_a_backscat'):
backscat=rs.rs_inv.beta_a_backscat
else:
backscat=rs.rs_inv.beta_a_backscat_par + rs.rs_inv.beta_a_backscat_perp
print rs.rs_inv.times.shape
print backscat.shape
if disp.get_value(bsfigname,"log_linear")=='log':
scale=np.log10(scale)
backscat[backscat<=0]=np.NAN;
backscat=np.log10(backscat)
if disp.get_value(field+'_image',"log_linear")=='log':
depol[depol<=0]=np.NAN;
depolscale=np.log10(depolscale)
depol=np.log10(depol)
print backscat.shape
qc_mask=None
if hasattr(rs.rs_inv,'qc_mask'):
qc_mask=np.ones_like(rs.rs_inv.qc_mask)
qcbits={'mol_lost':64,'mol_sn_ratio':16,'cloud_mask':128,'I2_lock_lost':4}
for name,maskbit in qcbits.items():
if disp.get_value('mask_image',name):
qc_mask = np.logical_and(rs.rs_inv.qc_mask & maskbit > 0,qc_mask)
#print np.sum(backscat<=0)
f=tinythumb(backscat,scale,imtimestr,dpi=usedpi,qcmask=qc_mask)
thumbname=os.path.join(filelocation,bsprefix + file_timetag + '_thumb'+preferredExtension)
print thumbname
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
f=tinythumb(depol,depolscale,imtimestr,dpi=usedpi,qcmask=qc_mask)
thumbname=os.path.join(filelocation,'depol' + file_timetag + '_thumb'+preferredExtension)
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
if ir1064 or hasattr(rs.rs_inv,'color_ratio'):
if hasattr(rs.rs_inv,'qc_mask'):
qc_mask=np.ones_like(rs.rs_inv.qc_mask)
qcbits={'mol_lost':64,'mol_sn_ratio':16,'cloud_mask':128,'I2_lock_lost':4,'1064_shutter':0x8000}
for name,maskbit in qcbits.items():
if disp.get_value('mask_image',name):
qc_mask = np.logical_and(rs.rs_inv.qc_mask & maskbit > 0,qc_mask)
cr=rs.rs_inv.color_ratio
scale = [float(disp.get_value('color_ratio_image','lo_color_lmt')),
float(disp.get_value('color_ratio_image','hi_color_lmt'))]
if disp.get_value('color_ratio_image',"log_linear")=='log':
scale=np.log10(scale)
cr[cr<=0]=np.NAN;
cr=np.log10(cr)
f=tinythumb(cr,scale,imtimestr,dpi=usedpi,qcmask=qc_mask)
thumbname=os.path.join(filelocation,'ratioIR' + file_timetag + '_thumb'+preferredExtension)
print thumbname
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
elif isRadar:
f=figs.figure('radar_backscatter_image')
#f.set_size_inches(f.get_size_inches(),forward=True)
hiresfile=os.path.join(filelocation,instrumentprefix+"_bscat" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
figs.close()
if not full24hour:
radarscale = [float(disp.get_value('radar_backscatter_image', 'lo_color_lmt')),
float(disp.get_value('radar_backscatter_image', 'hi_color_lmt'))]
radar=rs.Backscatter
if disp.get_value('radar_backscatter_image','log_linear')=='log':
radar[radar<=0]=np.NAN
radarscale=np.log10(radarscale)
radar=np.log10(radar)
f=tinythumb(radar,radarscale,imtimestr,dpi=usedpi)
thumbname=os.path.join(filelocation,instrumentprefix+'_bscat' + file_timetag + '_thumb'+preferredExtension)
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
elif isRaman:
if hasattr(rs,'backscatter'):
f=figs.figure('rl_backscatter_image')
#f.set_size_inches(f.get_size_inches(),forward=True)
hiresfile=os.path.join(filelocation,instrumentprefix+"_bscat" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
elif hasattr(rs,'beta'):
f=figs.figure('rl_backscatter_image')
#f.set_size_inches(f.get_size_inches(),forward=True)
hiresfile=os.path.join(filelocation,instrumentprefix+"_beta" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
if hasattr(rs,'linear_depol'):
f=figs.figure('rl_depol_image')
#f.set_size_inches(f.get_size_inches(),forward=True)
hiresfile=os.path.join(filelocation,instrumentprefix+"_dep" + file_timetag + preferredExtension)
ensureOnlyOne(hiresfile,extensionsList)
f.savefig(hiresfile,format=preferredFormat,bbox_inches='tight')
figs.close()
if not full24hour:
if hasattr(rs,'backscatter'):
ramanscale = [float(disp.get_value('rl_backscatter_image', 'lo_color_lmt')),
float(disp.get_value('rl_backscatter_image', 'hi_color_lmt'))]
raman=rs.backscatter.copy()
#print np.nanmax(raman),np.nanmin(raman),' is ramans actual range'
if disp.get_value('rl_backscatter_image','log_linear')=='log':
raman[raman<=0]=np.NAN
ramanscale=np.log10(ramanscale)
raman=np.log10(raman)
f=tinythumb(raman,ramanscale,imtimestr,dpi=usedpi)
thumbname=os.path.join(filelocation,instrumentprefix+'_bscat' + file_timetag + '_thumb'+preferredExtension)
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
if hasattr(rs,'beta'):
ramanscale = [float(disp.get_value('rl_backscatter_image', 'lo_color_lmt')),
float(disp.get_value('rl_backscatter_image', 'hi_color_lmt'))]
raman=rs.beta.copy()
#print np.nanmax(raman),np.nanmin(raman),' is ramans actual range'
if disp.get_value('rl_backscatter_image','log_linear')=='log':
raman[raman<=0]=np.NAN
ramanscale=np.log10(ramanscale)
raman=np.log10(raman)
f=tinythumb(raman,ramanscale,imtimestr,dpi=usedpi)
thumbname=os.path.join(filelocation,instrumentprefix+'_beta' + file_timetag + '_thumb'+preferredExtension)
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
if hasattr(rs,'linear_depol'):
ramanscale = [float(disp.get_value('rl_depol_image', 'lo_color_lmt')),
float(disp.get_value('rl_depol_image', 'hi_color_lmt'))]
raman=rs.linear_depol.copy()
#print np.nanmax(raman),np.nanmin(raman),' is ramans actual range'
if disp.get_value('rl_depol_image','log_linear')=='log':
raman[raman<=0]=np.NAN
ramanscale=np.log10(ramanscale)
raman=np.log10(raman)
f=tinythumb(raman,ramanscale,imtimestr,dpi=usedpi)
thumbname=os.path.join(filelocation,instrumentprefix+'_dep' + file_timetag + '_thumb'+preferredExtension)
ensureOnlyOne(thumbname,extensionsList)
f.savefig(thumbname,format=preferredFormat,bbox_inches='tight',dpi=usedpi)
trimborder(thumbname)
today=datetime.utcnow()
if not frappe and not ismf2ship and full24hour and imetime.year==today.year and imetime.month==today.month and imetime.day==today.day:
destpath=os.getenv("DAYSTORE",os.path.join('/','var','ftp','public_html','hsrl'))
destfile=os.path.join(destpath,full24hour + '_current'+preferredExtension)
fulln=hiresfile
if not fulln:
return
outf=file(destfile,'w')
inf=file(fulln,'r')
ensureOnlyOne(destfile,extensionsList)
outf.write(inf.read())
inf.close()
outf.close()
os.unlink(fulln)
elif ismf2ship:
if not hiresfile:
return
destpath='./'#os.getenv("DAYSTORE",os.path.join('/','var','ftp','public_html','hsrl'))
destfile=os.path.join(destpath,realend.strftime('hsrl_%Y%m%d%H%M')+preferredExtension)
fulln=hiresfile
outf=file(destfile,'w')
inf=file(fulln,'r')
ensureOnlyOne(destfile,extensionsList)
outf.write(inf.read())
inf.close()
outf.close()
os.unlink(fulln)
recompressImage(destfile,quality=30,optimize=True)
varlist=('rs_mean.times','rs_mean.latitude','rs_mean.longitude','rs_mean.transmitted_energy','rs_mean.seedvoltage',\
'rs_mean.seeded_shots','rs_mean.coolant_temperature','rs_mean.laserpowervalues','rs_mean.opticalbenchairpressure',\
'rs_mean.humidity','rs_mean.l3locking_stats','rs_mean.l3cavityvoltage','rs_mean.nonfiltered_energy','rs_mean.filtered_energy',\
'rs_mean.builduptime','rs_mean.one_wire_temperatures')
skip=0
while skip<16:
try:
print 'skip is ',skip
addExifComment(destfile,makedict(n,varlist,skip=skip))
break
except Exception as e:
skip+=1
print e
traceback.print_exc()
pass
sendSCPFile(destfile,host='198.129.80.15',path='/ftpdata/outgoing',user='******',keyfile=os.getenv('MF2_HSRL_KEY'))
elif frappe:
if not hiresfile:
return
sendFTPFile(hiresfile,host='catalog.eol.ucar.edu',path='/pub/incoming/catalog/frappe/',user='******',password='******')
def main():
from dpl_filters import QAFlagClonedSyncFilter
from dpl_narrators import QCFlagNarrator, QCLogNarrator
import dpl_zookeeper as qazoo
import dpl_librarian as qalib
#from parsing import fileParser,flagParser
from hsrl.dpl.dpl_hsrl import dpl_hsrl
import lg_dpl_toolbox.filters.time_frame as time_slicing
import sys
import json
td = timedelta(hours=2)
et = datetime.utcnow()
if len(sys.argv) > 3:
st = datetime.strptime(sys.argv[3], '%Y.%m.%dT%H:%M:%S')
else:
st = et - td
if len(sys.argv) > 4:
et = datetime.strptime(sys.argv[4], '%Y.%m.%dT%H:%M:%S')
from lg_dpl_toolbox.dpl.TimeSource import TimeGenerator
hsrllib = dpl_hsrl(instrument=sys.argv[2])
import lg_base.core.canvas_info as ci
canvas_info = ci.load_canvas_info()
#process_control=hsrllib.hsrl_process_control
number_x_pixels = canvas_info['canvas_pixels']['x']
timesource = TimeGenerator(start_time=st,
end_time=et,
time_step_count=number_x_pixels)
zookeeper = qazoo.QualityAssuranceZookeeper(
) #responsible for reading the files
librarian = qalib.QualityAssuranceLibrarian(
instrument=sys.argv[2]) #responsible for finding the files
hsrlnar = hsrllib(min_alt_m=0, max_alt_m=20000, timesource=timesource)
#f=fileParser()
#fp=flagParser(hsrlnar.altitudeAxis,binwidth=hsrlnar.hsrl_constants['binwidth'] * 1.5e8)
#x= f.parseFile(sys.argv[1])
x = zookeeper(librarian(
start_time=st, end_time=et)) #finding the files and reading them all
print x
json.dump(safeenc(x),
file('dump.json', 'w'),
indent=4,
separators=(',', ': '))
lognarr = QCLogNarrator(x, st, et)
print 'Log Content for', st, 'to', et, 'is:'
for e in lognarr:
print e['header']['date'], ':', e['content'].strip()
#assert(0)
qcnarr = QCFlagNarrator(
x,
altitude_axis=hsrlnar.altitudeAxis,
binwidth=hsrlnar.hsrl_constants['binwidth'] *
1.5e8) #narrator streams the read flags as a single timestep per entry
import lg_dpl_toolbox.filters.substruct as frame_substruct
qcnarr = QAFlagClonedSyncFilter(
qasource=qcnarr, timealtsource=timesource, timename='start'
) #takes the output of the narrator, and echos entries for each timestep (and potentially project range entries to altitude)
if False:
hsrlnarsplitter = frame_substruct.SubstructBrancher(hsrlnar)
#timesource=time_slicing.TimeGinsu(hsrlnarsplitter.narrateSubstruct('rs_inv'),'times',onlyTime=True)#,isEnd=True)
#qcnarr=frame_substruct.Retyper(qcnarr,hau.Time_Z_Group,dict(timevarname='times',altname='altitudes'))
qcnarr = frame_substruct.CountDeGinsu(
frame_substruct.FrameLength(
hsrlnarsplitter.narrateSubstruct('rs_inv'), 'times'), qcnarr)
dplc = frame_substruct.NestingCompositer(
hsrlnarsplitter.narrateSubstruct(None), dict(rs_hsrl_qc=qcnarr))
printkeys = ('rs_inv', 'rs_hsrl_qc', 'rs_hsrl_qc.qcflags.shape',
'rs_hsrl_qc.qcflags')
else:
dplc = qcnarr
printkeys = ('qcflags.shape', 'qcflags')
dplc = time_slicing.FrameCachedConcatenate(dplc)
for f in dplc:
print vars(f).keys()
for x in printkeys:
print x
print deepgetattr(f, x)
def main(inst, startdate, enddate, sig=None):
history = dict()
parts = (('thermal1', 'thermal2'), ('records', 'goodrecords',
'errorCount'))
alltherms = list(makelist(parts).keys()) + ['times']
began = datetime.utcnow()
import hsrl.dpl.dpl_hsrl as dpl_hsrl
from lg_dpl_toolbox.dpl.NetCDFZookeeper import GenericTemplateRemapNetCDFZookeeper
zoo = GenericTemplateRemapNetCDFZookeeper(inst,
forModule=dpl_hsrl,
keepfields=alltherms)
dpllib = dpl_hsrl.dpl_hsrl(inst, zoo=zoo)
dpl = dpllib(start_time_datetime=startdate,
end_time_datetime=enddate,
min_alt_m=0,
max_alt_m=500,
forimage=False,
raw_only=True,
with_profiles=False)
dpl = RawFieldSelection(dpl, alltherms, subscope='rs_raw')
for fr in dpl:
appendContent(history, fr)
print('timed at ' + repr(datetime.utcnow() - began))
print(repr(history.keys()))
#print(retlist)
for x in alltherms:
print(x)
if 'errorCount' in x:
history[x] = history[x].astype('float')
history[x][history[x] < 0.0] = -1
history[x][history[x] > 1e5] = -1
print('%f %f' % (np.nanmin(history[x]), np.nanmax(history[x])))
elif '_records' in x:
history[x] = history[x].astype('float')
history[x][history[x] <= 0] = -1
history[x][history[x] > 1e7] = -1
elif '_goodrecords' in x:
history[x] = history[x].astype('float')
history[x][history[x] <= 0] = 0
history[x][history[x] > 1e7] = 0
plt.figure()
plt.title(inst + " error")
starts = history['times']
plt.plot(starts, history['thermal1_errorCount'], 'b', starts,
history['thermal2_errorCount'], 'r')
plt.legend(('thermal1', 'thermal2'))
plt.grid()
plt.figure()
plt.title(inst + " badcount")
plt.plot(starts,
history['thermal1_records'] - history['thermal1_goodrecords'],
'b', starts,
history['thermal2_records'] - history['thermal2_goodrecords'],
'r')
plt.legend(('thermal1', 'thermal2'))
plt.grid()
print('with all graphs timed at ' + repr(datetime.utcnow() - began))
if sig is not None:
with sig:
sig.notifyAll()
plt.show()
def livestream(instrument,
hours,
starttime=None,
process_control=None,
display_defaults='min_plots.json',
maxalt_km=15,
minalt_km=0,
now_delay=None):
"""
Quick and simple livestream of HSRL data using DPL constructs.
:param instrument: instrument name ('gvhsrl','bagohsrl', etc)
:param hours: numeric number of hours to display
:param starttime: optional starttime, if stream should begin in the past. it will still stream thru all data, and approach now regardless of this
:type starttime: datetime
:param process_control: process control file or object for processing parameters
:param display_defaults: graphics output configuration file
:param maxalt_km: maximum altitude in km. default is 15
:param minalt_km: minimum altitude in km. default is 0
:param now_delay: offset to consider the now time. "now" minus this value is the rolling end of the window.
:type now_delay: timedelta
"""
import lg_dpl_toolbox.filters.substruct as frame_substruct
frame_substruct.SubstructBrancher.multiprocessable = False
from hsrl.dpl.dpl_hsrl import dpl_hsrl
import hsrl.dpl.dpl_artists as artists
import lg_dpl_toolbox.dpl.dpl_artists as tools_artists
import lg_dpl_toolbox.filters.threading_filter as tf
window = timedelta(seconds=60 * 60 * hours)
print 'display_defaults=', display_defaults
#time.sleep(5)
figcontainer = None
sigg = waitforready(single_process=True)
dplobj = dpl_hsrl(instrument=instrument, process_control=process_control)
dplgen = dplobj(start_time_datetime=starttime,
reverse_padding=timedelta(
seconds=60) if now_delay == None else now_delay,
window_width_timedelta=window,
min_alt_m=minalt_km * 1000.0,
max_alt_m=maxalt_km * 1000.0,
with_profiles=False)
#dplgen=signalready_filter(dplgen,sigg)
dplgen = tf.forking_filter(dplgen, 2)
dplgen = tf.aThreading_filter(dplgen, 2)
#dplgen=signalready_filter(dplgen,sigg)
maxtimeout = 30.0
dplgen = tf.wait_filter(dplgen,
waitfunc=partial(plt.ginput,
timeout=maxtimeout / 6),
maxduration=timedelta(seconds=maxtimeout))
artist = artists.dpl_images_artist(
framestream=tools_artists.dpl_window_caching_filter(
dplgen, window, includeIncompleteFrames=True),
instrument=instrument,
max_alt=maxalt_km * 1000.0,
processing_defaults=dplgen.hsrl_process_control,
display_defaults=display_defaults,
figurecontainer=figcontainer)
#artist=tf.aThreading_filter(artist,1)
#sleeptime=10
#artist=input_filter(artist,"Pausing for %i seconds. Press enter to quit: " %(sleeptime),sleeptime,readier=sigg)
return artist
def makeL3Cal(inst, start, end, graphs=None, usel3slopecenter=True):
from hsrl.dpl.dpl_hsrl import dpl_hsrl
dpl = dpl_hsrl(inst)
r = dpl(start_time_datetime=start,
end_time_datetime=end,
min_alt_m=0,
max_alt_m=5000,
forimage=False,
raw_only=True,
with_profiles=False)
myvars = ('times', 'interf_freq', 'filtered_energy', 'nonfiltered_energy',
'l3cavityvoltage', 'l3locking_stats', 'superseedlasercontrollog',
'molecular_cal_pulse')
content = dict()
for x in r:
for v in myvars:
if not hasattr(x.rs_raw, v):
print 'no ', v
if v not in content:
content[v] = getattr(x.rs_raw, v).copy()
else:
content[v].append(getattr(x.rs_raw, v))
interfoffset = content['interf_freq']
seedoffset = content['superseedlasercontrollog'][:, 7] * r.hsrl_constants[
'seedlaser_temp_to_freq']
seedoffset *= 1e9
ratio = content['filtered_energy'][:, 0] / content['nonfiltered_energy'][:,
0]
l3slope = content['l3locking_stats'][:, 0].copy()
ridx = np.argmin(ratio)
sidx = np.argmin(np.abs(l3slope))
if usel3slopecenter:
idx = sidx
else:
idx = ridx
print sidx, ridx, l3slope[idx], abs(
seedoffset[sidx] - seedoffset[ridx]), abs(interfoffset[sidx] -
interfoffset[ridx])
seedoffset -= seedoffset[idx]
interfoffset -= interfoffset[idx]
pc = np.polyfit(l3slope, seedoffset, 5)
#pc[-1]=0.0
if graphs is not None:
#plt.subplot(411);plt.plot(content['times'],ratio)
if 2 in graphs:
plt.subplot(412)
plt.plot(content['times'], seedoffset, 'b', content['times'],
interfoffset, 'r')
if 3 in graphs:
plt.subplot(413)
plt.plot(content['times'], content['l3locking_stats'][:, 0], 'b')
if 1 in graphs:
plt.subplot(411)
plt.plot(seedoffset, ratio, 'b') #content['molecular_cal_pulse'])
if 4 in graphs:
seedfit = np.polyval(pc, l3slope)
plt.subplot(414)
plt.plot(seedoffset, l3slope, 'b', seedfit, l3slope, 'g')
print 'fit', pc
return pc
plt.show()
def skimcals(inst, start, end):
from time import sleep
from hsrl.dpl.dpl_hsrl import dpl_hsrl
from hsrl.dpl.calibration.dpl_calibration import dpl_singlecalibration_narr
import hsrl.dpl.dpl_artists as artists
import lg_dpl_toolbox.filters.time_frame as time_slicing
import lg_dpl_toolbox.filters.substruct as frame_substruct
import hsrl.graphics.hsrl_display as du
dropcontent = ['rs_raw', 'rs_mean']
storage = 'cal_verify'
try:
os.makedirs(storage)
except OSError:
pass
getfigs=('sc_ratio_profile','raw_profiles','dark_corrected_profiles','corrected_profiles','dif_geo_profiles',\
'wfov_geo_adjust','geometry_correction','calibration_coefficients','lapse_rate','backscat_profile','sounding')
preferredFormat = 'jpg'
preferredExtension = '.' + preferredFormat
withImage = True
display_defaults = 'all_plots.json'
print 'display_defaults=', display_defaults
#time.sleep(5)
dplobj = dpl_hsrl(instrument=inst)
minalt_km = 0
maxalt_km = 20
altres_m = 45.0
cals = dplobj.cal(interval_start_time=start,
interval_end_time=end,
min_alt_m=minalt_km * 1000.0,
max_alt_m=maxalt_km * 1000.0,
altres_m=altres_m)
filename = 'cal_log_' + inst + start.strftime(
'_%Y%m%dT%H%M%S') + end.strftime('_%Y%m%dT%H%M%S') + '.txt'
outf = file(os.path.join(storage, filename), "w")
cols = []
for calv in cals:
s = calv['chunk_start_time']
e = calv['chunk_end_time']
if (e - s).total_seconds() < 300:
continue
mycalgen = dpl_singlecalibration_narr(inst, calv,
cals.hsrl_process_control,
cals.corr_adjusts)
dplgen = dplobj(start_time_datetime=s,
end_time_datetime=e,
min_alt_m=minalt_km * 1000.0,
max_alt_m=maxalt_km * 1000.0,
with_profiles=True,
do_inversion=False,
calsrc=mycalgen)
dplgen = frame_substruct.DropFrameContent(dplgen, dropcontent)
dplgen = time_slicing.FrameCachedConcatenate(dplgen)
if withImage:
(disp, conf) = du.get_display_defaults(display_defaults)
for f in disp.get_attrs():
disp.set_value(f, 'enable', 0)
for f in getfigs:
disp.set_value(f, 'enable', 1)
dplgen = artists.dpl_images_artist(
framestream=dplgen,
instrument=inst,
max_alt=maxalt_km * 1000.0,
processing_defaults=dplgen.hsrl_process_control,
display_defaults=disp)
haveData = False
for rs in dplgen:
if hasattr(rs, 'profiles') and hasattr(rs.profiles, 'inv'):
logData(outf, cols, rs.profiles.inv)
haveData = True
if withImage and haveData:
figs = dplgen.figs
datetag = s.strftime('%Y%m%dT%H%M%S') + e.strftime(
'_%Y%m%dT%H%M%S')
for f in getfigs:
if f in figs:
print 'getting fig', f
fig = figs.figure(f)
hiresfile = os.path.join(
storage,
'%s_%s_%s%s' % (inst, f, datetag, preferredExtension))
print hiresfile
fig.savefig(hiresfile,
format=preferredFormat,
bbox_inches='tight')
def main():
from hsrl.dpl.dpl_hsrl import dpl_hsrl
from lg_dpl_toolbox.dpl.NetCDFZookeeper import GenericTemplateRemapNetCDFZookeeper
import radar.dpl.MMCRMergeLibrarian as mmcr
import radar.dpl.RadarFilters as rf
import hsrl.dpl.dpl_artists as artists
import functools
import substruct
import resample_altitude
starttime=datetime(2006,12,24,22,0,0)
endtime=datetime(2006,12,25,0,0,0)
#stitcher=TimeStitch()
#restr=SubstructRestractor('rs_mmcr')
hsrllib=dpl_hsrl(instrument='ahsrl')
hsrlnar=hsrllib(start_time_datetime=starttime,end_time_datetime=endtime,min_alt_m=0.0,max_alt_m=20000.0,timeres_timedelta=timedelta(seconds=30),altres_m=15)
mmcrzoo=GenericTemplateRemapNetCDFZookeeper('eurmmcrmerge')
mmcrlib=mmcr.MMCRMergeLibrarian('ahsrl','eurmmcrmerge.C1.c1.',zoo=mmcrzoo)
mmcrnar=rf.RadarPrefilter(mmcrlib(start=starttime,end=endtime))
#mmcrnar=mmcr.MMCRMergeBackscatterToReflectivity(resample_altitude.ResampleXd(TimeGinsu(substruct.SubstructExtractor(mmcrnar,None),'times',stitcherbase=stitcher),'heights',hsrlnar.getAltitudeAxis))
mmcrnar=rf.RadarBackscatterToReflectivity(resample_altitude.ResampleXd(TimeGinsu(mmcrnar,'times'),'heights',hsrlnar.altitudeAxis.copy()))
hsrlnarsplitter=substruct.SubstructBrancher(hsrlnar)
#hsrlnar=TimeGinsu(substruct.SubstructExtractor(hsrlnar,'rs_inv',restractor=restr),'times',isEnd=False,stitchersync=stitcher)
hsrlnar=TimeGinsu(hsrlnarsplitter.narrateSubstruct('rs_inv'),'times',isEnd=False)
from dplkit.simple.blender import TimeInterpolatedMerge
merge=TimeInterpolatedMerge(hsrlnar,[mmcrnar],allow_nans=True,channels=['heights','Reflectivity','MeanDopplerVelocity','Backscatter','SpectralWidth'])
merge=substruct.Retyper(merge,functools.partial(hau.Time_Z_Group,timevarname='times',altname='heights'))
#stitcher.setFramestream(merge)
#restr.setFramestream(stitcher)
#curs=restr
curs=substruct.SubstructMerger('rs_inv',{
'rs_mean':hsrlnarsplitter.narrateSubstruct('rs_mean'),
'rs_raw':hsrlnarsplitter.narrateSubstruct('rs_raw'),
'rs_inv':hsrlnarsplitter.narrateSubstruct('rs_inv'),
'rs_mmcr':merge,
'rs_init':hsrlnarsplitter.narrateSubstruct('rs_init'),
'rs_static':hsrlnarsplitter.narrateSubstruct('rs_static'),
'rs_Cxx':hsrlnarsplitter.narrateSubstruct('rs_Cxx'),
'profiles':hsrlnarsplitter.narrateSubstruct('profiles',sparse=True),
}
,hau.Time_Z_Group,{'timevarname':'times','altname':'heights'})
artist=artists.dpl_images_artist(framestream=curs,
instrument='ahsrl',max_alt=30*1000.0,processing_defaults=curs.hsrl_process_control,
display_defaults='all_plots.json')
curs=artist
for frame in curs:
print 'frame',frame
print 'frame keys',vars(frame).keys()
print 'rs_inv',frame.rs_inv
print 'rs_mmcr',frame.rs_mmcr
print 'RefShape',frame.rs_mmcr.Reflectivity.shape
print 'MMCRTimes',frame.rs_mmcr.times.shape
print 'invTimes',frame.rs_inv.times.shape
print 'heights',frame.rs_mmcr.heights.shape
for x in range(frame.rs_mmcr.times.shape[0]-1):
if frame.rs_mmcr.times[x]==frame.rs_mmcr.times[x+1]:
print 'dupe at',x,frame.rs_mmcr.times[x]
#print type(frame),type(frame['Reflectivity']) if 'Reflectivity' in frame else 'no ref',type(frame['beta_a_backscat_par']) if 'beta_a_backscat_par' in frame else 'no backscat'
time.sleep(5)