def run_calcs(df, ctx): """Do our maths.""" # Convert sea level pressure to station pressure df['pressure'] = mcalc.add_height_to_pressure( df['slp'].values * units('millibars'), ctx['nt'].sts[ctx['station']]['elevation'] * units('m')).to( units('millibar')) # Compute the relative humidity df['relh'] = mcalc.relative_humidity_from_dewpoint( df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF')) # Compute the mixing ratio df['mixingratio'] = mcalc.mixing_ratio_from_relative_humidity( df['relh'].values, df['tmpf'].values * units('degF'), df['pressure'].values * units('millibars')) # Compute the saturation mixing ratio df['saturation_mixingratio'] = mcalc.saturation_mixing_ratio( df['pressure'].values * units('millibars'), df['tmpf'].values * units('degF')) df['vapor_pressure'] = mcalc.vapor_pressure( df['pressure'].values * units('millibars'), df['mixingratio'].values * units('kg/kg')).to(units('kPa')) df['saturation_vapor_pressure'] = mcalc.vapor_pressure( df['pressure'].values * units('millibars'), df['saturation_mixingratio'].values * units('kg/kg')).to(units('kPa')) df['vpd'] = df['saturation_vapor_pressure'] - df['vapor_pressure'] group = df.groupby('year') df = group.aggregate(np.average) df['dwpf'] = mcalc.dewpoint(df['vapor_pressure'].values * units('kPa')).to( units('degF')).m return df
def mcalc_feelslike(tmpf, dwpf, smps): """Compute a feels like temperature Args: temperature (temperature): The dry bulb temperature dewpoint (temperature): The dew point temperature speed (speed): the wind speed Returns: temperature (temperature): The feels like temperature """ is_not_scalar = isinstance(tmpf.m, (list, tuple, np.ndarray)) rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf) # NB: update this once metpy 0.11 is released app = mcalc.apparent_temperature(tmpf, rh, smps) if hasattr(app, "mask"): if is_not_scalar: app[app.mask] = tmpf[app.mask] else: app = tmpf elif hasattr(tmpf, "mask"): app = masked_array(app.m, app.units) app.mask = tmpf.mask return app
def get_df(ctx): """Figure out what data we need to fetch here""" nt = NetworkTable(ctx['network']) ctx['ugc'] = nt.sts[ctx['station']]['ugc_zone'] pgconn = get_dbconn('postgis') events = read_sql(""" SELECT generate_series(issue, expire, '1 minute'::interval) as valid, (phenomena ||'.'|| significance) as vtec from warnings WHERE ugc = %s and ( (phenomena = 'EH' and significance = 'W') or (phenomena = 'HT' and significance = 'Y') ) ORDER by issue ASC """, pgconn, params=(ctx['ugc'], ), index_col='valid') if events.empty: raise ValueError("No Alerts were found for UGC: %s" % (ctx['ugc'], )) pgconn = get_dbconn('asos') obs = read_sql(""" SELECT valid, tmpf::int as tmpf, dwpf::int as dwpf from alldata where station = %s and valid > %s and tmpf > 70 and dwpf > -20 ORDER by valid """, pgconn, params=(ctx['station'], str(events.index.values[0])), index_col='valid') ctx['title'] = ("%s [%s] (%s to %s)\n" "Frequency of NWS Heat Headline for %s by Heat Index") % ( nt.sts[ctx['station']]['name'], ctx['station'], str(events.index.values[0])[:10], str(obs.index.values[-1])[:10], ctx['ugc']) relh = mcalc.relative_humidity_from_dewpoint( obs['tmpf'].values * munits.degF, obs['dwpf'].values * munits.degF) obs['feel'] = mcalc.heat_index(obs['tmpf'].values * munits.degF, relh).magnitude if ctx['opt'] == 'yes': obs = obs[obs['feel'] > obs['tmpf']] obs['feel'] = obs['feel'].round(0) res = obs.join(events) res.fillna('None', inplace=True) counts = res[['feel', 'vtec']].groupby(['feel', 'vtec']).size() df = pd.DataFrame(counts) df.columns = ['count'] df.reset_index(inplace=True) ctx['df'] = df.pivot(index='feel', columns='vtec', values='count') ctx['df'].fillna(0, inplace=True) ctx['df']['Total'] = ctx['df'].sum(axis=1) for vtec in ['HT.Y', 'EH.W', 'None']: if vtec not in ctx['df'].columns: ctx['df'][vtec] = 0. ctx['df'][vtec + "%"] = ctx['df'][vtec] / ctx['df']['Total'] * 100.
def mcalc_feelslike(tmpf, dwpf, smps): """Compute a feels like temperature Args: temperature (temperature): The dry bulb temperature dewpoint (temperature): The dew point temperature speed (speed): the wind speed Returns: temperature (temperature): The feels like temperature """ rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf) hidx = mcalc.heat_index(tmpf, rh, mask_undefined=True) wcht = mcalc.windchill(tmpf, smps, mask_undefined=True) # Where heat index is masked, replace with temperature hidx[hidx.mask] = tmpf[hidx.mask] # where ever wcht is not masked, replace with wind chill hidx[~ wcht.mask] = wcht[~ wcht.mask] return hidx
def mcalc_feelslike(tmpf, dwpf, smps): """Compute a feels like temperature Args: temperature (temperature): The dry bulb temperature dewpoint (temperature): The dew point temperature speed (speed): the wind speed Returns: temperature (temperature): The feels like temperature """ rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf) hidx = mcalc.heat_index(tmpf, rh, mask_undefined=True) wcht = mcalc.windchill(tmpf, smps, mask_undefined=True) # Where heat index is masked, replace with temperature hidx[hidx.mask] = tmpf[hidx.mask] # where ever wcht is not masked, replace with wind chill hidx[~wcht.mask] = wcht[~wcht.mask] return hidx
def main(): """Go Main Go.""" pgconn = get_dbconn('asos') df = read_sql(""" SELECT date_trunc('hour', valid) as ts, avg(tmpf) as temp, avg(dwpf) as dew from alldata where station = 'DSM' and extract(month from valid) in (5, 6, 7, 8) and tmpf is not null and dwpf is not null GROUP by ts """, pgconn, index_col=None) df['relh'] = relative_humidity_from_dewpoint( df['temp'].values * units('degF'), df['dew'].values * units('degF') ) * 100. df['year'] = df['ts'].dt.year counts = df[['year', 'relh']].groupby('year').count() df2 = df[df['relh'] >= 87.] counts2 = df2[['year', 'relh']].groupby('year').count() counts['hits'] = counts2['relh'] counts['freq'] = counts['hits'] / counts['relh'] counts.to_csv('test.csv')
def calc(self): """Compute things not usually computed""" if self.data["relh"] is None and None not in [ self.data["tmpf"], self.data["dwpf"], ]: self.data["relh"] = bounded( mcalc.relative_humidity_from_dewpoint( self.data["tmpf"] * munits.degF, self.data["dwpf"] * munits.degF, ).to(munits.percent).magnitude, 0.5, 100.5, ) if (self.data["dwpf"] is None and None not in [self.data["tmpf"], self.data["relh"]] and self.data["relh"] >= 1 and self.data["relh"] <= 100): self.data["dwpf"] = bounded( mcalc.dewpoint_from_relative_humidity( self.data["tmpf"] * munits.degF, self.data["relh"] * munits.percent, ).to(munits.degF).magnitude, -100.0, 100.0, ) if self.data["feel"] is None and None not in [ self.data["tmpf"], self.data["relh"], self.data["sknt"], ]: self.data["feel"] = bounded( mcalc.apparent_temperature( self.data["tmpf"] * munits.degF, self.data["relh"] * munits.percent, self.data["sknt"] * munits.knots, ).to(munits.degF).magnitude, -150.0, 200.0, )
def calc(self): """Compute things not usually computed""" if (self.data['relh'] is None and None not in [self.data['tmpf'], self.data['dwpf']]): self.data['relh'] = bounded( mcalc.relative_humidity_from_dewpoint( self.data['tmpf'] * munits.degF, self.data['dwpf'] * munits.degF).to(munits.percent).magnitude, 0.5, 100.5) if (self.data['dwpf'] is None and None not in [self.data['tmpf'], self.data['relh']] and self.data['relh'] >= 1 and self.data['relh'] <= 100): self.data['dwpf'] = bounded( mcalc.dewpoint_rh(self.data['tmpf'] * munits.degF, self.data['relh'] * munits.percent).to( munits.degF).magnitude, -100., 100.) if (self.data['feel'] is None and None not in [ self.data['tmpf'], self.data['relh'], self.data['sknt'] ]): self.data['feel'] = bounded( mcalc.apparent_temperature( self.data['tmpf'] * munits.degF, self.data['relh'] * munits.percent, self.data['sknt'] * munits.knots).to(munits.degF).magnitude, -150., 200.)
def test_relative_humidity_from_dewpoint(): """Test Relative Humidity calculation.""" assert_almost_equal( relative_humidity_from_dewpoint(25. * units.degC, 15. * units.degC), 53.80 * units.percent, 2)
def test_relative_humidity_from_dewpoint_xarray(): """Test Relative Humidity calculation with xarray data arrays.""" temp = xr.DataArray(25., attrs={'units': 'degC'}) dewp = xr.DataArray(15., attrs={'units': 'degC'}) assert_almost_equal(relative_humidity_from_dewpoint(temp, dewp), 53.80 * units.percent, 2)
def test_relative_humidity_from_dewpoint(): """Test Relative Humidity calculation.""" assert_almost_equal(relative_humidity_from_dewpoint(25. * units.degC, 15. * units.degC), 53.80 * units.percent, 2)
def Station_Synthetical_Forecast_From_Cassandra( model='ECMWF', output_dir=None, t_range=[0,84], t_gap=3, points={'lon':[116.3833], 'lat':[39.9]}, initTime=None, draw_VIS=True,drw_thr=False, extra_info={ 'output_head_name':' ', 'output_tail_name':' ', 'point_name':' '} ): #+get all the directories needed try: dir_rqd=[ "ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_3_HOURS/", "ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_6_HOURS/", "ECMWF_HR/TCDC/", "ECMWF_HR/LCDC/", "ECMWF_HR/UGRD_100M/", "ECMWF_HR/VGRD_100M/", "NWFD_SCMOC/VIS/", utl.Cassandra_dir( data_type='surface',data_source=model,var_name='RAIN03'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='RAIN06'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='T2m'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='u10m'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='v10m'), ] except KeyError: raise ValueError('Can not find all required directories needed') try: dir_opt=[ utl.Cassandra_dir( data_type='surface',data_source=model,var_name='Td2m') ] name_opt=['Td2m'] except: dir_opt=[ utl.Cassandra_dir(data_type='surface',data_source=model,var_name='rh2m') ] name_opt=['rh2m'] #+get all the directories needed if(initTime == None): last_file={model:get_latest_initTime(dir_rqd[0]), 'SCMOC':get_latest_initTime(dir_rqd[6]), } else: last_file={model:initTime[0], 'SCMOC':initTime[1], } y_s={model:int('20'+last_file[model][0:2]), 'SCMOC':int('20'+last_file['SCMOC'][0:2])} m_s={model:int(last_file[model][2:4]), 'SCMOC':int(last_file['SCMOC'][2:4])} d_s={model:int(last_file[model][4:6]), 'SCMOC':int(last_file['SCMOC'][4:6])} h_s={model:int(last_file[model][6:8]), 'SCMOC':int(last_file['SCMOC'][6:8])} fhours = np.arange(t_range[0], t_range[1], t_gap) for ifhour in fhours: if (ifhour == fhours[0] ): time_all=datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour)) else: time_all=np.append(time_all,datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour))) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] t2m=utl.get_model_points_gy(dir_rqd[9], filenames, points,allExists=False) if(name_opt[0] == 'rh2m'): rh2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False) Td2m=mpcalc.dewpoint_rh(t2m['data'].values*units('degC'),rh2m['data'].values/100.) p_vapor=(rh2m['data'].values/100.)*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values)))) if(name_opt[0] == 'Td2m'): Td2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False) rh2m=mpcalc.relative_humidity_from_dewpoint(t2m['data'].values* units('degC'), Td2m['data'].values* units('degC')) p_vapor=(np.array(rh2m))*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values)))) Td2m=np.array(Td2m['data'].values)* units('degC') u10m=utl.get_model_points_gy(dir_rqd[10], filenames, points,allExists=False) v10m=utl.get_model_points_gy(dir_rqd[11], filenames, points,allExists=False) wsp10m=(u10m['data']**2+v10m['data']**2)**0.5 AT=1.07*t2m['data'].values+0.2*p_vapor-0.65*wsp10m-2.7 if((t_range[1]) > 72): fhours = np.arange(6, t_range[1], 6) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] r03=utl.get_model_points_gy(dir_rqd[8], filenames, points,allExists=False) else: r03=utl.get_model_points_gy(dir_rqd[7], filenames, points,allExists=False) fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file['SCMOC']+'.'+str(fhour).zfill(3) for fhour in fhours] VIS=utl.get_model_points_gy(dir_rqd[6], filenames, points,allExists=False,fill_null=True,Null_value=-0.001) if(last_file['SCMOC'] == last_file[model] and t_range[1] > 72): fhours = np.append(np.arange(3,72,3),np.arange(72, (t_range[1]), 6)) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model] and t_range[1] > 60): fhours = np.append(np.arange(3,60,3),np.arange(60, (t_range[1]), 6)) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model] and t_range[1] <= 60): fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] == last_file[model] and t_range[1] <= 72): fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] TCDC=utl.get_model_points_gy(dir_rqd[2], filenames2, points,allExists=False) LCDC=utl.get_model_points_gy(dir_rqd[3], filenames2, points,allExists=False) u100m=utl.get_model_points_gy(dir_rqd[4], filenames2, points,allExists=False) v100m=utl.get_model_points_gy(dir_rqd[5], filenames2, points,allExists=False) wsp100m=(u100m['data']**2+v100m['data']**2)**0.5 if(fhours[-1] < 120): gust10m=utl.get_model_points_gy(dir_rqd[0], filenames, points,allExists=False) if(fhours[-1] > 120): if(last_file['SCMOC'] == last_file[model]): fhours = np.arange(0, t_range[1], 6) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model]): fhours = np.arange(0, t_range[1], 6) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] gust10m=utl.get_model_points_gy(dir_rqd[1], filenames, points,allExists=False) sta_graphics.draw_Station_Synthetical_Forecast_From_Cassandra( t2m=t2m,Td2m=Td2m,AT=AT,u10m=u10m,v10m=v10m,u100m=u100m,v100m=v100m, gust10m=gust10m,wsp10m=wsp10m,wsp100m=wsp100m,r03=r03,TCDC=TCDC,LCDC=LCDC, draw_VIS=draw_VIS,VIS=VIS,drw_thr=drw_thr, time_all=time_all, model=model,points=points, output_dir=output_dir,extra_info=extra_info)
def main(): ### START OF USER SETTINGS BLOCK ### # FILE/DATA SETTINGS # file path to input datafile = '/home/jgodwin/python/sfc_observations/surface_observations.txt' timefile = '/home/jgodwin/python/sfc_observations/validtime.txt' # MAP SETTINGS # map names (for tracking purposes) maps = ['CONUS','Texas','Floater 1'] restart_domain = [True,False] # map boundaries west = [-120,-108,-108] east = [-70,-93,-85] south = [20,25,37] north = [50,38,52] # OUTPUT SETTINGS # save directory for output savedir = '/var/www/html/images/' # filenames ("_[variable].png" will be appended, so only a descriptor like "conus" is needed) savenames = ['conus','texas','floater1'] # TEST MODE SETTINGS test = False testnum = 3 ### END OF USER SETTINGS BLOCK ### for i in range(len(maps)): if test and i != testnum: continue print(maps[i]) # create the map projection cenlon = (west[i] + east[i]) / 2.0 cenlat = (south[i] + north[i]) / 2.0 sparallel = cenlat if cenlat > 0: cutoff = -30 flip = False elif cenlat < 0: cutoff = 30 flip = True if restart_domain: to_proj = ccrs.LambertConformal(central_longitude=cenlon,central_latitude=cenlat,standard_parallels=[sparallel],cutoff=cutoff) # open the data vt = open(timefile).read() with open(datafile) as f: data = pd.read_csv(f,header=0,names=['siteID','lat','lon','elev','slp','temp','sky','dpt','wx','wdr',\ 'wsp'],na_values=-99999) # filter data by lat/lon data = data[(data['lat'] >= south[i]-2.0) & (data['lat'] <= north[i]+2.0) & (data['lon'] >= west[i]-2.0)\ & (data['lon'] <= east[i]+2.0)] # remove questionable data data = data[(cToF(data['temp']) <= 120) & (cToF(data['dpt']) <= 80)] # project lat/lon onto final projection print("Creating map projection.") lon = data['lon'].values lat = data['lat'].values xp, yp, _ = to_proj.transform_points(ccrs.Geodetic(), lon, lat).T # remove missing data from pressure and interpolate # we'll give this a try and see if it can help with my CPU credit problem if restart_domain: print("Performing Cressman interpolation.") x_masked, y_masked, pres = remove_nan_observations(xp, yp, data['slp'].values) slpgridx, slpgridy, slp = interpolate_to_grid(x_masked, y_masked, pres, interp_type='cressman', minimum_neighbors=1, search_radius=400000, hres=100000) # get wind information and remove missing data wind_speed = (data['wsp'].values * units('knots')) wind_dir = data['wdr'].values * units.degree good_indices = np.where((~np.isnan(wind_dir)) & (~np.isnan(wind_speed))) x_masked = xp[good_indices] y_masked = yp[good_indices] wind_speed = wind_speed[good_indices] wind_dir = wind_dir[good_indices] u, v = wind_components(wind_speed, wind_dir) windgridx, windgridy, uwind = interpolate_to_grid(x_masked, y_masked, np.array(u), interp_type='cressman', search_radius=400000, hres=100000) _, _, vwind = interpolate_to_grid(x_masked, y_masked, np.array(v), interp_type='cressman', search_radius=400000, hres=100000) # get temperature information data['temp'] = cToF(data['temp']) x_masked, y_masked, t = remove_nan_observations(xp, yp, data['temp'].values) tempx, tempy, temp = interpolate_to_grid(x_masked, y_masked, t, interp_type='cressman', minimum_neighbors=3, search_radius=200000, hres=18000) temp = np.ma.masked_where(np.isnan(temp), temp) # get dewpoint information data['dpt'] = cToF(data['dpt']) x_masked,y_masked,td = remove_nan_observations(xp,yp,data['dpt'].values) dptx,dpty,dewp = interpolate_to_grid(x_masked,y_masked,td,interp_type='cressman',\ minimum_neighbors=3,search_radius=200000,hres=18000) dewp = np.ma.masked_where(np.isnan(dewp),dewp) # interpolate wind speed x_masked,y_masked,wspd = remove_nan_observations(xp,yp,data['wsp'].values) wspx,wspy,speed = interpolate_to_grid(x_masked,y_masked,wspd,interp_type='cressman',\ minimum_neighbors=3,search_radius=200000,hres=18000) speed = np.ma.masked_where(np.isnan(speed),speed) # derived values # station pressure data['pres'] = stationPressure(data['slp'],data['elev']) # theta-E data['thetae'] = equivalent_potential_temperature(data['pres'].values*units.hPa,data['temp'].values*units.degF,data['dpt'].values*units.degF) x_masked,y_masked,thetae = remove_nan_observations(xp,yp,data['thetae'].values) thex,they,thte = interpolate_to_grid(x_masked,y_masked,thetae,interp_type='cressman',\ minimum_neighbors=3,search_radius=200000,hres=18000) thte = np.ma.masked_where(np.isnan(thte),thte) # mixing ratio relh = relative_humidity_from_dewpoint(data['temp'].values*units.degF,data['dpt'].values*units.degF) mixr = mixing_ratio_from_relative_humidity(relh,data['temp'].values*units.degF,data['pres'].values*units.hPa) * 1000.0 x_masked,y_masked,mixrat = remove_nan_observations(xp,yp,mixr) mrx,mry,mrat = interpolate_to_grid(x_masked,y_masked,mixrat,interp_type='cressman',\ minimum_neighbors=3,search_radius=200000,hres=18000) mrat = np.ma.masked_where(np.isnan(mrat),mrat) # set up the state borders state_boundaries = cfeature.NaturalEarthFeature(category='cultural',\ name='admin_1_states_provinces_lines',scale='50m',facecolor='none') # SCALAR VARIABLES TO PLOT # variable names (will appear in plot title) variables = ['Temperature','Dewpoint','Wind Speed','Theta-E','Mixing Ratio'] # units (for colorbar label) unitlabels = ['F','F','kt','K','g/kg'] # list of actual variables to plot vardata = [temp,dewp,speed,thte,mrat] # tag in output filename varplots = ['temp','dewp','wspd','thte','mrat'] # levels: (lower,upper,step) levs = [[-20,105,5],[30,85,5],[0,70,5],[250,380,5],[0,22,2]] # colormaps colormaps = ['hsv_r','Greens','plasma','hsv_r','Greens'] for j in range(len(variables)): print("\t%s" % variables[j]) fig = plt.figure(figsize=(20, 10)) view = fig.add_subplot(1, 1, 1, projection=to_proj) # set up the map and plot the interpolated grids levels = list(range(levs[j][0],levs[j][1],levs[j][2])) cmap = plt.get_cmap(colormaps[j]) norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) labels = variables[j] + " (" + unitlabels[j] + ")" # add map features view.set_extent([west[i],east[i],south[i],north[i]]) view.add_feature(state_boundaries,edgecolor='black') view.add_feature(cfeature.OCEAN,zorder=-1) view.add_feature(cfeature.COASTLINE,zorder=2) view.add_feature(cfeature.BORDERS, linewidth=2,edgecolor='black') # plot the sea-level pressure cs = view.contour(slpgridx, slpgridy, slp, colors='k', levels=list(range(990, 1034, 4))) view.clabel(cs, inline=1, fontsize=12, fmt='%i') # plot the scalar background mmb = view.pcolormesh(tempx, tempy, vardata[j], cmap=cmap, norm=norm) fig.colorbar(mmb, shrink=.4, orientation='horizontal', pad=0.02, boundaries=levels, \ extend='both',label=labels) # plot the wind barbs view.barbs(windgridx, windgridy, uwind, vwind, alpha=.4, length=5,flip_barb=flip) # plot title and save view.set_title('%s (shaded), SLP, and Wind (valid %s)' % (variables[j],vt)) plt.savefig('/var/www/html/images/%s_%s.png' % (savenames[i],varplots[j]),bbox_inches='tight') # close everything fig.clear() view.clear() plt.close(fig) f.close() print("Script finished.")
def test_relative_humidity_from_dewpoint_with_f(): """Test Relative Humidity accepts temperature in Fahrenheit.""" assert_almost_equal(relative_humidity_from_dewpoint(70. * units.degF, 55. * units.degF), 58.935 * units.percent, 3)
def do(ts): """Process this date timestamp""" asos = get_dbconn('asos', user='******') iemaccess = get_dbconn('iem') icursor = iemaccess.cursor() df = read_sql(""" select station, network, iemid, drct, sknt, valid at time zone tzname as localvalid, tmpf, dwpf from alldata d JOIN stations t on (t.id = d.station) where (network ~* 'ASOS' or network = 'AWOS') and valid between %s and %s and t.tzname is not null and date(valid at time zone tzname) = %s ORDER by valid ASC """, asos, params=(ts - datetime.timedelta(days=2), ts + datetime.timedelta(days=2), ts.strftime("%Y-%m-%d")), index_col=None) # derive some parameters df['relh'] = mcalc.relative_humidity_from_dewpoint( df['tmpf'].values * munits.degF, df['dwpf'].values * munits.degF).to(munits.percent) df['feel'] = mcalc.apparent_temperature( df['tmpf'].values * munits.degF, df['relh'].values * munits.percent, df['sknt'].values * munits.knots ) df['u'], df['v'] = mcalc.get_wind_components( df['sknt'].values * munits.knots, df['drct'].values * munits.deg ) df['localvalid_lag'] = df.groupby('iemid')['localvalid'].shift(1) df['timedelta'] = df['localvalid'] - df['localvalid_lag'] ndf = df[pd.isna(df['timedelta'])] df.loc[ndf.index.values, 'timedelta'] = pd.to_timedelta( ndf['localvalid'].dt.hour * 3600. + ndf['localvalid'].dt.minute * 60., unit='s' ) df['timedelta'] = df['timedelta'] / np.timedelta64(1, 's') table = "summary_%s" % (ts.year,) for iemid, gdf in df.groupby('iemid'): if len(gdf.index) < 6: # print(" Quorum not meet for %s" % (gdf.iloc[0]['station'], )) continue ldf = gdf.copy() ldf.interpolate(inplace=True) totsecs = ldf['timedelta'].sum() avg_rh = clean((ldf['relh'] * ldf['timedelta']).sum() / totsecs, 1, 100) min_rh = clean(ldf['relh'].min(), 1, 100) max_rh = clean(ldf['relh'].max(), 1, 100) uavg = (ldf['u'] * ldf['timedelta']).sum() / totsecs vavg = (ldf['u'] * ldf['timedelta']).sum() / totsecs drct = clean( mcalc.get_wind_dir(uavg * munits.knots, vavg * munits.knots), 0, 360) avg_sknt = clean( (ldf['sknt'] * ldf['timedelta']).sum() / totsecs, 0, 150 # arb ) max_feel = clean(ldf['feel'].max(), -150, 200) avg_feel = clean( (ldf['feel'] * ldf['timedelta']).sum() / totsecs, -150, 200 ) min_feel = clean(ldf['feel'].min(), -150, 200) def do_update(): """Inline updating""" icursor.execute(""" UPDATE """ + table + """ SET avg_rh = %s, min_rh = %s, max_rh = %s, avg_sknt = %s, vector_avg_drct = %s, min_feel = %s, avg_feel = %s, max_feel = %s WHERE iemid = %s and day = %s """, (avg_rh, min_rh, max_rh, avg_sknt, drct, min_feel, avg_feel, max_feel, iemid, ts)) do_update() if icursor.rowcount == 0: print(('compute_daily Adding %s for %s %s %s' ) % (table, gdf.iloc[0]['station'], gdf.iloc[0]['network'], ts)) icursor.execute(""" INSERT into """ + table + """ (iemid, day) values (%s, %s) """, (iemid, ts)) do_update() icursor.close() iemaccess.commit() iemaccess.close()
def plotter(fdict): """ Go """ pgconn = get_dbconn("asos") ctx = get_autoplot_context(fdict, get_description()) station = ctx["zstation"] highlightyear = ctx["year"] sdate = datetime.date(ctx["syear"], 1, 1) edate = datetime.date(ctx["eyear"] + 1, 1, 1) ytd = ctx["ytd"] varname = ctx["var"] inc = ctx["inc"] doylimiter = get_doylimit(ytd, varname) tmpflimit = "and tmpf >= 50" if varname != "windchill" else "and tmpf < 50" if varname not in ["windchill", "heatindex"]: tmpflimit = "" df = read_sql( "SELECT to_char(valid, 'YYYYmmddHH24') as d, avg(tmpf)::int as tmpf, " "avg(dwpf)::int as dwpf, avg(coalesce(sknt, 0)) as sknt " f"from alldata WHERE station = %s {tmpflimit} " "and dwpf <= tmpf and valid > %s and valid < %s and report_type = 2 " f"{doylimiter} GROUP by d", pgconn, params=(station, sdate, edate), index_col=None, ) if df.empty: raise NoDataFound("No Data Found.") df["year"] = df["d"].apply(lambda x: int(x[:4])) df2 = df title2 = VDICT[varname] compop = np.greater_equal inctitle = "" if varname == "heatindex": df["heatindex"] = (heat_index( df["tmpf"].values * units("degF"), relative_humidity_from_dewpoint( df["tmpf"].values * units("degF"), df["dwpf"].values * units("degF"), ), ).to(units("degF")).m) inctitle = " [All Obs Included]" if inc == "no": df2 = df[df["heatindex"] > df["tmpf"]] inctitle = " [Only Additive]" else: df2 = df maxval = int(df2["heatindex"].max() + 1) LEVELS[varname] = np.arange(80, maxval) elif varname == "windchill": compop = np.less_equal df["year"] = df["d"].apply(lambda x: (int(x[:4]) - 1) if int(x[4:6]) < 7 else int(x[:4])) df["windchill"] = (windchill( df["tmpf"].values * units("degF"), df["sknt"].values * units("knot"), ).to(units("degF")).m) inctitle = " [All Obs Included]" if inc == "no": df2 = df[df["windchill"] < df["tmpf"]] inctitle = " [Only Additive]" else: df2 = df minval = int(df2["windchill"].min() - 1) LEVELS[varname] = np.arange(minval, minval + 51) else: maxval = int(df2[varname].max() + 1) LEVELS[varname] = np.arange(maxval - 31, maxval) bs = ctx["_nt"].sts[station]["archive_begin"] if bs is None: raise NoDataFound("Unknown station metadata.") minyear = df["year"].min() maxyear = df["year"].max() years = float((maxyear - minyear) + 1) x = [] y = [] y2 = [] fig = plt.figure(figsize=(9, 6)) ax = fig.add_axes([0.1, 0.1, 0.6, 0.8]) yloc = 1.0 xloc = 1.13 yrlabel = ("%s" % (highlightyear, ) if varname != "windchill" else "%s-%s" % (highlightyear, highlightyear + 1)) ax.text(xloc + 0.08, yloc + 0.04, "Avg:", transform=ax.transAxes, color="b") ax.text(xloc + 0.21, yloc + 0.04, yrlabel, transform=ax.transAxes, color="r") df3 = df2[df2["year"] == highlightyear] for level in LEVELS[varname]: x.append(level) y.append(len(df2[compop(df2[varname], level)]) / years) y2.append(len(df3[compop(df3[varname], level)])) if level % 2 == 0: ax.text(xloc, yloc, "%s" % (level, ), transform=ax.transAxes) ax.text( xloc + 0.08, yloc, "%.1f" % (y[-1], ), transform=ax.transAxes, color="b", ) ax.text( xloc + 0.21, yloc, "%.0f" % (y2[-1], ), transform=ax.transAxes, color="r", ) yloc -= 0.04 ax.text(xloc, yloc, "n=%s" % (len(df2.index), ), transform=ax.transAxes) for x0, y0, y02 in zip(x, y, y2): ax.plot([x0, x0], [y0, y02], color="k") rdf = pd.DataFrame({"level": x, "avg": y, "d%s" % (highlightyear, ): y2}) x = np.array(x, dtype=np.float64) ax.scatter(x, y, color="b", label="Avg") ax.scatter(x, y2, color="r", label=yrlabel) ax.grid(True) ymax = int(max([max(y), max(y2)])) ax.set_xlim(x[0] - 0.5, x[-1] + 0.5) dy = 24 * (int(ymax / 240) + 1) ax.set_yticks(range(0, ymax, dy)) ax.set_ylim(-0.5, ymax + 5) ax2 = ax.twinx() ax2.set_ylim(-0.5, ymax + 5) ax2.set_yticks(range(0, ymax, dy)) ax2.set_yticklabels(["%.0f" % (s, ) for s in np.arange(0, ymax, dy) / 24]) ax2.set_ylabel("Expressed in 24 Hour Days") ax.set_ylabel("Hours Per Year") ax.set_xlabel(r"%s $^\circ$F" % (VDICT[varname], )) title = "till %s" % (datetime.date.today().strftime("%-d %b"), ) title = "Entire Year" if ytd == "no" else title ax.set_title(("[%s] %s %s-%s\n" "%s Histogram (%s)%s") % ( station, ctx["_nt"].sts[station]["name"], minyear, maxyear, title2, title, inctitle, )) ax.legend(loc="best", scatterpoints=1) return fig, rdf
def get_weather(self): #------------------------------------------------------ #------------------------------------------------------ # Load TMY2 if self.file_ext == TMY2EXT: f = open(self.weatherpath + self.city + self.file_ext, 'r') # Header read for lat and lon head = f.readline() self.lat = int(head[39:41]) + int(head[42:44]) / 60.0 self.lon = int(head[47:50]) + int(head[51:53]) / 60.0 line = f.readline() ind = 0 while line: # Process the line self.tothor[ind] = float( line[17:21]) #Total horizontal solar Wh/m2 self.dirnorm[ind] = float( line[23:27]) #Direct normal solar Wh/m2 self.difhor[ind] = float( line[29:33]) #Diffuse Horizontal Solar Wh/m2 self.tdry[ind] = float(line[67:71]) * 0.1 #tdrybulb (deg C) self.rhs[ind] = float(line[79:82]) * 0.01 #relative humidity (%) self.tdew[ind] = float(line[73:77]) * 0.1 #tdew (deg C) to conform with TB code self.press[ind] = float(line[84:88]) #atmospheric pressure (mbar) mb = 100 Pascals #self.wind_speed[ind] = float(line[95:98]) * 0.1; #windspeed m/s #self.wind_dir[ind] = float(line[90:93]); #wind direction azimuth #self.cloud[ind] = float(line[59:61])/10.0; #Could cover fraction #wd.ocloud = getint(line,63,2)/10.0; #Opaque cloud cover fraction #wd.ceilht = getint(line,106,5); #Cloud ceiling height m # Calculate specfic humidity from dry bulb, dew point, and atm pressure using MetPy self.huss[ind] = mpcalc.specific_humidity_from_mixing_ratio( mpcalc.mixing_ratio_from_relative_humidity( mpcalc.relative_humidity_from_dewpoint( self.tdry[ind] * units.degC, self.tdew[ind] * units.degC), self.tdry[ind] * units.degC, self.press[ind] * units.mbar)) #Next line line = f.readline() ind = ind + 1 f.close() #------------------------------------------------------ #------------------------------------------------------ # Load TMY3 elif self.file_ext == TMY3EXT: f = open( self.weatherpath + TMY3NUMBER[CITY.index(self.city)] + self.file_ext, 'r') # Header read for lat and lon head = f.readline().split(',') self.lat = float(head[4]) self.lon = float(head[5]) #Burn a line for the second part of the header. line = f.readline() line = f.readline() ind = 0 while line: line = line.split(',') if len(line) < 20: print('line is short!') # Process the line self.tothor[ind] = float( line[4]) #Total horizontal solar Wh/m2 self.dirnorm[ind] = float(line[7]) #Direct normal solar Wh/m2 self.difhor[ind] = float( line[10]) #Diffuse Horizontal Solar Wh/m2 self.tdry[ind] = float(line[31]) #tdrybulb (deg C) self.rhs[ind] = float(line[37]) * 0.01 #relative humidity (%) self.tdew[ind] = float(line[34]) #tdew (deg C) to conform with TB code self.press[ind] = float(line[40]) #atmospheric pressure (mbar) mb = 100 Pascals #self.wind_speed[ind] = float(line[46]); #windspeed m/s #self.wind_dir[ind] = float(line[43]); #wind direction azimuth # Calculate specfic humidity from dry bulb, dew point, and atm pressure using MetPy self.huss[ind] = mpcalc.specific_humidity_from_mixing_ratio( mpcalc.mixing_ratio_from_relative_humidity( mpcalc.relative_humidity_from_dewpoint( self.tdry[ind] * units.degC, self.tdew[ind] * units.degC), self.tdry[ind] * units.degC, self.press[ind] * units.mbar)) #Next line line = f.readline() ind = ind + 1 f.close()
def append_cfs(res, lon, lat): """Append on needed CFS data.""" gridx, gridy = find_ij(lon, lat) lastyear = max(res["data"].keys()) thisyear = datetime.date.today().year lastdate = datetime.date(thisyear, 8, 31) if lastyear != thisyear: # We don't have any data yet for this year, so we add some res["data"][thisyear] = {"dates": [], "high": [], "low": [], "rh": []} else: # shrug if res["data"][lastyear]["dates"]: lastdate = datetime.datetime.strptime( res["data"][thisyear]["dates"][-1], "%Y-%m-%d" ).date() # go find the most recent CFS 0z file valid = datetime.date.today() attempt = 0 while True: testfn = valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d00.nc") if os.path.isfile(testfn): break valid -= datetime.timedelta(hours=24) attempt += 1 if attempt > 9: return None try: nc = ncopen(testfn, timeout=NCOPEN_TIMEOUT) except Exception as exp: LOG.error(exp) return None if nc is None: LOG.debug("Failing %s as nc is None", testfn) return None high = ( masked_array(nc.variables["high_tmpk"][:, gridy, gridx], units.degK) .to(units.degF) .m ) low = ( masked_array(nc.variables["low_tmpk"][:, gridy, gridx], units.degK) .to(units.degF) .m ) # RH hack # found ~20% bias with this value, so arb addition for now rh = ( relative_humidity_from_dewpoint( masked_array(high, units.degF), masked_array(low, units.degF) ).m * 100.0 + 20.0 ) rh = np.where(rh > 95, 95, rh) entry = res["data"][thisyear] # lastdate is either August 31 or a date after, so our first forecast # date is i+1 tidx = daily_offset(lastdate + datetime.timedelta(days=1)) for i in range(tidx, 365): lts = datetime.date(thisyear, 1, 1) + datetime.timedelta(days=i) if lts.month in [9, 10, 11]: entry["dates"].append(lts.strftime("%Y-%m-%d")) entry["high"].append(_i(high[i])) entry["low"].append(_i(low[i])) entry["rh"].append(_i(rh[i])) return res
#Load wind gust data da_wg = xr.open_dataset("/g/data/ma05/" + model + "/v1/forecast/spec/max_wndgust10m/" + year + "/" + month + "/max_wndgust10m-fc-spec-PT1H-" + model + "-v1-" + date + ".sub.nc").sel({ "time": dt.datetime.strptime(valid, "%Y-%m-%d %H:%M") + dt.timedelta(hours=wg_lead) }).max_wndgust10m #Calculate a few thermodynamic variables dp = get_dp(hur=da_rh, ta=da_ta, dp_mask=False) ta_unit = units.units.degC * da_ta dp_unit = units.units.degC * dp p_unit = units.units.hectopascals * p_3d hur_unit = mpcalc.relative_humidity_from_dewpoint(ta_unit, dp_unit)*\ 100*units.units.percent q_unit = mpcalc.mixing_ratio_from_relative_humidity(hur_unit,\ ta_unit,p_unit) #plot m = Basemap(llcrnrlon=lon.min(), llcrnrlat=lat.min(), urcrnrlon=lon.max(), urcrnrlat=lat.max(), projection="cyl", resolution="h") #omega = wrf.omega(q_unit, ta_unit.to("K"), da_w.values, p_3d*100) #omega = omega.assign_coords(dim_0=p, dim_2=lon, dim_1=lat) #omega = xr.where((da_z >= 1000) & (da_z <= 3000), omega, np.nan) #c = xr.plot.contour(omega.mean("dim_0"), levels=[10,20,50], colors=["#d0d1e6","#3690c0","#034e7b"])
xsect = radisonde_cross_section(stn_list, df) ###################################################################### # Calculate Variables for Plotting # -------------------------------- # # Use MetPy to calculate common variables for plotting a cross section, # specifically potential temperature and mixing ratio # potemp = mpcalc.potential_temperature( xsect['p_grid'] * units('hPa'), xsect['grid_data']['temperature'] * units('degC')) relhum = mpcalc.relative_humidity_from_dewpoint( xsect['grid_data']['temperature'] * units('degC'), xsect['grid_data']['dewpoint'] * units('degC')) mixrat = mpcalc.mixing_ratio_from_relative_humidity( relhum, xsect['grid_data']['temperature'] * units('degC'), xsect['p_grid'] * units('hPa')) ###################################################################### # Plot Cross Section # ------------------ # # Use standard Matplotlib to plot the now 2D cross section grid using the # data from xsect and those calculated above. Additionally, the actualy # radiosonde wind observations are plotted as barbs on this plot. #
def test_relative_humidity_from_dewpoint_with_f(): """Test Relative Humidity accepts temperature in Fahrenheit.""" assert_almost_equal( relative_humidity_from_dewpoint(70. * units.degF, 55. * units.degF), 58.935 * units.percent, 3)
def plotter(fdict): """ Go """ pgconn = get_dbconn('asos') ctx = get_autoplot_context(fdict, get_description()) station = ctx['zstation'] network = ctx['network'] nt = NetworkTable(network) year = ctx['year'] varname = ctx['var'] df = read_sql(""" SELECT extract(year from valid) as year, coalesce(mslp, alti * 33.8639, 1013.25) as slp, extract(doy from valid) as doy, tmpf, dwpf from alldata where station = %s and dwpf > -50 and dwpf < 90 and tmpf > -50 and tmpf < 120 and valid > '1950-01-01' and report_type = 2 """, pgconn, params=(station, ), index_col=None) # compute RH df['relh'] = mcalc.relative_humidity_from_dewpoint( df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF')) # saturation vapor pressure # Convert sea level pressure to station pressure df['pressure'] = mcalc.add_height_to_pressure( df['slp'].values * units('millibars'), nt.sts[station]['elevation'] * units('m')).to(units('millibar')) # Compute the relative humidity df['relh'] = mcalc.relative_humidity_from_dewpoint( df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF')) # Compute the mixing ratio df['mixing_ratio'] = mcalc.mixing_ratio_from_relative_humidity( df['relh'].values, df['tmpf'].values * units('degF'), df['pressure'].values * units('millibars')) # Compute the saturation mixing ratio df['saturation_mixingratio'] = mcalc.saturation_mixing_ratio( df['pressure'].values * units('millibars'), df['tmpf'].values * units('degF')) df['vapor_pressure'] = mcalc.vapor_pressure( df['pressure'].values * units('millibars'), df['mixing_ratio'].values * units('kg/kg')).to(units('kPa')) df['saturation_vapor_pressure'] = mcalc.vapor_pressure( df['pressure'].values * units('millibars'), df['saturation_mixingratio'].values * units('kg/kg')).to(units('kPa')) df['vpd'] = df['saturation_vapor_pressure'] - df['vapor_pressure'] dailymeans = df[['year', 'doy', varname]].groupby(['year', 'doy']).mean() dailymeans = dailymeans.reset_index() df2 = dailymeans[['doy', varname]].groupby('doy').describe() dyear = df[df['year'] == year] df3 = dyear[['doy', varname]].groupby('doy').describe() df3[(varname, 'diff')] = df3[(varname, 'mean')] - df2[(varname, 'mean')] (fig, ax) = plt.subplots(2, 1, figsize=(8, 6)) multiplier = 1000. if varname == 'mixing_ratio' else 10. ax[0].fill_between(df2[(varname, 'min')].index.values, df2[(varname, 'min')].values * multiplier, df2[(varname, 'max')].values * multiplier, color='gray') ax[0].plot(df2[(varname, 'mean')].index.values, df2[(varname, 'mean')].values * multiplier, label="Climatology") ax[0].plot(df3[(varname, 'mean')].index.values, df3[(varname, 'mean')].values * multiplier, color='r', label="%s" % (year, )) ax[0].set_title(("%s [%s]\nDaily Mean Surface %s") % (station, nt.sts[station]['name'], PDICT[varname])) lbl = ("Mixing Ratio ($g/kg$)" if varname == 'mixing_ratio' else PDICT[varname]) ax[0].set_ylabel(lbl) ax[0].set_xlim(0, 366) ax[0].set_ylim(bottom=0) ax[0].set_xticks( (1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 365)) ax[0].set_xticklabels(calendar.month_abbr[1:]) ax[0].grid(True) ax[0].legend(loc=2, fontsize=10) cabove = 'b' if varname == 'mixing_ratio' else 'r' cbelow = 'r' if cabove == 'b' else 'b' rects = ax[1].bar(df3[(varname, 'diff')].index.values, df3[(varname, 'diff')].values * multiplier, facecolor=cabove, edgecolor=cabove) for rect in rects: if rect.get_height() < 0.: rect.set_facecolor(cbelow) rect.set_edgecolor(cbelow) plunits = '$g/kg$' if varname == 'mixing_ratio' else 'hPa' ax[1].set_ylabel("%.0f Departure (%s)" % (year, plunits)) ax[1].set_xlim(0, 366) ax[1].set_xticks( (1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 365)) ax[1].set_xticklabels(calendar.month_abbr[1:]) ax[1].grid(True) return fig, df3
def main(): """Do Something""" pgconn = get_dbconn("iem") icursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor) fn = find_file() nc = ncopen(fn, timeout=300) stations = chartostring(nc.variables["stationId"][:]) providers = chartostring(nc.variables["dataProvider"][:]) names = chartostring(nc.variables["stationName"][:]) tmpk = nc.variables["temperature"][:] dwpk = nc.variables["dewpoint"][:] # Set some data bounds to keep mcalc from complaining dwpk = np.ma.where( np.ma.logical_or(np.ma.less(dwpk, 200), np.ma.greater(dwpk, 320)), np.nan, dwpk, ) tmpk = np.ma.where( np.ma.logical_or(np.ma.less(tmpk, 200), np.ma.greater(tmpk, 320)), np.nan, tmpk, ) relh = (mcalc.relative_humidity_from_dewpoint( tmpk * units.degK, dwpk * units.degK).magnitude * 100.0) obtime = nc.variables["observationTime"][:] pressure = nc.variables["stationPressure"][:] # altimeter = nc.variables["altimeter"][:] # slp = nc.variables["seaLevelPressure"][:] drct = nc.variables["windDir"][:] smps = nc.variables["windSpeed"][:] gmps = nc.variables["windGust"][:] # gmps_drct = nc.variables["windDirMax"][:] pcpn = nc.variables["precipAccum"][:] rtk1 = nc.variables["roadTemperature1"][:] rtk2 = nc.variables["roadTemperature2"][:] rtk3 = nc.variables["roadTemperature3"][:] rtk4 = nc.variables["roadTemperature4"][:] subk1 = nc.variables["roadSubsurfaceTemp1"][:] nc.close() db = {} for recnum, provider in enumerate(providers): this_station = stations[recnum] if not provider.endswith("DOT") and provider not in MY_PROVIDERS: continue name = names[recnum] if name == "": continue if provider == "MesoWest": # get the network from the last portion of the name network = name.split()[-1] if network != "VTWAC": continue else: network = provider2network(provider) if network is None: continue db[this_station] = {} ticks = obtime[recnum] ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks) db[this_station]["ts"] = ts.replace(tzinfo=pytz.utc) db[this_station]["network"] = network db[this_station]["pres"] = sanity_check(pressure[recnum], 0, 1000000) db[this_station]["tmpk"] = sanity_check(tmpk[recnum], 200, 330) db[this_station]["dwpk"] = sanity_check(dwpk[recnum], 200, 330) db[this_station]["relh"] = relh[recnum] db[this_station]["drct"] = sanity_check(drct[recnum], -1, 361) db[this_station]["smps"] = sanity_check(smps[recnum], -1, 200) db[this_station]["gmps"] = sanity_check(gmps[recnum], -1, 200) db[this_station]["rtk1"] = sanity_check(rtk1[recnum], 0, 500) db[this_station]["rtk2"] = sanity_check(rtk2[recnum], 0, 500) db[this_station]["rtk3"] = sanity_check(rtk3[recnum], 0, 500) db[this_station]["rtk4"] = sanity_check(rtk4[recnum], 0, 500) db[this_station]["subk"] = sanity_check(subk1[recnum], 0, 500) db[this_station]["pday"] = sanity_check(pcpn[recnum], -1, 5000) for sid in db: # print("Processing %s[%s]" % (sid, db[sid]['network'])) iem = Observation(sid, db[sid]["network"], db[sid]["ts"]) if db[sid]["tmpk"] is not None: iem.data["tmpf"] = temperature(db[sid]["tmpk"], "K").value("F") if db[sid]["dwpk"] is not None: iem.data["dwpf"] = temperature(db[sid]["dwpk"], "K").value("F") if db[sid]["relh"] is not None and db[sid]["relh"] is not np.ma.masked: iem.data["relh"] = float(db[sid]["relh"]) if db[sid]["drct"] is not None: iem.data["drct"] = db[sid]["drct"] if db[sid]["smps"] is not None: iem.data["sknt"] = speed(db[sid]["smps"], "MPS").value("KT") if db[sid]["gmps"] is not None: iem.data["gust"] = speed(db[sid]["gmps"], "MPS").value("KT") if db[sid]["pres"] is not None: iem.data["pres"] = (float(db[sid]["pres"]) / 100.00) * 0.02952 if db[sid]["rtk1"] is not None: iem.data["tsf0"] = temperature(db[sid]["rtk1"], "K").value("F") if db[sid]["rtk2"] is not None: iem.data["tsf1"] = temperature(db[sid]["rtk2"], "K").value("F") if db[sid]["rtk3"] is not None: iem.data["tsf2"] = temperature(db[sid]["rtk3"], "K").value("F") if db[sid]["rtk4"] is not None: iem.data["tsf3"] = temperature(db[sid]["rtk4"], "K").value("F") if db[sid]["subk"] is not None: iem.data["rwis_subf"] = temperature(db[sid]["subk"], "K").value("F") if db[sid]["pday"] is not None: iem.data["pday"] = round( distance(db[sid]["pday"], "MM").value("IN"), 2) if not iem.save(icursor): print(("MADIS Extract: %s found new station: %s network: %s" "") % (fn.split("/")[-1], sid, db[sid]["network"])) subprocess.call("python sync_stations.py %s" % (fn, ), shell=True) os.chdir("../../dbutil") subprocess.call("sh SYNC_STATIONS.sh", shell=True) os.chdir("../ingestors/madis") print("...done with sync.") del iem icursor.close() pgconn.commit() pgconn.close()
def test_relative_humidity_from_dewpoint_xarray(): """Test Relative Humidity calculation with xarray data arrays.""" temp = xr.DataArray(25., attrs={'units': 'degC'}) dewp = xr.DataArray(15., attrs={'units': 'degC'}) assert_almost_equal(relative_humidity_from_dewpoint(temp, dewp), 53.80 * units.percent, 2)
def wind_rh_according_to_4D_data( initTime=None, fhour=6, day_back=0, model='ECMWF', sta_fcs={ 'lon': [101.82, 101.32, 101.84, 102.23, 102.2681], 'lat': [28.35, 27.91, 28.32, 27.82, 27.8492], 'altitude': [3600, 3034.62, 3240, 1669, 1941.5], 'name': ['健美乡', '项脚乡', '\n锦屏镇', '\n马道镇', 'S9005 '] }, draw_zd=True, levels=[1000, 950, 925, 900, 850, 800, 700, 600, 500], map_ratio=19 / 9, zoom_ratio=1, south_China_sea=False, area='全国', city=False, output_dir=None, bkgd_type='satellite', data_source='MICAPS'): # micaps data directory if (area != '全国'): south_China_sea = False # prepare data if (area != '全国'): cntr_pnt, zoom_ratio = utl.get_map_area(area_name=area) cntr_pnt = np.append(np.mean(sta_fcs['lon']), np.mean(sta_fcs['lat'])) map_extent = [0, 0, 0, 0] map_extent[0] = cntr_pnt[0] - zoom_ratio * 1 * map_ratio map_extent[1] = cntr_pnt[0] + zoom_ratio * 1 * map_ratio map_extent[2] = cntr_pnt[1] - zoom_ratio * 1 map_extent[3] = cntr_pnt[1] + zoom_ratio * 1 bkgd_level = utl.cal_background_zoom_ratio(zoom_ratio) # micaps data directory if (data_source == 'MICAPS'): try: data_dir = [ utl.Cassandra_dir(data_type='high', data_source=model, var_name='HGT', lvl=''), utl.Cassandra_dir(data_type='high', data_source=model, var_name='RH', lvl=''), utl.Cassandra_dir(data_type='high', data_source=model, var_name='UGRD', lvl=''), utl.Cassandra_dir(data_type='high', data_source=model, var_name='VGRD', lvl=''), utl.Cassandra_dir(data_type='surface', data_source=model, var_name='u10m'), utl.Cassandra_dir(data_type='surface', data_source=model, var_name='v10m'), utl.Cassandra_dir(data_type='surface', data_source=model, var_name='Td2m'), utl.Cassandra_dir(data_type='surface', data_source=model, var_name='T2m') ] except KeyError: raise ValueError('Can not find all directories needed') # get filename if (initTime != None): filename = utl.model_filename(initTime, fhour) else: filename = utl.filename_day_back_model(day_back=day_back, fhour=fhour) initTime = filename[0:8] # retrieve data from micaps server gh = MICAPS_IO.get_model_3D_grid(directory=data_dir[0][0:-1], filename=filename, levels=levels) if (gh is None): return gh['data'].values = gh['data'].values * 10 rh = MICAPS_IO.get_model_3D_grid(directory=data_dir[1][0:-1], filename=filename, levels=levels, allExists=False) if rh is None: return u = MICAPS_IO.get_model_3D_grid(directory=data_dir[2][0:-1], filename=filename, levels=levels, allExists=False) if u is None: return v = MICAPS_IO.get_model_3D_grid(directory=data_dir[3][0:-1], filename=filename, levels=levels, allExists=False) if v is None: return u10m = MICAPS_IO.get_model_grid(directory=data_dir[4], filename=filename) if u10m is None: return v10m = MICAPS_IO.get_model_grid(directory=data_dir[5], filename=filename) if v10m is None: return td2m = MICAPS_IO.get_model_grid(directory=data_dir[6], filename=filename) if td2m is None: return t2m = MICAPS_IO.get_model_grid(directory=data_dir[7], filename=filename) if t2m is None: return if (draw_zd == True): validtime = (datetime.strptime('20' + initTime, '%Y%m%d%H') + timedelta(hours=fhour)).strftime("%Y%m%d%H") directory_obs = utl.Cassandra_dir(data_type='surface', data_source='OBS', var_name='PLOT_ALL') try: zd_sta = MICAPS_IO.get_station_data(filename=validtime + '0000.000', directory=directory_obs, dropna=True, cache=False) obs_valid = True except: zd_sta = MICAPS_IO.get_station_data(directory=directory_obs, dropna=True, cache=False) obs_valid = False zd_lon = zd_sta['lon'].values zd_lat = zd_sta['lat'].values zd_alt = zd_sta['Alt'].values zd_u, zd_v = mpcalc.wind_components( zd_sta['Wind_speed_2m_avg'].values * units('m/s'), zd_sta['Wind_angle_2m_avg'].values * units.deg) idx_zd = np.where((zd_lon > map_extent[0]) & (zd_lon < map_extent[1]) & (zd_lat > map_extent[2]) & (zd_lat < map_extent[3])) zd_sm_lon = zd_lon[idx_zd[0]] zd_sm_lat = zd_lat[idx_zd[0]] zd_sm_alt = zd_alt[idx_zd[0]] zd_sm_u = zd_u[idx_zd[0]] zd_sm_v = zd_v[idx_zd[0]] if (data_source == 'CIMISS'): # get filename if (initTime != None): filename = utl.model_filename(initTime, fhour, UTC=True) else: filename = utl.filename_day_back_model(day_back=day_back, fhour=fhour, UTC=True) try: # retrieve data from CMISS server gh = CIMISS_IO.cimiss_model_3D_grid( data_code=utl.CMISS_data_code(data_source=model, var_name='GPH'), init_time_str='20' + filename[0:8], valid_time=fhour, levattrs={ 'long_name': 'pressure_level', 'units': 'hPa', '_CoordinateAxisType': '-' }, fcst_levels=levels, fcst_ele="GPH", units='gpm') if gh is None: return rh = CIMISS_IO.cimiss_model_3D_grid( data_code=utl.CMISS_data_code(data_source=model, var_name='RHU'), init_time_str='20' + filename[0:8], valid_time=fhour, levattrs={ 'long_name': 'pressure_level', 'units': 'hPa', '_CoordinateAxisType': '-' }, fcst_levels=levels, fcst_ele="RHU", units='%') if rh is None: return u = CIMISS_IO.cimiss_model_3D_grid( data_code=utl.CMISS_data_code(data_source=model, var_name='WIU'), init_time_str='20' + filename[0:8], valid_time=fhour, levattrs={ 'long_name': 'pressure_level', 'units': 'hPa', '_CoordinateAxisType': '-' }, fcst_levels=levels, fcst_ele="WIU", units='m/s') if u is None: return v = CIMISS_IO.cimiss_model_3D_grid( data_code=utl.CMISS_data_code(data_source=model, var_name='WIV'), init_time_str='20' + filename[0:8], valid_time=fhour, levattrs={ 'long_name': 'pressure_level', 'units': 'hPa', '_CoordinateAxisType': '-' }, fcst_levels=levels, fcst_ele="WIV", units='m/s') if v is None: return if (model == 'ECMWF'): td2m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='DPT'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=0, fcst_ele="DPT", units='K') if td2m is None: return t2m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='TEF2'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=0, fcst_ele="TEF2", units='K') if t2m is None: return v10m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='WIV10'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=0, fcst_ele="WIV10", units='m/s') if v10m is None: return u10m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='WIU10'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=0, fcst_ele="WIU10", units='m/s') if u10m is None: return if (model == 'GRAPES_GFS'): rh2m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='RHF2'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=2, fcst_ele="RHF2", units='%') if rh2m is None: return v10m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='WIV10'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=10, fcst_ele="WIV10", units='m/s') if v10m is None: return u10m = CIMISS_IO.cimiss_model_by_time( '20' + filename[0:8], valid_time=fhour, data_code=utl.CMISS_data_code(data_source=model, var_name='WIU10'), levattrs={ 'long_name': 'height_above_ground', 'units': 'm', '_CoordinateAxisType': '-' }, fcst_level=10, fcst_ele="WIU10", units='m/s') if u10m is None: return except KeyError: raise ValueError('Can not find all data needed') if (draw_zd == True): if (initTime == None): initTime1 = CIMISS_IO.cimiss_get_obs_latest_time( data_code="SURF_CHN_MUL_HOR") initTime = (datetime.strptime('20' + initTime1, '%Y%m%d%H') - timedelta(days=day_back)).strftime("%Y%m%d%H")[2:] validtime = (datetime.strptime('20' + initTime, '%Y%m%d%H') + timedelta(hours=fhour)).strftime("%Y%m%d%H") data_code = utl.CMISS_data_code(data_source='OBS', var_name='PLOT_sfc') zd_sta = CIMISS_IO.cimiss_obs_by_time( times=validtime + '0000', data_code=data_code, sta_levels="011,012,013,014", elements= "Station_Id_C,Station_Id_d,lat,lon,Alti,TEM,WIN_D_Avg_2mi,WIN_S_Avg_2mi,RHU" ) obs_valid = True if (zd_sta is None): CIMISS_IO.cimiss_get_obs_latest_time(data_code=data_code, latestTime=6) zd_sta = CIMISS_IO.cimiss_obs_by_time(directory=directory_obs, dropna=True, cache=False) obs_valid = False zd_lon = zd_sta['lon'].values zd_lat = zd_sta['lat'].values zd_alt = zd_sta['Alti'].values zd_u, zd_v = mpcalc.wind_components( zd_sta['WIN_S_Avg_2mi'].values * units('m/s'), zd_sta['WIN_D_Avg_2mi'].values * units.deg) idx_zd = np.where((zd_lon > map_extent[0]) & (zd_lon < map_extent[1]) & (zd_lat > map_extent[2]) & (zd_lat < map_extent[3]) & (zd_sta['WIN_S_Avg_2mi'].values < 1000)) zd_sm_lon = zd_lon[idx_zd[0]] zd_sm_lat = zd_lat[idx_zd[0]] zd_sm_alt = zd_alt[idx_zd[0]] zd_sm_u = zd_u[idx_zd[0]] zd_sm_v = zd_v[idx_zd[0]] #maskout area delt_xy = rh['lon'].values[1] - rh['lon'].values[0] #+ to solve the problem of labels on all the contours mask1 = (rh['lon'] > map_extent[0] - delt_xy) & ( rh['lon'] < map_extent[1] + delt_xy) & ( rh['lat'] > map_extent[2] - delt_xy) & (rh['lat'] < map_extent[3] + delt_xy) mask2 = (u10m['lon'] > map_extent[0] - delt_xy) & ( u10m['lon'] < map_extent[1] + delt_xy) & ( u10m['lat'] > map_extent[2] - delt_xy) & (u10m['lat'] < map_extent[3] + delt_xy) #- to solve the problem of labels on all the contours rh = rh.where(mask1, drop=True) u = u.where(mask1, drop=True) v = v.where(mask1, drop=True) gh = gh.where(mask1, drop=True) u10m = u10m.where(mask2, drop=True) v10m = v10m.where(mask2, drop=True) #prepare interpolator Ex1 = np.squeeze(u['data'].values).flatten() Ey1 = np.squeeze(v['data'].values).flatten() Ez1 = np.squeeze(rh['data'].values).flatten() z = (np.squeeze(gh['data'].values)).flatten() coords = np.zeros((np.size(levels), u['lat'].size, u['lon'].size, 3)) coords[..., 1] = u['lat'].values.reshape((1, u['lat'].size, 1)) coords[..., 2] = u['lon'].values.reshape((1, 1, u['lon'].size)) coords = coords.reshape((Ex1.size, 3)) coords[:, 0] = z interpolator_U = LinearNDInterpolator(coords, Ex1, rescale=True) interpolator_V = LinearNDInterpolator(coords, Ey1, rescale=True) interpolator_RH = LinearNDInterpolator(coords, Ez1, rescale=True) #process sta_fcs 10m wind coords2 = np.zeros((np.size(sta_fcs['lon']), 3)) coords2[:, 0] = sta_fcs['altitude'] coords2[:, 1] = sta_fcs['lat'] coords2[:, 2] = sta_fcs['lon'] u_sta = interpolator_U(coords2) v_sta = interpolator_V(coords2) RH_sta = interpolator_RH(coords2) wsp_sta = (u_sta**2 + v_sta**2)**0.5 u10m_2D = u10m.interp(lon=('points', sta_fcs['lon']), lat=('points', sta_fcs['lat'])) v10m_2D = v10m.interp(lon=('points', sta_fcs['lon']), lat=('points', sta_fcs['lat'])) if (model == 'GRAPES_GFS' and data_source == 'CIMISS'): rh2m_2D = rh2m.interp(lon=('points', sta_fcs['lon']), lat=('points', sta_fcs['lat']))['data'].values else: td2m_2D = td2m.interp(lon=('points', sta_fcs['lon']), lat=('points', sta_fcs['lat'])) t2m_2D = t2m.interp(lon=('points', sta_fcs['lon']), lat=('points', sta_fcs['lat'])) if (data_source == 'MICAPS'): rh2m_2D = mpcalc.relative_humidity_from_dewpoint( t2m_2D['data'].values * units('degC'), td2m_2D['data'].values * units('degC')) * 100 else: rh2m_2D = mpcalc.relative_humidity_from_dewpoint( t2m_2D['data'].values * units('kelvin'), td2m_2D['data'].values * units('kelvin')) * 100 wsp10m_2D = (u10m_2D['data'].values**2 + v10m_2D['data'].values**2)**0.5 winddir10m = mpcalc.wind_direction(u10m_2D['data'].values * units('m/s'), v10m_2D['data'].values * units('m/s')) if (np.isnan(wsp_sta).any()): if (wsp_sta.size == 1): wsp_sta[np.isnan(wsp_sta)] = np.squeeze( wsp10m_2D[np.isnan(wsp_sta)]) RH_sta[np.isnan(RH_sta)] = np.squeeze( np.array(rh2m_2D)[np.isnan(RH_sta)]) else: wsp_sta[np.isnan(wsp_sta)] = np.squeeze(wsp10m_2D)[np.isnan( wsp_sta)] RH_sta[np.isnan(RH_sta)] = np.squeeze( np.array(rh2m_2D))[np.isnan(RH_sta)] u_sta, v_sta = mpcalc.wind_components(wsp_sta * units('m/s'), winddir10m) #process zd_sta 10m wind zd_fcst_obs = None if (draw_zd is True): coords3 = np.zeros((np.size(zd_sm_alt), 3)) coords3[:, 0] = zd_sm_alt coords3[:, 1] = zd_sm_lat coords3[:, 2] = zd_sm_lon u_sm_sta = interpolator_U(coords3) v_sm_sta = interpolator_V(coords3) wsp_sm_sta = (u_sm_sta**2 + v_sm_sta**2)**0.5 u10m_sm = u10m.interp(lon=('points', zd_sm_lon), lat=('points', zd_sm_lat)) v10m_sm = v10m.interp(lon=('points', zd_sm_lon), lat=('points', zd_sm_lat)) wsp10m_sta = np.squeeze( (u10m_sm['data'].values**2 + v10m_sm['data'].values**2)**0.5) winddir10m_sm = mpcalc.wind_direction( u10m_sm['data'].values * units('m/s'), v10m_sm['data'].values * units('m/s')) if (np.isnan(wsp_sm_sta).any()): wsp_sm_sta[np.isnan(wsp_sm_sta)] = wsp10m_sta[np.isnan(wsp_sm_sta)] u_sm_sta, v_sm_sta = mpcalc.wind_components(wsp_sm_sta * units('m/s'), winddir10m_sm) zd_fcst_obs = { 'lon': zd_sm_lon, 'lat': zd_sm_lat, 'altitude': zd_sm_alt, 'U': np.squeeze(np.array(u_sm_sta)), 'V': np.squeeze(np.array(v_sm_sta)), 'obs_valid': obs_valid, 'U_obs': np.squeeze(np.array(zd_sm_u)), 'V_obs': np.squeeze(np.array(zd_sm_v)) } #prepare for graphics sta_fcs_fcst = { 'lon': sta_fcs['lon'], 'lat': sta_fcs['lat'], 'altitude': sta_fcs['altitude'], 'name': sta_fcs['name'], 'RH': np.array(RH_sta), 'U': np.squeeze(np.array(u_sta)), 'V': np.squeeze(np.array(v_sta)) } fcst_info = gh.coords local_scale_graphics.draw_wind_rh_according_to_4D_data( sta_fcs_fcst=sta_fcs_fcst, zd_fcst_obs=zd_fcst_obs, fcst_info=fcst_info, map_extent=map_extent, draw_zd=draw_zd, bkgd_type=bkgd_type, bkgd_level=bkgd_level, output_dir=None)
plt.xlabel('Dantong data') plt.ylabel('ERA5') plt.title('Wind v (m/s)') #plt.show() plt.savefig("figs/wind_v_scatter.png") #RH ERA5_T = combined_df['t2m'].values * units.degC ERA5_Td = (combined_df['d2m'].values) * units.degC Dan_T = combined_df['Temp_C'].values * units.degC Dan_RH = combined_df['RH'].values / 100 * units('') Dan_Td = mpcalc.thermo.dewpoint_from_relative_humidity(Dan_T, Dan_RH) combined_df['Dan_Td'] = Dan_Td combined_df['RH_ERA5'] = mpcalc.relative_humidity_from_dewpoint( ERA5_T, ERA5_Td) * 100 combined_df.plot.scatter('RH', 'RH_ERA5') plt.xlabel('Dantong data') plt.ylabel('ERA5') plt.title('RH (%)') #plt.show() plt.savefig("figs/RH_scatter.png") #Tdew combined_df.plot.scatter('Dan_Td', 'd2m') plt.xlabel('Dantong data') plt.ylabel('ERA5') plt.title('Dew point (C)') #plt.show() plt.savefig("figs/Dew_point_scatter.png")
VP = df_selected_time['vaper_pressure'].values * units.hPa df_selected_time[ 'saturation_mixing_ratio'] = mpcalc.mixing_ratio( VPS, p) df_selected_time['mixing_ratio'] = mpcalc.mixing_ratio( VP, p) MRS = df_selected_time[ 'saturation_mixing_ratio'].values * units('g/kg') MR = df_selected_time['mixing_ratio'].values * units( 'g/kg') # Calculate RH #RH = mpcalc.relative_humidity_from_dewpoint(T[0], Td[0]) df_selected_time[ 'RH'] = mpcalc.relative_humidity_from_dewpoint(T, Td) df_selected_time[ 'RH_MR'] = mpcalc.relative_humidity_from_mixing_ratio( MR, T, p) print('df_selected_time after drop nan.\n{0}'.format( df_selected_time)) df_selected_time.to_csv( r'{0}{1}{2}_{3}_solution.csv'.format( dir_name, save_dir_name, filename_el[-5], selected_time[:13])) p = df_selected_time['pressure'].values * units.hPa T = df_selected_time['temperature'].values * units.degC Td = df_selected_time['dewpoint'].values * units.degC
def process_metar(mstr, now): """ Do the METAR Processing """ mtr = None while mtr is None: try: mtr = Metar(mstr, now.month, now.year) except MetarParserError as exp: try: msg = str(exp) except Exception as exp: return None tokens = ERROR_RE.findall(str(exp)) orig_mstr = mstr if tokens: for token in tokens[0].split(): mstr = mstr.replace(" %s" % (token, ), "") if orig_mstr == mstr: print("Can't fix badly formatted metar: " + mstr) return None else: print("MetarParserError: " + msg) return None except Exception as exp: print("Double Fail: %s %s" % (mstr, exp)) return None if mtr is None or mtr.time is None: return None ob = OB() ob.metar = mstr[:254] ob.valid = now if mtr.temp: ob.tmpf = mtr.temp.value("F") if mtr.dewpt: ob.dwpf = mtr.dewpt.value("F") if mtr.wind_speed: ob.sknt = mtr.wind_speed.value("KT") if mtr.wind_gust: ob.gust = mtr.wind_gust.value("KT") # Calc some stuff if ob.tmpf is not None and ob.dwpf is not None: ob.relh = relative_humidity_from_dewpoint( ob.tmpf * units('degF'), ob.dwpf * units('degF')).to(units('percent')).magnitude if ob.sknt is not None: ob.feel = apparent_temperature(ob.tmpf * units('degF'), ob.relh * units('percent'), ob.sknt * units('knots')).to( units('degF')).magnitude if mtr.wind_dir and mtr.wind_dir.value() != "VRB": ob.drct = mtr.wind_dir.value() if mtr.vis: ob.vsby = mtr.vis.value("SM") # see pull request #38 if mtr.press and mtr.press != mtr.press_sea_level: ob.alti = mtr.press.value("IN") if mtr.press_sea_level: ob.mslp = mtr.press_sea_level.value("MB") if mtr.precip_1hr: ob.p01i = mtr.precip_1hr.value("IN") # Do something with sky coverage for i in range(len(mtr.sky)): (c, h, _) = mtr.sky[i] setattr(ob, 'skyc%s' % (i + 1), c) if h is not None: setattr(ob, 'skyl%s' % (i + 1), h.value("FT")) if mtr.max_temp_6hr: ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F") if mtr.min_temp_6hr: ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F") if mtr.max_temp_24hr: ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F") if mtr.min_temp_24hr: ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F") if mtr.precip_3hr: ob.p03i = mtr.precip_3hr.value("IN") if mtr.precip_6hr: ob.p06i = mtr.precip_6hr.value("IN") if mtr.precip_24hr: ob.p24i = mtr.precip_24hr.value("IN") # Presentwx if mtr.weather: pwx = [] for wx in mtr.weather: val = "".join([a for a in wx if a is not None]) if val == "" or val == len(val) * "/": continue pwx.append(val) ob.wxcodes = pwx return ob
# OTHER OPTIONAL AXES TO PLOT # plot_irradiance # plot_precipitation # Download the station data station = '04360' #'04416' # '89606'# '03065' #04201 url, path = url_timeseries(2020, 4, 20, 00, 2020, 4, 27, 10, station) # yields an error (many not a time entries) download_and_save(path, url) df_synop, df_climat = synop_df(path, timeseries=True) # Temporary variables for ease temp = df_synop['TT'].values * units('degC') pres = df_synop['SLP'].values dewpoint = df_synop['TD'].values * units('degC') rh = mpcalc.relative_humidity_from_dewpoint(temp, dewpoint) * 100 ws = df_synop['ff'].values if 'max_gust' in df_synop.columns: wsmax = df_synop['max_gust'].values else: pass wd = df_synop['dd'].values date = pd.to_datetime(df_synop['time'].values).tolist() # ID For Plotting on Meteogram probe_id = df_synop.Station[0] data = { 'wind_speed': (np.array(ws) * units('knots')), 'wind_speed_max': (np.array(wsmax) * units('kph')).to(units('knots')), 'wind_direction': np.array(wd) * units('degrees'),
def f(fullname, selected_time): fullname_el = fullname.split('/') #fullname_el = fullname.split('\\') filename = fullname_el[-1] filename_el = filename.split('_') save_base_dir_name = '../1data/' save_dir_name = '{0}{1}/'.format(save_base_dir_name + dir_name, obs_year) print('site : {0}'.format(dir_name)) if os.path.isfile('{0}{1}_{2}_student.csv'\ .format(save_dir_name, filename_el[-5], selected_time[:13]))\ and os.path.isfile('{0}{1}_{2}_solution.csv'\ .format(save_dir_name, filename_el[-5], selected_time[:13])): write_log(log_file, '{3} ::: {0}{1}_{2} files are already exist'\ .format(save_dir_name, filename_el[-5], selected_time[:13], datetime.now())) else: try: f = lambda s: selected_time in s ids = df['time'].apply(f) df_selected_time = df[ids] df_selected_time = df_selected_time.sort_values('pressure', ascending=False) print('filename : {0}'.format(fullname)) print('df_selected_time.\n{0}'.format(df_selected_time)) df_selected_time.to_csv(r'{0}{1}_{2}_student.csv'\ .format(save_dir_name, filename_el[-5], selected_time[:13])) ################################################################################# ### We will pull the data out of the example dataset into individual variables and ### assign units. ################################################################################# p = df_selected_time['pressure'].values * units.hPa T = df_selected_time['temperature'].values * units.degC Td = df_selected_time['dewpoint'].values * units.degC wind_speed = df_selected_time['speed'].values * units.knots wind_dir = df_selected_time['direction'].values * units.degrees u, v = mpcalc.wind_components(wind_speed, wind_dir) # Calculate web bulb temperature df_selected_time['wet_bulb_T'] = mpcalc.wet_bulb_temperature( p, T, Td) # Calculate potential temperature df_selected_time['potential_T'] = mpcalc.potential_temperature( p, T) df_selected_time['potential_T_C'] = df_selected_time[ 'potential_T'].values - 273.15 # Calculate saturation vaper pressure df_selected_time[ 'saturation_vaper_pressure'] = mpcalc.saturation_vapor_pressure( T) df_selected_time[ 'vaper_pressure'] = mpcalc.saturation_vapor_pressure(Td) SVP = df_selected_time[ 'saturation_vaper_pressure'].values * units.hPa VP = df_selected_time['vaper_pressure'].values * units.hPa # Calculate mixing ratio df_selected_time['saturation_mixing_ratio'] = mpcalc.mixing_ratio( SVP, p) df_selected_time['mixing_ratio'] = mpcalc.mixing_ratio(VP, p) SMR = df_selected_time['saturation_mixing_ratio'].values * units( 'g/kg') MR = df_selected_time['mixing_ratio'].values * units('g/kg') # Calculate relative humidity df_selected_time['relative_humidity_from_dewpoint'] \ = mpcalc.relative_humidity_from_dewpoint(T, Td) df_selected_time['relative_humidity_from_mixing_ratio'] \ = mpcalc.relative_humidity_from_mixing_ratio(MR, T, p) # Calculate virtual temperature df_selected_time['virtual_temperature'] \ = mpcalc.virtual_temperature(T, MR) # Calculate virtual potential temperature df_selected_time['virtual_potential_temperature'] \ = mpcalc.virtual_potential_temperature(p, T, MR) print('df_selected_time after drop nan.\n{0}'.format( df_selected_time)) df_selected_time.to_csv(r'{0}{1}_{2}_solution.csv'\ .format(save_dir_name, filename_el[-5], selected_time[:13])) except Exception as err: write_log(err_log_file, '{4} ::: {0} with {1}{2} on {3}'\ .format(err, dir_name, filename, selected_time[:13], datetime.now())) print('Thread {0} is finished'.format(selected_time)) return 0 # Return a dummy value
def merge(atmos, surface): """Create a dictionary of data based on these two dataframes Args: atmos (DataFrame): atmospherics surface (DataFrame): surface data Returns: dictionary of values """ data = {} # Do what we can with the atmospheric data for _, row in atmos.iterrows(): nwsli = get_nwsli(row["Rpuid"]) if nwsli is None: if int(row["Rpuid"]) not in KNOWN_UNKNOWNS: print(("process_rwis: Unknown Rpuid: %s in atmos" "") % (row["Rpuid"], )) continue if nwsli not in data: data[nwsli] = {} # Timestamp ts = datetime.datetime.strptime(row["DtTm"], "%m/%d/%y %H:%M") data[nwsli]["valid"] = ts.replace(tzinfo=pytz.UTC) data[nwsli]["tmpf"] = get_temp(row["AirTemp"]) data[nwsli]["dwpf"] = get_temp(row["Dewpoint"]) if data[nwsli]["tmpf"] is not None and data[nwsli]["dwpf"] is not None: data[nwsli]["relh"] = (mcalc.relative_humidity_from_dewpoint( data[nwsli]["tmpf"] * units("degF"), data[nwsli]["dwpf"] * units("degF"), ).magnitude * 100.0) # Rh is unused data[nwsli]["sknt"] = get_speed(row["SpdAvg"]) data[nwsli]["gust"] = get_speed(row["SpdGust"]) if row["DirMin"] not in ["", 32767, np.nan]: data[nwsli]["drct"] = row["DirMin"] # DirMax is unused # Pressure is not reported # PcIntens # PcType # PcRate if row["PcAccum"] not in ["", -1, 32767, np.nan]: data[nwsli]["pday"] = row["PcAccum"] * 0.00098425 if row["Visibility"] not in ["", -1, 32767, np.nan]: data[nwsli]["vsby"] = row["Visibility"] / 1609.344 # Do what we can with the surface data for _, row in surface.iterrows(): nwsli = get_nwsli(row["Rpuid"]) if nwsli is None: if int(row["Rpuid"]) not in KNOWN_UNKNOWNS: print(("process_rwis: Unknown Rpuid: %s in sfc" "") % (row["Rpuid"], )) continue ts = datetime.datetime.strptime(row["DtTm"], "%m/%d/%y %H:%M") ts = ts.replace(tzinfo=pytz.UTC) if nwsli not in data: data[nwsli] = {"valid": ts} sensorid = int(row["Senid"]) key = "sfvalid%s" % (sensorid, ) data[nwsli][key] = ts key = "scond%s" % (sensorid, ) data[nwsli][key] = row["sfcond"] # sftemp -150 key = "tsf%s" % (sensorid, ) data[nwsli][key] = get_temp(row["sftemp"]) # frztemp 32767 # chemfactor 0 # chempct 101 # depth 32767 # icepct 101 # subsftemp NaN key = "tsub%s" % (sensorid, ) data[nwsli][key] = get_temp(row["subsftemp"]) # waterlevel NaN # Unnamed: 13 NaN # Unnamed: 14 NaN return data
def process_metar(mstr, now): """ Do the METAR Processing """ mtr = None while mtr is None: try: mtr = Metar(mstr, now.month, now.year) except MetarParserError as exp: try: msg = str(exp) except Exception as exp: return None tokens = ERROR_RE.findall(str(exp)) orig_mstr = mstr if tokens: for token in tokens[0].split(): mstr = mstr.replace(" %s" % (token, ), "") if orig_mstr == mstr: print("Can't fix badly formatted metar: " + mstr) return None else: print("MetarParserError: "+msg) return None except Exception as exp: print("Double Fail: %s %s" % (mstr, exp)) return None if mtr is None or mtr.time is None: return None ob = OB() ob.metar = mstr[:254] ob.valid = now if mtr.temp: ob.tmpf = mtr.temp.value("F") if mtr.dewpt: ob.dwpf = mtr.dewpt.value("F") if mtr.wind_speed: ob.sknt = mtr.wind_speed.value("KT") if mtr.wind_gust: ob.gust = mtr.wind_gust.value("KT") # Calc some stuff if ob.tmpf is not None and ob.dwpf is not None: ob.relh = relative_humidity_from_dewpoint( ob.tmpf * units('degF'), ob.dwpf * units('degF') ).to(units('percent')).magnitude if ob.sknt is not None: ob.feel = apparent_temperature( ob.tmpf * units('degF'), ob.relh * units('percent'), ob.sknt * units('knots') ).to(units('degF')).magnitude if mtr.wind_dir and mtr.wind_dir.value() != "VRB": ob.drct = mtr.wind_dir.value() if mtr.vis: ob.vsby = mtr.vis.value("SM") # see pull request #38 if mtr.press and mtr.press != mtr.press_sea_level: ob.alti = mtr.press.value("IN") if mtr.press_sea_level: ob.mslp = mtr.press_sea_level.value("MB") if mtr.precip_1hr: ob.p01i = mtr.precip_1hr.value("IN") # Do something with sky coverage for i in range(len(mtr.sky)): (c, h, _) = mtr.sky[i] setattr(ob, 'skyc%s' % (i+1), c) if h is not None: setattr(ob, 'skyl%s' % (i+1), h.value("FT")) if mtr.max_temp_6hr: ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F") if mtr.min_temp_6hr: ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F") if mtr.max_temp_24hr: ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F") if mtr.min_temp_24hr: ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F") if mtr.precip_3hr: ob.p03i = mtr.precip_3hr.value("IN") if mtr.precip_6hr: ob.p06i = mtr.precip_6hr.value("IN") if mtr.precip_24hr: ob.p24i = mtr.precip_24hr.value("IN") # Presentwx if mtr.weather: pwx = [] for wx in mtr.weather: val = "".join([a for a in wx if a is not None]) if val == "" or val == len(val) * "/": continue pwx.append(val) ob.wxcodes = pwx return ob
def Station_Snow_Synthetical_Forecast_From_Cassandra( model='ECMWF', output_dir=None, t_range=[0,84], t_gap=3, points={'lon':[116.3833], 'lat':[39.9]}, initTime=None, draw_VIS=True,drw_thr=False, extra_info={ 'output_head_name':' ', 'output_tail_name':' ', 'point_name':' '} ): #+get all the directories needed try: dir_rqd=[ "ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_3_HOURS/", "ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_6_HOURS/", "ECMWF_HR/SNOD/", "ECMWF_HR/SDEN/", "ECMWF_HR/UGRD_100M/", "ECMWF_HR/VGRD_100M/", "NWFD_SCMOC/VIS/", "NCEP_GFS_HR/SNOD/", "ECMWF_HR/SNOW06/", utl.Cassandra_dir( data_type='surface',data_source=model,var_name='T2m'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='u10m'), utl.Cassandra_dir( data_type='surface',data_source=model,var_name='v10m'), 'ECMWF_ENSEMBLE/RAW/SNOW06/' ] except KeyError: raise ValueError('Can not find all required directories needed') try: dir_opt=[ utl.Cassandra_dir( data_type='surface',data_source=model,var_name='Td2m') ] name_opt=['Td2m'] except: dir_opt=[ utl.Cassandra_dir(data_type='surface',data_source=model,var_name='rh2m') ] name_opt=['rh2m'] #+get all the directories needed if(initTime == None): last_file={model:get_latest_initTime(dir_rqd[0]), 'SCMOC':get_latest_initTime(dir_rqd[6]), } else: last_file={model:initTime[0], 'SCMOC':initTime[1], } y_s={model:int('20'+last_file[model][0:2]), 'SCMOC':int('20'+last_file['SCMOC'][0:2])} m_s={model:int(last_file[model][2:4]), 'SCMOC':int(last_file['SCMOC'][2:4])} d_s={model:int(last_file[model][4:6]), 'SCMOC':int(last_file['SCMOC'][4:6])} h_s={model:int(last_file[model][6:8]), 'SCMOC':int(last_file['SCMOC'][6:8])} fhours = np.arange(t_range[0], t_range[1], t_gap) for ifhour in fhours: if (ifhour == fhours[0] ): time_all=datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour)) else: time_all=np.append(time_all,datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour))) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] t2m=utl.get_model_points_gy(dir_rqd[9], filenames, points,allExists=False) if(name_opt[0] == 'rh2m'): rh2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False) Td2m=mpcalc.dewpoint_rh(t2m['data'].values*units('degC'),rh2m['data'].values/100.) p_vapor=(rh2m['data'].values/100.)*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values)))) if(name_opt[0] == 'Td2m'): Td2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False) rh2m=mpcalc.relative_humidity_from_dewpoint(t2m['data'].values* units('degC'), Td2m['data'].values* units('degC')) p_vapor=(np.array(rh2m))*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values)))) Td2m=np.array(Td2m['data'].values)* units('degC') #SN06_ensm=utl.get_model_points_gy(dir_rqd[12], filenames, points,allExists=False) ''' for i in range(0,len(SN06_ensm['forecast_period'])): SN06_std=np.std(np.squeeze(SN06_ensm['data'].values[i,:])) SN06_mean=np.mean(np.squeeze(SN06_ensm['data'].values[i,:])) if(i == 0): SN06_01=norm.pdf(0.01, SN06_mean, SN06_std) SN06_10=norm.pdf(0.1, SN06_mean, SN06_std) SN06_25=norm.pdf(0.25, SN06_mean, SN06_std) SN06_50=norm.pdf(0.5, SN06_mean, SN06_std) SN06_75=norm.pdf(0.75, SN06_mean, SN06_std) SN06_90=norm.pdf(0.9, SN06_mean, SN06_std) SN06_99=norm.pdf(0.99, SN06_mean, SN06_std) if(i > 0): SN06_01=[SN06_01,norm.pdf(0.01, SN06_mean, SN06_std)] SN06_10=[SN06_10,norm.pdf(0.1, SN06_mean, SN06_std)] SN06_25=[SN06_25,norm.pdf(0.25, SN06_mean, SN06_std)] SN06_50=[SN06_50,norm.pdf(0.5, SN06_mean, SN06_std)] SN06_75=[SN06_75,norm.pdf(0.75, SN06_mean, SN06_std)] SN06_90=[SN06_90,norm.pdf(0.9, SN06_mean, SN06_std)] SN06_99=[SN06_99,norm.pdf(0.99, SN06_mean, SN06_std)] SN06_ensm_stc={ 'SN06_01'=SN06_01 'SN06_10'=SN06_10 'SN06_25'=SN06_25 'SN06_50'=SN06_50 'SN06_75'=SN06_75 'SN06_90'=SN06_90 'SN06_99'=SN06_99 } ''' u10m=utl.get_model_points_gy(dir_rqd[10], filenames, points,allExists=False) v10m=utl.get_model_points_gy(dir_rqd[11], filenames, points,allExists=False) wsp10m=(u10m['data']**2+v10m['data']**2)**0.5 AT=1.07*t2m['data'].values+0.2*p_vapor-0.65*wsp10m-2.7 #https://en.wikipedia.org/wiki/Wind_chill TWC=13.12+0.6215*t2m['data'].values-11.37*(wsp10m**0.16)+0.3965*t2m['data'].values*(wsp10m**0.16) fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file['SCMOC']+'.'+str(fhour).zfill(3) for fhour in fhours] VIS=utl.get_model_points_gy(dir_rqd[6], filenames, points,allExists=False,fill_null=True,Null_value=-0.001) if(last_file['SCMOC'] == last_file[model] and t_range[1] > 72): fhours = np.append(np.arange(3,72,3),np.arange(72, (t_range[1]), 6)) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model] and t_range[1] > 60): fhours = np.append(np.arange(3,60,3),np.arange(60, (t_range[1]), 6)) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model] and t_range[1] <= 60): fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] == last_file[model] and t_range[1] <= 72): fhours = np.arange(t_range[0], t_range[1], t_gap) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] SNOD1=utl.get_model_points_gy(dir_rqd[2], filenames2, points,allExists=False) SNOD2=utl.get_model_points_gy(dir_rqd[7], filenames2, points,allExists=False) SDEN=utl.get_model_points_gy(dir_rqd[3], filenames2, points,allExists=False) SN06=utl.get_model_points_gy(dir_rqd[8], filenames2, points,allExists=False) u100m=utl.get_model_points_gy(dir_rqd[4], filenames2, points,allExists=False) v100m=utl.get_model_points_gy(dir_rqd[5], filenames2, points,allExists=False) wsp100m=(u100m['data']**2+v100m['data']**2)**0.5 if(fhours[-1] < 120): gust10m=utl.get_model_points_gy(dir_rqd[0], filenames, points,allExists=False) if(fhours[-1] > 120): if(last_file['SCMOC'] == last_file[model]): fhours = np.arange(0, t_range[1], 6) filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours] if(last_file['SCMOC'] != last_file[model]): fhours = np.arange(0, t_range[1], 6) filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours] gust10m=utl.get_model_points_gy(dir_rqd[1], filenames, points,allExists=False) sta_graphics.draw_Station_Snow_Synthetical_Forecast_From_Cassandra( TWC=TWC,AT=AT,u10m=u10m,v10m=v10m,u100m=u100m,v100m=v100m, gust10m=gust10m,wsp10m=wsp10m,wsp100m=wsp100m,SNOD1=SNOD1,SNOD2=SNOD2,SDEN=SDEN,SN06=SN06, draw_VIS=draw_VIS,VIS=VIS,drw_thr=drw_thr, time_all=time_all, model=model,points=points, output_dir=output_dir,extra_info=extra_info)
def main(): """Do Something""" pgconn = get_dbconn('iem') icursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor) fn = find_file() try: nc = netCDF4.Dataset(fn) except Exception as _: # File may be in progress of being read, wait a little bit time.sleep(20) nc = netCDF4.Dataset(fn) stations = nc.variables["stationId"][:] providers = nc.variables["dataProvider"][:] names = nc.variables["stationName"][:] tmpk = nc.variables["temperature"][:] dwpk = nc.variables["dewpoint"][:] relh = mcalc.relative_humidity_from_dewpoint( tmpk * units.degK, dwpk * units.degK).magnitude * 100. tmpk_dd = nc.variables["temperatureDD"][:] obtime = nc.variables["observationTime"][:] pressure = nc.variables["stationPressure"][:] # altimeter = nc.variables["altimeter"][:] # slp = nc.variables["seaLevelPressure"][:] drct = nc.variables["windDir"][:] smps = nc.variables["windSpeed"][:] gmps = nc.variables["windGust"][:] # gmps_drct = nc.variables["windDirMax"][:] pcpn = nc.variables["precipAccum"][:] rtk1 = nc.variables["roadTemperature1"][:] rtk2 = nc.variables["roadTemperature2"][:] rtk3 = nc.variables["roadTemperature3"][:] rtk4 = nc.variables["roadTemperature4"][:] subk1 = nc.variables["roadSubsurfaceTemp1"][:] db = {} for recnum, provider in enumerate(providers): this_provider = provider.tostring().replace('\x00', '') this_station = stations[recnum].tostring().replace('\x00', '') if (not this_provider.endswith('DOT') and this_provider not in MY_PROVIDERS): continue name = names[recnum].tostring().replace("'", "").replace( '\x00', '').replace('\xa0', ' ').strip() if name == '': continue if this_provider == 'MesoWest': # get the network from the last portion of the name network = name.split()[-1] if network != 'VTWAC': continue else: network = provider2network(this_provider) if network is None: continue db[this_station] = {} ticks = obtime[recnum] ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks) db[this_station]['ts'] = ts.replace(tzinfo=pytz.utc) db[this_station]['network'] = network db[this_station]['pres'] = sanity_check(pressure[recnum], 0, 1000000, -99) db[this_station]['tmpk'] = sanity_check(tmpk[recnum], 0, 500, -99) db[this_station]['dwpk'] = sanity_check(dwpk[recnum], 0, 500, -99) db[this_station]['tmpk_dd'] = tmpk_dd[recnum] db[this_station]['relh'] = relh[recnum] db[this_station]['drct'] = sanity_check(drct[recnum], -1, 361, -99) db[this_station]['smps'] = sanity_check(smps[recnum], -1, 200, -99) db[this_station]['gmps'] = sanity_check(gmps[recnum], -1, 200, -99) db[this_station]['rtk1'] = sanity_check(rtk1[recnum], 0, 500, -99) db[this_station]['rtk2'] = sanity_check(rtk2[recnum], 0, 500, -99) db[this_station]['rtk3'] = sanity_check(rtk3[recnum], 0, 500, -99) db[this_station]['rtk4'] = sanity_check(rtk4[recnum], 0, 500, -99) db[this_station]['subk'] = sanity_check(subk1[recnum], 0, 500, -99) db[this_station]['pday'] = sanity_check(pcpn[recnum], -1, 5000, -99) for sid in db: # print("Processing %s[%s]" % (sid, db[sid]['network'])) iem = Observation(sid, db[sid]['network'], db[sid]['ts']) # if not iem.load(icursor): # print 'Missing fp: %s network: %s station: %s' % (fp, # db[sid]['network'], # sid) # subprocess.call("python sync_stations.py %s" % (fp,), shell=True) # os.chdir("../../dbutil") # subprocess.call("sh SYNC_STATIONS.sh", shell=True) # os.chdir("../ingestors/madis") iem.data['tmpf'] = temperature(db[sid]['tmpk'], 'K').value('F') iem.data['dwpf'] = temperature(db[sid]['dwpk'], 'K').value('F') if db[sid]['relh'] >= 0: iem.data['relh'] = float(db[sid]['relh']) if db[sid]['drct'] >= 0: iem.data['drct'] = db[sid]['drct'] if db[sid]['smps'] >= 0: iem.data['sknt'] = speed(db[sid]['smps'], 'MPS').value('KT') if db[sid]['gmps'] >= 0: iem.data['gust'] = speed(db[sid]['gmps'], 'MPS').value('KT') if db[sid]['pres'] > 0: iem.data['pres'] = (float(db[sid]['pres']) / 100.00) * 0.02952 if db[sid]['rtk1'] > 0: iem.data['tsf0'] = temperature(db[sid]['rtk1'], 'K').value('F') if db[sid]['rtk2'] > 0: iem.data['tsf1'] = temperature(db[sid]['rtk2'], 'K').value('F') if db[sid]['rtk3'] > 0: iem.data['tsf2'] = temperature(db[sid]['rtk3'], 'K').value('F') if db[sid]['rtk4'] > 0: iem.data['tsf3'] = temperature(db[sid]['rtk4'], 'K').value('F') if db[sid]['subk'] > 0: iem.data['rwis_subf'] = temperature(db[sid]['subk'], 'K').value('F') if db[sid]['pday'] >= 0: iem.data['pday'] = round( distance(db[sid]['pday'], 'MM').value("IN"), 2) if not iem.save(icursor): print(("MADIS Extract: %s found new station: %s network: %s" "") % (fn.split("/")[-1], sid, db[sid]['network'])) subprocess.call("python sync_stations.py %s" % (fn, ), shell=True) os.chdir("../../dbutil") subprocess.call("sh SYNC_STATIONS.sh", shell=True) os.chdir("../ingestors/madis") print("...done with sync.") del iem nc.close() icursor.close() pgconn.commit() pgconn.close()
def merge(atmos, surface): """Create a dictionary of data based on these two dataframes Args: atmos (DataFrame): atmospherics surface (DataFrame): surface data Returns: dictionary of values """ data = {} # Do what we can with the atmospheric data for _, row in atmos.iterrows(): nwsli = get_nwsli(row['Rpuid']) if nwsli is None: if int(row['Rpuid']) not in KNOWN_UNKNOWNS: print(('process_rwis: Unknown Rpuid: %s in atmos' '') % (row['Rpuid'], )) continue if nwsli not in data: data[nwsli] = {} # Timestamp ts = datetime.datetime.strptime(row['DtTm'], '%m/%d/%y %H:%M') data[nwsli]['valid'] = ts.replace(tzinfo=pytz.UTC) data[nwsli]['tmpf'] = get_temp(row['AirTemp']) data[nwsli]['dwpf'] = get_temp(row['Dewpoint']) if data[nwsli]['tmpf'] is not None and data[nwsli]['dwpf'] is not None: data[nwsli]['relh'] = mcalc.relative_humidity_from_dewpoint( data[nwsli]['tmpf'] * units('degF'), data[nwsli]['dwpf'] * units('degF')).magnitude * 100. # Rh is unused data[nwsli]['sknt'] = get_speed(row['SpdAvg']) data[nwsli]['gust'] = get_speed(row['SpdGust']) if row['DirMin'] not in ['', 32767, np.nan]: data[nwsli]['drct'] = row['DirMin'] # DirMax is unused # Pressure is not reported # PcIntens # PcType # PcRate if row['PcAccum'] not in ['', -1, 32767, np.nan]: data[nwsli]['pday'] = row['PcAccum'] * 0.00098425 if row['Visibility'] not in ['', -1, 32767, np.nan]: data[nwsli]['vsby'] = row['Visibility'] / 1609.344 # Do what we can with the surface data for _, row in surface.iterrows(): nwsli = get_nwsli(row['Rpuid']) if nwsli is None: if int(row['Rpuid']) not in KNOWN_UNKNOWNS: print(('process_rwis: Unknown Rpuid: %s in sfc' '') % (row['Rpuid'], )) continue ts = datetime.datetime.strptime(row['DtTm'], '%m/%d/%y %H:%M') ts = ts.replace(tzinfo=pytz.UTC) if nwsli not in data: data[nwsli] = {'valid': ts} sensorid = int(row['Senid']) key = 'sfvalid%s' % (sensorid, ) data[nwsli][key] = ts key = 'scond%s' % (sensorid, ) data[nwsli][key] = row['sfcond'] # sftemp -150 key = 'tsf%s' % (sensorid, ) data[nwsli][key] = get_temp(row['sftemp']) # frztemp 32767 # chemfactor 0 # chempct 101 # depth 32767 # icepct 101 # subsftemp NaN key = 'tsub%s' % (sensorid, ) data[nwsli][key] = get_temp(row['subsftemp']) # waterlevel NaN # Unnamed: 13 NaN # Unnamed: 14 NaN return data
def main(): load_start = dt.datetime.now() #Try parsing arguments using argparse parser = argparse.ArgumentParser(description='wrf non-parallel convective diagnostics processer') parser.add_argument("-m",help="Model name",required=True) parser.add_argument("-r",help="Region name (default is aus)",default="aus") parser.add_argument("-t1",help="Time start YYYYMMDDHH",required=True) parser.add_argument("-t2",help="Time end YYYYMMDDHH",required=True) parser.add_argument("-e", help="CMIP5 experiment name (not required if using era5, erai or barra)", default="") parser.add_argument("--ens", help="CMIP5 ensemble name (not required if using era5, erai or barra)", default="r1i1p1") parser.add_argument("--group", help="CMIP6 modelling group name", default="") parser.add_argument("--project", help="CMIP6 modelling intercomparison project", default="CMIP") parser.add_argument("--ver6hr", help="Version on al33 for 6hr data", default="") parser.add_argument("--ver3hr", help="Version on al33 for 3hr data", default="") parser.add_argument("--issave",help="Save output (True or False, default is False)", default="False") parser.add_argument("--outname",help="Name of saved output. In the form *outname*_*t1*_*t2*.nc. Default behaviour is the model name",default=None) parser.add_argument("--al33",help="Should data be gathered from al33? Default is False, and data is gathered from r87. If True, then group is required",default="False") args = parser.parse_args() #Parse arguments from cmd line and set up inputs (date region model) model = args.m region = args.r t1 = args.t1 t2 = args.t2 issave = args.issave al33 = args.al33 if args.outname==None: out_name = model else: out_name = args.outname experiment = args.e ensemble = args.ens group = args.group project = args.project ver6hr = args.ver6hr ver3hr = args.ver3hr if region == "sa_small": start_lat = -38; end_lat = -26; start_lon = 132; end_lon = 142 elif region == "aus": start_lat = -44.525; end_lat = -9.975; start_lon = 111.975; end_lon = 156.275 elif region == "global": start_lat = -70; end_lat = 70; start_lon = -180; end_lon = 179.75 else: raise ValueError("INVALID REGION\n") domain = [start_lat,end_lat,start_lon,end_lon] try: time = [dt.datetime.strptime(t1,"%Y%m%d%H"),dt.datetime.strptime(t2,"%Y%m%d%H")] except: raise ValueError("INVALID START OR END TIME. SHOULD BE YYYYMMDDHH\n") if issave=="True": issave = True elif issave=="False": issave = False else: raise ValueError("\n INVALID ISSAVE...SHOULD BE True OR False") if al33=="True": al33 = True elif al33=="False": al33 = False else: raise ValueError("\n INVALID al33...SHOULD BE True OR False") #Load data print("LOADING DATA...") if model in ["ACCESS1-0","ACCESS1-3","GFDL-CM3","GFDL-ESM2M","CNRM-CM5","MIROC5",\ "MRI-CGCM3","IPSL-CM5A-LR","IPSL-CM5A-MR","GFDL-ESM2G","bcc-csm1-1","MIROC-ESM",\ "BNU-ESM"]: #Check that t1 and t2 are in the same year year = np.arange(int(t1[0:4]), int(t2[0:4])+1) ta, hur, hgt, terrain, p_3d, ps, ua, va, uas, vas, tas, ta2d, tp, lon, lat, \ date_list = read_cmip(model, experiment, \ ensemble, year, domain, cmip_ver=5, al33=al33, group=group, ver6hr=ver6hr, ver3hr=ver3hr) p = np.zeros(p_3d[0,:,0,0].shape) tp = tp.astype("float32", order="C") elif model in ["ACCESS-ESM1-5", "ACCESS-CM2"]: year = np.arange(int(t1[0:4]), int(t2[0:4])+1) ta, hur, hgt, terrain, p_3d, ps, ua, va, uas, vas, tas, ta2d, lon, lat, \ date_list = read_cmip(model, experiment,\ ensemble, year, domain, cmip_ver=6, group=group, project=project) p = np.zeros(p_3d[0,:,0,0].shape) else: raise ValueError("Model not recognised") ta = ta.astype("float32", order="C") hur = hur.astype("float32", order="C") hgt = hgt.astype("float32", order="C") terrain = terrain.astype("float32", order="C") p = p.astype("float32", order="C") ps = ps.astype("float32", order="C") ua = ua.astype("float32", order="C") va = va.astype("float32", order="C") uas = uas.astype("float32", order="C") vas = vas.astype("float32", order="C") tas= tas.astype("float32", order="C") ta2d = ta2d.astype("float32", order="C") lon = lon.astype("float32", order="C") lat = lat.astype("float32", order="C") gc.collect() #This param list was originally given to AD for ERA5 global lightning report #param = np.array(["mu_cape", "eff_cape","ncape","mu_cin", "muq", "s06", "s0500", "lr700_500", "mhgt", "ta500","tp","cp","laplacian","t_totals"]) #This param list is intended for application to GCMs based on AD ERA5 lightning report param = np.array(["mu_cape","s06","laplacian","t_totals","ta850","ta500","dp850","tp",\ "muq","lr700_500","mhgt","z500"]) #Set output array output_data = np.zeros((ps.shape[0], ps.shape[1], ps.shape[2], len(param))) #Assign p levels to a 3d array, with same dimensions as input variables (ta, hgt, etc.) #If the 3d p-lvl array already exists, then declare the variable "mdl_lvl" as true. try: p_3d; mdl_lvl = True full_p3d = p_3d except: mdl_lvl = False p_3d = np.moveaxis(np.tile(p,[ta.shape[2],ta.shape[3],1]),[0,1,2],[1,2,0]).\ astype(np.float32) print("LOAD TIME..."+str(dt.datetime.now()-load_start)) tot_start = dt.datetime.now() for t in np.arange(0,ta.shape[0]): output = np.zeros((1, ps.shape[1], ps.shape[2], len(param))) cape_start = dt.datetime.now() print(date_list[t]) if mdl_lvl: p_3d = full_p3d[t] dp = get_dp(hur=hur[t], ta=ta[t], dp_mask = False) #Insert surface arrays, creating new arrays with "sfc" prefix sfc_ta = np.insert(ta[t], 0, tas[t], axis=0) sfc_hgt = np.insert(hgt[t], 0, terrain, axis=0) sfc_dp = np.insert(dp, 0, ta2d[t], axis=0) sfc_p_3d = np.insert(p_3d, 0, ps[t], axis=0) sfc_ua = np.insert(ua[t], 0, uas[t], axis=0) sfc_va = np.insert(va[t], 0, vas[t], axis=0) #Sort by ascending p a,temp1,temp2 = np.meshgrid(np.arange(sfc_p_3d.shape[0]) , np.arange(sfc_p_3d.shape[1]),\ np.arange(sfc_p_3d.shape[2])) sort_inds = np.flip(np.lexsort([np.swapaxes(a,1,0),sfc_p_3d],axis=0), axis=0) sfc_hgt = np.take_along_axis(sfc_hgt, sort_inds, axis=0) sfc_dp = np.take_along_axis(sfc_dp, sort_inds, axis=0) sfc_p_3d = np.take_along_axis(sfc_p_3d, sort_inds, axis=0) sfc_ua = np.take_along_axis(sfc_ua, sort_inds, axis=0) sfc_va = np.take_along_axis(sfc_va, sort_inds, axis=0) sfc_ta = np.take_along_axis(sfc_ta, sort_inds, axis=0) #Calculate q and wet bulb for pressure level arrays with surface values sfc_ta_unit = units.units.degC*sfc_ta sfc_dp_unit = units.units.degC*sfc_dp sfc_p_unit = units.units.hectopascals*sfc_p_3d sfc_hur_unit = mpcalc.relative_humidity_from_dewpoint(sfc_ta_unit, sfc_dp_unit)*\ 100*units.units.percent sfc_q_unit = mpcalc.mixing_ratio_from_relative_humidity(sfc_hur_unit,\ sfc_ta_unit,sfc_p_unit) sfc_q = np.array(sfc_q_unit) #Now get most-unstable CAPE (max CAPE in vertical, ensuring parcels used are AGL) cape3d = wrf.cape_3d(sfc_p_3d,sfc_ta+273.15,\ sfc_q,sfc_hgt,\ terrain,ps[t],\ True,meta=False, missing=0) cape = cape3d.data[0] cin = cape3d.data[1] lfc = cape3d.data[2] lcl = cape3d.data[3] el = cape3d.data[4] #Mask values which are below the surface and above 350 hPa AGL cape[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan cin[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan lfc[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan lcl[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan el[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan #Get maximum (in the vertical), and get cin, lfc, lcl for the same parcel mu_cape_inds = np.tile(np.nanargmax(cape,axis=0), (cape.shape[0],1,1)) mu_cape = np.take_along_axis(cape, mu_cape_inds, 0)[0] muq = np.take_along_axis(sfc_q, mu_cape_inds, 0)[0] * 1000 #Calculate other parameters #Thermo thermo_start = dt.datetime.now() lr700_500 = get_lr_p(ta[t], p_3d, hgt[t], 700, 500) melting_hgt = get_t_hgt(sfc_ta,np.copy(sfc_hgt),0,terrain) melting_hgt = np.where((melting_hgt < 0) | (np.isnan(melting_hgt)), 0, melting_hgt) ta500 = get_var_p_lvl(np.copy(sfc_ta), sfc_p_3d, 500) ta850 = get_var_p_lvl(np.copy(sfc_ta), sfc_p_3d, 850) dp850 = get_var_p_lvl(np.copy(sfc_dp), sfc_p_3d, 850) v_totals = ta850 - ta500 c_totals = dp850 - ta500 t_totals = v_totals + c_totals #Winds winds_start = dt.datetime.now() s06 = get_shear_hgt(sfc_ua, sfc_va, np.copy(sfc_hgt), 0, 6000, terrain) #Laplacian x, y = np.meshgrid(lon,lat) dx, dy = mpcalc.lat_lon_grid_deltas(x,y) if mdl_lvl: z500 = get_var_p_lvl(hgt[t], p_3d, 500) laplacian = np.array(mpcalc.laplacian(z500,deltas=[dy,dx])*1e9) else: z500 = np.squeeze(hgt[t,p==500]) laplacian = np.array(mpcalc.laplacian(uniform_filter(z500, 4),deltas=[dy,dx])*1e9) #Fill output output = fill_output(output, t, param, ps, "mu_cape", mu_cape) output = fill_output(output, t, param, ps, "muq", muq) output = fill_output(output, t, param, ps, "s06", s06) output = fill_output(output, t, param, ps, "lr700_500", lr700_500) output = fill_output(output, t, param, ps, "ta500", ta500) output = fill_output(output, t, param, ps, "ta850", ta850) output = fill_output(output, t, param, ps, "dp850", dp850) output = fill_output(output, t, param, ps, "mhgt", melting_hgt) output = fill_output(output, t, param, ps, "tp", tp[t]) output = fill_output(output, t, param, ps, "laplacian", laplacian) output = fill_output(output, t, param, ps, "t_totals", t_totals) output = fill_output(output, t, param, ps, "z500", z500) output_data[t] = output print("SAVING DATA...") param_out = [] for param_name in param: temp_data = output_data[:,:,:,np.where(param==param_name)[0][0]] param_out.append(temp_data) #If the mhgt variable is zero everywhere, then it is likely that data has not been read. #In this case, all values are missing, set to zero. for t in np.arange(param_out[0].shape[0]): if param_out[np.where(param=="mhgt")[0][0]][t].max() == 0: for p in np.arange(len(param_out)): param_out[p][t] = np.nan if issave: save_netcdf(region, model, out_name, date_list, lat, lon, param, param_out, \ out_dtype = "f4", compress=True) print(dt.datetime.now() - tot_start)