def to_json(frame):
""" convert a pandas dataframe to json representation """
frame = frame.set_index(['location_id', 'nest_id', 'variable', 'height'])
logger = wrftools.get_logger()
groupby = frame.index.names
grouped = frame.groupby(level=groupby)
ngroups = grouped.ngroups
# as long as there is only one reference to a string
# concatenation is O(n), so not too bad a method
result = '['
for i, (names, group) in enumerate(grouped):
result += '{'
for j, name in enumerate(names):
result += '"%s":"%s"' % (groupby[j], name)
if j < len(names):
result += ',\n'
ts = [time.mktime(t.timetuple()) * 1000 for t in group.valid_time]
values = group.value
# This is a bit of a hack
# Truncate values to 3 decimal places
valstrs = ['%0.3f' % v for v in values]
series = map(list, zip(ts, valstrs))
result += '"data" : '
# Because the values have been converted to 3-decimal place strings
# They are enclosed in ' ', which we need to remove.
result += str(series).replace("'", "")
result += '}'
if i < ngroups - 1:
result += ',\n'
result += ']'
result = result.replace('nan', 'null')
result = result.replace("'", "")
return result
def to_json(frame):
""" convert a pandas dataframe to json representation """
frame = frame.set_index(['location_id','nest_id', 'variable', 'height'])
logger=wrftools.get_logger()
groupby = frame.index.names
grouped = frame.groupby(level=groupby)
ngroups = grouped.ngroups
# as long as there is only one reference to a string
# concatenation is O(n), so not too bad a method
result = '['
for i,(names, group) in enumerate(grouped):
result += '{'
for j,name in enumerate(names):
result+='"%s":"%s"' % (groupby[j], name)
if j<len(names):
result+=',\n'
ts = [time.mktime(t.timetuple())*1000 for t in group.valid_time]
values = group.value
# This is a bit of a hack
# Truncate values to 3 decimal places
valstrs = ['%0.3f' % v for v in values]
series = map(list, zip(ts,valstrs ))
result += '"data" : '
# Because the values have been converted to 3-decimal place strings
# They are enclosed in ' ', which we need to remove.
result += str(series).replace("'", "")
result += '}'
if i<ngroups-1:
result+=',\n'
result+=']'
result = result.replace('nan', 'null')
result = result.replace("'", "")
return result
def _get_index(lat, lon, lat2d, lon2d):
""" Finds the nearest mass point grid index to the point (lon, lat).
Works but is slow as just naively searches through the arrays point
by point. Would be much better to implement the ncl function
wrf_user_get_ij to use projection information to actually calculate
the point in porjected space.
Arguments:
@lat: the latitude of the target point
@lon: longitude of the target point
@lat2d: 2d array of latitudes of grid points
@lon2d: 2d array of longitudes of grid points
Returns (i,j) of nearest grid cell. Raises exception if outside"""
logger = wrftools.get_logger()
logger.debug("finding index of (%0.3f, %0.3f) " %(lat,lon))
west_east = lat2d.shape[0]
south_north = lat2d.shape[1]
logger.debug("dimensions of domain are: %d south_north, %d west_east" %(south_north, west_east))
if not _in_domain(lat, lon, lat2d, lon2d):
logger.error('point (%0.3f, %0.3f) not in model domain' % (lat, lon))
raise DomainError('point (%0.3f, %0.3f) not in model domain' %(lat, lon))
#
# slow, but will work. Just search through the arrays until we
# hit the nearest grid point
#
min_dist = 10000000 # if the point is further than 10M m away, don't bother!
min_x = 0
min_y = 0
for x in range(west_east-1):
for y in range(south_north-1):
point_lat = lat2d[x,y]
point_lon = lon2d[x,y]
d = tools.haversine(lat, lon, point_lat, point_lon)
if d < min_dist:
min_dist = d
min_x = x
min_y = y
if min_x==0 or min_y==0 or min_x>west_east or min_y>south_north:
logger.error("Point is on/off edge of of model domain, this should have been caught earlier!")
raise DomainError("Point is on/off edge of of model domain")
logger.debug('nearest grid index is x=%d, y=%d, %0.3f m away' %(min_x, min_y, min_dist))
logger.debug('latitude, longitude of original is (%0.3f, %0.3f)' %(lat, lon))
logger.debug('latitude, longitude of index is (%0.3f, %0.3f)' %(lat2d[min_x,min_y], lon2d[min_x,min_y]))
return (min_x, min_y, min_dist)
def get_ij_old(config):
logger = wrftools.get_logger()
logger.info('*** RUNNING NCL EXTRACTION ***')
domain_dir = config['domain_dir']
model_run = config['model_run']
ncl_nc = config['ncl_nc']
ncl_grb = config['ncl_grb']
ncl_code_dir = config['ncl_code_dir']
ncl_nc_code = config['ncl_nc_code']
ncl_grb_code = config['ncl_grb_code']
wrfout_dir = '%s/%s/' % (domain_dir, model_run)
init_time = config['init_time']
dom = config['dom']
grb_file = '%s/%s/archive/wrfpost_d%02d_%s.grb' % (
domain_dir, model_run, dom, init_time.strftime(archive_format))
nc_file = '%s/%s/wrfout/wrfout_d%02d_%s:00:00.nc' % (
domain_dir, model_run, dom, init_time.strftime(archive_format))
ncl_out_dir = '%s/%s/plots/%s/d%02d/' % (
domain_dir, model_run, init_time.strftime(archive_format), dom)
ncl_output = config['ncl_output']
namelist_wps = '%(domain_dir)s/%(model_run)s/namelist.wps' % config
if not os.path.exists(ncl_out_dir):
os.makedirs(ncl_out_dir)
#
# Communicate to NCL via environment variables
#
os.environ['DOMAIN_DIR'] = domain_dir
os.environ['MODEL_RUN'] = model_run
os.environ['NCL_OUT_DIR'] = ncl_out_dir
os.environ['NCL_OUTPUT'] = ncl_output
os.environ['NAMELIST_WPS'] = namelist_wps
os.environ['FCST_FILE'] = grb_file
cmd = "ncl %s/wrf_grib_series.ncl" % (ncl_code_dir)
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = p.stdout.readlines()
for line in lines:
if '(0,0)' in line:
thanet_i = line.split()[-1]
logger.debug(thanet_i)
if '(0,1)' in line:
aberdeen_i = line.split()[-1]
logger.debug(aberdeen_i)
if '(1,0)' in line:
thanet_j = line.split()[-1]
logger.debug(thanet_j)
if '(1,1)' in line:
aberdeen_j = line.split()[-1]
logger.debug(aberdeen_j)
return [(thanet_i, thanet_j), (aberdeen_i, abderdeen_j)]
def get_index(lat, lon, lat2d, lon2d):
""" Finds the nearest mass point grid index to the point (lon, lat).
Works but is slow as just naively searches through the arrays point
by point.
Arguments:
@lat: the latitude of the target point
@lon: longitude of the target point
@lat2d: 2d array of latitudes of grid points
@lon2d: 2d array of longitudes of grid points
Returns (i,j) of nearest grid cell. Raises exception if outside"""
logger = wrftools.get_logger()
logger.debug("finding index of (%0.3f, %0.3f) " % (lat, lon))
west_east = lat2d.shape[0]
south_north = lat2d.shape[1]
logger.debug("dimensions of domain are: %d south_north, %d west_east" %
(south_north, west_east))
if not in_domain(lat, lon, lat2d, lon2d):
raise DomainError('point (%0.3f, %0.3f) not in model domain' %
(lat, lon))
#
# slow, but will work. Just search through the arrays until we
# hit the nearest grid point
#
min_dist = 10000000 # if the point is further than 10M m away, don't bother!
min_x = 0
min_y = 0
for x in range(west_east - 1):
for y in range(south_north - 1):
point_lat = lat2d[x, y]
point_lon = lon2d[x, y]
d = tools.haversine(lat, lon, point_lat, point_lon)
if d < min_dist:
min_dist = d
min_x = x
min_y = y
if min_x == 0 or min_y == 0 or min_x > west_east or min_y > south_north:
logger.error(
"Point is on/off edge of of model domain, this should have been caught earlier!"
)
raise DomainError("Point is on/off edge of of model domain")
logger.debug('nearest grid index is x=%d, y=%d, %0.3f m away' %
(min_x, min_y, min_dist))
logger.debug('latitude, longitude of original is (%0.3f, %0.3f)' %
(lat, lon))
logger.debug('latitude, longitude of index is (%0.3f, %0.3f)' %
(lat2d[min_x, min_y], lon2d[min_x, min_y]))
return (min_x, min_y, min_dist)
def get_ij_old(config):
logger = wrftools.get_logger()
logger.info('*** RUNNING NCL EXTRACTION ***')
domain_dir = config['domain_dir']
model_run = config['model_run']
ncl_nc = config['ncl_nc']
ncl_grb = config['ncl_grb']
ncl_code_dir = config['ncl_code_dir']
ncl_nc_code = config['ncl_nc_code']
ncl_grb_code = config['ncl_grb_code']
wrfout_dir = '%s/%s/'%(domain_dir, model_run)
init_time = config['init_time']
dom = config['dom']
grb_file = '%s/%s/archive/wrfpost_d%02d_%s.grb' %(domain_dir, model_run, dom, init_time.strftime(archive_format))
nc_file = '%s/%s/wrfout/wrfout_d%02d_%s:00:00.nc' %(domain_dir, model_run, dom, init_time.strftime(archive_format))
ncl_out_dir = '%s/%s/plots/%s/d%02d/' %(domain_dir,model_run, init_time.strftime(archive_format), dom)
ncl_output = config['ncl_output']
namelist_wps = '%(domain_dir)s/%(model_run)s/namelist.wps' % config
if not os.path.exists(ncl_out_dir):
os.makedirs(ncl_out_dir)
#
# Communicate to NCL via environment variables
#
os.environ['DOMAIN_DIR'] = domain_dir
os.environ['MODEL_RUN'] = model_run
os.environ['NCL_OUT_DIR'] = ncl_out_dir
os.environ['NCL_OUTPUT'] = ncl_output
os.environ['NAMELIST_WPS'] = namelist_wps
os.environ['FCST_FILE'] = grb_file
cmd = "ncl %s/wrf_grib_series.ncl" % (ncl_code_dir)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = p.stdout.readlines()
for line in lines:
if '(0,0)' in line:
thanet_i = line.split()[-1]
logger.debug(thanet_i)
if '(0,1)' in line:
aberdeen_i = line.split()[-1]
logger.debug(aberdeen_i)
if '(1,0)' in line:
thanet_j = line.split()[-1]
logger.debug(thanet_j)
if '(1,1)' in line:
aberdeen_j = line.split()[-1]
logger.debug(aberdeen_j)
return [(thanet_i, thanet_j), (aberdeen_i, abderdeen_j)]
def _read_time_series(fname):
""" Reads formatted time series"""
logger = wrftools.get_logger()
logger.debug('reading tseries file %s' % fname)
#
# Current format of time series files is:
#fcst_ts_names = ['domain', 'model', 'model_run', 'nest_id', 'fcst_init','fcst_valid', 'fcst_var',
# 'location_id', 'fcst_lat', 'fcst_lon', 'fcst_hgt', 'fcst_val']
#
names = ['domain', 'model', 'model_run', 'grid_id', 'init_time', 'valid_time', 'var', 'obs_sid', 'lat', 'lon', 'hgt', 'val']
converters = {4: wrftools.strptime, 5: wrftools.strptime}
rec = np.genfromtxt(fname, delimiter=',', names=names, dtype=None, converters=converters)
return rec
def _in_domain(lat, lon, lat2d, lon2d):
"""Tests whether (lat,lon) is within domain defined by lat2d and lon2d.
Simply tests lat within lat2d and lon within lon2d
Returns boolean, true if the point is within the domain """
logger = wrftools.get_logger()
min_lon = np.min(lon2d)
max_lon = np.max(lon2d)
min_lat = np.min(lat2d)
max_lat = np.max(lat2d)
if (lat < min_lat) or (lat> max_lat) or (lon<min_lon) or (lon>max_lon):
logger.debug("point (%0.3f, %0.3f) is not within domain (%0.2f, %0.3f, %0.3f, %0.3f)" %(lat, lon, min_lat, max_lat, min_lon, max_lon))
return False
else:
return True
def _in_domain(lat, lon, lat2d, lon2d):
"""Tests whether (lat,lon) is within domain defined by lat2d and lon2d.
Simply tests lat within lat2d and lon within lon2d
Returns boolean, true if the point is within the domain """
logger = wrftools.get_logger()
min_lon = np.min(lon2d)
max_lon = np.max(lon2d)
min_lat = np.min(lat2d)
max_lat = np.max(lat2d)
if (lat < min_lat) or (lat > max_lat) or (lon < min_lon) or (lon >
max_lon):
logger.debug(
"point (%0.3f, %0.3f) is not within domain (%0.2f, %0.3f, %0.3f, %0.3f)"
% (lat, lon, min_lat, max_lat, min_lon, max_lon))
return False
else:
return True
def tseries_to_json(config):
"""Converts the time series, in the record-based format, to JSON strings for plotting """
logger=wrftools.get_logger()
domain_dir = config['domain_dir']
domain = config['domain']
dom = config['dom']
model_run = config['model_run']
init_time = config['init_time']
tseries_dir = config['tseries_dir']
tseries_file = '%s/tseries_d%02d_%s.nc' % (tseries_dir, dom,init_time.strftime("%Y-%m-%d_%H"))
json_dir = '%s/%s/json' % (domain_dir, model_run)
#json_file = '%s/fcst_data.json' % json_dir
if not os.path.exists(json_dir):
os.makedirs(json_dir)
ncdump.write_json_files([tseries_file], ncdump.GLOBAL_ATTS, ncdump.VAR_ATTS,ncdump.COORD_VARS, json_dir, ncdump.FILE_DATE_FMT)
logger.info('*** WRITTEN JSON DATA ***')
def _read_time_series(fname):
""" Reads formatted time series"""
logger = wrftools.get_logger()
logger.debug('reading tseries file %s' % fname)
#
# Current format of time series files is:
#fcst_ts_names = ['domain', 'model', 'model_run', 'nest_id', 'fcst_init','fcst_valid', 'fcst_var',
# 'location_id', 'fcst_lat', 'fcst_lon', 'fcst_hgt', 'fcst_val']
#
names = [
'domain', 'model', 'model_run', 'grid_id', 'init_time', 'valid_time',
'var', 'obs_sid', 'lat', 'lon', 'hgt', 'val'
]
converters = {4: wrftools.strptime, 5: wrftools.strptime}
rec = np.genfromtxt(fname,
delimiter=',',
names=names,
dtype=None,
converters=converters)
return rec
def tseries_to_json(config):
"""Converts the time series, in the record-based format, to JSON strings for plotting """
logger = wrftools.get_logger()
domain_dir = config['domain_dir']
domain = config['domain']
dom = config['dom']
model_run = config['model_run']
init_time = config['init_time']
tseries_dir = config['tseries_dir']
tseries_file = '%s/tseries_d%02d_%s.nc' % (
tseries_dir, dom, init_time.strftime("%Y-%m-%d_%H"))
json_dir = '%s/%s/json' % (domain_dir, model_run)
#json_file = '%s/fcst_data.json' % json_dir
if not os.path.exists(json_dir):
os.makedirs(json_dir)
ncdump.write_json_files([tseries_file], ncdump.GLOBAL_ATTS,
ncdump.VAR_ATTS, ncdump.COORD_VARS, json_dir,
ncdump.FILE_DATE_FMT)
logger.info('*** WRITTEN JSON DATA ***')
import wrftools
import sys
config = wrftools.read_namelist(sys.argv[1]).settings
base_dir = config['base_dir']
domain = config['domain']
domain_dir = '%s/%s' % (base_dir, domain)
config['domain_dir'] = domain_dir # domain directory
wrftools.create_logger(config)
logger = wrftools.get_logger()
logger.info('testing timing')
wrftools.timing(config)
import wrftools
import sys
config = wrftools.read_namelist(sys.argv[1]).settings
base_dir = config['base_dir']
domain = config['domain']
domain_dir = '%s/%s' % (base_dir, domain)
config['domain_dir'] = domain_dir # domain directory
wrftools.create_logger(config)
logger = wrftools.get_logger()
logger.info('testing timing')
wrftools.timing(config)
