def complete_configuration(self):
self.data = {}
dir = './'
#dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/'
file = 'CoV_wind_station_output_prox_penalty.nc' #file with _II has smaller exclusion zone
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['CoV_wind'][:,:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_wind'] = numpy.array(temp, dtype=float)
else:
self.data['ts_wind'] = temp
file = 'CoV_dsr_station_output_prox_penalty.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['CoV_dsr'][:,:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_solar'] = numpy.array(temp, dtype=float)
else:
self.data['ts_solar'] = temp
file = 'Aus_demand_sample_raw.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_demand'][:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_demand'] = numpy.array(temp, dtype=float)
else:
self.data['ts_demand'] = temp
wind_nan = numpy.isnan(self.data['ts_wind'])
solar_nan = numpy.isnan(self.data['ts_solar'])
demand_nan = numpy.isnan(self.data['ts_demand'])
wind_row = wind_nan.any(1)
solar_row = solar_nan.any(1)
combo = numpy.array([wind_row, solar_row, demand_nan])
combo_flat = combo.any(0)
self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :]
self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :]
self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False]
print self.data['ts_wind'].shape
print self.data['ts_solar'].shape
print self.data['ts_demand'].shape
self.ts_length = self.data['ts_wind'].shape[0]
return None
def Dataset(ncfile):
if isinstance(ncfile, str):
return pupynere.NetCDFFile(ncfile)
elif isinstance(ncfile, pupynere.NetCDFFile):
return ncfile
else:
raise TypeError, 'type %s not supported' % type(ncfile)
def createNetCDFFile(cfg, direct):
path = '%s/%s.%02d.nc' % (direct, cfg['ident'], cfg['month'])
os.system('/bin/rm -f %s' % path)
try:
fh = pupynere.NetCDFFile(path, 'w', time.ctime())
except IOError:
msg = 'Cannot create file %s' % path
print msg
if _Logger : _Logger.error(msg)
return None
fh.title = 'Some hopefully less useless junk'
fh.ident = cfg['ident']
fh.month = cfg['month']
fh.createDimension('dd', len(cfg['dd'])+1) # cycle, 0 is calm/vrb
fh.createDimension('ff', len(cfg['ff'])+1)
fh.createDimension('cig', len(cfg['cig'])+1)
fh.createDimension('vsby', len(cfg['vsby'])+1)
fh.createDimension('obv', 8)
fh.createDimension('int', 4) # check this
fh.createDimension('pcp', 4)
fh.createDimension('total', 1)
for var in fh.dimensions:
v = fh.createVariable(var, 'i', (var,))
v[:] = 0
all = fh.createVariable('all', 'i',
('dd', 'ff', 'obv', 'int', 'pcp', 'cig', 'vsby'))
all[:] = 0
for v in fh.variables:
fh.variables[v].units = 'count'
return fh
def test_variable_missing(self):
# first, create a netCDF files to test with
# float timeseries
ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2],
[5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6],
[4, 3, 2], [5, 5, 4]])
f1 = nc.NetCDFFile('test_missing_1.nc', 'w')
# put ts_wind, ts_time and float_1 into file 1
f1.createDimension('wx', ts_wind.shape[0])
f1.createDimension('wy', ts_wind.shape[1])
ts_wind_var = f1.createVariable('ts_wind_funny', 'float64',
('wx', 'wy'))
ts_wind_var[:, :] = ts_wind
f1.close()
config = {
'description': 'test variable missing',
'model': 'data.ncdata.py',
'section': 'Data',
'ts_float_list': 'ts_wind',
'ts_wind_file': 'test_missing_1.nc'
}
self.assertRaises(mureilexception.ConfigException,
self.data.set_config, config)
def _climate(ident, periods, dataDir):
result = {}
try:
month = time.gmtime(periods[0][1]['time']['from']).tm_mon
path = os.path.join(dataDir, '%s.%02d.nc' % (ident, month))
if not os.path.isfile(path):
raise Avn.AvnError('File %s does not exist' % path)
ncfh = pupynere.NetCDFFile(path)
cfg = AvnParser.getClimQCConfig(ident)
except IndexError:
return result
except IOError:
raise IOError, 'Cannot access %s' % path
for ix, p in periods:
off = _getOffset(cfg, p)
a0, text, off = _analyze(ncfh, off, cfg['showdetails'])
level = _getLevel(a0, cfg['alpha'])
if a0 < cfg['alpha']:
suggestions = [_analyze(ncfh, toff, cfg['showdetails']) \
for toff in _neighbors(ncfh, off)]
suggestions.sort()
suggestions.reverse()
best = filter(None, [_suggest(off, x[2]) for x in suggestions \
if x[0] >= cfg['alpha']])
if best:
text.extend(best)
else:
text.append('Cannot figure it out')
result[ix] = {'level': level, 'text': '\n'.join(text)}
ncfh.close()
return result
def data_for_time(self, t0):
""" Read data from the file corresponding to datetime t.
Returns xedge, yedge, and density for the file
"""
fname, i = self._time_lookup[
t0] #i is the frame id for this time in NetCDFFile f
f = nc.NetCDFFile(fname)
data = f.variables # dictionary of variable names to nc_var objects
dims = f.dimensions # dictionary of dimension names to sizes
x = data[self.x_name]
y = data[self.y_name]
t = data[self.t_name]
grid = data[self.grid_name]
indexer = [
slice(None),
] * len(grid.shape)
grid_dims = grid.dimensions # tuple of dimension names
name_to_idx = dict((k, i) for i, k in enumerate(grid_dims))
grid_t_idx = name_to_idx[t.dimensions[0]]
grid_x_idx = name_to_idx[x.dimensions[0]]
grid_y_idx = name_to_idx[y.dimensions[0]]
xedge = centers_to_edges(x)
yedge = centers_to_edges(y)
indexer[grid_t_idx] = i
density = grid[indexer].transpose()
f.close()
return xedge, yedge, density
def complete_configuration(self):
dir = './'
file = 'CoV_output.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
self.data = {}
self.data['ts_wind'] = numpy.array(f.variables['ts_wind'][:, :],
dtype=float)
self.data['ts_solar'] = numpy.array(f.variables['ts_solar'][:, :],
dtype=float)
self.data['ts_demand'] = numpy.array(f.variables['ts_demand'][:],
dtype=float)
self.ts_length = self.data['ts_wind'].shape[0]
self.is_configured = True
return None
def getFile(self, path):
try:
self._path = path
self._fh = pupynere.NetCDFFile(self._path, 'r')
self.issuetime = self._fh.variables['time'].getValue()
self._validtimes = \
self._fh.variables['validTimeList'][:self.NumData].tolist()
var = self._fh.variables['stationName']
self._sitedict = {}
for n in xrange(var.shape[0]):
ident = var[n, :].tostring().split('\x00')[0].rstrip()
self._sitedict[ident] = n
return True
except IOError:
_Logger.error('Error accessing %s', path)
return False
def _frame_times_for_file(self, fname):
""" Called once by init to set up frame lookup tables and yield
the frame start times. _frame_lookup goes from
datetime->(nc file, frame index)"""
f = nc.NetCDFFile(fname)
data = f.variables # dictionary of variable names to nc_var objects
dims = f.dimensions # dictionary of dimension names to sizes
t = data[self.t_name]
base_date = datetime.strptime(t.units,
"seconds since %Y-%m-%d %H:%M:%S")
for i in range(t.shape[0]):
frame_start = base_date + timedelta(0, float(t[i]), 0)
self._time_lookup[frame_start] = (fname, i)
yield frame_start
f.close()
def test_different_lengths(self):
# first, create a netCDF files to test with
# float timeseries
ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2],
[5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6],
[4, 3, 2], [5, 5, 4]])
ts_solar = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2],
[5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6],
[4, 3, 2], [5, 5, 4], [5, 5, 4], [5, 5, 4]])
f1 = nc.NetCDFFile('test_different_lengths.nc', 'w')
f1.createDimension('wx', ts_wind.shape[0])
f1.createDimension('wy', ts_wind.shape[1])
f1.createDimension('sx', ts_solar.shape[0])
f1.createDimension('sy', ts_solar.shape[1])
ts_wind_var = f1.createVariable('ts_wind', 'float64', ('wx', 'wy'))
ts_wind_var[:, :] = ts_wind
ts_solar_var = f1.createVariable('ts_solar', 'float64', ('sx', 'sy'))
ts_solar_var[:, :] = ts_solar
f1.close()
config = {
'description': 'test different length ts',
'model': 'data.ncdata.py',
'section': 'Data',
'ts_float_list': 'ts_wind ts_solar',
'ts_wind_file': 'test_different_lengths.nc',
'ts_solar_file': 'test_different_lengths.nc'
}
self.assertRaises(mureilexception.ConfigException,
self.data.set_config, config)
def getFile(self, path):
try:
self._path = path
self._fh = pupynere.NetCDFFile(self._path, 'r')
var = self._fh.variables['staName']
# get record numbers for all idents
self._sitedict = {}
for n, s in enumerate(var[:]):
ident = s.tostring().split('\x00')[0].rstrip()
try:
self._sitedict[ident].append(n)
except KeyError:
self._sitedict[ident] = [n]
# sort wrt valid time
for i in self._sitedict:
v = self._fh.variables['validTime']
tmp = [(int(v[n]), n) for n in self._sitedict[i]]
tmp.sort()
self._sitedict[i] = [t[1] for t in tmp][:self.NumData + 2]
return True
except IOError:
_Logger.error('Error accessing %s', path)
return False
def complete_configuration(self):
self.data = {}
dir = '/export/karoly2/Roger/ACCESS-A/'
file = 'RegGrid_wind_output_11point_gap.nc' #file with _II has smaller exclusion zone
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['wind_power'][:, :]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_wind'] = numpy.array(temp, dtype=float)
else:
self.data['ts_wind'] = temp
file = 'RegGrid_dsr_output_19point_gap.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['dsr'][:, :]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_solar'] = numpy.array(temp, dtype=float)
else:
self.data['ts_solar'] = temp
dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/'
file = 'Aus_demand_2010_2011.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_demand'][:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_demand'] = numpy.array(temp, dtype=float)
else:
self.data['ts_demand'] = temp
wind_nan = numpy.isnan(self.data['ts_wind'])
solar_nan = numpy.isnan(self.data['ts_solar'])
demand_nan = numpy.isnan(self.data['ts_demand'])
wind_row = wind_nan.any(1)
solar_row = solar_nan.any(1)
combo = numpy.array([wind_row, solar_row, demand_nan])
combo_flat = combo.any(0)
self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :]
self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :]
self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False]
print self.data['ts_wind'].shape
print self.data['ts_solar'].shape
print self.data['ts_demand'].shape
self.ts_length = self.data['ts_wind'].shape[0]
self.data['ts_solar_distances'] = numpy.array([
668730.4375, 909338.1875, 372287.25, 1125613.25, 211878.890625,
68551.4609375, 768108.3125, 513450.8125, 1239141.625,
242710.109375, 221302.75, 835822.375, 641565.125, 1371163.875,
404052.09375, 449818.1875, 949940.75, 1176451.625, 1239225.875,
811838.8125, 610120.125, 679184.1875, 1126859.125, 926546.,
987652.1875, 1093136., 923490.5625, 605613.3125, 681002.75,
1170811.375, 1214910.625, 881211.1875, 1177105.25, 1020829.875,
374478.46875, 884468.125, 1147496.5, 1412441., 557196.4375,
111162.4375, 1129901.25, 901584.5625, 207429.328125, 830748.625,
1097925., 1311133.125, 474638.53125, 231650.921875, 1030239.5625,
784340.375, 599865.75, 355621.71875, 801379.6875, 1050643.,
1298696.375, 535831.75, 1012697.625, 282052., 738578.625,
468747.75, 587530.1875, 874383.75, 1085080.5, 1165477.5,
739965.6875, 1043011.9375, 133451.609375, 466417.03125,
887445.1875, 843954.375, 882972.9375, 911230.0625, 256325.21875,
196988.21875, 722306.0625, 494064.375, 656777.1875, 590096.75,
838475., 576432.125, 1038951.0625, 709157.1875, 329522.5625,
246577.59375, 866532.3125, 684706.0625, 507050.0625, 353760.5,
528336.75, 308515.5, 294442.8125, 38012.9140625, 354666.53125,
473246.90625, 815155.4375, 729953.8125, 211116.015625, 548792.25,
134444.53125, 367146.65625, 252677.890625, 404498.71875,
615562.1875, 426158.125, 478903.125, 478492.6875, 204912.421875,
186742.734375, 449120.125, 467410.25, 242252.140625, 325181.5625,
223081.09375, 109695.5546875, 159941.5
],
dtype=float)
self.data['ts_wind_distances'] = numpy.array([
668730.4375, 909338.1875, 1058640.875, 516586.90625, 754774.625,
909874.625, 372287.25, 1125613.25, 212450.59375, 211878.890625,
611953.6875, 68551.4609375, 768108.3125, 938604.9375, 475540.34375,
341444.6875, 1206778.375, 1072823.25, 209707.75, 161579.765625,
637232.6875, 111058.5234375, 353789.75, 811775.375, 946141.8125,
513450.8125, 383905.8125, 1239141.625, 1106075.25, 274558.,
157525.8125, 242710.109375, 694877.25, 221302.75, 387798.59375,
835822.375, 992085.4375, 585522.375, 465096.75, 1306448.625,
1127716.375, 376003.46875, 262230.59375, 309341.46875, 734677.625,
341611.15625, 897978.625, 1027317.9375, 641565.125, 533193.8125,
1371163.875, 1242591.625, 464625.5, 374925.21875, 404052.09375,
811223.6875, 449818.1875, 949940.75, 1152356.75, 731487.6875,
632564.3125, 1328731.375, 1318661.25, 575718.375, 489012.5,
503957.4375, 877959.625, 567724., 1176451.625, 1085111.125,
1239225.875, 811838.8125, 723001.8125, 1228210., 1317506.875,
1061301.5, 678183., 604640.75, 610120.125, 1019876.0625,
679184.1875, 1064924.875, 1183351.125, 957665., 869637.5625,
1125113.625, 1221260.5, 1323786.875, 816667.5625, 719099.3125,
722332.8125, 908779.125, 1126859.125, 812075.8125, 926546.,
1267433.125, 1071478.125, 987652.1875, 992131.5625, 1093136.,
1200561.75, 944202.625, 841807.75, 831524.5625, 766421.25,
923490.5625, 797534.4375, 1268186.625, 1294879.625, 1179975.25,
1100428.25, 605613.3125, 871203.0625, 978584.5625, 1091826.375,
1060980.625, 956020.0625, 637515.875, 1057804.5, 681002.75,
1170811.375, 1295239.375, 1214910.625, 457551.96875, 766078.25,
881211.1875, 1001023.3125, 1178623., 1083117.625, 1063405.75,
511700.875, 1177105.25, 562428.6875, 660418.0625, 783159.8125,
1081792.375, 1020829.875, 374478.46875, 1181675.5, 444261.125,
884468.125, 1015654., 1147496.5, 1279800., 1412441., 557196.4375,
686802.875, 1046372.1875, 969600.625, 224758.90625, 1270513.,
111162.4375, 1129901.25, 321961.25, 829833.5625, 963491.6875,
1097242.375, 1231062., 1364937.25, 450944.96875, 584710.375,
989115.9375, 901584.5625, 60845.32421875, 1226836.625,
130785.546875, 1098943.5, 207429.328125, 830748.625, 964274.5,
1097925., 1231665.75, 1311133.125, 345572.96875, 474638.53125,
605817.375, 953379.625, 853869.9375, 709148.6875, 144821.765625,
1161816.75, 231650.921875, 1030239.5625, 281000.03125, 757300.375,
887036.6875, 1017876.0625, 1149449.375, 1281537.875, 405575.3125,
519712.8125, 641537.125, 784340.375, 599865.75, 263454.5625,
1193936., 1058048.625, 938030.5, 648269.0625, 355621.71875,
801379.6875, 924709.125, 1050643., 1178353.875, 1298696.375,
448873.90625, 535831.75, 639195.5625, 778615.625, 486159.1875,
1150266.875, 432368.78125, 1012697.625, 887173.875, 635939.25,
370392.96875, 475697.8125, 802408.9375, 914610.6875, 1042862.375,
1164123.875, 1287900.625, 551048.75, 625359.1875, 713181.125,
282052., 738578.625, 468747.75, 1176506., 1038450.75, 907944.3125,
594211.4375, 331333.875, 587530.1875, 874383.75, 976254.0625,
1085080.5, 1127779.875, 1165477.5, 739965.6875, 196089.296875,
762503.6875, 446178.5625, 1156598.25, 1043011.9375, 608477.6875,
302244., 891794.3125, 974806.9375, 971028.9375, 981571.,
1009794.625, 133451.609375, 114606.5859375, 466417.03125,
818209.125, 1002819.125, 1039978.75, 683042.9375, 329406.875,
887445.1875, 855617.0625, 843954.375, 853292.25, 882972.9375,
1000176.8125, 196278.171875, 64264.375, 88714.125, 552421.75,
869001.5625, 870218., 911230.0625, 429908.59375, 256325.21875,
740118.0625, 718609.1875, 721628.5625, 748889.0625, 978856.625,
310406.625, 196988.21875, 722306.0625, 161488.421875, 599724.625,
934020.25, 737569.25, 814771.6875, 494064.375, 251381.703125,
336539., 656777.1875, 609839.375, 590096.75, 600250.25,
893760.0625, 391393.0625, 297513.59375, 823148.3125, 838475.,
704558.125, 686599.5625, 948955.6875, 964129.25, 294521.5625,
576432.125, 645428.0625, 1038951.0625, 709157.1875, 594892.125,
329522.5625, 258209.796875, 246577.59375, 537336.3125,
483168.28125, 462575.5625, 798845.6875, 503718.4375, 418395.09375,
866532.3125, 777922.0625, 670250.3125, 762942.75, 881406.8125,
884783.375, 413056.8125, 413987.5, 473308.65625, 535773.375,
684706.0625, 301884.875, 191727.3125, 135755.625, 507050.0625,
416187.03125, 353760.5, 629375.8125, 604659.375, 528336.75,
923294.9375, 236056.421875, 848792.3125, 723014.9375, 432805.5,
588751.75, 266921.25, 308515.5, 371523.1875, 745499.75,
294442.8125, 161392.625, 38012.9140625, 354666.53125, 442686.21875,
330582.15625, 241744.234375, 473246.90625, 653481.125, 585862.,
815155.4375, 771210.125, 489359.5625, 579454.375, 453331.5,
139264.03125, 201134.484375, 298521.34375, 729953.8125,
327048.5625, 199195.953125, 91859.6015625, 115152.90625,
211116.015625, 421740.375, 632862.875, 548792.25, 697630.4375,
659870.3125, 412630.8125, 548571.125, 439476.25, 543691.,
11564.77832031, 134444.53125, 268900.28125, 598547.3125,
367146.65625, 358178.9375, 252677.890625, 184504.65625,
404498.71875, 600163.5, 535114.375, 615562.1875, 543468.25,
426158.125, 545614.5625, 401627.4375, 567999.5625, 136595.25,
255334.890625, 478903.125, 288917.75, 254081.71875, 285744.96875,
301214.53125, 381815.09375, 484310.0625, 478492.6875, 505826.21875,
425950.34375, 526906.8125, 336393.5625, 360000.46875,
204912.421875, 142902.09375, 186742.734375, 292174.5625,
375563.59375, 433015.375, 449120.125, 467410.25, 341196.125,
534745.5625, 289708.96875, 242252.140625, 125803.328125,
32823.0625, 242620.25, 325181.5625, 379624.03125, 253196.859375,
344051.40625, 129340.453125, 223081.09375, 129551.390625,
239891.03125, 261754.0625, 356155.5, 109695.5546875,
32649.04492188, 159941.5
],
dtype=float)
return None
def write_cf_netcdf_latlon(outfile,
t_start,
t,
xloc,
yloc,
lon_for_x,
lat_for_y,
ctr_lat,
ctr_lon,
grid,
grid_var_name,
grid_description,
format='i',
**kwargs):
""" Write a Climate and Forecast Metadata-compliant NetCDF file.
Grid is regular in lon, lat and so no map projection information is necessary.
Should display natively in conformant packages like McIDAS-V.
"""
import pupynere as nc
missing_value = -9999
nc_out = nc.NetCDFFile(outfile, 'w')
nc_out.createDimension('lon', xloc.shape[0])
nc_out.createDimension('lat', yloc.shape[0])
nc_out.createDimension('ntimes', t.shape[0]) #unlimited==None
# declare the coordinate reference system, WGS84 values
proj = nc_out.createVariable('crs', 'i', ())
proj.grid_mapping_name = 'latitude_longitude'
proj.longitude_of_prime_meridian = 0.0
proj.semi_major_axis = 6378137.0
proj.inverse_flattening = 298.257223563
y_coord = nc_out.createVariable('latitude', 'f', ('lat', ))
y_coord.units = "degrees_north"
y_coord.long_name = "latitude"
y_coord.standard_name = 'latitude'
x_coord = nc_out.createVariable('longitude', 'f', ('lon', ))
x_coord.units = "degrees_east"
x_coord.long_name = "longitude"
x_coord.standard_name = 'longitude'
times = nc_out.createVariable('time', 'f',
('ntimes', )) #, filters=no_compress)
times.long_name = "time"
times.units = "seconds since %s" % t_start.strftime('%Y-%m-%d %H:%M:%S')
# lons = nc_out.createVariable('lons', 'd', ('nx','ny') )#, filters=no_compress)
# lons.long_name="longitude"
# lons.standard_name="longitude"
# lons.units = "degrees_east"
#
# lats = nc_out.createVariable('lats', 'd', ('nx','ny') )#, filters=no_compress)
# lats.long_name="latitude"
# lats.standard_name="latitude"
# lats.units = "degrees_north"
lightning2d = nc_out.createVariable(
grid_var_name, format,
('ntimes', 'lon', 'lat')) #, filters=no_compress)
lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)'
lightning2d.units = kwargs['grid_units']
# lightning2d.coordinates='time lons lats'
lightning2d.grid_mapping = "crs"
lightning2d.missing_value = missing_value
x_coord[:] = xloc[:]
y_coord[:] = yloc[:]
times[:] = t[:]
# lons[:] = lon_for_x[:]
# lats[:] = lat_for_y[:]
for i in range(grid.shape[2]):
lightning2d[i, :, :] = grid[:, :, i]
nc_out.close()
def test_normal(self):
# first, create some netCDF files to test with
# create two files, and assign some variables to each
# float timeseries
ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2],
[5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6],
[4, 3, 2], [5, 5, 4]])
ts_solar = numpy.array([[4, 3.3], [6, 3], [4, 4], [2, 1], [4, 3.3],
[4, 3], [4, numpy.nan], [2, 1]])
# integer timeseries
ts_time = numpy.array([[1, 2, 3, 4], [4, 5, 6, 7], [7, 6, 5, 4],
[1, 2, 3, 4], [4, 5, 6, 7], [7, 6, 5, 4],
[4, 5, 6, 7], [4, 5, 6, 7]])
ts_vector = numpy.array([1, 2, 3, 4, 5, 6, 7, 3])
# float other
float_1 = numpy.array([1.0, numpy.nan, 3.0, 4.0])
# integer other
int_1 = numpy.array([443, 45, 43, 23, 34])
f1 = nc.NetCDFFile('test_normal_1.nc', 'w')
f2 = nc.NetCDFFile('test_normal_2.nc', 'w')
# put ts_wind, ts_time and float_1 into file 1
f1.createDimension('wx', ts_wind.shape[0])
f1.createDimension('wy', ts_wind.shape[1])
f1.createDimension('tx', ts_time.shape[0])
f1.createDimension('ty', ts_time.shape[1])
f1.createDimension('fx', float_1.shape[0])
# put ts_solar, ts_vector and int_1 into file 2
f2.createDimension('sx', ts_solar.shape[0])
f2.createDimension('sy', ts_solar.shape[1])
f2.createDimension('vx', ts_vector.shape[0])
f2.createDimension('ix', int_1.shape[0])
ts_wind_var = f1.createVariable('ts_wind', 'float64', ('wx', 'wy'))
ts_time_var = f1.createVariable('ts_time', 'int32', ('tx', 'ty'))
float_1_var = f1.createVariable('float_1', 'float32', ('fx', ))
ts_solar_var = f2.createVariable('ts_solar', 'float32', ('sx', 'sy'))
ts_vector_var = f2.createVariable('ts_vector', 'float32', ('vx', ))
int_1_var = f2.createVariable('funny_int_1', 'int32', ('ix', ))
ts_wind_var[:, :] = ts_wind
ts_time_var[:, :] = ts_time
float_1_var[:] = float_1
ts_solar_var[:, :] = ts_solar
ts_vector_var[:] = ts_vector
int_1_var[:] = int_1
f1.close()
f2.close()
config = {
'description': 'test normal',
'model': 'data.ncdata.py',
'section': 'Data',
'ts_float_list': 'ts_wind ts_solar',
'ts_int_list': 'ts_time ts_vector',
'other_float_list': 'float_1',
'other_int_list': 'int_1',
'ts_wind_file': 'test_normal_1.nc',
'ts_solar_file': 'test_normal_2.nc',
'ts_time_file': 'test_normal_1.nc',
'ts_vector_file': 'test_normal_2.nc',
'float_1_file': 'test_normal_1.nc',
'int_1_file': 'test_normal_2.nc',
'int_1_vbl': 'funny_int_1'
}
try:
self.data.set_config(config)
results = {}
for series_name in [
'ts_wind', 'ts_solar', 'ts_time', 'ts_vector', 'float_1',
'int_1'
]:
results[series_name] = self.data.get_timeseries(series_name)
ts_len = self.data.get_ts_length()
except mureilexception.MureilException as me:
print me.msg
self.assertEqual(False, True)
exp_ts_sel = [0, 1, 2, 4, 5, 7]
exp_ts_wind = numpy.array(ts_wind[exp_ts_sel], dtype=float)
exp_ts_solar = numpy.array(ts_solar[exp_ts_sel], dtype=float)
exp_ts_time = ts_time[exp_ts_sel]
exp_ts_vector = ts_vector[exp_ts_sel]
exp_float_1 = numpy.array(float_1, dtype=float)
exp_int_1 = int_1
self.assertTrue(numpy.allclose(exp_ts_wind, results['ts_wind']))
self.assertTrue(numpy.allclose(exp_ts_solar, results['ts_solar']))
self.assertTrue(numpy.allclose(exp_ts_time, results['ts_time']))
self.assertTrue(numpy.allclose(exp_ts_vector, results['ts_vector']))
self.assertTrue(numpy.allclose(exp_int_1, results['int_1']))
# Can't just compare exp_float_1 because it has a nan in it. Instead,
# check that the nan is there, and that the rest of the array is as
# expected.
self.assertTrue(
numpy.all(
numpy.isnan(exp_float_1) == numpy.isnan(results['float_1'])))
self.assertTrue(
numpy.allclose(exp_float_1[[0, 2, 3]],
results['float_1'][[0, 2, 3]]))
# and check that all the floats are float64
self.assertTrue(results['ts_wind'].dtype.name == 'float64')
self.assertTrue(results['ts_solar'].dtype.name == 'float64')
self.assertTrue(results['float_1'].dtype.name == 'float64')
self.assertTrue((ts_len == 6))
def complete_configuration(self):
self.data = {}
for list_type in [
'ts_float_list', 'ts_int_list', 'other_float_list',
'other_int_list'
]:
for series_name in self.config[list_type]:
infile = self.config['dir'] + self.config[series_name +
'_file']
try:
f = nc.NetCDFFile(infile)
except:
msg = ('File ' + infile + ' for data series ' +
series_name + ' was not opened.')
raise mureilexception.ConfigException(msg, {})
try:
vbl = f.variables[self.config[series_name + '_vbl']]
except:
msg = ('Variable ' + self.config[series_name + '_vbl'] +
' not found in file ' + infile)
raise mureilexception.ConfigException(msg, {})
dims = len(vbl.shape)
if (dims == 1):
temp = vbl[:]
elif (dims == 2):
temp = vbl[:, :]
else:
msg = 'Data series ' + series_name + ' has more than 2 dimensions, so is not handled.'
raise mureilexception.ConfigException(msg, {})
if 'float' in list_type:
if not mureiltypes.check_ndarray_float(temp, True):
self.data[series_name] = numpy.array(temp, dtype=float)
else:
self.data[series_name] = temp
else:
if not mureiltypes.check_ndarray_int(temp, True):
self.data[series_name] = numpy.array(temp, dtype=int)
else:
self.data[series_name] = temp
for list_type in ['ts_csv_list']:
for series_name in self.config[list_type]:
infile = self.config['dir'] + self.config[series_name +
'_file']
temp = []
try:
with open(infile, 'rU') as n:
reader = csv.reader(n)
for row in reader:
if reader.line_num == 1:
self.data[series_name + '_hdr'] = row[1:]
else:
if reader.line_num == 2:
temp = [map(float, (row[1:]))]
else:
temp.append(map(float, (row[1:])))
self.data[series_name] = numpy.array(temp, dtype=float)
if not mureiltypes.check_ndarray_float(temp, True):
self.data[series_name] = numpy.array(temp, dtype=float)
else:
self.data[series_name] = temp
except:
msg = ('File ' + infile + ' for data series ' +
series_name +
' was not opened or had an error in reading.')
raise mureilexception.ConfigException(msg, {})
# Now apply the NaN filter to the ts lists, but note that the integer
# ones are not identified as nan.
all_ts = self.config['ts_float_list'] + self.config[
'ts_int_list'] + self.config['ts_csv_list']
if len(all_ts) == 0:
self.ts_length = 0
logger.warning('No timeseries data defined')
else:
self.ts_length = self.data[all_ts[0]].shape[0]
# Start with an array of 'False'
nan_acc = (numpy.ones(self.ts_length) == 0)
# Accumulate 'True' entries in nan_acc where NaN found in timeseries
for ts_name in all_ts:
ts_nan = numpy.isnan(self.data[ts_name])
if ts_nan.ndim > 1:
ts_nan = ts_nan.any(1)
# Check all the timeseries are the same length
if not (len(ts_nan) == self.ts_length):
msg = ('Data series ' + ts_name +
' is length {:d}, not matching {:d} of '.format(
len(ts_nan), self.ts_length) + all_ts[0])
raise mureilexception.ConfigException(msg, {})
nan_acc = numpy.logical_or(nan_acc, ts_nan)
# Clean up the timeseries using slices
nan_acc = numpy.logical_not(nan_acc)
for ts_name in all_ts:
if self.data[ts_name].ndim == 1:
self.data[ts_name] = self.data[ts_name][nan_acc]
else:
self.data[ts_name] = self.data[ts_name][nan_acc, :]
self.ts_length = self.data[all_ts[0]].shape[0]
self.is_configured = True
return None
def complete_configuration(self):
self.data = {}
wind_gap = '11'
solar_gap = '21'
dir = '/mnt/meteo0/data/dargaville/rhuva/'
file = 'RegGrid_wind_output_'+wind_gap+'point_gap.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_wind'][:,:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_wind'] = numpy.array(temp, dtype=float)
else:
self.data['ts_wind'] = temp
file = 'RegGrid_dsr_output_'+solar_gap+'point_gap.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_solar'][:,:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_solar'] = numpy.array(temp, dtype=float)
else:
self.data['ts_solar'] = temp
file = 'Aus_demand_2010-2011.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_demand'][:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_demand'] = numpy.array(temp, dtype=float)
else:
self.data['ts_demand'] = temp
wind_nan = numpy.isnan(self.data['ts_wind'])
solar_nan = numpy.isnan(self.data['ts_solar'])
demand_nan = numpy.isnan(self.data['ts_demand'])
wind_row = wind_nan.any(1)
solar_row = solar_nan.any(1)
combo = numpy.array([wind_row, solar_row, demand_nan])
combo_flat = combo.any(0)
self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :]
self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :]
self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False]
print self.data['ts_wind'].shape
print self.data['ts_solar'].shape
print self.data['ts_demand'].shape
self.ts_length = self.data['ts_wind'].shape[0]
file = 'Dist-to-nearest-cap_wind_'+wind_gap+'point_gap.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_wind_distances'][:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_wind_distances'] = numpy.array(temp, dtype=float)
else:
self.data['ts_wind_distances'] = temp
file = 'Dist-to-nearest-cap_dsr_'+solar_gap+'point_gap.nc'
infile = dir + file
f = nc.NetCDFFile(infile)
temp = f.variables['ts_solar_distances'][:]
if not mureiltypes.check_ndarray_float(temp, True):
self.data['ts_solar_distances'] = numpy.array(temp, dtype=float)
else:
self.data['ts_solar_distances'] = temp
return None
def make_plot(filename,
grid_name,
x_name='x',
y_name='y',
t_name='time',
n_cols=6,
outpath='',
filename_prefix='LMA',
do_save=True,
image_type='pdf',
colormap='gist_earth'):
""" colormap: a string giving the name of a matplotlib built-in colormap,
or a matplotlib.colors.Colormap instance
"""
f = nc.NetCDFFile(filename)
data = f.variables # dictionary of variable names to nc_var objects
dims = f.dimensions # dictionary of dimension names to sizes
x = data[x_name]
y = data[y_name]
t = data[t_name]
grid = data[grid_name]
assert len(x.shape) == 1
assert len(y.shape) == 1
assert len(t.shape) == 1
grid_dims = grid.dimensions # tuple of dimension names
name_to_idx = dict((k, i) for i, k in enumerate(grid_dims))
grid_t_idx = name_to_idx[t.dimensions[0]]
grid_x_idx = name_to_idx[x.dimensions[0]]
grid_y_idx = name_to_idx[y.dimensions[0]]
n_frames = t.shape[0]
# n_cols = 6
n_rows = int(ceil(float(n_frames) / n_cols))
if type(colormap) == type(''):
colormap = get_cmap(colormap)
grey_color = (0.5, ) * 3
frame_color = (0.2, ) * 3
density_maxes = []
total_counts = []
all_t = []
xedge = centers_to_edges(x)
x_range = xedge.max() - xedge.min()
yedge = centers_to_edges(y)
y_range = yedge.max() - yedge.min()
dx = (xedge[1] - xedge[0])
# w, h = figaspect(float(n_rows)/n_cols) # breaks for large numbers of frames - has a hard-coded max figure size
w, h, n_rows_perpage, n_pages = multiples_figaspect(n_rows,
n_cols,
x_range,
y_range,
fig_width=8.5,
max_height=None)
# count_scale_factor = dx # / 1000.0
# max_count_baseline = 450 * count_scale_factor #/ 10.0
min_count, max_count = 1, grid[:].max() #max_count_baseline*(t[1]-t[0])
if (max_count == 0) | (max_count == 1):
max_count = min_count + 1
f.close()
default_vmin = -0.2
if np.log10(max_count) <= default_vmin:
vmin_count = np.log10(max_count) + default_vmin
else:
vmin_count = default_vmin
fig = Figure(figsize=(w, h))
canvas = FigureCanvasAgg(fig)
fig.set_canvas(canvas)
p = small_multiples_plot(fig=fig, rows=n_rows, columns=n_cols)
p.label_edges(True)
pad = 0.0 # for time labels in each frame
for ax in p.multiples.flat:
ax.set_axis_bgcolor('black')
ax.spines['top'].set_edgecolor(frame_color)
ax.spines['bottom'].set_edgecolor(frame_color)
ax.spines['left'].set_edgecolor(frame_color)
ax.spines['right'].set_edgecolor(frame_color)
# ax.yaxis.set_major_formatter(kilo_formatter)
# ax.xaxis.set_major_formatter(kilo_formatter)
base_date = datetime.strptime(t.units, "seconds since %Y-%m-%d %H:%M:%S")
time_delta = timedelta(0, float(t[0]), 0)
start_time = base_date + time_delta
indexer = [
slice(None),
] * len(grid.shape)
frame_start_times = []
for i in range(n_frames):
frame_start = base_date + timedelta(0, float(t[i]), 0)
frame_start_times.append(frame_start)
indexer[grid_t_idx] = i
density = grid[indexer]
# density,edges = np.histogramdd((x,y), bins=(xedge,yedge))
density_plot = p.multiples.flat[i].pcolormesh(xedge,
yedge,
np.log10(
density.transpose()),
vmin=vmin_count,
vmax=np.log10(max_count),
cmap=colormap)
label_string = frame_start.strftime('%H%M:%S')
text_label = p.multiples.flat[i].text(xedge[0] - pad + x_range * .015,
yedge[0] - pad + y_range * .015,
label_string,
color=grey_color,
size=6)
density_plot.set_rasterized(True)
density_maxes.append(density.max())
total_counts.append(density.sum())
all_t.append(frame_start)
print label_string, x.shape, density.max(), density.sum()
color_scale = ColorbarBase(p.colorbar_ax,
cmap=density_plot.cmap,
norm=density_plot.norm,
orientation='horizontal')
# color_scale.set_label('count per pixel')
color_scale.set_label('log10(count per pixel)')
view_x = (xedge.min(), xedge.max())
view_y = (yedge.min(), yedge.max())
print 'making multiples',
p.multiples.flat[0].axis(view_x + view_y)
filename = '%s-%s_%s_%05.2fkm_%05.1fs.%s' % (
filename_prefix, grid_name, start_time.strftime('%Y%m%d_%H%M%S'), dx,
time_delta.seconds, image_type)
filename = os.path.join(outpath, filename)
if do_save:
fig.savefig(filename, dpi=150)
return fig, p, frame_start_times, filename
print ' ... done'
def write_cf_netcdf(outfile,
t_start,
t,
xloc,
yloc,
lon_for_x,
lat_for_y,
ctr_lat,
ctr_lon,
grid,
grid_var_name,
grid_description,
format='i',
**kwargs):
""" Write a Climate and Forecast Metadata-compliant NetCDF file.
Should display natively in conformant packages like McIDAS-V.
"""
import pupynere as nc
missing_value = -9999
nc_out = nc.NetCDFFile(outfile, 'w')
nc_out.createDimension('nx', xloc.shape[0])
nc_out.createDimension('ny', yloc.shape[0])
nc_out.createDimension('ntimes', t.shape[0]) #unlimited==None
proj = nc_out.createVariable('Lambert_Azimuthal_Equal_Area', 'i', ())
proj.grid_mapping_name = 'lambert_azimuthal_equal_area'
proj.longitude_of_projection_origin = ctr_lon
proj.latitude_of_projection_origin = ctr_lat
proj.false_easting = 0.0
proj.false_northing = 0.0
# x_coord = nc_out.createVariable('longitude', 'f', ('nx',))
# x_coord.long_name="longitude"
# x_coord.standard_name="longitude"
# x_coord.units = "degrees_east"
x_coord = nc_out.createVariable('x', 'f', ('nx', ))
x_coord.units = "km"
x_coord.long_name = "x coordinate of projection"
x_coord.standard_name = 'projection_x_coordinate'
# y_coord = nc_out.createVariable('latitude', 'f', ('nx',))
# y_coord.long_name="latitude"
# y_coord.standard_name="latitude"
# y_coord.units = "degrees_north"
y_coord = nc_out.createVariable('y', 'f', ('ny', ))
y_coord.units = "km"
y_coord.long_name = "y coordinate of projection"
y_coord.standard_name = 'projection_y_coordinate'
times = nc_out.createVariable('time', 'f',
('ntimes', )) #, filters=no_compress)
times.long_name = "time"
times.units = "seconds since %s" % t_start.strftime('%Y-%m-%d %H:%M:%S')
lons = nc_out.createVariable('lons', 'd',
('nx', 'ny')) #, filters=no_compress)
lons.long_name = "longitude"
lons.standard_name = "longitude"
lons.units = "degrees_east"
lats = nc_out.createVariable('lats', 'd',
('nx', 'ny')) #, filters=no_compress)
lats.long_name = "latitude"
lats.standard_name = "latitude"
lats.units = "degrees_north"
lightning2d = nc_out.createVariable(
grid_var_name, format, ('ntimes', 'nx', 'ny')) #, filters=no_compress)
lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)'
lightning2d.units = 'dimensionless'
lightning2d.coordinates = 'time lons lats'
lightning2d.grid_mapping = "Lambert_Azimuthal_Equal_Area"
lightning2d.missing_value = missing_value
x_coord[:] = xloc[:]
y_coord[:] = yloc[:]
times[:] = t[:]
lons[:] = lon_for_x[:]
lats[:] = lat_for_y[:]
for i in range(grid.shape[2]):
lightning2d[i, :, :] = grid[:, :, i]
nc_out.close()
def write_AWIPS_netcdf_grid(outfile,
t_start,
t,
xloc,
yloc,
lon_for_x,
lat_for_y,
ctr_lat,
ctr_lon,
grid,
grid_var_name,
grid_description,
format='i',
refresh_minutes=1,
grid_units='dimensionless'):
""" Write an AWIPS-formatted NetCDF grid for lightning data.
TYPEMAP = { NC_BYTE: ('b', 1),
NC_CHAR: ('c', 1),
NC_SHORT: ('h', 2),
NC_INT: ('i', 4), # this is also long
NC_FLOAT: ('f', 4),
NC_DOUBLE: ('d', 8) }
"""
missing_value = -9999
# all_levels = (1,)
level_name = "SFC"
# ----------
# Dimensions
# ----------
nc_out = nc.NetCDFFile(outfile, 'w')
nc_out.createDimension('record', None) #unlimited==None
nc_out.createDimension('n_valtimes', t.shape[0])
nc_out.createDimension('data_variables', 1)
nc_out.createDimension('namelen', 132)
nc_out.createDimension('charsPerLevel', 20)
nc_out.createDimension('levels', 1)
# for level in all_levels:
# nc_out.createDimension('levels_{0}'.format(level), level)
nc_out.createDimension('x', xloc.shape[0])
nc_out.createDimension('y', yloc.shape[0])
# -----------------
# Global attributes
# -----------------
nc_out.Data_record_time = t_start.strftime('%Y%m%d.%H%M')
nc_out.depictorName = "WTLMA_ltng_grid"
nc_out.projIndex = 8
nc_out.projName = "CYLINDRICAL_EQUIDISTANT"
nc_out.centralLon = ctr_lon
nc_out.centralLat = ctr_lat
nc_out.xMin = lon_for_x.min()
nc_out.xMax = lon_for_x.max()
nc_out.yMin = lat_for_y.min()
nc_out.yMax = lat_for_y.max()
nc_out.lat00 = nc_out.yMin
nc_out.lon00 = nc_out.xMin
nc_out.latNxNy = nc_out.yMax
nc_out.lonNxNy = nc_out.xMax
nc_out.dxKm = nc_out.xMin
nc_out.dyKm = nc_out.yMax
nc_out.latDxDy = (nc_out.yMin + nc_out.yMax) / 2.0
nc_out.lonDxDy = (nc_out.xMin + nc_out.xMax) / 2.0
nc_out.UnitSquareSideSize = xloc[1] - xloc[0]
nc_out.RefreshPeriod = 60 * int(refresh_minutes)
# nc_out.NbRefreshPeriods = int(10*60/nc_out.RefreshPeriod)
charsPerLevel = nc_out.createVariable(grid_var_name + 'Levels', 'c',
('levels', 'charsPerLevel'))
inventory = nc_out.createVariable(grid_var_name + 'Inventory', 'c',
('n_valtimes', 'charsPerLevel'))
charsPerLevel[0, 0:len(level_name)] = level_name
for i in range(nc_out.dimensions['n_valtimes']):
inventory[i, :] = '1' * nc_out.dimensions[
'charsPerLevel'] # What is this? Why AWIPS, Why?
the_epoch = datetime.datetime(1970, 1, 1, 0, 0, 0)
reftime_since_the_epoch = int((t_start - the_epoch).total_seconds())
valtimeref_delta = nc_out.createVariable('valtimeMINUSreftime', 'i',
('n_valtimes', ))
valtime = nc_out.createVariable('valtimeMINUSreftime', 'i',
('n_valtimes', ))
reftime = nc_out.createVariable('valtimeMINUSreftime', 'i', ())
reftime.long_name = "reference time"
reftime.units = "seconds since (1970-1-1 00:00:00.0)"
reftime.assignValue(reftime_since_the_epoch)
valtime.long_name = "valid time"
valtime.units = "seconds since (1970-1-1 00:00:00.0)"
valtime[:] = t + reftime_since_the_epoch
valtimeref_delta[:] = t
lightning2d = nc_out.createVariable(
grid_var_name, format,
('record', 'levels', 'y', 'x')) #, filters=no_compress)
lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)'
lightning2d.units = grid_units
lightning2d.uiname = grid_var_name
lightning2d._FillValue = missing_value
lightning2d._n3D = 1
lightning2d.levels = level_name
# Loop over times
for i in range(grid.shape[2]):
lightning2d[i, 0, :, :] = grid[:, :, i].T
nc_out.close()
