def complete_configuration(self): self.data = {} dir = './' #dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/' file = 'CoV_wind_station_output_prox_penalty.nc' #file with _II has smaller exclusion zone infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['CoV_wind'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'CoV_dsr_station_output_prox_penalty.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['CoV_dsr'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp file = 'Aus_demand_sample_raw.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] return None
def Dataset(ncfile): if isinstance(ncfile, str): return pupynere.NetCDFFile(ncfile) elif isinstance(ncfile, pupynere.NetCDFFile): return ncfile else: raise TypeError, 'type %s not supported' % type(ncfile)
def createNetCDFFile(cfg, direct): path = '%s/%s.%02d.nc' % (direct, cfg['ident'], cfg['month']) os.system('/bin/rm -f %s' % path) try: fh = pupynere.NetCDFFile(path, 'w', time.ctime()) except IOError: msg = 'Cannot create file %s' % path print msg if _Logger : _Logger.error(msg) return None fh.title = 'Some hopefully less useless junk' fh.ident = cfg['ident'] fh.month = cfg['month'] fh.createDimension('dd', len(cfg['dd'])+1) # cycle, 0 is calm/vrb fh.createDimension('ff', len(cfg['ff'])+1) fh.createDimension('cig', len(cfg['cig'])+1) fh.createDimension('vsby', len(cfg['vsby'])+1) fh.createDimension('obv', 8) fh.createDimension('int', 4) # check this fh.createDimension('pcp', 4) fh.createDimension('total', 1) for var in fh.dimensions: v = fh.createVariable(var, 'i', (var,)) v[:] = 0 all = fh.createVariable('all', 'i', ('dd', 'ff', 'obv', 'int', 'pcp', 'cig', 'vsby')) all[:] = 0 for v in fh.variables: fh.variables[v].units = 'count' return fh
def test_variable_missing(self): # first, create a netCDF files to test with # float timeseries ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, 4]]) f1 = nc.NetCDFFile('test_missing_1.nc', 'w') # put ts_wind, ts_time and float_1 into file 1 f1.createDimension('wx', ts_wind.shape[0]) f1.createDimension('wy', ts_wind.shape[1]) ts_wind_var = f1.createVariable('ts_wind_funny', 'float64', ('wx', 'wy')) ts_wind_var[:, :] = ts_wind f1.close() config = { 'description': 'test variable missing', 'model': 'data.ncdata.py', 'section': 'Data', 'ts_float_list': 'ts_wind', 'ts_wind_file': 'test_missing_1.nc' } self.assertRaises(mureilexception.ConfigException, self.data.set_config, config)
def _climate(ident, periods, dataDir): result = {} try: month = time.gmtime(periods[0][1]['time']['from']).tm_mon path = os.path.join(dataDir, '%s.%02d.nc' % (ident, month)) if not os.path.isfile(path): raise Avn.AvnError('File %s does not exist' % path) ncfh = pupynere.NetCDFFile(path) cfg = AvnParser.getClimQCConfig(ident) except IndexError: return result except IOError: raise IOError, 'Cannot access %s' % path for ix, p in periods: off = _getOffset(cfg, p) a0, text, off = _analyze(ncfh, off, cfg['showdetails']) level = _getLevel(a0, cfg['alpha']) if a0 < cfg['alpha']: suggestions = [_analyze(ncfh, toff, cfg['showdetails']) \ for toff in _neighbors(ncfh, off)] suggestions.sort() suggestions.reverse() best = filter(None, [_suggest(off, x[2]) for x in suggestions \ if x[0] >= cfg['alpha']]) if best: text.extend(best) else: text.append('Cannot figure it out') result[ix] = {'level': level, 'text': '\n'.join(text)} ncfh.close() return result
def data_for_time(self, t0): """ Read data from the file corresponding to datetime t. Returns xedge, yedge, and density for the file """ fname, i = self._time_lookup[ t0] #i is the frame id for this time in NetCDFFile f f = nc.NetCDFFile(fname) data = f.variables # dictionary of variable names to nc_var objects dims = f.dimensions # dictionary of dimension names to sizes x = data[self.x_name] y = data[self.y_name] t = data[self.t_name] grid = data[self.grid_name] indexer = [ slice(None), ] * len(grid.shape) grid_dims = grid.dimensions # tuple of dimension names name_to_idx = dict((k, i) for i, k in enumerate(grid_dims)) grid_t_idx = name_to_idx[t.dimensions[0]] grid_x_idx = name_to_idx[x.dimensions[0]] grid_y_idx = name_to_idx[y.dimensions[0]] xedge = centers_to_edges(x) yedge = centers_to_edges(y) indexer[grid_t_idx] = i density = grid[indexer].transpose() f.close() return xedge, yedge, density
def complete_configuration(self): dir = './' file = 'CoV_output.nc' infile = dir + file f = nc.NetCDFFile(infile) self.data = {} self.data['ts_wind'] = numpy.array(f.variables['ts_wind'][:, :], dtype=float) self.data['ts_solar'] = numpy.array(f.variables['ts_solar'][:, :], dtype=float) self.data['ts_demand'] = numpy.array(f.variables['ts_demand'][:], dtype=float) self.ts_length = self.data['ts_wind'].shape[0] self.is_configured = True return None
def getFile(self, path): try: self._path = path self._fh = pupynere.NetCDFFile(self._path, 'r') self.issuetime = self._fh.variables['time'].getValue() self._validtimes = \ self._fh.variables['validTimeList'][:self.NumData].tolist() var = self._fh.variables['stationName'] self._sitedict = {} for n in xrange(var.shape[0]): ident = var[n, :].tostring().split('\x00')[0].rstrip() self._sitedict[ident] = n return True except IOError: _Logger.error('Error accessing %s', path) return False
def _frame_times_for_file(self, fname): """ Called once by init to set up frame lookup tables and yield the frame start times. _frame_lookup goes from datetime->(nc file, frame index)""" f = nc.NetCDFFile(fname) data = f.variables # dictionary of variable names to nc_var objects dims = f.dimensions # dictionary of dimension names to sizes t = data[self.t_name] base_date = datetime.strptime(t.units, "seconds since %Y-%m-%d %H:%M:%S") for i in range(t.shape[0]): frame_start = base_date + timedelta(0, float(t[i]), 0) self._time_lookup[frame_start] = (fname, i) yield frame_start f.close()
def test_different_lengths(self): # first, create a netCDF files to test with # float timeseries ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, 4]]) ts_solar = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, 4], [5, 5, 4], [5, 5, 4]]) f1 = nc.NetCDFFile('test_different_lengths.nc', 'w') f1.createDimension('wx', ts_wind.shape[0]) f1.createDimension('wy', ts_wind.shape[1]) f1.createDimension('sx', ts_solar.shape[0]) f1.createDimension('sy', ts_solar.shape[1]) ts_wind_var = f1.createVariable('ts_wind', 'float64', ('wx', 'wy')) ts_wind_var[:, :] = ts_wind ts_solar_var = f1.createVariable('ts_solar', 'float64', ('sx', 'sy')) ts_solar_var[:, :] = ts_solar f1.close() config = { 'description': 'test different length ts', 'model': 'data.ncdata.py', 'section': 'Data', 'ts_float_list': 'ts_wind ts_solar', 'ts_wind_file': 'test_different_lengths.nc', 'ts_solar_file': 'test_different_lengths.nc' } self.assertRaises(mureilexception.ConfigException, self.data.set_config, config)
def getFile(self, path): try: self._path = path self._fh = pupynere.NetCDFFile(self._path, 'r') var = self._fh.variables['staName'] # get record numbers for all idents self._sitedict = {} for n, s in enumerate(var[:]): ident = s.tostring().split('\x00')[0].rstrip() try: self._sitedict[ident].append(n) except KeyError: self._sitedict[ident] = [n] # sort wrt valid time for i in self._sitedict: v = self._fh.variables['validTime'] tmp = [(int(v[n]), n) for n in self._sitedict[i]] tmp.sort() self._sitedict[i] = [t[1] for t in tmp][:self.NumData + 2] return True except IOError: _Logger.error('Error accessing %s', path) return False
def complete_configuration(self): self.data = {} dir = '/export/karoly2/Roger/ACCESS-A/' file = 'RegGrid_wind_output_11point_gap.nc' #file with _II has smaller exclusion zone infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['wind_power'][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'RegGrid_dsr_output_19point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['dsr'][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/' file = 'Aus_demand_2010_2011.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] self.data['ts_solar_distances'] = numpy.array([ 668730.4375, 909338.1875, 372287.25, 1125613.25, 211878.890625, 68551.4609375, 768108.3125, 513450.8125, 1239141.625, 242710.109375, 221302.75, 835822.375, 641565.125, 1371163.875, 404052.09375, 449818.1875, 949940.75, 1176451.625, 1239225.875, 811838.8125, 610120.125, 679184.1875, 1126859.125, 926546., 987652.1875, 1093136., 923490.5625, 605613.3125, 681002.75, 1170811.375, 1214910.625, 881211.1875, 1177105.25, 1020829.875, 374478.46875, 884468.125, 1147496.5, 1412441., 557196.4375, 111162.4375, 1129901.25, 901584.5625, 207429.328125, 830748.625, 1097925., 1311133.125, 474638.53125, 231650.921875, 1030239.5625, 784340.375, 599865.75, 355621.71875, 801379.6875, 1050643., 1298696.375, 535831.75, 1012697.625, 282052., 738578.625, 468747.75, 587530.1875, 874383.75, 1085080.5, 1165477.5, 739965.6875, 1043011.9375, 133451.609375, 466417.03125, 887445.1875, 843954.375, 882972.9375, 911230.0625, 256325.21875, 196988.21875, 722306.0625, 494064.375, 656777.1875, 590096.75, 838475., 576432.125, 1038951.0625, 709157.1875, 329522.5625, 246577.59375, 866532.3125, 684706.0625, 507050.0625, 353760.5, 528336.75, 308515.5, 294442.8125, 38012.9140625, 354666.53125, 473246.90625, 815155.4375, 729953.8125, 211116.015625, 548792.25, 134444.53125, 367146.65625, 252677.890625, 404498.71875, 615562.1875, 426158.125, 478903.125, 478492.6875, 204912.421875, 186742.734375, 449120.125, 467410.25, 242252.140625, 325181.5625, 223081.09375, 109695.5546875, 159941.5 ], dtype=float) self.data['ts_wind_distances'] = numpy.array([ 668730.4375, 909338.1875, 1058640.875, 516586.90625, 754774.625, 909874.625, 372287.25, 1125613.25, 212450.59375, 211878.890625, 611953.6875, 68551.4609375, 768108.3125, 938604.9375, 475540.34375, 341444.6875, 1206778.375, 1072823.25, 209707.75, 161579.765625, 637232.6875, 111058.5234375, 353789.75, 811775.375, 946141.8125, 513450.8125, 383905.8125, 1239141.625, 1106075.25, 274558., 157525.8125, 242710.109375, 694877.25, 221302.75, 387798.59375, 835822.375, 992085.4375, 585522.375, 465096.75, 1306448.625, 1127716.375, 376003.46875, 262230.59375, 309341.46875, 734677.625, 341611.15625, 897978.625, 1027317.9375, 641565.125, 533193.8125, 1371163.875, 1242591.625, 464625.5, 374925.21875, 404052.09375, 811223.6875, 449818.1875, 949940.75, 1152356.75, 731487.6875, 632564.3125, 1328731.375, 1318661.25, 575718.375, 489012.5, 503957.4375, 877959.625, 567724., 1176451.625, 1085111.125, 1239225.875, 811838.8125, 723001.8125, 1228210., 1317506.875, 1061301.5, 678183., 604640.75, 610120.125, 1019876.0625, 679184.1875, 1064924.875, 1183351.125, 957665., 869637.5625, 1125113.625, 1221260.5, 1323786.875, 816667.5625, 719099.3125, 722332.8125, 908779.125, 1126859.125, 812075.8125, 926546., 1267433.125, 1071478.125, 987652.1875, 992131.5625, 1093136., 1200561.75, 944202.625, 841807.75, 831524.5625, 766421.25, 923490.5625, 797534.4375, 1268186.625, 1294879.625, 1179975.25, 1100428.25, 605613.3125, 871203.0625, 978584.5625, 1091826.375, 1060980.625, 956020.0625, 637515.875, 1057804.5, 681002.75, 1170811.375, 1295239.375, 1214910.625, 457551.96875, 766078.25, 881211.1875, 1001023.3125, 1178623., 1083117.625, 1063405.75, 511700.875, 1177105.25, 562428.6875, 660418.0625, 783159.8125, 1081792.375, 1020829.875, 374478.46875, 1181675.5, 444261.125, 884468.125, 1015654., 1147496.5, 1279800., 1412441., 557196.4375, 686802.875, 1046372.1875, 969600.625, 224758.90625, 1270513., 111162.4375, 1129901.25, 321961.25, 829833.5625, 963491.6875, 1097242.375, 1231062., 1364937.25, 450944.96875, 584710.375, 989115.9375, 901584.5625, 60845.32421875, 1226836.625, 130785.546875, 1098943.5, 207429.328125, 830748.625, 964274.5, 1097925., 1231665.75, 1311133.125, 345572.96875, 474638.53125, 605817.375, 953379.625, 853869.9375, 709148.6875, 144821.765625, 1161816.75, 231650.921875, 1030239.5625, 281000.03125, 757300.375, 887036.6875, 1017876.0625, 1149449.375, 1281537.875, 405575.3125, 519712.8125, 641537.125, 784340.375, 599865.75, 263454.5625, 1193936., 1058048.625, 938030.5, 648269.0625, 355621.71875, 801379.6875, 924709.125, 1050643., 1178353.875, 1298696.375, 448873.90625, 535831.75, 639195.5625, 778615.625, 486159.1875, 1150266.875, 432368.78125, 1012697.625, 887173.875, 635939.25, 370392.96875, 475697.8125, 802408.9375, 914610.6875, 1042862.375, 1164123.875, 1287900.625, 551048.75, 625359.1875, 713181.125, 282052., 738578.625, 468747.75, 1176506., 1038450.75, 907944.3125, 594211.4375, 331333.875, 587530.1875, 874383.75, 976254.0625, 1085080.5, 1127779.875, 1165477.5, 739965.6875, 196089.296875, 762503.6875, 446178.5625, 1156598.25, 1043011.9375, 608477.6875, 302244., 891794.3125, 974806.9375, 971028.9375, 981571., 1009794.625, 133451.609375, 114606.5859375, 466417.03125, 818209.125, 1002819.125, 1039978.75, 683042.9375, 329406.875, 887445.1875, 855617.0625, 843954.375, 853292.25, 882972.9375, 1000176.8125, 196278.171875, 64264.375, 88714.125, 552421.75, 869001.5625, 870218., 911230.0625, 429908.59375, 256325.21875, 740118.0625, 718609.1875, 721628.5625, 748889.0625, 978856.625, 310406.625, 196988.21875, 722306.0625, 161488.421875, 599724.625, 934020.25, 737569.25, 814771.6875, 494064.375, 251381.703125, 336539., 656777.1875, 609839.375, 590096.75, 600250.25, 893760.0625, 391393.0625, 297513.59375, 823148.3125, 838475., 704558.125, 686599.5625, 948955.6875, 964129.25, 294521.5625, 576432.125, 645428.0625, 1038951.0625, 709157.1875, 594892.125, 329522.5625, 258209.796875, 246577.59375, 537336.3125, 483168.28125, 462575.5625, 798845.6875, 503718.4375, 418395.09375, 866532.3125, 777922.0625, 670250.3125, 762942.75, 881406.8125, 884783.375, 413056.8125, 413987.5, 473308.65625, 535773.375, 684706.0625, 301884.875, 191727.3125, 135755.625, 507050.0625, 416187.03125, 353760.5, 629375.8125, 604659.375, 528336.75, 923294.9375, 236056.421875, 848792.3125, 723014.9375, 432805.5, 588751.75, 266921.25, 308515.5, 371523.1875, 745499.75, 294442.8125, 161392.625, 38012.9140625, 354666.53125, 442686.21875, 330582.15625, 241744.234375, 473246.90625, 653481.125, 585862., 815155.4375, 771210.125, 489359.5625, 579454.375, 453331.5, 139264.03125, 201134.484375, 298521.34375, 729953.8125, 327048.5625, 199195.953125, 91859.6015625, 115152.90625, 211116.015625, 421740.375, 632862.875, 548792.25, 697630.4375, 659870.3125, 412630.8125, 548571.125, 439476.25, 543691., 11564.77832031, 134444.53125, 268900.28125, 598547.3125, 367146.65625, 358178.9375, 252677.890625, 184504.65625, 404498.71875, 600163.5, 535114.375, 615562.1875, 543468.25, 426158.125, 545614.5625, 401627.4375, 567999.5625, 136595.25, 255334.890625, 478903.125, 288917.75, 254081.71875, 285744.96875, 301214.53125, 381815.09375, 484310.0625, 478492.6875, 505826.21875, 425950.34375, 526906.8125, 336393.5625, 360000.46875, 204912.421875, 142902.09375, 186742.734375, 292174.5625, 375563.59375, 433015.375, 449120.125, 467410.25, 341196.125, 534745.5625, 289708.96875, 242252.140625, 125803.328125, 32823.0625, 242620.25, 325181.5625, 379624.03125, 253196.859375, 344051.40625, 129340.453125, 223081.09375, 129551.390625, 239891.03125, 261754.0625, 356155.5, 109695.5546875, 32649.04492188, 159941.5 ], dtype=float) return None
def write_cf_netcdf_latlon(outfile, t_start, t, xloc, yloc, lon_for_x, lat_for_y, ctr_lat, ctr_lon, grid, grid_var_name, grid_description, format='i', **kwargs): """ Write a Climate and Forecast Metadata-compliant NetCDF file. Grid is regular in lon, lat and so no map projection information is necessary. Should display natively in conformant packages like McIDAS-V. """ import pupynere as nc missing_value = -9999 nc_out = nc.NetCDFFile(outfile, 'w') nc_out.createDimension('lon', xloc.shape[0]) nc_out.createDimension('lat', yloc.shape[0]) nc_out.createDimension('ntimes', t.shape[0]) #unlimited==None # declare the coordinate reference system, WGS84 values proj = nc_out.createVariable('crs', 'i', ()) proj.grid_mapping_name = 'latitude_longitude' proj.longitude_of_prime_meridian = 0.0 proj.semi_major_axis = 6378137.0 proj.inverse_flattening = 298.257223563 y_coord = nc_out.createVariable('latitude', 'f', ('lat', )) y_coord.units = "degrees_north" y_coord.long_name = "latitude" y_coord.standard_name = 'latitude' x_coord = nc_out.createVariable('longitude', 'f', ('lon', )) x_coord.units = "degrees_east" x_coord.long_name = "longitude" x_coord.standard_name = 'longitude' times = nc_out.createVariable('time', 'f', ('ntimes', )) #, filters=no_compress) times.long_name = "time" times.units = "seconds since %s" % t_start.strftime('%Y-%m-%d %H:%M:%S') # lons = nc_out.createVariable('lons', 'd', ('nx','ny') )#, filters=no_compress) # lons.long_name="longitude" # lons.standard_name="longitude" # lons.units = "degrees_east" # # lats = nc_out.createVariable('lats', 'd', ('nx','ny') )#, filters=no_compress) # lats.long_name="latitude" # lats.standard_name="latitude" # lats.units = "degrees_north" lightning2d = nc_out.createVariable( grid_var_name, format, ('ntimes', 'lon', 'lat')) #, filters=no_compress) lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)' lightning2d.units = kwargs['grid_units'] # lightning2d.coordinates='time lons lats' lightning2d.grid_mapping = "crs" lightning2d.missing_value = missing_value x_coord[:] = xloc[:] y_coord[:] = yloc[:] times[:] = t[:] # lons[:] = lon_for_x[:] # lats[:] = lat_for_y[:] for i in range(grid.shape[2]): lightning2d[i, :, :] = grid[:, :, i] nc_out.close()
def test_normal(self): # first, create some netCDF files to test with # create two files, and assign some variables to each # float timeseries ts_wind = numpy.array([[1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, numpy.nan], [1, 2, 3.4], [4, 5, 6], [4, 3, 2], [5, 5, 4]]) ts_solar = numpy.array([[4, 3.3], [6, 3], [4, 4], [2, 1], [4, 3.3], [4, 3], [4, numpy.nan], [2, 1]]) # integer timeseries ts_time = numpy.array([[1, 2, 3, 4], [4, 5, 6, 7], [7, 6, 5, 4], [1, 2, 3, 4], [4, 5, 6, 7], [7, 6, 5, 4], [4, 5, 6, 7], [4, 5, 6, 7]]) ts_vector = numpy.array([1, 2, 3, 4, 5, 6, 7, 3]) # float other float_1 = numpy.array([1.0, numpy.nan, 3.0, 4.0]) # integer other int_1 = numpy.array([443, 45, 43, 23, 34]) f1 = nc.NetCDFFile('test_normal_1.nc', 'w') f2 = nc.NetCDFFile('test_normal_2.nc', 'w') # put ts_wind, ts_time and float_1 into file 1 f1.createDimension('wx', ts_wind.shape[0]) f1.createDimension('wy', ts_wind.shape[1]) f1.createDimension('tx', ts_time.shape[0]) f1.createDimension('ty', ts_time.shape[1]) f1.createDimension('fx', float_1.shape[0]) # put ts_solar, ts_vector and int_1 into file 2 f2.createDimension('sx', ts_solar.shape[0]) f2.createDimension('sy', ts_solar.shape[1]) f2.createDimension('vx', ts_vector.shape[0]) f2.createDimension('ix', int_1.shape[0]) ts_wind_var = f1.createVariable('ts_wind', 'float64', ('wx', 'wy')) ts_time_var = f1.createVariable('ts_time', 'int32', ('tx', 'ty')) float_1_var = f1.createVariable('float_1', 'float32', ('fx', )) ts_solar_var = f2.createVariable('ts_solar', 'float32', ('sx', 'sy')) ts_vector_var = f2.createVariable('ts_vector', 'float32', ('vx', )) int_1_var = f2.createVariable('funny_int_1', 'int32', ('ix', )) ts_wind_var[:, :] = ts_wind ts_time_var[:, :] = ts_time float_1_var[:] = float_1 ts_solar_var[:, :] = ts_solar ts_vector_var[:] = ts_vector int_1_var[:] = int_1 f1.close() f2.close() config = { 'description': 'test normal', 'model': 'data.ncdata.py', 'section': 'Data', 'ts_float_list': 'ts_wind ts_solar', 'ts_int_list': 'ts_time ts_vector', 'other_float_list': 'float_1', 'other_int_list': 'int_1', 'ts_wind_file': 'test_normal_1.nc', 'ts_solar_file': 'test_normal_2.nc', 'ts_time_file': 'test_normal_1.nc', 'ts_vector_file': 'test_normal_2.nc', 'float_1_file': 'test_normal_1.nc', 'int_1_file': 'test_normal_2.nc', 'int_1_vbl': 'funny_int_1' } try: self.data.set_config(config) results = {} for series_name in [ 'ts_wind', 'ts_solar', 'ts_time', 'ts_vector', 'float_1', 'int_1' ]: results[series_name] = self.data.get_timeseries(series_name) ts_len = self.data.get_ts_length() except mureilexception.MureilException as me: print me.msg self.assertEqual(False, True) exp_ts_sel = [0, 1, 2, 4, 5, 7] exp_ts_wind = numpy.array(ts_wind[exp_ts_sel], dtype=float) exp_ts_solar = numpy.array(ts_solar[exp_ts_sel], dtype=float) exp_ts_time = ts_time[exp_ts_sel] exp_ts_vector = ts_vector[exp_ts_sel] exp_float_1 = numpy.array(float_1, dtype=float) exp_int_1 = int_1 self.assertTrue(numpy.allclose(exp_ts_wind, results['ts_wind'])) self.assertTrue(numpy.allclose(exp_ts_solar, results['ts_solar'])) self.assertTrue(numpy.allclose(exp_ts_time, results['ts_time'])) self.assertTrue(numpy.allclose(exp_ts_vector, results['ts_vector'])) self.assertTrue(numpy.allclose(exp_int_1, results['int_1'])) # Can't just compare exp_float_1 because it has a nan in it. Instead, # check that the nan is there, and that the rest of the array is as # expected. self.assertTrue( numpy.all( numpy.isnan(exp_float_1) == numpy.isnan(results['float_1']))) self.assertTrue( numpy.allclose(exp_float_1[[0, 2, 3]], results['float_1'][[0, 2, 3]])) # and check that all the floats are float64 self.assertTrue(results['ts_wind'].dtype.name == 'float64') self.assertTrue(results['ts_solar'].dtype.name == 'float64') self.assertTrue(results['float_1'].dtype.name == 'float64') self.assertTrue((ts_len == 6))
def complete_configuration(self): self.data = {} for list_type in [ 'ts_float_list', 'ts_int_list', 'other_float_list', 'other_int_list' ]: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] try: f = nc.NetCDFFile(infile) except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened.') raise mureilexception.ConfigException(msg, {}) try: vbl = f.variables[self.config[series_name + '_vbl']] except: msg = ('Variable ' + self.config[series_name + '_vbl'] + ' not found in file ' + infile) raise mureilexception.ConfigException(msg, {}) dims = len(vbl.shape) if (dims == 1): temp = vbl[:] elif (dims == 2): temp = vbl[:, :] else: msg = 'Data series ' + series_name + ' has more than 2 dimensions, so is not handled.' raise mureilexception.ConfigException(msg, {}) if 'float' in list_type: if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp else: if not mureiltypes.check_ndarray_int(temp, True): self.data[series_name] = numpy.array(temp, dtype=int) else: self.data[series_name] = temp for list_type in ['ts_csv_list']: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] temp = [] try: with open(infile, 'rU') as n: reader = csv.reader(n) for row in reader: if reader.line_num == 1: self.data[series_name + '_hdr'] = row[1:] else: if reader.line_num == 2: temp = [map(float, (row[1:]))] else: temp.append(map(float, (row[1:]))) self.data[series_name] = numpy.array(temp, dtype=float) if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened or had an error in reading.') raise mureilexception.ConfigException(msg, {}) # Now apply the NaN filter to the ts lists, but note that the integer # ones are not identified as nan. all_ts = self.config['ts_float_list'] + self.config[ 'ts_int_list'] + self.config['ts_csv_list'] if len(all_ts) == 0: self.ts_length = 0 logger.warning('No timeseries data defined') else: self.ts_length = self.data[all_ts[0]].shape[0] # Start with an array of 'False' nan_acc = (numpy.ones(self.ts_length) == 0) # Accumulate 'True' entries in nan_acc where NaN found in timeseries for ts_name in all_ts: ts_nan = numpy.isnan(self.data[ts_name]) if ts_nan.ndim > 1: ts_nan = ts_nan.any(1) # Check all the timeseries are the same length if not (len(ts_nan) == self.ts_length): msg = ('Data series ' + ts_name + ' is length {:d}, not matching {:d} of '.format( len(ts_nan), self.ts_length) + all_ts[0]) raise mureilexception.ConfigException(msg, {}) nan_acc = numpy.logical_or(nan_acc, ts_nan) # Clean up the timeseries using slices nan_acc = numpy.logical_not(nan_acc) for ts_name in all_ts: if self.data[ts_name].ndim == 1: self.data[ts_name] = self.data[ts_name][nan_acc] else: self.data[ts_name] = self.data[ts_name][nan_acc, :] self.ts_length = self.data[all_ts[0]].shape[0] self.is_configured = True return None
def complete_configuration(self): self.data = {} wind_gap = '11' solar_gap = '21' dir = '/mnt/meteo0/data/dargaville/rhuva/' file = 'RegGrid_wind_output_'+wind_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'RegGrid_dsr_output_'+solar_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp file = 'Aus_demand_2010-2011.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] file = 'Dist-to-nearest-cap_wind_'+wind_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_wind_distances'] = temp file = 'Dist-to-nearest-cap_dsr_'+solar_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_solar_distances'] = temp return None
def make_plot(filename, grid_name, x_name='x', y_name='y', t_name='time', n_cols=6, outpath='', filename_prefix='LMA', do_save=True, image_type='pdf', colormap='gist_earth'): """ colormap: a string giving the name of a matplotlib built-in colormap, or a matplotlib.colors.Colormap instance """ f = nc.NetCDFFile(filename) data = f.variables # dictionary of variable names to nc_var objects dims = f.dimensions # dictionary of dimension names to sizes x = data[x_name] y = data[y_name] t = data[t_name] grid = data[grid_name] assert len(x.shape) == 1 assert len(y.shape) == 1 assert len(t.shape) == 1 grid_dims = grid.dimensions # tuple of dimension names name_to_idx = dict((k, i) for i, k in enumerate(grid_dims)) grid_t_idx = name_to_idx[t.dimensions[0]] grid_x_idx = name_to_idx[x.dimensions[0]] grid_y_idx = name_to_idx[y.dimensions[0]] n_frames = t.shape[0] # n_cols = 6 n_rows = int(ceil(float(n_frames) / n_cols)) if type(colormap) == type(''): colormap = get_cmap(colormap) grey_color = (0.5, ) * 3 frame_color = (0.2, ) * 3 density_maxes = [] total_counts = [] all_t = [] xedge = centers_to_edges(x) x_range = xedge.max() - xedge.min() yedge = centers_to_edges(y) y_range = yedge.max() - yedge.min() dx = (xedge[1] - xedge[0]) # w, h = figaspect(float(n_rows)/n_cols) # breaks for large numbers of frames - has a hard-coded max figure size w, h, n_rows_perpage, n_pages = multiples_figaspect(n_rows, n_cols, x_range, y_range, fig_width=8.5, max_height=None) # count_scale_factor = dx # / 1000.0 # max_count_baseline = 450 * count_scale_factor #/ 10.0 min_count, max_count = 1, grid[:].max() #max_count_baseline*(t[1]-t[0]) if (max_count == 0) | (max_count == 1): max_count = min_count + 1 f.close() default_vmin = -0.2 if np.log10(max_count) <= default_vmin: vmin_count = np.log10(max_count) + default_vmin else: vmin_count = default_vmin fig = Figure(figsize=(w, h)) canvas = FigureCanvasAgg(fig) fig.set_canvas(canvas) p = small_multiples_plot(fig=fig, rows=n_rows, columns=n_cols) p.label_edges(True) pad = 0.0 # for time labels in each frame for ax in p.multiples.flat: ax.set_axis_bgcolor('black') ax.spines['top'].set_edgecolor(frame_color) ax.spines['bottom'].set_edgecolor(frame_color) ax.spines['left'].set_edgecolor(frame_color) ax.spines['right'].set_edgecolor(frame_color) # ax.yaxis.set_major_formatter(kilo_formatter) # ax.xaxis.set_major_formatter(kilo_formatter) base_date = datetime.strptime(t.units, "seconds since %Y-%m-%d %H:%M:%S") time_delta = timedelta(0, float(t[0]), 0) start_time = base_date + time_delta indexer = [ slice(None), ] * len(grid.shape) frame_start_times = [] for i in range(n_frames): frame_start = base_date + timedelta(0, float(t[i]), 0) frame_start_times.append(frame_start) indexer[grid_t_idx] = i density = grid[indexer] # density,edges = np.histogramdd((x,y), bins=(xedge,yedge)) density_plot = p.multiples.flat[i].pcolormesh(xedge, yedge, np.log10( density.transpose()), vmin=vmin_count, vmax=np.log10(max_count), cmap=colormap) label_string = frame_start.strftime('%H%M:%S') text_label = p.multiples.flat[i].text(xedge[0] - pad + x_range * .015, yedge[0] - pad + y_range * .015, label_string, color=grey_color, size=6) density_plot.set_rasterized(True) density_maxes.append(density.max()) total_counts.append(density.sum()) all_t.append(frame_start) print label_string, x.shape, density.max(), density.sum() color_scale = ColorbarBase(p.colorbar_ax, cmap=density_plot.cmap, norm=density_plot.norm, orientation='horizontal') # color_scale.set_label('count per pixel') color_scale.set_label('log10(count per pixel)') view_x = (xedge.min(), xedge.max()) view_y = (yedge.min(), yedge.max()) print 'making multiples', p.multiples.flat[0].axis(view_x + view_y) filename = '%s-%s_%s_%05.2fkm_%05.1fs.%s' % ( filename_prefix, grid_name, start_time.strftime('%Y%m%d_%H%M%S'), dx, time_delta.seconds, image_type) filename = os.path.join(outpath, filename) if do_save: fig.savefig(filename, dpi=150) return fig, p, frame_start_times, filename print ' ... done'
def write_cf_netcdf(outfile, t_start, t, xloc, yloc, lon_for_x, lat_for_y, ctr_lat, ctr_lon, grid, grid_var_name, grid_description, format='i', **kwargs): """ Write a Climate and Forecast Metadata-compliant NetCDF file. Should display natively in conformant packages like McIDAS-V. """ import pupynere as nc missing_value = -9999 nc_out = nc.NetCDFFile(outfile, 'w') nc_out.createDimension('nx', xloc.shape[0]) nc_out.createDimension('ny', yloc.shape[0]) nc_out.createDimension('ntimes', t.shape[0]) #unlimited==None proj = nc_out.createVariable('Lambert_Azimuthal_Equal_Area', 'i', ()) proj.grid_mapping_name = 'lambert_azimuthal_equal_area' proj.longitude_of_projection_origin = ctr_lon proj.latitude_of_projection_origin = ctr_lat proj.false_easting = 0.0 proj.false_northing = 0.0 # x_coord = nc_out.createVariable('longitude', 'f', ('nx',)) # x_coord.long_name="longitude" # x_coord.standard_name="longitude" # x_coord.units = "degrees_east" x_coord = nc_out.createVariable('x', 'f', ('nx', )) x_coord.units = "km" x_coord.long_name = "x coordinate of projection" x_coord.standard_name = 'projection_x_coordinate' # y_coord = nc_out.createVariable('latitude', 'f', ('nx',)) # y_coord.long_name="latitude" # y_coord.standard_name="latitude" # y_coord.units = "degrees_north" y_coord = nc_out.createVariable('y', 'f', ('ny', )) y_coord.units = "km" y_coord.long_name = "y coordinate of projection" y_coord.standard_name = 'projection_y_coordinate' times = nc_out.createVariable('time', 'f', ('ntimes', )) #, filters=no_compress) times.long_name = "time" times.units = "seconds since %s" % t_start.strftime('%Y-%m-%d %H:%M:%S') lons = nc_out.createVariable('lons', 'd', ('nx', 'ny')) #, filters=no_compress) lons.long_name = "longitude" lons.standard_name = "longitude" lons.units = "degrees_east" lats = nc_out.createVariable('lats', 'd', ('nx', 'ny')) #, filters=no_compress) lats.long_name = "latitude" lats.standard_name = "latitude" lats.units = "degrees_north" lightning2d = nc_out.createVariable( grid_var_name, format, ('ntimes', 'nx', 'ny')) #, filters=no_compress) lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)' lightning2d.units = 'dimensionless' lightning2d.coordinates = 'time lons lats' lightning2d.grid_mapping = "Lambert_Azimuthal_Equal_Area" lightning2d.missing_value = missing_value x_coord[:] = xloc[:] y_coord[:] = yloc[:] times[:] = t[:] lons[:] = lon_for_x[:] lats[:] = lat_for_y[:] for i in range(grid.shape[2]): lightning2d[i, :, :] = grid[:, :, i] nc_out.close()
def write_AWIPS_netcdf_grid(outfile, t_start, t, xloc, yloc, lon_for_x, lat_for_y, ctr_lat, ctr_lon, grid, grid_var_name, grid_description, format='i', refresh_minutes=1, grid_units='dimensionless'): """ Write an AWIPS-formatted NetCDF grid for lightning data. TYPEMAP = { NC_BYTE: ('b', 1), NC_CHAR: ('c', 1), NC_SHORT: ('h', 2), NC_INT: ('i', 4), # this is also long NC_FLOAT: ('f', 4), NC_DOUBLE: ('d', 8) } """ missing_value = -9999 # all_levels = (1,) level_name = "SFC" # ---------- # Dimensions # ---------- nc_out = nc.NetCDFFile(outfile, 'w') nc_out.createDimension('record', None) #unlimited==None nc_out.createDimension('n_valtimes', t.shape[0]) nc_out.createDimension('data_variables', 1) nc_out.createDimension('namelen', 132) nc_out.createDimension('charsPerLevel', 20) nc_out.createDimension('levels', 1) # for level in all_levels: # nc_out.createDimension('levels_{0}'.format(level), level) nc_out.createDimension('x', xloc.shape[0]) nc_out.createDimension('y', yloc.shape[0]) # ----------------- # Global attributes # ----------------- nc_out.Data_record_time = t_start.strftime('%Y%m%d.%H%M') nc_out.depictorName = "WTLMA_ltng_grid" nc_out.projIndex = 8 nc_out.projName = "CYLINDRICAL_EQUIDISTANT" nc_out.centralLon = ctr_lon nc_out.centralLat = ctr_lat nc_out.xMin = lon_for_x.min() nc_out.xMax = lon_for_x.max() nc_out.yMin = lat_for_y.min() nc_out.yMax = lat_for_y.max() nc_out.lat00 = nc_out.yMin nc_out.lon00 = nc_out.xMin nc_out.latNxNy = nc_out.yMax nc_out.lonNxNy = nc_out.xMax nc_out.dxKm = nc_out.xMin nc_out.dyKm = nc_out.yMax nc_out.latDxDy = (nc_out.yMin + nc_out.yMax) / 2.0 nc_out.lonDxDy = (nc_out.xMin + nc_out.xMax) / 2.0 nc_out.UnitSquareSideSize = xloc[1] - xloc[0] nc_out.RefreshPeriod = 60 * int(refresh_minutes) # nc_out.NbRefreshPeriods = int(10*60/nc_out.RefreshPeriod) charsPerLevel = nc_out.createVariable(grid_var_name + 'Levels', 'c', ('levels', 'charsPerLevel')) inventory = nc_out.createVariable(grid_var_name + 'Inventory', 'c', ('n_valtimes', 'charsPerLevel')) charsPerLevel[0, 0:len(level_name)] = level_name for i in range(nc_out.dimensions['n_valtimes']): inventory[i, :] = '1' * nc_out.dimensions[ 'charsPerLevel'] # What is this? Why AWIPS, Why? the_epoch = datetime.datetime(1970, 1, 1, 0, 0, 0) reftime_since_the_epoch = int((t_start - the_epoch).total_seconds()) valtimeref_delta = nc_out.createVariable('valtimeMINUSreftime', 'i', ('n_valtimes', )) valtime = nc_out.createVariable('valtimeMINUSreftime', 'i', ('n_valtimes', )) reftime = nc_out.createVariable('valtimeMINUSreftime', 'i', ()) reftime.long_name = "reference time" reftime.units = "seconds since (1970-1-1 00:00:00.0)" reftime.assignValue(reftime_since_the_epoch) valtime.long_name = "valid time" valtime.units = "seconds since (1970-1-1 00:00:00.0)" valtime[:] = t + reftime_since_the_epoch valtimeref_delta[:] = t lightning2d = nc_out.createVariable( grid_var_name, format, ('record', 'levels', 'y', 'x')) #, filters=no_compress) lightning2d.long_name = grid_description #'LMA VHF event counts (vertically integrated)' lightning2d.units = grid_units lightning2d.uiname = grid_var_name lightning2d._FillValue = missing_value lightning2d._n3D = 1 lightning2d.levels = level_name # Loop over times for i in range(grid.shape[2]): lightning2d[i, 0, :, :] = grid[:, :, i].T nc_out.close()