def __str__(self): return """{0} Creation Status: {1} Source Seed: {2}\tSource NN: {3}\tSource NB: {4}\t Source Lp: {5} Box Matrix: {6} Box String: {7} Radis: {8} Number of Nodes: {9}\tNumber of Surface Nodes: {10}\tNumber of Bonds {11} Source Z: {12}\tSource Delta Z: {13} Z: {14}\tDelta Z: {15} """.format( self.data, chartostring(self.get("creation_status", 0)), self.get("seed_src", 0), self.get("NN_src", 0), self.get("NB_src", 0), self.get("Lp_src", 0), self.get("box_mat", 0), chartostring(self.get("box_str", 0)), self.get("radius", 0), self.get("NN", 0), self.get("NSN", 0), self.get("NB", 0), self.get("Z_src", 0), self.get("DeltaZ_src", 0), self.get("Z", 0), self.get("DeltaZ", 0), )
def drifter_meta(): imei = {} wmo = {} deployment = {} with Dataset(app.config['DRIFTER_AGG_URL'], 'r') as ds: for idx, b in enumerate(ds.variables['buoy'][:]): bid = str(chartostring(b))[:-3] for data, key in [ [imei, 'imei'], [wmo, 'wmo'], [deployment, 'deployment'] ]: d = str(chartostring(ds.variables[key][idx][0])) if data.get(d) is None: data[d] = [bid] else: data[d].append(bid) return { 'imei': imei, 'wmo': wmo, 'deployment': deployment, }
def observation_meta(): with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds: ship = chartostring(ds['ship'][:]) trip = chartostring(ds['trip'][:]) data = { 'ship': sorted(np.unique(ship).tolist()), 'trip': sorted(np.unique(trip).tolist()), } return data
def main(): global ncfile, prefix parser = argparse.ArgumentParser(description="A commandline tool to convert WRF timeseries files to netCDF4") parser.add_argument('netcdf', metavar="netCDF4 file", type=str, nargs=1, help="NetCDF file base name. Create it first with ts_make_ncs.sh") parser.add_argument('-d', '--domain', metavar="domain", type=int, nargs=1, help="WRF domain number", default=0) args = parser.parse_args() for varname in ['TS']: # ,'UU','VV','TH','QV','PH']: logging.info( "{}".format(varname) ) ncfile = cdf.Dataset( args.netcdf[0] + "." + varname + ".nc", "r+" ) do_tslist() prefix = ncfile.variables['prefix'] for stationi in range(nstations): filename = "{}.d{:02d}.".format( cdf.chartostring( prefix[stationi] ), args.domain[0] ) if not os.path.isfile( filename + "TS" ): continue if varname in ['TS',]: if not ntimes: simplecount(filename + "TS", args.domain[0]) do_tsfile ( filename + "TS", stationi ) else: do_profile( filename + varname, stationi, varname ) if varname in ['TS',]: flush_tsfile() else: ncfile.variables[varname][:,:,:] = profile [:,:,:] ncfile.close()
def class4(class4_id, projection, resolution, extent): dataset_url = app.config["CLASS4_URL"] % class4_id proj = pyproj.Proj(init=projection) view = _get_view(extent) rmse = [] lat = [] lon = [] point_id = [] identifiers = [] with Dataset(dataset_url, 'r') as ds: lat_in = ds['latitude'][:] lon_in = ds['longitude'][:] ids = map(str.strip, chartostring(ds['id'][:])) for i in range(0, lat_in.shape[0]): x, y = proj(lon_in[i], lat_in[i]) p = Point(y, x) if view.envelope.intersects(p): lat.append(float(lat_in[i])) lon.append(float(lon_in[i])) identifiers.append(ids[i]) best = ds['best_estimate'][i, 0, :] obsv = ds['observation'][i, 0, :] point_id.append(i) rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean())) rmse = np.ma.hstack(rmse) rmse_norm = np.clip(rmse / 1.5, 0, 1) loc = zip(lon, lat) points = [] for idx, ll in enumerate(loc): if np.ma.is_masked(rmse[idx]): continue points.append({ 'type': "Feature", 'geometry': { 'type': "Point", 'coordinates': ll, }, 'properties': { 'name': "%s" % identifiers[idx], 'id': "%s/%d" % (class4_id, point_id[idx]), 'error': float(rmse[idx]), 'error_norm': float(rmse_norm[idx]), 'type': 'class4', 'resolution': 0, }, }) result = { 'type': "FeatureCollection", 'features': points, } return result
def list_class4(d): dataset_url = app.config["CLASS4_URL"] % d with Dataset(dataset_url, 'r') as ds: lat = ds['latitude'][:] lon = ds['longitude'][:] ids = map(str.strip, chartostring(ds['id'][:])) rmse = [] for i in range(0, lat.shape[0]): best = ds['best_estimate'][i, 0, :] obsv = ds['observation'][i, 0, :] rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean())) rmse = np.ma.hstack(rmse) maxval = rmse.mean() + 2 * rmse.std() rmse_norm = rmse / maxval loc = zip(lat, lon) points = [] for idx, ll in enumerate(loc): if np.ma.is_masked(rmse[idx]): continue points.append({ 'name': "%s" % ids[idx], 'loc': "%f,%f" % ll, 'id': "%s/%d" % (d, idx), 'rmse': float(rmse[idx]), 'rmse_norm': float(rmse_norm[idx]), }) points = sorted(points, key=lambda k: k['id']) return points
def runTest(self): """testing functions for converting arrays of chars to fixed-len strings""" nc = Dataset(FILE_NAME) assert nc.dimensions['n1'].isunlimited() == True v = nc.variables['strings'] v2 = nc.variables['strings2'] v3 = nc.variables['strings3'] assert v.dtype.str[1:] in ['S1','U1'] assert v.shape == (nrecs,n2,nchar) for nrec in range(nrecs): data2 = chartostring(v[nrec],encoding='ascii') assert_array_equal(data2,datau[nrec]) data2 = v2[:] data2[0] = v2[0] data2[0,1] = v2[0,1] assert_array_equal(data2,datau) data3 = v3[:] assert_array_equal(data3,datau) # these slices should return a char array, not a string array data4 = v2[:,:,0] assert(data4.dtype.itemsize == 1) assert_array_equal(data4, datac[:,:,0]) data5 = v2[0,0:nchar,0] assert(data5.dtype.itemsize == 1) assert_array_equal(data5, datac[0,0:nchar,0]) # test turning auto-conversion off. v2.set_auto_chartostring(False) data6 = v2[:] assert(data6.dtype.itemsize == 1) assert_array_equal(data6, datac) nc.set_auto_chartostring(False) data7 = v3[:] assert(data7.dtype.itemsize == 1) assert_array_equal(data7, datac) nc.close()
def maketransectdf(transect, dt_from=None, dt_to=None): """Read some transect data""" transectidx, filter, vardict, ds = prepare_vardict(transect, 'transect', False, dt_from, dt_to) if transectidx is None: alongshore = np.zeros(0) areaname = np.zeros(0) mean_high_water = np.zeros(0) mean_low_water = np.zeros(0) else: alongshore = ds.variables['alongshore'][transectidx] areaname = str(netCDF4.chartostring(ds.variables['areaname'][transectidx])).strip() mean_high_water = ds.variables['mean_high_water'][transectidx] mean_low_water = ds.variables['mean_low_water'][transectidx] ds.close() transectdf = pandas.DataFrame( index=[transect], data=dict( transect=transect, areaname=areaname, mean_high_water=mean_high_water, mean_low_water=mean_low_water ) ) return transectdf
def get_data_from_file(path, field_name = 'water_discharge'): """ returns streamflow, times, x_indices, y_indices """ date_format = '%Y_%m_%d_%H_%M' print path ds = nc.Dataset(path) #dims: time, cell_index discharge = ds.variables[field_name][:] times = ds.variables['time'][:] times = chartostring(times) if ds.variables.has_key('x_index'): x_indices = ds.variables['x_index'][:] y_indices = ds.variables['y_index'][:] else: x_indices = ds.variables['x-index'][:] y_indices = ds.variables['y-index'][:] date_times = map(lambda x: datetime.strptime( x , date_format ), times) return discharge, date_times, x_indices, y_indices
def do_tsfile(filename, stationi): if not os.path.isfile( filename): logging.info( "Skipping station %s : TS", cdf.chartostring( prefix[stationi] ) ) return logging.debug( "{} starting".format( filename ) ) logging.debug( "parsing ts file" ) # Parse TS file, see the README.tslist in the WRF run directory for details fileTS = open( filename, 'r' ) # Header # NZCM McMurdo 2 7 mcm (-77.850, 166.710) ( 153, 207) (-77.768, 166.500) 81.8 meters # # Those are name of the station, grid ID, time-series ID, station lat/lon, grid indices (nearest grid point to # the station location), grid lat/lon, elevation. line = fileTS.next() fields = re.search(r"\( *(\d+) *, *(\d+) *\) *\( *(-?\d+\.\d+) *, *(-?\d+\.\d+) *\) * (-?\d+(\.\d+)?) *meters", line).groups() ncfile.variables['gj' ][stationi] = fields[0] ncfile.variables['gi' ][stationi] = fields[1] ncfile.variables['glat'][stationi] = fields[2] ncfile.variables['glon'][stationi] = fields[3] ncfile.variables['elevation'][stationi] = fields[4] # Body # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 # id, ts_hour, id_tsloc, ix, iy, t, q, u, v, psfc, glw, gsw, hfx, lh, tsk, tslb(1), rainc, rainnc, clw timei = -1 for line in fileTS: timei += 1 fields = line.split() time [ timei ] = fields[ 1] T2m [ timei,stationi ] = fields[ 5] Q2m [ timei,stationi ] = fields[ 6] U10m [ timei,stationi ] = fields[ 7] V10m [ timei,stationi ] = fields[ 8] psfc [ timei,stationi ] = fields[ 9] glw [ timei,stationi ] = fields[10] gsw [ timei,stationi ] = fields[11] hfx [ timei,stationi ] = fields[12] lh [ timei,stationi ] = fields[13] tsk [ timei,stationi ] = fields[14] tslb1 [ timei,stationi ] = fields[15] rainc [ timei,stationi ] = fields[16] rainnc[ timei,stationi ] = fields[17] clw [ timei,stationi ] = fields[18] tc2m [ timei,stationi ] = fields[19] tp2m [ timei,stationi ] = fields[20] fileTS.close() logging.info( "{} done".format( filename ) )
def xtimeGetYear(xtime): """Get an array of years from an xtime array, ignoring any partial year information""" # First parse the xtime character array into a string xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function years = np.zeros( (len(xtimestr),) ) for i in range(len(xtimestr)): years[i] = ( int(xtimestr[i].split('-')[0]) ) # Get the year part and make it an integer return years
def test_sweep_mode(): assert 'standard_name' in radar.sweep_mode assert radar.sweep_mode['data'].shape == (1, 32) assert radar.sweep_mode['data'].dtype.char == 'S' str_array = netCDF4.chartostring(radar.sweep_mode['data']) try: assert np.all(str_array == ['azimuth_surveillance']) except AssertionError: assert np.all(str_array == [b'azimuth_surveillance'])
def read_json(open_file, var_name): qp = nc.chartostring((open_file.variables[var_name][:])) try: parsed_json = json.loads(qp[0]) return parsed_json except: parsed_json = 'n/a' return parsed_json
def check_valid_time_format(section, dataset, text, var_name): """ Check that a variable is a valid UTC time formatted string. """ if var_name in dataset.variables: s = str(netCDF4.chartostring(dataset.variables[var_name][:])) try: time.strptime(s, '%Y-%m-%dT%H:%M:%SZ') except: tup = (text, var_name, s, 'yyyy-mm-ddThh:mm:ssZ') t = "%s '%s' has an invalid format: %s should be %s" % (tup) log_error(section, t)
def insert_station(sesh, cf, i, name, x, y): station = Station( name=str(chartostring(name[i])), x=x[i], x_units=x.units, y=y[i], y_units=y.units, ) sesh.add(station) return station
def runTest(self): """testing functions for converting arrays of chars to fixed-len strings""" nc = Dataset(FILE_NAME) assert nc.dimensions['n1'].isunlimited() == True v = nc.variables['strings'] assert v.dtype.str[1:] == 'S1' assert v.shape == (nrecs,n2,nchar) for nrec in range(nrecs): data2 = chartostring(v[nrec]) assert_array_equal(data2,data[nrec]) nc.close()
def check_dataset_metadata_in_storage_unit(nco, dataset_dir): assert len(nco.variables['dataset']) == 1 # 1 time slice stored_metadata = nco.variables['dataset'][0] if not isinstance(stored_metadata, str): stored_metadata = netCDF4.chartostring(stored_metadata) stored_metadata = str(np.char.decode(stored_metadata)) ds_filename = dataset_dir / 'agdc-metadata.yaml' stored = yaml.safe_load(stored_metadata) [(_, original)] = read_documents(ds_filename) assert len(stored['lineage']['source_datasets']) == 1 assert next(iter(stored['lineage']['source_datasets'].values())) == original
def find_station(sesh, cf, i, name, x, y): return ( sesh.query(Station) .filter_by( name=str(chartostring(name[i])), x=x[i], x_units=x.units, y=y[i], y_units=y.units, ) .first() )
def check_dataset_metadata_in_storage_unit(nco, dataset_dir): assert len(nco.variables['dataset']) == 1 # 1 time slice stored_metadata = nco.variables['dataset'][0] if not isinstance(stored_metadata, str): stored_metadata = netCDF4.chartostring(stored_metadata) stored_metadata = str(np.char.decode(stored_metadata)) ds_filename = dataset_dir / 'agdc-metadata.yaml' with ds_filename.open() as f: orig_metadata = f.read() stored = make_pgsqljson_match_yaml_load(yaml.safe_load(stored_metadata)) original = make_pgsqljson_match_yaml_load(yaml.safe_load(orig_metadata)) assert len(stored['lineage']['source_datasets']) == 1 assert next(iter(stored['lineage']['source_datasets'].values())) == original
def xtime2numtime(xtime): # First parse the xtime character array into a string xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function dt = [] for stritem in xtimestr: itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s results = [int(i) for i in itemarray] if (results[0] < 1900): # datetime has a bug where years less than 1900 are invalid on some systems results[0] += 1900 dt.append( datetime.datetime(*results) ) # * notation passes in the array as arguments numtime = netCDF4.date2num(dt, units='seconds since '+str(dt[0])) # use the netCDF4 module's function for converting a datetime to a time number return numtime
def test_create_string_variable(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype) var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg',), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert 'str_var' in nco.variables assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def runTest(self): """testing compound attributes""" f = Dataset(self.file, 'r') v = f.variables[VAR_NAME] g = f.groups[GROUP_NAME] vv = g.variables[VAR_NAME2] assert_array_almost_equal(v.missing_values['speed'], missvals['speed']) assert_array_almost_equal(v.missing_values['direction'],\ missvals['direction']) assert_array_almost_equal(vv.missing_values['speed'], missvals['speed']) assert_array_almost_equal(vv.missing_values['direction'],\ missvals['direction']) assert_array_equal(v.units['speed'], windunits['speed'].squeeze()) assert_array_equal(v.units['direction'],\ windunits['direction'].squeeze()) assert_array_equal(vv.units['speed'], windunits['speed'].squeeze()) assert_array_equal(vv.units['direction'],\ windunits['direction'].squeeze()) assert(chartostring(v.units['speed']).item().rstrip().decode(default_encoding) == 'm/s') assert(chartostring(v.units['direction']).item().rstrip().decode(default_encoding) == 'degrees') assert(chartostring(vv.units['speed']).item().rstrip().decode(default_encoding) == 'm/s') assert(chartostring(vv.units['direction']).item().rstrip().decode(default_encoding) == 'degrees') f.close()
def xtime2numtimeMy(xtime): """Define a function to convert xtime character array to numeric time values using local arithmetic""" # First parse the xtime character array into a string xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function numtime = np.zeros( (len(xtimestr),) ) ii = 0 dayOfMonthStart = [1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] for stritem in xtimestr: itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s results = [int(i) for i in itemarray] numtime[ii] = results[0] + (dayOfMonthStart[results[1]-1]-1 + results[2]) / 365.0 # decimal year ii += 1 return numtime
def test_non_scalar_variables(self) : var = self.dataset.variables['regcodes'] self.assertTrue(var.long_name == "region codes") self.assertTrue(var.shape == (3,)) data = var[:] expected = np.array([1,2,3], dtype=np.int32) expected.shape = (3,) self.assertTrue(np.array_equal(data, expected)) var = self.dataset.variables['regions'] self.assertTrue(var.long_name == "region names") self.assertTrue(var.shape == (3,10)) data = var[:] data = nc4.chartostring(data) self.assertTrue(data[0].startswith(b"Europe")) self.assertTrue(data[1].startswith(b"Americas")) self.assertTrue(data[2].startswith(b"Asia")) var = self.dataset.variables['digits'] self.assertTrue(var.long_name == "decimal digits") self.assertTrue(var.shape == (10,)) data = var[:] data = nc4.chartostring(data) self.assertTrue(data == b"0123456789") var = self.dataset.variables['dna_code'] self.assertTrue(var.long_name == "DNA code") self.assertTrue(var.shape == (2,3,4)) data = var[:] sample = nc4.chartostring(data[0]) self.assertTrue(sample[0] == b"ACTG") self.assertTrue(sample[1] == b"ACGG") self.assertTrue(sample[2] == b"ATGC") sample = nc4.chartostring(data[1]) self.assertTrue(sample[0] == b"CTGA") self.assertTrue(sample[1] == b"GCTA") self.assertTrue(sample[2] == b"TGCA")
def maketransectdf(transect, url='http://opendap.deltares.nl/thredds/dodsC/opendap/rijkswaterstaat/jarkus/profiles/transect.nc'): """Read some transect data""" ds = netCDF4.Dataset(url) transectidx = bisect.bisect_left(ds.variables['id'],transect) if ds.variables['id'][transectidx] != transect: idfound = ds.variables['id'][transectidx] ds.close() raise ValueError("Could not find transect data for transect {}, closest is {}".format(transect, idfound)) alongshore = ds.variables['alongshore'][transectidx] areaname = netCDF4.chartostring(ds.variables['areaname'][transectidx]) mean_high_water = ds.variables['mean_high_water'][transectidx] mean_low_water = ds.variables['mean_low_water'][transectidx] ds.close() transectdf = pandas.DataFrame(index=[transect], data=dict(transect=transect, areaname=areaname, mean_high_water=mean_high_water, mean_low_water=mean_low_water)) return transectdf
def xtime2numtime(xtime): """Define a function to convert xtime character array to numeric time values using datetime objects""" # First parse the xtime character array into a string xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function dt = [] for stritem in xtimestr: itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s results = [int(i) for i in itemarray] if (results[0] < 1900): # datetime has a bug where years less than 1900 are invalid on some systems results[0] += 1900 dt.append( datetime.datetime(*results) ) # * notation passes in the array as arguments # use the netCDF4 module's function for converting a datetime to a time number numtime = netCDF4.date2num(dt, units='seconds since '+str(dt[0])) numtime /= (3600.0 * 24.0 * 365.0) numtime -= numtime[0] # return years from start return numtime
def normalize_array(var): """ Returns a normalized data array from a NetCDF4 variable. This is mostly used to normalize string types between py2 and py3. It has no effect on types other than chars/strings """ if np.issubdtype(var.dtype, 'S1'): if var.dtype == str: # Python 2 on netCDF4 'string' variables needs this. # Python 3 returns false for np.issubdtype(var.dtype, 'S1') return var[:] def decoder(x): return str(x.decode('utf-8')) vfunc = np.vectorize(decoder) return vfunc(nc4.chartostring(var[:])) else: return var[:]
def test_insert_station(test_session_with_empty_db, tiny_dsg_dataset): var_name = tiny_dsg_dataset.dependent_varnames()[0] instance_dim = tiny_dsg_dataset.instance_dim(var_name) name = tiny_dsg_dataset.id_instance_var(var_name) lat = tiny_dsg_dataset.spatial_instance_var(var_name, 'X') lon = tiny_dsg_dataset.spatial_instance_var(var_name, 'Y') i = 0 assert instance_dim.size > 0 check_insert( insert_station, test_session_with_empty_db, tiny_dsg_dataset, i, name, lon, lat, name=str(chartostring(name[i])), x=lon[i], x_units=lon.units, y=lat[i], y_units=lat.units )
def GetFileDateTime(file_obj, time, string=True): """ Returns a string of the date time from model time index""" tindex = GetTimeIndex(file_obj, time) print(tindex) try: valid_time = netCDF4.chartostring(file_obj.variables['Times'][tindex]) valid_DT = datetime.datetime.strptime( valid_time, '%Y-%m-%d_%H:%M:%S') + datetime.datetime( 1900, 1, 1, 0, 0, 0) except: return 'Error Parsing netcdf file time variable' if string: valid_DT else: return valid_time
def xtime2numtimeMy(xtime): """Define a function to convert xtime character array to numeric time values using local arithmetic""" # First parse the xtime character array into a string xtimestr = netCDF4.chartostring( xtime ) # convert from the character array to an array of strings using the netCDF4 module's function numtime = np.zeros((len(xtimestr), )) ii = 0 dayOfMonthStart = [1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] for stritem in xtimestr: itemarray = stritem.strip().replace('_', '-').replace(':', '-').split( '-') # Get an array of strings that are Y,M,D,h,m,s results = [int(i) for i in itemarray] numtime[ii] = results[0] + (dayOfMonthStart[results[1] - 1] - 1 + results[2]) / 365.0 # decimal year ii += 1 return numtime
def render(self, paths, variable, metric, region): xs, ys = [], [] for path in paths: with netCDF4.Dataset(path) as dataset: mi = index(dataset.variables["metric_names"][:], metric) ai = index(dataset.variables["area_names"][:], region) names = np.char.strip( netCDF4.chartostring( dataset.variables["forecast_names"][:])) fi = names == "forecast" y = dataset.variables[variable][:, fi, :, mi, ai] y = y.mean(axis=(0, 2)) x = dataset.variables["forecasts"][fi] xs.append(x) ys.append(y) x = np.mean(xs, axis=0) y = np.mean(ys, axis=0) self.source.data = {"x": x, "y": y}
def readTimeSeries (ncFile, ncVar = 'zeta', verbose=1): """ Reads fort.61.nc-like file """ if verbose: msg( 'i','Reading [' + ncVar + '] from ' + ncFile) if not os.path.exists (ncFile): msg( 'e','File ' + ncFile + ' does not exist.') return nc = netCDF4.Dataset( ncFile ) fld = nc.variables[ncVar][:] missingVal = nc.variables[ncVar]._FillValue try: fld.unshare_mask() except: pass fld [np.where(fld == missingVal)] = np.nan lon = nc.variables['x'][:] lat = nc.variables['y'][:] tim = nc.variables['time'][:] nam = nc.variables['station_name'][:] stations = netCDF4.chartostring(nam) # Python3 requirement? ncTitle = nc.getncattr('title') try: baseDate = datetime.strptime(nc.variables['time'].base_date[0:19], '%Y-%m-%d %H:%M:%S') except: # when 00 sec is not written at all baseDate = datetime.strptime(nc.variables['time'].base_date[0:19], '%Y-%m-%d %H:%M ') realtime = np.array([baseDate + timedelta(seconds=int(tim[i])) for i in range(len(tim))]) return {'lat' : lat, 'lon' : lon, 'time' : realtime, 'base_date' : baseDate, 'zeta' : fld, 'stations' : stations, 'title' : ncTitle}
def read_point_files(data_files, variables, start_date): for j, files in enumerate(data_files): for i, name in enumerate(files): print(name.split('/')[-1]) nc_file = netCDF4.Dataset(name, 'r') # Initializations for first iteration of run if j == 0 and i == 0: # Station information stations = {} stations['name'] = netCDF4.chartostring( nc_file.variables['station_name'][:, :]).tolist() stations['lon'] = np.squeeze(nc_file.variables['longitude'][:]) stations['lat'] = np.squeeze(nc_file.variables['latitude'][:]) nstations = len(stations['name']) # Model output data (nfiles = number of timesnaps, 1 timesnap per file) nfiles = len(files) data = {} for var in variables: data[var] = np.empty((nfiles, nstations)) # Time information data['time'] = np.empty(nfiles) data['ref_date'] = nc_file.variables['time'].getncattr( 'units').replace('days since ', '') data['ref_date'] = datetime.datetime.strptime( data['ref_date'], '%Y-%m-%d %H:%M:%S') # Get time and output variables if j == 0: data['time'][i] = nc_file.variables['time'][:] for var in variables: if var in nc_file.variables: print(var) data[var][i][:] = nc_file.variables[var][:] data['datetime'], data['date'] = output_time_to_date( data['time'], data['ref_date'], start_date) return data, stations
def timestamps(self): if self.__timestamp_cache.get("timestamps") is None: for v in ['Times']: if v in self._dataset.variables: var = self._dataset.variables[v] break tz = pytz.timezone(var.time_zone) timestamps = np.array( map( lambda datestr: dateutil.parser.parse(datestr).replace(tzinfo=tz), chartostring(var[:]) ) ) timestamps.flags.writeable = False self.__timestamp_cache["timestamps"] = timestamps return self.__timestamp_cache.get("timestamps")
def timestamps(self): """ Loads, caches, and returns the time dimension from a dataset. """ if self.__timestamp_cache.get("timestamps") is None: for v in ['Times']: if v in self.nc_data.dataset.variables: var = self.nc_data.get_dataset_variable(v) break tz = pytz.timezone(var.time_zone) time_list = list( map( lambda dateStr: dateutil.parser.parse(dateStr).replace( tzinfo=tz), netcdf.chartostring(var[:]))) timestamps = np.array(time_list) timestamps.setflags(write=False) # Make immutable self.__timestamp_cache["timestamps"] = timestamps return self.__timestamp_cache.get("timestamps")
def openfile(self): # takes care of opening files print self.path+self.filename if (os.path.isfile(self.path+self.filename)): # file exists self.rootcdf = Dataset(self.path+self.filename, 'a') # open a dataset if self.rtype=='full': self.stns = chartostring(self.rootcdf.variables['station_name'][:]) self.stns = (np.ma.masked_array(self.stns, self.stns=='')).compressed() self.stntoid = dict(zip(self.stns,np.arange(0,len(self.stns)))) self.instodid.clear() self.obstoid.clear() else: self.rootcdf = Dataset(self.path+self.filename, 'w') # open a dataset self.newfile() self.stntoid.clear() self.instodid.clear() self.obstoid.clear()
def read_mmd(sensor, pathname): """ :type sensor: str :type pathname: str :rtype : tuple """ mmd = Dataset(pathname) mmd.set_auto_maskandscale(True) nx = len(mmd.dimensions['atsr.nx']) ny = len(mmd.dimensions['atsr.ny']) m_ids = mmd.variables['matchup.id'][:] mmd.variables[sensor + '.matchup_elem'].set_auto_maskandscale(False) mmd.variables[sensor + '.matchup_line'].set_auto_maskandscale(False) m_elems = mmd.variables['atsr.3.matchup_elem'][:] m_lines = mmd.variables['atsr.3.matchup_line'][:] m_source_filenames = chartostring(mmd.variables[sensor + '.l1b_filename'][:]) return mmd, nx, ny, m_ids, m_source_filenames, m_elems, m_lines
def getProfiles(full_path,SearchLimits,StartDate,EndDate,FillValue): s_lim, n_lim, w_lim, e_lim = SearchLimits[:] # Unpack SearchLimits. filename_list = glob.glob(full_path) # Search for files in full_path. matching_file_list = [] # Make an empty list to hold filenames. Argo_Dataframe = pd.DataFrame(columns=["PRES","PSAL","TEMPR","LAT","LON","JULD","ID"]) # Make an empty dataframe to hold profile data. for filename in filename_list: # For each file in full_path... nc = netCDF4.Dataset(filename) # Call the Dataset constructor. try: # Try to read the file. LAT = nc.variables['LATITUDE'][:] LON = nc.variables['LONGITUDE'][:] JULD = nc.variables['JULD'][:] # See which profiles have the correct lat,lon,date. LAT_good_loci = np.where((s_lim <= LAT) & (LAT <= n_lim)) LON_good_loci = np.where((w_lim <= LON) & (LON <= e_lim)) JULD_good_loci = np.where((StartDate <= JULD) & (JULD < (EndDate+1))) # (The "+1" is to include the EndDate.) loci_of_good_profiles = np.intersect1d(np.intersect1d(LAT_good_loci,LON_good_loci),JULD_good_loci) if (len(loci_of_good_profiles) > 0): # If there is at least one good profile in this file... matching_file_list.append(filename) # Record filename to list. TEMPR = nc.variables['TEMP_ADJUSTED'][:] # Read all profiles in the file. PSAL = nc.variables['PSAL_ADJUSTED'][:] PRES = nc.variables['PRES_ADJUSTED'][:] ID = netCDF4.chartostring(nc.variables['PLATFORM_NUMBER'][:]) ID = ID.astype(np.int) for locus in loci_of_good_profiles: # Write each good profile into temporary cell array... if not all(x == FillValue for x in PRES[locus]): # As long is profile is not just fill values. Profile_Dataframe = pd.DataFrame({"PRES":[PRES[locus]],"PSAL":[PSAL[locus]],"TEMPR":[TEMPR[locus]],"LAT":LAT[locus],"LON":LON[locus],"JULD":JULD[locus],"ID":ID[locus]},index=[0]) Argo_Dataframe= pd.concat([Argo_Dataframe, Profile_Dataframe],axis=0,sort=False) # Combine temporary with general dataframe. except Exception as error_message: # Throw an exception if unable to read file. print(error_message) nc.close() # Close the file. print('\nMatching files:\n'+'\n'.join(matching_file_list)) # Print a list of matching files. print(Argo_Dataframe) # Print the dataframe. print(Argo_Dataframe.shape) return(matching_file_list,Argo_Dataframe) # Return the list and the dataframe.
def __init__(self, data_file, errors_file=None, batch_size=32, tile_shape=(128,128), random_seed=None): self.data_file = data_file self.errors_file = errors_file self.batch_size = batch_size self.tile_shape = tile_shape self.img_shape = tile_shape # for compatibility self.ds = netCDF4.Dataset(data_file, 'r') self.N = self.ds["image"].shape[0] self.n_channels = self.ds["image"].shape[-1] self.image_shape = self.ds["image"].shape[1:3] self.timestamps = [str(ts) for ts in netCDF4.chartostring(self.ds["timestamp"][:])] with open(errors_file) as f: self.errors = json.load(f) self.prng = np.random.RandomState(seed=random_seed)
def read_data(self): with nc4.Dataset(self.filename) as f: grp_resp = f.groups['response_data'] resp = {} pts = {} skipvar = ["x", "y", "z", "time", "points"] t = grp_resp.variables['time'][:] pts_list = [] for point in nc4.chartostring(grp_resp.variables['points'][:]): pts_list.append(point) for ptid, x in enumerate(grp_resp.variables['x'][:]): y = grp_resp.variables['y'][:][ptid] z = grp_resp.variables['z'][:][ptid] pts[pts_list[ptid]] = (x, y, z) resp[pts_list[ptid]] = {} for var in grp_resp.variables: if not (var in skipvar): resp[pts_list[ptid]][var] = grp_resp.variables[ var][:, ptid] return (resp, t, pts)
def test_find_or_insert_stations( test_session_with_empty_db, tiny_dsg_dataset): var_name = tiny_dsg_dataset.dependent_varnames()[0] stations = find_or_insert_stations( test_session_with_empty_db, tiny_dsg_dataset, var_name) instance_dim = tiny_dsg_dataset.instance_dim(var_name) name = tiny_dsg_dataset.id_instance_var(var_name) lat = tiny_dsg_dataset.spatial_instance_var(var_name, 'X') lon = tiny_dsg_dataset.spatial_instance_var(var_name, 'Y') assert len(stations) == instance_dim.size for i, station in enumerate(stations): check_properties( station, name=str(chartostring(name[i])), x=lon[i], x_units=lon.units, y=lat[i], y_units=lat.units )
def read_line_data_header_variables(self, L): """ Reads the header data from a NetCDF file. Args: L (object): Line object Returns: none """ L.bank = self.nc.variables['Header.LineData.Bank'][0] L.v_source = self.nc.variables['Header.LineData.VSource'][0] L.v_system = self.nc.variables['Header.LineData.VSystem'][0] L.z_source = self.nc.variables['Header.LineData.ZSource'][0] L.frest = self.nc.variables['Header.LineData.LineRestFrequency'][0] L.dop_track = self.nc.variables['Header.LineData.DopplerTrack'][0] L.vbary = self.nc.variables['Header.LineData.VBarycenter'][0] L.vobs = self.nc.variables['Header.LineData.VObservatory'][0] L.fsky = self.nc.variables['Header.LineData.SkyFrequency'][0] L.flo1 = self.nc.variables['Header.LineData.LO1Frequency'][0] L.flo2 = self.nc.variables['Header.LineData.LO2Frequency'][0] L.fif1 = self.nc.variables['Header.LineData.IF1Frequency'][0] L.fif2 = self.nc.variables['Header.LineData.IF2Frequency'][0] L.fsyn = self.nc.variables['Header.LineData.SynthesizerFrequency'][0] L.foff = self.nc.variables['Header.LineData.FrequencyOffset'][0] L.voff = self.nc.variables['Header.LineData.VelocityOffset'][0] L.sideband = self.nc.variables['Header.LineData.Sideband'][0] L.nchan = self.nc.variables['Header.LineData.NChannels'][0] L.bw = self.nc.variables['Header.LineData.Bandwidth'][0] L.dfdc = self.nc.variables['Header.LineData.DeltaFrequency'][0] L.dvdc = self.nc.variables['Header.LineData.DeltaVelocity'][0] L.v0_lsr = self.nc.variables['Header.LineData.LSRFrameV0'][0] L.v0_bary = self.nc.variables['Header.LineData.BarycenterFrameV0'][0] L.v0_sky = self.nc.variables['Header.LineData.SkyFrameV0'][0] L.xtype = self.nc.variables['Header.LineData.XType'][0] L.iarray = self.nc.variables['Header.LineData.ChannelNumber'][:] # the following are the generic spectrum axis parameters L.ch0 = self.nc.variables['Header.SpectrumAxis.CRPIX'][0] L.xv0 = self.nc.variables['Header.SpectrumAxis.CRVAL'][0] L.dxdc = self.nc.variables['Header.SpectrumAxis.CDELT'][0] L.xname = netCDF4.chartostring( self.nc.variables['Header.SpectrumAxis.CTYPE'][:]) L.xarray = self.nc.variables['Header.SpectrumAxis.CAXIS'][:]
def test_hourly_aggregator_with_nonqc(self): output_file, bad_files = hourly_aggregator( files_to_aggregate=INPUT_FILES, site_code='NRSROT', qcflags=(0, 1, 2), input_dir=TEST_ROOT, output_dir='/tmp', download_url_prefix='http://test.download.url', opendap_url_prefix='http://test.opendap.url') self.assertRegex( output_file, r'IMOS_ANMN-NRS_BOSTUZ_20181213_NRSROT_FV02_hourly-timeseries-including-non-QC' r'_END-20190523_C-\d{8}\.nc') dataset = Dataset(output_file) self.assertEqual(dataset['source_file'].download_url_prefix, 'http://test.download.url') self.assertEqual(dataset['source_file'].opendap_url_prefix, 'http://test.opendap.url') for f in chartostring(dataset['source_file'][:]): self.assertIn(f, INPUT_FILES)
def readDSGfile(netCDFfiles): ds = Dataset(netCDFfiles[1], 'a') vs = ds.get_variables_by_attributes( standard_name='sea_water_pressure_due_to_sea_water') vs = ds.get_variables_by_attributes(long_name='actual depth') pres_var = vs[0] var_temp = ds.variables["TEMP"] plot_var = var_temp[:] plt.plot(plot_var) plt.show() time_var = ds.variables["TIME"] time = num2date(time_var[:], units=time_var.units, calendar=time_var.calendar) index_var = ds.variables["instrument_index"] idx = index_var[:] instrument_id_var = ds.variables["instrument_id"] #print(idx) i = 0 for x in chartostring(instrument_id_var[:]): print(i, x, time[idx == 1], plot_var[idx == i]) plt.plot(time[idx == i], plot_var[idx == i]) # , marker='.' i += 1 #plt.gca().invert_yaxis() plt.grid(True) # close the netCDF file ds.close() plt.show()
def fromNetCDF(self, ncfilename): nc_fid = netCDF4.Dataset(ncfilename, 'r') timestamp = datetime.strptime( \ nc_fid.getncattr( 'sliceCenterTimeUTC' ), \ "%Y-%m-%d_%H:%M:00" ) #print timestamp.isoformat() time_resol = timedelta( minutes = \ int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) ) stations = netCDF4.chartostring( \ nc_fid.variables[ 'stationId'][ : ] ) discharge = nc_fid.variables['discharge'][:] queryTime = nc_fid.variables['queryTime'][:] quality = nc_fid.variables['discharge_quality'][:] # for s, d, q in zip( stations, discharge, queryTime ): # print 'USGS.' + s.rstrip(), d, \ # datetime.utcfromtimestamp( q ).isoformat(), \ # datetime.utcfromtimestamp( q ).tzname(), \ # q # self.centralTimeStamp = timestamp # self.sliceTimeResolution = time_resol # self.obvStationTimeValue = [] stationTimeValue = [] for s, d, q, qual in zip(stations, discharge, queryTime, quality): stationTimeValue.append( \ ( 'USGS.' + s.strip(), \ datetime.utcfromtimestamp( q ), d, qual ) ) nc_fid.close() return self(timestamp, time_resol, stationTimeValue)
def read_file(netcdf_file_name): """Reads class-activation maps from NetCDF file. :param netcdf_file_name: Path to input file. :return: gradcam_dict: Dictionary with the following keys. gradcam_dict['class_activation_matrix']: See doc for `write_file`. gradcam_dict['example_id_strings']: Same. gradcam_dict['model_file_name']: Same. gradcam_dict['activation_layer_name']: Same. gradcam_dict['vector_output_layer_name']: Same. gradcam_dict['output_neuron_indices']: Same. gradcam_dict['ideal_activation']: Same. """ dataset_object = netCDF4.Dataset(netcdf_file_name) gradcam_dict = { CLASS_ACTIVATIONS_KEY: dataset_object.variables[CLASS_ACTIVATIONS_KEY][:], EXAMPLE_IDS_KEY: [ str(id) for id in netCDF4.chartostring( dataset_object.variables[EXAMPLE_IDS_KEY][:]) ], MODEL_FILE_KEY: str(getattr(dataset_object, MODEL_FILE_KEY)), ACTIVATION_LAYER_KEY: str(getattr(dataset_object, ACTIVATION_LAYER_KEY)), VECTOR_OUT_LAYER_KEY: str(getattr(dataset_object, VECTOR_OUT_LAYER_KEY)), OUTPUT_NEURONS_KEY: numpy.array(getattr(dataset_object, OUTPUT_NEURONS_KEY), dtype=int), IDEAL_ACTIVATION_KEY: getattr(dataset_object, IDEAL_ACTIVATION_KEY) } dataset_object.close() return gradcam_dict
def openfile(self): # takes care of opening files print self.path + self.filename if (os.path.isfile(self.path + self.filename)): # file exists self.rootcdf = Dataset(self.path + self.filename, 'a') # open a dataset if self.rtype == 'full': self.stns = chartostring( self.rootcdf.variables['station_name'][:]) self.stns = (np.ma.masked_array(self.stns, self.stns == '')).compressed() self.stntoid = dict( zip(self.stns, np.arange(0, len(self.stns)))) self.instodid.clear() self.obstoid.clear() else: self.rootcdf = Dataset(self.path + self.filename, 'w') # open a dataset self.newfile() self.stntoid.clear() self.instodid.clear() self.obstoid.clear()
def process(ncfn): """Process this file """ pgconn = get_dbconn('iem') icursor = pgconn.cursor() xref = {} icursor.execute("""SELECT id, network from stations where network ~* 'ASOS' or network = 'AWOS' and country = 'US'""") for row in icursor: xref[row[0]] = row[1] icursor.close() nc = ncopen(ncfn) data = {} for vname in [ 'stationId', 'observationTime', 'temperature', 'dewpoint', 'altimeter', # Pa 'windDir', 'windSpeed', # mps 'windGust', 'visibility', # m 'precipAccum', 'presWx', 'skyCvr', 'skyCovLayerBase', 'autoRemark', 'operatorRemark' ]: data[vname] = nc.variables[vname][:] for qc in ['QCR', 'QCD']: vname2 = vname + qc if vname2 in nc.variables: data[vname2] = nc.variables[vname2][:] for vname in ['temperature', 'dewpoint']: data[vname + "C"] = temperature(data[vname], 'K').value('C') data[vname] = temperature(data[vname], 'K').value('F') for vname in ['windSpeed', 'windGust']: data[vname] = speed(data[vname], 'MPS').value('KT') data['altimeter'] = pressure(data['altimeter'], 'PA').value("IN") data['skyCovLayerBase'] = distance(data['skyCovLayerBase'], 'M').value("FT") data['visibility'] = distance(data['visibility'], 'M').value("MI") data['precipAccum'] = distance(data['precipAccum'], 'MM').value("IN") stations = chartostring(data['stationId'][:]) presentwxs = chartostring(data['presWx'][:]) skycs = chartostring(data['skyCvr'][:]) autoremarks = chartostring(data['autoRemark'][:]) opremarks = chartostring(data['operatorRemark'][:]) def decision(i, fieldname, tolerance): """Our decision if we are going to take a HFMETAR value or not""" if data[fieldname][i] is np.ma.masked: return None if data["%sQCR" % (fieldname, )][i] == 0: return data[fieldname][i] # Now we have work to do departure = np.ma.max(np.ma.abs(data['%sQCD' % (fieldname, )][i, :])) # print("departure: %s tolerance: %s" % (departure, tolerance)) if departure <= tolerance: return data[fieldname][i] return None for i, sid in tqdm(enumerate(stations), total=len(stations), disable=(not sys.stdout.isatty())): sid3 = sid[1:] if sid[0] == 'K' else sid ts = datetime.datetime(1970, 1, 1) + datetime.timedelta( seconds=data['observationTime'][i]) ts = ts.replace(tzinfo=pytz.utc) mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M")) network = xref.get(sid3, 'ASOS') iem = Observation(sid3, network, ts) # 06019G23KT val = decision(i, 'windDir', 15) if val is not None: iem.data['drct'] = int(val) mtr += "%03i" % (iem.data['drct'], ) else: mtr += "///" val = decision(i, 'windSpeed', 10) if val is not None: iem.data['sknt'] = int(val) mtr += "%02i" % (iem.data['sknt'], ) else: mtr += "//" val = decision(i, 'windGust', 10) if val is not None and val > 0: iem.data['gust'] = int(val) mtr += "G%02i" % (iem.data['gust'], ) mtr += "KT " val = decision(i, 'visibility', 4) if val is not None: iem.data['vsby'] = float(val) mtr += "%sSM " % (vsbyfmt(iem.data['vsby']), ) presentwx = presentwxs[i] if presentwx != '': # database storage is comma delimited iem.data['wxcodes'] = presentwx.split(" ") mtr += "%s " % (presentwx, ) for _i, (skyc, _l) in enumerate(zip(skycs[i], data['skyCovLayerBase'][i])): if skyc != '': iem.data['skyc%s' % (_i + 1, )] = skyc if skyc != 'CLR': iem.data['skyl%s' % (_i + 1, )] = int(_l) mtr += "%s%03i " % (skyc, int(_l) / 100) else: mtr += "CLR " t = "" tgroup = "T" val = decision(i, 'temperature', 10) if val is not None: # Recall the pain enabling this # iem.data['tmpf'] = float(data['temperature'][i]) tmpc = float(data['temperatureC'][i]) t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else (0 - tmpc)) tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.) val = decision(i, 'dewpoint', 10) if val is not None: # iem.data['dwpf'] = float(data['dewpoint'][i]) tmpc = float(data['dewpointC'][i]) if t != "": t = "%s%s%02i " % (t, "M" if tmpc < 0 else "", tmpc if tmpc > 0 else 0 - tmpc) tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.) if len(t) > 4: mtr += t val = decision(i, 'altimeter', 20) if val is not None: iem.data['alti'] = float(round(val, 2)) mtr += "A%4i " % (iem.data['alti'] * 100., ) mtr += "RMK " val = decision(i, 'precipAccum', 25) if val is not None: if val >= 0.01: iem.data['phour'] = float(round(val, 2)) mtr += "P%04i " % (iem.data['phour'] * 100., ) elif val < 0.01 and val > 0: # Trace mtr += "P0000 " iem.data['phour'] = TRACE_VALUE if tgroup != "T": mtr += "%s " % (tgroup, ) if autoremarks[i] != '' or opremarks[i] != '': mtr += "%s %s " % (autoremarks[i], opremarks[i]) mtr += "MADISHF" # Eat our own dogfood try: Metar.Metar(mtr) iem.data['raw'] = mtr except Exception as exp: print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp)) continue for key in iem.data: if isinstance(iem.data[key], np.float32): print("key: %s type: %s" % (key, type(iem.data[key]))) icursor = pgconn.cursor() if not iem.save(icursor, force_current_log=True, skip_current=True): print(("extract_hfmetar: unknown station? %s %s %s\n%s") % (sid3, network, ts, mtr)) icursor.close() pgconn.commit()
stationobs_units['temp_sounding'] = stringtoarr('Kelvin',NUMCHARS) stationobs_units['press_sounding'] = stringtoarr('hPa',NUMCHARS) statdat.units = stationobs_units # close and reopen the file. f.close() f = Dataset('compound_example.nc') print f statdat = f.variables['station_obs'] # print out data in variable. print 'data in a variable of compound type:' print '----' for data in statdat[:]: for name in statdat.dtype.names: if data[name].dtype.kind == 'S': # a string # convert array of characters back to a string for display. print name,': value =',chartostring(data[name]),\ ': units=',chartostring(statdat.units[name]) elif data[name].dtype.kind == 'V': # a nested compound type print name,data[name].dtype.names,': value=',data[name],': units=',\ tuple([chartostring(s) for s in tuple(statdat.units[name])]) else: # a numeric type. print name,': value=',data[name],': units=',chartostring(statdat.units[name]) print '----' f.close() f = Dataset('tst_vlen.nc','w') vlen_t = f.createVLType(numpy.int32, 'phony_vlen') x = f.createDimension('x',3) y = f.createDimension('y',4) vlvar = f.createVariable('phony_vlen_var', vlen_t, ('y','x')) import random
def process(ncfn): """Process this file """ pgconn = get_dbconn("iem") icursor = pgconn.cursor() xref = {} icursor.execute("SELECT id, network from stations where " "network ~* 'ASOS' or network = 'AWOS' and country = 'US'") for row in icursor: xref[row[0]] = row[1] icursor.close() nc = ncopen(ncfn) data = {} for vname in [ "stationId", "observationTime", "temperature", "dewpoint", "altimeter", # Pa "windDir", "windSpeed", # mps "windGust", # mps "visibility", # m "precipAccum", "presWx", "skyCvr", "skyCovLayerBase", "autoRemark", "operatorRemark", ]: data[vname] = nc.variables[vname][:] for qc in ["QCR", "QCD"]: vname2 = vname + qc if vname2 in nc.variables: data[vname2] = nc.variables[vname2][:] for vname in ["temperature", "dewpoint"]: data[vname + "C"] = temperature(data[vname], "K").value("C") data[vname] = temperature(data[vname], "K").value("F") for vname in ["windSpeed", "windGust"]: data[vname] = (masked_array(data[vname], units("meter / second")).to( units("knots")).magnitude) data["altimeter"] = pressure(data["altimeter"], "PA").value("IN") data["skyCovLayerBase"] = distance(data["skyCovLayerBase"], "M").value("FT") data["visibility"] = distance(data["visibility"], "M").value("MI") data["precipAccum"] = distance(data["precipAccum"], "MM").value("IN") stations = chartostring(data["stationId"][:]) presentwxs = chartostring(data["presWx"][:]) skycs = chartostring(data["skyCvr"][:]) autoremarks = chartostring(data["autoRemark"][:]) opremarks = chartostring(data["operatorRemark"][:]) def decision(i, fieldname, tolerance): """Our decision if we are going to take a HFMETAR value or not""" if data[fieldname][i] is np.ma.masked: return None if data["%sQCR" % (fieldname, )][i] == 0: return data[fieldname][i] # Now we have work to do departure = np.ma.max(np.ma.abs(data["%sQCD" % (fieldname, )][i, :])) # print("departure: %s tolerance: %s" % (departure, tolerance)) if departure <= tolerance: return data[fieldname][i] return None for i, sid in tqdm( enumerate(stations), total=len(stations), disable=(not sys.stdout.isatty()), ): if len(sid) < 3: continue sid3 = sid[1:] if sid.startswith("K") else sid ts = datetime.datetime(1970, 1, 1) + datetime.timedelta( seconds=data["observationTime"][i]) ts = ts.replace(tzinfo=pytz.UTC) mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M")) network = xref.get(sid3, "ASOS") iem = Observation(sid3, network, ts) # 06019G23KT val = decision(i, "windDir", 15) if val is not None: iem.data["drct"] = int(val) mtr += "%03i" % (iem.data["drct"], ) else: mtr += "///" val = decision(i, "windSpeed", 10) if val is not None: iem.data["sknt"] = int(val) mtr += "%02i" % (iem.data["sknt"], ) else: mtr += "//" val = decision(i, "windGust", 10) if val is not None and val > 0: iem.data["gust"] = int(val) mtr += "G%02i" % (iem.data["gust"], ) mtr += "KT " val = decision(i, "visibility", 4) if val is not None: iem.data["vsby"] = float(val) mtr += "%sSM " % (vsbyfmt(iem.data["vsby"]), ) presentwx = presentwxs[i] if presentwx != "": # database storage is comma delimited iem.data["wxcodes"] = presentwx.split(" ") mtr += "%s " % (presentwx, ) for _i, (skyc, _l) in enumerate(zip(skycs[i], data["skyCovLayerBase"][i])): if skyc != "": iem.data["skyc%s" % (_i + 1, )] = skyc if skyc != "CLR": iem.data["skyl%s" % (_i + 1, )] = int(_l) mtr += "%s%03i " % (skyc, int(_l) / 100) else: mtr += "CLR " t = "" tgroup = "T" val = decision(i, "temperature", 10) if val is not None: # Recall the pain enabling this # iem.data['tmpf'] = float(data['temperature'][i]) tmpc = float(data["temperatureC"][i]) t = "%s%02i/" % ( "M" if tmpc < 0 else "", tmpc if tmpc > 0 else (0 - tmpc), ) tgroup += "%s%03i" % ( "1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.0, ) val = decision(i, "dewpoint", 10) if val is not None: # iem.data['dwpf'] = float(data['dewpoint'][i]) tmpc = float(data["dewpointC"][i]) if t != "": t = "%s%s%02i " % ( t, "M" if tmpc < 0 else "", tmpc if tmpc > 0 else 0 - tmpc, ) tgroup += "%s%03i" % ( "1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.0, ) if len(t) > 4: mtr += t val = decision(i, "altimeter", 20) if val is not None: iem.data["alti"] = float(round(val, 2)) mtr += "A%4i " % (iem.data["alti"] * 100.0, ) mtr += "RMK " val = decision(i, "precipAccum", 25) if val is not None: if val > 0.009: iem.data["phour"] = float(round(val, 2)) mtr += "P%04i " % (iem.data["phour"] * 100.0, ) elif val > 0: # Trace mtr += "P0000 " iem.data["phour"] = TRACE_VALUE if tgroup != "T": mtr += "%s " % (tgroup, ) if autoremarks[i] != "" or opremarks[i] != "": mtr += "%s %s " % (autoremarks[i], opremarks[i]) mtr += "MADISHF" # Eat our own dogfood try: Metar.Metar(mtr) iem.data["raw"] = mtr except Exception as exp: print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp)) continue for key in iem.data: if isinstance(iem.data[key], np.float32): print("key: %s type: %s" % (key, type(iem.data[key]))) icursor = pgconn.cursor() if not iem.save(icursor, force_current_log=True, skip_current=True): print(("extract_hfmetar: unknown station? %s %s %s\n%s") % (sid3, network, ts, mtr)) icursor.close() pgconn.commit()
print 'Basal homologous temperature at divide (deg C) = {}'.format(basalTemp[ind][0]-basalPmpTemp[ind][0]) print 'EISMINT models basal temperature at divide (m):' print ' 3d models (6 of them): -13.34 +/- 0.56' print '====================================' print '' # Plot the results fig = plt.figure(1, facecolor='w') markersize = 30.0 fig.add_subplot(1,1,1) plt.scatter(xCell,yCell,markersize,thk[:], marker='h', edgecolors='none') plt.colorbar() plt.axis('equal') plt.title('Modeled thickness (m) \n at time ' + netCDF4.chartostring(xtime)[timelev].strip() ) plt.draw() if options.saveimage: plotname = 'halfar-results.png' plt.savefig(plotname, dpi=150) print 'Saved plot as ' + plotname if options.hidefigs: print "Plot display disabled with -n argument." else: print 'Showing plot... Close plot window to exit.' plt.show()
def read_cfradial(filename, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, include_fields=None, delay_field_loading=False, **kwargs): """ Read a Cfradial netCDF file. Parameters ---------- filename : str Name of CF/Radial netCDF file to read data from. field_names : dict, optional Dictionary mapping field names in the file names to radar field names. Unlike other read functions, fields not in this dictionary or having a value of None are still included in the radar.fields dictionary, to exclude them use the `exclude_fields` parameter. Fields which are mapped by this dictionary will be renamed from key to value. additional_metadata : dict of dicts, optional This parameter is not used, it is included for uniformity. file_field_names : bool, optional True to force the use of the field names from the file in which case the `field_names` parameter is ignored. False will use to `field_names` parameter to rename fields. exclude_fields : list or None, optional List of fields to exclude from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields specified by include_fields. include_fields : list or None, optional List of fields to include from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields not specified by exclude_fields. delay_field_loading : bool True to delay loading of field data from the file until the 'data' key in a particular field dictionary is accessed. In this case the field attribute of the returned Radar object will contain LazyLoadDict objects not dict objects. Delayed field loading will not provide any speedup in file where the number of gates vary between rays (ngates_vary=True) and is not recommended. Returns ------- radar : Radar Radar object. Notes ----- This function has not been tested on "stream" Cfradial files. """ # test for non empty kwargs _test_arguments(kwargs) # create metadata retrieval object filemetadata = FileMetadata('cfradial', field_names, additional_metadata, file_field_names, exclude_fields) # read the data ncobj = netCDF4.Dataset(filename) ncvars = ncobj.variables # 4.1 Global attribute -> move to metadata dictionary metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()]) if 'n_gates_vary' in metadata: metadata['n_gates_vary'] = 'false' # corrected below # 4.2 Dimensions (do nothing) # 4.3 Global variable -> move to metadata dictionary if 'volume_number' in ncvars: metadata['volume_number'] = int(ncvars['volume_number'][:]) else: metadata['volume_number'] = 0 global_vars = { 'platform_type': 'fixed', 'instrument_type': 'radar', 'primary_axis': 'axis_z' } # ignore time_* global variables, these are calculated from the time # variable when the file is written. for var, default_value in global_vars.items(): if var in ncvars: metadata[var] = str(netCDF4.chartostring(ncvars[var][:])) else: metadata[var] = default_value # 4.4 coordinate variables -> create attribute dictionaries time = _ncvar_to_dict(ncvars['time']) _range = _ncvar_to_dict(ncvars['range']) # 4.5 Ray dimension variables # 4.6 Location variables -> create attribute dictionaries latitude = _ncvar_to_dict(ncvars['latitude']) longitude = _ncvar_to_dict(ncvars['longitude']) altitude = _ncvar_to_dict(ncvars['altitude']) if 'altitude_agl' in ncvars: altitude_agl = _ncvar_to_dict(ncvars['altitude_agl']) else: altitude_agl = None # 4.7 Sweep variables -> create atrribute dictionaries sweep_mode = _ncvar_to_dict(ncvars['sweep_mode']) fixed_angle = _ncvar_to_dict(ncvars['fixed_angle']) sweep_start_ray_index = _ncvar_to_dict(ncvars['sweep_start_ray_index']) sweep_end_ray_index = _ncvar_to_dict(ncvars['sweep_end_ray_index']) if 'sweep_number' in ncvars: sweep_number = _ncvar_to_dict(ncvars['sweep_number']) else: nsweeps = len(sweep_start_ray_index['data']) sweep_number = filemetadata('sweep_number') sweep_number['data'] = np.arange(nsweeps, dtype='float32') warnings.warn("Warning: File violates CF/Radial convention. " + "Missing sweep_number variable") if 'target_scan_rate' in ncvars: target_scan_rate = _ncvar_to_dict(ncvars['target_scan_rate']) else: target_scan_rate = None if 'rays_are_indexed' in ncvars: rays_are_indexed = _ncvar_to_dict(ncvars['rays_are_indexed']) else: rays_are_indexed = None if 'ray_angle_res' in ncvars: ray_angle_res = _ncvar_to_dict(ncvars['ray_angle_res']) else: ray_angle_res = None # first sweep mode determines scan_type try: mode = netCDF4.chartostring(sweep_mode['data'][0])[()].decode('utf-8') except AttributeError: # Python 3, all strings are already unicode. mode = netCDF4.chartostring(sweep_mode['data'][0])[()] # options specified in the CF/Radial standard if mode == 'rhi': scan_type = 'rhi' elif mode == 'vertical_pointing': scan_type = 'vpt' elif mode == 'azimuth_surveillance': scan_type = 'ppi' elif mode == 'elevation_surveillance': scan_type = 'rhi' elif mode == 'manual_ppi': scan_type = 'ppi' elif mode == 'manual_rhi': scan_type = 'rhi' # fallback types elif 'sur' in mode: scan_type = 'ppi' elif 'sec' in mode: scan_type = 'sector' elif 'rhi' in mode: scan_type = 'rhi' elif 'ppi' in mode: scan_type = 'ppi' else: scan_type = 'other' # 4.8 Sensor pointing variables -> create attribute dictionaries azimuth = _ncvar_to_dict(ncvars['azimuth']) elevation = _ncvar_to_dict(ncvars['elevation']) if 'scan_rate' in ncvars: scan_rate = _ncvar_to_dict(ncvars['scan_rate']) else: scan_rate = None if 'antenna_transition' in ncvars: antenna_transition = _ncvar_to_dict(ncvars['antenna_transition']) else: antenna_transition = None # 4.9 Moving platform geo-reference variables # Aircraft specific varaibles if 'rotation' in ncvars: rotation = _ncvar_to_dict(ncvars['rotation']) else: rotation = None if 'tilt' in ncvars: tilt = _ncvar_to_dict(ncvars['tilt']) else: tilt = None if 'roll' in ncvars: roll = _ncvar_to_dict(ncvars['roll']) else: roll = None if 'drift' in ncvars: drift = _ncvar_to_dict(ncvars['drift']) else: drift = None if 'heading' in ncvars: heading = _ncvar_to_dict(ncvars['heading']) else: heading = None if 'pitch' in ncvars: pitch = _ncvar_to_dict(ncvars['pitch']) else: pitch = None if 'georefs_applied' in ncvars: georefs_applied = _ncvar_to_dict(ncvars['georefs_applied']) else: georefs_applied = None # 4.10 Moments field data variables -> field attribute dictionary if 'ray_n_gates' in ncvars: # all variables with dimensions of n_points are fields. keys = [k for k, v in ncvars.items() if v.dimensions == ('n_points', )] else: # all variables with dimensions of 'time', 'range' are fields keys = [ k for k, v in ncvars.items() if v.dimensions == ('time', 'range') ] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) if field_name is None: if exclude_fields is not None and key in exclude_fields: if key not in include_fields: continue if include_fields is None or key in include_fields: field_name = key else: continue fields[field_name] = _ncvar_to_dict(ncvars[key], delay_field_loading) if 'ray_n_gates' in ncvars: shape = (len(ncvars['time']), len(ncvars['range'])) ray_n_gates = ncvars['ray_n_gates'][:] ray_start_index = ncvars['ray_start_index'][:] for dic in fields.values(): _unpack_variable_gate_field_dic(dic, shape, ray_n_gates, ray_start_index) # 4.5 instrument_parameters sub-convention -> instrument_parameters dict # 4.6 radar_parameters sub-convention -> instrument_parameters dict keys = [k for k in _INSTRUMENT_PARAMS_DIMS.keys() if k in ncvars] instrument_parameters = dict((k, _ncvar_to_dict(ncvars[k])) for k in keys) if instrument_parameters == {}: # if no parameters set to None instrument_parameters = None # 4.7 lidar_parameters sub-convention -> skip # 4.8 radar_calibration sub-convention -> radar_calibration keys = _find_all_meta_group_vars(ncvars, 'radar_calibration') radar_calibration = dict((k, _ncvar_to_dict(ncvars[k])) for k in keys) if radar_calibration == {}: radar_calibration = None # do not close file is field loading is delayed if not delay_field_loading: ncobj.close() return Radar(time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters, radar_calibration=radar_calibration, altitude_agl=altitude_agl, scan_rate=scan_rate, antenna_transition=antenna_transition, target_scan_rate=target_scan_rate, rays_are_indexed=rays_are_indexed, ray_angle_res=ray_angle_res, rotation=rotation, tilt=tilt, roll=roll, drift=drift, heading=heading, pitch=pitch, georefs_applied=georefs_applied)
def ohc_timeseries(config, streamMap=None, variableMap=None): """ Performs analysis of ocean heat content (OHC) from time-series output. config is an instance of an MpasAnalysisConfigParser containing configuration options. config is an instance of MpasAnalysisConfigParser containing configuration options. If present, streamMap is a dictionary of MPAS-O stream names that map to their mpas_analysis counterparts. If present, variableMap is a dictionary of MPAS-O variable names that map to their mpas_analysis counterparts. Author: Xylar Asay-Davis, Milena Veneziani Last Modified: 01/07/2017 """ # read parameters from config file casename = config.get('case', 'casename') ref_casename_v0 = config.get('case', 'ref_casename_v0') indir_v0data = config.get('paths', 'ref_archive_v0_ocndir') compare_with_obs = config.getboolean('ohc_timeseries', 'compare_with_obs') plots_dir = config.get('paths', 'plots_dir') yr_offset = config.getint('time', 'yr_offset') N_movavg = config.getint('ohc_timeseries', 'N_movavg') regions = config.getExpression('regions', 'regions') plot_titles = config.getExpression('regions', 'plot_titles') iregions = config.getExpression('ohc_timeseries', 'regionIndicesToPlot') indir = config.get('paths', 'archive_dir_ocn') namelist_filename = config.get('input', 'ocean_namelist_filename') namelist = NameList(namelist_filename, path=indir) streams_filename = config.get('input', 'ocean_streams_filename') streams = StreamsFile(streams_filename, streamsdir=indir) # Note: input file, not a mesh file because we need dycore specific fields # such as refBottomDepth and namelist fields such as config_density0, as # well as simulationStartTime, that are not guaranteed to be in the mesh file. try: inputfile = streams.readpath('restart')[0] except ValueError: raise IOError( 'No MPAS-O restart file found: need at least one restart file for OHC calculation' ) # get a list of timeSeriesStats output files from the streams file, # reading only those that are between the start and end dates startDate = config.get('time', 'timeseries_start_date') endDate = config.get('time', 'timeseries_end_date') streamName = streams.find_stream(streamMap['timeSeriesStats']) infiles = streams.readpath(streamName, startDate=startDate, endDate=endDate) print 'Reading files {} through {}'.format(infiles[0], infiles[-1]) # Define/read in general variables print ' Read in depth and compute specific depth indexes...' f = netcdf_dataset(inputfile, mode='r') # reference depth [m] depth = f.variables['refBottomDepth'][:] # simulation start time simStartTime = netCDF4.chartostring(f.variables['simulationStartTime'][:]) simStartTime = str(simStartTime) f.close() # specific heat [J/(kg*degC)] cp = namelist.getfloat('config_specific_heat_sea_water') # [kg/m3] rho = namelist.getfloat('config_density0') fac = 1e-22 * rho * cp k700m = np.where(depth > 700.)[0][0] - 1 k2000m = np.where(depth > 2000.)[0][0] - 1 kbtm = len(depth) - 1 # Load data print ' Load ocean data...' varList = [ 'avgLayerTemperature', 'sumLayerMaskValue', 'avgLayerArea', 'avgLayerThickness' ] ds = xr.open_mfdataset( infiles, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, varmap=variableMap)) ds = remove_repeated_time_index(ds) # convert the start and end dates to datetime objects using # the Date class, which ensures the results are within the # supported range time_start = Date(startDate).to_datetime(yr_offset) time_end = Date(endDate).to_datetime(yr_offset) # select only the data in the specified range of years ds = ds.sel(Time=slice(time_start, time_end)) # Select year-1 data and average it (for later computing anomalies) time_start_yr1 = Date(simStartTime).to_datetime(yr_offset) if time_start_yr1 < time_start: startDate_yr1 = simStartTime endDate_yr1 = startDate_yr1[0:5] + '12-31' + startDate_yr1[10:] infiles_yr1 = streams.readpath(streamName, startDate=startDate_yr1, endDate=endDate_yr1) ds_yr1 = xr.open_mfdataset( infiles_yr1, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, varmap=variableMap)) ds_yr1 = remove_repeated_time_index(ds_yr1) else: time_start = datetime.datetime(time_start.year, 1, 1) time_end = datetime.datetime(time_start.year, 12, 31) ds_yr1 = ds.sel(Time=slice(time_start, time_end)) mean_yr1 = ds_yr1.mean('Time') print ' Compute temperature anomalies...' avgLayerTemperature = ds.avgLayerTemperature avgLayerTemperature_yr1 = mean_yr1.avgLayerTemperature avgLayTemp_anomaly = avgLayerTemperature - avgLayerTemperature_yr1 year_start = (pd.to_datetime(ds.Time.min().values)).year year_end = (pd.to_datetime(ds.Time.max().values)).year time_start = datetime.datetime(year_start, 1, 1) time_end = datetime.datetime(year_end, 12, 31) if ref_casename_v0 != 'None': print ' Load in OHC for ACMEv0 case...' infiles_v0data = '{}/OHC.{}.year*.nc'.format(indir_v0data, ref_casename_v0) ds_v0 = xr.open_mfdataset( infiles_v0data, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset)) ds_v0 = remove_repeated_time_index(ds_v0) year_end_v0 = (pd.to_datetime(ds_v0.Time.max().values)).year if year_start <= year_end_v0: ds_v0_tslice = ds_v0.sel(Time=slice(time_start, time_end)) else: print ' Warning: v0 time series lies outside current bounds of v1 time series. Skipping it.' ref_casename_v0 = 'None' sumLayerMaskValue = ds.sumLayerMaskValue avgLayerArea = ds.avgLayerArea avgLayerThickness = ds.avgLayerThickness print ' Compute OHC and make plots...' for index in range(len(iregions)): iregion = iregions[index] # Compute volume of each layer in the region: layerArea = sumLayerMaskValue[:, iregion, :] * \ avgLayerArea[:, iregion, :] layerVolume = layerArea * avgLayerThickness[:, iregion, :] # Compute OHC: ohc = layerVolume * avgLayTemp_anomaly[:, iregion, :] # OHC over 0-bottom depth range: ohc_tot = ohc.sum('nVertLevels') ohc_tot = fac * ohc_tot # OHC over 0-700m depth range: ohc_700m = fac * ohc[:, 0:k700m].sum('nVertLevels') # OHC over 700m-2000m depth range: ohc_2000m = fac * ohc[:, k700m + 1:k2000m].sum('nVertLevels') # OHC over 2000m-bottom depth range: ohc_btm = ohc[:, k2000m + 1:kbtm].sum('nVertLevels') ohc_btm = fac * ohc_btm title = 'OHC, {}, 0-bottom (thick-), 0-700m (thin-), 700-2000m (--),' \ ' 2000m-bottom (-.) \n {}'.format(plot_titles[iregion], casename) xlabel = 'Time [years]' ylabel = '[x$10^{22}$ J]' if ref_casename_v0 != 'None': figname = '{}/ohc_{}_{}_{}.png'.format(plots_dir, regions[iregion], casename, ref_casename_v0) ohc_v0_tot = ds_v0_tslice.ohc_tot ohc_v0_700m = ds_v0_tslice.ohc_700m ohc_v0_2000m = ds_v0_tslice.ohc_2000m ohc_v0_btm = ds_v0_tslice.ohc_btm title = '{} (r), {} (b)'.format(title, ref_casename_v0) timeseries_analysis_plot( config, [ ohc_tot, ohc_700m, ohc_2000m, ohc_btm, ohc_v0_tot, ohc_v0_700m, ohc_v0_2000m, ohc_v0_btm ], N_movavg, title, xlabel, ylabel, figname, lineStyles=[ 'r-', 'r-', 'r--', 'r-.', 'b-', 'b-', 'b--', 'b-.' ], lineWidths=[2, 1, 1.5, 1.5, 2, 1, 1.5, 1.5]) if not compare_with_obs and ref_casename_v0 == 'None': figname = '{}/ohc_{}_{}.png'.format(plots_dir, regions[iregion], casename) timeseries_analysis_plot(config, [ohc_tot, ohc_700m, ohc_2000m, ohc_btm], N_movavg, title, xlabel, ylabel, figname, lineStyles=['r-', 'r-', 'r--', 'r-.'], lineWidths=[2, 1, 1.5, 1.5])
def main(): """Go Main Go""" pgconn = get_dbconn('iem') icursor = pgconn.cursor() now = datetime.datetime.utcnow() - datetime.timedelta(hours=3) fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"), ) table = "current_qc" if not os.path.isfile(fn): sys.exit() nc = ncopen(fn) ids = chartostring(nc.variables["stationName"][:]) nc_tmpk = nc.variables["temperature"] nc_dwpk = nc.variables["dewpoint"] nc_alti = nc.variables["altimeter"] tmpkqcd = nc.variables["temperatureQCD"] dwpkqcd = nc.variables["dewpointQCD"] altiqcd = nc.variables["altimeterQCD"] for j in range(ids.shape[0]): sid = ids[j] if len(sid) < 4: continue if sid[0] == "K": ts = datetime.datetime(1970, 1, 1) + datetime.timedelta( seconds=int(nc.variables["timeObs"][j])) ts = ts.replace(tzinfo=pytz.utc) (tmpf, tmpf_qc_av, tmpf_qc_sc) = ('Null', 'Null', 'Null') (dwpf, dwpf_qc_av, dwpf_qc_sc) = ('Null', 'Null', 'Null') (alti, alti_qc_av, alti_qc_sc) = ('Null', 'Null', 'Null') if (not np.ma.is_masked(nc_tmpk[j]) and not np.ma.is_masked(tmpkqcd[j, 0]) and not np.ma.is_masked(tmpkqcd[j, 6])): tmpf = check(temperature(nc_tmpk[j], 'K').value('F')) tmpf_qc_av = figure(nc_tmpk[j], tmpkqcd[j, 0]) tmpf_qc_sc = figure(nc_tmpk[j], tmpkqcd[j, 6]) if (not np.ma.is_masked(nc_dwpk[j]) and not np.ma.is_masked(dwpkqcd[j, 0]) and not np.ma.is_masked(dwpkqcd[j, 6])): dwpf = check(temperature(nc_dwpk[j], 'K').value('F')) dwpf_qc_av = figure(nc_dwpk[j], dwpkqcd[j, 0]) dwpf_qc_sc = figure(nc_dwpk[j], dwpkqcd[j, 6]) if not np.ma.is_masked(nc_alti[j]): alti = check(nc_alti[j] / 100.0 * 0.0295298) alti_qc_av = figure_alti(alti, altiqcd[j, 0] * 0.0295298) alti_qc_sc = figure_alti(alti, altiqcd[j, 6] * 0.0295298) sql = """ UPDATE %s SET tmpf = %s, tmpf_qc_av = %s, tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s, dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s, alti_qc_sc = %s, valid = '%s' WHERE station = '%s' """ % (table, tmpf, tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av, dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc, ts.strftime("%Y-%m-%d %H:%M+00"), sid[1:]) sql = sql.replace("--", "Null").replace("nan", "Null") try: icursor.execute(sql) except Exception as exp: print(exp) print(sql) nc.close() icursor.close() pgconn.commit() pgconn.close()
def read_kazr(filename, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, include_fields=None): """ Read K-band ARM Zenith Radar (KAZR) NetCDF ingest data. Parameters ---------- filename : str Name of NetCDF file to read data from. field_names : dict, optional Dictionary mapping field names in the file names to radar field names. Unlike other read functions, fields not in this dictionary or having a value of None are still included in the radar.fields dictionary, to exclude them use the `exclude_fields` parameter. Fields which are mapped by this dictionary will be renamed from key to value. additional_metadata : dict of dicts, optional This parameter is not used, it is included for uniformity. file_field_names : bool, optional True to force the use of the field names from the file in which case the `field_names` parameter is ignored. False will use to `field_names` parameter to rename fields. exclude_fields : list or None, optional List of fields to exclude from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields specified by include_fields. include_fields : list or None, optional List of fields to include from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields not specified by exclude_fields. Returns ------- radar : Radar Radar object. """ # create metadata retrieval object filemetadata = FileMetadata( 'cfradial', field_names, additional_metadata, file_field_names, exclude_fields, include_fields) # read the data ncobj = netCDF4.Dataset(filename) ncvars = ncobj.variables # 4.1 Global attribute -> move to metadata dictionary metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()]) metadata['n_gates_vary'] = 'false' # 4.2 Dimensions (do nothing) # 4.3 Global variable -> move to metadata dictionary if 'volume_number' in ncvars: metadata['volume_number'] = int(ncvars['volume_number'][:]) else: metadata['volume_number'] = 0 global_vars = {'platform_type': 'fixed', 'instrument_type': 'radar', 'primary_axis': 'axis_z'} # ignore time_* global variables, these are calculated from the time # variable when the file is written. for var, default_value in global_vars.items(): if var in ncvars: metadata[var] = str(netCDF4.chartostring(ncvars[var][:])) else: metadata[var] = default_value # 4.4 coordinate variables -> create attribute dictionaries time = cfradial._ncvar_to_dict(ncvars['time']) _range = cfradial._ncvar_to_dict(ncvars['range']) # 4.5 Ray dimension variables # 4.6 Location variables -> create attribute dictionaries # the only difference in this section to cfradial.read_cfradial is the # minor variable name differences: # latitude -> lat # longitude -> lon # altitdue -> alt latitude = cfradial._ncvar_to_dict(ncvars['lat']) longitude = cfradial._ncvar_to_dict(ncvars['lon']) altitude = cfradial._ncvar_to_dict(ncvars['alt']) # 4.7 Sweep variables -> create atrribute dictionaries # this is the section that needed the most work since the initial NetCDF # file did not contain any sweep information sweep_number = filemetadata('sweep_number') sweep_number['data'] = np.array([0], dtype=np.int32) sweep_mode = filemetadata('sweep_mode') sweep_mode['data'] = np.array(['vertical_pointing'], dtype=np.str) fixed_angle = filemetadata('fixed_angle') fixed_angle['data'] = np.array([90.0], dtype=np.float32) sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_start_ray_index['data'] = np.array([0], dtype=np.int32) sweep_end_ray_index = filemetadata('sweep_end_ray_index') sweep_end_ray_index['data'] = np.array( [ncvars['time'].size - 1], dtype=np.int32) # first sweep mode determines scan_type # this module is specific to vertically-pointing data scan_type = 'vpt' # 4.8 Sensor pointing variables -> create attribute dictionaries # this section also required some changes since the initial NetCDF did not # contain any sensor pointing variables azimuth = filemetadata('azimuth') azimuth['data'] = 0.0 * np.ones(ncvars['time'].size, dtype=np.float32) elevation = filemetadata('elevation') elevation['data'] = 90.0 * np.ones(ncvars['time'].size, dtype=np.float32) # 4.9 Moving platform geo-reference variables # 4.10 Moments field data variables -> field attribute dictionary # all variables with dimensions of 'time', 'range' are fields keys = [k for k, v in ncvars.items() if v.dimensions == ('time', 'range')] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) if field_name is None: if exclude_fields is not None and key in exclude_fields: continue if include_fields is not None and not key in include_fields: continue field_name = key fields[field_name] = cfradial._ncvar_to_dict(ncvars[key]) # 4.5 instrument_parameters sub-convention -> instrument_parameters dict # this section needed multiple changes and/or additions since the # instrument parameters were primarily located in the global attributes # this section is likely still incomplete omega = float(ncobj.radar_operating_frequency.split()[0]) frequency = filemetadata('frequency') frequency['data'] = np.array([omega / 1e9], dtype=np.float32) prt_mode = filemetadata('prt_mode') prt_mode['data'] = np.array(['fixed'], dtype=np.str) prf = float(ncobj.pulse_repetition_frequency.split()[0]) prt = filemetadata('prt') prt['data'] = (1.0 / prf) * np.ones(ncvars['time'].size, dtype=np.float32) v_nq = float(ncobj.nyquist_velocity.split()[0]) nyquist_velocity = filemetadata('nyquist_velocity') nyquist_velocity['data'] = v_nq * np.ones(ncvars['time'].size, dtype=np.float32), samples = int(ncobj.num_spectral_averages) n_samples = filemetadata('n_samples') n_samples['data'] = samples * np.ones(ncvars['time'].size, dtype=np.int32) # 4.6 radar_parameters sub-convention -> instrument_parameters dict # this section needed multiple changes and/or additions since the # radar instrument parameters were primarily located in the global # attributes # this section is likely still incomplete instrument_parameters = { 'frequency': frequency, 'prt_mode': prt_mode, 'prt': prt, 'nyquist_velocity': nyquist_velocity, 'n_samples': n_samples, } # 4.7 lidar_parameters sub-convention -> skip # 4.8 radar_calibration sub-convention -> skip # close NetCDF object ncobj.close() return Radar( time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters)
def test_write_two_datasets(self): # Dictionaries representing data sets catalogue = Catalogue( identifier='bob', datasets=[ CatalogueDataSet( name='SomeData', path='/where/is/it', subsets=[ CatalogueDataSubset(layout=DataStorageFiles(patterns=[ 'something_fancy-with % signs and everything.nc', 'and another array member' ]), matches=[ CatalogueFileEntry( name='myname.nc', time=datetime( 2016, 05, 20, 13, 05), tags=['onetag', 'twotags'], size=1356, checksum='ABC129549'), CatalogueFileEntry( name='othername.nc', time=datetime( 2016, 05, 20, 14, 05), tags=['hello'], size=None, checksum=None), CatalogueFileEntry( name='bob.nc', time=datetime( 2016, 05, 20, 15, 05), tags=None, size=55552, checksum='checkitout') ], value_errors=[ 'whatwentwrong?', 'we have a problem' ], archive_unused=[ 'why is this even here :-/' ]) ], non_matching=[ '/bob/is/not/here', '/neither/is/ermintrude' ]), CatalogueDataSet(name='Other Stuff Here', path='/somewhere/here', subsets=[], non_matching=['harrumph']), ]) # temporary file to use for output testfile = tempfile.NamedTemporaryFile(suffix='.nc') # instantiate writer and do output writer = CatalogueWriterNetCDF() writer.save(testfile.name, catalogue) # TEMP: show netcdf contents # print '\n' + subprocess.Popen(['ncdump', testfile.name], stdout=subprocess.PIPE).communicate()[0] # TEMP: show cfchecks output # print '\n' + subprocess.Popen(['cfchecks', testfile.name], # stdout=subprocess.PIPE).communicate()[0] # load with netCDF library netcdf = Dataset(testfile.name) # Check top level self.assertEqual('bob', netcdf.identifier) self.assertTrue('datasets' in netcdf.dimensions) self.assertEqual(2, len(netcdf.dimensions['datasets'])) self.assertFalse(netcdf.dimensions['datasets'].isunlimited()) self.assertEqual(2, len(netcdf.groups)) # Groups for each data set group0 = netcdf.groups['datasets_00000000'] group1 = netcdf.groups['datasets_00000001'] # Check dataset group attributes self.assertEqual('SomeData', group0.getncattr('name')) self.assertEqual('/where/is/it', group0.getncattr('path')) # Check numbered subsets self.assertEqual(1, len(group0.dimensions['subsets'])) self.assertFalse(group0.dimensions['subsets'].isunlimited()) subset0 = group0.groups['subsets_00000000'] # Storage spec # self.assertEqual('eumopps.storage.DataStorageFiles', subset0.storage) # Layout subgroup layout = subset0.groups['layout'] self.assertEqual(2, len(layout.dimensions['patterns'])) self.assertFalse(layout.dimensions['patterns'].isunlimited()) self.assertEqual(2, len(layout.variables['patterns'])) self.assertEqual('something_fancy-with % signs and everything.nc', chartostring(layout.variables['patterns'][0])) self.assertEqual('and another array member', chartostring(layout.variables['patterns'][1])) # Matches arrays matches = subset0.groups['matches'] self.assertEqual(3, matches.dimensions['list_count'].size) self.assertEqual('seconds since 1850-01-01 00:00:00 UTC', matches.variables['time'].units) self.assertEqual(5250575100, matches.variables['time'][0]) self.assertEqual('myname.nc', chartostring(matches.variables['name'][0])) self.assertEqual('onetag', chartostring(matches.variables['tags'][0][0])) self.assertEqual('twotags', chartostring(matches.variables['tags'][0][1])) self.assertIsInstance(matches.variables['size'][0], numpy.int64) self.assertEqual(1356, matches.variables['size'][0]) self.assertEqual('ABC129549', chartostring(matches.variables['checksum'][0])) self.assertEqual('othername.nc', chartostring(matches.variables['name'][1])) self.assertEqual(5250578700, matches.variables['time'][1]) self.assertEqual('hello', chartostring(matches.variables['tags'][1][0])) self.assertEqual('', chartostring(matches.variables['tags'][2][1])) self.assertTrue(matches.variables['size'][1].mask) self.assertEqual('', chartostring(matches.variables['checksum'][1])) self.assertEqual('bob.nc', chartostring(matches.variables['name'][2])) self.assertEqual(5250582300, matches.variables['time'][2]) self.assertEqual('', chartostring(matches.variables['tags'][2][0])) self.assertEqual('', chartostring(matches.variables['tags'][2][1])) self.assertTrue(isinstance(matches.variables['size'][2], numpy.int64)) self.assertEqual(55552, matches.variables['size'][2]) self.assertEqual('checkitout', chartostring(matches.variables['checksum'][2])) # Value errors array of strings self.assertEqual(2, len(subset0.variables['value_errors'])) self.assertEqual('whatwentwrong?', chartostring(subset0.variables['value_errors'][0])) self.assertEqual('we have a problem', chartostring(subset0.variables['value_errors'][1])) # 'archive_unused' array of strings self.assertEqual(1, len(subset0.variables['archive_unused'])) self.assertEqual('why is this even here :-/', chartostring(subset0.variables['archive_unused'][0])) # Second data set example self.assertEqual(2, len(group0.dimensions['non_matching'])) self.assertFalse(group0.dimensions['non_matching'].isunlimited()) self.assertEqual('/bob/is/not/here', chartostring(group0.variables['non_matching'][0])) self.assertEqual('/neither/is/ermintrude', chartostring(group0.variables['non_matching'][1])) self.assertEqual('Other Stuff Here', group1.getncattr('name')) self.assertEqual('/somewhere/here', group1.getncattr('path')) self.assertEqual(1, len(group1.dimensions['non_matching'])) self.assertFalse(group1.dimensions['non_matching'].isunlimited()) self.assertEqual('harrumph', chartostring(group1.variables['non_matching'][0]))
def load_data(self): indices = self.class4[:, 1].astype(int) # Expecting specific class4 ID format: "class4_YYYMMDD_*.nc" with Dataset( current_app.config["CLASS4_FNAME_PATTERN"] % (self.class4[0][0][7:11], self.class4[0][0]), 'r') as ds: self.latitude = ds['latitude'][indices] self.longitude = ds['longitude'][indices] self.ids = list(map(str.strip, chartostring(ds['id'][indices]))) self.variables = list( map(str.strip, chartostring(ds['varname'][:]))) self.variable_units = list( map(str.strip, chartostring(ds['unitname'][:]))) forecast_data = [] observed_data = [] climatology_data = [] depths = [] for i in indices: f_data = [] o_data = [] c_data = [] for j in range(0, len(self.variables)): if self.forecast == 'best': f_data.append(ds['best_estimate'][i, j, :]) else: f_data.append(ds['forecast'][i, j, int(self.forecast), :]) o_data.append(ds['observation'][i, j, :]) c_data.append(ds['climatology'][i, j, :]) forecast_data.append(np.ma.vstack(f_data)) observed_data.append(np.ma.vstack(o_data)) climatology_data.append(np.ma.vstack(c_data)) depths.append(ds['depth'][i, :]) self.depth_unit = ds['depth'].units self.forecast_data = np.ma.array(forecast_data) self.observed_data = np.ma.array(observed_data) self.climatology_data = np.ma.array(climatology_data) self.depths = np.ma.vstack(depths) additional_model_data = [] additional_model_names = [] for m in self.models: additional_model_names.append(m.split("_")[2]) # Expecting specific class4 ID format: "class4_YYYMMDD_*.nc" with Dataset( current_app.config["CLASS4_FNAME_PATTERN"] % (m[7:11], m), 'r') as ds: m_data = [] for i in indices: data = [] for j in range(0, len(self.variables)): data.append(ds['best_estimate'][i, j, :]) m_data.append(np.ma.vstack(data)) additional_model_data.append(np.ma.array(m_data)) self.additional_model_data = np.ma.array(additional_model_data) self.additional_model_names = additional_model_names
def decode_strs(ncin, vname): """Reads a bunch of strings and returns as a list""" return [ x.decode().strip() for x in netCDF4.chartostring(ncin.variables[vname][:]) ]