def test_read_example_data(): # read any example data files for fname in glob(pjoin(TEST_DATA_PATH, '*.nc')): f = netcdf_file(fname, 'r') f.close() f = netcdf_file(fname, 'r', mmap=False) f.close()
def get_data(dir, run_id = 'def', plot = False): #if 'inputs' not in locals(): inputs = netcdf.netcdf_file(dir + '{}_inputs.cdf'.format(run_id),'r',mmap = False) spectra = netcdf.netcdf_file(dir + '{}_spectra.cdf'.format(run_id),'r',mmap = False) #neutrals = netcdf.netcdf_file(dir + '{}_neutrals.cdf'.format(run_id),'r',mmap = False) #weights = netcdf.netcdf_file(dir + '{}_fida_weights.cdf'.format(run_id),'r',mmap = False) neutrals=weights=None fida = +spectra.variables['fida'].data wave = +spectra.variables['lambda'].data halo = +spectra.variables['halo'].data BE = [+spectra.variables['full'].data, +spectra.variables['half'].data, +spectra.variables['third'].data] if plot: fig, ax = pt.subplots(nrows = len(BE)+1) for i in range(fida.shape[0]): for ax_tmp, dat_tmp in zip(ax,BE): ax_tmp.plot(wave,dat_tmp[i,:]) ax[len(BE)].plot(wave,halo[i,:]) fig.canvas.draw();fig.show() # ax[0].plot(fida[i,:]) i = np.argmin(np.abs([np.mean(inputs.variables['z_grid'].data[i,:,:]) for i in range(inputs.variables['z_grid'].data.shape[0])])) z = inputs.variables['z_grid'].data[i,:,:] x_grid = inputs.variables['x_grid'].data[i,:,:] y_grid = inputs.variables['y_grid'].data[i,:,:] if plot: dat_grid = neutrals.variables['halodens'].data[0,i,:,:] n_halos = neutrals.variables['halodens'].shape[0] fig, ax = pt.subplots(nrows = n_halos) for j in range(n_halos): im = ax[j].pcolormesh(x_grid, y_grid, neutrals.variables['halodens'].data[j,i,:,:]) pt.colorbar(im,ax=ax[j]) fig.canvas.draw();fig.show return inputs, neutrals, spectra, weights
def plot_arm_speed(axis, startTime=-1): rootName = 'siemensSensors' f = netcdf.netcdf_file(rootName+'Data.nc', 'r') data1 = f.variables[rootName+'.data.'+'carouselSpeedSetpoint'].data[startSample:] data2 = f.variables[rootName+'.data.'+'carouselSpeedSmoothed'].data[startSample:] ts_trigger = f.variables[rootName+'.data.ts_trigger'].data[startSample:]*1.0e-9 # Load the actual arm speed from the arm gyro rootName = 'armboneLisaSensors' fiile = netcdf.netcdf_file(rootName+'Data.nc', 'r') rawdata4 = fiile.variables['armboneLisaSensors.GyroState.gr'].data[startSample:] ts_trigger4 = fiile.variables['armboneLisaSensors.GyroState.ts_trigger'].data[startSample:]*1.0e-9 #fullscale = 2000 # deg/sec #data4 = -1.0 * rawdata4 / (2**15) * fullscale * pi/180 - 0.0202 # Rad/s data4 = rawdata4 if startTime == -1: startTime = ts_trigger[0] times = ts_trigger-startTime times4 = ts_trigger4-startTime pylab.hold(True) plot(times, data2, '.-', label='On Motor Side of Belt') plot(times4, data4,'.-', label='From Gyro on Arm') plot(times, data1, '.-', label='Setpoint (Echoed)') ylabel('Arm rotation speed [Rad/s]') xlabel('Time [s]') #legend(['Setpoint (Echoed)', 'Setpoint (Sent)', 'On Motor Side of Belt', 'From Gyro on Arm']) title('Plot of Signals Related to Arm Speed') return startTime
def grab_data(rule): files = glob.glob(rule) filebases = [] for i in range(0, len(files)): #Grab the part after the / of any path, and before the _k*.nc filebase = re.split('_', re.split('/', files[i])[-1] )[0] #If we haven't seen this base before, keep it around. if not(filebases.count(filebase)): filebases.append(filebase) numbases = len(filebases) data = numpy.ones(numbases, dtype = [('omega1', float), ('B', float), ('kmin', float), ('kmax', float), ('kpeak', float), ('peakgr', float)]) path = "" for dir in (re.split('/', files[0])[0:-1]): path = path + dir + '/' for i in range(0,numbases): data[i]['omega1'] = float(re.split('B', re.split('w', filebases[i])[1])[0]) data[i]['B'] = float(re.split('B', filebases[i])[1]) kminname = path + filebases[i] + "_kmin.nc" kmaxname = path + filebases[i] + "_kmax.nc" kpeakname = path + filebases[i] + "_kpeak.nc" if(os.path.exists(kminname)): ncfile = netcdf.netcdf_file(kminname, 'r') data[i]['kmin'] = ncfile.kz ncfile.close() else: data[i]['kmin'] = numpy.nan if(os.path.exists(kmaxname)): ncfile = netcdf.netcdf_file(kmaxname, 'r') data[i]['kmax'] = ncfile.kz ncfile.close() else: data[i]['kmax'] = numpy.nan if(os.path.exists(kpeakname)): ncfile = netcdf.netcdf_file(kpeakname, 'r') data[i]['kpeak'] = ncfile.kz data[i]['peakgr'] = ncfile.variables['lambda'][0,0] ncfile.close() else: data[i]['kpeak'] = numpy.nan data[i]['peakgr'] = numpy.nan #Sort the array so things are nicer data = numpy.sort(data, order=['omega1', 'B']) return data
def test_read_example_data(): # read any example data files for fname in glob(pjoin(TEST_DATA_PATH, '*.nc')): with netcdf_file(fname, 'r') as f: pass with netcdf_file(fname, 'r', mmap=False) as f: pass
def test_read_write_sio(): eg_sio1 = BytesIO() with make_simple(eg_sio1, 'w') as f1: str_val = eg_sio1.getvalue() eg_sio2 = BytesIO(str_val) with netcdf_file(eg_sio2) as f2: check_simple(f2) # Test that error is raised if attempting mmap for sio eg_sio3 = BytesIO(str_val) assert_raises(ValueError, netcdf_file, eg_sio3, 'r', True) # Test 64-bit offset write / read eg_sio_64 = BytesIO() with make_simple(eg_sio_64, 'w', version=2) as f_64: str_val = eg_sio_64.getvalue() eg_sio_64 = BytesIO(str_val) with netcdf_file(eg_sio_64) as f_64: check_simple(f_64) assert_equal(f_64.version_byte, 2) # also when version 2 explicitly specified eg_sio_64 = BytesIO(str_val) with netcdf_file(eg_sio_64, version=2) as f_64: check_simple(f_64) assert_equal(f_64.version_byte, 2)
def test_maskandscale(): t = np.linspace(20, 30, 15) t[3] = 100 tm = np.ma.masked_greater(t, 99) fname = pjoin(TEST_DATA_PATH, 'example_2.nc') with netcdf_file(fname, maskandscale=True) as f: Temp = f.variables['Temperature'] assert_equal(Temp.missing_value, 9999) assert_equal(Temp.add_offset, 20) assert_equal(Temp.scale_factor, np.float32(0.01)) found = Temp[:].compressed() del Temp # Remove ref to mmap, so file can be closed. expected = np.round(tm.compressed(), 2) assert_allclose(found, expected) with in_tempdir(): newfname = 'ms.nc' f = netcdf_file(newfname, 'w', maskandscale=True) f.createDimension('Temperature', len(tm)) temp = f.createVariable('Temperature', 'i', ('Temperature',)) temp.missing_value = 9999 temp.scale_factor = 0.01 temp.add_offset = 20 temp[:] = tm f.close() with netcdf_file(newfname, maskandscale=True) as f: Temp = f.variables['Temperature'] assert_equal(Temp.missing_value, 9999) assert_equal(Temp.add_offset, 20) assert_equal(Temp.scale_factor, np.float32(0.01)) expected = np.round(tm.compressed(), 2) found = Temp[:].compressed() del Temp assert_allclose(found, expected)
def load_steady_state_experiments_data(): samplesToSkip = 3 print('loading data...') rootName = 'lineAngleSensor2' f = netcdf.netcdf_file(rootName+'Data.nc', 'r') memberName = 'azimuth' azimuth = f.variables[rootName+'.data.'+memberName].data[samplesToSkip:] memberName = 'elevation' elevation = f.variables[rootName+'.data.'+memberName].data[samplesToSkip:] ts_trigger_las = f.variables[rootName+'.data.ts_trigger'].data[samplesToSkip:]*1.0e-9 rootName = 'siemensSensors' f = netcdf.netcdf_file(rootName+'Data.nc', 'r') setpoint = f.variables[rootName+'.data.'+'carouselSpeedSetpoint'].data[samplesToSkip:] speed = f.variables[rootName+'.data.'+'carouselSpeedSmoothed'].data[samplesToSkip:] ts_trigger_siemens = f.variables[rootName+'.data.ts_trigger'].data[samplesToSkip:]*1.0e-9 # Chose intersection of the two time ranges startTime = max(ts_trigger_las[0],ts_trigger_siemens[0]) endTime = min(ts_trigger_las[-1],ts_trigger_siemens[-1]) ts_trigger_las -= startTime ts_trigger_siemens -= startTime startTime_new = 0 endTime_new = endTime-startTime times = len(ts_trigger_las)+len(ts_trigger_siemens) # Since we didn't have the resampler turned on... t = numpy.linspace(startTime_new, endTime_new, times) elevation_resampled = numpy.interp(t, ts_trigger_las, elevation) speed_resampled = numpy.interp(t, ts_trigger_siemens, speed) return speed_resampled, elevation_resampled
def load_timeseries(filenames,region,months,bias): data=np.ma.zeros([len(filenames),len(months)*30]) tmp=np.zeros([len(months)*30]) i=0 [j_s,j_e,i_s,i_e]=region for f in filenames: # print i,os.path.basename(f) try: f2=f.replace('field16','field16_1') for j,monthstr in enumerate(months): f_month=f[:-10]+monthstr+'.nc' f2_month=f2[:-10]+monthstr+'.nc' var1=netcdf_file(f_month,'r').variables['field16'][:,0,4:-7,4:-4] var1=np.ma.masked_values(var1,-1.07374e+09) var2=netcdf_file(f2_month,'r').variables['field16_1'][:,0,4:-7,4:-4] var2=np.ma.masked_values(var2,-1.07374e+09) tmp[j*30:(j+1)*30]=((var1+var2)/2.-bias)[:,j_s:j_e,i_s:i_e].mean(1).mean(1) if tmp.max()>350.0 or tmp.min()<170 or not np.all(np.isfinite(tmp)): print 'error: wierd vals',f continue else: # print tmp.min(),tmp.max() data[i,:]=tmp i=i+1 except: print 'Error, cannot load files',f raise #continue return data[:i,:]
def _load_scipy(self, scipy_nc, *args, **kwdargs): """ Interprets a netcdf file-like object using scipy.io.netcdf. The contents of the netcdf object are loaded into memory. """ try: nc = netcdf.netcdf_file(scipy_nc, mode='r', *args, **kwdargs) except: scipy_nc = StringIO(scipy_nc) scipy_nc.seek(0) nc = netcdf.netcdf_file(scipy_nc, mode='r', *args, **kwdargs) def from_scipy_variable(sci_var): return Variable(dims = sci_var.dimensions, data = sci_var.data, attributes = sci_var._attributes) object.__setattr__(self, 'attributes', AttributesDict()) self.attributes.update(nc._attributes) object.__setattr__(self, 'dimensions', OrderedDict()) dimensions = OrderedDict((k, len(d)) for k, d in nc.dimensions.iteritems()) self.dimensions.update(dimensions) object.__setattr__(self, 'variables', OrderedDict()) OrderedDict = OrderedDict((vn, from_scipy_variable(v)) for vn, v in nc.variables.iteritems()) self.variables.update()
def test_read_write_sio(): eg_sio1 = BytesIO() f1 = make_simple(eg_sio1, 'w') str_val = eg_sio1.getvalue() f1.close() eg_sio2 = BytesIO(str_val) f2 = netcdf_file(eg_sio2) for testargs in gen_for_simple(f2): yield testargs f2.close() # Test that error is raised if attempting mmap for sio eg_sio3 = BytesIO(str_val) yield assert_raises, ValueError, netcdf_file, eg_sio3, 'r', True # Test 64-bit offset write / read eg_sio_64 = BytesIO() f_64 = make_simple(eg_sio_64, 'w', version=2) str_val = eg_sio_64.getvalue() f_64.close() eg_sio_64 = BytesIO(str_val) f_64 = netcdf_file(eg_sio_64) for testargs in gen_for_simple(f_64): yield testargs yield assert_equal, f_64.version_byte, 2 # also when version 2 explicitly specified eg_sio_64 = BytesIO(str_val) f_64 = netcdf_file(eg_sio_64, version=2) for testargs in gen_for_simple(f_64): yield testargs yield assert_equal, f_64.version_byte, 2
def write_netcdf(path, fields, attrs={}, append=False, format='NETCDF3_64BIT'): """ Write the data and grid information for *fields* to *path* as NetCDF. If the *append* keyword argument in True, append the data to an existing file, if it exists. Otherwise, clobber an existing files. """ assert(format in _VALID_NETCDF_FORMATS) if os.path.isfile(path) and append: mode = 'a' else: mode = 'w' if format == 'NETCDF3_CLASSIC': root = nc.netcdf_file(path, mode, version=1) elif format == 'NETCDF3_64BIT': root = nc.netcdf_file(path, mode, version=2) else: root = nc4.Dataset(path, mode, format=format) _set_netcdf_attributes(root, attrs) _set_netcdf_structured_dimensions(root, fields.shape) _set_netcdf_variables(root, fields) root.close()
def get_fidasim_output(self,): '''Open the relevant netcdf files SRH: 23June2015 ''' self.inputs = netcdf.netcdf_file(self.directory + '{}_inputs.cdf'.format(self.run_id),'r',mmap = False) self.spectra = netcdf.netcdf_file(self.directory + '{}_spectra.cdf'.format(self.run_id),'r',mmap = False) self.neutrals = netcdf.netcdf_file(self.directory + '{}_neutrals.cdf'.format(self.run_id),'r',mmap = False)
def WriteTMPave(biofile,physfile, outfile): nc=NC.netcdf_file(biofile,"r"); DIMS=nc.dimensions; jpk = DIMS['depth'] jpj = DIMS['lat' ] jpi = DIMS['lon' ] ncOUT=NC.netcdf_file(outfile,"w") setattr(ncOUT,"Convenctions","COARDS") setattr(ncOUT,"DateStart",nc.DateStart) setattr(ncOUT,"Date__End",nc.Date__End) ncOUT.createDimension('time', 1) ncOUT.createDimension('lon' ,jpi) ncOUT.createDimension('lat' ,jpj) ncOUT.createDimension('depth',jpk) for var in ['lon','lat','depth']: ncvar=ncOUT.createVariable(var,'f',(var,)) ncvar[:]=nc.variables[var].data nc.close() setattr(ncOUT.variables['lon'],"long_name","Longitude") setattr(ncOUT.variables['lat'],"long_name","Latitude") for var in ['N1p','N3n','O2o'] : ncIN = NC.netcdf_file(biofile,"r") ncvar=ncOUT.createVariable(var,'f',('time','depth','lat','lon')) ncvar[:]=ncIN.variables[var].data.copy() setattr(ncvar,"long_name",var) setattr(ncvar,"missing_value",1.e+20) ncIN.close() for var in ['votemper','vosaline'] : ncIN = NC.netcdf_file(physfile,"r") ncvar=ncOUT.createVariable(var,'f',('time','depth','lat','lon')) ncvar[:]=ncIN.variables[var].data.copy() setattr(ncvar,"long_name",var) setattr(ncvar,"missing_value",1.e+20) ncIN.close() AGGREGATE_DICT={'P_l':['P1l','P2l','P3l','P4l']} for var in AGGREGATE_DICT.keys(): ncvar=ncOUT.createVariable(var,'f',('time','depth','lat','lon')) junk = np.zeros((1,jpk,jpj,jpi),np.float32) for lvar in AGGREGATE_DICT[var]: ncIN = NC.netcdf_file(biofile,"r") junk +=ncIN.variables[lvar].data.copy() ncIN.close() tmask= junk > 1.e+19 junk[tmask] = 1.e+20 ncvar[:]=junk setattr(ncvar,"long_name",var) setattr(ncvar,"missing_value",1.e+20) ncOUT.close()
def write_2d_file(M2d, varname, outfile, mask, fillValue=1.0e20): """ Dumps a 2D array in a NetCDF file. Arguments: * M2d * the 2D array to dump * varname * the variable name on NetCDF file * outfile * file that will be created. If it is an existing file, it will be opened in 'append' mode. * mask * a mask object consistent with M2d array * fillvalue * (optional) value to set missing_value attribute. When the file is opened in 'append' mode this method tries to adapt to existing dimension names (for example it works both with 'lon' or 'longitude') Does not return anything.""" if os.path.exists(outfile): ncOUT = NC.netcdf_file(outfile, "a") print "appending ", varname, " in ", outfile else: ncOUT = NC.netcdf_file(outfile, "w") jpk, jpj, jpi = mask.shape ncOUT.createDimension("longitude", jpi) ncOUT.createDimension("latitude", jpj) ncOUT.createDimension("depth", jpk) ncvar = ncOUT.createVariable("longitude", "f", ("longitude",)) setattr(ncvar, "units", "degrees_east") setattr(ncvar, "long_name", "longitude") setattr(ncvar, "standard_name", "longitude") setattr(ncvar, "axis", "X") setattr(ncvar, "valid_min", -5.5625) setattr(ncvar, "valid_max", 36.25) setattr(ncvar, "_CoordinateAxisType", "Lon") ncvar[:] = mask.xlevels[0, :] ncvar = ncOUT.createVariable("latitude", "f", ("latitude",)) setattr(ncvar, "units", "degrees_north") setattr(ncvar, "long_name", "latitude") setattr(ncvar, "standard_name", "latitude") setattr(ncvar, "axis", "Y") setattr(ncvar, "valid_min", 30.1875) setattr(ncvar, "valid_max", 45.9375) setattr(ncvar, "_CoordinateAxisType", "Lat") ncvar[:] = mask.ylevels[:, 0] ncvar = ncOUT.createVariable(varname, "f", (lat_dimension_name(ncOUT), lon_dimension_name(ncOUT))) setattr(ncvar, "fillValue", fillValue) setattr(ncvar, "missing_value", fillValue) setattr(ncvar, "coordinates", "latitude longitude") ncvar[:] = M2d setattr(ncOUT, "latitude_min", 30.0) setattr(ncOUT, "latitude_max", 46.0) setattr(ncOUT, "longitude_min", -6.0) setattr(ncOUT, "longitude_max", 37.0) ncOUT.close()
def test_timestep_diff(data, dt, eps=2.e-4): """ checking if the results are close to the referential ones (stored in refdata folder) """ filename = "timesteptest_dt=" + str(dt) + ".nc" f_test = netcdf.netcdf_file(filename, "r") f_ref = netcdf.netcdf_file(os.path.join("long_test/refdata", filename), "r") for var in ["t", "z", "th_d", "T", "p", "r_v", "rhod"]: assert np.isclose(f_test.variables[var][:], f_ref.variables[var][:], atol=0, rtol=eps).all(), "differs e.g. " + str(var) + "; max(ref diff) = " + str(np.where(f_ref.variables[var][:] != 0., abs((f_test.variables[var][:]-f_ref.variables[var][:])/f_ref.variables[var][:]), 0.).max())
def use_netcdf_files(): nc = netcdf.netcdf_file('/home/nicholas/data/netcdf_files/CFLX_2000_2009.nc', 'r') all_data = nc.variables['Cflx'][:, :45, :180] nc.close() all_data = all_data * 1000 * 24 * 60 * 60 all_data = ma.masked_values(all_data, 1e20) nc = netcdf.netcdf_file('/home/nicholas/data/netcdf_files/ORCA2.0_grid.nc', 'r') mask = nc.variables['mask'][0, :45, :180] nc.close() mask = ma.masked_values(mask, -1e34) return all_data, mask
def get_los_data(dir='/u/haskeysr/FIDASIM/RESULTS/D3D/155196/00500/MAIN_ION330/', run_id = 'def', plot = False): #if 'inputs' not in locals(): inputs = netcdf.netcdf_file(dir + '{}_inputs.cdf'.format(run_id),'r',mmap = False) los_wght = inputs.variables['los_wght'].data #spectra = netcdf.netcdf_file(dir + '{}_spectra.cdf'.format(run_id),'r',mmap = False) neutrals = netcdf.netcdf_file(dir + '{}_neutrals.cdf'.format(run_id),'r',mmap = False) halo_dens = neutrals.variables['halodens'].data #weights = netcdf.netcdf_file(dir + '{}_fida_weights.cdf'.format(run_id),'r',mmap = False) print 'hello' print 'hello2' print 'hello3' 1/0 return halo_dens, los_wght
def test_byte_gatts(): # Check that global "string" atts work like they did before py3k # unicode and general bytes confusion with in_tempdir(): filename = 'g_byte_atts.nc' f = netcdf_file(filename, 'w') f._attributes['holy'] = b'grail' f._attributes['witch'] = 'floats' f.close() f = netcdf_file(filename, 'r') assert_equal(f._attributes['holy'], b'grail') assert_equal(f._attributes['witch'], b'floats') f.close()
def doHeatContent(thetaFile, thetaVar, rhoFile, rhoVar, nodata, thresholdDepth, outFile, outFormat, options): # open file fhTheta = netcdf.netcdf_file(thetaFile, 'r') if fhTheta is None: exitMessage("Could not open file {0}. Exit 2.".format(thetaFile), 2) fhRho = netcdf.netcdf_file(rhoFile, 'r') if fhRho is None: exitMessage("Could not open file {0}; Exit(2).".format(rhoFile), 2) thetao = fhTheta.variables[thetaVar][:] # [time, levels, lat, lon] rho = fhRho.variables[rhoVar][:] levelsTmp = fhTheta.variables['lev_bnds'][:] levels = numpy.ravel(0.5*(levelsTmp[:,0] + levelsTmp[:,1] )) # mapHeat : time, lat, lon mapHeat = numpy.zeros( (thetao.shape[0], thetao.shape[2], thetao.shape[3]) ) - 1 # Compressing loops... timelatlon=[] for itime in range(thetao.shape[0]): for ilat in range(thetao.shape[2]): for ilon in range(thetao.shape[3]): if thetao[itime, 0, ilat, ilon] < nodata: timelatlon.append((itime, ilat, ilon)) # loop over time, lat and lon counter=0 for ill in timelatlon: profileTheta = thetao[ ill[0], : ,ill[1], ill[2] ].ravel() profileRho = rho[ ill[0], : ,ill[1], ill[2] ].ravel() heat = 0 heat = computeHeatPotential( profileTheta, profileRho, levels, thresholdDepth, nodata ) mapHeat[ ill[0], ill[1], ill[2] ] = heat gdal.TermProgress_nocb( counter/float(len(timelatlon)) ) counter = counter+1 gdal.TermProgress_nocb(1) # save result outDrv = gdal.GetDriverByName(outformat) outDS = outDrv.Create(outFile, mapHeat.shape[2], mapHeat.shape[1], mapHeat.shape[0], GDT_Float32, options) outDS.SetProjection(latlon()) for itime in range(thetao.shape[0]): data = numpy.ravel(mapHeat[itime, :, :]) outDS.GetRasterBand(itime+1).WriteArray( numpy.flipud( data.reshape((mapHeat.shape[1], mapHeat.shape[2])) ) ) gdal.TermProgress_nocb( itime/float(thetao.shape[0]) ) gdal.TermProgress_nocb(1) outDS = None
def create_ave_pp_header(datestr): ''' Generates output files, in PROFILES directory Files refer to a time frame and will contain all the variables Works in append mode, if the file name exists.''' ave_Pprofiles = OUTPUT_DIR_PRO + IOnames.Output.prefix + datestr + ".profiles.nc" if os.path.exists(ave_Pprofiles): ncOUT_Pprofiles = NC.netcdf_file(ave_Pprofiles,"a") else: ncOUT_Pprofiles = NC.netcdf_file(ave_Pprofiles,"w") ncOUT_Pprofiles.createDimension("Ncruise" ,nCruise) ncOUT_Pprofiles.createDimension("z" ,jpk) setattr(ncOUT_Pprofiles,"CruiseIndex",CruiseDescr) return ncOUT_Pprofiles
def test_byte_gatts(): # Check that global "string" atts work like they did before py3k # unicode and general bytes confusion filename = pjoin(TEST_DATA_PATH, 'g_byte_atts.nc') f = netcdf_file(filename, 'w') f._attributes['holy'] = b'grail' f._attributes['witch'] = 'floats' f.close() f = netcdf_file(filename, 'r') assert_equal(f._attributes['holy'], b'grail') assert_equal(f._attributes['witch'], b'floats') f.close() os.remove(filename)
def size_check(datadir): """Determine how large the merged arrays will be, for preallocation. """ files = glob.glob(datadir + '*.nc') numFiles = len(files) ncid = netcdf.netcdf_file(files[0]) nele = ncid.dimensions['nele'] node = ncid.dimensions['node'] ncid.close() timeDim = 0 for i in xrange(numFiles): ncid = netcdf.netcdf_file(files[i]) timeDim += len(ncid.variables['time'].data) ncid.close() return nele, node, timeDim
def data(request): # initial condition SO2_g_init = 200e-12 O3_g_init = 50e-9 H2O2_g_init = 500e-12 outfreq = 1000 z_max = 250. dt = .01 w = 1. # turn on chemistry chem_dsl = True chem_dsc = True chem_rct = True chem_spn = 10 # define output for moments and chemistry out_bin_chem = '{"radii": {"rght": 1, "moms": [0, 1, 3], "drwt": "wet", "nbin": 1, "lnli": "lin", "left": 0}, "chem": {"rght": 1, "moms": ["O3_a", "H2O2_a", "SO2_a", "H", "OH", "HSO3_a", "SO3_a", "HSO4_a", "SO4_a", "S_VI"], "drwt": "wet", "nbin": 1, "lnli": "lin", "left": 0}, "radiidry": {"rght": 1, "moms": [0, 1, 3], "drwt": "dry", "nbin": 1, "lnli": "lin", "left": 0}}' out_bin = '{"radii": {"rght": 1, "moms": [0, 1, 3], "drwt": "wet", "nbin": 1, "lnli": "lin", "left": 0}, "radiidry": {"rght": 1, "moms": [0, 1, 3], "drwt": "dry", "nbin": 1, "lnli": "lin", "left": 0}}' # running parcel model for open / closed chem system ... parcel(dt = dt, z_max = z_max, w = w, outfreq = outfreq,\ SO2_g_0 = SO2_g_init, O3_g_0 = O3_g_init, H2O2_g_0 = H2O2_g_init,\ chem_sys = 'open', outfile="test_chem_open.nc",\ chem_dsl = chem_dsl, chem_dsc = chem_dsc, chem_rct = chem_rct, chem_spn = chem_spn, \ out_bin = out_bin_chem) parcel(dt = dt, z_max = z_max, w = w, outfreq = outfreq,\ SO2_g_0 = SO2_g_init, O3_g_0 = O3_g_init, H2O2_g_0 = H2O2_g_init,\ chem_sys = 'closed', outfile="test_chem_closed.nc",\ chem_dsl = chem_dsl, chem_dsc = chem_dsc, chem_rct = chem_rct, chem_spn = chem_spn, \ out_bin = out_bin_chem) parcel(dt = dt, z_max = z_max, w = w, outfreq = outfreq, SO2_g_0=0, O3_g_0=0, H2O2_g_0=0,\ outfile="test_chem_off.nc", out_bin = out_bin) # TODO - why do I have to repeat this import here? from scipy.io import netcdf data = {'open' : netcdf.netcdf_file("test_chem_open.nc", "r"),\ 'closed' : netcdf.netcdf_file("test_chem_closed.nc", "r"),\ 'off' : netcdf.netcdf_file("test_chem_off.nc", "r")} def removing_files(): for name, netcdf in data.iteritems(): subprocess.call(["rm", "test_chem_" + name + ".nc"]) request.addfinalizer(removing_files) return data
def data(request): """ Run parcel simulation and return opened netdcf file """ # copy options from chem_conditions p_dict = copy.deepcopy(parcel_dict) # modify options from chem_conditions p_dict['outfreq'] = p_dict['z_max'] / p_dict['w'] / p_dict['dt'] / 4 p_dict['outfile'] = "test_mass.nc" p_dict['chem_dsl'] = True p_dict['out_bin'] = \ '{"wradii": {"rght": 1e-4, "left": 1e-10, "drwt": "wet", "lnli": "lin", "nbin": 500, "moms": [0, 3]}, \ "dradii": {"rght": 1e-4, "left": 1e-10, "drwt": "dry", "lnli": "lin", "nbin": 500, "moms": [0, 3]}}' # run parcel model parcel(**p_dict) data = netcdf.netcdf_file(p_dict['outfile'], "r") # remove all netcdf files after all tests def removing_files(): subprocess.call(["rm", p_dict['outfile']]) request.addfinalizer(removing_files) return data
def saveCurrentToFile( filename, I, variables, potentials ): f = netcdf.netcdf_file(filename, 'w') sh = I.shape n = f.createDimension('n_potentials', sh[0]) xd = f.createDimension('x', sh[1]) yd = f.createDimension('y', sh[2]) zd = f.createDimension('z', sh[3]) vector = f.createDimension('vector', 3) posd = f.createVariable('position', dimensions=('vector', 'x', 'y', 'z'), type=np.dtype('f')) Id = f.createVariable('I', dimensions=('n_potentials', 'x', 'y', 'z'), type=np.dtype('f')) bias = f.createVariable('bias', dimensions=('n_potentials', ), type=np.dtype('f')) temperature = f.createVariable('temperature', dimensions=('n_potentials', ), type=np.dtype('f')) broadening = f.createVariable('broadening', dimensions=('n_potentials', ), type=np.dtype('f')) for i, potential in enumerate(potentials): bias[i] = potential['voltage'] temperature[i] = potential['temperature'] broadening[i] = potential['state_broadening'] Id[:,:,:,:] = I posd[:,:,:,:] = variables f.close()
def read_netcdf(nc_file, reshape=False, just_grid=False): """ Reads the NetCDF file *nc_file*, and writes it to the fields of a new RasterModelGrid, which it then returns. Check the names of the fields in the returned grid with grid.at_nodes.keys(). """ try: root = nc.netcdf_file(nc_file, 'r', version=2) except TypeError: root = nc4.Dataset(nc_file, 'r', format='NETCDF4') shape = _read_netcdf_grid_shape(root) spacing = _read_netcdf_grid_spacing(root) assert(len(shape) == 2) assert(len(spacing) == 2) if spacing[0] != spacing[1]: raise NotRasterGridError() grid = RasterModelGrid(num_rows=shape[0], num_cols=shape[1], dx=spacing[0]) if not just_grid: fields = _read_netcdf_structured_data(root) for (name, values) in fields.items(): grid.add_field('node', name, values) root.close() return grid
def createSampleXRange(M,N,filename,bounds=None,xdim=None,ydim=None): if xdim is None: xdim = 1.0 if ydim is None: ydim = 1.0 if bounds is None: xmin = 0.5 xmax = xmin + (N-1)*xdim ymin = 0.5 ymax = ymin + (M-1)*ydim else: xmin,xmax,ymin,ymax = bounds data = np.arange(0,M*N).reshape(M,N).astype(np.int32) cdf = netcdf.netcdf_file(filename,'w') cdf.createDimension('side',2) cdf.createDimension('xysize',M*N) dim = cdf.createVariable('dimension','i',('side',)) dim[:] = np.array([N,M]) spacing = cdf.createVariable('spacing','i',('side',)) spacing[:] = np.array([xdim,ydim]) zrange = cdf.createVariable('z_range',INVERSE_NETCDF_TYPES[str(data.dtype)],('side',)) zrange[:] = np.array([data.min(),data.max()]) x_range = cdf.createVariable('x_range','d',('side',)) x_range[:] = np.array([xmin,xmax]) y_range = cdf.createVariable('y_range','d',('side',)) y_range[:] = np.array([ymin,ymax]) z = cdf.createVariable('z',INVERSE_NETCDF_TYPES[str(data.dtype)],('xysize',)) z[:] = data.flatten() cdf.close() return data
def read_raw(self, var): ''' If profile is None the entire 2d array (depth,time) is returned If a profile object is provided, a 1d array (dimension depth) is returned corresponding to the pointprofile provided ''' if not self.__file_already_read : ncIN = NC.netcdf_file(self.filename,'r') self._timesInFile = ncIN.variables['TIME'].data.copy() self._VAR = ncIN.variables[var ].data.copy() self._QC = ncIN.variables[var + "_QC" ].data.copy() if ncIN.variables.has_key('PRES'): self._PRES = ncIN.variables['PRES'].data.copy() pres_qc = ncIN.variables['PRES_QC'].data.copy() else: self._PRES = ncIN.variables['DEPH'].data.copy() pres_qc = ncIN.variables['DEPH_QC'].data.copy() self._TEMP = ncIN.variables['TEMP'].data.copy() self._PSAL = ncIN.variables['PSAL'].data.copy() temp_qc = ncIN.variables['TEMP_QC'].data.copy() psal_qc = ncIN.variables['PSAL_QC'].data.copy() ncIN.close() self._good_data = (temp_qc == 1 ) & (psal_qc == 1 ) & (pres_qc == 1 ) & (self._QC == 1) self.__file_already_read = True tconv = T90conv(self._TEMP) self._RHO = seawater.dens(self._PSAL,tconv, self._PRES) return self._VAR, self._PRES, self._QC, self._timesInFile
def test_read_withMissingValue(): # For a variable with missing_value but not _FillValue, the missing_value # should be used fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc') with netcdf_file(fname, maskandscale=True) as f: vardata = f.variables['var4_missingValue'][:] assert_mask_matches(vardata, [False, True, False])
var.update(tmp_result) result = var status = pyferret.stop() if verbose: if status: print "PyFerret stopped." else: print "PyFerret failed to stop." return result regrid_primitive = run_worker(regrid_once_primitive) if __name__ == '__main__': import scipy.io.netcdf as netcdf ncfile_low = netcdf.netcdf_file("land_mask_lowres.nc") newvar = dict(data=ncfile_low.variables['land_mask'].data, coords=[ncfile_low.variables[dim].data for dim in ncfile_low.variables['land_mask'].\ dimensions], dimunits=[ncfile_low.variables[dim].units for dim in ncfile_low.variables['land_mask'].\ dimensions]) ncfile_high = netcdf.netcdf_file("land_mask_highres.nc") var_high = dict(data=ncfile_high.variables['land_mask'].data, coords=[ncfile_high.variables[dim].data for dim in ncfile_high.variables['land_mask'].\ dimensions], dimunits=[ncfile_high.variables[dim].units for dim in ncfile_high.variables['land_mask'].\ dimensions])
def produce(self, data): base_url = data["http"]["url"] pad = data["http"]["padProfileID"] fd, temp_nc = mkstemp() os.close(fd) try: # Iterate over the floats for floats in data["floats"]: # Handle null profile number for this float - just get the last profile if data["floats"][floats] is None: url = pystache.render(base_url, { "floatID": floats }).rsplit("/", 1)[0] idx = requests.get(url) if idx.status_code == 404: raise Exception( "Internet", "Could not load FTP index {0}".format(url)) data["floats"][floats] = \ int(idx.content.rsplit("<a href=", 1)[1]. split("_")[1].split(".")[0]) - 1 # Iterate over the profiles for this float profile_found = True while profile_found: sequence_no = data["floats"][floats] + 1 profile_found = False profile_quality = None for quality in ["D", "R"]: url = self.format_url(base_url, floats, sequence_no, quality, pad) with open(temp_nc, 'wb') as handle: r = requests.get(url, stream=True) if r.status_code == 404: continue if not r.ok: logging.info("Problem with %s", url) continue profile_found = True profile_quality = quality for block in r.iter_content(1024): handle.write(block) break if profile_found: try: last_modified = datetime.strftime( datetime.strptime(r.headers["Last-Modified"], "%a, %d %b %Y %H:%M:%S %Z"), "%Y-%m-%dT%H:%M:%SZ") f = netcdf.netcdf_file(temp_nc, "rw") # Add some attributes we want to have when we get to Erddap setattr(f, "FLOAT_WMO_ID", str(floats)) setattr(f, "FLOAT_PROFILE_SEQUENCE_NUMBER", str(sequence_no)) setattr(f, "SOURCE_URL", url) setattr(f, "QUALITY", quality) setattr(f, "LAST_MODIFIED_DATETIME", last_modified) f.close() logging.info("%s %s %s %s %s", f.FLOAT_WMO_ID, f.FLOAT_PROFILE_SEQUENCE_NUMBER, f.SOURCE_URL, f.LAST_MODIFIED_DATETIME, f.QUALITY) with open(temp_nc, "rb") as nc_file: nc_encoded = base64.b64encode(nc_file.read()) result = { "source_url": url, "last_modified": last_modified, "netcdf": nc_encoded, "float": floats, "quality": profile_quality, "sequence_no": sequence_no } data["floats"][floats] += 1 yield (data, result) except AssertionError: pass for i in range(5 * 10): time.sleep(0.1) finally: try: os.remove(temp_nc) except OSError: pass
def extract_nc3_by_name(filename, dsname): nc_data = netcdf.netcdf_file(filename, "r") ds = np.array(nc_data.variables[dsname][:]) nc_data.close() return ds
import numpy as np from scipy.io import netcdf f_path = 'precip.nc' #your file from the NCEP reanalysis plotter # Retrieve data from NetCDF file with netcdf.netcdf_file(f_path, 'r') as f: lon = f.variables['lon'][::] # copy as list lat = f.variables[ 'lat'][::-1] # invert the latitude vector -> South to North air = f.variables['prate'][0, ::-1, :] # Shift 'lon' from [0,360] to [-180,180], make numpy array tmp_lon = np.array([ lon[n] - 360 if l >= 180 else lon[n] for n, l in enumerate(lon) ]) # => [0,180]U[-180,2.5] i_east, = np.where(tmp_lon >= 0) # indices of east lon i_west, = np.where(tmp_lon < 0) # indices of west lon lon = np.hstack((tmp_lon[i_west], tmp_lon[i_east])) # stack the 2 halves # Correspondingly, shift the 'air' array tmp_air = np.array(air) air = np.hstack((tmp_air[:, i_west], tmp_air[:, i_east]))
def getNetCDFHeader(cls, filename): """Get the header information from a GMT NetCDF3 file. :param fname: File name of GMT NetCDF3 grid :returns: - GeoDict specifying spatial extent, resolution, and shape of grid inside NetCDF file. - xvar array specifying X coordinates of data columns - xvar array specifying Y coordinates of data rows """ cdf = netcdf.netcdf_file(filename) geodict = {} xvarname = None registration = 'gridline' if hasattr(cdf, 'node_offset') and getattr(cdf, 'node_offset') == 1: registration = 'pixel' if 'x' in cdf.variables.keys(): xvarname = 'x' yvarname = 'y' elif 'lon' in cdf.variables.keys(): xvarname = 'lon' yvarname = 'lat' if xvarname is not None: xvar = cdf.variables[xvarname].data.copy() #GMT grids can have longitudes from 0-360, we're correcting for that here #however, sometimes GMT (or at least netcdf) allows an array for longitude centers #where the max value exceeds +180.0. Here we're also trapping for that situation, #and hopefully not catastrophically breaking a bunch of other stuff. if (xvar > 180.01).any(): xvar = xvar - 360 yvar = cdf.variables[yvarname].data.copy() geodict['nx'] = len(xvar) geodict['ny'] = len(yvar) geodict['xmin'] = xvar.min() geodict['xmax'] = xvar.max() geodict['ymin'] = yvar.min() geodict['ymax'] = yvar.max() newx = np.linspace(geodict['xmin'], geodict['xmax'], num=geodict['nx']) newy = np.linspace(geodict['ymin'], geodict['ymax'], num=geodict['ny']) geodict['dx'] = newx[1] - newx[0] geodict['dy'] = newy[1] - newy[0] elif 'x_range' in cdf.variables.keys(): geodict['xmin'] = cdf.variables['x_range'].data[0] geodict['xmax'] = cdf.variables['x_range'].data[1] geodict['ymin'] = cdf.variables['y_range'].data[0] geodict['ymax'] = cdf.variables['y_range'].data[1] geodict['nx'], geodict['ny'] = cdf.variables['dimension'].data #geodict['dx'],geodict['dy'] = cdf.variables['spacing'].data xvar = np.linspace(geodict['xmin'], geodict['xmax'], num=geodict['nx']) yvar = np.linspace(geodict['ymin'], geodict['ymax'], num=geodict['ny']) geodict['dx'] = xvar[1] - xvar[0] geodict['dy'] = yvar[1] - yvar[0] else: raise DataSetException( 'No support for CDF data file with variables: %s' % str(cdf.variables.keys())) #We are going to represent all grids internally as grid-line registered #The difference between pixel and gridline-registered grids is depicted well here: #http://gmt.soest.hawaii.edu/doc/5.1.0/GMT_Docs.html#grid-registration-the-r-option if registration == 'pixel': geodict['xmin'] += geodict['dx'] / 2.0 geodict['xmax'] -= geodict['dx'] / 2.0 geodict['ymin'] += geodict['dy'] / 2.0 geodict['ymax'] -= geodict['dy'] / 2.0 #because dx/dy are not explicitly defined in netcdf headers, here we'll assume #that those values are adjustable, and we'll preserve the shape and extent. gd = GeoDict(geodict, adjust='res') return (gd, xvar, yvar)
def write_coordinates(self, atoms, filename=''): """ write amber coordinates in netCDF format, only rectangular unit cells are allowed""" if filename == '': filename = self.incoordfile fout = netcdf.netcdf_file(filename, 'w') # dimension fout.Conventions = 'AMBERRESTART' fout.ConventionVersion = "1.0" fout.title = 'Ase-generated-amber-restart-file' fout.application = "AMBER" fout.program = "ASE" fout.programVersion = "1.0" fout.createDimension('cell_spatial', 3) fout.createDimension('label', 5) fout.createDimension('cell_angular', 3) fout.createDimension('time', 1) time = fout.createVariable('time', 'd', ('time', )) time.units = 'picosecond' time[0] = 0 fout.createDimension('spatial', 3) spatial = fout.createVariable('spatial', 'c', ('spatial', )) spatial[:] = np.asarray(list('xyz')) # spatial = 'xyz' natom = len(atoms) fout.createDimension('atom', natom) coordinates = fout.createVariable('coordinates', 'd', ('atom', 'spatial')) coordinates.units = 'angstrom' coordinates[:] = atoms.get_positions()[:] if atoms.get_velocities() is not None: velocities = fout.createVariable('velocities', 'd', ('atom', 'spatial')) velocities.units = 'angstrom/picosecond' velocities[:] = atoms.get_velocities()[:] # title cell_angular = fout.createVariable('cell_angular', 'c', ('cell_angular', 'label')) cell_angular[0] = np.asarray(list('alpha')) cell_angular[1] = np.asarray(list('beta ')) cell_angular[2] = np.asarray(list('gamma')) # title cell_spatial = fout.createVariable('cell_spatial', 'c', ('cell_spatial', )) cell_spatial[0], cell_spatial[1], cell_spatial[2] = 'a', 'b', 'c' # data cell_lengths = fout.createVariable('cell_lengths', 'd', ('cell_spatial', )) cell_lengths.units = 'angstrom' cell_lengths[0] = atoms.get_cell()[0, 0] cell_lengths[1] = atoms.get_cell()[1, 1] cell_lengths[2] = atoms.get_cell()[2, 2] cell_angles = fout.createVariable('cell_angles', 'd', ('cell_angular', )) box_alpha, box_beta, box_gamma = 90.0, 90.0, 90.0 cell_angles[0] = box_alpha cell_angles[1] = box_beta cell_angles[2] = box_gamma cell_angles.units = 'degree' fout.close()
diag = 'atmos_daily' diago = 'eflx2d' npert = np.size(pert) var1_name = 'vcomp' #if adjust, vcomp must be var1 var2_name = 'temp' varo_name = 'vTtr_zm' adj = True #T if vcomp, F if omega (need different adjustment) sind = range(npert) init = True init3d = False pind = 0 for si in sind: atmdir = indir + exper + pert[si] + plat + reg + '/history/' stafile = atmdir + '00000.atmos_average.nc' fs = [] fs.append(nc.netcdf_file(stafile, 'r', mmap=True)) bk = fs[-1].variables['bk'][:].astype(np.float64) pk = fs[-1].variables['pk'][:].astype(np.float64) lat = fs[-1].variables['lat'][:].astype(np.float64) lon = fs[-1].variables['lon'][:].astype(np.float64) pfull = fs[-1].variables['pfull'][pind:].astype(np.float64) #zsurf = fs[-1].variables['zsurf'][:].astype(np.float64) fs[-1].close() nlat = np.size(lat) nlon = np.size(lon) nlev = np.size(pfull) #%% filename = atmdir + '00000.atmos_daily.nc' fs.append(nc.netcdf_file(filename, 'r', mmap=True)) vcomp = fs[-1].variables['vcomp'][500:, pind:, :, :].astype( np.float64) #t,p,lat,lon
def test_read_withFillValNaN(): fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc') with netcdf_file(fname, maskandscale=True) as f: vardata = f.variables['var5_fillvalNaN'][:] assert_mask_matches(vardata, [False, True, False])
def read_E_out(self, ref_file, rec_file): """read data from the output file, produce the received E signal, return the complex E """ #print 'reading file:',file f = nc.netcdf_file(ref_file, 'r') #print 'finish reading.' if (self.dimension == 2): y = np.copy(f.variables['a_y'].data) z = np.copy(f.variables['a_z'].data) z_idx = f.dimensions['a_nz'] / 2 - 1 # FWR2D output doesn't include the central ray WKB phase advance in # paraxial region. The complete result needs to take into account # this additional phase ephi_wkb = f.variables['p_phaser'][...] +\ 1j * f.variables['p_phasei'][...] D_ephi = ephi_wkb[0] / ephi_wkb[-1] E_ref = np.copy(f.variables['a_Er'][1,z_idx,:] + \ 1j*f.variables['a_Ei'][1,z_idx,:])*D_ephi**2 f.close() receiver = c5.C5_reader(rec_file) E_rec = receiver.E_re_interp(z, y) + 1j * receiver.E_im_interp(z, y) E_out = np.trapz(E_ref * np.conj(E_rec[z_idx, :]), x=y) return E_out elif (self.dimension == 3): y = np.copy(f.variables['a_y'][:]) z = np.copy(f.variables['a_z'][:]) E_ref_re_interp = interp2d(y, z, f.variables['a_Er'][1, :, :], kind='cubic', fill_value=0) E_ref_im_interp = interp2d(y, z, f.variables['a_Ei'][1, :, :], kind='cubic', fill_value=0) f.close() receiver = c5.C5_reader(rec_file) #use area average to estimate E_ref*np.conj(E_receive) integrated over y,z dimension. ymin = np.max([y[0], receiver.X1D[0]]) ymax = np.min([y[-1], receiver.X1D[-1]]) zmin = np.max([z[0], receiver.Y1D[0]]) zmax = np.min([z[-1], receiver.Y1D[-1]]) y_fine = np.linspace(ymin, ymax, 200) z_fine = np.linspace(zmin, zmax, 200) self.E_ref = E_ref_re_interp(y_fine, z_fine)+ \ 1j*E_ref_im_interp(y_fine, z_fine) self.E_rec = receiver.E_re_interp(z_fine,y_fine)+ \ 1j*receiver.E_im_interp(z_fine,y_fine) E_conv = self.E_ref * np.conj(self.E_rec) dy = y_fine[1] - y_fine[0] dz = z_fine[1] - z_fine[0] # integration over y direction SEy = np.sum(E_conv[:, :-1] + E_conv[:, 1:], axis=1) / 2 * dy # integration over z direction E_out = np.sum(SEy[:-1] + SEy[1:]) / 2 * dz return E_out
def test_read_withValuesNearFillValue(): # Regression test for ticket #5626 fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc') with netcdf_file(fname, maskandscale=True) as f: vardata = f.variables['var1_fillval0'][:] assert_mask_matches(vardata, [False, True, False])
def doit(): with netcdf_file(filename, mmap=True) as f: return f.variables['lat'][:]
] fig, axarr = plt.subplots(3, 1, sharex='col', figsize=(8, 16)) for i, ncfile in enumerate(filenames1): a = 6400000.0 * afac #/20.0 g = 9.8 kappa = 0.286 M_air = 28.98 k = 1.38E-23 R = 8.3144621 p0 = 100000 * pfac R_spec = R / M_air * 1000 # 1000g per kg cp = 1003 f = netcdf.netcdf_file(ncfile, 'r', mmap=False) time = f.variables['time'] lat = f.variables['lat'] lon = f.variables['lon'] lev = f.variables['lev'] #* pfac t = f.variables['ta'] u = f.variables['ua'] v = f.variables['va'] o = f.variables['wap'] lev = lev[:] #* pfac p = lev[:] print lev[:] nlat = lat.shape[0] nlon = lon.shape[0]
def main(infile, outdir, fname_fmt='%Y%m%d.tif', params={}, unscale=False, skip_existing=False, progress=None): """Extract all time slices from netCDF file naming them after date Parameters ---------- ncfname : str path to netCDF file outdir : str output directory fname_fmt : str format string for datetime.datetime.strftime() that generates the target file name e.g. '%Y%m%d.tif' will give '20160101.tif' params : dict parameters passed to processing.runalg('gdalogr:translate') unscale : bool apply GDAL's -unscale option and also fix the nodata value skip_existing : bool skip existing files if False, existing files will be overwritten progress : QGIS progress, optional if run in QGIS """ # set common parameters params = params.copy() params['INPUT'] = infile params['EXPAND'] = 0 _common_extra = params.get('EXTRA', '') # get time data with netcdf.netcdf_file(infile, 'r') as ds: timevar = ds.variables['time'] timedata = netcdf_utils.num2date(timevar.data, timevar) # set extent lonlim = _get_minmax(ds.variables['lon'].data) latlim = _get_minmax(ds.variables['lat'].data) params['PROJWIN'] = '{0[0]},{0[1]},{1[0]},{1[1]}'.format( lonlim, latlim) # unscale if unscale: if '-unscale' not in _common_extra: _common_extra += ' -unscale' # fixing nodata value (gdal_translate defficiency) new_nodata = _get_scaled_nodata(ds, progress) if new_nodata is not None: _common_extra += ' -a_nodata {}'.format(new_nodata) # export time slices to tif files for i, date in enumerate(timedata): # set specific parameters fname = date.strftime(fname_fmt) params['OUTPUT'] = os.path.join(outdir, fname) params['EXTRA'] = (_common_extra + ' -b {}'.format(i + 1)).strip() # skip existing if skip_existing and os.path.exists(params['OUTPUT']): continue # run gdal_translate with parameters processing.runalg("gdalogr:translate", params) if not os.path.exists(params['OUTPUT']): raise ValueError('Algorithm failed. ' 'No output file was created. ' 'Parameters to gdalogr:translate were:\n' '{}'.format(params.__repr__())) # log if progress is not None: progress.setConsoleInfo("Time slice {} to file {}".format( date, fname))
for v in range(0, nvar): nsteps = 0 for y in range(ystart, yend + 1): yst = str(10000 + y)[1:5] for m in range(0, 12): mst = str(101 + m)[1:3] #myfile = os.path.abspath(mydir+'/'+mycases[c]+'_'+mysites[c]+'_'+mycompsets[c]+ \ # ".clm2."+hst+"."+yst+"-"+mst+".nc") #myfile = os.path.abspath(mydir+'/'+runnames[c]+".clm2."+hst+"."+yst+"-"+mst+".nc") myfile = os.path.abspath(mydir + '/' + runnames[c] + "." + options.model_name + "." + hst + "." + yst + "-" + mst + ".nc") #get units/long names from first file if (os.path.exists(myfile)): if (y == ystart and m == 0 and c == 0): nffile = netcdf.netcdf_file(myfile, "r") varout = nffile.variables[myvars[v]] var_long_names.append( varout.long_name.decode('utf_8')) nffile.close() if (float(options.scale_factor) < -900): if ('gC/m^2/s' in varout.units): myscalefactors.append(3600 * 24) var_units.append('g.C/m2/day') else: myscalefactors.append(1.0) var_units.append( varout.units.decode('utf_8').replace( '^', '')) else: myscalefactors.append(
import numpy as np from scipy.io import netcdf import math # set general text options for plotting plt.rc('text' , usetex=True) plt.rc('font' , size=12) plt.rc('legend' , fontsize=12) plt.rc('text.latex' , preamble=r'\usepackage{cmbright}') # almost_black = '#262626' abrupt2xCO2_prp = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt2xCO2_echam6_prp_mean_anomalies.nc') abrupt2xCO2_bot = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt2xCO2_echam6_BOT_mean_anomalies.nc') abrupt4xCO2_prp = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt4xCO2_echam6_prp_mean_anomalies.nc') abrupt4xCO2_bot = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt4xCO2_echam6_BOT_mean_anomalies.nc') abrupt8xCO2_prp = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt8xCO2_echam6_prp_mean_anomalies.nc') abrupt8xCO2_bot = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt8xCO2_echam6_BOT_mean_anomalies.nc') abrupt16xCO2_prp = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt16xCO2_echam6_prp_mean_anomalies.nc') abrupt16xCO2_bot = netcdf.netcdf_file('../Data/mpiesm-1.2.prp_abrupt16xCO2_echam6_BOT_mean_anomalies.nc') amip4xCO2 = netcdf.netcdf_file('../Data/BOT_echam-6.3.02p4_amip4xCO2_1979-2008_timmean_fldmean_anomaly.nc') # --------------------------------------------------- fig, axes = plt.subplots(1,1, figsize=(5,7)) id=np.arange(0,299)
from scipy.io import netcdf import matplotlib.pyplot as plt import numpy as np import math import sys if len(sys.argv) != 2: print "Error! Wrong number of arguments" exit(1) filename = sys.argv[1] #output file f = netcdf.netcdf_file(filename, 'r', mmap=False) theta = f.variables['theta_coil'][()] zeta = f.variables['zeta_coil'][()] nfp = f.variables['nfp'][()] #number of filed periodicity?? net_poloidal_current_Amperes = f.variables['net_poloidal_current_Amperes'][( )] #?? current_potential = f.variables['current_potential'][()] f.close() plt.figure() plt.contour(zeta, theta, np.transpose(current_potential[9, :, :]), 10) plt.show()
def main(): #---------------- # Initializations #---------------- dataDir = '/data1/ancillary_data/fl0/eol/' dataFileTag = 'flab' fileExtTag = 'cdf' outDataDir = '/data1/ancillary_data/fl0/eol/' yearstr = '2016' #-------------------------- # Initialize variable lists #-------------------------- tempList = [] timeList = [] rhList = [] pressList = [] tempDPList = [] wdirList = [] wspdList = [] wmaxList = [] wsdevList = [] rmind = [] #-------------------- # Search through base # directory for files #-------------------- files = glob.glob(dataDir + 'flab.' + yearstr + '*.' + fileExtTag) if not files: print 'No files found for year: %s' % yearstr sys.exit() else: print ' %d files found for year: %s' % (len(files), yearstr) #------------------------- # Loop through found files #------------------------- for indvfile in files: cdfname = netcdf.netcdf_file(indvfile, 'r', mmap=False) # Open netcdf file # Get variables base_time = cdfname.variables['base_time'] time_offset = cdfname.variables['time_offset'] temp = cdfname.variables['tdry'] rh = cdfname.variables['rh'] press = cdfname.variables['pres'] tempDP = cdfname.variables['dp'] wdir = cdfname.variables['wdir'] wspd = cdfname.variables['wspd'] wmax = cdfname.variables['wmax'] wsdev = cdfname.variables['wsdev'] cdfname.close() #---------------------------------- # Create an actual time vector from # basetime and offset (unix time) #---------------------------------- total_time = base_time[()] + time_offset.data #total_time = [dt.datetime.utcfromtimestamp(indtime) for indtime in total_time] total_time = total_time.tolist() # Convert numpy array to list #------------------------------------------------------------ # There seems to be a timing issue in some of the data files. # time_offset can have unusually large numbers. Need to check # for this and delete observations. #------------------------------------------------------------ for ind, indvtime in enumerate(total_time): # Identify erroneous time_offset values try: total_time[ind] = dt.datetime.utcfromtimestamp(indvtime) except ValueError: total_time[ind] = -9999 rmind.append(ind) #------------------------------------------------------ # Remove observations with erroneous time_offset values #------------------------------------------------------ total_time[:] = [ item for ind, item in enumerate(total_time) if ind not in rmind ] # List temp.data = np.delete(temp.data, rmind) # Numpy array rh.data = np.delete(rh.data, rmind) # Numpy array press.data = np.delete(press.data, rmind) # Numpy array tempDP.data = np.delete(tempDP.data, rmind) # Numpy array wdir.data = np.delete(wdir.data, rmind) wspd.data = np.delete(wspd.data, rmind) wmax.data = np.delete(wmax.data, rmind) wsdev.data = np.delete(wsdev.data, rmind) # Numpy array #--------------------- # Append to main lists #--------------------- timeList.extend(total_time) tempList.extend(temp.data) rhList.extend(rh.data) pressList.extend(press.data) tempDPList.extend(tempDP.data) wdirList.extend(wdir.data) wspdList.extend(wspd.data) wmaxList.extend(wmax.data) wsdevList.extend(wsdev.data) #------------------------ # Sort list based on time # This returns a tuple #------------------------ timeList, tempList, rhList, pressList, tempDPList, wdirList, wspdList, wmaxList, wsdevList = zip( *sorted( zip(timeList, tempList, rhList, pressList, tempDPList, wdirList, wspdList, wmaxList, wsdevList))) #----------------------------------- # Construct a vector of string years #----------------------------------- years = ["{0:02d}".format(indtime.year) for indtime in timeList] months = ["{0:02d}".format(indtime.month) for indtime in timeList] days = ["{0:02d}".format(indtime.day) for indtime in timeList] hours = ["{0:02d}".format(indtime.hour) for indtime in timeList] minutes = ["{0:02d}".format(indtime.minute) for indtime in timeList] #-------------------------- # Write data to output file #-------------------------- with open(outDataDir + 'fl0_met_data_' + years[0] + '_v2.txt', 'w') as fopen: fopen.write( 'Year Month Day Hour Minute Temperature[C] RelativeHumidity[%] Pressure[mbars] DewPointTemperature[C] WinDir[rN] WinSpeed[m/s] WindSpeedMax[m/s]\n' ) for line in zip(years, months, days, hours, minutes, tempList, rhList, pressList, tempDPList, wdirList, wspdList, wmaxList): fopen.write( '%-4s %-5s %-3s %-4s %-6s %-14.1f %-19.1f %-15.1f %-22.6f %-15.4f %-15.4f %-15.4f\n' % line)
header = ['ncols 3600\n'] #header.append('nrows 2053\n') #Oahu header.append('nrows 1500\n') #header=['ncols 720\n']# % 30min #header.append('nrows 360\n')# % 30min #header.append('xllcorner -158.40027777733\n') #Oahu #header.append('yllcorner 21.22999999938\n') #Oahu header.append('xllcorner -180\n') header.append('yllcorner -60\n') #header.append('cellsize 0.000277777778\n') header.append('cellsize 0.1\n') #header.append('cellsize 0.5\n')# %30min header.append('NODATA_value -9999\n') NetCDF_fn = NetCDF_fn1 + str(sim_first + 1) + '.nc' # set input netcdf name f = netcdf.netcdf_file(NetCDF_fn, 'r') #if nutrient == 'Discharge': # valueAcc = f.variables['discharge'][:].astype(np.float32) #else: # valueAcc = f.variables[nutrient][:].astype(np.float16)/1000 #print valueAcc.shape value = f.variables[nutrient][[0][:][:]].astype(np.float32) #/1000 maxRunAvr = np.concatenate(value) f.close() np.putmask(maxRunAvr, maxRunAvr > 0, 0) #stdRas = np.copy(averageRas) #rangeRas = np.copy(averageRas) for r in range(0, 1500): #for r in range(0, 2053): #Oahu print r
#!/usr/bin/env python3 import numpy as np import matplotlib.pyplot as plt import json from scipy.io import netcdf with netcdf.netcdf_file('MSR-2.nc', mmap=False) as netcdf_file: variables = netcdf_file.variables print(*variables, sep=', ') print(variables['time'].dimensions) print(variables['Average_O3_column'].dimensions) print(variables['time'].units, variables['Average_O3_column'].units) Dushanbelong = 68.75 Dushanbelat = 38.57 a = np.searchsorted(variables['longitude'].data, Dushanbelong, sorter=None)#Вычисляем узел сетки с похожими коориданатми b = np.searchsorted(variables['latitude'].data, Dushanbelat, sorter=None) plt.plot(variables['time'][:], variables['Average_O3_column'][:, b, a], label = 'All data', color='grey')#рисуем график зависимости озона от времени plt.plot(variables['time'][::12], variables['Average_O3_column'][::12, b, a], label = 'January', color='orange')#для января plt.plot(variables['time'][6::12], variables['Average_O3_column'][6::12, b, a], label = 'July', color='purple') plt.xlabel("Months since 1970-01-15") plt.ylabel('Dobson units') plt.grid() plt.legend() plt.suptitle('Monthly average data 1979-2018') plt.savefig('ozon.png') plt.show() minall=min(variables['Average_O3_column'][:, b, a])#вычисляем минимальные/максимальные/средние значения для данных за весь период, январь и июль avgall=np.mean(variables['Average_O3_column'][:, b, a]) maxall=max(variables['Average_O3_column'][:, b, a]) minjan=min(variables['Average_O3_column'][::12, b, a]) avgjan=np.mean(variables['Average_O3_column'][::12, b, a])
def readNCFromTar(tar, file, var): TF = tarfile.open(tar, 'r') member = [m for m in TF.getmembers() if file in m.name][0] nc = netcdf.netcdf_file(TF.extractfile(member), 'r') return nc.variables[var]
def read_fesom_mesh(path, alpha, beta, gamma, read_diag=True): ''' .. note:: Deprecated in pyfesom 0.1 `read_fesom_mesh` will be removed in future, it is replaced by `load_mesh`. ''' mesh=fesom_mesh() mesh.path=path mesh.alpha=alpha mesh.beta=beta mesh.gamma=gamma nod2dfile=mesh.path+'nod2d.out' elm2dfile=mesh.path+'elem2d.out' aux3dfile=mesh.path+'aux3d.out' nod3dfile=mesh.path+'nod3d.out' file_content = pd.read_csv(nod2dfile, delim_whitespace=True, skiprows=1, \ names=['node_number','x','y','flag'] ) mesh.x2=file_content.x.values mesh.y2=file_content.y.values mesh.n2d=len(mesh.x2) file_content = pd.read_csv(elm2dfile, delim_whitespace=True, skiprows=1, \ names=['first_elem','second_elem','third_elem']) mesh.elem=file_content.values-1 mesh.e2d=np.shape(mesh.elem)[0] mesh.n3d=int(open(mesh.path+'nod3d.out').readline().rstrip()) df = pd.read_csv(nod3dfile, delim_whitespace=True, skiprows=1, \ names=['node_number','x','y','z','flag']) zcoord = -df.z.values mesh.zlevs = np.unique(zcoord) with open(aux3dfile) as f: mesh.nlev=int(next(f)) mesh.n32=np.array([next(f) for x in range(mesh.n2d*mesh.nlev)]).astype(int).reshape(mesh.n2d,mesh.nlev) mesh.topo=np.zeros(shape=(mesh.n2d)) for prof in mesh.n32: ind_nan = prof[prof>0] ind_nan=ind_nan[-1] mesh.topo[prof[0]-1]=zcoord[ind_nan-1] ########################################### #here we compute the volumes of the triangles #this should be moved into fesom generan mesh output netcdf file # r_earth=6371000.0 rad=np.pi/180 edx=mesh.x2[mesh.elem] edy=mesh.y2[mesh.elem] ed=np.array([edx, edy]) jacobian2D=ed[:, :, 1]-ed[:, :, 0] jacobian2D=np.array([jacobian2D, ed[:, :, 2]-ed[:, :, 0]]) for j in range(2): mind = [i for (i, val) in enumerate(jacobian2D[j,0,:]) if val > 355] pind = [i for (i, val) in enumerate(jacobian2D[j,0,:]) if val < -355] jacobian2D[j,0,mind]=jacobian2D[j,0,mind]-360 jacobian2D[j,0,pind]=jacobian2D[j,0,pind]+360 jacobian2D=jacobian2D*r_earth*rad for k in range(2): jacobian2D[k,0,:]=jacobian2D[k,0,:]*np.cos(edy*rad).mean(axis=1) mesh.voltri = abs(np.linalg.det(np.rollaxis(jacobian2D, 2)))/2. # compute the 2D lump operator cnt=np.array((0,)*mesh.n2d) mesh.lump2=np.array((0.,)*mesh.n2d) for i in range(3): for j in range(mesh.e2d): n=mesh.elem[j,i] #cnt[n]=cnt[n]+1 mesh.lump2[n]=mesh.lump2[n]+mesh.voltri[j] mesh.lump2=mesh.lump2/3. ########################################### #here we read the 3D cluster volumes if (read_diag): f = netcdf.netcdf_file(mesh.path+'mesh.diag.nc', 'r') mesh.cluster_vol3=f.variables['cluster_vol'].data mesh.cluster_vol2=f.variables['cluster_area'].data f.close() else: mesh.cluster_vol3=0 mesh.cluster_vol2=0 #we should rotate the mesh to the geographical coordinates (mesh.x2,mesh.y2)=ut.scalar_r2g(mesh.alpha,mesh.beta,mesh.gamma,mesh.x2,mesh.y2) d=mesh.x2[mesh.elem].max(axis=1)-mesh.x2[mesh.elem].min(axis=1) mesh.no_cyclic_elem = [i for (i, val) in enumerate(d) if val < 100] return mesh
def PyExec(self): # Get file path file_name = self.getPropertyValue("InputFile") # Load trajectory file trajectory = netcdf.netcdf_file(file_name, mode="r") logger.information("Loading particle id's and coordinate array...") start_time = time.time() # netcdf object containing the particle id numbers description = (trajectory.variables["description"])[:] # Convert description object to string via for loop. The original object has strange formatting particleID = '' for i in description: particleID += i.decode('UTF-8') # Extract particle id's from string using regular expressions p_atoms = re.findall(r"A\('[a-z]+\d+',\d+", particleID) # Many-to-one structures. Identify the set of atomic species present (list structure 'elements') # in the simulation and repackage particles into a dictionary # 'particles_to_species' with structure id number -> species atoms_to_species = {} species_to_atoms = {} elements = [] # Populate the particles_to_species dictionary and the elements list for j in p_atoms: key = re.findall(r"',\d+", j)[0] key = int(re.findall(r"\d+", key)[0]) element = re.findall(r"[a-z]+", j)[0] if element not in elements: elements.append(str(element)) atoms_to_species[key] = str(element) # Initialise lists in the species_to_particles dictionary for j in elements: species_to_atoms[j] = [] # Populate the species_to_particles dictionary for j in p_atoms: key = re.findall(r"',\d+", j)[0] key = int(re.findall(r"\d+", key)[0]) element = re.findall(r"[a-z]+", j)[0] species_to_atoms[element].append(key) # Coordinate array. Shape: timesteps x (# of particles) x (# of spatial dimensions) configuration = trajectory.variables["configuration"] # Extract useful simulation parameters # Number of species present in the simulation n_species = len(elements) # Number of particles present in the simulation n_particles = len(atoms_to_species) # Number of timesteps in the simulation n_timesteps = int(configuration.shape[0]) # Number of spatial dimensions n_dimensions = int(configuration.shape[2]) logger.information(str(time.time() - start_time) + " s") logger.information("Transforming coordinates...") start_time = time.time() # Box size for each timestep. Shape: timesteps x (3 consecutive 3-vectors) box_size = trajectory.variables["box_size"] # Reshape the paralellepipeds into 3x3 tensors for coordinate transformations. # Shape: timesteps x 3 vectors x (# of spatial dimensions) box_size_tensors = np.array( [box_size[j].reshape((3, 3)) for j in range(n_timesteps)]) # Copy the configuration object into a numpy array configuration_copy = np.array( [configuration[i] for i in range(n_timesteps)]) # Swap the time and particle axes configuration_copy = np.swapaxes(configuration_copy, 0, 1) #/1.12484770 # Get scaled coordinates (assumes orthogonal simulation box) scaled_coords = np.zeros(np.shape(configuration_copy)) for i in range(n_particles): for j in range(n_timesteps): for k in range(n_dimensions): scaled_coords[i, j, k] = configuration_copy[ i, j, k] / box_size_tensors[j, k, k] # # Transform particle trajectories (configuration array) to Cartesian coordinates at each time step logger.information(str(time.time() - start_time) + " s") logger.information("Calculating velocities...") start_time = time.time() # Initialise velocity arrray. Note that the first dimension is 2 elements shorter than the coordinate array. # Use finite difference methods to evaluate the time-derivative to 1st order # Shape: (# of particles) x (timesteps-2) x (# of spatial dimensions) velocities = np.zeros((n_particles, n_timesteps - 1, n_dimensions)) v1 = scaled_coords[:, 1:-1, :] - scaled_coords[:, :-2, :] - np.round( scaled_coords[:, 1:-1, :] - scaled_coords[:, :-2, :]) v2 = scaled_coords[:, 2:, :] - scaled_coords[:, 1:-1, :] - np.round( scaled_coords[:, 2:, :] - scaled_coords[:, 1:-1, :]) velocities[:, :-1, :] = (v1 + v2) / 2. # Transform velocities (configuration array) back to Cartesian coordinates at each time step velocities = np.array([[ np.dot(box_size_tensors[j + 1], np.transpose(velocities[i, j])) for j in range(n_timesteps - 1) ] for i in range(n_particles)]) logger.information(str(time.time() - start_time) + " s") logger.information( "Calculating velocity cross-correlations (resource intensive calculation)..." ) start_time = time.time() correlation_length = n_timesteps - 1 correlations = np.zeros((n_species, n_species, correlation_length)) # Array for counting particle pairings correlation_count = np.zeros((n_species, n_species)) # Compute cross-correlations for each pair of particles in each spatial coordinate for i in range(n_particles): for j in range(i + 1, n_particles): # Retrieve particle indices from the 'particles' dictionary and # determine the relevant position in the 'correlations' matrices k = elements.index(atoms_to_species[i]) l = elements.index(atoms_to_species[j]) # Check for the order of elements (ensures upper triangular matrix form & consistent order of operations) if k <= l: correlation_temp = self.cross_correlation( velocities[i], velocities[j]) correlations[k, l] += correlation_temp correlation_count[k, l] += 1 else: correlation_temp = self.cross_correlation( velocities[j], velocities[i]) correlations[l, k] += correlation_temp correlation_count[l, k] += 1 logger.information(str(time.time() - start_time) + " s") # Neutron coherent scattering lengths (femtometres) # Sources: # https://www.ncnr.nist.gov/resources/n-lengths/list.html # http://www.ati.ac.at/~neutropt/scattering/RecommendedScatteringLengthsOfElements.PDF Coh_b = { 'h': -3.7409, 'he': 3.26, 'li': -1.90, 'be': 7.79, 'b': 5.30, 'c': 6.6484, 'n': 9.36, 'o': 5.805, 'f': 5.654, 'ne': 4.60, 'na': 3.63, 'mg': 5.375, 'al': 3.449, 'si': 4.15071, 'p': 5.13, 's': 2.847, 'cl': 9.5792, 'ar': 1.909, 'k': 3.67, 'ca': 4.70, 'sc': 12.29, 'ti': -3.37, 'v': -0.443, 'cr': 3.635, 'mn': -3.73, 'fe': 9.45, 'co': 2.49, 'ni': 10.3, 'cu': 7.718, 'zn': 5.680, 'ga': 7.288, 'ge': 8.185, 'as': 6.58, 'se': 7.970, 'br': 6.79, 'kr': 7.81, 'rb': 7.08, 'sr': 7.02, 'y': 7.75, 'zr': 7.16, 'nb': 7.054, 'mo': 6.715, 'tc': 6.8, 'ru': 7.02, 'rh': 5.90, 'pd': 5.91, 'ag': 5.922, 'cd': 4.83, 'in': 4.065, 'sn': 6.225, 'sb': 5.57, 'te': 5.68, 'i': 5.28, 'xe': 4.69, 'cs': 5.42, 'ba': 5.07, 'la': 8.24, 'ce': 4.84, 'pr': 4.58, 'nd': 7.69, 'pm': 12.6, 'sm': 0.00, 'eu': 5.3, 'gd': 9.5, 'tb': 7.34, 'dy': 16.9, 'ho': 8.44, 'er': 7.79, 'tm': 7.07, 'yb': 12.41, 'lu': 7.21, 'hf': 7.77, 'ta': 6.91, 'w': 4.755, 're': 9.2, 'os': 10.7, 'ir': 10.6, 'pt': 9.60, 'au': 7.63, 'hg': 12.66, 'tl': 8.776, 'pb': 9.401, 'bi': 8.532, 'po': None, 'at': None, 'rn': None, 'fr': None, 'ra': 10.0, 'ac': None, 'th': 10.31, 'pa': 9.1, 'u': 8.417, 'np': 10.55, 'pu': None, 'am': 8.3, 'cm': 9.5 } logger.information( "Averaging correlation Fourier transforms & scaling with the coherent neutron scattering lenghts..." ) start_time = time.time() # Scaling cross-correlations with the scattering lengths for i in range(n_species): for j in range(i, n_species): correlations[i, j] = correlations[i, j] * Coh_b[ elements[i]] * Coh_b[elements[j]] / correlation_count[i, j] logger.information(str(time.time() - start_time) + " s") # Generate a list of row names according to the atomic species present in the simulation row_names = [] for i in range(n_species): for j in range(i, n_species): row_names.append(elements[i].capitalize() + ' and ' + elements[j].capitalize()) # Initialise & populate the output_ws workspace nrows = int((n_species * n_species - n_species) / 2 + n_species) #nbins=(np.shape(correlations)[2]) yvals = np.empty(0) for i in range(n_species): for j in range(i, n_species): # Add folded correlations to the array passed to the workspace yvals = np.append(yvals, self.fold_correlation(correlations[i, j])) # Timesteps between coordinate positions step = float(self.getPropertyValue("Timestep")) xvals = np.arange(0, np.ceil((n_timesteps - 1) / 2.0)) * step / 1000.0 evals = np.zeros(np.shape(yvals)) output_name = self.getPropertyValue("OutputWorkspace") output_ws = CreateWorkspace(OutputWorkspace=output_name, DataX=xvals, DataY=yvals, DataE=evals, NSpec=nrows, VerticalAxisUnit="Text", VerticalAxisValues=row_names, UnitX="ps") # Set output workspace to output_ws self.setProperty('OutputWorkspace', output_ws)
def test_read_write_files(): # test round trip for example file cwd = os.getcwd() try: tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) with make_simple('simple.nc', 'w') as f: pass # read the file we just created in 'a' mode with netcdf_file('simple.nc', 'a') as f: check_simple(f) # add something f._attributes['appendRan'] = 1 # To read the NetCDF file we just created:: with netcdf_file('simple.nc') as f: # Using mmap is the default assert_(f.use_mmap) check_simple(f) assert_equal(f._attributes['appendRan'], 1) # Read it in append (and check mmap is off) with netcdf_file('simple.nc', 'a') as f: assert_(not f.use_mmap) check_simple(f) assert_equal(f._attributes['appendRan'], 1) # Now without mmap with netcdf_file('simple.nc', mmap=False) as f: # Using mmap is the default assert_(not f.use_mmap) check_simple(f) # To read the NetCDF file we just created, as file object, no # mmap. When n * n_bytes(var_type) is not divisible by 4, this # raised an error in pupynere 1.0.12 and scipy rev 5893, because # calculated vsize was rounding up in units of 4 - see # http://www.unidata.ucar.edu/software/netcdf/docs/netcdf.html with open('simple.nc', 'rb') as fobj: with netcdf_file(fobj) as f: # by default, don't use mmap for file-like assert_(not f.use_mmap) check_simple(f) # Read file from fileobj, with mmap with open('simple.nc', 'rb') as fobj: with netcdf_file(fobj, mmap=True) as f: assert_(f.use_mmap) check_simple(f) # Again read it in append mode (adding another att) with open('simple.nc', 'r+b') as fobj: with netcdf_file(fobj, 'a') as f: assert_(not f.use_mmap) check_simple(f) f.createDimension('app_dim', 1) var = f.createVariable('app_var', 'i', ('app_dim',)) var[:] = 42 # And... check that app_var made it in... with netcdf_file('simple.nc') as f: check_simple(f) assert_equal(f.variables['app_var'][:], 42) except: os.chdir(cwd) shutil.rmtree(tmpdir) raise os.chdir(cwd) shutil.rmtree(tmpdir)
def test_read_with2dVar(): fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc') with netcdf_file(fname, maskandscale=True) as f: vardata = f.variables['var7_2d'][:] assert_mask_matches(vardata, [[True, False], [False, False], [False, True]])
def pyseidon_to_netcdf(fvcom, filename, debug): """ saves fvcom object in a pickle file inputs: - fvcom = fvcom pyseidon object - filename = file name, string """ #Define bounding box if debug: print "Computing bounding box..." if fvcom.Grid._ax == []: lon = fvcom.Grid.lon[:] lat = fvcom.Grid.lat[:] fvcom.Grid._ax = [lon.min(), lon.max(), lat.min(), lat.max()] filename = filename + ".nc" f = netcdf.netcdf_file(filename, 'w') #history attribut f.history = fvcom.History[:] #create dimensions if not fvcom.Variables._3D: ##2D dimensions dims = { 'three': 3, 'four': 4, 'nele': fvcom.Grid.nele, 'node': fvcom.Grid.nnode, 'siglay': 2, 'siglev': 3, 'time': fvcom.Variables.julianTime.shape[0] } else: ##3D dimensions dims = { 'three': 3, 'four': 4, 'nele': fvcom.Grid.nele, 'node': fvcom.Grid.nnode, 'siglay': fvcom.Grid.nlevel, 'siglev': fvcom.Grid.nlevel + 1, 'vertshear': fvcom.Grid.nlevel - 1, 'time': fvcom.Variables.julianTime.shape[0] } for key in dims.keys(): f.createDimension(key, dims[key]) #list of potential 2D var varname = [ 'el', 'ua', 'va', 'julianTime', 'matlabTime', 'depth_av_flow_dir', 'hori_velo_norm', 'depth_av_vorticity', 'depth_av_power_density', 'depth_av_power_assessment' ] #list of potential 2D grid var gridname = [ 'a1u', 'a2u', 'trinodes', 'triele', 'xc', 'x', 'yc', 'y', 'lonc', 'lon', 'latc', 'lat', 'aw0', 'awy', 'awx', 'h', 'hc', 'depth2D' ] #list of potential 3D var * grid var if fvcom.Variables._3D: varname = varname + [ 'u', 'v', 'flow_dir', 'velo_norm', 'verti_shear', 'vorticity', 'power_density' ] gridname = gridname + ['siglay', 'siglev', 'depth'] #load in netcdf file if debug: print "Loading variables' matrices in nc file..." for var in varname: if var in [ 'ua', 'va', 'depth_av_flow_dir', 'depth_av_vorticity', 'depth_av_power_density', 'depth_av_power_assessment', 'hori_velo_norm' ]: try: tmp_var = f.createVariable(var, 'float', ('time', 'nele')) f.variables[var][:] = getattr(fvcom.Variables, var)[:] except AttributeError: pass if var in ['julianTime', 'matlabTime']: try: tmp_var = f.createVariable(var, 'float', ('time', )) f.variables[var][:] = getattr(fvcom.Variables, var)[:] except AttributeError: pass if var == 'el': try: tmp_var = f.createVariable(var, 'float', ('time', 'node')) f.variables[var][:] = getattr(fvcom.Variables, var)[:] except AttributeError: pass if fvcom.Variables._3D: if var in [ 'u', 'v', 'flow_dir', 'velo_norm', 'vorticity', 'power_density' ]: try: tmp_var = f.createVariable(var, 'float', ('time', 'siglay', 'nele')) f.variables[var][:] = getattr(fvcom.Variables, var)[:] except AttributeError: pass if var in ['verti_shear']: try: tmp_var = f.createVariable(var, 'float', ('time', 'vertshear', 'nele')) f.variables[var][:] = getattr(fvcom.Variables, var)[:] except AttributeError: pass if debug: print "Loading grid' matrices in nc file..." for grd in gridname: if grd in ['xc', 'yc', 'lonc', 'latc', 'hc']: try: tmp_var = f.createVariable(grd, 'float', ('nele', )) tmp_var[:] = getattr(fvcom.Grid, grd)[:] #f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd == 'depth2D': try: depth2D = f.createVariable(grd, 'float', ('time', 'nele')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd in ['x', 'y', 'lon', 'lat', 'h']: try: tmp_var = f.createVariable(grd, 'float', ('node', )) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd in ['triele', 'trinodes']: try: tmp_var = f.createVariable(grd, 'i', ('nele', 'three')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd in ['a1u', 'a2u']: try: tmp_var = f.createVariable(grd, 'i', ('four', 'nele')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd in ['aw0', 'awy', 'awx']: try: tmp_var = f.createVariable(grd, 'i', ('three', 'nele')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if fvcom.Variables._3D: if grd == 'siglay': try: siglay = f.createVariable(grd, 'float', ('siglay', 'node')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd == 'siglev': try: siglev = f.createVariable(grd, 'float', ('siglev', 'node')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass if grd == 'depth': try: depth = f.createVariable(grd, 'float', ('time', 'siglay', 'nele')) f.variables[grd][:] = getattr(fvcom.Grid, grd)[:] except AttributeError: pass f.close() if debug: print "...done"
def read_amber_traj(parm7_file, nc_file, mdfrc_file, mden_file): """The amber trajectory includes: * nc, NetCDF format, stores coordinates * mdfrc, NetCDF format, stores forces * mden, text format, stores energies * parm7, text format, stores types """ flag = False amber_types = [] with open(parm7_file) as f: for line in f: if line.startswith("%FLAG"): flag = line.startswith("%FLAG AMBER_ATOM_TYPE") elif flag: if line.startswith("%FORMAT"): fmt = re.findall(r'\d+', line) fmt0 = int(fmt[0]) fmt1 = int(fmt[1]) else: for ii in range(fmt0): start_index = ii * fmt1 end_index = (ii + 1) * fmt1 if end_index >= len(line): continue amber_types.append(line[start_index:end_index].strip()) with netcdf.netcdf_file(nc_file, 'r') as f: coords = np.array(f.variables["coordinates"][:]) cell_lengths = np.array(f.variables["cell_lengths"][:]) cell_angles = np.array(f.variables["cell_angles"][:]) if np.all(cell_angles > 89.99) and np.all(cell_angles < 90.01): # only support 90 # TODO: support other angles shape = cell_lengths.shape cells = np.zeros((shape[0], 3, 3)) for ii in range(3): cells[:, ii, ii] = cell_lengths[:, ii] else: raise RuntimeError("Unsupported cells") with netcdf.netcdf_file(mdfrc_file, 'r') as f: forces = np.array(f.variables["forces"][:]) # energy energies = [] with open(mden_file) as f: for line in f: if line.startswith("L6"): s = line.split() if s[2] != "E_pot": energies.append(float(s[2])) atom_names, atom_types, atom_numbs = np.unique(amber_types, return_inverse=True, return_counts=True) data = {} data['atom_names'] = list(atom_names) data['atom_numbs'] = list(atom_numbs) data['atom_types'] = atom_types data['forces'] = forces * force_convert data['energies'] = np.array(energies) * energy_convert data['coords'] = coords data['cells'] = cells data['orig'] = np.array([0, 0, 0]) return data
thePrec = np.transpose(thePrec) # print(thePrec) # exit() #print (thePrec.shape) #exit() lats = extract_h4_by_name(the_filename, 'Grid/lat') lons = extract_h4_by_name(the_filename, 'Grid/lon') isif = thePrec.shape allday_prec.append(thePrec) allday_prec = np.array(allday_prec) daily_prec = np.nanmean(allday_prec, axis=0) outfile = output_dir + 'daily_precipitation_' + idate + '.nc4' # create nc file fid = netcdf.netcdf_file(outfile, 'w') # create dimension variable, so we can use it in the netcdf fid.createDimension('longitude', isif[1]) fid.createDimension('latitude', isif[0]) nc_var = fid.createVariable('nlat', 'f4', ('latitude', )) nc_var[:] = lats nc_var.long_name = 'latitude' nc_var.standard_name = 'latitude' nc_var.units = 'degrees_north' nc_var = fid.createVariable('nlon', 'f4', ('longitude', )) nc_var[:] = lons nc_var.long_name = 'longitude' nc_var.standard_name = 'longitude' nc_var.units = 'degrees_east'
yrstr.append(yrC) ind = range(nsim) nfile = np.size(yrstr) """ check before this line """ #%% #npert = 1 land_mask = 0 for vi in range(nvar): for i in ind: atmdir = indir + exper + sim[i] + '/' + plat + 'pp/' + diag + '/' stafile = atmdir + diag + '.static.nc' fs = [] fs.append(nc.netcdf_file(stafile, 'r', mmap=True)) bk = fs[-1].variables['bk'][:].astype(np.float64) pk = fs[-1].variables['pk'][:].astype(np.float64) lat = fs[-1].variables['lat'][:].astype(np.float64) lon = fs[-1].variables['lon'][:].astype(np.float64) phalf = fs[-1].variables['phalf'][:].astype(np.float64) zsurf = fs[-1].variables['zsurf'][:].astype(np.float64) if ('land_mask' in fs[-1].variables): land_mask = fs[-1].variables['land_mask'][:].astype(np.float64) fs[-1].close() nlat = np.size(lat) nlon = np.size(lon) nphalf = np.size(phalf) #%% filedir = atmdir + indir_sub files = []
if os.path.isfile(interppath): ''' print('Interpolated data found at: %s' %interppath) interpdict = np.load(interppath).item() # load here the above pickle ncdict = wrf.extract_vars(wrfdata, None, ('GRNHFX')) ''' print('Interpolated data found at: %s' % interppath) interpfile = open(interppath, 'rb') interpdict = pickle.load(interpfile) # load here the above pickle else: print( 'WARNING: no interpolated data found - generating: SLOW ROUTINE!') wrfpath = plume.wrfdir + 'wrfout_' + Case wrfdata = netcdf.netcdf_file(wrfpath, mode='r') ncdict = wrf.extract_vars(wrfdata, None, ('GRNHFX', 'W', 'QVAPOR', 'T', 'PHB', 'PH', 'U', 'P', 'PB', 'V', 'tr17_1'), meta=False) ncdict['PM25'] = ncdict.pop('tr17_1') #get height and destagger vars zstag = (ncdict['PHB'] + ncdict['PH']) // 9.81 z = wrf.destagger(zstag, 1) u = wrf.destagger(ncdict['U'], 3) w = wrf.destagger(ncdict['W'], 1) v = wrf.destagger(ncdict['V'], 2) #list variables to interpolate nT, nZ, nY, nX = np.shape(z)
def test_read_example_data(): # read any example data files for fname in glob(pjoin(TEST_DATA_PATH, '*.nc')): f = netcdf_file(fname, 'r') f = netcdf_file(fname, 'r', mmap=False)