def bom_ascii_to_nc(year,dates,froot): debug = False latvec = None lonvec = None ncobj = None latslice = slice(None,None,None) lonslice = slice(None,None,None) missed_dates = [] adict = {} latvec,lonvec,adict = get_basics() miss = adict['missing'] for dti,dt in enumerate(dates): index = (slice(dti,None,None),latslice,lonslice) # Initialise the netcdf object if ncobj is None: if debug: print "Define ncobj:",dt dni_var = 'solar_dni' ghi_var = 'solar_ghi' obs_var = 'obs_time' ncobj = nb.nc_open(froot+'.nc','w',format='NETCDF4_CLASSIC') nb.nc_set_timelatlon(ncobj,None,len(latvec),len(lonvec)) nb.nc_set_var(ncobj,dni_var,fill=miss,zlib=True) nb.nc_set_var(ncobj,ghi_var,fill=miss,zlib=True) nb.nc_set_var(ncobj,obs_var,fill=miss,zlib=True) nb.nc_set_var(ncobj,'crs',dims=(),dtype="i4") # Grid mapping container nb.nc_add_data(ncobj,'latitude',latvec) nb.nc_add_data(ncobj,'longitude',lonvec) dni_nc = ncobj.variables[dni_var] ghi_nc = ncobj.variables[ghi_var] obs_nc = ncobj.variables[obs_var] # Calculate the observation time layer obs_data = create_obs_time(latvec,lonvec,dt) obs_nc[dti,:,:] = obs_data # Add time values nb.nc_add_time(ncobj,dates) # Create an ordered metadata dictionary meta = create_meta(dates[0],dates[-1],adict) # Output the file-specific metadata to a JSON file regardless of the # requested output formats jh.json_dump(meta, froot+'.json') # Setting attributes here is optional in this example because it is # known that netcdf_json_wrapper.py will be called next with # 'froot+".json"' as one of the input files. nb.nc_set_attributes(ncobj,meta) if debug: print "Added attributes" nb.nc_close(ncobj) print "Wrote: "+froot+".nc" return froot
def asciigrid_to_nc(arcfilename,fileroot): """ The main routine that calls the calls other routines to prepare the data and metadata and create the netCDF file. """ # Read ascii grid file asciihead,asciidata,asciitail = split_asciigrid(arcfilename) meta = arcasciihdr_to_dict(asciihead) # Create numpy components latvec,lonvec,datadict = set_latlon(meta) data,datadict = asciigrid_to_numpy(asciidata,meta,datadict) # Resample or edit array # Add a default no_data value if required. miss = -999.0 if 'nodata_value' in meta: miss = float(meta['nodata_value']) datadict['missing'] = miss #data,latvec,lonvec,datadict = resample_array(data,latvec,lonvec,datadict) # Prepare time, variable name and metadata d1,d2,datadict = set_datetime(arcfilename,datadict) datadict = set_varname(arcfilename,datadict) attributes = set_attributes(arcfilename,meta,datadict) # Netcdf options # http://netcdf4-python.googlecode.com/svn/trunk/docs/netCDF4-module.html nc_format = 'NETCDF4_CLASSIC' nc_compress = True debug = False # Write netcdf file if os.path.exists(fileroot+'.nc'): os.remove(fileroot+'.nc') varname = datadict['varname'] vartype = datadict['datatype'] fillval = datadict['missing'] timevec = [d1] ncobj = nb.ncopen(fileroot+'.nc','w',format=nc_format) nb.set_timelatlon(ncobj,None,len(latvec),len(lonvec)) # unlimited time nb.set_variable(ncobj,varname,dtype=vartype,fill=fillval,zlib=nc_compress) nb.set_variable(ncobj,'crs',dims=(),dtype="i4") # Grid mapping container nb.add_time(ncobj,timevec) nb.add_data(ncobj,'latitude',latvec) nb.add_data(ncobj,'longitude',lonvec) if debug: print varname,data.shape nb.show_dimensions(ncobj) # nb.add_data should work but is presently broken. Use direct method #nb.add_data(ncobj,varname,data) #ncobj.variables[varname][0,:,:] = data # 2D numpy array ncobj.variables[varname][:] = data # 3D numpy array nb.set_attributes(ncobj,attributes) nb.ncclose(ncobj) print 'Wrote:',fileroot+'.nc' # Write metadata to json if os.path.exists(fileroot+'.json'): os.remove(fileroot+'.json') jh.json_dump(attributes,fileroot+'.json')
def bom_ascii_to_nc(year,dates,froot): latvec = None lonvec = None ncobj = None latslice = slice(None,None,None) lonslice = slice(None,None,None) missed_dates = [] for dti,dt in enumerate(dates): dni,ghi = get_solar_files(str(year),dt) index = (slice(dti,None,None),latslice,lonslice) if dni is not None: # Split the input file into metadata and data components dni_head,dni_rows,dni_history = split_bom_file(dni) # Resample the data dni_data,dni_lat,dni_lon,dni_dict = resample_data(dni_rows,dni_head) if ghi is not None: # Split the input file into metadata and data components ghi_head,ghi_rows,ghi_history = split_bom_file(ghi) # Resample the data ghi_data,ghi_lat,ghi_lon,ghi_dict = resample_data(ghi_rows,ghi_head) # Skip initial dates until we get a valid file because we need lat,lon if latvec is None and dni is not None: latvec = dni_lat lonvec = dni_lon miss = dni_dict['missing'] if latvec is None and ghi is not None: latvec = ghi_lat lonvec = ghi_lon miss = ghi_dict['missing'] if latvec is None: missed_dates.append(dt) continue # Initialise the netcdf object if ncobj is None: dni_var = 'solar_dni' ghi_var = 'solar_ghi' obs_var = 'obs_time' ncobj = nb.nc_open(froot+'.nc','w') #ncobj = nb.nc_open(froot+'.nc','w',format='NETCDF4_CLASSIC') nb.nc_set_timelatlon(ncobj,None,len(latvec),len(lonvec)) nb.nc_set_var(ncobj,dni_var,fill=miss) nb.nc_set_var(ncobj,ghi_var,fill=miss) nb.nc_set_var(ncobj,obs_var,fill=miss) #nb.nc_set_var(ncobj,dni_var,fill=miss,zlib=True) #nb.nc_set_var(ncobj,ghi_var,fill=miss,zlib=True) #nb.nc_set_var(ncobj,obs_var,fill=miss,zlib=True) nb.nc_set_var(ncobj,'crs',dims=(),dtype="i4") # Grid mapping container nb.nc_add_data(ncobj,'latitude',latvec) nb.nc_add_data(ncobj,'longitude',lonvec) dni_nc = ncobj.variables[dni_var] ghi_nc = ncobj.variables[ghi_var] obs_nc = ncobj.variables[obs_var] # Add observation time layers for any missed dates for mi,md in enumerate(missed_dates): obs_data = create_obs_time(latvec,lonvec,md) #nb.nc_add_data(ncobj,obs_var,obs_data, # index=(slice(mi,None,None),latslice,lonslice)) obs_nc[mi,:,:] = obs_data missed_dates = [] # Calculate the observation time layer obs_data = create_obs_time(latvec,lonvec,dt) obs_nc[dti,:,:] = obs_data # Add data if dni is not None: #nb.nc_add_data(ncobj,dni_var,dni_data,index=index) dni_nc[dti,:,:] = dni_data if ghi is not None: #nb.nc_add_data(ncobj,ghi_var,ghi_data,index=index) ghi_nc[dti,:,:] = ghi_data # Add time values nb.nc_add_time(ncobj,dates) # Create an ordered metadata dictionary meta = create_meta(dates[0],dates[-1],dni_dict) # Output the file-specific metadata to a JSON file regardless of the # requested output formats jh.json_dump(meta, froot+'.json') # Setting attributes here is optional in this example because it is # known that netcdf_json_wrapper.py will be called next with # 'froot+".json"' as one of the input files. nb.nc_set_attributes(ncobj,meta) print "Added attributes" nb.nc_close(ncobj) print "Wrote: "+froot+".nc" return froot
def asciigrid_to_nc(arcfilename, fileroot): """ The main routine that calls the calls other routines to prepare the data and metadata and create the netCDF file. """ # Read ascii grid file asciihead, asciidata, asciitail = split_asciigrid(arcfilename) meta = arcasciihdr_to_dict(asciihead) # Create numpy components latvec, lonvec, datadict = set_latlon(meta) data, datadict = asciigrid_to_numpy(asciidata, meta, datadict) # Resample or edit array # Add a default no_data value if required. miss = -999.0 if 'nodata_value' in meta: miss = float(meta['nodata_value']) datadict['missing'] = miss #data,latvec,lonvec,datadict = resample_array(data,latvec,lonvec,datadict) # Prepare time, variable name and metadata d1, d2, datadict = set_datetime(arcfilename, datadict) datadict = set_varname(arcfilename, datadict) attributes = set_attributes(arcfilename, meta, datadict) # Netcdf options # http://netcdf4-python.googlecode.com/svn/trunk/docs/netCDF4-module.html nc_format = 'NETCDF4_CLASSIC' nc_compress = True debug = False # Write netcdf file if os.path.exists(fileroot + '.nc'): os.remove(fileroot + '.nc') varname = datadict['varname'] vartype = datadict['datatype'] fillval = datadict['missing'] timevec = [d1] ncobj = nb.ncopen(fileroot + '.nc', 'w', format=nc_format) nb.set_timelatlon(ncobj, None, len(latvec), len(lonvec)) # unlimited time nb.set_variable(ncobj, varname, dtype=vartype, fill=fillval, zlib=nc_compress) nb.set_variable(ncobj, 'crs', dims=(), dtype="i4") # Grid mapping container nb.add_time(ncobj, timevec) nb.add_data(ncobj, 'latitude', latvec) nb.add_data(ncobj, 'longitude', lonvec) if debug: print varname, data.shape nb.show_dimensions(ncobj) # nb.add_data should work but is presently broken. Use direct method #nb.add_data(ncobj,varname,data) #ncobj.variables[varname][0,:,:] = data # 2D numpy array ncobj.variables[varname][:] = data # 3D numpy array nb.set_attributes(ncobj, attributes) nb.ncclose(ncobj) print 'Wrote:', fileroot + '.nc' # Write metadata to json if os.path.exists(fileroot + '.json'): os.remove(fileroot + '.json') jh.json_dump(attributes, fileroot + '.json')
def bom_ascii_to_nc(year, dates, froot): debug = False latvec = None lonvec = None ncobj = None latslice = slice(None, None, None) lonslice = slice(None, None, None) missed_dates = [] adict = {} for dti, dt in enumerate(dates): dni, ghi = get_solar_files(str(year), dt) index = (slice(dti, None, None), latslice, lonslice) if dni is not None: # Split the input file into metadata and data components dni_head, dni_rows, dni_history = split_bom_file(dni) # Resample the data dni_data, dni_lat, dni_lon, dni_dict = resample_data( dni_rows, dni_head) else: if debug: print "No dni data:", dt if ghi is not None: # Split the input file into metadata and data components ghi_head, ghi_rows, ghi_history = split_bom_file(ghi) # Resample the data ghi_data, ghi_lat, ghi_lon, ghi_dict = resample_data( ghi_rows, ghi_head) else: if debug: print "No ghi data:", dt # Skip initial dates until we get a valid file because we need lat,lon if latvec is None and dni is not None: if debug: print "Using dni_lat:", dt latvec = dni_lat lonvec = dni_lon adict = dni_dict miss = adict['missing'] if latvec is None and ghi is not None: if debug: print "Using ghi_lat:", dt latvec = ghi_lat lonvec = ghi_lon adict = ghi_dict miss = adict['missing'] if latvec is None: if debug: print "Save miss:", dt missed_dates.append(dt) continue # Initialise the netcdf object if ncobj is None: if debug: print "Define ncobj:", dt dni_var = 'solar_dni' ghi_var = 'solar_ghi' obs_var = 'obs_time' ncobj = nb.nc_open(froot + '.nc', 'w', format='NETCDF4_CLASSIC') nb.nc_set_timelatlon(ncobj, None, len(latvec), len(lonvec)) nb.nc_set_var(ncobj, dni_var, fill=miss, zlib=True) nb.nc_set_var(ncobj, ghi_var, fill=miss, zlib=True) nb.nc_set_var(ncobj, obs_var, fill=miss, zlib=True) nb.nc_set_var(ncobj, 'crs', dims=(), dtype="i4") # Grid mapping container nb.nc_add_data(ncobj, 'latitude', latvec) nb.nc_add_data(ncobj, 'longitude', lonvec) dni_nc = ncobj.variables[dni_var] ghi_nc = ncobj.variables[ghi_var] obs_nc = ncobj.variables[obs_var] # Add observation time layers for any missed dates for mi, md in enumerate(missed_dates): if debug: print "Add missed:", md obs_data = create_obs_time(latvec, lonvec, md) #nb.nc_add_data(ncobj,obs_var,obs_data, # index=(slice(mi,None,None),latslice,lonslice)) obs_nc[mi, :, :] = obs_data missed_dates = [] # Calculate the observation time layer obs_data = create_obs_time(latvec, lonvec, dt) obs_nc[dti, :, :] = obs_data # Add data if dni is not None: #nb.nc_add_data(ncobj,dni_var,dni_data,index=index) if debug: print "Add dni:", dni dni_nc[dti, :, :] = dni_data if ghi is not None: if debug: print "Add ghi:", ghi #nb.nc_add_data(ncobj,ghi_var,ghi_data,index=index) ghi_nc[dti, :, :] = ghi_data # Add time values nb.nc_add_time(ncobj, dates) # Create an ordered metadata dictionary meta = create_meta(dates[0], dates[-1], adict) # Output the file-specific metadata to a JSON file regardless of the # requested output formats jh.json_dump(meta, froot + '.json') # Setting attributes here is optional in this example because it is # known that netcdf_json_wrapper.py will be called next with # 'froot+".json"' as one of the input files. nb.nc_set_attributes(ncobj, meta) if debug: print "Added attributes" nb.nc_close(ncobj) print "Wrote: " + froot + ".nc" return froot