def USGSgh2netcdf(ncfile, meta, time, gauge_height):
"""
Convert the USGS files to netcdf4 format
"""
shpfile = ncfile[:-2] + 'shp'
varname = 'gauge_height'
longname = 'Gauge height'
units = 'm'
ncdict = []
ii = -1
for tt, dd in zip(time, gauge_height):
ii += 1
timeout = othertime.MinutesSince(tt, basetime=datetime(1970, 1, 1))
#ncdict=netcdfio.createObsDict(varname,longname,units,[dd],[timeout],\
# [meta['Latitude'][ii]],[meta['Longitude'][ii]],[0.0],[meta['Station ID'][ii]],[meta['StationName'][ii]],ncdict=ncdict )
ncdict=netcdfio.createObsDict(varname,longname,units,[dd],[timeout],\
[meta['Latitude'][ii]],[meta['Longitude'][ii]],[meta['elevation'][ii]],[meta['Station ID'][ii]],[meta['StationName'][ii]],ncdict=ncdict )
## Global atts
globalatts = {'Title': 'USGS gauge height data'}
# Write to a netcdf file
## write to a single netcdf file with multi groups
netcdfio.writePointData2Netcdf(ncfile, ncdict, globalatts)
## write to multi netcdf files
#netcdfio.writePointData2Netcdf_new(ncfile,ncdict,globalatts)
# Write to a shape file
netcdfio.pointNC2shp(ncfile, shpfile)
def USGS2netcdf(ncfile, meta, time, discharge):
"""
Convert the USGS files to netcdf4 format
"""
shpfile = ncfile[:-2] + 'shp'
varname = 'discharge'
longname = 'Stream Discharge Rate'
units = 'm3 s-1'
ncdict = []
ii = -1
for tt, dd in zip(time, discharge):
ii += 1
timeout = othertime.MinutesSince(tt, basetime=datetime(1970, 1, 1))
ncdict=netcdfio.createObsDict(varname,longname,units,[dd],[timeout],\
[meta['Latitude'][ii]],[meta['Longitude'][ii]],[0.0],[meta['Station ID'][ii]],[meta['StationName'][ii]],ncdict=ncdict )
## Global atts
globalatts = {'Title': 'USGS stream gage discharge data'}
# Write to a netcdf file
netcdfio.writePointData2Netcdf(ncfile, ncdict, globalatts)
# Write to a shape file
netcdfio.pointNC2shp(ncfile, shpfile)
def downloadTCOONTide(starttime,endtime,staid):
data=downloadTide(starttime,endtime,staid)
dire=os.path.dirname(__file__)
dire=os.getcwd()
ncfile = dire+'/DATA/TCOONTide.nc'
globalatts = {'title':'TCOON oceanographic observation data'}
netcdfio.writePointData2Netcdf(ncfile,data,globalatts)
print 'end writing tide data...'
def USGS2netcdf(ncfile,meta,time,discharge):
"""
Convert the USGS files to netcdf4 format
"""
shpfile = ncfile[:-2]+'shp'
varname = 'discharge'
longname = 'Stream Discharge Rate'
units = 'm3 s-1'
ncdict=[]
ii=-1
for tt,dd in zip(time,discharge):
ii+=1
timeout = othertime.MinutesSince(tt,basetime=datetime(1970,1,1))
ncdict=netcdfio.createObsDict(varname,longname,units,[dd],[timeout],\
[meta['Latitude'][ii]],[meta['Longitude'][ii]],[0.0],[meta['Station ID'][ii]],[meta['StationName'][ii]],ncdict=ncdict )
## Global atts
globalatts={'Title':'USGS stream gage discharge data'}
# Write to a netcdf file
netcdfio.writePointData2Netcdf(ncfile,ncdict,globalatts)
# Write to a shape file
netcdfio.pointNC2shp(ncfile,shpfile)
#####
## Example call
#
####
## Input variables
#stationids = ['08066500',\
# '08078000',\
# '08067500',\
# '08076000',\
# '08075000',\
# '08074500',\
# '08076500',\
# '08073600',\
# '08075770']
#
#starttime = '2000-01-01'
#endtime = '2012-01-01'
#ncfile = 'C:/Projects/GOMGalveston/DATA/River/USGS_Rivers_20002012.nc'
####
#
#getUSGSnwis(stationids,starttime,endtime,ncfile)
def noaaish2nc(latlon, yearrange, localdir, ncfile, shpfile):
""" Reads in the noaa data and spits it out to a netcdf file"""
varnames = ['Tair', 'Pair', 'Uwind', 'Vwind', 'RH', 'rain', 'cloud']
# Read in the semi-processed data
timestart = yearrange[0]
timeend = yearrange[1]
data = readall(latlon, [timestart, timeend], localdir)
data = dataQC(data, varnames)
# Create the dictionary format necessary to parse the data to a grouped netcdf file
ncdict = []
for dd in data:
for vv in dd.keys():
if vv in varnames:
# Start building the dictionary
if np.size(dd['Longitude']) > 0 or np.size(dd['Latitude']) > 0:
if dd[vv].has_key('Height'):
ele = dd[vv]['Height']
else:
ele = 0.0
coords = [{'Name':'longitude','Value':dd['Longitude'],'units':'degrees East'},\
{'Name':'latitude','Value':dd['Latitude'],'units':'degrees North'},\
{'Name':'elevation','Value':ele,'units':'metres','positive':'up'},\
{'Name':'time','Value':dd[vv]['Time'],'units':'minutes since 1970-01-01 00:00:00'}]
attribs = {'StationID':dd['StationID'],'StationName':dd['StationName'],'Data':dd[vv]['Data'],\
'coordinates':'time, elevation, longitude, latitude','long_name':dd[vv]['Longname'],\
'units':dd[vv]['Units'],'coords':coords}
ncdict.append({vv: attribs})
else:
print 'Station: %s missing coordinate info - skipping' % dd[
'StationName']
globalatts = {'title':'NCDC/NWS integrated surface hourly observation data',\
'history':'Created on '+datetime.ctime(datetime.now()),\
'source':'ftp://ftp.ncdc.noaa.gov/pub/data/ish/'}
# Write the output to a netcdf file
netcdfio.writePointData2Netcdf(ncfile, ncdict, globalatts)
# Write the metadata to a shapefile
netcdfio.pointNC2shp(ncfile, shpfile)
return ncdict
def noaaish2nc(latlon,yearrange,localdir,ncfile,shpfile):
""" Reads in the noaa data and spits it out to a netcdf file"""
varnames = ['Tair','Pair','Uwind','Vwind','RH','rain','cloud']
# Read in the semi-processed data
timestart=yearrange[0]
timeend=yearrange[1]
data = readall(latlon,[timestart,timeend],localdir)
data = dataQC(data,varnames)
# Create the dictionary format necessary to parse the data to a grouped netcdf file
ncdict=[]
for dd in data:
for vv in dd.keys():
if vv in varnames:
# Start building the dictionary
if np.size(dd['Longitude'])>0 or np.size(dd['Latitude'])>0:
if dd[vv].has_key('Height'):
ele = dd[vv]['Height']
else:
ele=0.0
coords = [{'Name':'longitude','Value':dd['Longitude'],'units':'degrees East'},\
{'Name':'latitude','Value':dd['Latitude'],'units':'degrees North'},\
{'Name':'elevation','Value':ele,'units':'metres','positive':'up'},\
{'Name':'time','Value':dd[vv]['Time'],'units':'minutes since 1970-01-01 00:00:00'}]
attribs = {'StationID':dd['StationID'],'StationName':dd['StationName'],'Data':dd[vv]['Data'],\
'coordinates':'time, elevation, longitude, latitude','long_name':dd[vv]['Longname'],\
'units':dd[vv]['Units'],'coords':coords}
ncdict.append({vv:attribs})
else:
print 'Station: %s missing coordinate info - skipping'%dd['StationName']
globalatts = {'title':'NCDC/NWS integrated surface hourly observation data',\
'history':'Created on '+datetime.ctime(datetime.now()),\
'source':'ftp://ftp.ncdc.noaa.gov/pub/data/ish/'}
# Write the output to a netcdf file
netcdfio.writePointData2Netcdf(ncfile,ncdict,globalatts)
# Write the metadata to a shapefile
netcdfio.pointNC2shp(ncfile,shpfile)
return ncdict
def main(vartype, bbox, timestart, timeend, ncfile, dbfile=None):
varlookup ={'CurrentsActive':'currents','WaterLevelActive':'water_surface_height_above_reference_datum',\
'Salinity':'sea_water_salinity','Conductivity':'sea_water_electrical_conductivity'}
# Find the stations
staInfo = stationInfo(vartype)
for vv in list(staInfo.keys()):
if staInfo[vv]['lon'] >= bbox[0] and staInfo[vv]['lon'] <= bbox[
1] and staInfo[vv]['lat'] >= bbox[2] and staInfo[vv][
'lat'] <= bbox[3]:
print('Station %s inside lat-lon range.' % staInfo[vv]['name'])
else:
staInfo.pop(vv)
ncdata = []
for ID in list(staInfo.keys()):
print('Getting %s data from %s' % (vartype, staInfo[ID]['name']))
# Grab the station data
data = getAllTime(ID, varlookup[vartype], timestart, timeend)
###
if len(data) > 0:
# Convert the output to the format required for the netcdf file
if vartype == 'CurrentsActive':
ncdatatmp = parseADCPdata(data, staInfo, ID)
elif vartype == 'WaterLevelActive':
ncdatatmp = parseWaterLev(data, staInfo, ID)
ncdata += ncdatatmp
# Write to the output netcdf
globalatts = {'title':'US-IOOS observation data',\
'history':'Created on '+datetime.ctime(datetime.now()),\
'source':'http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/index.jsp'}
netcdfio.writePointData2Netcdf(ncfile, ncdata, globalatts)
# Write the metadata to a shapefile
#shpfile = ncfile.split('.')[0]+'.shp'
#netcdfio.pointNC2shp(ncfile,shpfile)
# Update the database
#createObsDB(dbfile)
if not dbfile == None:
print('Updating database: %s' % dbfile)
netcdfio.netcdfObs2DB(ncfile, dbfile)
def main(vartype,bbox,timestart,timeend,ncfile,dbfile=None):
varlookup ={'CurrentsActive':'currents','WaterLevelActive':'water_surface_height_above_reference_datum',\
'Salinity':'sea_water_salinity','Conductivity':'sea_water_electrical_conductivity'}
# Find the stations
staInfo = stationInfo(vartype)
for vv in staInfo.keys():
if staInfo[vv]['lon']>=bbox[0] and staInfo[vv]['lon']<=bbox[1] and staInfo[vv]['lat']>=bbox[2] and staInfo[vv]['lat']<=bbox[3]:
print 'Station %s inside lat-lon range.' % staInfo[vv]['name']
else:
staInfo.pop(vv)
ncdata=[]
for ID in staInfo.keys():
print 'Getting %s data from %s' % (vartype,staInfo[ID]['name'])
# Grab the station data
data = getAllTime(ID,varlookup[vartype],timestart,timeend)
###
if len(data)>0:
# Convert the output to the format required for the netcdf file
if vartype == 'CurrentsActive':
ncdatatmp=parseADCPdata(data,staInfo,ID)
elif vartype == 'WaterLevelActive':
ncdatatmp=parseWaterLev(data,staInfo,ID)
ncdata+=ncdatatmp
# Write to the output netcdf
globalatts = {'title':'US-IOOS observation data',\
'history':'Created on '+datetime.ctime(datetime.now()),\
'source':'http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/index.jsp'}
netcdfio.writePointData2Netcdf(ncfile,ncdata,globalatts)
# Write the metadata to a shapefile
#shpfile = ncfile.split('.')[0]+'.shp'
#netcdfio.pointNC2shp(ncfile,shpfile)
# Update the database
#createObsDB(dbfile)
if not dbfile == None:
print 'Updating database: %s'%dbfile
netcdfio.netcdfObs2DB(ncfile,dbfile)
endtimes.append('%s-%s-%s' % (year, month, end_day))
#stationids = ['8775870', \
# '8775792', \
# '8775283', \
# '8775237', \
# '8775296', \
# '8776139', \
# '8775244', \
# '8775241']
stationids = ['8776604', \
'8778490', \
'8779280', \
'8779770']
vartype = 'temperature'
#vartype = 'salinity'
for staid in stationids:
ff = open('temp/availiability/%s_%s.txt' % (vartype, staid), 'w+')
print 'Reading %s data at station %s ...' % (vartype, staid)
## write to txt file for future reference
print >> ff, 'Reading %s data at station %s ...' % (vartype, staid)
data = downloadObs(starttimes, endtimes, staid, ff, vartype)
#pdb.set_trace()
ncfile = 'temp/NOAA_%s_%s.nc' % (vartype, staid)
globalatts = {'title': 'NOAA oceanographic observation data'}
netcdfio.writePointData2Netcdf(ncfile, data, globalatts)
print 'end writing tide data...'
