def getJulDatefromNc(ncfile):
#from netCDF4 import Dataset as NetCDFFile
from astropy.time import Time
from datetime import datetime
nc = NetCDFFile(ncfile)
retval = -1
try:
utdate = getattr(nc, 'date')
uttime = getattr(nc, 'start_time')
utdate = utdate.replace("\"", "")
utString = "%s %s" % (utdate, uttime)
t = Time(datetime.strptime(utString, "%m/%d/%Y %H:%M:%S"),
scale="utc",
format="datetime")
retval = t.jd
except Exception:
#try:
utdate = nc.variables['Header.TimePlace.UTDate'].getValue()
t = Time(float(utdate), scale="utc", format="decimalyear")
retval = t.jd
#except Exception:
nc.close()
return retval
def save_2D(filenameout,var,lat,lon,miss_val):
import numpy as np
from scipy.io.netcdf import netcdf_file as NetCDFFile
n = NetCDFFile(filenameout, 'w',) # open it for writing
n.title = 'saved netcdf variable'
n.createDimension('lat',len(lat))
n.createDimension('lon',len(lon))
latitude = n.createVariable('lat','f',('lat',))
latitude.longname = 'latitude'
latitude.units = 'degrees_north'
latitude[:] = lat.astype(np.float32)
longitude = n.createVariable('lon','f',('lon',))
longitude.longname = 'longitude'
longitude.units = 'degrees_east'
longitude[:] = lon[:].astype(np.float32)
varnc = n.createVariable('varnc','f',('lat','lon',))
varnc.missing_value = np.array(miss_val,np.float32)
varnc.FillValue = np.array(miss_val,np.float32)
varnc[:,:]=var[:,:].astype(np.float32)
n.close()
def get_mask(self):
self.array_mean = ma.mean(self.array)
self.array_stdev = ma.std(self.array)
self.array_range = ma.max(self.array) - ma.min(self.array)
print "The mean is %f, the stdev is %f, the range is %f." %(self.array_mean, self.array_stdev, self.array_range)
from scipy.io.netcdf import netcdf_file as NetCDFFile
### get landmask
nc = NetCDFFile(os.getcwd()+ '/../data/netcdf_files/ORCA2_landmask.nc','r')
self.mask = ma.masked_values(nc.variables['MASK'][:, :self.time_len, :self.lat_len, :180], -9.99999979e+33)
nc.close()
self.xxx, self.yyy, self.zzz = np.lib.index_tricks.mgrid[0:self.time_len, 0:self.lat_len, 0:180]
def getObsNum(ncfile):
nc = NetCDFFile(ncfile)
try:
onum = nc.variables['Header.Dcs.ObsNum'].getValue()
except:
raise Exception(
"This does not seems to be a LMT file. Use ASTE functions instead")
nc.close()
return onum
def getOffsetFromMap(ncfile):
nc = NetCDFFile(ncfile)
try:
signalv = nc.variables['signal']
azOffset = signalv.offset_x
elOffset = signalv.offset_y
except KeyError:
raise Exception(
"This does not seems to be a LMT file. Use ASTE functions instead")
nc.close()
return azOffset, elOffset
def __init__(self,filename):
self.filename=filename
try:
self.ncf=NetCDFFile(filename)
self.readmaps(self.ncf)
except IOError:
raise NameError("Failed reading map {}".format(self.filename))
self.ncf.close()
def load_ncep_data(self, air_temp_file=None, spec_hum_file=None, trop_pres_file=None, geo_height_file=None, surface_pres_file=None):
"""Loads NCEP Reanalysis data into NetCdf objects. Returns a named tuple with objects."""
filenames = (air_temp_file, spec_hum_file, trop_pres_file, geo_height_file, surface_pres_file)
dflt_search_globs = ("air.*.nc", "shum.*.nc", "pres.tropp.*.nc", "hgt.*.nc", "pres.sfc.*.nc")
data_list = []
for data_filename, srch_glob, file_type_name in zip(filenames, dflt_search_globs, NCEPData._fields):
if not data_filename:
srch_res = glob(srch_glob)
if len(srch_res) != 1:
raise Exception("For file type: %s expected to find 1 NCEP file, instead found: %s, using glob: %s" % (file_type_name, srch_res, srch_glob))
data_filename = srch_res[0]
logger.debug("Loading NCEP file for %s: %s" % (file_type_name, data_filename))
# Add filename for use elsewhere
ncep_obj = NetCDFFile(data_filename, "r")
ncep_obj.__dict__["filename"] = data_filename
data_list.append(ncep_obj)
return NCEPData(*data_list)
def load_ncep_data(self, air_temp_file=None, spec_hum_file=None, trop_pres_file=None, geo_height_file=None, surface_pres_file=None):
"""Loads NCEP Reanalysis data into NetCdf objects. Returns a named tuple with objects."""
filenames = (air_temp_file, spec_hum_file, trop_pres_file, geo_height_file, surface_pres_file)
dflt_search_globs = ("air.*.nc", "shum.*.nc", "pres.tropp.*.nc", "hgt.*.nc", "pres.sfc.*.nc")
data_list = []
for data_filename, srch_glob, file_type_name in zip(filenames, dflt_search_globs, NCEPData._fields):
if not data_filename:
srch_res = glob(srch_glob)
if len(srch_res) != 1:
raise Exception("For file type: %s expected to find 1 NCEP file, instead found: %s, using glob: %s" % (file_type_name, srch_res, srch_glob))
data_filename = srch_res[0]
logger.debug("Loading NCEP file for %s: %s" % (file_type_name, data_filename))
# Add filename for use elsewhere
ncep_obj = NetCDFFile(data_filename, "r")
ncep_obj.__dict__["filename"] = data_filename
data_list.append(ncep_obj)
return NCEPData(*data_list)
def read(filename):
import numpy as np
from numpy import ma
# from Scientific.IO.NetCDF import NetCDFFile
from scipy.io.netcdf import netcdf_file as NetCDFFile
"""
read a netcdf file and return a dictionnary with
key: name of the variables: value (array)
array is masked is missing_values defined
in the netcdf
"""
nc = NetCDFFile(filename, 'r') # open it for writing
# number of dimensions
# ndim = nc.dimensions.__len__()
# dim_dict = nc.dimensions
var_dict = {}
for i in nc.variables.iterkeys():
exec("cdf_var = nc.variables['" + i + "']")
#exec(i + " = nc.variables['" + i + "'].getValue()")
exec(i + " = nc.variables['" + i + "'][...]")
exec(i + " = np.squeeze(" + i + ")")
if '_FillValue' in dir(cdf_var):
miss_val = np.float(cdf_var._FillValue)
exec(i + " = ma.masked_values(" + i + ",miss_val)")
if 'FillValue' in dir(cdf_var):
miss_val = np.float(cdf_var.FillValue)
exec(i + " = ma.masked_values(" + i + ",miss_val)")
if '_missing_value' in dir(cdf_var):
miss_val = np.float(cdf_var._missing_value)
exec(i + " = ma.masked_values(" + i + ",miss_val)")
if 'missing_value' in dir(cdf_var):
miss_val = np.float(cdf_var.missing_value)
exec(i + " = ma.masked_values(" + i + ",miss_val)")
exec("var_dict['" + i + "'] = " + i + ".astype(np.float32)")
nc.close()
return var_dict
def __init__(self, ncFileName, source=None):
if os.path.exists(ncFileName):
self.filename = ncFileName
self.filtered = False
if not source is None:
self.source = source
try:
ncFile = NetCDFFile(ncFileName)
self.loadFromNcFile(ncFile)
ncFile.close()
except RuntimeError:
#try:
savFile = readsav(ncFileName)
self.loadFromIDLSav(savFile)
#except Exception:
#raise Exception ("AzTEC Map Error. Unknown format")
#
self.fixWeightMap()
self.calculateSNMap()
else:
raise Exception("AztecMap Error. No such file or directory")
class Azmap(object):
"""This is a short version of AztecMap."""
def __init__(self,filename):
self.filename=filename
try:
self.ncf=NetCDFFile(filename)
self.readmaps(self.ncf)
except IOError:
raise NameError("Failed reading map {}".format(self.filename))
self.ncf.close()
def readmaps(self,ncf):
try:
self.signal=(np.array(ncf.variables['signal'][:]).T)[::-1,::-1]
self.weight=(np.array(ncf.variables['weight'][:]).T)[::-1,::-1]
self.fSignal=(np.array(ncf.variables['filteredSignal'][:]).T)[::-1,::-1]
self.fWeight=(np.array(ncf.variables['filteredWeight'][:]).T)[::-1,::-1]
except Exception:
raise Exception("File {} is damaged".format(self.filename))
finally:
# Si todo va bien, que cierre el ncf,
# si ocurre una excepcion, que cierre el ncf
ncf.close()
def save_3D(filenameout,var,varname,lat,lon,miss_val,time_it):
import numpy as np
from scipy.io.netcdf import netcdf_file as NetCDFFile
"""
savenetcdf_3D(filenameout,var,varname,lat,lon,miss_val,time_it)
save a 3D (time,lat,lon) numpy array into netcdf
arguments:
filenameout: string
var: np.array
varname: string
lat: np.array
lon: np.array
miss_val: real
time_it: np.array (dtype=int)
"""
n = NetCDFFile(filenameout, 'w',) # open it for writing
n.title = 'saved netcdf variable'
n.createDimension('time',None)
n.createDimension('lat',len(lat))
n.createDimension('lon',len(lon))
latitude = n.createVariable('lat','f',('lat',))
latitude.longname = 'latitude'
latitude.units = 'degrees_north'
latitude[:] = lat[:].astype(np.float32)
longitude = n.createVariable('lon','f',('lon',))
longitude.longname = 'longitude'
longitude.units = 'degrees_east'
longitude[:] = lon[:].astype(np.float32)
time = n.createVariable('time','i',('time',))
time.units = 'days since 1979-1-1 00:00:0.0'
time.delta_t = '0000-00-01 00:00:00'
varnc = n.createVariable(varname,'f',('time','lat','lon',))
varnc.missing_value = np.array(miss_val,np.float32)
varnc._FillValue = np.array(miss_val,np.float32)
for l in range(0,var.shape[0]):
time[l]=time_it[l,]
varnc[l,:,:]=var[l,:,:].astype(np.float32)
n.close()
def Parse_extents(src_fn):
out = {}
src_ext = os.path.splitext(src_fn)[1]
if src_ext == None:
print '[ ERROR ] input should be MODIS hdf, MODIS or ERA hdf5, ERA netcdf (CF-1.0), shapefile, or geoTIFF'
sys.exit( 1 )
elif src_ext == '.shp':
toprint = 'reading shapefile extents'
src = ogr.Open(src_fn)
layer = src.GetLayer()
extent = layer.GetExtent()
out['dx'] = 'shapefile'
out['dy'] = 'shapefile'
out['srs'] = layer.GetSpatialRef()
out['xmin'] = extent[0]
out['xmax'] = extent[1]
out['ymin'] = extent[2]
out['ymax'] = extent[3]
elif src_ext in ('.tif', '.hdf'):
import gdal
ras = gdal.Open(src_fn)
if src_ext=='.hdf':
toprint = 'reading MODIS hdf grid extents'
# get subdataset (all have the same params in modis)
dset0 = ras.GetSubDatasets()[0][0] # first listed subdataset
del ras
ras = gdal.Open(dset0)
elif src_ext=='.tif':
toprint = 'reading GeoTIFF extents'
srs = osr.SpatialReference()
tmp = srs.ImportFromWkt(ras.GetProjection())
if tmp == 1:
print '[ERROR] FAILED to import projection from raster file'
sys.exit( 1 )
gt = ras.GetGeoTransform()
out['srs'] = srs
out['dx'] = gt[1]
out['dy'] = abs(gt[-1]) # abs if this should be negative
out['xmin'] = gt[0]
out['ymax'] = gt[3]
out['xmax'] = out['xmin'] + ras.RasterXSize*out['dx']
out['ymin'] = out['ymax'] - ras.RasterYSize*out['dy']
del ras
elif src_ext == '.hdf5':
toprint = 'reading hdf5 stack'
import h5py
fn = os.path.split(src_fn)[-1]
prj_name = fn.split('_')[0]
sds = os.path.splitext(fn)[0].split(prj_name+'_')[1]
hdf = h5py.File(src_fn,'r')
out['dx'] = hdf[sds].attrs['dx']
out['dy'] = np.abs(hdf[sds].attrs['dy'])
out['srs'] = osr.SpatialReference()
out['srs'].ImportFromWkt(hdf[sds].attrs['projection'])
out['xmin'] = np.min(hdf['x'])
out['xmax'] = np.max(hdf['x'])+out['dx']
out['ymin'] = np.min(hdf['y'])-out['dy']
out['ymax'] = np.max(hdf['y'])
elif src_ext == '.nc':
#from Scientific.IO.NetCDF import NetCDFFile
from scipy.io.netcdf import netcdf_file as NetCDFFile
#Can this simply drop in like this?
toprint = 'converting NetCDF Grid'
toprint = 'assuming Geographic coordinates, WGS84'
get4326 = osr.SpatialReference();get4326.ImportFromEPSG(4326)
epsg4326 = get4326
out['srs'] = epsg4326
ras = NetCDFFile(src_fn,'r')
lat = ras.variables['latitude'][:]
lon = ras.variables['longitude'][:]
ras.close()
out['dx'] = float(lat[0] - lat[1])
out['dy'] = float(np.abs(lon[0] - lon[1]))
out['xmin'] = float(np.min(lon)) - 0.5 * out['dx']
out['ymax'] = float(np.max(lat)) + 0.5 * out['dy']
out['xmax'] = float(np.max(lon)) + 0.5 * out['dx']
out['ymin'] = float(np.min(lat)) - 0.5 * out['dy']
else:
print '[ ERROR ] input file not of type tif|hdf|nc'
sys.exit( 1 )
return out
def writeToNc(self, ncFileName):
ncFile = NetCDFFile(ncFileName, 'w')
ncFile.createDimension("nrows", self.RaCoords.shape[0])
ncFile.createDimension("ncols", self.DecCoords.shape[0])
dims = ("nrows", "ncols")
drow = ("nrows", )
dcol = ("ncols", )
ncFile.createVariable("signal", "d", dims)
ncFile.createVariable("weight", "d", dims)
ncFile.createVariable("kernel", "d", dims)
ncFile.createVariable("rowCoordsPhys", "d", drow)
ncFile.createVariable("colCoordsPhys", "d", dcol)
ncFile.createVariable("xCoordsAbs", "d", dims)
ncFile.createVariable("yCoordsAbs", "d", dims)
ncFile.createVariable("filteredSignal", "d", dims)
ncFile.createVariable("filteredWeight", "d", dims)
ncFile.createVariable("filteredKernel", "d", dims)
setattr(ncFile, "source", "%s" % self.source)
setattr(ncFile, "MasterGrid[0]", self.SourceRa / 180.0 * (npy.pi))
setattr(ncFile, "MasterGrid[1]", self.SourceDec / 180.0 * (npy.pi))
ncFile.variables['signal'][:] = self.signal.value
ncFile.variables['weight'][:] = self.weight.value
ncFile.variables['rowCoordsPhys'][:] = self.RaCoords / 180.0 * (npy.pi)
ncFile.variables['colCoordsPhys'][:] = self.DecCoords / 180.0 * (
npy.pi)
ncFile.variables['kernel'][:] = self.kernel
ncFile.variables['xCoordsAbs'][:] = self.AbsRaCoords.to("rad").value
ncFile.variables['yCoordsAbs'][:] = self.AbsDecCoords.to("rad").value
if self.filtered:
ncFile.variables['filteredSignal'][:] = self.fSignal.value
ncFile.variables['filteredWeight'][:] = self.fWeight.value
ncFile.variables['filteredKernel'][:] = self.fKernel
else:
ncFile.variables['filteredSignal'][:] = self.signal.value
ncFile.variables['filteredWeight'][:] = self.weight.value
ncFile.variables['filteredKernel'][:] = self.kernel
ncFile.sync()
ncFile.close()
def createBstatsFile(ncfile, outFile=None):
from scipy.interpolate import interp1d
from xml.etree.ElementTree import Element, SubElement
if outFile is None:
outFile = ncfile.replace(".nc", ".bstats")
jdate = getJulDatefromNc(ncfile)
season = getAztecSeason(jdate)
if season.find("LMT") > -1:
nc = NetCDFFile(ncfile)
elSignal = npy.mean(nc.variables['Data.AztecBackend.TelElAct'][:])
elSignal = elSignal.flatten()
meanEl = npy.degrees(npy.median(elSignal))
nc.close()
namePrefix = "Data.AztecBackend."
else:
meanEl = None
namePrefix = ""
calpath = os.path.join(os.getenv("AZTEC_MACANA_PATH"), "calibration/")
bololistfile = os.path.join(calpath, "parameters_%s" % season,
"bololist.csv")
tausave = os.path.join(calpath, "parameters_%s" % season,
"fit_parameters_bolodc2tau_%s.sav" % season)
ressave = os.path.join(
calpath, "parameters_%s" % season,
"fit_parameters_bolodc2responsivity_%s.sav" % season)
bololist = npy.loadtxt(bololistfile,
delimiter=",",
comments="#",
dtype={
'names': (
'boloname',
'flags',
),
'formats': ('|S10', 'i4')
})
nbolo = len(bololist)
boloInfo = []
jbolo = 1
for ibolo in bololist:
boloInfo.append({
'boloName': namePrefix + ibolo[0],
'valid': ibolo[1],
'id': jbolo
})
jbolo += 1
tauInfo = readsav(tausave)
resInfo = readsav(ressave)
for ibolo, i in zip(boloInfo, range(nbolo)):
if "offset" in tauInfo.keys():
ibolo['tau_offset'] = tauInfo['offset'][i]
ibolo['tau_offset_err'] = tauInfo['offset_err'][i]
ibolo['tau_slope'] = tauInfo['slope'][i]
ibolo['tau_slope_err'] = tauInfo['slope_err'][i]
ibolo['tau_quad'] = 0.0
ibolo['tau_quad_err'] = 0.0
elif "p0" in tauInfo.keys():
ibolo['tau_offset'] = tauInfo['p0'][i]
ibolo['tau_offset_err'] = tauInfo['p0_err'][i]
ibolo['tau_slope'] = tauInfo['p1'][i]
ibolo['tau_slope_err'] = tauInfo['p1_err'][i]
ibolo['tau_quad'] = tauInfo['p2'][i]
ibolo['tau_quad_err'] = tauInfo['p2_err'][i]
else:
raise Exception("Cannot get Tau2dc information properly.")
ibolo['res_offset'] = resInfo['offset'][i]
ibolo['res_offset_err'] = resInfo['offset_err'][i]
ibolo['res_slope'] = resInfo['slope'][i]
ibolo['res_slope_err'] = resInfo['slope_err'][i]
#Now interpolate the gain values
interpolateParams(jdate, boloInfo, meanEl=meanEl)
boloInfo2Xml(outFile, boloInfo)