def do_file(self, filename, data):
print "Creating ", filename
# create compact file and initialize basic settings
root, is_new = nc.open(filename + self.extension) # The filename does not contain the extension
if is_new:
sample, n = nc.open(data[(data.keys())[0].completepath()])
shape = sample.variables['data'].shape
nc.getdim(root,'northing', shape[1])
nc.getdim(root,'easting', shape[2])
nc.getdim(root,'timing')
v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
v_lon[:] = nc.getvar(sample, 'lon')[:]
v_lat[:] = nc.getvar(sample, 'lat')[:]
nc.close(sample)
if isinstance(data,collections.Mapping):
for d in data:
self.do_var(root, d, data[d])
else:
self.create_var(root, 'data', data)
# save the content inside the compact file
if not root is None: nc.close(root)
f = File(localname=localname, downloaded=True, begin_download=begin_time, end_download=datetime.now())
f.save()
return f
def do_var(self, root, var_name, files):
count = 0
max_count = len(files[c])
print "Compacting ", var_name
shape = nc.getvar(root,'lat').shape
begin_time = datetime.now()
for f in files:
# join the distributed content
ch = f.channel()
v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
v_ch_t = nc.getvar(root,var_name + 'time', 'f4', ('timing',))
try:
rootimg, n = nc.open(f.completepath())
data = (nc.getvar(rootimg, 'data'))[:]
# Force all the channels to the same shape
data = matrix.adapt(data, shape)
if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
index = v_ch.shape[0]
v_ch[index,:] = data
v_ch_t[index] = calendar.timegm(f.image_datetime().utctimetuple())
nc.close(rootimg)
nc.sync(root)
print "\r","\t" + str(count/max_count * 100).zfill(2) + "%",
except RuntimeError, e:
print f.completepath()
def cut(lat,lon): import numpy as np root, is_new = nc.open(localpath+"/dem.nc") dem = nc.getvar(root, 'dem', 'i2', lat.dimensions) x, y = p.pixels_from_coordinates(lat[:], lon[:], dem.shape[0], dem.shape[1]) # In the transformation the 'y' dimension is twisted because the map is inverted. result = p.transform_data(dem,x,dem.shape[0] - y) nc.close(root) return result, x, y
def latlon(self): try: root, n = nc.open(self.completepath()) lat = nc.getvar(root,'lat')[:] lon = nc.getvar(root, 'lon')[:] nc.close(root) except RuntimeError: show(self.completepath()) lat, lon = None, None return lat, lon
def get_month(month): if gdal_supported: ds = gdal.Open(localpath + "/tifs/"+str(month).zfill(2)+"_longlat_wgs84.tif") linke = ds.ReadAsArray() else: root, n = nc.open(destiny) data = nc.getvar(root, "linketurbidity") linke = data[month -1] # The linke turbidity is obtained when the image pixel value is divied by 20. return linke/20.
def do(self, files): for c in files: for f in files[c]: if f.channel() == '01': root, is_new = nc.open(f.completepath()) var = nc.getvar(root, 'data') data = np.zeros(var.shape) for i in range(var.shape[0]): data = var[i] data = self.calibrated_coefficient * ((data / self.counts_shift) - self.space_measurement) var[i] = data nc.close(root) return files
def do_file(self, filename, stream):
# create compact file and initialize basic settings
begin_time = self.getdatetimenow()
root, is_new = nc.open(filename)
if is_new:
sample, n = nc.open(stream.files.all()[0].file.completepath())
shape = sample.variables['data'].shape
nc.getdim(root,'northing', shape[1])
nc.getdim(root,'easting', shape[2])
nc.getdim(root,'timing')
v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
v_lon[:] = nc.getvar(sample, 'lon')[:]
v_lat[:] = nc.getvar(sample, 'lat')[:]
nc.close(sample)
nc.sync(root)
self.do_var(root, 'data', stream)
# save the content inside the compact file
if not root is None: nc.close(root)
f = File(localname=filename)
f.save()
return f
def rows2netcdf(rows, filename, index):
root, is_new = nc.open(filename)
if not is_new:
measurements = nc.getvar(root, 'measurements', 'f4', ('timing','northing','easting'), 4)
slots = nc.getvar(root, 'slots')
times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ]
instant_radiation = rows2slots(rows,2)
i_e = 0
i_m = 0
while i_e < len(times) and i_m < len(instant_radiation):
#print i_e,"/", len(times) -1, " | ", i_m, "/", len(instant_radiation) - 1
# When date estimated before date measured
if times[i_e].date() < instant_radiation[i_m][1][0].date():
i_e += 1
# When date estimated after date measured
elif times[i_e].date() > instant_radiation[i_m][1][0].date():
i_m += 1
else:
if slots[i_e] < instant_radiation[i_m][0]:
measurements[i_e, index,:] = np.array([-1, -1])
i_e += 1
elif slots[i_e] > instant_radiation[i_m][0]:
i_m += 1
else:
value = instant_radiation[i_m][1][1]
row_in_slot = instant_radiation[i_m][1][2]
measurements[i_e, index,:] = np.array([value, value])
i_e += 1
i_m += 1
days = [ t.date() for t in times ]
nc.getdim(root, 'diarying')
diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting'))
estimated = nc.getvar(root, 'globalradiation')[:]
error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting'), 4)
error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting'), 4)
error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
error[:, index, :] = np.sqrt((error_diff[:, index, :])**2)
print error_diff.shape
nc.sync(root)
for i in range(len(days)):
d = days[i].day
print index, i, d, error_diff[i,index], diary_error[d, index]
diary_error[d, index,:] += error_diff[i,index,:]
diary_error[:, index,:] = np.sqrt((diary_error[:, index,:])**2)
nc.close(root)
def merge_tails():
import glob
import numpy as np
#from scipy import misc
say("Merging all the tails in one netCDF file...")
root, is_new = nc.open(localpath+ "/dem.nc")
n_d = nc.getdim(root,'northing', 180 * 120) # 180 degrees * 120 px/degree
n_e = nc.getdim(root,'easting', 360 * 120) # 360 degrees * 120 px/degree
data = nc.getvar(root, 'dem', 'i2', ('northing','easting',))
files = glob.glob(localpath + "/all10/all10/*10g")
tails = [[file_coordinates(f),f] for f in files]
for t, f in tails:
print t, f
tmp = np.fromfile(f,np.int16)
dynamic = tmp.shape[0]/10800
tmp = tmp.reshape(dynamic,10800)
#misc.imsave(f+'.png',tmp)
data[t[0]:t[0]+dynamic,t[1]:t[1]+10800] = tmp
nc.close(root)
def rows2netcdf(rows, filename, index):
root, is_new = nc.open(filename)
if not is_new:
measurements = nc.clonevar(root, 'globalradiation', 'measurements')
slots = nc.getvar(root, 'slots')
times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ]
instant_radiation = rows2slots(rows,2)
earth_failures = 0
i_e = 0
i_m = 0
while i_e < len(times) and i_m < len(instant_radiation):
# When date estimated before date measured
if times[i_e].date() < instant_radiation[i_m][1][0].date():
i_e += 1
# When date estimated after date measured
elif times[i_e].date() > instant_radiation[i_m][1][0].date():
i_m += 1
else:
if slots[i_e] < instant_radiation[i_m][0]:
# TODO: This should be completed with a 0 error from the estimation.
measurements[i_e, index,:] = np.array([0, 0])
earth_failures += 1
i_e += 1
elif slots[i_e] > instant_radiation[i_m][0]:
i_m += 1
else:
value = instant_radiation[i_m][1][1]
#row_in_slot = instant_radiation[i_m][1][2]
measurements[i_e, index,:] = np.array([value, value])
i_e += 1
i_m += 1
while i_e < len(times):
# TODO: This should be completed with a 0 error from the estimation.
measurements[i_e, index,:] = np.array([0, 0])
earth_failures += 1
i_e += 1
show("Detected %i of %i estimated times without earth measure.\n" % (earth_failures, len(slots)))
error.rmse(root, index)
nc.close(root)
def do(self, stream):
from libs.sat.goes import calibration
resultant_stream = stream.clone()
for fs in stream.files.all():
f = fs.file
if f.channel() == '01':
sat = f.satellite()
root, is_new = nc.open(f.completepath())
nc.getdim(root,'coefficient',1)
var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4)
var[0] = calibration.counts_shift.coefficient(sat)
var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4)
var[0] = calibration.space_measurement.coefficient(sat)
var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4)
var[0] = calibration.prelaunch.coefficient(sat)[0]
var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4)
dt = f.datetime()
var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month)
#data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement))
nc.close(root)
fs.processed=True
fs.save()
return resultant_stream
def do(self, stream):
from libs.sat.goes import calibration
resultant_stream = stream.clone()
for ms in stream.materials.all():
m = ms.material
if hasattr(m, 'channel') and hasattr(m, 'satellite') and m.channel() == '01':
sat = m.satellite()
root = nc.open(m.completepath())[0]
nc.getdim(root,'coefficient',1)
var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4)
var[0] = calibration.counts_shift.coefficient(sat)
var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4)
var[0] = calibration.space_measurement.coefficient(sat)
var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4)
var[0] = calibration.prelaunch.coefficient(sat)[0]
var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4)
dt = m.datetime()
var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month)
#data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement))
nc.close(root)
ms.processed=True
ms.save()
return resultant_stream
def do_var(self, root, var_name, stream):
count = 0
file_statuses = stream.sorted_files()
shape = nc.getvar(root,'lat').shape
for fs in file_statuses:
# join the distributed content
ch = fs.file.channel()
v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',))
try:
rootimg, n = nc.open(fs.file.completepath())
data = (nc.getvar(rootimg, 'data'))[:]
# Force all the channels to the same shape
if not (data.shape[1:3] == shape):
print data.shape[1:3], shape
data = matrix.adapt(data, shape)
if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
index = v_ch.shape[0]
v_ch[index,:] = data
v_ch_t[index] = calendar.timegm(fs.file.datetime().utctimetuple())
nc.close(rootimg)
nc.sync(root)
except RuntimeError, e:
print fs.file.completepath()
show("BIAS:\t%.5f\t( %.5f %%)" % (bias, bias * ghi_ratio))
rmse = error.rmse_es(estimated, measured, s)
show("RMSE:\t%.5f\t( %.5f %%)" % (rmse, rmse * ghi_ratio))
mae = error.mae(estimated, measured, s)
show("MAE:\t%.5f\t( %.5f %%)" % (mae, mae * ghi_ratio))
show("----------\n")
error.rmse(root, s)
def workwith(year=2011, month=05, filename="goes13.all.BAND_02.nc"):
show("=======================")
show("Year: " , year)
show("Month: " , month)
show("Filename: ", filename)
show("-----------------------\n")
root = nc.open(filename)[0]
lat = (nc.getvar(root, 'lat'))[:]
lon = (nc.getvar(root, 'lon'))[:]
data = calibrated_data(root)
process_temporal_data(lat, lon, root)
process_atmospheric_data(data, root)
process_ground_albedo(lat, data, root)
process_radiation(root)
process_validate(root)
#draw.getpng(draw.matrixtogrey(data[15]),'prueba.png')
nc.close(root)
show("Process finished.\n")
show(s['Name'])
bias = diff.mean()
show("BIAS:", bias, "(", bias * ghi_ratio, "%)")
rmse = np.sqrt((diff**2).mean())
show("RMSE:", rmse, "(", rmse * ghi_ratio, "%)")
mae = np.absolute(diff).mean()
show("MAE:", mae, "(", mae * ghi_ratio, "%)")
def workwith(year=2011, month=05, filename="goes13.all.BAND_02.nc"):
show("=======================")
show("Year: " , year)
show("Month: " , month)
show("Filename: ", filename)
show("-----------------------\n")
root, is_new = nc.open(filename)
lat = (nc.getvar(root, 'lat'))[:]
lon = (nc.getvar(root, 'lon'))[:]
data = nc.getvar(root, 'data')[:]
process_temporal_data(lat, lon, root)
process_atmospheric_data(lat, lon, data, root)
process_ground_albedo(lat, lon, data, root)
process_radiation(lat, lon, data, root)
#s = process_validate(year, month, root)
#draw.getpng(draw.matrixtogrey(data[15]),'prueba.png')
nc.close(root)
show("Process finished.\n")
import glob
from osgeo import gdal
from libs.file import netcdf as nc
files = glob.glob("libs/linke/tifs/*.tif")
shape = gdal.Open(files[0]).ReadAsArray().shape
root, n = nc.open("linketurbidity.nc")
nc.getdim(root, "northing", shape[0])
nc.getdim(root, "easting", shape[1])
nc.getdim(root, "monthing")
linke = nc.getvar(root, "linketurbidity", "f4", ("monthing", "northing", "easting",), 4)
for i in range(len(files)):
print i, '->', files[i]
linke[i,:] = gdal.Open(files[i]).ReadAsArray()
nc.close(root)
def diff(estimated, measured, station): return estimated[:,station,0] - measured[:,station,0] def ghi_mean(measured, station): return measured[:,station,0].mean() def ghi_ratio(measured, station): return 100 / ghi_mean(measured, station) def bias(estimated, measured, station): t_diff = diff(estimated, measured, station) return t_diff.mean() def rmse_es(estimated, measured, station): t_diff = diff(estimated, measured, station) return np.sqrt((t_diff**2).mean()) def mae(estimated, measured, station): t_diff = diff(estimated, measured, station) return np.absolute(t_diff).mean() filename = sys.argv[1] if len(sys.argv) == 2 else None if not filename == None: try: index = int(sys.argv[2]) root,n = nc.open(filename) rmse(root, index) nc.close(root) except Exception, e: show(e)
