def do_var(self, root, var_name, files):
count = 0
max_count = len(files[c])
print "Compacting ", var_name
shape = nc.getvar(root,'lat').shape
begin_time = datetime.now()
for f in files:
# join the distributed content
ch = f.channel()
v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
v_ch_t = nc.getvar(root,var_name + 'time', 'f4', ('timing',))
try:
rootimg, n = nc.open(f.completepath())
data = (nc.getvar(rootimg, 'data'))[:]
# Force all the channels to the same shape
data = matrix.adapt(data, shape)
if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
index = v_ch.shape[0]
v_ch[index,:] = data
v_ch_t[index] = calendar.timegm(f.image_datetime().utctimetuple())
nc.close(rootimg)
nc.sync(root)
print "\r","\t" + str(count/max_count * 100).zfill(2) + "%",
except RuntimeError, e:
print f.completepath()
def process_temporal_data(lat, lon, root):
times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ]
indexes = range(len(times))
for i in indexes:
show("\rTemporal data: preprocessing image %d / %d " % (i, len(indexes)-1))
dt = times[i]
# Calculate some geometry parameters
# Parameters that need the datetime: gamma, tst_hour, slots, linketurbidity
gamma = geo.getdailyangle(geo.getjulianday(dt),geo.gettotaldays(dt))
tst_hour = geo.gettsthour(geo.getdecimalhour(dt), GREENWICH_LON, lon, geo.gettimeequation(gamma))
declination = geo.getdeclination(gamma)
slots = geo.getslots(dt,IMAGE_PER_HOUR)
omega = geo.gethourlyangle(tst_hour, lat/abs(lat))
solarangle = geo.getzenithangle(declination,lat,omega)
solarelevation = geo.getelevation(solarangle)
excentricity = geo.getexcentricity(gamma)
save_temporal_data(root,i,gamma,tst_hour,declination,solarangle,solarelevation,excentricity,slots)
say("Projecting Linke's turbidity index... ")
linke.cut_projected(root)
say("Calculating the satellital zenith angle... ")
satellitalzenithangle = geo.getsatellitalzenithangle(lat, lon, SAT_LON)
dem.cut_projected(root)
v_satellitalzenithangle = nc.getvar(root,'satellitalzenithangle', 'f4', ('northing','easting',),4)
v_satellitalzenithangle[:] = satellitalzenithangle
nc.sync(root)
def process_temporal_data(lat, lon, root):
times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ]
indexes = range(len(times))
gamma = nc.clonevar(root,'data_time', 'gamma')
nc.sync(root)
tst_hour = nc.clonevar(root,'data', 'tst_hour')
declination = nc.clonevar(root, 'gamma', 'declination')
solarangle = nc.clonevar(root, 'data', 'solarangle')
solarelevation = nc.clonevar(root, 'solarangle', 'solarelevation')
excentricity = nc.clonevar(root, 'gamma', 'excentricity')
slots = nc.getvar(root,'slots', 'u1', ('timing',))
nc.sync(root)
for i in indexes:
show("\rTemporal data: preprocessing image %d / %d " % (i, len(indexes)-1))
dt = times[i]
# Calculate some geometry parameters
# Parameters that need the datetime: gamma, tst_hour, slots, linketurbidity
gamma[i] = geo.getdailyangle(geo.getjulianday(dt),geo.gettotaldays(dt))
tst_hour[i,:] = geo.gettsthour(geo.getdecimalhour(dt), GREENWICH_LON, lon, geo.gettimeequation(gamma[i]))
declination[i] = geo.getdeclination(gamma[i])
slots[i] = geo.getslots(dt,IMAGE_PER_HOUR)
omega = geo.gethourlyangle(tst_hour[i], lat/abs(lat))
solarangle[i] = geo.getzenithangle(declination[i],lat,omega)
solarelevation[i] = geo.getelevation(solarangle[i])
excentricity[i] = geo.getexcentricity(gamma[i])
nc.sync(root)
say("Projecting Linke's turbidity index... ")
linke.cut_projected(root)
say("Calculating the satellital zenith angle... ")
satellitalzenithangle = geo.getsatellitalzenithangle(lat, lon, SAT_LON)
dem.cut_projected(root)
v_satellitalzenithangle = nc.clonevar(root,'lat', 'satellitalzenithangle')
v_satellitalzenithangle[:] = satellitalzenithangle
nc.sync(root)
v_satellitalzenithangle = None
def process_atmospheric_irradiance(lat, lon, data, root):
i0met = geti0met()
dc = nc.getvar(root,'dc')[:]
satellitalzenithangle = nc.getvar(root,'satellitalzenithangle')[:]
excentricity = nc.getvar(root,'excentricity')[:]
say("Calculating atmospheric irradiance... ")
atmosphericradiance = geo.getatmosphericradiance(1367.0, i0met,dc, satellitalzenithangle)
atmosphericalbedo = geo.getalbedo(atmosphericradiance, i0met, excentricity, satellitalzenithangle)
satellitalelevation = geo.getelevation(satellitalzenithangle)
v_atmosphericalbedo = nc.getvar(root, 'atmosphericalbedo', 'f4', ('timing','northing','easting',),4)
v_atmosphericalbedo[:] = atmosphericalbedo
v_satellitalelevation = nc.getvar(root, 'satellitalelevation', 'f4', ('northing','easting',),4)
v_satellitalelevation[:] = satellitalelevation
nc.sync(root)
def process_albedos(data, root):
i0met = geti0met()
excentricity = nc.getvar(root,'excentricity')[:]
solarangle = nc.getvar(root,'solarangle')[:]
atmosphericalbedo = nc.getvar(root,'atmosphericalbedo')[:]
t_earth = nc.getvar(root,'t_earth')[:]
t_sat = nc.getvar(root,'t_sat')[:]
say("Calculating observed albedo, apparent albedo, effective albedo and cloud albedo... ")
observedalbedo = geo.getalbedo(data, i0met , excentricity, solarangle)
v_albedo = nc.clonevar(root, 'data', 'observedalbedo')
v_albedo[:] = observedalbedo
nc.sync(root)
apparentalbedo = geo.getapparentalbedo(observedalbedo,atmosphericalbedo, t_earth, t_sat)
v_albedo = nc.clonevar(root, 'data', 'apparentalbedo')
v_albedo[:] = apparentalbedo
nc.sync(root)
effectivealbedo = geo.geteffectivealbedo(solarangle)
v_albedo = nc.clonevar(root, 'data', 'effectivealbedo')
v_albedo[:] = effectivealbedo
nc.sync(root)
cloudalbedo = geo.getcloudalbedo(effectivealbedo,atmosphericalbedo,t_earth,t_sat)
v_albedo = nc.clonevar(root, 'data', 'cloudalbedo')
v_albedo[:] = cloudalbedo
nc.sync(root)
v_albedo = None
def rmse(root, index):
times = [
datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:]
]
days = [t.date() for t in times]
days.sort()
days_index = [d.day for d in set(days)]
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', (
'time',
'yc_cut',
'xc_cut',
), 4)
error = nc.getvar(root, 'error', 'f4', (
'time',
'yc_cut',
'xc_cut',
), 4)
diary_error = nc.getvar(root, 'diaryerror', 'f4', (
'diarying',
'yc_cut',
'xc_cut',
), 4)
error_diff[:] = np.zeros(estimated.shape)
error[:] = np.zeros(estimated.shape)
diary_error[:] = np.zeros(
(days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
#the_max = measurements[:].max()
error_diff[:,
index, :] = measurements[:, index, :] - estimated[:, index, :]
error[:, index, :] = np.abs(error_diff[:, index, :])
nc.sync(root)
max_value_in_day = np.zeros([days_amount]) + 1
for i in range(len(days)):
d_i = days_index.index(days[i].day)
max_value_in_day[d_i] = measurements[
i, index, 0] if max_value_in_day[d_i] < measurements[
i, index, 0] else max_value_in_day[d_i]
diary_error[d_i, index, :] += np.array([error_diff[i, index, 0]**2, 1])
count = diary_error[:, index, 1]
count[count == 0] = 1
diary_error[:, index, 0] = np.sqrt(diary_error[:, index, 0] / count)
diary_error[:, index,
1] = diary_error[:, index, 0] / max_value_in_day * 100
show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean())
for i in range(len(days)):
d_i = days_index.index(days[i].day)
error[i, index, 1] = error[i, index, 1] / max_value_in_day[d_i] * 100
result = np.sum(error[:, index, 1]**2)
result = np.sqrt(result / error.shape[0])
show("Half-hour RMS error: %.2f \n" % result)
#diary_error[:, index,1] = diary_error[:, index,0]
nc.sync(root)
nc.close(root)
def process_atmospheric_irradiance(root):
i0met = geti0met()
dc = nc.getvar(root,'dc')[:]
satellitalzenithangle = nc.getvar(root,'satellitalzenithangle')[:]
excentricity = nc.getvar(root,'excentricity')[:]
say("Calculating atmospheric irradiance... ")
atmosphericradiance = geo.getatmosphericradiance(1367.0, i0met,dc, satellitalzenithangle)
atmosphericalbedo = geo.getalbedo(atmosphericradiance, i0met, excentricity, satellitalzenithangle)
satellitalelevation = geo.getelevation(satellitalzenithangle)
v_atmosphericalbedo = nc.clonevar(root, 'data', 'atmosphericalbedo')
v_atmosphericalbedo[:] = atmosphericalbedo
v_satellitalelevation = nc.clonevar(root, 'lon', 'satellitalelevation')
v_satellitalelevation[:] = satellitalelevation
nc.sync(root)
v_atmosphericalbedo, v_satellitalelevation = None, None
def save_temporal_data(root,index,gamma,tst_hour,declination,solarangle,solarelevation,excentricity,slots):
v_gamma = nc.getvar(root,'gamma', 'f4', ('timing',),4)
v_gamma[index] = gamma
v_tst_hour = nc.getvar(root,'tst_hour', 'f4', ('timing','northing','easting',),4)
v_tst_hour[index,:] = tst_hour
v_declination = nc.getvar(root,'declination', 'f4', ('timing',),4)
v_declination[index] = declination
v_solarangle = nc.getvar(root, 'solarangle', 'f4', ('timing','northing','easting',),4)
v_solarangle[index] = solarangle
v_solarelevation = nc.getvar(root, 'solarelevation', 'f4', ('timing','northing','easting',),4)
v_solarelevation[index] = solarelevation
v_excentricity = nc.getvar(root, 'excentricity', 'f4', ('timing','northing','easting',),4)
v_excentricity[index] = excentricity
v_slots = nc.getvar(root,'slots', 'u1', ('timing',))
v_slots[index] = slots
nc.sync(root)
def rows2netcdf(rows, filename, index):
root, is_new = nc.open(filename)
if not is_new:
measurements = nc.getvar(root, 'measurements', 'f4', ('timing','northing','easting'), 4)
slots = nc.getvar(root, 'slots')
times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ]
instant_radiation = rows2slots(rows,2)
i_e = 0
i_m = 0
while i_e < len(times) and i_m < len(instant_radiation):
#print i_e,"/", len(times) -1, " | ", i_m, "/", len(instant_radiation) - 1
# When date estimated before date measured
if times[i_e].date() < instant_radiation[i_m][1][0].date():
i_e += 1
# When date estimated after date measured
elif times[i_e].date() > instant_radiation[i_m][1][0].date():
i_m += 1
else:
if slots[i_e] < instant_radiation[i_m][0]:
measurements[i_e, index,:] = np.array([-1, -1])
i_e += 1
elif slots[i_e] > instant_radiation[i_m][0]:
i_m += 1
else:
value = instant_radiation[i_m][1][1]
row_in_slot = instant_radiation[i_m][1][2]
measurements[i_e, index,:] = np.array([value, value])
i_e += 1
i_m += 1
days = [ t.date() for t in times ]
nc.getdim(root, 'diarying')
diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting'))
estimated = nc.getvar(root, 'globalradiation')[:]
error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting'), 4)
error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting'), 4)
error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
error[:, index, :] = np.sqrt((error_diff[:, index, :])**2)
print error_diff.shape
nc.sync(root)
for i in range(len(days)):
d = days[i].day
print index, i, d, error_diff[i,index], diary_error[d, index]
diary_error[d, index,:] += error_diff[i,index,:]
diary_error[:, index,:] = np.sqrt((diary_error[:, index,:])**2)
nc.close(root)
def process_irradiance(lat, lon, data, root):
excentricity = nc.getvar(root,'excentricity')[:]
solarangle = nc.getvar(root,'solarangle')[:]
solarelevation = nc.getvar(root,'solarelevation')[:]
linketurbidity = nc.getvar(root,'linketurbidity')[0]
terrain = nc.getvar(root,'dem')[:]
say("Calculating beam, diffuse and global irradiance... ")
# The average extraterrestrial irradiance is 1367.0 Watts/meter^2
bc = geo.getbeamirradiance(1367.0,excentricity,solarangle,solarelevation,linketurbidity,terrain)
dc = geo.getdiffuseirradiance(1367.0,excentricity,solarelevation,linketurbidity)
gc = geo.getglobalirradiance(bc,dc)
v_bc = nc.getvar(root, 'bc', 'f4', ('timing','northing','easting',),4)
v_bc[:] = bc
v_dc = nc.getvar(root, 'dc', 'f4', ('timing','northing','easting',),4)
v_dc[:] = dc
v_gc = nc.getvar(root, 'gc', 'f4', ('timing','northing','easting',),4)
v_gc[:] = gc
nc.sync(root)
def process_ground_albedo(lat, data, root):
slots = nc.getvar(root, "slots")[:]
declination = nc.getvar(root, "declination")[:]
#The day is divided in _slots_ to avoid the minutes diferences between days.
# TODO: Related with the solar hour at the noon if the pictures are taken every 15 minutes (meteosat)
say("Calculating the noon window... ")
slot_window_in_hours = 4
# On meteosat are 96 image per day
image_per_hour = IMAGE_PER_HOUR
image_per_day = 24 * image_per_hour
# and 48 image to the noon
noon_slot = image_per_day / 2
half_window = image_per_hour * slot_window_in_hours/2
min_slot = noon_slot - half_window
max_slot = noon_slot + half_window
# Create the condition used to work only with the data inside that window
say("Filtering the data outside the calculated window... ")
condition = ((slots >= min_slot) & (slots < max_slot)) # TODO: Meteosat: From 40 to 56 inclusive (the last one is not included)
apparentalbedo = nc.getvar(root, "apparentalbedo")[:]
m_apparentalbedo = np.ma.masked_array(apparentalbedo[condition], data[condition] <= (geti0met()/np.pi) * 0.03)
# To do the nexts steps needs a lot of memory
say("Calculating the ground reference albedo... ")
# TODO: Should review the p5_apparentalbedo parameters and shapes
p5_apparentalbedo = np.ma.masked_array(m_apparentalbedo, m_apparentalbedo < stats.scoreatpercentile(m_apparentalbedo, 5))
groundreferencealbedo = geo.getsecondmin(p5_apparentalbedo)
# Calculate the solar elevation using times, latitudes and omega
say("Calculating solar elevation... ")
r_alphanoon = geo.getsolarelevation(declination, lat, 0)
r_alphanoon = r_alphanoon * 2./3.
r_alphanoon[r_alphanoon > 40] = 40
r_alphanoon[r_alphanoon < 15] = 15
solarelevation = nc.getvar(root, "solarelevation")[:]
say("Calculating the apparent albedo second minimum... ")
groundminimumalbedo = geo.getsecondmin(np.ma.masked_array(apparentalbedo[condition], solarelevation[condition] < r_alphanoon[condition]))
aux_2g0 = 2 * groundreferencealbedo
aux_05g0 = 0.5 * groundreferencealbedo
groundminimumalbedo[groundminimumalbedo > aux_2g0] = aux_2g0[groundminimumalbedo > aux_2g0]
groundminimumalbedo[groundminimumalbedo < aux_05g0] = aux_05g0[groundminimumalbedo < aux_05g0]
say("Synchronizing with the NetCDF4 file... ")
f_groundalbedo = nc.clonevar(root, 'lat', 'groundalbedo')
f_groundalbedo[:] = groundminimumalbedo
nc.sync(root)
f_groundalbedo = None
def process_optical_fading(lat, lon, data, root):
solarelevation = nc.getvar(root,'solarelevation')[:]
terrain = nc.getvar(root,'dem')[:]
satellitalelevation = nc.getvar(root, 'satellitalelevation')[:]
linketurbidity = nc.getvar(root,'linketurbidity')[0]
say("Calculating optical path and optical depth... ")
# The maximum height of the non-transparent atmosphere is at 8434.5 mts
solar_opticalpath = geo.getopticalpath(geo.getcorrectedelevation(solarelevation),terrain, 8434.5)
solar_opticaldepth = geo.getopticaldepth(solar_opticalpath)
satellital_opticalpath = geo.getopticalpath(geo.getcorrectedelevation(satellitalelevation),terrain, 8434.5)
satellital_opticaldepth = geo.getopticaldepth(satellital_opticalpath)
say("Calculating sun-earth and earth-satellite transmitances... ")
t_earth = geo.gettransmitance(linketurbidity, solar_opticalpath, solar_opticaldepth, solarelevation)
t_sat = geo.gettransmitance(linketurbidity, satellital_opticalpath, satellital_opticaldepth, satellitalelevation)
v_earth = nc.getvar(root, 't_earth', 'f4', ('timing','northing','easting',),4)
v_earth[:] = t_earth
v_sat = nc.getvar(root, 't_sat', 'f4', ('northing','easting',),4)
v_sat[:] = t_sat
nc.sync(root)
def process_albedos(lat, lon, data, root):
i0met = geti0met()
excentricity = nc.getvar(root,'excentricity')[:]
solarangle = nc.getvar(root,'solarangle')[:]
atmosphericalbedo = nc.getvar(root,'atmosphericalbedo')[:]
t_earth = nc.getvar(root,'t_earth')[:]
t_sat = nc.getvar(root,'t_sat')[:]
say("Calculating observed albedo, apparent albedo, effective albedo and cloud albedo... ")
observedalbedo = geo.getalbedo(data, i0met , excentricity, solarangle)
apparentalbedo = geo.getapparentalbedo(observedalbedo,atmosphericalbedo, t_earth, t_sat)
effectivealbedo = geo.geteffectivealbedo(solarangle)
cloudalbedo = geo.getcloudalbedo(effectivealbedo,atmosphericalbedo,t_earth,t_sat)
v_observedalbedo = nc.getvar(root, 'observedalbedo', 'f4', ('timing','northing','easting',),4)
v_observedalbedo[:] = observedalbedo
v_apparentalbedo = nc.getvar(root, 'apparentalbedo', 'f4', ('timing','northing','easting',),4)
v_apparentalbedo[:] = apparentalbedo
v_effectivealbedo = nc.getvar(root, 'effectivealbedo', 'f4', ('timing','northing','easting',),4)
v_effectivealbedo[:] = effectivealbedo
v_cloudalbedo = nc.getvar(root, 'cloudalbedo', 'f4', ('timing','northing','easting',),4)
v_cloudalbedo[:] = cloudalbedo
nc.sync(root)
def process_radiation(root):
apparentalbedo = nc.getvar(root, "apparentalbedo")[:]
groundalbedo = nc.getvar(root, "groundalbedo")[:]
cloudalbedo = nc.getvar(root, "cloudalbedo")[:]
say("Calculating the cloud index... ")
cloudindex = geo.getcloudindex(apparentalbedo, groundalbedo, cloudalbedo)
apparentalbedo = None
groundalbedo = None
cloudalbedo = None
f_var = nc.clonevar(root, 'cloudalbedo', 'cloudinessindex')
f_var[:] = cloudindex
nc.sync(root)
say("Calculating the clear sky... ")
clearsky = geo.getclearsky(cloudindex)
cloudindex = None
f_var = nc.clonevar(root, 'cloudalbedo', 'clearskyindex')
f_var[:] = clearsky
nc.sync(root)
say("Calculating the global radiation... ")
clearskyglobalradiation = nc.getvar(root, 'gc')[:]
globalradiation = clearsky * clearskyglobalradiation
f_var = nc.clonevar(root, 'gc', 'globalradiation')
say("Saving the global radiation... ")
f_var[:] = globalradiation
nc.sync(root)
f_var = None
def do_file(self, filename, stream):
# create compact file and initialize basic settings
begin_time = self.getdatetimenow()
root, is_new = nc.open(filename)
if is_new:
sample, n = nc.open(stream.files.all()[0].file.completepath())
shape = sample.variables['data'].shape
nc.getdim(root,'northing', shape[1])
nc.getdim(root,'easting', shape[2])
nc.getdim(root,'timing')
v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
v_lon[:] = nc.getvar(sample, 'lon')[:]
v_lat[:] = nc.getvar(sample, 'lat')[:]
nc.close(sample)
nc.sync(root)
self.do_var(root, 'data', stream)
# save the content inside the compact file
if not root is None: nc.close(root)
f = File(localname=filename)
f.save()
return f
def do_var(self, root, var_name, stream):
count = 0
file_statuses = stream.sorted_files()
shape = nc.getvar(root,'lat').shape
for fs in file_statuses:
# join the distributed content
ch = fs.file.channel()
v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',))
try:
rootimg, n = nc.open(fs.file.completepath())
data = (nc.getvar(rootimg, 'data'))[:]
# Force all the channels to the same shape
if not (data.shape[1:3] == shape):
print data.shape[1:3], shape
data = matrix.adapt(data, shape)
if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
index = v_ch.shape[0]
v_ch[index,:] = data
v_ch_t[index] = calendar.timegm(fs.file.datetime().utctimetuple())
nc.close(rootimg)
nc.sync(root)
except RuntimeError, e:
print fs.file.completepath()
def rmse(root, index):
times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ]
days = [ t.date() for t in times ]
days.sort()
days_index = [d.day for d in set(days)]
days_amount = len(days_index)
d_dim = nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting',), 4)
error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting',), 4)
diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting',), 4)
error_diff[:] = np.zeros(estimated.shape)
error[:] = np.zeros(estimated.shape)
diary_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
the_max = measurements[:].max()
error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
error[:, index, :] = np.abs(error_diff[:, index, :])
nc.sync(root)
max_value_in_day = np.zeros([days_amount]) + 1
for i in range(len(days)):
d_i = days_index.index(days[i].day)
max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i]
diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1])
count = diary_error[:, index, 1]
count[count == 0] = 1
diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count)
diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100
show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean())
for i in range(len(days)):
d_i = days_index.index(days[i].day)
error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100
result = np.sum(error[:, index, 1] ** 2)
result = np.sqrt(result / error.shape[0])
show("Half-hour RMS error: %.2f \n" % result)
#diary_error[:, index,1] = diary_error[:, index,0]
nc.sync(root)
def dailyerrors(root, stations):
times = [
datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:]
]
days = [t.date() for t in times]
days.sort()
days_index = [d.day for d in set(days)]
days_index.sort()
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = root.getvar('errordiff', 'f4', (
'time',
'yc_cut',
'xc_cut',
), 4)
RMS_daily_error = root.getvar('RMSdailyerror', 'f4', (
'diarying',
'yc_cut',
'xc_cut',
), 4)
BIAS_daily_error = root.getvar('BIASdailyerror', 'f4', (
'diarying',
'yc_cut',
'xc_cut',
), 4)
error_diff[:] = np.zeros(estimated.shape)
RMS_daily_error[:] = np.zeros(
(days_amount, estimated.shape[1], estimated.shape[2]))
BIAS_daily_error[:] = np.zeros(
(days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
for s in stations:
index = stations.index(s)
show('Station: %s \n' % stations[index])
error_diff[:,
index, :] = measurements[:, index, :] - estimated[:,
index, :]
nc.sync(root)
sum_value_in_day = np.zeros((days_amount))
for i in range(len(days)):
d_i = days_index.index(days[i].day)
if not measurements[i, index, 0] == 0.0:
sum_value_in_day[d_i] += measurements[i, index, 0]
RMS_daily_error[d_i, index, :] += np.array(
[error_diff[i, index, 0]**2, 1])
BIAS_daily_error[d_i, index, :] += error_diff[i, index, 0]
count = RMS_daily_error[:, index, 1]
count[count == 0] = 1
RMS_daily_error[:, index, 0] = np.sqrt(
RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100
BIAS_daily_error[:, index, 0] = (BIAS_daily_error[:, index, 0] /
count) / sum_value_in_day * 100
RMS_daily_error[:, index, 1] = RMS_daily_error[:, index, 0]
BIAS_daily_error[:, index, 1] = BIAS_daily_error[:, index, 0]
print 'RMS :', RMS_daily_error[:, index, 0]
print 'BIAS', BIAS_daily_error[:, index, 0]
print 'sum value in day: ', sum_value_in_day[:]
show("\rDiary RMS daily error: %.2f\n" %
(RMS_daily_error[:, index, 0]).mean())
nc.sync(root)
nc.close(root)
