def process_temporal_data(lat, lon, root): times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ] indexes = range(len(times)) gamma = nc.clonevar(root,'data_time', 'gamma') nc.sync(root) tst_hour = nc.clonevar(root,'data', 'tst_hour') declination = nc.clonevar(root, 'gamma', 'declination') solarangle = nc.clonevar(root, 'data', 'solarangle') solarelevation = nc.clonevar(root, 'solarangle', 'solarelevation') excentricity = nc.clonevar(root, 'gamma', 'excentricity') slots = nc.getvar(root,'slots', 'u1', ('timing',)) nc.sync(root) for i in indexes: show("\rTemporal data: preprocessing image %d / %d " % (i, len(indexes)-1)) dt = times[i] # Calculate some geometry parameters # Parameters that need the datetime: gamma, tst_hour, slots, linketurbidity gamma[i] = geo.getdailyangle(geo.getjulianday(dt),geo.gettotaldays(dt)) tst_hour[i,:] = geo.gettsthour(geo.getdecimalhour(dt), GREENWICH_LON, lon, geo.gettimeequation(gamma[i])) declination[i] = geo.getdeclination(gamma[i]) slots[i] = geo.getslots(dt,IMAGE_PER_HOUR) omega = geo.gethourlyangle(tst_hour[i], lat/abs(lat)) solarangle[i] = geo.getzenithangle(declination[i],lat,omega) solarelevation[i] = geo.getelevation(solarangle[i]) excentricity[i] = geo.getexcentricity(gamma[i]) nc.sync(root) say("Projecting Linke's turbidity index... ") linke.cut_projected(root) say("Calculating the satellital zenith angle... ") satellitalzenithangle = geo.getsatellitalzenithangle(lat, lon, SAT_LON) dem.cut_projected(root) v_satellitalzenithangle = nc.clonevar(root,'lat', 'satellitalzenithangle') v_satellitalzenithangle[:] = satellitalzenithangle nc.sync(root) v_satellitalzenithangle = None
def process_temporal_data(lat, lon, root): times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ] indexes = range(len(times)) for i in indexes: show("\rTemporal data: preprocessing image %d / %d " % (i, len(indexes)-1)) dt = times[i] # Calculate some geometry parameters # Parameters that need the datetime: gamma, tst_hour, slots, linketurbidity gamma = geo.getdailyangle(geo.getjulianday(dt),geo.gettotaldays(dt)) tst_hour = geo.gettsthour(geo.getdecimalhour(dt), GREENWICH_LON, lon, geo.gettimeequation(gamma)) declination = geo.getdeclination(gamma) slots = geo.getslots(dt,IMAGE_PER_HOUR) omega = geo.gethourlyangle(tst_hour, lat/abs(lat)) solarangle = geo.getzenithangle(declination,lat,omega) solarelevation = geo.getelevation(solarangle) excentricity = geo.getexcentricity(gamma) save_temporal_data(root,i,gamma,tst_hour,declination,solarangle,solarelevation,excentricity,slots) say("Projecting Linke's turbidity index... ") linke.cut_projected(root) say("Calculating the satellital zenith angle... ") satellitalzenithangle = geo.getsatellitalzenithangle(lat, lon, SAT_LON) dem.cut_projected(root) v_satellitalzenithangle = nc.getvar(root,'satellitalzenithangle', 'f4', ('northing','easting',),4) v_satellitalzenithangle[:] = satellitalzenithangle nc.sync(root)
def do_file(self, filename, data): print "Creating ", filename # create compact file and initialize basic settings root, is_new = nc.open(filename + self.extension) # The filename does not contain the extension if is_new: sample, n = nc.open(data[(data.keys())[0].completepath()]) shape = sample.variables['data'].shape nc.getdim(root,'northing', shape[1]) nc.getdim(root,'easting', shape[2]) nc.getdim(root,'timing') v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4) v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4) v_lon[:] = nc.getvar(sample, 'lon')[:] v_lat[:] = nc.getvar(sample, 'lat')[:] nc.close(sample) if isinstance(data,collections.Mapping): for d in data: self.do_var(root, d, data[d]) else: self.create_var(root, 'data', data) # save the content inside the compact file if not root is None: nc.close(root) f = File(localname=localname, downloaded=True, begin_download=begin_time, end_download=datetime.now()) f.save() return f
def process_radiation(root): apparentalbedo = nc.getvar(root, "apparentalbedo")[:] groundalbedo = nc.getvar(root, "groundalbedo")[:] cloudalbedo = nc.getvar(root, "cloudalbedo")[:] say("Calculating the cloud index... ") cloudindex = geo.getcloudindex(apparentalbedo, groundalbedo, cloudalbedo) apparentalbedo = None groundalbedo = None cloudalbedo = None f_var = nc.clonevar(root, 'cloudalbedo', 'cloudinessindex') f_var[:] = cloudindex nc.sync(root) say("Calculating the clear sky... ") clearsky = geo.getclearsky(cloudindex) cloudindex = None f_var = nc.clonevar(root, 'cloudalbedo', 'clearskyindex') f_var[:] = clearsky nc.sync(root) say("Calculating the global radiation... ") clearskyglobalradiation = nc.getvar(root, 'gc')[:] globalradiation = clearsky * clearskyglobalradiation f_var = nc.clonevar(root, 'gc', 'globalradiation') say("Saving the global radiation... ") f_var[:] = globalradiation nc.sync(root) f_var = None
def process_albedos(data, root): i0met = geti0met() excentricity = nc.getvar(root,'excentricity')[:] solarangle = nc.getvar(root,'solarangle')[:] atmosphericalbedo = nc.getvar(root,'atmosphericalbedo')[:] t_earth = nc.getvar(root,'t_earth')[:] t_sat = nc.getvar(root,'t_sat')[:] say("Calculating observed albedo, apparent albedo, effective albedo and cloud albedo... ") observedalbedo = geo.getalbedo(data, i0met , excentricity, solarangle) v_albedo = nc.clonevar(root, 'data', 'observedalbedo') v_albedo[:] = observedalbedo nc.sync(root) apparentalbedo = geo.getapparentalbedo(observedalbedo,atmosphericalbedo, t_earth, t_sat) v_albedo = nc.clonevar(root, 'data', 'apparentalbedo') v_albedo[:] = apparentalbedo nc.sync(root) effectivealbedo = geo.geteffectivealbedo(solarangle) v_albedo = nc.clonevar(root, 'data', 'effectivealbedo') v_albedo[:] = effectivealbedo nc.sync(root) cloudalbedo = geo.getcloudalbedo(effectivealbedo,atmosphericalbedo,t_earth,t_sat) v_albedo = nc.clonevar(root, 'data', 'cloudalbedo') v_albedo[:] = cloudalbedo nc.sync(root) v_albedo = None
def do_var(self, root, var_name, files): count = 0 max_count = len(files[c]) print "Compacting ", var_name shape = nc.getvar(root,'lat').shape begin_time = datetime.now() for f in files: # join the distributed content ch = f.channel() v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4) v_ch_t = nc.getvar(root,var_name + 'time', 'f4', ('timing',)) try: rootimg, n = nc.open(f.completepath()) data = (nc.getvar(rootimg, 'data'))[:] # Force all the channels to the same shape data = matrix.adapt(data, shape) if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]: index = v_ch.shape[0] v_ch[index,:] = data v_ch_t[index] = calendar.timegm(f.image_datetime().utctimetuple()) nc.close(rootimg) nc.sync(root) print "\r","\t" + str(count/max_count * 100).zfill(2) + "%", except RuntimeError, e: print f.completepath()
def calibrated_data(root): data = nc.getvar(root, 'data')[:] counts_shift = nc.getvar(root, 'counts_shift')[0] space_measurement = nc.getvar(root, 'space_measurement')[0] prelaunch = nc.getvar(root, 'prelaunch')[0] postlaunch = nc.getvar(root, 'postlaunch')[0] print prelaunch, "= 0.6118208", postlaunch, "= 1.181", counts_shift, space_measurement # INFO: Without the postlaunch coefficient the RMSE go to 15% return postlaunch * prelaunch * (np.float32(data) / counts_shift - space_measurement)
def latlon(self): try: root, n = nc.open(self.completepath()) lat = nc.getvar(root,'lat')[:] lon = nc.getvar(root, 'lon')[:] nc.close(root) except RuntimeError: show(self.completepath()) lat, lon = None, None return lat, lon
def cut_projected(root): lat = nc.getvar(root, "lat") lon = nc.getvar(root, "lon") time = nc.getvar(root, "time") months = list(set([(datetime.fromtimestamp(int(t))).month for t in time])) months_d = nc.getdim(root, "monthing") months_cut = nc.getvar(root, "months", "i2", ("monthing",)) linke = nc.getvar(root, "linketurbidity", "f4", ("monthing", "northing", "easting"), 4) linke_x, linke_y = project_coordinates(lat[:], lon[:]) months_cut[:] = np.array(list(months)) for i in range(len(months)): linke[i] = cut_month(linke_x, linke_y, months[i]) return linke[:], linke_x, linke_y
def process_atmospheric_irradiance(lat, lon, data, root): i0met = geti0met() dc = nc.getvar(root,'dc')[:] satellitalzenithangle = nc.getvar(root,'satellitalzenithangle')[:] excentricity = nc.getvar(root,'excentricity')[:] say("Calculating atmospheric irradiance... ") atmosphericradiance = geo.getatmosphericradiance(1367.0, i0met,dc, satellitalzenithangle) atmosphericalbedo = geo.getalbedo(atmosphericradiance, i0met, excentricity, satellitalzenithangle) satellitalelevation = geo.getelevation(satellitalzenithangle) v_atmosphericalbedo = nc.getvar(root, 'atmosphericalbedo', 'f4', ('timing','northing','easting',),4) v_atmosphericalbedo[:] = atmosphericalbedo v_satellitalelevation = nc.getvar(root, 'satellitalelevation', 'f4', ('northing','easting',),4) v_satellitalelevation[:] = satellitalelevation nc.sync(root)
def process_atmospheric_irradiance(root): i0met = geti0met() dc = nc.getvar(root,'dc')[:] satellitalzenithangle = nc.getvar(root,'satellitalzenithangle')[:] excentricity = nc.getvar(root,'excentricity')[:] say("Calculating atmospheric irradiance... ") atmosphericradiance = geo.getatmosphericradiance(1367.0, i0met,dc, satellitalzenithangle) atmosphericalbedo = geo.getalbedo(atmosphericradiance, i0met, excentricity, satellitalzenithangle) satellitalelevation = geo.getelevation(satellitalzenithangle) v_atmosphericalbedo = nc.clonevar(root, 'data', 'atmosphericalbedo') v_atmosphericalbedo[:] = atmosphericalbedo v_satellitalelevation = nc.clonevar(root, 'lon', 'satellitalelevation') v_satellitalelevation[:] = satellitalelevation nc.sync(root) v_atmosphericalbedo, v_satellitalelevation = None, None
def cut_projected(root): lat = nc.getvar(root, 'lat') lon = nc.getvar(root, 'lon') time = nc.getvar(root, 'data_time') months = list(set([ (datetime.fromtimestamp(int(t))).month for t in time ])) months_d = nc.getdim(root, 'monthing') months_cut = nc.getvar(root, 'months', 'i2', ('monthing',)) dims = list(lat.dimensions) dims.insert(0, 'monthing') linke = nc.getvar(root, 'linketurbidity', 'f4', tuple(dims),4) linke_x, linke_y = project_coordinates(lat[:], lon[:]) months_cut[:] = np.array(list(months)) for i in range(len(months)): linke[i] = cut_month(linke_x, linke_y, months[i]) return linke[:], linke_x, linke_y
def cut(lat,lon): import numpy as np root, is_new = nc.open(localpath+"/dem.nc") dem = nc.getvar(root, 'dem', 'i2', lat.dimensions) x, y = p.pixels_from_coordinates(lat[:], lon[:], dem.shape[0], dem.shape[1]) # In the transformation the 'y' dimension is twisted because the map is inverted. result = p.transform_data(dem,x,dem.shape[0] - y) nc.close(root) return result, x, y
def get_month(month): if gdal_supported: ds = gdal.Open(localpath + "/tifs/"+str(month).zfill(2)+"_longlat_wgs84.tif") linke = ds.ReadAsArray() else: root, n = nc.open(destiny) data = nc.getvar(root, "linketurbidity") linke = data[month -1] # The linke turbidity is obtained when the image pixel value is divied by 20. return linke/20.
def process_irradiance(root): excentricity = nc.getvar(root,'excentricity')[:] solarangle = nc.getvar(root,'solarangle')[:] solarelevation = nc.getvar(root,'solarelevation')[:] linketurbidity = nc.getvar(root,'linketurbidity')[0] terrain = nc.getvar(root,'dem')[:] say("Calculating beam, diffuse and global irradiance... ") # The average extraterrestrial irradiance is 1367.0 Watts/meter^2 bc = geo.getbeamirradiance(1367.0,excentricity,solarangle,solarelevation,linketurbidity,terrain) dc = geo.getdiffuseirradiance(1367.0,excentricity,solarelevation,linketurbidity) gc = geo.getglobalirradiance(bc,dc) v_bc = nc.clonevar(root, 'data', 'bc') v_bc[:] = bc v_dc = nc.clonevar(root, 'data', 'dc') v_dc[:] = dc v_gc = nc.clonevar(root, 'data', 'gc') v_gc[:] = gc nc.sync(root) v_gc, v_dc, v_bc = None, None, None
def process_ground_albedo(lat, data, root): slots = nc.getvar(root, "slots")[:] declination = nc.getvar(root, "declination")[:] #The day is divided in _slots_ to avoid the minutes diferences between days. # TODO: Related with the solar hour at the noon if the pictures are taken every 15 minutes (meteosat) say("Calculating the noon window... ") slot_window_in_hours = 4 # On meteosat are 96 image per day image_per_hour = IMAGE_PER_HOUR image_per_day = 24 * image_per_hour # and 48 image to the noon noon_slot = image_per_day / 2 half_window = image_per_hour * slot_window_in_hours/2 min_slot = noon_slot - half_window max_slot = noon_slot + half_window # Create the condition used to work only with the data inside that window say("Filtering the data outside the calculated window... ") condition = ((slots >= min_slot) & (slots < max_slot)) # TODO: Meteosat: From 40 to 56 inclusive (the last one is not included) apparentalbedo = nc.getvar(root, "apparentalbedo")[:] m_apparentalbedo = np.ma.masked_array(apparentalbedo[condition], data[condition] <= (geti0met()/np.pi) * 0.03) # To do the nexts steps needs a lot of memory say("Calculating the ground reference albedo... ") # TODO: Should review the p5_apparentalbedo parameters and shapes p5_apparentalbedo = np.ma.masked_array(m_apparentalbedo, m_apparentalbedo < stats.scoreatpercentile(m_apparentalbedo, 5)) groundreferencealbedo = geo.getsecondmin(p5_apparentalbedo) # Calculate the solar elevation using times, latitudes and omega say("Calculating solar elevation... ") r_alphanoon = geo.getsolarelevation(declination, lat, 0) r_alphanoon = r_alphanoon * 2./3. r_alphanoon[r_alphanoon > 40] = 40 r_alphanoon[r_alphanoon < 15] = 15 solarelevation = nc.getvar(root, "solarelevation")[:] say("Calculating the apparent albedo second minimum... ") groundminimumalbedo = geo.getsecondmin(np.ma.masked_array(apparentalbedo[condition], solarelevation[condition] < r_alphanoon[condition])) aux_2g0 = 2 * groundreferencealbedo aux_05g0 = 0.5 * groundreferencealbedo groundminimumalbedo[groundminimumalbedo > aux_2g0] = aux_2g0[groundminimumalbedo > aux_2g0] groundminimumalbedo[groundminimumalbedo < aux_05g0] = aux_05g0[groundminimumalbedo < aux_05g0] say("Synchronizing with the NetCDF4 file... ") f_groundalbedo = nc.clonevar(root, 'lat', 'groundalbedo') f_groundalbedo[:] = groundminimumalbedo nc.sync(root) f_groundalbedo = None
def process_optical_fading(root): solarelevation = nc.getvar(root,'solarelevation')[:] terrain = nc.getvar(root,'dem')[:] satellitalelevation = nc.getvar(root, 'satellitalelevation')[:] linketurbidity = nc.getvar(root,'linketurbidity')[0] say("Calculating optical path and optical depth... ") # The maximum height of the non-transparent atmosphere is at 8434.5 mts solar_opticalpath = geo.getopticalpath(geo.getcorrectedelevation(solarelevation),terrain, 8434.5) solar_opticaldepth = geo.getopticaldepth(solar_opticalpath) satellital_opticalpath = geo.getopticalpath(geo.getcorrectedelevation(satellitalelevation),terrain, 8434.5) satellital_opticaldepth = geo.getopticaldepth(satellital_opticalpath) say("Calculating sun-earth and earth-satellite transmitances... ") t_earth = geo.gettransmitance(linketurbidity, solar_opticalpath, solar_opticaldepth, solarelevation) t_sat = geo.gettransmitance(linketurbidity, satellital_opticalpath, satellital_opticaldepth, satellitalelevation) v_earth = nc.clonevar(root, 'data', 't_earth') v_earth[:] = t_earth v_sat = nc.clonevar(root, 'satellitalelevation', 't_sat') v_sat[:] = t_sat nc.sync(root) v_earth, v_sat = None, None
def process_validate(root): from libs.statistics import error estimated = nc.getvar(root, 'globalradiation') measured = nc.getvar(root, 'measurements') stations = [0] for s in stations: show("==========\n") say("Station %i (%i slots)" % (s, measured[:,s,0].size)) show("----------") show("mean (measured):\t", error.ghi_mean(measured, s)) show("mean (estimated):\t", estimated[:,s,0].mean()) ghi_ratio = error.ghi_ratio(measured, s) bias = error.bias(estimated, measured, s) show("BIAS:\t%.5f\t( %.5f %%)" % (bias, bias * ghi_ratio)) rmse = error.rmse_es(estimated, measured, s) show("RMSE:\t%.5f\t( %.5f %%)" % (rmse, rmse * ghi_ratio)) mae = error.mae(estimated, measured, s) show("MAE:\t%.5f\t( %.5f %%)" % (mae, mae * ghi_ratio)) show("----------\n") error.rmse(root, s)
def do(self, files): for c in files: for f in files[c]: if f.channel() == '01': root, is_new = nc.open(f.completepath()) var = nc.getvar(root, 'data') data = np.zeros(var.shape) for i in range(var.shape[0]): data = var[i] data = self.calibrated_coefficient * ((data / self.counts_shift) - self.space_measurement) var[i] = data nc.close(root) return files
def rows2netcdf(rows, filename, index): root, is_new = nc.open(filename) if not is_new: measurements = nc.clonevar(root, 'globalradiation', 'measurements') slots = nc.getvar(root, 'slots') times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ] instant_radiation = rows2slots(rows,2) earth_failures = 0 i_e = 0 i_m = 0 while i_e < len(times) and i_m < len(instant_radiation): # When date estimated before date measured if times[i_e].date() < instant_radiation[i_m][1][0].date(): i_e += 1 # When date estimated after date measured elif times[i_e].date() > instant_radiation[i_m][1][0].date(): i_m += 1 else: if slots[i_e] < instant_radiation[i_m][0]: # TODO: This should be completed with a 0 error from the estimation. measurements[i_e, index,:] = np.array([0, 0]) earth_failures += 1 i_e += 1 elif slots[i_e] > instant_radiation[i_m][0]: i_m += 1 else: value = instant_radiation[i_m][1][1] #row_in_slot = instant_radiation[i_m][1][2] measurements[i_e, index,:] = np.array([value, value]) i_e += 1 i_m += 1 while i_e < len(times): # TODO: This should be completed with a 0 error from the estimation. measurements[i_e, index,:] = np.array([0, 0]) earth_failures += 1 i_e += 1 show("Detected %i of %i estimated times without earth measure.\n" % (earth_failures, len(slots))) error.rmse(root, index) nc.close(root)
def rmse(root, index): times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [t.date() for t in times] days.sort() days_index = [d.day for d in set(days)] days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = nc.getvar(root, 'errordiff', 'f4', ( 'time', 'yc_cut', 'xc_cut', ), 4) error = nc.getvar(root, 'error', 'f4', ( 'time', 'yc_cut', 'xc_cut', ), 4) diary_error = nc.getvar(root, 'diaryerror', 'f4', ( 'diarying', 'yc_cut', 'xc_cut', ), 4) error_diff[:] = np.zeros(estimated.shape) error[:] = np.zeros(estimated.shape) diary_error[:] = np.zeros( (days_amount, estimated.shape[1], estimated.shape[2])) nc.sync(root) #the_max = measurements[:].max() error_diff[:, index, :] = measurements[:, index, :] - estimated[:, index, :] error[:, index, :] = np.abs(error_diff[:, index, :]) nc.sync(root) max_value_in_day = np.zeros([days_amount]) + 1 for i in range(len(days)): d_i = days_index.index(days[i].day) max_value_in_day[d_i] = measurements[ i, index, 0] if max_value_in_day[d_i] < measurements[ i, index, 0] else max_value_in_day[d_i] diary_error[d_i, index, :] += np.array([error_diff[i, index, 0]**2, 1]) count = diary_error[:, index, 1] count[count == 0] = 1 diary_error[:, index, 0] = np.sqrt(diary_error[:, index, 0] / count) diary_error[:, index, 1] = diary_error[:, index, 0] / max_value_in_day * 100 show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean()) for i in range(len(days)): d_i = days_index.index(days[i].day) error[i, index, 1] = error[i, index, 1] / max_value_in_day[d_i] * 100 result = np.sum(error[:, index, 1]**2) result = np.sqrt(result / error.shape[0]) show("Half-hour RMS error: %.2f \n" % result) #diary_error[:, index,1] = diary_error[:, index,0] nc.sync(root) nc.close(root)
def do_file(self, filename, stream): # create compact file and initialize basic settings begin_time = self.getdatetimenow() root, is_new = nc.open(filename) if is_new: sample, n = nc.open(stream.files.all()[0].file.completepath()) shape = sample.variables['data'].shape nc.getdim(root,'northing', shape[1]) nc.getdim(root,'easting', shape[2]) nc.getdim(root,'timing') v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4) v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4) v_lon[:] = nc.getvar(sample, 'lon')[:] v_lat[:] = nc.getvar(sample, 'lat')[:] nc.close(sample) nc.sync(root) self.do_var(root, 'data', stream) # save the content inside the compact file if not root is None: nc.close(root) f = File(localname=filename) f.save() return f
def do(self, stream): from libs.sat.goes import calibration resultant_stream = stream.clone() for fs in stream.files.all(): f = fs.file if f.channel() == '01': sat = f.satellite() root, is_new = nc.open(f.completepath()) nc.getdim(root,'coefficient',1) var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4) var[0] = calibration.counts_shift.coefficient(sat) var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4) var[0] = calibration.space_measurement.coefficient(sat) var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4) var[0] = calibration.prelaunch.coefficient(sat)[0] var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4) dt = f.datetime() var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month) #data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement)) nc.close(root) fs.processed=True fs.save() return resultant_stream
def do(self, stream): from libs.sat.goes import calibration resultant_stream = stream.clone() for ms in stream.materials.all(): m = ms.material if hasattr(m, 'channel') and hasattr(m, 'satellite') and m.channel() == '01': sat = m.satellite() root = nc.open(m.completepath())[0] nc.getdim(root,'coefficient',1) var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4) var[0] = calibration.counts_shift.coefficient(sat) var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4) var[0] = calibration.space_measurement.coefficient(sat) var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4) var[0] = calibration.prelaunch.coefficient(sat)[0] var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4) dt = m.datetime() var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month) #data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement)) nc.close(root) ms.processed=True ms.save() return resultant_stream
def process_validate(year, month, times, root): tst_hour_step = 1/24. globalradiation = nc.getvar(root, 'globalradiation') clearskyradiation = nc.getvar(root, 'clearskyglobalradiation') timestamp = np.array([date2num(dt) for dt in times]) tst_hour = nc.getvar(root, 'tst_hour') stations = pgs.getmeasuresinstations(year, month) for s in stations: l,c = dp.getpos(float(s['latitude']), float(s['longitude'])) globalradiationinposition = globalradiation[:,l,c] tst_datehour = gettstdatetime(timestamp,tst_hour[:,l,c]) estimated = [] measured = [] for m in s['measures']: cond = ((tst_datehour > m['timestamp'] - tst_hour_step) & (tst_datehour <= m['timestamp'])) tst_filtered = tst_datehour[cond] if tst_filtered.size >= 4 and m['ghi'] > 0: estimated.append(globalradiationinposition[cond].mean()) measured.append(m['ghi']) measured = np.array(measured) estimated = np.array(estimated) diff = estimated - measured ghi_mean = measured.mean() ghi_ratio = 100 / ghi_mean show("----------") show("Length:", measured.size) show("mean(GHI)", ghi_mean) show("mean(estimated)", estimated.mean()) show(s['Name']) bias = diff.mean() show("BIAS:", bias, "(", bias * ghi_ratio, "%)") rmse = np.sqrt((diff**2).mean()) show("RMSE:", rmse, "(", rmse * ghi_ratio, "%)") mae = np.absolute(diff).mean() show("MAE:", mae, "(", mae * ghi_ratio, "%)")
def do_var(self, root, var_name, stream): count = 0 file_statuses = stream.sorted_files() shape = nc.getvar(root,'lat').shape for fs in file_statuses: # join the distributed content ch = fs.file.channel() v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4) v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',)) try: rootimg, n = nc.open(fs.file.completepath()) data = (nc.getvar(rootimg, 'data'))[:] # Force all the channels to the same shape if not (data.shape[1:3] == shape): print data.shape[1:3], shape data = matrix.adapt(data, shape) if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]: index = v_ch.shape[0] v_ch[index,:] = data v_ch_t[index] = calendar.timegm(fs.file.datetime().utctimetuple()) nc.close(rootimg) nc.sync(root) except RuntimeError, e: print fs.file.completepath()
def process_albedos(lat, lon, data, root): i0met = geti0met() excentricity = nc.getvar(root,'excentricity')[:] solarangle = nc.getvar(root,'solarangle')[:] atmosphericalbedo = nc.getvar(root,'atmosphericalbedo')[:] t_earth = nc.getvar(root,'t_earth')[:] t_sat = nc.getvar(root,'t_sat')[:] say("Calculating observed albedo, apparent albedo, effective albedo and cloud albedo... ") observedalbedo = geo.getalbedo(data, i0met , excentricity, solarangle) apparentalbedo = geo.getapparentalbedo(observedalbedo,atmosphericalbedo, t_earth, t_sat) effectivealbedo = geo.geteffectivealbedo(solarangle) cloudalbedo = geo.getcloudalbedo(effectivealbedo,atmosphericalbedo,t_earth,t_sat) v_observedalbedo = nc.getvar(root, 'observedalbedo', 'f4', ('timing','northing','easting',),4) v_observedalbedo[:] = observedalbedo v_apparentalbedo = nc.getvar(root, 'apparentalbedo', 'f4', ('timing','northing','easting',),4) v_apparentalbedo[:] = apparentalbedo v_effectivealbedo = nc.getvar(root, 'effectivealbedo', 'f4', ('timing','northing','easting',),4) v_effectivealbedo[:] = effectivealbedo v_cloudalbedo = nc.getvar(root, 'cloudalbedo', 'f4', ('timing','northing','easting',),4) v_cloudalbedo[:] = cloudalbedo nc.sync(root)
def save_temporal_data(root,index,gamma,tst_hour,declination,solarangle,solarelevation,excentricity,slots): v_gamma = nc.getvar(root,'gamma', 'f4', ('timing',),4) v_gamma[index] = gamma v_tst_hour = nc.getvar(root,'tst_hour', 'f4', ('timing','northing','easting',),4) v_tst_hour[index,:] = tst_hour v_declination = nc.getvar(root,'declination', 'f4', ('timing',),4) v_declination[index] = declination v_solarangle = nc.getvar(root, 'solarangle', 'f4', ('timing','northing','easting',),4) v_solarangle[index] = solarangle v_solarelevation = nc.getvar(root, 'solarelevation', 'f4', ('timing','northing','easting',),4) v_solarelevation[index] = solarelevation v_excentricity = nc.getvar(root, 'excentricity', 'f4', ('timing','northing','easting',),4) v_excentricity[index] = excentricity v_slots = nc.getvar(root,'slots', 'u1', ('timing',)) v_slots[index] = slots nc.sync(root)
def process_radiation(lat, lon, data, root): apparentalbedo = nc.getvar(root, "apparentalbedo")[:] groundalbedo = nc.getvar(root, "groundalbedo")[:] cloudalbedo = nc.getvar(root, "cloudalbedo")[:] say("Calculating the cloud index... ") cloudindex = geo.getcloudindex(apparentalbedo, groundalbedo, cloudalbedo) f_cloudindex = nc.getvar(root, 'cloudinessindex', 'f4', ('timing','northing','easting',),4) f_cloudindex[:] = cloudindex say("Calculating the clear sky... ") clearsky = geo.getclearsky(cloudindex) f_clearsky = nc.getvar(root, 'clearskyindex', 'f4', ('timing','northing','easting',),4) f_clearsky[:] = clearsky say("Calculating the global radiation... ") clearskyglobalradiation = nc.getvar(root, 'gc')[:] globalradiation = clearsky * clearskyglobalradiation f_globalradiation = nc.getvar(root, 'globalradiation', 'f4', ('timing','northing','easting',),4) f_globalradiation[:] = globalradiation
def merge_tails(): import glob import numpy as np #from scipy import misc say("Merging all the tails in one netCDF file...") root, is_new = nc.open(localpath+ "/dem.nc") n_d = nc.getdim(root,'northing', 180 * 120) # 180 degrees * 120 px/degree n_e = nc.getdim(root,'easting', 360 * 120) # 360 degrees * 120 px/degree data = nc.getvar(root, 'dem', 'i2', ('northing','easting',)) files = glob.glob(localpath + "/all10/all10/*10g") tails = [[file_coordinates(f),f] for f in files] for t, f in tails: print t, f tmp = np.fromfile(f,np.int16) dynamic = tmp.shape[0]/10800 tmp = tmp.reshape(dynamic,10800) #misc.imsave(f+'.png',tmp) data[t[0]:t[0]+dynamic,t[1]:t[1]+10800] = tmp nc.close(root)
def rows2netcdf(rows, filename, index): root, is_new = nc.open(filename) if not is_new: measurements = nc.getvar(root, 'measurements', 'f4', ('timing','northing','easting'), 4) slots = nc.getvar(root, 'slots') times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ] instant_radiation = rows2slots(rows,2) i_e = 0 i_m = 0 while i_e < len(times) and i_m < len(instant_radiation): #print i_e,"/", len(times) -1, " | ", i_m, "/", len(instant_radiation) - 1 # When date estimated before date measured if times[i_e].date() < instant_radiation[i_m][1][0].date(): i_e += 1 # When date estimated after date measured elif times[i_e].date() > instant_radiation[i_m][1][0].date(): i_m += 1 else: if slots[i_e] < instant_radiation[i_m][0]: measurements[i_e, index,:] = np.array([-1, -1]) i_e += 1 elif slots[i_e] > instant_radiation[i_m][0]: i_m += 1 else: value = instant_radiation[i_m][1][1] row_in_slot = instant_radiation[i_m][1][2] measurements[i_e, index,:] = np.array([value, value]) i_e += 1 i_m += 1 days = [ t.date() for t in times ] nc.getdim(root, 'diarying') diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting')) estimated = nc.getvar(root, 'globalradiation')[:] error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting'), 4) error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting'), 4) error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:] error[:, index, :] = np.sqrt((error_diff[:, index, :])**2) print error_diff.shape nc.sync(root) for i in range(len(days)): d = days[i].day print index, i, d, error_diff[i,index], diary_error[d, index] diary_error[d, index,:] += error_diff[i,index,:] diary_error[:, index,:] = np.sqrt((diary_error[:, index,:])**2) nc.close(root)
def dailyerrors(root, stations): times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [t.date() for t in times] days.sort() days_index = [d.day for d in set(days)] days_index.sort() days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = root.getvar('errordiff', 'f4', ( 'time', 'yc_cut', 'xc_cut', ), 4) RMS_daily_error = root.getvar('RMSdailyerror', 'f4', ( 'diarying', 'yc_cut', 'xc_cut', ), 4) BIAS_daily_error = root.getvar('BIASdailyerror', 'f4', ( 'diarying', 'yc_cut', 'xc_cut', ), 4) error_diff[:] = np.zeros(estimated.shape) RMS_daily_error[:] = np.zeros( (days_amount, estimated.shape[1], estimated.shape[2])) BIAS_daily_error[:] = np.zeros( (days_amount, estimated.shape[1], estimated.shape[2])) nc.sync(root) for s in stations: index = stations.index(s) show('Station: %s \n' % stations[index]) error_diff[:, index, :] = measurements[:, index, :] - estimated[:, index, :] nc.sync(root) sum_value_in_day = np.zeros((days_amount)) for i in range(len(days)): d_i = days_index.index(days[i].day) if not measurements[i, index, 0] == 0.0: sum_value_in_day[d_i] += measurements[i, index, 0] RMS_daily_error[d_i, index, :] += np.array( [error_diff[i, index, 0]**2, 1]) BIAS_daily_error[d_i, index, :] += error_diff[i, index, 0] count = RMS_daily_error[:, index, 1] count[count == 0] = 1 RMS_daily_error[:, index, 0] = np.sqrt( RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 BIAS_daily_error[:, index, 0] = (BIAS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 RMS_daily_error[:, index, 1] = RMS_daily_error[:, index, 0] BIAS_daily_error[:, index, 1] = BIAS_daily_error[:, index, 0] print 'RMS :', RMS_daily_error[:, index, 0] print 'BIAS', BIAS_daily_error[:, index, 0] print 'sum value in day: ', sum_value_in_day[:] show("\rDiary RMS daily error: %.2f\n" % (RMS_daily_error[:, index, 0]).mean()) nc.sync(root) nc.close(root)