def do_file(self, filename, data): print "Creating ", filename # create compact file and initialize basic settings root, is_new = nc.open(filename + self.extension) # The filename does not contain the extension if is_new: sample, n = nc.open(data[(data.keys())[0].completepath()]) shape = sample.variables['data'].shape nc.getdim(root,'northing', shape[1]) nc.getdim(root,'easting', shape[2]) nc.getdim(root,'timing') v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4) v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4) v_lon[:] = nc.getvar(sample, 'lon')[:] v_lat[:] = nc.getvar(sample, 'lat')[:] nc.close(sample) if isinstance(data,collections.Mapping): for d in data: self.do_var(root, d, data[d]) else: self.create_var(root, 'data', data) # save the content inside the compact file if not root is None: nc.close(root) f = File(localname=localname, downloaded=True, begin_download=begin_time, end_download=datetime.now()) f.save() return f
def do_var(self, root, var_name, files): count = 0 max_count = len(files[c]) print "Compacting ", var_name shape = nc.getvar(root,'lat').shape begin_time = datetime.now() for f in files: # join the distributed content ch = f.channel() v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4) v_ch_t = nc.getvar(root,var_name + 'time', 'f4', ('timing',)) try: rootimg, n = nc.open(f.completepath()) data = (nc.getvar(rootimg, 'data'))[:] # Force all the channels to the same shape data = matrix.adapt(data, shape) if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]: index = v_ch.shape[0] v_ch[index,:] = data v_ch_t[index] = calendar.timegm(f.image_datetime().utctimetuple()) nc.close(rootimg) nc.sync(root) print "\r","\t" + str(count/max_count * 100).zfill(2) + "%", except RuntimeError, e: print f.completepath()
def cut(lat,lon): import numpy as np root, is_new = nc.open(localpath+"/dem.nc") dem = nc.getvar(root, 'dem', 'i2', lat.dimensions) x, y = p.pixels_from_coordinates(lat[:], lon[:], dem.shape[0], dem.shape[1]) # In the transformation the 'y' dimension is twisted because the map is inverted. result = p.transform_data(dem,x,dem.shape[0] - y) nc.close(root) return result, x, y
def latlon(self): try: root, n = nc.open(self.completepath()) lat = nc.getvar(root,'lat')[:] lon = nc.getvar(root, 'lon')[:] nc.close(root) except RuntimeError: show(self.completepath()) lat, lon = None, None return lat, lon
def get_month(month): if gdal_supported: ds = gdal.Open(localpath + "/tifs/"+str(month).zfill(2)+"_longlat_wgs84.tif") linke = ds.ReadAsArray() else: root, n = nc.open(destiny) data = nc.getvar(root, "linketurbidity") linke = data[month -1] # The linke turbidity is obtained when the image pixel value is divied by 20. return linke/20.
def do(self, files): for c in files: for f in files[c]: if f.channel() == '01': root, is_new = nc.open(f.completepath()) var = nc.getvar(root, 'data') data = np.zeros(var.shape) for i in range(var.shape[0]): data = var[i] data = self.calibrated_coefficient * ((data / self.counts_shift) - self.space_measurement) var[i] = data nc.close(root) return files
def do_file(self, filename, stream): # create compact file and initialize basic settings begin_time = self.getdatetimenow() root, is_new = nc.open(filename) if is_new: sample, n = nc.open(stream.files.all()[0].file.completepath()) shape = sample.variables['data'].shape nc.getdim(root,'northing', shape[1]) nc.getdim(root,'easting', shape[2]) nc.getdim(root,'timing') v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4) v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4) v_lon[:] = nc.getvar(sample, 'lon')[:] v_lat[:] = nc.getvar(sample, 'lat')[:] nc.close(sample) nc.sync(root) self.do_var(root, 'data', stream) # save the content inside the compact file if not root is None: nc.close(root) f = File(localname=filename) f.save() return f
def rows2netcdf(rows, filename, index): root, is_new = nc.open(filename) if not is_new: measurements = nc.getvar(root, 'measurements', 'f4', ('timing','northing','easting'), 4) slots = nc.getvar(root, 'slots') times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ] instant_radiation = rows2slots(rows,2) i_e = 0 i_m = 0 while i_e < len(times) and i_m < len(instant_radiation): #print i_e,"/", len(times) -1, " | ", i_m, "/", len(instant_radiation) - 1 # When date estimated before date measured if times[i_e].date() < instant_radiation[i_m][1][0].date(): i_e += 1 # When date estimated after date measured elif times[i_e].date() > instant_radiation[i_m][1][0].date(): i_m += 1 else: if slots[i_e] < instant_radiation[i_m][0]: measurements[i_e, index,:] = np.array([-1, -1]) i_e += 1 elif slots[i_e] > instant_radiation[i_m][0]: i_m += 1 else: value = instant_radiation[i_m][1][1] row_in_slot = instant_radiation[i_m][1][2] measurements[i_e, index,:] = np.array([value, value]) i_e += 1 i_m += 1 days = [ t.date() for t in times ] nc.getdim(root, 'diarying') diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting')) estimated = nc.getvar(root, 'globalradiation')[:] error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting'), 4) error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting'), 4) error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:] error[:, index, :] = np.sqrt((error_diff[:, index, :])**2) print error_diff.shape nc.sync(root) for i in range(len(days)): d = days[i].day print index, i, d, error_diff[i,index], diary_error[d, index] diary_error[d, index,:] += error_diff[i,index,:] diary_error[:, index,:] = np.sqrt((diary_error[:, index,:])**2) nc.close(root)
def merge_tails(): import glob import numpy as np #from scipy import misc say("Merging all the tails in one netCDF file...") root, is_new = nc.open(localpath+ "/dem.nc") n_d = nc.getdim(root,'northing', 180 * 120) # 180 degrees * 120 px/degree n_e = nc.getdim(root,'easting', 360 * 120) # 360 degrees * 120 px/degree data = nc.getvar(root, 'dem', 'i2', ('northing','easting',)) files = glob.glob(localpath + "/all10/all10/*10g") tails = [[file_coordinates(f),f] for f in files] for t, f in tails: print t, f tmp = np.fromfile(f,np.int16) dynamic = tmp.shape[0]/10800 tmp = tmp.reshape(dynamic,10800) #misc.imsave(f+'.png',tmp) data[t[0]:t[0]+dynamic,t[1]:t[1]+10800] = tmp nc.close(root)
def rows2netcdf(rows, filename, index): root, is_new = nc.open(filename) if not is_new: measurements = nc.clonevar(root, 'globalradiation', 'measurements') slots = nc.getvar(root, 'slots') times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ] instant_radiation = rows2slots(rows,2) earth_failures = 0 i_e = 0 i_m = 0 while i_e < len(times) and i_m < len(instant_radiation): # When date estimated before date measured if times[i_e].date() < instant_radiation[i_m][1][0].date(): i_e += 1 # When date estimated after date measured elif times[i_e].date() > instant_radiation[i_m][1][0].date(): i_m += 1 else: if slots[i_e] < instant_radiation[i_m][0]: # TODO: This should be completed with a 0 error from the estimation. measurements[i_e, index,:] = np.array([0, 0]) earth_failures += 1 i_e += 1 elif slots[i_e] > instant_radiation[i_m][0]: i_m += 1 else: value = instant_radiation[i_m][1][1] #row_in_slot = instant_radiation[i_m][1][2] measurements[i_e, index,:] = np.array([value, value]) i_e += 1 i_m += 1 while i_e < len(times): # TODO: This should be completed with a 0 error from the estimation. measurements[i_e, index,:] = np.array([0, 0]) earth_failures += 1 i_e += 1 show("Detected %i of %i estimated times without earth measure.\n" % (earth_failures, len(slots))) error.rmse(root, index) nc.close(root)
def do(self, stream): from libs.sat.goes import calibration resultant_stream = stream.clone() for fs in stream.files.all(): f = fs.file if f.channel() == '01': sat = f.satellite() root, is_new = nc.open(f.completepath()) nc.getdim(root,'coefficient',1) var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4) var[0] = calibration.counts_shift.coefficient(sat) var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4) var[0] = calibration.space_measurement.coefficient(sat) var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4) var[0] = calibration.prelaunch.coefficient(sat)[0] var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4) dt = f.datetime() var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month) #data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement)) nc.close(root) fs.processed=True fs.save() return resultant_stream
def do(self, stream): from libs.sat.goes import calibration resultant_stream = stream.clone() for ms in stream.materials.all(): m = ms.material if hasattr(m, 'channel') and hasattr(m, 'satellite') and m.channel() == '01': sat = m.satellite() root = nc.open(m.completepath())[0] nc.getdim(root,'coefficient',1) var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4) var[0] = calibration.counts_shift.coefficient(sat) var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4) var[0] = calibration.space_measurement.coefficient(sat) var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4) var[0] = calibration.prelaunch.coefficient(sat)[0] var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4) dt = m.datetime() var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month) #data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement)) nc.close(root) ms.processed=True ms.save() return resultant_stream
def do_var(self, root, var_name, stream): count = 0 file_statuses = stream.sorted_files() shape = nc.getvar(root,'lat').shape for fs in file_statuses: # join the distributed content ch = fs.file.channel() v_ch = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4) v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',)) try: rootimg, n = nc.open(fs.file.completepath()) data = (nc.getvar(rootimg, 'data'))[:] # Force all the channels to the same shape if not (data.shape[1:3] == shape): print data.shape[1:3], shape data = matrix.adapt(data, shape) if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]: index = v_ch.shape[0] v_ch[index,:] = data v_ch_t[index] = calendar.timegm(fs.file.datetime().utctimetuple()) nc.close(rootimg) nc.sync(root) except RuntimeError, e: print fs.file.completepath()
show("BIAS:\t%.5f\t( %.5f %%)" % (bias, bias * ghi_ratio)) rmse = error.rmse_es(estimated, measured, s) show("RMSE:\t%.5f\t( %.5f %%)" % (rmse, rmse * ghi_ratio)) mae = error.mae(estimated, measured, s) show("MAE:\t%.5f\t( %.5f %%)" % (mae, mae * ghi_ratio)) show("----------\n") error.rmse(root, s) def workwith(year=2011, month=05, filename="goes13.all.BAND_02.nc"): show("=======================") show("Year: " , year) show("Month: " , month) show("Filename: ", filename) show("-----------------------\n") root = nc.open(filename)[0] lat = (nc.getvar(root, 'lat'))[:] lon = (nc.getvar(root, 'lon'))[:] data = calibrated_data(root) process_temporal_data(lat, lon, root) process_atmospheric_data(data, root) process_ground_albedo(lat, data, root) process_radiation(root) process_validate(root) #draw.getpng(draw.matrixtogrey(data[15]),'prueba.png') nc.close(root) show("Process finished.\n")
show(s['Name']) bias = diff.mean() show("BIAS:", bias, "(", bias * ghi_ratio, "%)") rmse = np.sqrt((diff**2).mean()) show("RMSE:", rmse, "(", rmse * ghi_ratio, "%)") mae = np.absolute(diff).mean() show("MAE:", mae, "(", mae * ghi_ratio, "%)") def workwith(year=2011, month=05, filename="goes13.all.BAND_02.nc"): show("=======================") show("Year: " , year) show("Month: " , month) show("Filename: ", filename) show("-----------------------\n") root, is_new = nc.open(filename) lat = (nc.getvar(root, 'lat'))[:] lon = (nc.getvar(root, 'lon'))[:] data = nc.getvar(root, 'data')[:] process_temporal_data(lat, lon, root) process_atmospheric_data(lat, lon, data, root) process_ground_albedo(lat, lon, data, root) process_radiation(lat, lon, data, root) #s = process_validate(year, month, root) #draw.getpng(draw.matrixtogrey(data[15]),'prueba.png') nc.close(root) show("Process finished.\n")
import glob from osgeo import gdal from libs.file import netcdf as nc files = glob.glob("libs/linke/tifs/*.tif") shape = gdal.Open(files[0]).ReadAsArray().shape root, n = nc.open("linketurbidity.nc") nc.getdim(root, "northing", shape[0]) nc.getdim(root, "easting", shape[1]) nc.getdim(root, "monthing") linke = nc.getvar(root, "linketurbidity", "f4", ("monthing", "northing", "easting",), 4) for i in range(len(files)): print i, '->', files[i] linke[i,:] = gdal.Open(files[i]).ReadAsArray() nc.close(root)
def diff(estimated, measured, station): return estimated[:,station,0] - measured[:,station,0] def ghi_mean(measured, station): return measured[:,station,0].mean() def ghi_ratio(measured, station): return 100 / ghi_mean(measured, station) def bias(estimated, measured, station): t_diff = diff(estimated, measured, station) return t_diff.mean() def rmse_es(estimated, measured, station): t_diff = diff(estimated, measured, station) return np.sqrt((t_diff**2).mean()) def mae(estimated, measured, station): t_diff = diff(estimated, measured, station) return np.absolute(t_diff).mean() filename = sys.argv[1] if len(sys.argv) == 2 else None if not filename == None: try: index = int(sys.argv[2]) root,n = nc.open(filename) rmse(root, index) nc.close(root) except Exception, e: show(e)