Exemple #1
0
def rmse(root, index):
    times = [
        datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:]
    ]
    days = [t.date() for t in times]
    days.sort()
    days_index = [d.day for d in set(days)]
    days_amount = len(days_index)
    nc.getdim(root, 'diarying', days_amount)
    nc.sync(root)
    measurements = nc.getvar(root, 'measurements')
    estimated = nc.getvar(root, 'globalradiation')
    error_diff = nc.getvar(root, 'errordiff', 'f4', (
        'time',
        'yc_cut',
        'xc_cut',
    ), 4)
    error = nc.getvar(root, 'error', 'f4', (
        'time',
        'yc_cut',
        'xc_cut',
    ), 4)
    diary_error = nc.getvar(root, 'diaryerror', 'f4', (
        'diarying',
        'yc_cut',
        'xc_cut',
    ), 4)
    error_diff[:] = np.zeros(estimated.shape)
    error[:] = np.zeros(estimated.shape)
    diary_error[:] = np.zeros(
        (days_amount, estimated.shape[1], estimated.shape[2]))
    nc.sync(root)
    #the_max = measurements[:].max()
    error_diff[:,
               index, :] = measurements[:, index, :] - estimated[:, index, :]
    error[:, index, :] = np.abs(error_diff[:, index, :])
    nc.sync(root)
    max_value_in_day = np.zeros([days_amount]) + 1
    for i in range(len(days)):
        d_i = days_index.index(days[i].day)
        max_value_in_day[d_i] = measurements[
            i, index, 0] if max_value_in_day[d_i] < measurements[
                i, index, 0] else max_value_in_day[d_i]
        diary_error[d_i, index, :] += np.array([error_diff[i, index, 0]**2, 1])
    count = diary_error[:, index, 1]
    count[count == 0] = 1
    diary_error[:, index, 0] = np.sqrt(diary_error[:, index, 0] / count)
    diary_error[:, index,
                1] = diary_error[:, index, 0] / max_value_in_day * 100
    show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean())
    for i in range(len(days)):
        d_i = days_index.index(days[i].day)
        error[i, index, 1] = error[i, index, 1] / max_value_in_day[d_i] * 100
    result = np.sum(error[:, index, 1]**2)
    result = np.sqrt(result / error.shape[0])
    show("Half-hour RMS error: %.2f \n" % result)
    #diary_error[:, index,1] = diary_error[:, index,0]
    nc.sync(root)
    nc.close(root)
def cut_projected(root):
	lat = nc.getvar(root, 'lat')
	lon = nc.getvar(root, 'lon')
	time = nc.getvar(root, 'data_time')
	months = list(set([ (datetime.fromtimestamp(int(t))).month for t in time ]))
	nc.getdim(root, 'monthing')
	months_cut = nc.getvar(root, 'months', 'i2', ('monthing',))
	dims = list(lat.dimensions)
	dims.insert(0, 'monthing')
	linke = nc.getvar(root, 'linketurbidity', 'f4', tuple(dims),4)
	linke_x, linke_y = project_coordinates(lat[:], lon[:])
	months_cut[:] = np.array(list(months))
	for i in range(len(months)):
		linke[i] = cut_month(linke_x, linke_y, months[i])
	return linke[:], linke_x, linke_y
def rows2netcdf(rows, filename, index):
	root, is_new = nc.open(filename)
	if not is_new:
		measurements = nc.getvar(root, 'measurements', 'f4', ('timing','northing','easting'), 4)
		slots = nc.getvar(root, 'slots')
		times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time') ]
		instant_radiation = rows2slots(rows,2)
		i_e = 0
		i_m = 0
		while i_e < len(times) and i_m < len(instant_radiation):
			#print i_e,"/", len(times) -1, " | ", i_m, "/", len(instant_radiation) - 1
			# When date estimated before date measured
			if times[i_e].date() < instant_radiation[i_m][1][0].date():
				i_e += 1
			# When date estimated after date measured
			elif times[i_e].date() > instant_radiation[i_m][1][0].date():
				i_m += 1
			else:
				if slots[i_e] < instant_radiation[i_m][0]:
					measurements[i_e, index,:] = np.array([-1, -1])
					i_e += 1
				elif slots[i_e] > instant_radiation[i_m][0]:
					i_m += 1
				else:
					value = instant_radiation[i_m][1][1]
					row_in_slot = instant_radiation[i_m][1][2]
					measurements[i_e, index,:] = np.array([value, value])
					i_e += 1
					i_m += 1
	days = [ t.date() for t in times ]
	nc.getdim(root, 'diarying')
	diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing', 'easting'))
	estimated = nc.getvar(root, 'globalradiation')[:]
	error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing','easting'), 4)
	error = nc.getvar(root, 'error', 'f4', ('timing','northing','easting'), 4)
	error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
	error[:, index, :] = np.sqrt((error_diff[:, index, :])**2)
	print error_diff.shape
	nc.sync(root)
	for	i in range(len(days)):
		d = days[i].day
		print index, i, d, error_diff[i,index], diary_error[d, index]
		diary_error[d, index,:] += error_diff[i,index,:]
	diary_error[:, index,:] = np.sqrt((diary_error[:, index,:])**2)
	nc.close(root)
def merge_tails():
	import glob
	import numpy as np
	#from scipy import misc
	say("Merging all the tails in one netCDF file...")
	root, is_new = nc.open(localpath+ "/dem.nc")
	n_d = nc.getdim(root,'northing', 180 * 120) # 180 degrees * 120 px/degree
	n_e = nc.getdim(root,'easting', 360 * 120) # 360 degrees * 120 px/degree
	data = nc.getvar(root, 'dem', 'i2', ('northing','easting',))
	files = glob.glob(localpath + "/all10/all10/*10g")
	tails = [[file_coordinates(f),f] for f in files]
	for t, f in tails:
		print t, f
		tmp = np.fromfile(f,np.int16)
		dynamic = tmp.shape[0]/10800
		tmp = tmp.reshape(dynamic,10800)
		#misc.imsave(f+'.png',tmp)
		data[t[0]:t[0]+dynamic,t[1]:t[1]+10800] = tmp
	nc.close(root)
	def do_file(self, filename, data):
		print "Creating ", filename
		# create compact file and initialize basic settings
		root, is_new = nc.open(filename + self.extension) # The filename does not contain the extension
		if is_new:
			sample, n = nc.open(data[(data.keys())[0].completepath()])
			shape = sample.variables['data'].shape
			nc.getdim(root,'northing', shape[1])
			nc.getdim(root,'easting', shape[2])
			nc.getdim(root,'timing')
			v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
			v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
			v_lon[:] = nc.getvar(sample, 'lon')[:]
			v_lat[:] = nc.getvar(sample, 'lat')[:]
			nc.close(sample)
		if isinstance(data,collections.Mapping):
			for d in data:
				self.do_var(root, d, data[d])
		else:
			self.create_var(root, 'data', data)
		# save the content inside the compact file
		if not root is None: nc.close(root)
		f = File(localname=localname, downloaded=True, begin_download=begin_time, end_download=datetime.now())
		f.save()
		return f
def rmse(root, index):
	times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'data_time')[:] ]
	days = [ t.date() for t in times ]
	days.sort()
	days_index = [d.day for d in set(days)]
	days_amount = len(days_index)
	nc.getdim(root, 'diarying', days_amount)
	nc.sync(root)
	measurements = nc.getvar(root, 'measurements')
	estimated = nc.getvar(root, 'globalradiation')
	error_diff = nc.getvar(root, 'errordiff', 'f4', ('timing','northing_cut','easting_cut',), 4)
	error = nc.getvar(root, 'error', 'f4', ('timing','northing_cut','easting_cut',), 4)
	diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'northing_cut', 'easting_cut',), 4)
	error_diff[:] = np.zeros(estimated.shape)
	error[:] = np.zeros(estimated.shape)
	diary_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2]))
	nc.sync(root)
	#the_max = measurements[:].max()
	error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
	error[:, index, :] = np.abs(error_diff[:, index, :])
	nc.sync(root)
	max_value_in_day = np.zeros([days_amount]) + 1
	for i in range(len(days)):
		d_i = days_index.index(days[i].day)
		max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i]
		diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1])
	count = diary_error[:, index, 1]
	count[count == 0] = 1
	diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count)
	diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100
	show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean())
	for i in range(len(days)):
		d_i = days_index.index(days[i].day)
		error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100
	result = np.sum(error[:, index, 1] ** 2)
	result = np.sqrt(result / error.shape[0])
	show("Half-hour RMS error: %.2f \n" % result)
	#diary_error[:, index,1] = diary_error[:, index,0]
	nc.sync(root)
def cut_projected(root):
    lat = nc.getvar(root, "lat")
    lon = nc.getvar(root, "lon")
    time = nc.getvar(root, "time")
    months = list(set([(datetime.fromtimestamp(int(t))).month for t in time]))
    months_d = nc.getdim(root, "monthing")
    months_cut = nc.getvar(root, "months", "i2", ("monthing",))
    linke = nc.getvar(root, "linketurbidity", "f4", ("monthing", "northing", "easting"), 4)
    linke_x, linke_y = project_coordinates(lat[:], lon[:])
    months_cut[:] = np.array(list(months))
    for i in range(len(months)):
        linke[i] = cut_month(linke_x, linke_y, months[i])
    return linke[:], linke_x, linke_y
	def do(self, stream):
		from libs.sat.goes import calibration
		resultant_stream = stream.clone()
		for fs in stream.files.all():
			f = fs.file
			if f.channel() == '01':
				sat = f.satellite()
				root, is_new = nc.open(f.completepath())
				nc.getdim(root,'coefficient',1)
				var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4)
				var[0] = calibration.counts_shift.coefficient(sat)
				var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4)
				var[0] = calibration.space_measurement.coefficient(sat)
				var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4)
				var[0] = calibration.prelaunch.coefficient(sat)[0]
				var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4)
				dt = f.datetime()
				var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month)
				#data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement))
				nc.close(root)
			fs.processed=True
			fs.save()
		return resultant_stream
	def do(self, stream):
		from libs.sat.goes import calibration
		resultant_stream = stream.clone()
		for ms in stream.materials.all():
			m = ms.material
			if hasattr(m, 'channel') and hasattr(m, 'satellite') and m.channel() == '01':
				sat = m.satellite()
				root = nc.open(m.completepath())[0]
				nc.getdim(root,'coefficient',1)
				var = nc.getvar(root, 'counts_shift', 'f4', ('coefficient',), 4)
				var[0] = calibration.counts_shift.coefficient(sat)
				var = nc.getvar(root, 'space_measurement', 'f4', ('coefficient',), 4)
				var[0] = calibration.space_measurement.coefficient(sat)
				var = nc.getvar(root, 'prelaunch', 'f4', ('coefficient',), 4)
				var[0] = calibration.prelaunch.coefficient(sat)[0]
				var = nc.getvar(root, 'postlaunch', 'f4', ('coefficient',), 4)
				dt = m.datetime()
				var[0] = calibration.postlaunch.coefficient(sat, dt.year, dt.month)
				#data = np.float32(self.calibrated_coefficient) * ((data / np.float32(self.counts_shift)) - np.float32(self.space_measurement))
				nc.close(root)
			ms.processed=True
			ms.save()
		return resultant_stream
	def do_file(self, filename, stream):
		# create compact file and initialize basic settings
		begin_time = self.getdatetimenow()
		root, is_new = nc.open(filename)
		if is_new:
			sample, n = nc.open(stream.files.all()[0].file.completepath())
			shape = sample.variables['data'].shape
			nc.getdim(root,'northing', shape[1])
			nc.getdim(root,'easting', shape[2])
			nc.getdim(root,'timing')
			v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
			v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
			v_lon[:] = nc.getvar(sample, 'lon')[:]
			v_lat[:] = nc.getvar(sample, 'lat')[:]
			nc.close(sample)
			nc.sync(root)
		self.do_var(root, 'data', stream)
		# save the content inside the compact file
		if not root is None: nc.close(root)
		f = File(localname=filename)
		f.save()
		return f
Exemple #11
0
def dailyerrors(root, stations):
    times = [
        datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:]
    ]
    days = [t.date() for t in times]
    days.sort()
    days_index = [d.day for d in set(days)]
    days_index.sort()
    days_amount = len(days_index)
    nc.getdim(root, 'diarying', days_amount)
    nc.sync(root)
    measurements = nc.getvar(root, 'measurements')
    estimated = nc.getvar(root, 'globalradiation')
    error_diff = root.getvar('errordiff', 'f4', (
        'time',
        'yc_cut',
        'xc_cut',
    ), 4)
    RMS_daily_error = root.getvar('RMSdailyerror', 'f4', (
        'diarying',
        'yc_cut',
        'xc_cut',
    ), 4)
    BIAS_daily_error = root.getvar('BIASdailyerror', 'f4', (
        'diarying',
        'yc_cut',
        'xc_cut',
    ), 4)
    error_diff[:] = np.zeros(estimated.shape)
    RMS_daily_error[:] = np.zeros(
        (days_amount, estimated.shape[1], estimated.shape[2]))
    BIAS_daily_error[:] = np.zeros(
        (days_amount, estimated.shape[1], estimated.shape[2]))
    nc.sync(root)
    for s in stations:
        index = stations.index(s)
        show('Station: %s \n' % stations[index])
        error_diff[:,
                   index, :] = measurements[:, index, :] - estimated[:,
                                                                     index, :]
        nc.sync(root)
        sum_value_in_day = np.zeros((days_amount))
        for i in range(len(days)):
            d_i = days_index.index(days[i].day)
            if not measurements[i, index, 0] == 0.0:
                sum_value_in_day[d_i] += measurements[i, index, 0]
                RMS_daily_error[d_i, index, :] += np.array(
                    [error_diff[i, index, 0]**2, 1])
                BIAS_daily_error[d_i, index, :] += error_diff[i, index, 0]
        count = RMS_daily_error[:, index, 1]
        count[count == 0] = 1
        RMS_daily_error[:, index, 0] = np.sqrt(
            RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100
        BIAS_daily_error[:, index, 0] = (BIAS_daily_error[:, index, 0] /
                                         count) / sum_value_in_day * 100
        RMS_daily_error[:, index, 1] = RMS_daily_error[:, index, 0]
        BIAS_daily_error[:, index, 1] = BIAS_daily_error[:, index, 0]
        print 'RMS :', RMS_daily_error[:, index, 0]
        print 'BIAS', BIAS_daily_error[:, index, 0]
        print 'sum value in day: ', sum_value_in_day[:]
        show("\rDiary RMS daily error: %.2f\n" %
             (RMS_daily_error[:, index, 0]).mean())

    nc.sync(root)
    nc.close(root)
import glob
from osgeo import gdal
from libs.file import netcdf as nc

files = glob.glob("libs/linke/tifs/*.tif")
shape = gdal.Open(files[0]).ReadAsArray().shape

root, n = nc.open("linketurbidity.nc")
nc.getdim(root, "northing", shape[0])
nc.getdim(root, "easting", shape[1])
nc.getdim(root, "monthing")
linke = nc.getvar(root, "linketurbidity", "f4", ("monthing", "northing", "easting",), 4)

for i in range(len(files)):
	print i, '->', files[i]
	linke[i,:] = gdal.Open(files[i]).ReadAsArray()

nc.close(root)