def initialize(radiance_filename, radiation_filename, callback=lambda r: r):
ref, _ = nc.open(radiation_filename)
ref_radiation = nc.getvar(ref, 'globalradiation')
with nc.loader(radiance_filename) as radiance_root:
radiance = nc.getvar(radiance_root, 'radiance', source=ref_radiation)
radiance[0, :] = callback(radiance[0, :])
nc.close(ref)
def test_get_not_existing_dim_multiple_file(self):
# check if get the dimension in a single file.
root = nc.open('unittest0*.nc')[0]
self.assertFalse(root.has_dimension('the_12th_dimension'))
self.assertEquals(len(nc.getdim(root, 'the_12th_dimension', 123)), 5)
self.assertTrue(root.has_dimension('the_12th_dimension'))
nc.close(root)
def test_character_variables_in_single_file(self):
# check if get and set the numpy string matrix in single files.
root = nc.open('unittest00.nc')[0]
var = nc.getvar(root, 'auditTrail')
self.assertEquals(var.shape, (1, 2, 80))
self.assertEquals(var, self.auditTrail)
self.auditTrail[:].data[0:6] = 'CHANGE'
var[0, 0:6] = np.array(list('CHANGE'))
self.assertEquals(var, self.auditTrail)
nc.close(root)
def test_get_existing_var_multiple_file(self):
# check if get the variable with multiples files.
root = nc.open('unittest0*.nc')[0]
self.assertNotIn('data', root.variables)
var = nc.getvar(root, 'data')
self.assertEquals(var.shape, (5, 100, 200))
self.assertIn('data', root.variables)
are_equals = (var[:] == self.data)
self.assertTrue(are_equals.all())
nc.close(root)
def test_get_existing_var_single_file(self):
# check if get the variable in a single file.
root = nc.open('unittest00.nc')[0]
self.assertNotIn('data', root.variables)
var = nc.getvar(root, 'data')
self.assertEquals(var.shape, (1, 100, 200))
self.assertIn('data', root.variables)
are_equals = (var[:] == np.zeros(var.shape) + 1.)
self.assertTrue(are_equals.all())
nc.close(root)
def test_open_close_existent_file(self):
# check if open an existent file.
root, is_new = nc.open('unittest00.nc')
self.assertEquals(root.files, ['unittest00.nc'])
self.assertEquals(root.pattern, 'unittest00.nc')
self.assertEquals(len(root.roots), 1)
self.assertFalse(is_new)
self.assertFalse(root.read_only)
# check if close an existent file.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_open_close_using_with(self):
# check if open the pattern selection using using a package instance.
with nc.loader('unittest0*.nc') as root:
self.assertEquals(root.files,
['unittest0%i.nc' % i for i in range(5)])
self.assertEquals(root.pattern, 'unittest0*.nc')
self.assertEquals(len(root.roots), 5)
self.assertFalse(root.is_new)
self.assertFalse(root.read_only)
# check if close the package with all the files.
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_multiple_file_var_operations(self):
# check if get and set the numpy matrix.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root, 'data')
self.assertEquals(var.__class__, nc.DistributedNCVariable)
self.assertEquals(var[:].__class__, np.ndarray)
tmp = var[:]
var[:] = var[:] + 1
nc.close(root)
# check if value was saved into the file.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root, 'data')
self.assertTrue(var, tmp + 1)
nc.close(root)
def test_get_non_existing_var_multiple_file(self):
# check if get the variable with multiples files.
root = nc.open('unittest0*.nc')[0]
self.assertNotIn('new_variable', root.variables)
var = nc.getvar(root, 'new_variable',
'f4', ('time', 'yc', 'xc'),
digits=3, fill_value=1.2)
self.assertEquals(var.shape, (5, 100, 200))
self.assertIn('new_variable', root.variables)
ref = np.zeros(var.shape) + 1.2
# the comparison is true if the error is less than 0.002
are_equals = (var[:] - ref) < 0.002
self.assertTrue(are_equals.all())
nc.close(root)
def test_open_close_readonly_file(self):
# set the file to be readonly.
filename = 'ro_unittest.nc'
if os.path.isfile(filename):
os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
# check if create and open a new file.
root, is_new = nc.open(filename)
self.assertEquals(root.files, [filename])
self.assertEquals(root.pattern, filename)
self.assertEquals(len(root.roots), 1)
self.assertFalse(is_new)
self.assertTrue(root.read_only)
# check if close the readonly file.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_multiple_file_new_var_operations(self):
# check if create a new var.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root, 'new_variable',
'f4', ('time', 'yc', 'xc'),
digits=3, fill_value=1.0)
self.assertEquals(var.__class__, nc.DistributedNCVariable)
self.assertEquals(var[:].__class__, np.ndarray)
tmp = var[:]
var[:] = var[:] + 1
nc.close(root)
# check if value was saved into the files.
root = nc.open('unittest00.nc')[0]
var = nc.getvar(root, 'new_variable')
self.assertTrue(var, tmp + 1)
nc.close(root)
def test_open_close_new_file(self):
# delete the filename from the system.
filename = 'unittest-1.nc'
if os.path.isfile(filename):
os.remove(filename)
# check if create and open a new file.
root, is_new = nc.open(filename)
self.assertEquals(root.files, ['unittest-1.nc'])
self.assertEquals(root.pattern, 'unittest-1.nc')
self.assertEquals(len(root.roots), 1)
self.assertTrue(is_new)
self.assertFalse(root.read_only)
# check if close the created file.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_character_variables_in_multiple_file(self):
# check if get and set the numpy string matrix in multiple files.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root, 'auditTrail')
self.assertEquals(var.shape, (5, 2, 80))
result = np.vstack([[self.auditTrail] for i in range(5)])
self.assertEquals(var, result)
for i in range(5):
result[i, i % 2].data[0:6] = 'CHANGE'
var[i, i % 2, 0:6] = np.array(list('CHANGE'))
self.assertEquals(var, result)
nc.close(root)
# check if was writed to each file.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root, 'auditTrail')
self.assertEquals(var, result)
nc.close(root)
def interpolate_radiance(radiance_files, radiance_filename):
before = search_closest(radiance_files, radiance_filename, lambda s: s - 1)
after = search_closest(radiance_files, radiance_filename, lambda s: s + 1)
extrems = filter(lambda x: x, [before, after])
if extrems:
ref_filename = max(extrems)
files = map(lambda e: rev_key[e], extrems)
root, is_new = nc.open(files)
radiation = nc.getvar(root, 'globalradiation')
if len(extrems) > 1:
radiation = np.average(radiation[:], axis=0,
weights=calculate_weights(radiance_filename,
files))
else:
radiation = radiation[:].mean()
initialize(radiance_filename, rev_key[ref_filename],
lambda r: radiation * TO_MJRAD)
nc.close(root)
def interpolate_radiance(radiance_files, radiance_filename):
before = search_closest(radiance_files, radiance_filename, lambda s: s - 1)
after = search_closest(radiance_files, radiance_filename, lambda s: s + 1)
extrems = filter(lambda x: x, [before, after])
if extrems:
ref_filename = max(extrems)
files = map(lambda e: rev_key[e], extrems)
root, is_new = nc.open(files)
radiation = nc.getvar(root, 'globalradiation')
if len(extrems) > 1:
radiation = np.average(radiation[:],
axis=0,
weights=calculate_weights(
radiance_filename, files))
else:
radiation = radiation[:].mean()
initialize(radiance_filename, rev_key[ref_filename],
lambda r: radiation * TO_MJRAD)
nc.close(root)
def test_open_close_file_with_readonly_restriction(self):
# check the file is NOT read only.
filename = 'unittest00.nc'
can_write = os.access(filename, os.W_OK)
self.assertTrue(can_write)
# check if open an existent file.
root, is_new = nc.open('unittest00.nc', read_only=True)
self.assertEquals(root.files, ['unittest00.nc'])
self.assertEquals(root.pattern, 'unittest00.nc')
self.assertEquals(len(root.roots), 1)
self.assertFalse(is_new)
self.assertTrue(root.read_only)
with self.assertRaisesRegexp(Exception, u'NetCDF: Write to read only'):
var = nc.getvar(root, 'data')
var[:] = 0
# check if close an existent file.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def dailyerrors(root, stations):
times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ]
days = [ t.date() for t in times ]
days.sort()
days_index = [d.day for d in set(days)]
days_index.sort()
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4)
RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)
BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)
error_diff[:] = np.zeros(estimated.shape)
RMS_daily_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2]))
BIAS_daily_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
for s in stations:
index = stations.index(s)
show('Station: %s \n' % stations[index])
error_diff[:, index, :] = measurements[:, index, :] - estimated[:, index, :]
nc.sync(root)
sum_value_in_day = np.zeros((days_amount))
for i in range(len(days)):
d_i = days_index.index(days[i].day)
if not measurements[i, index, 0] == 0.0:
sum_value_in_day[d_i] += measurements[i,index ,0]
RMS_daily_error[d_i, index ,:] += np.array([ error_diff[i, index ,0] ** 2,1])
BIAS_daily_error[d_i, index ,:] += error_diff[i, index ,0]
count = RMS_daily_error[:, index, 1]
count[count == 0] = 1
RMS_daily_error[:, index,0] = np.sqrt(RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100
BIAS_daily_error[:, index,0] = (BIAS_daily_error[:, index, 0] / count) / sum_value_in_day * 100
RMS_daily_error[:, index,1] = RMS_daily_error[:, index,0]
BIAS_daily_error[:, index,1] = BIAS_daily_error[:, index,0]
print 'RMS :', RMS_daily_error[:, index, 0]
print 'BIAS', BIAS_daily_error[:, index, 0]
print 'sum value in day: ', sum_value_in_day[:]
show("\rDiary RMS daily error: %.2f\n" % (RMS_daily_error[:, index, 0]).mean())
nc.sync(root)
nc.close(root)
def test_multiple_file_new_var_operations(self):
# check if create a new var.
root = nc.open('unittest0*.nc')[0]
var = nc.getvar(root,
'new_variable',
'f4', ('time', 'yc', 'xc'),
digits=3,
fill_value=1.0)
self.assertTrue((var[:] == 1.0).all())
self.assertEquals(var.__class__, nc.DistributedNCVariable)
self.assertEquals(var[:].__class__, np.ndarray)
tmp = var[:]
var[:] = var[:] + 1
nc.close(root)
# check if value was saved into the files.
root = nc.open('unittest00.nc')[0]
var = nc.getvar(root, 'new_variable')
self.assertEquals(var, tmp + 1)
nc.close(root)
def test_open_close_multiple_files_with_readonly_restriction(self):
# check the files are NOT read only.
filenames = map(lambda i: 'unittest0%i.nc' % i, range(5))
can_write = map(lambda f: os.access(f, os.W_OK), filenames)
self.assertTrue(all(can_write))
# check if open the pattern selection using using a package instance.
root, is_new = nc.open('unittest0*.nc', read_only=True)
self.assertEquals(root.files, ['unittest0%i.nc' % i for i in range(5)])
self.assertEquals(root.pattern, 'unittest0*.nc')
self.assertEquals(len(root.roots), 5)
self.assertFalse(is_new)
self.assertTrue(root.read_only)
with self.assertRaisesRegexp(Exception, u'NetCDF: Write to read only'):
var = nc.getvar(root, 'data')
var[:] = 0
# check if close the package with all the files.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_open_close_file_with_readonly_restriction(self):
# check the file is NOT read only.
filename = 'unittest00.nc'
can_write = os.access(filename, os.W_OK)
self.assertTrue(can_write)
# check if open an existent file.
root, is_new = nc.open('unittest00.nc', read_only=True)
self.assertEquals(root.files, ['unittest00.nc'])
self.assertEquals(root.pattern, 'unittest00.nc')
self.assertEquals(len(root.roots), 1)
self.assertFalse(is_new)
self.assertTrue(root.read_only)
with self.assertRaisesRegexp(Exception, u'NetCDF: Write to read only'):
var = nc.getvar(root, 'data')
var[:] = 0
# check if close an existent file.
nc.close(root)
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def rmse(root, index):
times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ]
days = [ t.date() for t in times ]
days.sort()
days_index = [d.day for d in set(days)]
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4)
error = nc.getvar(root, 'error', 'f4', ('time','yc_cut','xc_cut',), 4)
diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)
error_diff[:] = np.zeros(estimated.shape)
error[:] = np.zeros(estimated.shape)
diary_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
#the_max = measurements[:].max()
error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:]
error[:, index, :] = np.abs(error_diff[:, index, :])
nc.sync(root)
max_value_in_day = np.zeros([days_amount]) + 1
for i in range(len(days)):
d_i = days_index.index(days[i].day)
max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i]
diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1])
count = diary_error[:, index, 1]
count[count == 0] = 1
diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count)
diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100
show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean())
for i in range(len(days)):
d_i = days_index.index(days[i].day)
error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100
result = np.sum(error[:, index, 1] ** 2)
result = np.sqrt(result / error.shape[0])
show("Half-hour RMS error: %.2f \n" % result)
#diary_error[:, index,1] = diary_error[:, index,0]
nc.sync(root)
nc.close(root)
def dailyerrors(root, stations):
times = [
datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:]
]
days = [t.date() for t in times]
days.sort()
days_index = [d.day for d in set(days)]
days_index.sort()
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', (
'time',
'yc_cut',
'xc_cut',
), 4)
RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', (
'diarying',
'yc_cut',
'xc_cut',
), 4)
BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', (
'diarying',
'yc_cut',
'xc_cut',
), 4)
error_diff[:] = np.zeros(estimated.shape)
RMS_daily_error[:] = np.zeros(
(days_amount, estimated.shape[1], estimated.shape[2]))
BIAS_daily_error[:] = np.zeros(
(days_amount, estimated.shape[1], estimated.shape[2]))
nc.sync(root)
for s in stations:
index = stations.index(s)
show('Station: %s \n' % stations[index])
error_diff[:,
index, :] = measurements[:, index, :] - estimated[:,
index, :]
nc.sync(root)
sum_value_in_day = np.zeros((days_amount))
for i in range(len(days)):
d_i = days_index.index(days[i].day)
if not measurements[i, index, 0] == 0.0:
sum_value_in_day[d_i] += measurements[i, index, 0]
RMS_daily_error[d_i, index, :] += np.array(
[error_diff[i, index, 0]**2, 1])
BIAS_daily_error[d_i, index, :] += error_diff[i, index, 0]
count = RMS_daily_error[:, index, 1]
count[count == 0] = 1
RMS_daily_error[:, index, 0] = np.sqrt(
RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100
BIAS_daily_error[:, index, 0] = (BIAS_daily_error[:, index, 0] /
count) / sum_value_in_day * 100
RMS_daily_error[:, index, 1] = RMS_daily_error[:, index, 0]
BIAS_daily_error[:, index, 1] = BIAS_daily_error[:, index, 0]
print 'RMS :', RMS_daily_error[:, index, 0]
print 'BIAS', BIAS_daily_error[:, index, 0]
print 'sum value in day: ', sum_value_in_day[:]
show("\rDiary RMS daily error: %.2f\n" %
(RMS_daily_error[:, index, 0]).mean())
nc.sync(root)
nc.close(root)
def test_get_existing_dim_multiple_file(self):
# check if get the dimension in a single file.
root = nc.open('unittest0*.nc')[0]
self.assertEquals(len(nc.getdim(root, 'time')), 5)
nc.close(root)
def dump(self):
nc.close(self.root)
def dump(self):
nc.close(self.root)
def dump(self):
for k in self._attrs.keys():
self._attrs.pop(k, None)
nc.close(self.root)
def test_get_existing_dim_multiple_file(self):
# check if get the dimension in a single file.
root = nc.open('unittest0*.nc')[0]
self.assertEquals(len(nc.getdim(root, 'time')), 5)
nc.close(root)
max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i]
diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1])
count = diary_error[:, index, 1]
count[count == 0] = 1
diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count)
diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100
show("\rDiary RMS error: {:.2f}".format(diary_error[:, index, 1].mean()))
for i in range(len(days)):
d_i = days_index.index(days[i].day)
error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100
result = np.sum(error[:, index, 1] ** 2)
result = np.sqrt(result / error.shape[0])
show("Half-hour RMS error: {:.2f} \n".format(result))
#diary_error[:, index,1] = diary_error[:, index,0]
nc.sync(root)
nc.close(root)
def dailyerrors(root, stations):
times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ]
days = [ t.date() for t in times ]
days.sort()
days_index = [d.day for d in set(days)]
days_index.sort()
days_amount = len(days_index)
nc.getdim(root, 'diarying', days_amount)
nc.sync(root)
measurements = nc.getvar(root, 'measurements')
estimated = nc.getvar(root, 'globalradiation')
error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4)
RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)
BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)
