def initialize(radiance_filename, radiation_filename, callback=lambda r: r): ref, _ = nc.open(radiation_filename) ref_radiation = nc.getvar(ref, 'globalradiation') with nc.loader(radiance_filename) as radiance_root: radiance = nc.getvar(radiance_root, 'radiance', source=ref_radiation) radiance[0, :] = callback(radiance[0, :]) nc.close(ref)
def test_get_not_existing_dim_multiple_file(self): # check if get the dimension in a single file. root = nc.open('unittest0*.nc')[0] self.assertFalse(root.has_dimension('the_12th_dimension')) self.assertEquals(len(nc.getdim(root, 'the_12th_dimension', 123)), 5) self.assertTrue(root.has_dimension('the_12th_dimension')) nc.close(root)
def test_character_variables_in_single_file(self): # check if get and set the numpy string matrix in single files. root = nc.open('unittest00.nc')[0] var = nc.getvar(root, 'auditTrail') self.assertEquals(var.shape, (1, 2, 80)) self.assertEquals(var, self.auditTrail) self.auditTrail[:].data[0:6] = 'CHANGE' var[0, 0:6] = np.array(list('CHANGE')) self.assertEquals(var, self.auditTrail) nc.close(root)
def test_get_existing_var_multiple_file(self): # check if get the variable with multiples files. root = nc.open('unittest0*.nc')[0] self.assertNotIn('data', root.variables) var = nc.getvar(root, 'data') self.assertEquals(var.shape, (5, 100, 200)) self.assertIn('data', root.variables) are_equals = (var[:] == self.data) self.assertTrue(are_equals.all()) nc.close(root)
def test_get_existing_var_single_file(self): # check if get the variable in a single file. root = nc.open('unittest00.nc')[0] self.assertNotIn('data', root.variables) var = nc.getvar(root, 'data') self.assertEquals(var.shape, (1, 100, 200)) self.assertIn('data', root.variables) are_equals = (var[:] == np.zeros(var.shape) + 1.) self.assertTrue(are_equals.all()) nc.close(root)
def test_open_close_existent_file(self): # check if open an existent file. root, is_new = nc.open('unittest00.nc') self.assertEquals(root.files, ['unittest00.nc']) self.assertEquals(root.pattern, 'unittest00.nc') self.assertEquals(len(root.roots), 1) self.assertFalse(is_new) self.assertFalse(root.read_only) # check if close an existent file. nc.close(root) with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def test_open_close_using_with(self): # check if open the pattern selection using using a package instance. with nc.loader('unittest0*.nc') as root: self.assertEquals(root.files, ['unittest0%i.nc' % i for i in range(5)]) self.assertEquals(root.pattern, 'unittest0*.nc') self.assertEquals(len(root.roots), 5) self.assertFalse(root.is_new) self.assertFalse(root.read_only) # check if close the package with all the files. with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def test_multiple_file_var_operations(self): # check if get and set the numpy matrix. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'data') self.assertEquals(var.__class__, nc.DistributedNCVariable) self.assertEquals(var[:].__class__, np.ndarray) tmp = var[:] var[:] = var[:] + 1 nc.close(root) # check if value was saved into the file. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'data') self.assertTrue(var, tmp + 1) nc.close(root)
def test_get_non_existing_var_multiple_file(self): # check if get the variable with multiples files. root = nc.open('unittest0*.nc')[0] self.assertNotIn('new_variable', root.variables) var = nc.getvar(root, 'new_variable', 'f4', ('time', 'yc', 'xc'), digits=3, fill_value=1.2) self.assertEquals(var.shape, (5, 100, 200)) self.assertIn('new_variable', root.variables) ref = np.zeros(var.shape) + 1.2 # the comparison is true if the error is less than 0.002 are_equals = (var[:] - ref) < 0.002 self.assertTrue(are_equals.all()) nc.close(root)
def test_open_close_readonly_file(self): # set the file to be readonly. filename = 'ro_unittest.nc' if os.path.isfile(filename): os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) # check if create and open a new file. root, is_new = nc.open(filename) self.assertEquals(root.files, [filename]) self.assertEquals(root.pattern, filename) self.assertEquals(len(root.roots), 1) self.assertFalse(is_new) self.assertTrue(root.read_only) # check if close the readonly file. nc.close(root) with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def test_multiple_file_new_var_operations(self): # check if create a new var. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'new_variable', 'f4', ('time', 'yc', 'xc'), digits=3, fill_value=1.0) self.assertEquals(var.__class__, nc.DistributedNCVariable) self.assertEquals(var[:].__class__, np.ndarray) tmp = var[:] var[:] = var[:] + 1 nc.close(root) # check if value was saved into the files. root = nc.open('unittest00.nc')[0] var = nc.getvar(root, 'new_variable') self.assertTrue(var, tmp + 1) nc.close(root)
def test_open_close_new_file(self): # delete the filename from the system. filename = 'unittest-1.nc' if os.path.isfile(filename): os.remove(filename) # check if create and open a new file. root, is_new = nc.open(filename) self.assertEquals(root.files, ['unittest-1.nc']) self.assertEquals(root.pattern, 'unittest-1.nc') self.assertEquals(len(root.roots), 1) self.assertTrue(is_new) self.assertFalse(root.read_only) # check if close the created file. nc.close(root) with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def test_character_variables_in_multiple_file(self): # check if get and set the numpy string matrix in multiple files. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'auditTrail') self.assertEquals(var.shape, (5, 2, 80)) result = np.vstack([[self.auditTrail] for i in range(5)]) self.assertEquals(var, result) for i in range(5): result[i, i % 2].data[0:6] = 'CHANGE' var[i, i % 2, 0:6] = np.array(list('CHANGE')) self.assertEquals(var, result) nc.close(root) # check if was writed to each file. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'auditTrail') self.assertEquals(var, result) nc.close(root)
def interpolate_radiance(radiance_files, radiance_filename): before = search_closest(radiance_files, radiance_filename, lambda s: s - 1) after = search_closest(radiance_files, radiance_filename, lambda s: s + 1) extrems = filter(lambda x: x, [before, after]) if extrems: ref_filename = max(extrems) files = map(lambda e: rev_key[e], extrems) root, is_new = nc.open(files) radiation = nc.getvar(root, 'globalradiation') if len(extrems) > 1: radiation = np.average(radiation[:], axis=0, weights=calculate_weights(radiance_filename, files)) else: radiation = radiation[:].mean() initialize(radiance_filename, rev_key[ref_filename], lambda r: radiation * TO_MJRAD) nc.close(root)
def interpolate_radiance(radiance_files, radiance_filename): before = search_closest(radiance_files, radiance_filename, lambda s: s - 1) after = search_closest(radiance_files, radiance_filename, lambda s: s + 1) extrems = filter(lambda x: x, [before, after]) if extrems: ref_filename = max(extrems) files = map(lambda e: rev_key[e], extrems) root, is_new = nc.open(files) radiation = nc.getvar(root, 'globalradiation') if len(extrems) > 1: radiation = np.average(radiation[:], axis=0, weights=calculate_weights( radiance_filename, files)) else: radiation = radiation[:].mean() initialize(radiance_filename, rev_key[ref_filename], lambda r: radiation * TO_MJRAD) nc.close(root)
def test_open_close_file_with_readonly_restriction(self): # check the file is NOT read only. filename = 'unittest00.nc' can_write = os.access(filename, os.W_OK) self.assertTrue(can_write) # check if open an existent file. root, is_new = nc.open('unittest00.nc', read_only=True) self.assertEquals(root.files, ['unittest00.nc']) self.assertEquals(root.pattern, 'unittest00.nc') self.assertEquals(len(root.roots), 1) self.assertFalse(is_new) self.assertTrue(root.read_only) with self.assertRaisesRegexp(Exception, u'NetCDF: Write to read only'): var = nc.getvar(root, 'data') var[:] = 0 # check if close an existent file. nc.close(root) with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def dailyerrors(root, stations): times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [ t.date() for t in times ] days.sort() days_index = [d.day for d in set(days)] days_index.sort() days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4) RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4) BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4) error_diff[:] = np.zeros(estimated.shape) RMS_daily_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2])) BIAS_daily_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2])) nc.sync(root) for s in stations: index = stations.index(s) show('Station: %s \n' % stations[index]) error_diff[:, index, :] = measurements[:, index, :] - estimated[:, index, :] nc.sync(root) sum_value_in_day = np.zeros((days_amount)) for i in range(len(days)): d_i = days_index.index(days[i].day) if not measurements[i, index, 0] == 0.0: sum_value_in_day[d_i] += measurements[i,index ,0] RMS_daily_error[d_i, index ,:] += np.array([ error_diff[i, index ,0] ** 2,1]) BIAS_daily_error[d_i, index ,:] += error_diff[i, index ,0] count = RMS_daily_error[:, index, 1] count[count == 0] = 1 RMS_daily_error[:, index,0] = np.sqrt(RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 BIAS_daily_error[:, index,0] = (BIAS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 RMS_daily_error[:, index,1] = RMS_daily_error[:, index,0] BIAS_daily_error[:, index,1] = BIAS_daily_error[:, index,0] print 'RMS :', RMS_daily_error[:, index, 0] print 'BIAS', BIAS_daily_error[:, index, 0] print 'sum value in day: ', sum_value_in_day[:] show("\rDiary RMS daily error: %.2f\n" % (RMS_daily_error[:, index, 0]).mean()) nc.sync(root) nc.close(root)
def test_multiple_file_new_var_operations(self): # check if create a new var. root = nc.open('unittest0*.nc')[0] var = nc.getvar(root, 'new_variable', 'f4', ('time', 'yc', 'xc'), digits=3, fill_value=1.0) self.assertTrue((var[:] == 1.0).all()) self.assertEquals(var.__class__, nc.DistributedNCVariable) self.assertEquals(var[:].__class__, np.ndarray) tmp = var[:] var[:] = var[:] + 1 nc.close(root) # check if value was saved into the files. root = nc.open('unittest00.nc')[0] var = nc.getvar(root, 'new_variable') self.assertEquals(var, tmp + 1) nc.close(root)
def test_open_close_multiple_files_with_readonly_restriction(self): # check the files are NOT read only. filenames = map(lambda i: 'unittest0%i.nc' % i, range(5)) can_write = map(lambda f: os.access(f, os.W_OK), filenames) self.assertTrue(all(can_write)) # check if open the pattern selection using using a package instance. root, is_new = nc.open('unittest0*.nc', read_only=True) self.assertEquals(root.files, ['unittest0%i.nc' % i for i in range(5)]) self.assertEquals(root.pattern, 'unittest0*.nc') self.assertEquals(len(root.roots), 5) self.assertFalse(is_new) self.assertTrue(root.read_only) with self.assertRaisesRegexp(Exception, u'NetCDF: Write to read only'): var = nc.getvar(root, 'data') var[:] = 0 # check if close the package with all the files. nc.close(root) with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def rmse(root, index): times = [ datetime.utcfromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [ t.date() for t in times ] days.sort() days_index = [d.day for d in set(days)] days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4) error = nc.getvar(root, 'error', 'f4', ('time','yc_cut','xc_cut',), 4) diary_error = nc.getvar(root, 'diaryerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4) error_diff[:] = np.zeros(estimated.shape) error[:] = np.zeros(estimated.shape) diary_error[:] = np.zeros((days_amount, estimated.shape[1], estimated.shape[2])) nc.sync(root) #the_max = measurements[:].max() error_diff[:, index, :] = measurements[:,index,:] - estimated[:,index,:] error[:, index, :] = np.abs(error_diff[:, index, :]) nc.sync(root) max_value_in_day = np.zeros([days_amount]) + 1 for i in range(len(days)): d_i = days_index.index(days[i].day) max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i] diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1]) count = diary_error[:, index, 1] count[count == 0] = 1 diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count) diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100 show("\rDiary RMS error: %.2f" % (diary_error[:, index, 1]).mean()) for i in range(len(days)): d_i = days_index.index(days[i].day) error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100 result = np.sum(error[:, index, 1] ** 2) result = np.sqrt(result / error.shape[0]) show("Half-hour RMS error: %.2f \n" % result) #diary_error[:, index,1] = diary_error[:, index,0] nc.sync(root) nc.close(root)
def dailyerrors(root, stations): times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [t.date() for t in times] days.sort() days_index = [d.day for d in set(days)] days_index.sort() days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = nc.getvar(root, 'errordiff', 'f4', ( 'time', 'yc_cut', 'xc_cut', ), 4) RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', ( 'diarying', 'yc_cut', 'xc_cut', ), 4) BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', ( 'diarying', 'yc_cut', 'xc_cut', ), 4) error_diff[:] = np.zeros(estimated.shape) RMS_daily_error[:] = np.zeros( (days_amount, estimated.shape[1], estimated.shape[2])) BIAS_daily_error[:] = np.zeros( (days_amount, estimated.shape[1], estimated.shape[2])) nc.sync(root) for s in stations: index = stations.index(s) show('Station: %s \n' % stations[index]) error_diff[:, index, :] = measurements[:, index, :] - estimated[:, index, :] nc.sync(root) sum_value_in_day = np.zeros((days_amount)) for i in range(len(days)): d_i = days_index.index(days[i].day) if not measurements[i, index, 0] == 0.0: sum_value_in_day[d_i] += measurements[i, index, 0] RMS_daily_error[d_i, index, :] += np.array( [error_diff[i, index, 0]**2, 1]) BIAS_daily_error[d_i, index, :] += error_diff[i, index, 0] count = RMS_daily_error[:, index, 1] count[count == 0] = 1 RMS_daily_error[:, index, 0] = np.sqrt( RMS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 BIAS_daily_error[:, index, 0] = (BIAS_daily_error[:, index, 0] / count) / sum_value_in_day * 100 RMS_daily_error[:, index, 1] = RMS_daily_error[:, index, 0] BIAS_daily_error[:, index, 1] = BIAS_daily_error[:, index, 0] print 'RMS :', RMS_daily_error[:, index, 0] print 'BIAS', BIAS_daily_error[:, index, 0] print 'sum value in day: ', sum_value_in_day[:] show("\rDiary RMS daily error: %.2f\n" % (RMS_daily_error[:, index, 0]).mean()) nc.sync(root) nc.close(root)
def test_get_existing_dim_multiple_file(self): # check if get the dimension in a single file. root = nc.open('unittest0*.nc')[0] self.assertEquals(len(nc.getdim(root, 'time')), 5) nc.close(root)
def dump(self): nc.close(self.root)
def dump(self): for k in self._attrs.keys(): self._attrs.pop(k, None) nc.close(self.root)
max_value_in_day[d_i] = measurements[i,index,0] if max_value_in_day[d_i] < measurements[i,index,0] else max_value_in_day[d_i] diary_error[d_i, index,:] += np.array([ error_diff[i,index,0] ** 2,1]) count = diary_error[:, index, 1] count[count == 0] = 1 diary_error[:, index,0] = np.sqrt(diary_error[:, index, 0] / count) diary_error[:, index,1] = diary_error[:, index,0] / max_value_in_day * 100 show("\rDiary RMS error: {:.2f}".format(diary_error[:, index, 1].mean())) for i in range(len(days)): d_i = days_index.index(days[i].day) error[i,index,1] = error[i,index,1] / max_value_in_day[d_i] * 100 result = np.sum(error[:, index, 1] ** 2) result = np.sqrt(result / error.shape[0]) show("Half-hour RMS error: {:.2f} \n".format(result)) #diary_error[:, index,1] = diary_error[:, index,0] nc.sync(root) nc.close(root) def dailyerrors(root, stations): times = [ datetime.fromtimestamp(int(t)) for t in nc.getvar(root, 'time')[:] ] days = [ t.date() for t in times ] days.sort() days_index = [d.day for d in set(days)] days_index.sort() days_amount = len(days_index) nc.getdim(root, 'diarying', days_amount) nc.sync(root) measurements = nc.getvar(root, 'measurements') estimated = nc.getvar(root, 'globalradiation') error_diff = nc.getvar(root, 'errordiff', 'f4', ('time', 'yc_cut', 'xc_cut',), 4) RMS_daily_error = nc.getvar(root, 'RMSdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4) BIAS_daily_error = nc.getvar(root, 'BIASdailyerror', 'f4', ('diarying', 'yc_cut', 'xc_cut',), 4)