def test_compare_column_and_integrated_profile(self):
from CALIOPy.utils import integrate_profile, mask_data
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532",
"Caliop_V4_QC_NO_PRESSURE")
col_data = cis.read_data(
self.TEST_FILENAME,
"Column_Optical_Depth_Tropospheric_Aerosols_532", "Caliop_V4_QC")
backscatter = cis.read_data(self.TEST_FILENAME,
"Total_Backscatter_Coefficient_532",
"Caliop_V4_QC_NO_PRESSURE")
int_backscatter = cis.read_data(
self.TEST_FILENAME, "Column_Integrated_Attenuated_Backscatter_532",
"Caliop_V4_QC")
# Integrate the profile
int_data = integrate_profile(ext_data)
my_int_back = integrate_profile(backscatter)
assert np.abs((my_int_back.data - int_backscatter.data[:, 0]
).mean()) < 0.016, "Backscatter integrations differ"
print(np.abs((my_int_back.data - int_backscatter.data[:, 0]).mean()))
#-0.0154302524737
# There are a *few* values which are quite different, but I really don't know why, and they also have fairly
# large uncertainty. On average the differences are very small.
print(np.abs(np.mean(int_data.data - col_data.data[:, 0])))
assert np.abs(np.mean(int_data.data - col_data.data[:, 0])
) < 0.015, 'Extinction integrations are different'
assert np.array_equal(
int_data.data.mask,
col_data.data[:, 0].mask), 'Arrays have different masks'
def compare_column_and_integrated_profile(self):
from CALIOPy.utils import integrate_profile
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532")
col_data = cis.read_data(self.TEST_FILENAME,
"Column_Optical_Depth_Aerosols_532")
int_data = integrate_profile(ext_data)
assert_almost_equal(int_data.data, col_data.data, delta=0.05)
def setUpClass(cls):
# Read this in separately so that the post-processing doesn't get called when the second dataset is appended
# to the UngriddedDataList (via the .coords() call...)
cls.extinction_qc = cis.read_data(cls.TEST_FILENAME,
"Extinction_QC_Flag_532",
"Caliop_V4")
cls.cad_score = cis.read_data(cls.TEST_FILENAME, "CAD_Score",
"Caliop_V4")
def get_clean_clear_sky_toa_net_sw(infile, product=None):
"""
This is $F_{sw,clear}^{\top,0}$ as defined in A.7.1 of ECHAM6_userguide.pdf
:param infile: An ECHAM-HAM echam filepath
:param product:
:return:
"""
forcing_file = get_stream_file(infile, 'forcing')
clean_clear_sky_toa_sw_forcing = cis.read_data(infile, 'sraf0', product) - \
cis.read_data(forcing_file, 'FSW_CLEAR_TOP', product)
return clean_clear_sky_toa_sw_forcing
def get_clean_all_sky_surface_net_sw(infile, product=None):
"""
This is $F_{sw,all}^{\bot,0}$ as defined in A.7.1 of ECHAM6_userguide.pdf
:param infile: An ECHAM-HAM echam filepath
:param product:
:return:
"""
forcing_file = get_stream_file(infile, 'forcing')
clean_all_sky_toa_sw_forcing = cis.read_data(infile, 'srads', product) - \
cis.read_data(forcing_file, 'FSW_TOTAL_SUR', product)
return clean_all_sky_toa_sw_forcing
def get_clean_all_sky_toa_net_lw(infile, product=None):
"""
This is $F_{lw,all}^{\top,0}$ as defined in A.7.1 of ECHAM6_userguide.pdf
:param infile: An ECHAM-HAM echam filepath
:param product:
:return:
"""
forcing_file = get_stream_file(infile, 'forcing')
clean_all_sky_toa_sw_forcing = cis.read_data(infile, 'trad0', product) - \
cis.read_data(forcing_file, 'FLW_TOTAL_TOP', product)
return clean_all_sky_toa_sw_forcing
def get_clean_clear_sky_surface_net_lw(infile, product=None):
"""
This is $F_{lw,clear}^{\bot,0}$ as defined in A.7.1 of ECHAM6_userguide.pdf
:param infile: An ECHAM-HAM echam filepath
:param product:
:return:
"""
forcing_file = get_stream_file(infile, 'forcing')
clean_clear_sky_toa_sw_forcing = cis.read_data(infile, 'trafs', product) - \
cis.read_data(forcing_file, 'FLW_CLEAR_SUR', product)
return clean_clear_sky_toa_sw_forcing
def test_vertical_spacing(self):
from CALIOPy.utils import VERTICAL_SPACING, remove_air_pressure
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532", "Caliop_V4_QC")
remove_air_pressure(ext_data)
alt_diff = -np.diff(ext_data.coord('altitude').data[0, :] / 1000)
assert_almost_equal(VERTICAL_SPACING.data, alt_diff, decimal=3)
def test_masking_profile_vars(self):
raw_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532",
"Caliop_V4_QC_NO_PRESSURE")
assert raw_data.shape == (639, 399)
assert np.all(raw_data.data > 0.0)
assert raw_data.data.count() == 15382
def subset_aerosol_cci_over_africa():
from subset_by_region.utils import stack_data_list
subsetted_data = []
for f in files:
d = cis.read_data(f, "AOD550")
subsetted_data.append(subset_africa(d))
subset = stack_data_list(subsetted_data)
return subset
def test_can_concatenate_aircraft_files(self):
from cis import read_data
from cis.test.integration_test_data import valid_GASSP_aircraft_files_with_different_timestamps,\
valid_GASSP_aircraft_var_with_different_timestamps
data = read_data(valid_GASSP_aircraft_files_with_different_timestamps,
valid_GASSP_aircraft_var_with_different_timestamps)
time_coord = data.coord(axis='T')
assert_that(len(time_coord.data), equal_to(63609))
def test_can_concatenate_aircraft_files(self):
from cis import read_data
from cis.test.integration_test_data import valid_GASSP_aircraft_files_with_different_timestamps,\
valid_GASSP_aircraft_var_with_different_timestamps
data = read_data(valid_GASSP_aircraft_files_with_different_timestamps,
valid_GASSP_aircraft_var_with_different_timestamps)
time_coord = data.coord(axis='T')
assert_that(len(time_coord.data), equal_to(63609))
def load_var(self, infile, product=None):
# Take the specific variable product over the general one if there is one
product = self.product or product
if callable(self.load):
cube = self.load(self.stream_file(infile), product=product)
else:
cube = cis.read_data(self.stream_file(infile), self.load, product)
return cube
def test_compare_column_and_integrated_profile(self):
from CALIOPy.utils import integrate_profile, mask_data, remove_air_pressure
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532",
"Caliop_V4_NO_PRESSURE")
col_data = cis.read_data(
self.TEST_FILENAME,
"Column_Optical_Depth_Tropospheric_Aerosols_532")
cum_prob = cis.read_data(self.TEST_FILENAME,
"Column_IAB_Cumulative_Probability")
backscatter = cis.read_data(self.TEST_FILENAME,
"Total_Backscatter_Coefficient_532")
int_backscatter = cis.read_data(
self.TEST_FILENAME, "Column_Integrated_Attenuated_Backscatter_532")
remove_air_pressure(ext_data)
remove_air_pressure(col_data)
remove_air_pressure(cum_prob)
remove_air_pressure(backscatter)
remove_air_pressure(int_backscatter)
# Mask both datasets
col_data, cum_data, int_backscatter = mask_data(
[col_data, cum_prob, int_backscatter], self.cad_score,
self.extinction_qc)
ext_data, backscatter = mask_data([ext_data, backscatter],
self.cad_score, self.extinction_qc)
# Integrate the profile
int_data = integrate_profile(ext_data)
my_int_back = integrate_profile(backscatter)
assert np.abs((my_int_back.data - int_backscatter.data[:, 0]
).mean()) < 0.016, "Backscatter integrations differ"
#-0.0154302524737
print(np.abs((my_int_back.data - int_backscatter.data[:, 0]).mean()))
# There are a *few* values which are quite different, but I really don't know why, and they also have fairly
# large uncertainty. On average the differences are very small.
print(np.abs(np.mean(int_data.data - col_data.data[:, 0])))
assert np.abs(np.mean(int_data.data - col_data.data[:, 0])
) < 0.015, 'Extinction integrations are different'
assert np.array_equal(
int_data.data.mask,
col_data.data[:, 0].mask), 'Arrays have different masks'
def test_polar_projection(self):
from cis import read_data
import cartopy.crs as ccrs
d = read_data(valid_aerosol_cci_filename, valid_aerosol_cci_variable)
ax = d.plot(projection=ccrs.NorthPolarStereo())
self.check_graphic()
def test_integrate_profile(self):
from CALIOPy.utils import integrate_profile, mask_data
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532",
"Caliop_V4_QC_NO_PRESSURE")
int_data = integrate_profile(ext_data)
assert np.all(int_data.data > 0.0)
assert int_data.data.count() == 639
# This isn't the same as the above test because we're doing the masking before the integration
assert_almost_equal(int_data.data.mean(), 0.15009987)
def test_mercator_projection(self):
from cis import read_data
import cartopy.crs as ccrs
d = read_data(valid_aerosol_cci_filename, valid_aerosol_cci_variable)
ax = d.plot(projection=ccrs.Mercator())
ax.bluemarble()
self.check_graphic()
def test_can_concatenate_files_with_different_time_stamps(self):
from cis import read_data
import numpy as np
from cis.test.integration_test_data import valid_GASSP_station_files_with_different_timestamps,\
valid_GASSP_station_var_with_different_timestamps
var = valid_GASSP_station_var_with_different_timestamps
filename = valid_GASSP_station_files_with_different_timestamps
data = read_data(filename, var)
time_coord = data.coord(axis='T')
assert_that(np.min(time_coord.data), close_to(149107 + 54690.0/86400, 1e-5))
assert_that(np.max(time_coord.data), close_to(149110 + 81330.0/86400, 1e-5))
def get_bc_ppe_data(cache_path='.', dre=False, normalize_params=True):
"""
Load the example BC PPE AAOD data and parameters, downloading if not present
:param str cache_path: Path to load/store the PPE data
:param bool dre: Output the Direct Radiative Effect data also
:param bool normalize_params: Normalize the PPE parameters between 0.-1.
:return:
"""
import pandas as pd
import numpy as np
import cis
bc_ppe_cache = os.path.join(cache_path, 'BC_PPE_PD_AAOD_monthly.nc')
if not os.path.isfile(bc_ppe_cache):
urllib.request.urlretrieve("https://zenodo.org/record/3856645/files/BC_PPE_PD_AAOD_monthly.nc?download=1", bc_ppe_cache)
params_cache = os.path.join(cache_path, 'aerocommmppe_bcdesign.csv')
if not os.path.isfile(params_cache):
urllib.request.urlretrieve("https://zenodo.org/record/3856645/files/aerocommmppe_bcdesign.csv?download=1", params_cache)
ppe_params = pd.read_csv(params_cache)
ppe_aaod = cis.read_data(bc_ppe_cache, 'ABS_2D_550nm', 'NetCDF_Gridded')
# Ensure the job dimension is at the front
ppe_aaod.transpose((1, 0, 2, 3))
if normalize_params:
# These scaling parameters are log-uniformly distributed
ppe_params[['BCnumber', 'Wetdep']] = np.log(ppe_params[['BCnumber', 'Wetdep']])
ppe_params = ppe_params.apply(normalize, axis=0)
if dre:
ppe_dre_cache = os.path.join(cache_path, 'BC_PPE_PD_FORCING_monthly.nc')
if not os.path.isfile(ppe_dre_cache):
urllib.request.urlretrieve("https://zenodo.org/record/3856645/files/BC_PPE_PD_FORCING_monthly.nc?download=1", ppe_dre_cache)
ppe_dre = cis.read_data(ppe_dre_cache, 'FSW_TOTAL_TOP', 'NetCDF_Gridded')
return ppe_params, ppe_aaod, ppe_dre
else:
return ppe_params, ppe_aaod
def density(file):
d = cis.read_data(file, 'poc_mask')
binned = d.aggregate(longitude=[-180,180,1], latitude=[-90,90,1])
binned.append(d.aggregate(longitude=[-180,180,1], latitude=[-90,90,1], how='sum'))
for info in binned:
info.data = info.data.filled(0.0)
density = make_from_cube(binned[3]/binned[2])
count = binned[2]
sums = binned[3]
density.save_data('./density_data/'+file[0][-38:-16]+'_density_1km.nc')
count.save_data('./density_data/'+file[0][-38:-16]+'_count_1km.nc')
sums.save_data('./density_data/'+file[0][-38:-16]+'_sum_1km.nc')
return None
def test_orographic_projection(self):
from cis import read_data
import cartopy.crs as ccrs
d = read_data(valid_aerosol_cci_filename, valid_aerosol_cci_variable)
ax = plt.subplot(1, 1, 1, projection=ccrs.Orthographic(0, 90))
ax = d.plot(ax=ax)
ax.set_global()
self.check_graphic()
def test_can_concatenate_files_with_different_time_stamps(self):
from cis import read_data
import numpy as np
from cis.test.integration_test_data import valid_GASSP_station_files_with_different_timestamps,\
valid_GASSP_station_var_with_different_timestamps
var = valid_GASSP_station_var_with_different_timestamps
filename = valid_GASSP_station_files_with_different_timestamps
data = read_data(filename, var)
time_coord = data.coord(axis='T')
assert_that(np.min(time_coord.data),
close_to(149107 + 54690.0 / 86400, 1e-5))
assert_that(np.max(time_coord.data),
close_to(149110 + 81330.0 / 86400, 1e-5))
def test_integrate_profile(self):
from CALIOPy.utils import integrate_profile, remove_air_pressure, mask_data
ext_data = cis.read_data(self.TEST_FILENAME,
"Extinction_Coefficient_532",
"Caliop_V4_NO_PRESSURE")
remove_air_pressure(ext_data)
int_data = integrate_profile(ext_data)
assert np.all(int_data.data > 0.0)
assert int_data.data.shape == (4224, )
# Now check that masking this data produces something sensible
masked_int_data, = mask_data([int_data], self.cad_score,
self.extinction_qc)
assert masked_int_data.data.count() == 639
assert_almost_equal(masked_int_data.data.mean(), 0.1535116)
def density(file):
d = cis.read_data(file, 'poc_mask', 'cis')
binned = d.aggregate(longitude=[-180, 180, 1], latitude=[-90, 90, 1])
binned.append(
d.aggregate(longitude=[-180, 180, 1], latitude=[-90, 90, 1],
how='sum'))
for info in binned:
info.data = info.data.filled(0.0)
density = make_from_cube(binned[3] / binned[2])
count = binned[2]
sums = binned[3]
outname = './density_data/' + os.path.basename(file[0])
density.save_data(outname + '_density_5km.nc')
count.save_data(outname + '_count_5km.nc')
sums.save_data(outname + '_sum_5km.nc')
return None
def get_aeronet_data(cache_path='.'):
import cis
from cis.data_io.aeronet import AERONET_HEADER_LENGTH
aerocom_cache = os.path.join(cache_path, 'aerocom_2017_global.lev15')
if not os.path.isfile(aerocom_cache):
_download_aeronet(aerocom_cache)
# Fix the header length which seems to be different for downloaded files...
AERONET_HEADER_LENGTH["AERONET/3"] = 6
# TODO: I might want to interpolate between 440 and 630nm to get 550nm...
aaod = cis.read_data(aerocom_cache, 'Absorption_AOD440nm')
# Pop off the altitude coordinate which we're not interested in
_ = aaod._coords.pop(aaod._coords.index(aaod.coord('altitude')))
return aaod
def check_output_col_grid(self, sample_file, sample_var, output_file, output_vars, expected_shape=None):
"""
Check that the output grid matches the sample grid in shape.
:param sample_file:
:param sample_var:
:param output_file:
:param output_vars:
:return:
"""
from cis import read_data, read_data_list
from operator import mul
if expected_shape is None:
sample_shape = read_data(sample_file, sample_var).data.shape
else:
sample_shape = expected_shape
output = read_data_list(output_file, output_vars)
for output_var in output:
output_shape = output_var.data.shape
# This copes with dims in different orders, length 1 values being taken out etc
assert_that(reduce(mul, sample_shape), is_(reduce(mul, output_shape)))
def check_output_col_grid(self, sample_file, sample_var, output_file, output_vars, expected_shape=None):
"""
Check that the output grid matches the sample grid in shape.
:param sample_file:
:param sample_var:
:param output_file:
:param output_vars:
:return:
"""
from cis import read_data
from operator import mul
if expected_shape is None:
sample_shape = read_data(self._clean_sample_file_name(sample_file), sample_var).data.shape
else:
sample_shape = expected_shape
self.ds = Dataset(self._clean_sample_file_name(output_file))
for output_var in output_vars:
output_shape = self.ds.variables[output_var].shape
# This copes with dims in different orders, length 1 values being taken out etc
assert_that(reduce(mul, sample_shape), is_(reduce(mul, output_shape)))
self.ds.close()
def read_data(self):
# Only need to do this once
if self.data is None:
filename = valid_echamham_geopotential_height_filename
var = valid_echamham_geopotential_height_variable
self.data = read_data(filename, var)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import cis
from glob import glob
import xarray as xr
from cis.data_io.gridded_data import make_from_cube
import matplotlib.pyplot as plt
from cartopy import crs as ccrs
import matplotlib.patches as patch
dataDIR = '/gws/nopw/j04/impala/users/dwatsonparris/POC_analysis/density_data/*count*'
filelist = glob(dataDIR)
counts=[]
for file in filelist:
counts.append(cis.read_data(file, 'poc_mask_num_points', 'NetCDF_Gridded'))
total_counts = make_from_cube(sum(counts))
dataDIR = '/gws/nopw/j04/impala/users/dwatsonparris/POC_analysis/density_data/*sum*'
filelist = glob(dataDIR)
sums=[]
for file in filelist:
sums.append(cis.read_data(file, 'poc_mask', 'NetCDF_Gridded'))
total_sum = make_from_cube(sum(sums))
lon = {'cal':[230,240], 'per':[270,280], 'nam':[360,10]}
lat = {'cal':[20,30], 'per':[-20,-10], 'nam':[-20,-10]}
fig=plt.figure(figsize=(15,15))#, dpi=300)
ax = plt.axes(projection=ccrs.PlateCarree())
ax.coastlines()
total_counts.plot(cmap='cool', ax=ax)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import cis
from glob import glob
import xarray as xr
from cis.data_io.gridded_data import make_from_cube
import matplotlib.pyplot as plt
from cartopy import crs as ccrs
import matplotlib.patches as patch
dataDIR = './density_data/*count*'
filelist = glob(dataDIR)
counts = []
for file in filelist:
counts.append(cis.read_data(file, 'poc_mask_num_points'))
total_counts = make_from_cube(sum(counts))
dataDIR = './density_data/*sum*'
filelist = glob(dataDIR)
sums = []
for file in filelist:
sums.append(cis.read_data(file, 'poc_mask'))
total_sum = make_from_cube(sum(sums))
lon = {'cal': [230, 240], 'per': [270, 280], 'nam': [360, 10]}
lat = {'cal': [20, 30], 'per': [-20, -10], 'nam': [-20, -10]}
fig = plt.figure(figsize=(15, 15)) #, dpi=300)
ax = plt.axes(projection=ccrs.PlateCarree())
ax.coastlines()
total_counts.plot(cmap='cool', ax=ax)
filename='L1_ccn0.2_Cloud_condensation_nuclei_at_a_supersaturation_of_0.2000.nc'
filename=dir_file+filename
#----------------------------------------------------
var=get_variables(filename)
#output from cis should be in a set form
"""
#var_name=[]
#for i in var:
# var_name=np.append(var_name,i)
"""
var=list(var)
data0=read_data(filename,var[0])
mean=data0.collapsed(['latitude','longitude'],iris.analysis.MEAN)
mean2=data0.collapsed(['model_level_number'],iris.analysis.MEAN)
mean2.plot(yaxis='latitude')
plt.show()
time.sleep(20)
plt.close()
#print data0.info()
#print var
#----------------------------------------------------
"""
cube=iris.load(filename)
cube=cube[0].collapsed('model_level_number',iris.analysis.MEAN)
def read_and_subset_data(filename):
d = cis.read_data(filename, "AOD550")
return subset_region(d, make_african_land())
def test_log_colorbar_scale(self):
from cis import read_data
d = read_data(valid_aerosol_cci_filename, valid_aerosol_cci_variable)
d.plot(logv=True)
self.check_graphic()
def test_can_specify_yaxis_altitude(self):
from cis import read_data
d = read_data(valid_GASSP_aeroplane_filename, valid_GASSP_aeroplane_variable)
d.plot(yaxis='altitude')
self.check_graphic()
def test_read_data_raises_error_on_more_than_one_variable_returned():
file = cis_test_files['caliop_L1'].master_filename
read_data(file, cis_test_files['caliop_L1'].all_variable_names)
def test_read_data_raises_error_on_invalid_file():
read_data(invalid_filename, cis_test_files['Aerosol_CCI'].all_variable_names)
def test_read_data_raises_error_on_invalid_file_wildcard():
read_data(invalid_filename + '*', cis_test_files['caliop_L1'].all_variable_names)
def test_read_data_raises_error_on_invalid_variable():
file = cis_test_files['modis_L3'].master_filename
read_data(file, 'invalid_variable')