def test_moments_kernel_aggregate_cis_ungridded(self):
# Takes 1s
variable = '*'
filename = valid_cis_ungridded_output_filename
lon_min, lon_max, lon_delta = 1, 3, 0.3
lat_min, lat_max, lat_delta = 41, 42, 0.1
grid = 'x=[%s,%s,%s],y=[%s,%s,%s]' % (lon_min, lon_max, lon_delta,
lat_min, lat_max, lat_delta)
arguments = [
'aggregate',
variable + ':' + escape_colons(filename) + ':kernel=moments', grid,
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_grid_aggregation(lat_min,
lat_max,
lat_delta,
lon_min,
lon_max,
lon_delta,
lat_name='latitude',
lon_name='longitude')
expected_vars = [
'AOD870', 'AOD870_std_dev', 'AOD870_num_points', 'AOD550',
'AOD550_std_dev', 'AOD550_num_points'
]
self.check_output_contains_variables(self.OUTPUT_FILENAME,
expected_vars)
def test_netCDF_gridded_hybrid_height(self):
# Takes 2s
variable = valid_hybrid_height_variable
filename = valid_hybrid_height_filename
arguments = ['aggregate', variable + ':' + filename + ':kernel=mean', 't,x,y', '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_output_contains_variables(self.GRIDDED_OUTPUT_FILENAME, variable.split(','))
def test_aggregate_netCDF_gridded_HadGem(self):
# Takes 1s
variable = 'od550aer'
filename = valid_hadgem_filename
arguments = ['aggregate', variable + ':' + filename + ':kernel=mean', 't', '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_output_contains_variables(self.GRIDDED_OUTPUT_FILENAME, variable.split(','))
def test_aggregate_cis_gridded(self):
# Takes 1s
variable = '*'
filename = valid_cis_gridded_output_filename
arguments = ['aggregate', variable + ':' + filename + ':kernel=mean', 'x,y', '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_output_contains_variables(self.GRIDDED_OUTPUT_FILENAME, ['TAU_2D_550nm'])
def test_aggregate_netCDF_gridded_HadGem(self):
# Test aggregating a gridded file, it should work but throw a deprecation
# Takes 1s
variable = '*'
filename = valid_hadgem_filename
arguments = ['aggregate', variable + ':' + escape_colons(filename) + ':kernel=mean', 'x,y', '-o', self.OUTPUT_FILENAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_output_contains_variables(self.OUTPUT_FILENAME, ['od550aer'])
def test_aggregate_netCDF_gridded_HadGem(self):
# Takes 1s
variable = 'od550aer'
filename = valid_hadgem_filename
grid = 'x,y'
arguments = ['aggregate', variable + ':' + filename + ':kernel=moments', grid, '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
expected_vars = ['od550aer', 'od550aer_std_dev', 'od550aer_num_points']
self.check_output_contains_variables(self.GRIDDED_OUTPUT_FILENAME, expected_vars)
def do_spatial_aggregate(self, variable, filename, lat_start, lat_end,
lat_delta, lon_start, lon_end, lon_delta):
grid = 'x=[%s,%s,%s],y=[%s,%s,%s]' % (lon_start, lon_end, lon_delta,
lat_start, lat_end, lat_delta)
arguments = [
'aggregate',
variable + ':' + escape_colons(filename) + ':kernel=mean', grid,
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
def do_temporal_aggregate(self, variable, filename, t_start, t_end,
str_delta):
grid = 't=[%s,%s,%s]' % (t_start.isoformat(), t_end.isoformat(),
str_delta)
arguments = [
'aggregate',
variable + ':' + escape_colons(filename) + ':kernel=mean', grid,
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
def test_GIVEN_no_kernel_specified_WHEN_aggregate_THEN_moments_used_as_default(self):
# Takes 1s
variable = 'AOD550'
filename = valid_cis_ungridded_output_filename
lon_min, lon_max, lon_delta = 1, 3, 0.3
lat_min, lat_max, lat_delta = 41, 42, 0.1
grid = 'x=[%s,%s,%s],y=[%s,%s,%s]' % (lon_min, lon_max, lon_delta, lat_min, lat_max, lat_delta)
arguments = ['aggregate', variable + ':' + filename, grid, '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_grid_aggregation(lat_min, lat_max, lat_delta, lon_min, lon_max, lon_delta,
lat_name='latitude', lon_name='longitude')
expected_vars = ['AOD550', 'AOD550_std_dev', 'AOD550_num_points']
self.check_output_contains_variables(self.GRIDDED_OUTPUT_FILENAME, expected_vars)
def test_moments_kernel_aggregate_cis_ungridded(self):
# Takes 1s
variable = '*'
filename = valid_cis_ungridded_output_filename
lon_min, lon_max, lon_delta = 1, 3, 0.3
lat_min, lat_max, lat_delta = 41, 42, 0.1
grid = 'x=[%s,%s,%s],y=[%s,%s,%s]' % (lon_min, lon_max, lon_delta, lat_min, lat_max, lat_delta)
arguments = ['aggregate', variable + ':' + escape_colons(filename) + ':kernel=moments', grid, '-o', self.OUTPUT_FILENAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_grid_aggregation(lat_min, lat_max, lat_delta, lon_min, lon_max, lon_delta,
lat_name='latitude', lon_name='longitude')
expected_vars = ['AOD870', 'AOD870_std_dev', 'AOD870_num_points',
'AOD550', 'AOD550_std_dev', 'AOD550_num_points']
self.check_output_contains_variables(self.OUTPUT_FILENAME, expected_vars)
def test_aggregate_netCDF_gridded_HadGem(self):
# Test aggregating a gridded file, it should work but throw a deprecation
# Takes 1s
variable = '*'
filename = valid_hadgem_filename
arguments = [
'aggregate',
variable + ':' + escape_colons(filename) + ':kernel=mean', 'x,y',
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_output_contains_variables(self.OUTPUT_FILENAME,
['od550aer'])
def test_aggregate_NCAR_RAF_with_named_time_variable_standard_name(self):
# Takes 28s
variable = "LATC,LONC,GGALTC,Time,PSXC,WSC,ATX,ATHR2,CONCD_LWI"
filename = valid_NCAR_NetCDF_RAF_filename
time_min, time_max = dt.datetime(2009, 1, 14, 20, 15), dt.datetime(2009, 1, 15, 2, 45)
time_delta = dt.timedelta(minutes=30)
str_delta = 'PT30M'
grid = 'time=[%s,%s,%s]' % (time_min.isoformat(), time_max.isoformat(), str_delta)
arguments = ['aggregate', variable + ':' + escape_colons(filename) + ':kernel=mean', grid, '-o', self.OUTPUT_FILENAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_temporal_aggregation(time_min, time_max, time_delta)
self.check_output_contains_variables(self.OUTPUT_FILENAME, variable.split(','))
def test_plot_aggregated_aeronet(self):
# Aggregated aeronet has multiple length 1 dimensions, so we want to make sure we can plot it OK.
# JASCIS-183
variable = 'Solar_Zenith_Angle'
filename = valid_aeronet_filename
agg_args = ['aggregate', variable + ':' + filename,
't=[2003-09-24T07:00:00,2003-11-04T07:00:00,P1D]', '-o', self.OUTPUT_FILENAME]
args = parse_args(agg_args)
aggregate_cmd(args)
out_name = 'aeronet_out.png'
args = ['plot', variable + ':' + self.OUTPUT_FILENAME,
'--xaxis', 'time', '--yaxis', variable, '-o', out_name]
main_arguments = parse_args(args)
plot_cmd(main_arguments)
os.remove(out_name)
def test_aggregate_Cloud_CCI_for_comparison_with_collocation(self):
"""
Takes ~80s on aopposxlap18. This test is primarily to compare the speed of ungridded->gridded collocation and
aggregation, which should in principle be just as quick as each other. This test mirrors the
test_cloud_cci_onto_NetCDF_Gridded test in test_colocate.py.
"""
variables = ','.join([valid_cloud_cci_variable, valid_cloud_cci_8_bit_variable])
filename = valid_cloud_cci_filename
grid = 'x=[0, 358.125, 1.875],y=[-90, 90, 1.25],t=[2007-06-01,2007-06-30,PT3H]'
arguments = ['aggregate', variables + ':' + escape_colons(filename) + ':kernel=mean', grid, '-o', self.OUTPUT_FILENAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
# This check can't deal with the combined temporal and spatial aggregation so skip it
#self.check_grid_aggregation(0, 358.125, 1.875, -90, 90, 1.25)
self.check_output_contains_variables(self.OUTPUT_FILENAME, variables.split(','))
def test_plot_aggregated_aeronet(self):
# Aggregated aeronet has multiple length 1 dimensions, so we want to make sure we can plot it OK.
# JASCIS-183
variable = 'Solar_Zenith_Angle'
filename = valid_aeronet_filename
agg_args = [
'aggregate', variable + ':' + escape_colons(filename),
't=[2003-09-24T07:00:00,2003-11-04T07:00:00,P1D]', '-o',
self.OUTPUT_FILENAME
]
args = parse_args(agg_args)
aggregate_cmd(args)
out_name = 'aeronet_out.png'
args = [
'plot', variable + ':' + self.OUTPUT_FILENAME, '--xaxis', 'time',
'--yaxis', variable, '-o', out_name
]
main_arguments = parse_args(args)
plot_cmd(main_arguments)
os.remove(out_name)
def test_aggregate_Cloud_CCI_for_comparison_with_collocation(self):
"""
Takes ~80s on aopposxlap18. This test is primarily to compare the speed of ungridded->gridded collocation and
aggregation, which should in principle be just as quick as each other. This test mirrors the
test_cloud_cci_onto_NetCDF_Gridded test in test_colocate.py.
"""
variables = ','.join(
[valid_cloud_cci_variable, valid_cloud_cci_8_bit_variable])
filename = valid_cloud_cci_filename
grid = 'x=[0, 358.125, 1.875],y=[-90, 90, 1.25],t=[2007-06-01,2007-06-30,PT3H]'
arguments = [
'aggregate',
variables + ':' + escape_colons(filename) + ':kernel=mean', grid,
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
# This check can't deal with the combined temporal and spatial aggregation so skip it
#self.check_grid_aggregation(0, 358.125, 1.875, -90, 90, 1.25)
self.check_output_contains_variables(self.OUTPUT_FILENAME,
variables.split(','))
def test_aggregate_NCAR_RAF_with_named_time_variable_standard_name(self):
# Takes 28s
variable = "LATC,LONC,GGALTC,Time,PSXC,WSC,ATX,ATHR2,CONCD_LWI"
filename = valid_NCAR_NetCDF_RAF_filename
time_min, time_max = dt.datetime(2009, 1, 14, 20,
15), dt.datetime(2009, 1, 15, 2, 45)
time_delta = dt.timedelta(minutes=30)
str_delta = 'PT30M'
grid = 'time=[%s,%s,%s]' % (time_min.isoformat(), time_max.isoformat(),
str_delta)
arguments = [
'aggregate',
variable + ':' + escape_colons(filename) + ':kernel=mean', grid,
'-o', self.OUTPUT_FILENAME
]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
self.check_temporal_aggregation(time_min, time_max, time_delta)
self.check_output_contains_variables(self.OUTPUT_FILENAME,
variable.split(','))
def do_temporal_aggregate(self, variable, filename, t_start, t_end, str_delta):
grid = 't=[%s,%s,%s]' % (t_start.isoformat(), t_end.isoformat(), str_delta)
arguments = ['aggregate', variable + ':' + filename + ':kernel=mean', grid, '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
def do_spatial_aggregate(self, variable, filename, lat_start, lat_end, lat_delta, lon_start, lon_end, lon_delta):
grid = 'x=[%s,%s,%s],y=[%s,%s,%s]' % (lon_start, lon_end, lon_delta, lat_start, lat_end, lat_delta)
arguments = ['aggregate', variable + ':' + filename + ':kernel=mean', grid, '-o', self.OUTPUT_NAME]
main_arguments = parse_args(arguments)
aggregate_cmd(main_arguments)
