def test_collocated_NetCDF_Gridded_onto_GASSP(self): # First do a collocation of ECHAMHAM onto GASSP vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = escape_colons(valid_echamham_filename) sample_file = escape_colons(valid_GASSP_aeroplane_filename) sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn[missing_data_for_missing_sample=True],variable=%s' % sample_var arguments = [ 'col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', 'collocated_gassp' ] main_arguments = parse_args(arguments) col_cmd(main_arguments) # Check collocation is the same self.ds = netCDF4.Dataset('collocated_gassp.nc') col_var1 = self.ds.variables[valid_echamham_variable_1][:] col_var2 = self.ds.variables[valid_echamham_variable_2][:] # A hand calculated selection of values expected_col1 = numpy.ma.masked_invalid([ float('Nan'), float('Nan'), float('Nan'), 0.0814601778984, 0.0814601778984 ]) compare_masked_arrays(expected_col1, col_var1[:][0:5]) expected_col2 = numpy.ma.masked_invalid([ float('Nan'), float('Nan'), float('Nan'), 0.0741240680218, 0.0741240680218 ]) compare_masked_arrays(expected_col2, col_var2[:][0:5]) # Then do an evaluation using the collocated data: args = [ 'eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, 'collocated_gassp.nc'), "%s=gassp_alias:%s" % (valid_GASSP_aeroplane_variable, escape_colons(valid_GASSP_aeroplane_filename)), "(%s + %s) / gassp_alias " % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds.close() # Check correct self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = numpy.ma.masked_invalid( [0.00196121983491, 0.00197255626472, 0.00120850731992]) assert_that(calculated_result.shape, is_((311, ))) # Check the first 3 vald values compare_masked_arrays(expected_result, calculated_result[:][10:13]) os.remove('collocated_gassp.nc')
def test_collocated_NetCDF_Gridded_onto_GASSP(self): # Takes 2s # First do a collocation of ECHAMHAM onto GASSP sample_file = valid_GASSP_aeroplane_filename sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = [ 'col', '%s:%s' % (valid_echamham_variable_1, escape_colons(valid_echamham_filename)), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', 'collocated_gassp' ] main_arguments = parse_args(arguments) col_cmd(main_arguments) # Then do a statistics calculation using the collocated data: args = [ 'stats', "%s:%s" % (valid_echamham_variable_1, 'collocated_gassp.nc'), "%s:%s" % (valid_GASSP_aeroplane_variable, escape_colons(valid_GASSP_aeroplane_filename)), '-o', self.OUTPUT_FILENAME ] arguments = parse_args(args) stats_cmd(arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, self.output_vars) os.remove('collocated_gassp.nc')
def test_GASSP_onto_CALIOP_L2(self): variable = valid_GASSP_aeroplane_variable filename = valid_GASSP_aeroplane_filename sample_file = valid_caliop_l2_filename collocator_and_opts = 'box[h_sep=10km],kernel=mean' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments)
def test_GASSP_onto_CALIOP_L1(self): # Ran for 14hr, not finished. (with no h_sep) variable = valid_GASSP_aeroplane_variable filename = valid_GASSP_aeroplane_filename sample_file = valid_caliop_l1_filename collocator_and_opts = 'box[h_sep=10km],kernel=mean' arguments = ['col', variable + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments)
def test_GIVEN_single_variable_WHEN_collocate_THEN_successful_collocation(self): variable = cis_test_files["NCAR_NetCDF_RAF"].data_variable_name filename = cis_test_files["NCAR_NetCDF_RAF"].master_filename sample_file = valid_hadgem_filename collocator_and_opts = 'bin,kernel=mean' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, ['ATX'])
def test_GIVEN_multiple_variables_on_same_coords_plus_dim_vars_WHEN_collocate_THEN_successful_collocation(self): variables = valid_aerosol_cci_variable, valid_aerosol_cci_variable_2, 'time' filename = valid_aerosol_cci_filename sample_file = valid_echamham_filename collocator_and_opts = 'bin,kernel=mean' arguments = ['col', ','.join(variables) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, variables)
def test_GASSP_onto_Aeronet(self): # Takes 73s variable = valid_GASSP_aeroplane_variable filename = valid_GASSP_aeroplane_filename sample_file = valid_aeronet_filename collocator_and_opts = 'box[h_sep=10km],kernel=mean' arguments = ['col', variable + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, variable.split(","))
def test_GIVEN_single_variable_WHEN_collocate_box_THEN_successful_collocation(self): # Takes 13 s variable = cis_test_files["NCAR_NetCDF_RAF"].data_variable_name filename = cis_test_files["NCAR_NetCDF_RAF"].master_filename sample_file = valid_echamham_filename collocator_and_opts = 'box[h_sep=10],kernel=mean' arguments = ['col', variable + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, ['ATX'])
def test_GASSP_onto_GASSP(self): # Takes 6.5mins vars = valid_GASSP_station_vars[:2] filename = valid_GASSP_station_filename sample_file = valid_GASSP_aeroplane_filename collocator_and_opts = 'box[h_sep=10km],kernel=mean' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars)
def test_Aeronet_onto_CloudSat(self): # Takes 5hrs variable = "AOT_440,AOT_870" filename = valid_aeronet_filename sample_file = valid_cloudsat_PRECIP_file collocator_and_opts = 'box[h_sep=700km],kernel=mean' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, variable.split(","))
def test_GASSP_aux_coord_onto_Aeronet(self): # Takes 73s filename = cis_test_files['GASSP_aux_coord'].master_filename variable = cis_test_files['GASSP_aux_coord'].data_variable_name sample_file = valid_aeronet_filename collocator_and_opts = 'box[h_sep=10km],kernel=mean' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, variable.split(","))
def test_CALIOP_L1_onto_NetCDF_Gridded(self): vars = [valid_caliop_l1_variable] filename = valid_caliop_l1_filename sample_file = valid_hadgem_filename sample_var = valid_hadgem_variable collocator_and_opts = 'bin,kernel=mean,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_Aeronet_onto_GASSP(self): # Takes 81 s # Two vars takes 160s - shouldn't? variable = "AOT_440,AOT_870" filename = valid_aeronet_filename sample_file = valid_GASSP_aeroplane_filename collocator_and_opts = 'box[h_sep=10m],kernel=mean' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, variable.split(","))
def test_GASSP_aux_onto_NetCDF_Gridded(self): filename = cis_test_files['GASSP_aux_coord'].master_filename var = cis_test_files['GASSP_aux_coord'].data_variable_name sample_file = valid_echamham_filename sample_var = valid_echamham_variable_1 collocator_and_opts = 'bin,kernel=mean' arguments = ['col', var + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, [var]) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, [var])
def test_NetCDF_Gridded_onto_CALIOP_L2(self): vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = valid_caliop_l2_filename sample_var = valid_caliop_l2_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_hybrid_pressure_onto_GASSP_li(self): # Takes 1s vars = [valid_hybrid_pressure_variable] filename = valid_hybrid_pressure_filename sample_file = valid_GASSP_aircraft_files_with_different_timestamps[0] sample_var = valid_GASSP_aircraft_var_with_different_timestamps collocator_and_opts = 'lin,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, ['mmrbc']) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, ['mmrbc'])
def test_hybrid_height_onto_GASSP_li(self): # Takes 1s vars = [valid_hybrid_height_flat_variable] filename = valid_hybrid_height_flat_filename sample_file = valid_GASSP_aeroplane_filename sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'lin,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_HadGem_onto_cis_gridded_output_lin(self): # Takes 1s vars = [valid_hadgem_variable] filename = valid_hadgem_filename sample_file = valid_cis_gridded_output_filename sample_var = valid_cis_gridded_output_variable collocator_and_opts = 'lin,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_NetCDF_Gridded_onto_ASCII_no_variable(self): # Takes 4s vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = cis_test_files["ascii"].master_filename sample_var = cis_test_files["ascii"].data_variable_name collocator_and_opts = 'nn' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_ASCII_onto_NetCDF_Gridded(self): # Takes 5s vars = ["values"] filename = valid_ascii_filename sample_file = valid_hadgem_filename sample_var = valid_hadgem_variable collocator_and_opts = 'bin,kernel=mean' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, ["value"]) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, ["value"])
def test_cloudsat_RVOD_onto_NetCDF_Gridded(self): # Takes 290s vars = [valid_cloudsat_RVOD_sdata_variable, valid_cloudsat_RVOD_vdata_variable] filename = valid_cloudsat_RVOD_file sample_file = valid_hadgem_filename sample_var = valid_hadgem_variable collocator_and_opts = 'bin,kernel=mean' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_GASSP_onto_Aeronet_using_moments_kernel(self): # Takes 73s variable = valid_GASSP_aeroplane_variable filename = valid_GASSP_aeroplane_filename sample_file = valid_aeronet_filename collocator_and_opts = 'box[h_sep=10km],kernel=moments' arguments = ['col', variable + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) out_vars = [variable, variable + '_std_dev', variable + '_num_points'] self.check_output_contains_variables(self.OUTPUT_FILENAME, out_vars) self.check_output_col_grid(sample_file, valid_aeronet_variable, self.OUTPUT_FILENAME, out_vars)
def test_GASSP_plane_onto_NetCDF_Gridded(self): # Takes 15s vars = valid_GASSP_aeroplane_vars filename = valid_GASSP_aeroplane_filename sample_file = valid_echamham_filename sample_var = valid_echamham_variable_1 collocator_and_opts = 'bin,kernel=mean,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_MODIS_L3_onto_NetCDF_Gridded(self): # Takes 27s vars = [valid_modis_l3_variable] filename = valid_modis_l3_filename sample_file = valid_hadgem_filename sample_var = valid_hadgem_variable collocator_and_opts = 'bin,kernel=mean,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_hybrid_pressure_onto_GASSP_nn(self): # Takes 1s vars = [valid_hybrid_pressure_variable] filename = valid_hybrid_pressure_filename sample_file = valid_GASSP_aeroplane_filename sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, ['mmrbc']) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, ['mmrbc'])
def test_MODIS_L2_onto_NetCDF_Gridded(self): # Takes 20s vars = ['Solar_Zenith', 'Optical_Depth_Ratio_Small_Land_And_Ocean'] filename = valid_modis_l2_filename sample_file = valid_hadgem_filename sample_var = valid_hadgem_variable collocator_and_opts = 'bin,kernel=mean,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_NetCDF_Gridded_onto_MODIS_L3(self): # Takes 47s vars = valid_hadgem_variable, filename = valid_hadgem_filename sample_file = valid_modis_l3_filename sample_var = valid_modis_l3_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_NetCDF_Gridded_onto_cis_output_data(self): # Takes 3s vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = valid_cis_ungridded_output_filename sample_var = valid_cis_ungridded_output_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_NetCDF_Gridded_onto_NCAR_RAF(self): # Takes 30s vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = valid_NCAR_NetCDF_RAF_filename sample_var = valid_NCAR_NetCDF_RAF_variable collocator_and_opts = 'nn' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_GASSP_ship_onto_NetCDF_Gridded(self): # Takes 10mins vars = valid_GASSP_ship_vars filename = valid_GASSP_ship_filename sample_file = valid_echamham_filename sample_var = valid_echamham_variable_1 collocator_and_opts = 'bin,kernel=mean' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_Aeronet_onto_NetCDF_Gridded(self): # JASCIS-120 # Takes 6s vars = ["AOT_440", "AOT_870"] filename = valid_aeronet_filename sample_file = valid_echamham_filename sample_var = valid_echamham_variable_1 collocator_and_opts = 'bin,kernel=mean' arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars)
def test_NetCDF_Gridded_onto_GASSP_aux_coord_li(self): # Takes 1s vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = cis_test_files['GASSP_aux_coord'].master_filename sample_var = cis_test_files['GASSP_aux_coord'].data_variable_name collocator_and_opts = 'lin,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + escape_colons(filename), escape_colons(sample_file) + ':collocator=' + collocator_and_opts, '-o', self.OUTPUT_FILENAME] main_arguments = parse_args(arguments) col_cmd(main_arguments) self.check_output_contains_variables(self.OUTPUT_FILENAME, vars) self.check_output_col_grid(sample_file, sample_var, self.OUTPUT_FILENAME, vars) self.check_output_vars_are_different(self.OUTPUT_FILENAME, vars)