def test_collocated_NetCDF_Gridded_onto_GASSP(self): # First do a collocation of ECHAMHAM onto GASSP vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = escape_colons(valid_echamham_filename) sample_file = escape_colons(valid_GASSP_aeroplane_filename) sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn[missing_data_for_missing_sample=True],variable=%s' % sample_var arguments = [ 'col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', 'collocated_gassp' ] main_arguments = parse_args(arguments) col_cmd(main_arguments) # Check collocation is the same self.ds = netCDF4.Dataset('collocated_gassp.nc') col_var1 = self.ds.variables[valid_echamham_variable_1][:] col_var2 = self.ds.variables[valid_echamham_variable_2][:] # A hand calculated selection of values expected_col1 = numpy.ma.masked_invalid([ float('Nan'), float('Nan'), float('Nan'), 0.0814601778984, 0.0814601778984 ]) compare_masked_arrays(expected_col1, col_var1[:][0:5]) expected_col2 = numpy.ma.masked_invalid([ float('Nan'), float('Nan'), float('Nan'), 0.0741240680218, 0.0741240680218 ]) compare_masked_arrays(expected_col2, col_var2[:][0:5]) # Then do an evaluation using the collocated data: args = [ 'eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, 'collocated_gassp.nc'), "%s=gassp_alias:%s" % (valid_GASSP_aeroplane_variable, escape_colons(valid_GASSP_aeroplane_filename)), "(%s + %s) / gassp_alias " % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds.close() # Check correct self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = numpy.ma.masked_invalid( [0.00196121983491, 0.00197255626472, 0.00120850731992]) assert_that(calculated_result.shape, is_((311, ))) # Check the first 3 vald values compare_masked_arrays(expected_result, calculated_result[:][10:13]) os.remove('collocated_gassp.nc')
def test_Aeronet_wavelength_calculation(self): # Example from the CIS Phase 3 Software spec: # ... a user should be able to write a plugin to calculate the Aeronet AOD at 550nm from the AOD at 500 nm as # AOD550 = AOD500 * (550/500)^(-1*Angstrom500-870)" # Takes 3s args = [ 'eval', 'AOT_500,500-870Angstrom=a550to870:' + escape_colons(another_valid_aeronet_filename), 'AOT_500 * (550.0/500)**(-1*a550to870)', '1', '-o', self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) # Check correct: self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] expected_result = [ 0.2341039087, 0.2285401152, 0.2228799533, 0.1953746746, 0.2094051561, 0.1696889668, 0.3137791803, 0.2798929273, 0.1664194279, 0.1254619092, 0.1258309124, 0.1496960031, 0.0768447737, 0.0550896430, 0.0534543107, 0.0538315909, 0.0666742975, 0.0512935449, 0.0699585189, 0.0645033944 ] assert_that(calculated_result.shape, is_((3140, ))) assert numpy.allclose(expected_result, calculated_result[0:20])
def test_CloudSat(self): args = ['eval', "%s,%s:%s" % (valid_cloudsat_RVOD_sdata_variable, valid_cloudsat_RVOD_vdata_variable, escape_colons(valid_cloudsat_RVOD_file)), '%s/%s' % (valid_cloudsat_RVOD_sdata_variable, valid_cloudsat_RVOD_vdata_variable), 'ppm', '-o', 'cloudsat_var:' + self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['cloudsat_var'].units, is_('ppm'))
def test_can_specify_units_gridded_no_output_var(self): args = ['eval', "%s:%s" % (valid_hadgem_variable, escape_colons(valid_hadgem_filename)), "od550aer", "ppm", "-o", self.OUTPUT_FILENAME, "-a", "att1=val1"] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['calculated_variable'].units, is_('ppm')) assert_that(self.ds.variables['calculated_variable'].att1, is_('val1'))
def test_can_specify_units_gridded(self): args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), 'kg m^-3', '-o', 'var_out:' + self.OUTPUT_FILENAME, '-a', 'att1=val1,att2=val2'] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['var_out'].units, is_('kg m^-3'))
def test_can_specify_output_variable(self): args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), 'kg m^-3', '-o', 'var_out:' + self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert 'var_out' in self.ds.variables
def test_can_specify_attributes_gridded(self): args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, valid_echamham_filename), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), 'kg m^-3', '-o', 'var_out:' + self.OUTPUT_NAME, '-a', 'att1=val1,att2=val2'] arguments = parse_args(args) evaluate_cmd(arguments) ds = netCDF4.Dataset(self.GRIDDED_OUTPUT_FILENAME) assert_that(ds.variables['var_out'].att1, is_('val1')) assert_that(ds.variables['var_out'].att2, is_('val2'))
def test_can_specify_units_gridded_no_output_var(self): args = [ 'eval', "%s:%s" % (valid_hadgem_variable, escape_colons(valid_hadgem_filename)), "od550aer", "ppm", "-o", self.OUTPUT_FILENAME, "-a", "att1=val1" ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['calculated_variable'].units, is_('ppm')) assert_that(self.ds.variables['calculated_variable'].att1, is_('val1'))
def test_can_specify_output_variable(self): args = [ 'eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), 'kg m^-3', '-o', 'var_out:' + self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert 'var_out' in self.ds.variables
def test_ECHAMHAM_wavelength_sum(self): args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) # Check correct: self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = [0.007633533, 0.007646653, 0.007749859, 0.007744226, 0.007761176] assert_that(calculated_result.shape, is_((96, 192))) assert numpy.allclose(expected_result, calculated_result[:][0][0:5])
def test_can_specify_units_gridded(self): args = [ 'eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), 'kg m^-3', '-o', 'var_out:' + self.OUTPUT_FILENAME, '-a', 'att1=val1,att2=val2' ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['var_out'].units, is_('kg m^-3'))
def test_CloudSat(self): args = [ 'eval', "%s,%s:%s" % (valid_cloudsat_RVOD_sdata_variable, valid_cloudsat_RVOD_vdata_variable, escape_colons(valid_cloudsat_RVOD_file)), '%s/%s' % (valid_cloudsat_RVOD_sdata_variable, valid_cloudsat_RVOD_vdata_variable), 'ppm', '-o', 'cloudsat_var:' + self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) assert_that(self.ds.variables['cloudsat_var'].units, is_('ppm'))
def test_collocated_NetCDF_Gridded_onto_GASSP(self): # First do a collocation of ECHAMHAM onto GASSP vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = escape_colons(valid_echamham_filename) sample_file = escape_colons(valid_GASSP_aeroplane_filename) sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn[missing_data_for_missing_sample=True],variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', 'collocated_gassp'] main_arguments = parse_args(arguments) col_cmd(main_arguments) # Check collocation is the same self.ds = netCDF4.Dataset('collocated_gassp.nc') col_var1 = self.ds.variables[valid_echamham_variable_1][:] col_var2 = self.ds.variables[valid_echamham_variable_2][:] # A hand calculated selection of values expected_col1 = numpy.ma.masked_invalid( [float('Nan'), float('Nan'), float('Nan'), 0.0814601778984, 0.0814601778984]) compare_masked_arrays(expected_col1, col_var1[:][0:5]) expected_col2 = numpy.ma.masked_invalid( [float('Nan'), float('Nan'), float('Nan'), 0.0741240680218, 0.0741240680218]) compare_masked_arrays(expected_col2, col_var2[:][0:5]) # Then do an evaluation using the collocated data: args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, 'collocated_gassp.nc'), "%s=gassp_alias:%s" % (valid_GASSP_aeroplane_variable, escape_colons(valid_GASSP_aeroplane_filename)), "(%s + %s) / gassp_alias " % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) self.ds.close() # Check correct self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = numpy.ma.masked_invalid([0.00196121983491, 0.00197255626472, 0.00120850731992]) assert_that(calculated_result.shape, is_((311,))) # Check the first 3 vald values compare_masked_arrays(expected_result, calculated_result[:][10:13]) os.remove('collocated_gassp.nc')
def test_collocated_NetCDF_Gridded_onto_GASSP(self): # First do a collocation of ECHAMHAM onto GASSP vars = valid_echamham_variable_1, valid_echamham_variable_2 filename = valid_echamham_filename sample_file = valid_GASSP_aeroplane_filename sample_var = valid_GASSP_aeroplane_variable collocator_and_opts = 'nn,variable=%s' % sample_var arguments = ['col', ",".join(vars) + ':' + filename, sample_file + ':collocator=' + collocator_and_opts, '-o', 'collocated_gassp'] main_arguments = parse_args(arguments) col_cmd(main_arguments) # Check collocation is the same self.ds = netCDF4.Dataset('collocated_gassp.nc') col_var1 = self.ds.variables[valid_echamham_variable_1][:] col_var2 = self.ds.variables[valid_echamham_variable_2][:] # A hand calculated selection of values expected_col1 = numpy.ma.masked_invalid( [float('Nan'), float('Nan'), float('Nan'), 0.0815568640828, 0.0815568640828]) compare_masked_arrays(expected_col1, col_var1[:][0:5]) expected_col2 = numpy.ma.masked_invalid( [float('Nan'), float('Nan'), float('Nan'), 0.0605319179595, 0.0605319179595]) compare_masked_arrays(expected_col2, col_var2[:][0:5]) # Then do an evaluation using the collocated data: args = ['eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, 'collocated_gassp.nc'), "%s=gassp_alias:%s" % (valid_GASSP_aeroplane_variable, valid_GASSP_aeroplane_filename), "(%s + %s) / gassp_alias " % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) self.ds.close() # Check correct self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = numpy.ma.masked_invalid([float('Nan'), float('Nan'), float('Nan'), 0.0004584692, 0.000697334]) assert_that(calculated_result.shape, is_((311,))) compare_masked_arrays(expected_result, calculated_result[:][0:5]) os.remove('collocated_gassp.nc')
def test_Aeronet_wavelength_calculation(self): # Example from the CIS Phase 3 Software spec: # ... a user should be able to write a plugin to calculate the Aeronet AOD at 550nm from the AOD at 500 nm as # AOD550 = AOD500 * (550/500)^(-1*Angstrom500-870)" # Takes 3s args = ['eval', 'AOT_500,500-870Angstrom=a550to870:' + escape_colons(another_valid_aeronet_filename), 'AOT_500 * (550.0/500)**(-1*a550to870)', '1', '-o', self.OUTPUT_FILENAME] arguments = parse_args(args) evaluate_cmd(arguments) # Check correct: self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] expected_result = [0.2341039087, 0.2285401152, 0.2228799533, 0.1953746746, 0.2094051561, 0.1696889668, 0.3137791803, 0.2798929273, 0.1664194279, 0.1254619092, 0.1258309124, 0.1496960031, 0.0768447737, 0.0550896430, 0.0534543107, 0.0538315909, 0.0666742975, 0.0512935449, 0.0699585189, 0.0645033944] assert_that(calculated_result.shape, is_((3140,))) assert numpy.allclose(expected_result, calculated_result[0:20])
def test_ECHAMHAM_wavelength_sum(self): args = [ 'eval', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), '%s+%s' % (valid_echamham_variable_1, valid_echamham_variable_2), '1', '-o', self.OUTPUT_FILENAME ] arguments = parse_args(args) evaluate_cmd(arguments) # Check correct: self.ds = netCDF4.Dataset(self.OUTPUT_FILENAME) calculated_result = self.ds.variables['calculated_variable'][:] # A hand calculated selection of values expected_result = [ 0.007633533, 0.007646653, 0.007749859, 0.007744226, 0.007761176 ] assert_that(calculated_result.shape, is_((96, 192))) assert numpy.allclose(expected_result, calculated_result[:][0][0:5])