def test_unknown_output_type(self):
"""Test what happens if you give an unknown output type."""
plugin = SpotDatabase("kitten", self.data_directory + "/test.csv",
"improver", "time", "index")
message = ("Unrecognised output type. Current options are 'sqlite'"
" or 'csv', 'kitten' given.")
with self.assertRaisesRegexp(ValueError, message):
plugin.process(self.cubes)
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
second_cube = self.cube.copy()
second_cube.coord("percentile").points = np.array([60.0])
cubelist = CubeList([self.cube, second_cube])
self.cubes = cubelist.concatenate()
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"IMPRO", 0)
def test_no_exception_if_pivot_dim_set(self):
"""Test it raises an exception if a 2D cube is input and
no pivot_dim."""
plugin = SpotDatabase("csv",
"output",
"improver",
"time",
"IMPRO",
0,
pivot_dim="percentile")
self.assertFalse(plugin.check_input_dimensions(self.cubes[0]))
class Test_map_primary_index(IrisTest):
"""Test the map_primary_index method"""
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(
280,
number_of_sites=1,
)
self.plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
"index",
primary_map=['validity_date', 'validity_time'],
primary_func=[
lambda x: dt.utcfromtimestamp(x).date(),
lambda x: dt.utcfromtimestamp(x).hour * 100
])
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
self.input_df = self.input_df.set_index(["time"])
def test_single_cube(self):
"""Basic test using one input cube."""
# Set up expected dataframe.
validity_date = dt.utcfromtimestamp(1487311200).date()
expected_data = [[validity_date, 600, 280.]]
columns = ['validity_date', 'validity_time', "values"]
expected_df = pd.DataFrame(expected_data, columns=columns)
expected_df = expected_df.set_index(["validity_date", "validity_time"])
# Call the method.
self.plugin.map_primary_index(self.input_df)
assert_frame_equal(expected_df, self.input_df)
def test_multiple_times_cube(self):
"""Test using one input cube, with one site and multiple times."""
# Set up expected dataframe.
validity_date = dt.utcfromtimestamp(1487311200).date()
expected_data = [[validity_date, 600, 280.],
[validity_date, 700, 281.]]
columns = ['validity_date', 'validity_time', "values"]
expected_df = pd.DataFrame(expected_data, columns=columns)
expected_df = expected_df.set_index(["validity_date", "validity_time"])
# Set up input dataframe
data = [[1487311200, 280.], [1487311200 + 3600, 281.]]
columns = ["time", "values"]
input_df = pd.DataFrame(data, columns=columns)
input_df = input_df.set_index(["time"])
self.plugin.map_primary_index(input_df)
assert_frame_equal(expected_df, input_df)
def test_repr_after_dataframe_created(self):
"""Basic test of string representation after a database has been
created and added to self."""
expected_result = ('<SpotDatabase: csv, output, improver, time, index,'
' primary_map=None, '
'primary_func=None, '
'pivot_dim=None, '
'pivot_map=None, '
'column_dims=None, '
'column_maps=None>')
plugin = SpotDatabase("csv", "output", "improver", "time", "index")
plugin.df = pd.DataFrame()
result = str(plugin)
self.assertEqual(expected_result, result)
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
"index",
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
pivot_dim="forecast_period")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
class Test_determine_schema(IrisTest):
"""A set of tests for the determine_schema method"""
def setUp(self):
"""Set up the plugin and dataframe needed for this test"""
cubes = CubeList([set_up_spot_cube(280)])
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
self.dataframe = self.plugin.to_dataframe(cubes)
def test_full_schema(self):
"""Basic test using a basic dataframe as input"""
schema = self.plugin.determine_schema("improver")
expected_schema = 'CREATE TABLE "improver" (\n"index" INTEGER,\n '\
'"values" REAL,\n CONSTRAINT improver_pk '\
'PRIMARY KEY ("index")\n)'
self.assertEqual(schema, expected_schema)
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.cube2 = set_up_spot_cube(281,
number_of_sites=1,
validity_time=1487311200 + 3600,
forecast_period=1)
self.cube3 = set_up_spot_cube(282,
number_of_sites=1,
validity_time=1487311200 + 7200,
forecast_period=2)
self.cubelist = CubeList([self.cube])
self.cubelist_multiple = CubeList([self.cube, self.cube2, self.cube3])
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(
280,
number_of_sites=1,
)
self.plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
"index",
primary_map=['validity_date', 'validity_time'],
primary_func=[
lambda x: dt.utcfromtimestamp(x).date(),
lambda x: dt.utcfromtimestamp(x).hour * 100
])
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
self.input_df = self.input_df.set_index(["time"])
def test_basic_repr(self):
"""Basic test of string representation"""
expected_result = ('<SpotDatabase: csv, output, improver, time, index,'
' primary_map=None, '
'primary_func=None, '
'pivot_dim=None, '
'pivot_map=None, '
'column_dims=None, '
'column_maps=None>')
result = str(SpotDatabase("csv", "output", "improver", "time",
"index"))
self.assertEqual(expected_result, result)
def test_exception_if_pivot_dim_set(self):
"""Test it raises an exception if a 2D cube is input and
no pivot_dim."""
plugin = SpotDatabase("csv",
"output",
"improver",
"time",
"IMPRO",
0,
pivot_dim="percentile")
cube = set_up_spot_cube(280, number_of_sites=3)
second_cube = cube.copy()
second_cube.coord("percentile").points = np.array([60.0])
cubelist = CubeList([cube, second_cube])
cubes = cubelist.concatenate()
message = "Dimensions that are not described by the pivot_dim or "\
"coord_to_slice_over must only have one point in. "\
"Dimension '2' has length '3' and is associated with the "\
"'index' coordinate."
with self.assertRaisesRegex(ValueError, message):
plugin.check_input_dimensions(cubes[0])
def test_all_optional_args(self):
"""Test we create a datafram even when we have all optional
arguements set"""
# Set up expected dataframe.
data = [[600, "air_temperature", 280.]]
columns = ["validity_time", "cf_name", "T+000"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["validity_time", "cf_name"], inplace=True)
expected_df.columns.name = "forecast_period"
plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
primary_map=['validity_time'],
primary_func=[lambda x: dt.utcfromtimestamp(x).hour * 100],
pivot_dim='forecast_period',
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
column_dims=['name'],
column_maps=['cf_name'],
coord_to_slice_over="index")
# Call the method.
plugin.to_dataframe(self.cubelist)
assert_frame_equal(plugin.df, expected_df)
def setUp(self):
"""Set up the plugin and dataframe needed for this test"""
self.cubes = CubeList([set_up_spot_cube(280)])
self.data_directory = mkdtemp()
self.plugin = SpotDatabase("csv", self.data_directory + "/test.csv",
"improver", "time", "index")
def setUp(self):
"""Set up the plugin and dataframe needed for this test"""
cubes = CubeList([set_up_spot_cube(280)])
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
self.dataframe = self.plugin.to_dataframe(cubes)
class Test_to_dataframe(IrisTest):
"""Test the to_dataframe method"""
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.cube2 = set_up_spot_cube(281,
number_of_sites=1,
validity_time=1487311200 + 3600,
forecast_period=1)
self.cube3 = set_up_spot_cube(282,
number_of_sites=1,
validity_time=1487311200 + 7200,
forecast_period=2)
self.cubelist = CubeList([self.cube])
self.cubelist_multiple = CubeList([self.cube, self.cube2, self.cube3])
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
@ManageWarnings(ignored_messages=IGNORED_MESSAGES,
warning_types=WARNING_TYPES)
def test_no_optional_args(self):
"""Test we create a datafram even when we have no optional
arguements set"""
# Set up expected dataframe.
data = [[280.]]
columns = ["values"]
expected_df = pd.DataFrame(data, index=[1487311200], columns=columns)
# Call the method.
self.plugin.to_dataframe(self.cubelist)
assert_frame_equal(self.plugin.df, expected_df)
@ManageWarnings(ignored_messages=IGNORED_MESSAGES,
warning_types=WARNING_TYPES)
def test_all_optional_args(self):
"""Test we create a datafram even when we have all optional
arguements set"""
# Set up expected dataframe.
data = [[600, "air_temperature", 280.]]
columns = ["validity_time", "cf_name", "T+000"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["validity_time", "cf_name"], inplace=True)
expected_df.columns.name = "forecast_period"
plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
primary_map=['validity_time'],
primary_func=[lambda x: dt.utcfromtimestamp(x).hour * 100],
pivot_dim='forecast_period',
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
column_dims=['name'],
column_maps=['cf_name'],
coord_to_slice_over="index")
# Call the method.
plugin.to_dataframe(self.cubelist)
assert_frame_equal(plugin.df, expected_df)
@ManageWarnings(ignored_messages=IGNORED_MESSAGES,
warning_types=WARNING_TYPES)
def test_all_optional_args_multiple_input_cubes(self):
"""Test we create a dataframe even when we have no optional
arguements set and multiple cubes"""
# Set up expected dataframe.
data = [[600, "air_temperature", 280., np.nan, np.nan],
[700, "air_temperature", np.nan, 281., np.nan],
[800, "air_temperature", np.nan, np.nan, 282.]]
columns = ["validity_time", "cf_name", "T+000", "T+001", "T+002"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["validity_time", "cf_name"], inplace=True)
expected_df.columns.name = "forecast_period"
plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
primary_map=['validity_time'],
primary_func=[lambda x: dt.utcfromtimestamp(x).hour * 100],
pivot_dim='forecast_period',
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
column_dims=['name'],
column_maps=['cf_name'],
coord_to_slice_over="index")
# Call the method.
plugin.to_dataframe(self.cubelist_multiple)
assert_frame_equal(plugin.df, expected_df)
@staticmethod
@ManageWarnings(ignored_messages=IGNORED_MESSAGES,
warning_types=WARNING_TYPES)
def test_all_optional_args_multiple_sites():
"""Test we create a datafram even when we have all optional
arguements set and multiple spots"""
# Set up expected dataframe.
data = [[600, "air_temperature", 0, 280.],
[600, "air_temperature", 1, 280.],
[600, "air_temperature", 2, 280.]]
columns = ["validity_time", "cf_name", "site", "T+000"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["validity_time", "cf_name", "site"],
inplace=True)
expected_df.columns.name = "forecast_period"
plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
primary_map=['validity_time'],
primary_func=[lambda x: dt.utcfromtimestamp(x).hour * 100],
pivot_dim='forecast_period',
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
column_dims=['name', "index"],
column_maps=['cf_name', "site"],
coord_to_slice_over="index")
# Call the method.
cube = set_up_spot_cube(
280,
number_of_sites=3,
)
plugin.to_dataframe(CubeList([cube]))
assert_frame_equal(plugin.df, expected_df)
class Test_insert_extra_mapped_column(IrisTest):
"""Test the insert_extra_mapped_column method"""
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.plugin = SpotDatabase("csv", "output", "improver", "time",
"index")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
def test_extra_column_in_df(self):
"""Basic test using a column and dim for something that already exist.
In this case the function does nothing to the dataframe."""
# Set up expected dataframe.
data = [[1487311200, 280.]]
columns = ["time", "values"]
expected_df = pd.DataFrame(data, columns=columns)
# Call the method.
self.plugin.insert_extra_mapped_column(self.input_df, self.cube,
"values", [380.])
assert_frame_equal(self.input_df, expected_df)
def test_extra_column_from_coord(self):
"""Test for when we are taking data from a coordinate in the cube"""
data = [[1487311200, 100, 280.]]
columns = ["time", "altitude_of_site", "values"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["altitude_of_site"], append=True, inplace=True)
# Call the method.
self.plugin.insert_extra_mapped_column(self.input_df, self.cube,
"altitude", "altitude_of_site")
assert_frame_equal(self.input_df, expected_df)
def test_extra_column_from_coord_multiple_points(self):
"""Test for when we are taking data from a coordinate in the cube.
This test has multiple sites in the cube."""
data = [[1487311200, 100, 280.], [1487311200, 101, 280.],
[1487311200, 102, 280.]]
columns = ["time", "altitude_of_site", "values"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["altitude_of_site"], append=True, inplace=True)
# Call the method.
cube = set_up_spot_cube(280, number_of_sites=3)
data = [[1487311200, 280.], [1487311200, 280.], [1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
self.plugin.insert_extra_mapped_column(self.input_df, cube, "altitude",
"altitude_of_site")
assert_frame_equal(self.input_df, expected_df)
def test_extra_column_from_cube_name(self):
"""Test for when we are taking data from the cube name."""
data = [[1487311200, "air_temperature", 280.]]
columns = ["time", "name_of_cube", "values"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["name_of_cube"], append=True, inplace=True)
# Call the method.
self.plugin.insert_extra_mapped_column(self.input_df, self.cube,
"name", "name_of_cube")
assert_frame_equal(self.input_df, expected_df)
def test_extra_column_from_cube_attribute(self):
"""Test for when we are taking data from the cube name."""
data = [[1487311200, "K", 280.]]
columns = ["time", "cube_units", "values"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["cube_units"], append=True, inplace=True)
# Call the method.
self.plugin.insert_extra_mapped_column(self.input_df, self.cube,
"units", "cube_units")
assert_frame_equal(self.input_df, expected_df)
def test_static_extra_column(self):
"""Test for when we add a new column that does not come from the cube.
"""
data = [[1487311200, "IMPRO_nbhood", 280.]]
columns = ["time", "experiment_id", "values"]
expected_df = pd.DataFrame(data, columns=columns)
expected_df.set_index(["experiment_id"], append=True, inplace=True)
# Call the method.
self.cube.attributes["source_grid"] = "ukvx"
self.plugin.insert_extra_mapped_column(self.input_df, self.cube,
"IMPRO_nbhood", "experiment_id")
assert_frame_equal(self.input_df, expected_df)
class Test_pivot_table(IrisTest):
"""Test the pivot_table method"""
def setUp(self):
"""Set up the plugin and dataframe needed for these tests"""
self.cube = set_up_spot_cube(280, number_of_sites=1)
self.plugin = SpotDatabase(
"csv",
"output",
"improver",
"time",
"index",
pivot_map=lambda x: 'T+{:03d}'.format(int(x / 3600)),
pivot_dim="forecast_period")
data = [[1487311200, 280.]]
columns = ["time", "values"]
self.input_df = pd.DataFrame(data, columns=columns)
def test_single_cube(self):
"""Basic test using one input cube."""
# Set up expected dataframe.
expected_data = [[280.]]
expected_df = pd.DataFrame(expected_data, columns=["T+000"])
expected_df.columns.name = "forecast_period"
# Call the method.
result = self.plugin.pivot_table(self.cube, self.input_df)
assert_frame_equal(expected_df, result)
def test_multiple_times_cube(self):
"""Test using one input cube, with one site and multiple times."""
# Set up expected dataframe.
expected_data = [[280., np.nan], [np.nan, 281.]]
expected_df = pd.DataFrame(expected_data, columns=["T+000", "T+001"])
expected_df.columns.name = "forecast_period"
# Set up cube with multiple lead times in.
second_cube = set_up_spot_cube(
281,
number_of_sites=1,
validity_time=1487311200 + 3600,
forecast_period=1,
)
merged_cube = CubeList([self.cube, second_cube])
merged_cube = merged_cube.concatenate()
# Set up input dataframe
data = [[1487311200, 280.], [1487311200 + 3600, 281.]]
columns = ["time", "values"]
input_df = pd.DataFrame(data, columns=columns)
result = self.plugin.pivot_table(merged_cube[0], input_df)
assert_frame_equal(expected_df, result)
def test_extra_columns(self):
"""Test with an extra column in the input dataframe.
In this case the extra column gets removed when the table
is pivoted.
"""
# Set up expected dataframe.
expected_data = [[280., np.nan], [np.nan, 281.]]
expected_df = pd.DataFrame(expected_data, columns=["T+000", "T+001"])
expected_df.columns.name = "forecast_period"
# Set up cube with multiple lead times in.
second_cube = set_up_spot_cube(
281,
number_of_sites=1,
validity_time=1487311200 + 3600,
forecast_period=1,
)
merged_cube = CubeList([self.cube, second_cube])
merged_cube = merged_cube.concatenate()
# Set up input dataframe
data = [[1487311200, 3001, 280.], [1487311200 + 3600, 3002, 281.]]
columns = ["time", "wmo_site", "values"]
input_df = pd.DataFrame(data, columns=columns)
result = self.plugin.pivot_table(merged_cube[0], input_df)
assert_frame_equal(expected_df, result)
