def setUp(self):
# tmp file to aggregate to
_, self.nc_out_filename = tempfile.mkstemp()
pwd = os.path.dirname(__file__)
self.files = sorted(glob.glob(os.path.join(pwd, "data", "*.nc")))
self.config = Config.from_nc(self.files[0])
def test_collapse_second_dim(self):
config = Config.from_nc(self.inputs[0])
config.dims["b"].update({"flatten": True, "index_by": "b"})
l = generate_aggregation_list(config, self.inputs)
evaluate_aggregation_list(config, l, self.filename)
with nc.Dataset(self.filename) as nc_out: # type: nc.Dataset
# This is the more practically useful method of aggregation,
# where, for example, the dimension "a" might represent time
# and dim "b" is maybe satellite, or event, etc. (something that,
# at any point in time, there could be an arbitrary number of).
# flatten b dimension, should turn out like:
# [[0 -- --]
# [1 -- --]
# [2 -- --]
# [3 3 --]
# [4 4 --]
# [5 5 --]
# [6 6 6]
# [7 7 7]
# [8 8 8]]
c = nc_out.variables["c"][:]
self.assertEqual(c.shape, (9, 3))
self.assertEqual(np.sum(c), 90)
self.assertEqual(np.ma.count_masked(c), 9)
for i, a in enumerate(["a", "b", "c"]):
self.assertEqual(nc_out.variables["b"][i], a)
def setUpClass(cls):
super(TestEvaluateAggregationList, cls).setUpClass()
pwd = os.path.dirname(__file__)
cls.start_time = datetime(2017, 3, 16, 15, 27)
cls.end_time = datetime(2017, 3, 16, 15, 28)
cls.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
cls.config = Config.from_nc(cls.files[0])
cls.config.dims["report_number"].update({
"index_by": "OB_time",
"min": cls.
start_time, # for convenience, will convert according to index_by units if this is datetime
"max": cls.end_time,
"other_dim_indicies": {
"samples_per_record": 0
},
"expected_cadence": {
"report_number": 1,
"number_samples_per_report": 10,
},
})
_, cls.filename = tempfile.mkstemp()
agg_list = generate_aggregation_list(cls.config, cls.files)
logger.info(agg_list)
evaluate_aggregation_list(cls.config, agg_list, cls.filename)
cls.output = nc.Dataset(cls.filename, "r")
def setUp(self):
_, self.file = tempfile.mkstemp()
pwd = os.path.dirname(__file__)
self.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
self.config = Config.from_nc(self.files[0])
self.config.dims["time"].update(
{"index_by": "time", "expected_cadence": {"time": 10},}
)
def setUp(self):
_, self.file = tempfile.mkstemp()
pwd = os.path.dirname(__file__)
self.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
self.config = Config.from_nc(self.files[0])
self.config.dims["report_number"].update(
{
"index_by": "OB_time",
"other_dim_indicies": {"samples_per_record": 0},
"expected_cadence": {
"report_number": 1,
"number_samples_per_report": 10,
},
}
)
def setUpClass(cls):
super(TestEvaluateAggregationList, cls).setUpClass()
pwd = os.path.dirname(__file__)
cls.start_time = datetime(2017, 6, 8, 16, 45)
cls.end_time = datetime(2017, 6, 8, 16, 50)
cls.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
cls.config = Config.from_nc(cls.files[0])
cls.config.dims["report_number"].update(
{
"index_by": "L1a_SciData_TimeStamp",
"min": cls.start_time, # for convenience, will convert according to index_by units if this is datetime
"max": cls.end_time,
"expected_cadence": {"report_number": 1, "sensor_unit": 0},
}
)
_, cls.filename = tempfile.mkstemp()
agg_list = generate_aggregation_list(cls.config, cls.files)
evaluate_aggregation_list(cls.config, agg_list, cls.filename)
cls.output = nc.Dataset(cls.filename, "r")
def setUpClass(cls):
super(TestEvaluateAggregationList, cls).setUpClass()
pwd = os.path.dirname(__file__)
cls.start_time = datetime(2017, 4, 14, 19, 23)
cls.end_time = datetime(2017, 4, 14, 20, 30)
cls.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
cls.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
cls.config = Config.from_nc(cls.files[0])
cls.config.dims["time"].update(
{
"index_by": "time",
"min": cls.start_time, # for convenience, will convert according to index_by units if this is datetime
"max": cls.end_time,
"expected_cadence": {"time": 10},
}
)
_, cls.filename = tempfile.mkstemp()
agg_list = generate_aggregation_list(cls.config, cls.files)
evaluate_aggregation_list(cls.config, agg_list, cls.filename)
cls.output = nc.Dataset(cls.filename, "r")
def test_default_multi_dim(self):
config = Config.from_nc(self.inputs[0])
l = generate_aggregation_list(config, self.inputs)
evaluate_aggregation_list(config, l, self.filename)
with nc.Dataset(self.filename) as nc_out: # type: nc.Dataset
# this is the default aggregation produced by aggregation
# along both unlimited dimensions. This isn't really practically
# useful, but, by our "basic" definition of aggregation along unlitimed
# dimensions is correct. Need to make sure we get what's expected.
# [[0 -- -- -- -- --]
# [1 -- -- -- -- --]
# [2 -- -- -- -- --]
# [-- 3 3 -- -- --]
# [-- 4 4 -- -- --]
# [-- 5 5 -- -- --]
# [-- -- -- 6 6 6]
# [-- -- -- 7 7 7]
# [-- -- -- 8 8 8]]
c = nc_out.variables["c"][:]
self.assertEqual(c.shape, (9, 6))
self.assertEqual(np.sum(c), 90)
self.assertEqual(np.ma.count_masked(c), 36)
def setUpClass(cls):
super(TestEvaluateAggregationList, cls).setUpClass()
pwd = os.path.dirname(__file__)
cls.start_time = datetime(2018, 1, 17, 15, 5)
cls.end_time = datetime(2018, 1, 17, 15, 56)
cls.files = glob.glob(os.path.join(pwd, "data", "*.nc"))
cls.config = Config.from_nc(cls.files[0])
cls.config.dims["report_number"].update(
{
"index_by": "ELF_StartStopTime",
"min": cls.start_time, # for convenience, will convert according to index_by units if this is datetime
"max": cls.end_time,
"expected_cadence": {
"report_number": 1.0 / (5.0 * 60.0),
"number_of_time_bounds": 1.0 / ((5.0 * 60.0) - 1),
},
"size": None,
}
)
_, cls.filename = tempfile.mkstemp()
agg_list = generate_aggregation_list(cls.config, cls.files)
evaluate_aggregation_list(cls.config, agg_list, cls.filename)
cls.output = nc.Dataset(cls.filename, "r")
def setUp(self):
_, self.file = tempfile.mkstemp()
pwd = os.path.dirname(__file__)
self.files = glob.glob(os.path.join(pwd, "data", "*.nc"))[:2]
self.config = Config.from_nc(self.files[0])
def setUp(self):
self.config = Config.from_nc(test_input_file)
def setUp(self):
self.config = Config.from_nc(another_input_file)