class TestBufferIngestDataStream(unittest.TestCase):
def setUp(self):
"""
Returns
-------
"""
"""
setup the buffer and do config stuff
:return: Nothing
"""
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(self.env, self.cluster, self.config)
self.observation = Observation(
name='test_observation',
start=OBS_START_TME,
duration=OBS_DURATION,
demand=OBS_DEMAND,
workflow=OBS_WORKFLOW,
data_rate=5,
)
def testBasicIngest(self):
"""
Test the 'ingest_data_stream' event, which is to be called in the
Scheduler. The changes we expect in this are simple - after n
timesteps, the HotBuffer.currect_capacity will have reduced
n*observation_ingest_rate.
We test a couple of requirements here.
"""
self.observation.status = RunStatus.RUNNING
ret = self.env.process(
self.buffer.ingest_data_stream(
self.observation
)
)
self.env.run(until=1)
self.assertEqual(495, self.buffer.hot[BUFFER_ID].current_capacity)
self.env.run(until=10)
self.assertEqual(self.env.now, 10)
self.assertEqual(450, self.buffer.hot[BUFFER_ID].current_capacity)
self.assertEqual(
self.buffer.hot[BUFFER_ID].observations["stored"][0],
self.observation
)
def testIngestPrerequisites(self):
"""
In order to call ingest_data_stream, the observation must be marked as
"RunStatus.RUNNING" - otherwise we will be processing an observation
that hasn't started!
-------
"""
ret = self.env.process(
self.buffer.ingest_data_stream(
self.observation
)
)
self.assertRaises(
RuntimeError, self.env.run, until=1
)
def testIngestObservationNotRunning(self):
"""
The buffer won't ingest if the observation is not marked as
RunStatus.RUNNING
"""
self.assertEqual(RunStatus.WAITING, self.observation.status)
self.env.process(self.buffer.ingest_data_stream(self.observation))
self.assertRaises(
RuntimeError, self.env.run, until=1
)
#
def testIngestEdgeCase(self):
"""
Buffer must accept ingest at rate up to 'max ingest data rate' but
raises an exception if the ingest rate for an observation is greater
(this means we have an error).
In addition, we are coordinating this ingest between the scheduler and
the telescope and the cluster so these actors also need to work
together in some way, which this test will also attempt to do .
"""
self.observation.status = RunStatus.RUNNING
self.observation.ingest_data_rate = 20
ret = self.env.process(
self.buffer.ingest_data_stream(
self.observation
)
)
self.assertRaises(ValueError, self.env.run, until=1)
def test_ingest_capacity_checks(self):
"""
The buffer checks the hot and cold buffer for capacity;
need to make sure that if either the hot buffer or the cold buffer do
not have enough room for the observation, it is not scheduled.
Returns
-------
"""
self.buffer.hot[BUFFER_ID].current_capacity = 2
self.assertFalse(
self.buffer.check_buffer_capacity(self.observation)
)
self.buffer.hot[BUFFER_ID].current_capacity = 100
self.buffer.cold[BUFFER_ID].current_capacity = 1
self.assertFalse(
self.buffer.check_buffer_capacity(self.observation)
)
class TestBufferIngestDataStream(unittest.TestCase):
def setUp(self):
"""
Returns
-------
"""
"""
setup the buffer and do config stuff
:return: Nothing
"""
self.env = simpy.Environment()
self.cluster = Cluster(self.env, CLUSTER_CONFIG)
self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG)
self.observation = Observation(
name='test_observation',
start=OBS_START_TME,
duration=OBS_DURATION,
demand=OBS_DEMAND,
workflow=OBS_WORKFLOW,
type='continuum',
data_rate=5,
)
def testBasicIngest(self):
"""
Test the 'ingest_data_stream' event, which is to be called in the
Scheduler. The changes we expect in this are simple - after n
timesteps, the HotBuffer.currect_capacity will have reduced
n*observation_ingest_rate.
"""
self.observation.status = RunStatus.RUNNING
ret = self.env.process(self.buffer.ingest_data_stream(
self.observation))
self.env.run(until=1)
self.assertEqual(495, self.buffer.hot.current_capacity)
self.env.run(until=10)
self.assertEqual(450, self.buffer.hot.current_capacity)
self.assertEqual(self.buffer.hot.stored_observations[0],
self.observation)
def testIngestObservationNotRunning(self):
"""
The buffer won't ingest if the observation is not marked as
RunStatus.RUNNING
"""
self.assertEqual(RunStatus.WAITING, self.observation.status)
self.env.process(self.buffer.ingest_data_stream(self.observation))
# self.assertRaises(
# RuntimeError, self.env.process, self.buffer.ingest_data_stream(
# self.observation
# )
# )
self.assertRaises(RuntimeError, self.env.run, until=1)
# self.assertEqual(500, self.buffer.hot.current_capacity)
def testIngestEdgeCase(self):
"""
class TestBufferRequests(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(
env=self.env, cluster=self.cluster, config=self.config
)
self.planner = Planner(
self.env, PLAN_ALGORITHM, self.cluster, SHADOWPlanning('heft')
)
self.observation = Observation(
'scheduler_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
data_rate=2
)
def tearDown(self):
pass
def test_buffer_hot_to_cold(self):
"""
This tests an ingest, and then, once we have a successful ingest,
the movement from one buffer to the other.
Using the current situation, we should have the observation finished by
timestep [TBC], and then the observation moved across by timestep [TBC]
Returns
-------
"""
self.observation.status = RunStatus.RUNNING
self.env.process(self.buffer.ingest_data_stream(self.observation))
self.env.run(until=10)
self.assertEqual(480, self.buffer.hot[BUFFER_ID].current_capacity)
# Moving data from one to the other
self.assertEqual(250, self.buffer.cold[BUFFER_ID].current_capacity)
self.assertTrue(self.observation in
self.buffer.hot[BUFFER_ID].observations["stored"])
self.env.process(self.buffer.move_hot_to_cold(0))
self.env.run(until=15)
self.assertEqual(240, self.buffer.cold[BUFFER_ID].current_capacity)
self.assertEqual(490, self.buffer.hot[BUFFER_ID].current_capacity)
self.env.run(until=20)
self.assertEqual(230, self.buffer.cold[BUFFER_ID].current_capacity)
self.env.run(until=22)
self.assertEqual(500, self.buffer.hot[BUFFER_ID].current_capacity)
self.assertEqual(230, self.buffer.cold[BUFFER_ID].current_capacity)
self.assertListEqual(
[self.observation],
self.buffer.cold[BUFFER_ID].observations['stored']
)
def test_hot_transfer_observation(self):
"""
When passed an observation, over a period of time ensure that the
complete data set is removed.
Only when all data has finished being transferred do we add the
observation to ColdBuffer.observations.
Observation duration is 10; ingest rate is 5.
Observation.total_data_size => 50
ColdBuffer.max_data_rate => 2; therefore
Time until Observation is moved => 25.
Returns
-------
"""
self.buffer.hot[BUFFER_ID].current_capacity = 450
self.observation.total_data_size = 50
data_left_to_transfer = self.observation.total_data_size
self.buffer.hot[BUFFER_ID].observations["stored"].append(
self.observation)
data_left_to_transfer = self.buffer.hot[BUFFER_ID].transfer_observation(
self.observation,
self.buffer.cold[BUFFER_ID].max_data_rate,
data_left_to_transfer
)
self.assertEqual(48, data_left_to_transfer)
self.assertTrue(
self.observation in self.buffer.hot[BUFFER_ID].observations[
"stored"]
)
self.assertEqual(452, self.buffer.hot[BUFFER_ID].current_capacity)
timestep = 24
while data_left_to_transfer > 0:
data_left_to_transfer = self.buffer.hot[
BUFFER_ID].transfer_observation(
self.observation,
self.buffer.cold[BUFFER_ID].max_data_rate,
data_left_to_transfer
)
self.assertEqual(0, data_left_to_transfer)
self.assertEqual(500, self.buffer.hot[BUFFER_ID].current_capacity)
def test_cold_receive_data(self):
"""
When passed an observation, over a period of time ensure that the
complete data set is added to the Cold Buffer.
Only when all data has finished being transferred do we add the
observation to ColdBuffer.observations.
Observation duration is 10; ingest rate is 5.
Observation.total_data_size => 50
ColdBuffer.max_data_rate => 2; therefore
Time until Observation is moved => 25.
Returns
-------
"""
self.observation.total_data_size = 50
data_left_to_transfer = self.observation.total_data_size
data_left_to_transfer = self.buffer.cold[BUFFER_ID].receive_observation(
self.observation,
data_left_to_transfer
)
self.assertEqual(48, data_left_to_transfer)
self.assertFalse(
self.observation in self.buffer.cold[BUFFER_ID].observations[
'stored']
)
while data_left_to_transfer > 0:
data_left_to_transfer = self.buffer.cold[
BUFFER_ID].receive_observation(
self.observation,
data_left_to_transfer
)
self.assertTrue(
self.observation in self.buffer.cold[BUFFER_ID].observations[
'stored']
)
self.assertEqual(None,
self.buffer.cold[BUFFER_ID].observations['transfer'])
# @unittest.skip("Functionality has changed")
def testWorkflowAddedToQueue(self):
"""
We only add a workflow to the queue once an observation has finished
(and, therefore, after we have finished generating a plan for it).
:return: None
"""
# Calling planner.run() will store the generate plan in the observation object
# calling next() runs the iterator immediately after generator is
# called
self.observation.ast = 0
self.observation.plan = self.planner.run(self.observation,
self.buffer, None)
self.assertTrue(self.observation.plan is not None)
class TestColdBufferRequests(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=BUFFER_CONFIG)
self.planner = Planner(self.env, PLAN_ALGORITHM, self.cluster)
self.observation = Observation('scheduler_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type='continuum',
data_rate=2)
def tearDown(self):
pass
def testHotColdInteraction(self):
"""
Testing the results of running 'buffer.request_data_from(observation)'.
Returns
-------
"""
# TODO THIS NEED TO CHANGE
# TODO Hot Cold transfer should be automatic, not instigated by the
# scheduler. THis ensures that the scheduler only needs to check the
# cold buffer, and that movement of data from the hot buffer to the
# cold buffer is 'automatic' (that is, once data has been through
# the hot buffer completely and INGEST run on that data, we can move
# it to a large buffer store).
# Prelimns
self.observation.status = RunStatus.RUNNING
self.env.process(self.buffer.ingest_data_stream(self.observation))
self.env.run(until=10)
self.assertEqual(480, self.buffer.hot.current_capacity)
# Moving data from one to the other
self.assertEqual(250, self.buffer.cold.current_capacity)
self.env.process(self.buffer.request_data_from(self.observation))
self.env.run(until=15)
self.assertEqual(240, self.buffer.cold.current_capacity)
self.assertEqual(490, self.buffer.hot.current_capacity)
self.env.run(until=40)
self.assertEqual(230, self.buffer.cold.current_capacity)
self.assertEqual(500, self.buffer.hot.current_capacity)
self.assertListEqual([self.observation], self.buffer.cold.observations)
def testHotColdErrors(self):
"""
We haven't processed the observation yet, so there shouldn't be
anything in the Hot Buffer to request
"""
self.env.process(self.buffer.request_data_from(self.observation))
self.assertRaises(
RuntimeError,
self.env.run,
until=10,
)
def testWorkflowAddedToQueue(self):
"""
We only add a workflow to the queue once an observation has finished
(and, therefore, after we have finished generating a plan for it).
:return: None
"""
# Calling planner.run() will store the generate plan in the observation object
# calling next() runs the iterator immediately after generator is called
next(self.planner.run(self.observation))
# Buffer observation queue should be empty
self.assertTrue(self.buffer.observations_for_processing.empty())
# self.buffer.add_observation_to_waiting_workflows(self.observation)
self.assertTrue(self.buffer.observations_for_processing.size() == 1)