def __init__(self,
metadata,
start_time,
end_time,
autoscaler_config_file=None,
metrics_client=None,
billing_frequency=timedelta(seconds=1),
refund_outbid=True) -> None:
""" Maintains all of the state for a clusterman simulation
:param metadata: a SimulationMetadata object
:param start_time: an arrow object indicating the start of the simulation
:param end_time: an arrow object indicating the end of the simulation
:param autoscaler_config_file: a filename specifying a list of existing SFRs or SFR configs
:param billing_frequency: a timedelta object indicating how often to charge for an instance
:param refund_outbid: if True, do not incur any cost for an instance lost to an outbid event
"""
self.autoscaler: Optional[Autoscaler] = None
self.metadata = metadata
self.metrics_client = metrics_client
self.start_time = start_time
self.current_time = start_time
self.end_time = end_time
self.instance_prices: Mapping[InstanceMarket,
PiecewiseConstantFunction] = defaultdict(
lambda: PiecewiseConstantFunction())
self.cost_per_hour: SimFn = PiecewiseConstantFunction()
self.aws_cpus: SimFn = PiecewiseConstantFunction()
self.mesos_cpus: SimFn = PiecewiseConstantFunction()
self.mesos_cpus_allocated: SimFn = PiecewiseConstantFunction()
self.markets: Set = set()
self.billing_frequency = billing_frequency
self.refund_outbid = refund_outbid
if autoscaler_config_file:
self._make_autoscaler(autoscaler_config_file)
self.aws_clusters = self.autoscaler.pool_manager.resource_groups.values(
) # type: ignore
period = self.autoscaler.signal.period_minutes # type: ignore
print(f'Autoscaler configured; will run every {period} minutes')
else:
self.aws_clusters = [SimulatedAWSCluster(self)]
print('No autoscaler configured; using metrics for cluster size')
# The event queue holds all of the simulation events, ordered by time
self.event_queue: List[Event] = []
# Don't use add_event here or the end_time event will get discarded
heappush(self.event_queue,
Event(self.start_time, msg='Simulation begins'))
heappush(self.event_queue, Event(self.end_time, msg='Simulation ends'))
def spot_prices():
instance_price = defaultdict(lambda: PiecewiseConstantFunction())
instance_price[MARKETS[0]].add_breakpoint(arrow.get(0), 0.5)
instance_price[MARKETS[1]].add_breakpoint(arrow.get(0), 2.5)
instance_price[MARKETS[2]].add_breakpoint(arrow.get(0), 0.1)
instance_price[MARKETS[3]].add_breakpoint(arrow.get(0), 0.5)
return instance_price
def test_combine_with_breakpoints_in_one_fn(op):
fn1 = PiecewiseConstantFunction(1)
fn1.add_breakpoint(2, 7)
fn1.add_breakpoint(4, 4)
fn1.add_breakpoint(7, 1)
fn2 = PiecewiseConstantFunction(2)
fn3 = op(fn1, fn2)
assert fn3._initial_value == op(fn1._initial_value, fn2._initial_value)
assert fn3.breakpoints[2] == op(fn1.breakpoints[2], fn2._initial_value)
assert fn3.breakpoints[4] == op(fn1.breakpoints[4], fn2._initial_value)
assert fn3.breakpoints[7] == op(fn1.breakpoints[7], fn2._initial_value)
def _make_mock_simulator():
instance_prices = defaultdict(lambda: PiecewiseConstantFunction())
instance_prices[_MARKETS[0]].add_breakpoint(arrow.get(0), 0.5)
instance_prices[_MARKETS[1]].add_breakpoint(arrow.get(0), 0.7)
instance_prices[_MARKETS[2]].add_breakpoint(arrow.get(0), 0.6)
instance_prices[_MARKETS[3]].add_breakpoint(arrow.get(0), 0.55)
instance_prices[_MARKETS[4]].add_breakpoint(arrow.get(0), 0.65)
instance_prices[_MARKETS[5]].add_breakpoint(arrow.get(0), 0.75)
instance_prices[_MARKETS[6]].add_breakpoint(arrow.get(0), 0.8)
instance_prices[_MARKETS[7]].add_breakpoint(arrow.get(0), 0.9)
return mock.Mock(
instance_prices=instance_prices,
current_time=arrow.get(0),
)
def get_data(
self,
key: str,
start_time: Optional[Arrow] = None,
end_time: Optional[Arrow] = None,
step: Optional[timedelta] = None,
) -> 'SortedDict[Arrow, float]':
""" Compute the capacity for the cluster in the specified time range, grouped into chunks
:param key: the type of data to retreive; must correspond to a key in REPORT_TYPES
:param start_time: the lower bound of the range (if None, use simulation start time)
:param end_time: the upper bound of the range (if None, use simulation end time)
:param step: the width of time for each chunk
:returns: a list of CPU capacities for the cluster from start_time to end_time
"""
start_time = start_time or self.start_time
end_time = end_time or self.end_time
if key == 'cpus':
return self.mesos_cpus.values(start_time, end_time, step)
elif key == 'cpus_allocated':
return self.mesos_cpus_allocated.values(start_time, end_time, step)
elif key == 'unused_cpus':
# If an agent hasn't joined the cluster yet, we'll treat it as "unused" in the simulation
unused_cpus = self.aws_cpus - self.mesos_cpus_allocated
return unused_cpus.values(start_time, end_time, step)
elif key == 'cost':
return self.cost_per_hour.integrals(start_time,
end_time,
step,
transform=hour_transform)
elif key == 'unused_cpus_cost':
# Here we treat CPUs that haven't joined the Mesos cluster as un-allocated. It's arguable
# if that's the right way to do this or not.
percent_unallocated = (self.aws_cpus -
self.mesos_cpus_allocated) / self.aws_cpus
percent_cost = percent_unallocated * self.cost_per_hour
return percent_cost.integrals(start_time,
end_time,
step,
transform=hour_transform)
elif key == 'cost_per_cpu':
cost_per_cpu = self.cost_per_hour / self.aws_cpus
return cost_per_cpu.values(start_time, end_time, step)
elif key == 'oversubscribed':
max_fn = piecewise_max(self.mesos_cpus_allocated - self.aws_cpus,
PiecewiseConstantFunction())
return max_fn.values(start_time, end_time, step)
else:
raise ValueError(f'Data key {key} is not recognized')
def fn():
return PiecewiseConstantFunction(1)
def test_piecewise_max():
fn1 = PiecewiseConstantFunction(1)
fn1.add_breakpoint(2, 7)
fn1.add_breakpoint(4, 4)
fn1.add_breakpoint(7, 1)
fn2 = PiecewiseConstantFunction(2)
fn2.add_breakpoint(-1, 3)
fn2.add_breakpoint(7, 1)
fn2.add_breakpoint(8, 0)
fn3 = piecewise_max(fn1, fn2)
assert fn3._initial_value == 2
assert fn3.breakpoints[-1] == 3
assert fn3.breakpoints[2] == 7
assert fn3.breakpoints[4] == 4
assert fn3.breakpoints[7] == 1
def test_combine_with_breakpoints_in_both_fns(op):
fn1 = PiecewiseConstantFunction(1)
fn1.add_breakpoint(2, 7)
fn1.add_breakpoint(4, 4)
fn1.add_breakpoint(7, 1)
fn2 = PiecewiseConstantFunction(2)
fn2.add_breakpoint(-1, 4)
fn2.add_breakpoint(7, 1)
fn2.add_breakpoint(8, 0)
fn3 = op(fn1, fn2)
assert fn3._initial_value == op(fn1._initial_value, fn2._initial_value)
assert fn3.breakpoints[-1] == op(fn1._initial_value, fn2.breakpoints[-1])
assert fn3.breakpoints[2] == op(fn1.breakpoints[2], fn2.breakpoints[-1])
assert fn3.breakpoints[4] == op(fn1.breakpoints[4], fn2.breakpoints[-1])
assert fn3.breakpoints[7] == op(fn1.breakpoints[7], fn2.breakpoints[7])
try:
assert fn3.breakpoints[8] == op(fn1.breakpoints[7], fn2.breakpoints[8])
except ZeroDivisionError:
assert fn3.breakpoints[8] == 0
def test_combine_no_breakpoints(op):
fn1 = PiecewiseConstantFunction(1)
fn2 = PiecewiseConstantFunction(2)
fn3 = op(fn1, fn2)
assert fn3._initial_value == op(fn1._initial_value, fn2._initial_value)
assert len(fn3.breakpoints) == 0
def test_one_step_integral_two_changes():
fn = PiecewiseConstantFunction()
fn.add_delta(1, 10)
fn.add_delta(1, -5)
assert fn.integral(0, 2) == 5
def test_one_step_integral(xval, result):
fn = PiecewiseConstantFunction()
fn.add_delta(xval, 1)
assert fn.integral(0, 2) == result
assert fn.integral(3, 4) == 1
assert fn.integral(-2, -1) == 0
def test_constant_integral(initial_value):
fn = PiecewiseConstantFunction(initial_value)
assert fn.integral(0, 1) == initial_value