def test_market_capacities(mock_spot_fleet_resource_group, mock_subnet): assert mock_spot_fleet_resource_group.market_capacities == { InstanceMarket('c3.8xlarge', mock_subnet['Subnet']['AvailabilityZone']): 8, InstanceMarket('i2.4xlarge', mock_subnet['Subnet']['AvailabilityZone']): 3, }
def cluster(simulator): cluster = SimulatedAWSCluster(simulator) cluster.simulator.current_time.shift(seconds=+42) cluster.modify_size({ InstanceMarket('m4.4xlarge', 'us-west-1a'): 4, InstanceMarket('i2.8xlarge', 'us-west-1a'): 2, InstanceMarket('i2.8xlarge', 'us-west-2a'): 1, }) cluster.ebs_storage += 3000 return cluster
def test_modify_size(cluster): cluster.simulator.current_time.shift(seconds=+76) added_instances, removed_instances = cluster.modify_size({ InstanceMarket('m4.4xlarge', 'us-west-1a'): 1, InstanceMarket('i2.8xlarge', 'us-west-1a'): 4, }) assert len(added_instances) == 2 assert len(removed_instances) == 4 assert len(cluster) == 5
def test_remove_instances(cluster): cluster.simulator.current_time.shift(seconds=+42) cluster.modify_size({ InstanceMarket('m4.4xlarge', 'us-west-1a'): 1, InstanceMarket('i2.8xlarge', 'us-west-1a'): 1, }) assert len(cluster) == 2 assert cluster.cpus == 48 assert cluster.mem == 308 assert cluster.disk == 9400
def test_get_options_for_instance_type(mock_asrg): mock_asrg._group_config['AvailabilityZones'] = ['us-west-1a', 'us-west-2a'] result = mock_asrg._get_options_for_instance_type('m5.4xlarge') assert len(result) == 2 assert all([ r.agent.total_resources == ClustermanResources( cpus=16, mem=64 * 1024, disk=DEFAULT_VOLUME_SIZE_GB * 1024, gpus=0, ) for r in result ]) assert result[0].instance.market == InstanceMarket('m5.4xlarge', 'us-west-1a') assert result[1].instance.market == InstanceMarket('m5.4xlarge', 'us-west-2a')
def sfrg(context): spot_fleet_request_config = { 'AllocationStrategy': 'diversified', 'LaunchSpecifications': [ { 'InstanceType': 'c3.4xlarge', 'SpotPrice': 1.01, 'WeightedCapacity': 1, 'SubnetId': 'us-west-1a', }, { 'InstanceType': 'c3.4xlarge', 'SpotPrice': 0.41, 'WeightedCapacity': 2, 'SubnetId': 'us-west-1b', }, { 'InstanceType': 'i2.8xlarge', 'SpotPrice': 0.57, 'WeightedCapacity': 3, 'SubnetId': 'us-west-2a', }, { 'InstanceType': 'm4.4xlarge', 'SpotPrice': 2.02, 'WeightedCapacity': 0.5, 'SubnetId': 'us-west-2b', }, { 'InstanceType': 'r4.2xlarge', 'SpotPrice': 1.2, 'WeightedCapacity': 1, 'SubnetId': 'us-west-1c', }, { 'InstanceType': 'd2.2xlarge', 'SpotPrice': 0.6, 'WeightedCapacity': 1.5, 'SubnetId': 'us-west-1c', }, { 'InstanceType': 'r4.4xlarge', 'SpotPrice': 0.57, 'WeightedCapacity': 2, 'SubnetId': 'us-west-2c', }, { 'InstanceType': 'd2.4xlarge', 'SpotPrice': 1.5, 'WeightedCapacity': 0.8, 'SubnetId': 'us-west-2c', }, ], } with mock.patch( 'clusterman.simulator.simulated_spot_fleet_resource_group.get_instance_market', side_effect=lambda spec: InstanceMarket(spec['InstanceType'], spec['SubnetId']), ): context.spot_fleet = SimulatedSpotFleetResourceGroup(spot_fleet_request_config, _make_mock_simulator())
def _populate_cluster_size_events(simulator, start_time, end_time): capacity_metrics = simulator.metrics_client.get_metric_values( f'fulfilled_capacity', METADATA, start_time.timestamp, end_time.timestamp, use_cache=False, extra_dimensions=get_cluster_dimensions( simulator.metadata.cluster, simulator.metadata.pool, simulator.metadata.scheduler, ), ) for i, (timestamp, data) in enumerate(capacity_metrics['fulfilled_capacity']): market_data = {} for market_str, value in data.items(): market = InstanceMarket.parse(market_str) weight = get_market_resources(market).cpus // staticconf.read_int( 'cpus_per_weight') market_data[market] = int(value) // weight simulator.markets |= set(market_data.keys()) use_join_delay = ( i != 0) # Want to start the cluster out at the expected capacity simulator.add_event( ModifyClusterSizeEvent(arrow.get(timestamp), market_data, use_join_delay))
def _get_options_for_instance_type( self, instance_type: str, weight: Optional[float] = None, ) -> List[ClusterNodeMetadata]: """ Generate a list of possible ClusterNode types that could be added to this ASG, given a particular instance type """ options = [] az_options = self._group_config['AvailabilityZones'] for az in az_options: instance_market = InstanceMarket(instance_type, az) weight = weight or self.market_weight(instance_market) options.append(ClusterNodeMetadata( agent=AgentMetadata(total_resources=ClustermanResources.from_instance_type(instance_type)), instance=InstanceMetadata(market=instance_market, weight=weight), )) return options
import behave import staticconf.testing from hamcrest import assert_that from hamcrest import close_to from hamcrest import contains from hamcrest import equal_to from clusterman.aws.markets import InstanceMarket from clusterman.run import setup_logging from clusterman.simulator.event import InstancePriceChangeEvent from clusterman.simulator.event import ModifyClusterSizeEvent from clusterman.simulator.simulator import SimulationMetadata from clusterman.simulator.simulator import Simulator _MARKETS = { 'a': InstanceMarket('c3.8xlarge', 'us-west-2a'), 'b': InstanceMarket('c3.8xlarge', 'us-west-2b'), 'c': InstanceMarket('c3.8xlarge', 'us-west-2c'), } @behave.given( 'market (?P<market_id>[a-cA-C]) has (?P<count>\d+) instances? at time (?P<time>\d+)' ) def setup_instance(context, market_id, count, time): if not hasattr(context, 'market_counts'): context.market_counts = [(0, {})] last_time = context.market_counts[-1][0] if int(time) != last_time: context.market_counts.append(
import arrow import behave import mock from hamcrest import assert_that from hamcrest import close_to from hamcrest import equal_to from hamcrest import greater_than_or_equal_to from clusterman.aws.markets import InstanceMarket from clusterman.math.piecewise import PiecewiseConstantFunction from clusterman.simulator.simulated_spot_fleet_resource_group import SimulatedSpotFleetResourceGroup _MARKETS = [ InstanceMarket('c3.4xlarge', 'us-west-1a'), InstanceMarket('c3.4xlarge', 'us-west-1b'), InstanceMarket('i2.8xlarge', 'us-west-2a'), InstanceMarket('m4.4xlarge', 'us-west-2b'), InstanceMarket('r4.2xlarge', 'us-west-1c'), InstanceMarket('d2.2xlarge', 'us-west-1c'), InstanceMarket('r4.4xlarge', 'us-west-2c'), InstanceMarket('d2.4xlarge', 'us-west-2c'), ] def _make_mock_simulator(): instance_prices = defaultdict(lambda: PiecewiseConstantFunction()) instance_prices[_MARKETS[0]].add_breakpoint(arrow.get(0), 0.5) instance_prices[_MARKETS[1]].add_breakpoint(arrow.get(0), 0.7) instance_prices[_MARKETS[2]].add_breakpoint(arrow.get(0), 0.6)
# limitations under the License. import arrow import mock import pytest from clusterman.aws.markets import get_market_resources from clusterman.aws.markets import InstanceMarket from clusterman.interfaces.cluster_connector import AgentMetadata from clusterman.interfaces.cluster_connector import AgentState from clusterman.interfaces.cluster_connector import ClustermanResources from clusterman.simulator.simulated_aws_cluster import Instance from clusterman.simulator.simulated_cluster_connector import SimulatedClusterConnector from clusterman.simulator.simulated_spot_fleet_resource_group import SimulatedSpotFleetResourceGroup TEST_MARKET = InstanceMarket('c3.4xlarge', 'us-west-2a') @pytest.fixture def ssfrg_config(): return { 'LaunchSpecifications': [], 'AllocationStrategy': 'diversified' } @pytest.fixture def mock_ssfrg(ssfrg_config): ssfrg = SimulatedSpotFleetResourceGroup(ssfrg_config, None) instances = [Instance(TEST_MARKET, arrow.get(0), join_time=arrow.get(0)) for i in range(10)] ssfrg.instances = {instance.id: instance for instance in instances}
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict import arrow import mock import pytest from clusterman.aws.markets import InstanceMarket from clusterman.math.piecewise import PiecewiseConstantFunction from clusterman.simulator.simulated_spot_fleet_resource_group import SimulatedSpotFleetResourceGroup MARKETS = [ InstanceMarket('c3.4xlarge', 'us-west-1a'), InstanceMarket('c3.4xlarge', 'us-west-1b'), InstanceMarket('i2.8xlarge', 'us-west-2a'), InstanceMarket('m4.4xlarge', 'us-west-2b'), ] @pytest.fixture def spot_fleet_request_config(): return { 'AllocationStrategy': 'diversified', 'LaunchSpecifications': [ { 'InstanceType': 'c3.4xlarge', 'SpotPrice': 1.01,
def get_fake_instance_market(spec): return InstanceMarket(spec['InstanceType'], spec['SubnetId'])
def test_valid_market(fake_markets): InstanceMarket('foo', 'bar')
def test_market_weight(mock_asrg, instance_type): market_weight = mock_asrg.market_weight( InstanceMarket(instance_type, 'us-west-2a')) assert market_weight == 1.0
def test_market_capacities(mock_resource_group): assert mock_resource_group.market_capacities == { InstanceMarket('c3.4xlarge', 'us-west-2a'): 5 }
def mock_instance(request): market = InstanceMarket('m4.4xlarge', 'us-west-1a') instance = Instance(market, arrow.get(0)) instance.end_time = request.param return instance
def test_invalid_market(): with pytest.raises(ValueError): InstanceMarket('foo', 'bar')