def test_images_with_zero_weight_not_cached(self): image_weights = { 'Ubuntu': 5, 'CoreOS': 10, 'Windows': 0, } sb._load_image_weights_file.image_weights = image_weights CONF.set_override('default_image_weight', 0, 'strategy') test_scenario = { 'num_images': 30, 'images': TEST_IMAGES, 'nodes': ( [sb.NodeInput('c-%d' % n, 'compute', False, False, 'aaaa') for n in range(0, 30)]) } picked_images = ( sb.choose_weighted_images_forced_distribution(**test_scenario)) # Make sure no images were picked with a zero weight. expected_names = [name for name, weight in six.iteritems(image_weights) if weight > 0] for image in picked_images: self.assertIn(image.name, expected_names, "Found an unexpected image cached. Image had a " "zero weight. Image name %s" % (image.name))
def test_images_with_zero_weight_not_cached(self): image_weights = { 'Ubuntu': 5, 'CoreOS': 10, 'Windows': 0, } sb._load_image_weights_file.image_weights = image_weights CONF.set_override('default_image_weight', 0, 'strategy') test_scenario = { 'num_images': 30, 'images': TEST_IMAGES, 'nodes': ( [sb.NodeInput('c-%d' % n, 'compute', False, False, 'aaaa') for n in range(0, 30)]) } picked_images = ( sb.choose_weighted_images_forced_distribution(**test_scenario)) # Make sure no images were picked with a zero weight. expected_names = [name for name, weight in image_weights.iteritems() if weight > 0] for image in picked_images: self.assertIn(image.name, expected_names, "Found an unexpected image cached. Image had a " "zero weight. Image name %s" % (image.name))
def cache_nodes(nodes, num_nodes_needed, images): available_nodes = nodes_available_for_caching(nodes) # Choose the images to cache in advance, based on how many nodes we should # use for caching. chosen_images = sb.choose_weighted_images_forced_distribution(num_nodes_needed, images, nodes) # If we're not meeting or exceeding our proportion goal, # schedule (node, image) pairs to cache until we would meet # our proportion goal. nodes_to_cache = [] random.shuffle(available_nodes) for n in range(0, num_nodes_needed): node = available_nodes.pop() image = chosen_images.pop() nodes_to_cache.append(sb.CacheNode(node.node_uuid, image.uuid, image.checksum)) return nodes_to_cache
def cache_nodes(nodes, num_nodes_needed, images): available_nodes = nodes_available_for_caching(nodes) # Choose the images to cache in advance, based on how many nodes we should # use for caching. chosen_images = sb.choose_weighted_images_forced_distribution( num_nodes_needed, images, nodes) # If we're not meeting or exceeding our proportion goal, # schedule (node, image) pairs to cache until we would meet # our proportion goal. nodes_to_cache = [] random.shuffle(available_nodes) for n in range(0, num_nodes_needed): node = available_nodes.pop() image = chosen_images.pop() nodes_to_cache.append( sb.CacheNode(node.node_uuid, image.uuid, image.checksum)) return nodes_to_cache
def test_choose_weighted_images_forced_distribution(self): test_scenarios = { '10-all nodes available': { 'num_images': 10, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 10)] }, '100-all nodes available': { 'num_images': 50, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 100)] }, '1000-all nodes available': { 'num_images': 1000, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 1000)] }, 'all nodes available - num of nodes machine image weight sum': { 'num_images': self.IMAGE_WEIGHT_SUM, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, self.IMAGE_WEIGHT_SUM)] }, } # Provides a list which coupled with random selection should # closely match the image weights. Therefore already cached nodes # in these scenarios already closely match the distribution. weighted_image_uuids = [] for image in TEST_IMAGES: weighted_image_uuids.extend( [image.uuid for n in range(0, self.WEIGHTED_IMAGES[image.name])]) images_by_uuids = {image.uuid: image for image in TEST_IMAGES} # Generate some more varied scenarios. for num_nodes in [1, 2, 3, 5, 10, 20, 50, 100, 1000, 10000]: new_scenario = { 'images': TEST_IMAGES, 'num_images': int(math.floor(num_nodes * 0.25)), 'nodes': [] } for n in range(0, num_nodes): cached = n % 4 == 0 provisioned = n % 7 == 0 cached_image_uuid = random.choice(weighted_image_uuids) generated_node = sb.NodeInput("c-%d" % (n), 'compute', provisioned, cached, cached_image_uuid) new_scenario['nodes'].append(generated_node) test_scenarios['%d-random scenario' % (num_nodes)] = new_scenario # Now test each scenario. for name, values in six.iteritems(test_scenarios): print("Testing against '%s' scenario." % (name)) picked_images = sb.choose_weighted_images_forced_distribution( **values) picked_distribution = collections.defaultdict(lambda: 0) for image in picked_images: picked_distribution[image.name] += 1 self.assertEqual(values['num_images'], len(picked_images), "Didn't get the expected number of selected " "images from " "choose_weighted_images_forced_distribution") num_already_cached = len([node for node in values['nodes'] if node.cached and not node.provisioned]) scale = (num_already_cached + values['num_images']) / sum( [self.WEIGHTED_IMAGES[image.name] for image in TEST_IMAGES]) already_cached = collections.defaultdict(lambda: 0) for node in values['nodes']: if node.cached and not node.provisioned: image = images_by_uuids[node.cached_image_uuid] already_cached[image.name] += 1 targeted_distribution = { image.name: (picked_distribution[image.name] + already_cached[image.name]) for image in TEST_IMAGES } print(''.join(["Picked distribution: %s\n" % ( str(picked_distribution)), "Already cached distribution: %s\n" % ( str(already_cached)), "Targeted distribution: %s\n" % ( str(targeted_distribution)), "Image weights: %s\n" % str(self.WEIGHTED_IMAGES), "scale factor: %f" % scale])) for image in values['images']: print("Inspecting image '%s'." % (image.name)) image_weight = self.WEIGHTED_IMAGES[image.name] num_image_already_cached = len([ node for node in values['nodes'] if node.cached and not node.provisioned and node.cached_image_uuid == image.uuid]) expected_num_of_selected_images = ( int(math.floor(scale * image_weight)) - num_image_already_cached) # Sometimes an underweighted image will be cached a great deal # more than should be given the current weights. Clamp this # the expectation to zero. if expected_num_of_selected_images < 0: expected_num_of_selected_images = 0 num_picked = len( [pi for pi in picked_images if pi.name == image.name]) failure_msg = ( "The number of selected images for image " "'%(image_name)s' did not match expectations. " "Expected %(expected)d and got %(actual)d. " % {'image_name': image.name, 'expected': expected_num_of_selected_images, 'actual': num_picked}) self.assertAlmostEqual(num_picked, expected_num_of_selected_images, delta=1, msg=failure_msg)
def test_choose_weighted_images_forced_distribution(self): test_scenarios = { '10-all nodes available': { 'num_images': 10, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 10)] }, '100-all nodes available': { 'num_images': 50, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 100)] }, '1000-all nodes available': { 'num_images': 1000, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, 1000)] }, 'all nodes available - num of nodes machine image weight sum': { 'num_images': self.IMAGE_WEIGHT_SUM, 'images': TEST_IMAGES, 'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False, 'aaaa') for n in range(0, self.IMAGE_WEIGHT_SUM)] }, } # Provides a list which coupled with random selection should # closely match the image weights. Therefore already cached nodes # in these scenarios already closely match the distribution. weighted_image_uuids = [] for image in TEST_IMAGES: weighted_image_uuids.extend( [image.uuid for n in range(0, self.WEIGHTED_IMAGES[image.name])]) images_by_uuids = {image.uuid: image for image in TEST_IMAGES} # Generate some more varied scenarios. for num_nodes in [1, 2, 3, 5, 10, 20, 50, 100, 1000, 10000]: new_scenario = { 'images': TEST_IMAGES, 'num_images': int(math.floor(num_nodes * 0.25)), 'nodes': [] } for n in range(0, num_nodes): cached = n % 4 == 0 provisioned = n % 7 == 0 cached_image_uuid = random.choice(weighted_image_uuids) generated_node = sb.NodeInput("c-%d" % (n), 'compute', provisioned, cached, cached_image_uuid) new_scenario['nodes'].append(generated_node) test_scenarios['%d-random scenario' % (num_nodes)] = new_scenario # Now test each scenario. for name, values in test_scenarios.iteritems(): print("Testing against '%s' scenario." % (name)) picked_images = sb.choose_weighted_images_forced_distribution( **values) picked_distribution = collections.defaultdict(lambda: 0) for image in picked_images: picked_distribution[image.name] += 1 self.assertEqual(values['num_images'], len(picked_images), "Didn't get the expected number of selected " "images from " "choose_weighted_images_forced_distribution") num_already_cached = len([node for node in values['nodes'] if node.cached and not node.provisioned]) scale = (num_already_cached + values['num_images']) / sum( [self.WEIGHTED_IMAGES[image.name] for image in TEST_IMAGES]) already_cached = collections.defaultdict(lambda: 0) for node in values['nodes']: if node.cached and not node.provisioned: image = images_by_uuids[node.cached_image_uuid] already_cached[image.name] += 1 targeted_distribution = { image.name: (picked_distribution[image.name] + already_cached[image.name]) for image in TEST_IMAGES } print(''.join(["Picked distribution: %s\n" % ( str(picked_distribution)), "Already cached distribution: %s\n" % ( str(already_cached)), "Targeted distribution: %s\n" % ( str(targeted_distribution)), "Image weights: %s\n" % str(self.WEIGHTED_IMAGES), "scale factor: %f" % scale])) for image in values['images']: print("Inspecting image '%s'." % (image.name)) image_weight = self.WEIGHTED_IMAGES[image.name] num_image_already_cached = len([ node for node in values['nodes'] if node.cached and not node.provisioned and node.cached_image_uuid == image.uuid]) expected_num_of_selected_images = ( int(math.floor(scale * image_weight)) - num_image_already_cached) # Sometimes an underweighted image will be cached a great deal # more than should be given the current weights. Clamp this # the expectation to zero. if expected_num_of_selected_images < 0: expected_num_of_selected_images = 0 num_picked = len( [pi for pi in picked_images if pi.name == image.name]) failure_msg = ( "The number of selected images for image " "'%(image_name)s' did not match expectations. " "Expected %(expected)d and got %(actual)d. " % {'image_name': image.name, 'expected': expected_num_of_selected_images, 'actual': num_picked}) self.assertAlmostEqual(num_picked, expected_num_of_selected_images, delta=1, msg=failure_msg)