def test_images_with_zero_weight_not_cached(self):
image_weights = {
'Ubuntu': 5,
'CoreOS': 10,
'Windows': 0,
}
sb._load_image_weights_file.image_weights = image_weights
CONF.set_override('default_image_weight', 0, 'strategy')
test_scenario = {
'num_images': 30,
'images': TEST_IMAGES,
'nodes': (
[sb.NodeInput('c-%d' % n, 'compute', False, False, 'aaaa') for
n in range(0, 30)])
}
picked_images = (
sb.choose_weighted_images_forced_distribution(**test_scenario))
# Make sure no images were picked with a zero weight.
expected_names = [name for name, weight in six.iteritems(image_weights)
if weight > 0]
for image in picked_images:
self.assertIn(image.name, expected_names,
"Found an unexpected image cached. Image had a "
"zero weight. Image name %s" % (image.name))
def test_images_with_zero_weight_not_cached(self):
image_weights = {
'Ubuntu': 5,
'CoreOS': 10,
'Windows': 0,
}
sb._load_image_weights_file.image_weights = image_weights
CONF.set_override('default_image_weight', 0, 'strategy')
test_scenario = {
'num_images': 30,
'images': TEST_IMAGES,
'nodes': (
[sb.NodeInput('c-%d' % n, 'compute', False, False, 'aaaa') for
n in range(0, 30)])
}
picked_images = (
sb.choose_weighted_images_forced_distribution(**test_scenario))
# Make sure no images were picked with a zero weight.
expected_names = [name for name, weight in image_weights.iteritems()
if weight > 0]
for image in picked_images:
self.assertIn(image.name, expected_names,
"Found an unexpected image cached. Image had a "
"zero weight. Image name %s" % (image.name))
def cache_nodes(nodes, num_nodes_needed, images):
available_nodes = nodes_available_for_caching(nodes)
# Choose the images to cache in advance, based on how many nodes we should
# use for caching.
chosen_images = sb.choose_weighted_images_forced_distribution(num_nodes_needed, images, nodes)
# If we're not meeting or exceeding our proportion goal,
# schedule (node, image) pairs to cache until we would meet
# our proportion goal.
nodes_to_cache = []
random.shuffle(available_nodes)
for n in range(0, num_nodes_needed):
node = available_nodes.pop()
image = chosen_images.pop()
nodes_to_cache.append(sb.CacheNode(node.node_uuid, image.uuid, image.checksum))
return nodes_to_cache
def cache_nodes(nodes, num_nodes_needed, images):
available_nodes = nodes_available_for_caching(nodes)
# Choose the images to cache in advance, based on how many nodes we should
# use for caching.
chosen_images = sb.choose_weighted_images_forced_distribution(
num_nodes_needed, images, nodes)
# If we're not meeting or exceeding our proportion goal,
# schedule (node, image) pairs to cache until we would meet
# our proportion goal.
nodes_to_cache = []
random.shuffle(available_nodes)
for n in range(0, num_nodes_needed):
node = available_nodes.pop()
image = chosen_images.pop()
nodes_to_cache.append(
sb.CacheNode(node.node_uuid, image.uuid, image.checksum))
return nodes_to_cache
def test_choose_weighted_images_forced_distribution(self):
test_scenarios = {
'10-all nodes available': {
'num_images': 10,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 10)]
},
'100-all nodes available': {
'num_images': 50,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 100)]
},
'1000-all nodes available': {
'num_images': 1000,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 1000)]
},
'all nodes available - num of nodes machine image weight sum': {
'num_images': self.IMAGE_WEIGHT_SUM,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, self.IMAGE_WEIGHT_SUM)]
},
}
# Provides a list which coupled with random selection should
# closely match the image weights. Therefore already cached nodes
# in these scenarios already closely match the distribution.
weighted_image_uuids = []
for image in TEST_IMAGES:
weighted_image_uuids.extend(
[image.uuid
for n in range(0, self.WEIGHTED_IMAGES[image.name])])
images_by_uuids = {image.uuid: image for image in TEST_IMAGES}
# Generate some more varied scenarios.
for num_nodes in [1, 2, 3, 5, 10, 20, 50, 100, 1000, 10000]:
new_scenario = {
'images': TEST_IMAGES,
'num_images': int(math.floor(num_nodes * 0.25)),
'nodes': []
}
for n in range(0, num_nodes):
cached = n % 4 == 0
provisioned = n % 7 == 0
cached_image_uuid = random.choice(weighted_image_uuids)
generated_node = sb.NodeInput("c-%d" % (n),
'compute',
provisioned,
cached,
cached_image_uuid)
new_scenario['nodes'].append(generated_node)
test_scenarios['%d-random scenario' % (num_nodes)] = new_scenario
# Now test each scenario.
for name, values in six.iteritems(test_scenarios):
print("Testing against '%s' scenario." % (name))
picked_images = sb.choose_weighted_images_forced_distribution(
**values)
picked_distribution = collections.defaultdict(lambda: 0)
for image in picked_images:
picked_distribution[image.name] += 1
self.assertEqual(values['num_images'], len(picked_images),
"Didn't get the expected number of selected "
"images from "
"choose_weighted_images_forced_distribution")
num_already_cached = len([node for node in values['nodes']
if node.cached and not node.provisioned])
scale = (num_already_cached + values['num_images']) / sum(
[self.WEIGHTED_IMAGES[image.name] for image in TEST_IMAGES])
already_cached = collections.defaultdict(lambda: 0)
for node in values['nodes']:
if node.cached and not node.provisioned:
image = images_by_uuids[node.cached_image_uuid]
already_cached[image.name] += 1
targeted_distribution = {
image.name: (picked_distribution[image.name] +
already_cached[image.name])
for image in TEST_IMAGES
}
print(''.join(["Picked distribution: %s\n" % (
str(picked_distribution)),
"Already cached distribution: %s\n" % (
str(already_cached)),
"Targeted distribution: %s\n" % (
str(targeted_distribution)),
"Image weights: %s\n" % str(self.WEIGHTED_IMAGES),
"scale factor: %f" % scale]))
for image in values['images']:
print("Inspecting image '%s'." % (image.name))
image_weight = self.WEIGHTED_IMAGES[image.name]
num_image_already_cached = len([
node for node in values['nodes'] if node.cached and
not node.provisioned and
node.cached_image_uuid == image.uuid])
expected_num_of_selected_images = (
int(math.floor(scale * image_weight)) -
num_image_already_cached)
# Sometimes an underweighted image will be cached a great deal
# more than should be given the current weights. Clamp this
# the expectation to zero.
if expected_num_of_selected_images < 0:
expected_num_of_selected_images = 0
num_picked = len(
[pi for pi in picked_images if pi.name == image.name])
failure_msg = (
"The number of selected images for image "
"'%(image_name)s' did not match expectations. "
"Expected %(expected)d and got %(actual)d. " %
{'image_name': image.name,
'expected': expected_num_of_selected_images,
'actual': num_picked})
self.assertAlmostEqual(num_picked,
expected_num_of_selected_images,
delta=1,
msg=failure_msg)
def test_choose_weighted_images_forced_distribution(self):
test_scenarios = {
'10-all nodes available': {
'num_images': 10,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 10)]
},
'100-all nodes available': {
'num_images': 50,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 100)]
},
'1000-all nodes available': {
'num_images': 1000,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, 1000)]
},
'all nodes available - num of nodes machine image weight sum': {
'num_images': self.IMAGE_WEIGHT_SUM,
'images': TEST_IMAGES,
'nodes': [sb.NodeInput('c-%d' % (n), 'compute', False, False,
'aaaa') for n in range(0, self.IMAGE_WEIGHT_SUM)]
},
}
# Provides a list which coupled with random selection should
# closely match the image weights. Therefore already cached nodes
# in these scenarios already closely match the distribution.
weighted_image_uuids = []
for image in TEST_IMAGES:
weighted_image_uuids.extend(
[image.uuid
for n in range(0, self.WEIGHTED_IMAGES[image.name])])
images_by_uuids = {image.uuid: image for image in TEST_IMAGES}
# Generate some more varied scenarios.
for num_nodes in [1, 2, 3, 5, 10, 20, 50, 100, 1000, 10000]:
new_scenario = {
'images': TEST_IMAGES,
'num_images': int(math.floor(num_nodes * 0.25)),
'nodes': []
}
for n in range(0, num_nodes):
cached = n % 4 == 0
provisioned = n % 7 == 0
cached_image_uuid = random.choice(weighted_image_uuids)
generated_node = sb.NodeInput("c-%d" % (n),
'compute',
provisioned,
cached,
cached_image_uuid)
new_scenario['nodes'].append(generated_node)
test_scenarios['%d-random scenario' % (num_nodes)] = new_scenario
# Now test each scenario.
for name, values in test_scenarios.iteritems():
print("Testing against '%s' scenario." % (name))
picked_images = sb.choose_weighted_images_forced_distribution(
**values)
picked_distribution = collections.defaultdict(lambda: 0)
for image in picked_images:
picked_distribution[image.name] += 1
self.assertEqual(values['num_images'], len(picked_images),
"Didn't get the expected number of selected "
"images from "
"choose_weighted_images_forced_distribution")
num_already_cached = len([node for node in values['nodes']
if node.cached and not node.provisioned])
scale = (num_already_cached + values['num_images']) / sum(
[self.WEIGHTED_IMAGES[image.name] for image in TEST_IMAGES])
already_cached = collections.defaultdict(lambda: 0)
for node in values['nodes']:
if node.cached and not node.provisioned:
image = images_by_uuids[node.cached_image_uuid]
already_cached[image.name] += 1
targeted_distribution = {
image.name: (picked_distribution[image.name] +
already_cached[image.name])
for image in TEST_IMAGES
}
print(''.join(["Picked distribution: %s\n" % (
str(picked_distribution)),
"Already cached distribution: %s\n" % (
str(already_cached)),
"Targeted distribution: %s\n" % (
str(targeted_distribution)),
"Image weights: %s\n" % str(self.WEIGHTED_IMAGES),
"scale factor: %f" % scale]))
for image in values['images']:
print("Inspecting image '%s'." % (image.name))
image_weight = self.WEIGHTED_IMAGES[image.name]
num_image_already_cached = len([
node for node in values['nodes'] if node.cached and
not node.provisioned and
node.cached_image_uuid == image.uuid])
expected_num_of_selected_images = (
int(math.floor(scale * image_weight)) -
num_image_already_cached)
# Sometimes an underweighted image will be cached a great deal
# more than should be given the current weights. Clamp this
# the expectation to zero.
if expected_num_of_selected_images < 0:
expected_num_of_selected_images = 0
num_picked = len(
[pi for pi in picked_images if pi.name == image.name])
failure_msg = (
"The number of selected images for image "
"'%(image_name)s' did not match expectations. "
"Expected %(expected)d and got %(actual)d. " %
{'image_name': image.name,
'expected': expected_num_of_selected_images,
'actual': num_picked})
self.assertAlmostEqual(num_picked,
expected_num_of_selected_images,
delta=1,
msg=failure_msg)
