def _schedule(self, elevated, topic, request_spec, *args, **kwargs):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_type = request_spec.get("instance_type", None)
if not instance_type:
msg = _("Scheduler only understands InstanceType-based" "provisioning.")
raise NotImplementedError(msg)
cost_functions = self.get_cost_functions()
ram_requirement_mb = instance_type["memory_mb"]
disk_requirement_gb = instance_type["local_gb"]
options = self._get_configuration_options()
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.zone_manager.get_all_host_data(elevated)
unfiltered_hosts = unfiltered_hosts_dict.items()
num_instances = request_spec.get("num_instances", 1)
selected_hosts = []
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
filtered_hosts = self._filter_hosts(topic, request_spec, unfiltered_hosts, options)
if not filtered_hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(filtered_hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
weighted_host = least_cost.weighted_sum(cost_functions, filtered_hosts, options)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.hostinfo.consume_resources(disk_requirement_gb, ram_requirement_mb)
# Next, tack on the host weights from the child zones
if not request_spec.get("local_zone", False):
json_spec = json.dumps(request_spec)
all_zones = self._zone_get_all(elevated)
child_results = self._call_zone_method(elevated, "select", specs=json_spec, zones=all_zones)
selected_hosts.extend(self._adjust_child_weights(child_results, all_zones))
selected_hosts.sort(key=operator.attrgetter("weight"))
return selected_hosts[:num_instances]
def test_weighted_sum_single_function(self):
fn_tuples = [(1.0, offset)]
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, ]=
# [10512, 11024, 13072, 18192]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list, options)
self.assertEqual(weighted_host.weight, 10512)
self.assertEqual(weighted_host.host_state.host, "host1")
def test_weighted_sum_single_function(self):
fn_tuples = [(1.0, offset), ]
hostinfo_list = self.zone_manager.get_all_host_data(None).items()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, ]=
# [10000, 11536, 13072, 18192]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10000)
self.assertEqual(weighted_host.host, 'host1')
def test_weighted_sum_single_function(self):
fn_tuples = [(1.0, offset), ]
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, ]=
# [10512, 11024, 13072, 18192]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10512)
self.assertEqual(weighted_host.host_state.host, 'host1')
def test_weighted_sum_single_function(self):
fn_tuples = [
(1.0, offset),
]
hostinfo_list = self.zone_manager.get_all_host_data(None).items()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, ]=
# [10000, 11536, 13072, 18192]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10000)
self.assertEqual(weighted_host.host, 'host1')
def test_basic_costing(self):
hosts = [
FakeHost(1, 512 * MB, 100),
FakeHost(2, 256 * MB, 400),
FakeHost(3, 512 * MB, 100),
]
weighted_fns = [
(1, lambda h: h.free_ram), # Fill-first, free_ram is a *cost*
(2, lambda h: h.io), # Avoid high I/O
]
costs = least_cost.weighted_sum(
domain=hosts, weighted_fns=weighted_fns)
# Each 256 MB unit of free-ram contributes 0.5 points by way of:
# cost = weight * (score/max_score) = 1 * (256/512) = 0.5
# Each 100 iops of IO adds 0.5 points by way of:
# cost = 2 * (100/400) = 2 * 0.25 = 0.5
expected = [1.5, 2.5, 1.5]
self.assertEqual(expected, costs)
def test_weighted_sum_happy_day(self):
fn_tuples = [(1.0, offset), (1.0, scale)]
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512
# host2: free_ram_mb=1024
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, scale]=
# [10512, 11024, 13072, 18192]
# [1024, 2048, 6144, 16384]
# adjusted [ 1.0 * x + 1.0 * y] =
# [11536, 13072, 19216, 34576]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list, options)
self.assertEqual(weighted_host.weight, 11536)
self.assertEqual(weighted_host.host_state.host, "host1")
def test_weighted_sum_happy_day(self):
fn_tuples = [(1.0, offset), (1.0, scale)]
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512
# host2: free_ram_mb=1024
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, scale]=
# [10512, 11024, 13072, 18192]
# [1024, 2048, 6144, 16384]
# adjusted [ 1.0 * x + 1.0 * y] =
# [11536, 13072, 19216, 34576]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 11536)
self.assertEqual(weighted_host.host_state.host, 'host1')
def test_weighted_sum_happy_day(self):
fn_tuples = [(1.0, offset), (1.0, scale)]
hostinfo_list = self.zone_manager.get_all_host_data(None).items()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, scale]=
# [10000, 11536, 13072, 18192]
# [0, 768, 1536, 4096]
# adjusted [ 1.0 * x + 1.0 * y] =
# [10000, 12304, 14608, 22288]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10000)
self.assertEqual(weighted_host.host, 'host1')
def test_weighted_sum_happy_day(self):
fn_tuples = [(1.0, offset), (1.0, scale)]
hostinfo_list = self.zone_manager.get_all_host_data(None).items()
# host1: free_ram_mb=0
# host2: free_ram_mb=1536
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# [offset, scale]=
# [10000, 11536, 13072, 18192]
# [0, 768, 1536, 4096]
# adjusted [ 1.0 * x + 1.0 * y] =
# [10000, 12304, 14608, 22288]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10000)
self.assertEqual(weighted_host.host, 'host1')
def test_empty_domain(self):
domain = []
weighted_fns = []
result = least_cost.weighted_sum(domain, weighted_fns)
expected = []
self.assertEqual(expected, result)
def _schedule(self, elevated, topic, request_spec, *args, **kwargs):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_type = request_spec.get("instance_type", None)
if not instance_type:
msg = _("Scheduler only understands InstanceType-based" \
"provisioning.")
raise NotImplementedError(msg)
cost_functions = self.get_cost_functions()
ram_requirement_mb = instance_type['memory_mb']
disk_requirement_bg = instance_type['local_gb']
options = self._get_configuration_options()
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.zone_manager.get_all_host_data(elevated)
unfiltered_hosts = unfiltered_hosts_dict.items()
num_instances = request_spec.get('num_instances', 1)
selected_hosts = []
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
filtered_hosts = self._filter_hosts(topic, request_spec,
unfiltered_hosts, options)
if not filtered_hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(filtered_hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
weighted_host = least_cost.weighted_sum(cost_functions,
filtered_hosts, options)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.hostinfo.consume_resources(disk_requirement_bg,
ram_requirement_mb)
# Next, tack on the host weights from the child zones
if not request_spec.get('local_zone', False):
json_spec = json.dumps(request_spec)
all_zones = self._zone_get_all(elevated)
child_results = self._call_zone_method(elevated, "select",
specs=json_spec, zones=all_zones)
selected_hosts.extend(self._adjust_child_weights(
child_results, all_zones))
selected_hosts.sort(key=operator.attrgetter('weight'))
return selected_hosts[:num_instances]
def _schedule(self, context, topic, request_spec, *args, **kwargs):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
elevated = context.elevated()
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_properties = request_spec['instance_properties']
instance_type = request_spec.get("instance_type", None)
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
filter_properties = kwargs.get('filter_properties', {})
filter_properties.update({'context': context,
'request_spec': request_spec,
'config_options': config_options,
'instance_type': instance_type})
self.populate_filter_properties(request_spec,
filter_properties)
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.host_manager.get_all_host_states(
elevated, topic)
# Note: remember, we are using an iterator here. So only
# traverse this list once. This can bite you if the hosts
# are being scanned in a filter or weighing function.
hosts = unfiltered_hosts_dict.itervalues()
num_instances = request_spec.get('num_instances', 1)
selected_hosts = []
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
hosts = self.host_manager.filter_hosts(hosts,
filter_properties)
if not hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions,
hosts, filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(
instance_properties)
selected_hosts.sort(key=operator.attrgetter('weight'))
return selected_hosts[:num_instances]
def _schedule(self, context, topic, request_spec, *args, **kwargs):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
elevated = context.elevated()
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_properties = request_spec['instance_properties']
instance_type = request_spec.get("instance_type", None)
if not instance_type:
msg = _("Scheduler only understands InstanceType-based" \
"provisioning.")
raise NotImplementedError(msg)
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
filter_properties = kwargs.get('filter_properties', {})
filter_properties.update({
'context': context,
'request_spec': request_spec,
'config_options': config_options,
'instance_type': instance_type
})
self.populate_filter_properties(request_spec, filter_properties)
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.host_manager.get_all_host_states(
elevated, topic)
hosts = unfiltered_hosts_dict.itervalues()
num_instances = request_spec.get('num_instances', 1)
selected_hosts = []
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
hosts = self.host_manager.filter_hosts(hosts, filter_properties)
if not hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions, hosts,
filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(instance_properties)
# Next, tack on the host weights from the child zones
if not filter_properties.get('local_zone_only', False):
json_spec = json.dumps(request_spec)
all_zones = self._zone_get_all(elevated)
child_results = self._call_zone_method(elevated,
"select",
specs=json_spec,
zones=all_zones)
selected_hosts.extend(
self._adjust_child_weights(child_results, all_zones))
selected_hosts.sort(key=operator.attrgetter('weight'))
return selected_hosts[:num_instances]
def _schedule(self,
context,
topic,
request_spec,
filter_properties,
instance_uuids=None):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
elevated = context.elevated()
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_properties = request_spec['instance_properties']
instance_type = request_spec.get("instance_type", None)
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
# check retry policy. Rather ugly use of instance_uuids[0]...
# but if we've exceeded max retries... then we really only
# have a single instance.
properties = instance_properties.copy()
if instance_uuids:
properties['uuid'] = instance_uuids[0]
self._populate_retry(filter_properties, properties)
filter_properties.update({
'context': context,
'request_spec': request_spec,
'config_options': config_options,
'instance_type': instance_type
})
self.populate_filter_properties(request_spec, filter_properties)
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.host_manager.get_all_host_states(
elevated, topic)
# Note: remember, we are using an iterator here. So only
# traverse this list once. This can bite you if the hosts
# are being scanned in a filter or weighing function.
hosts = unfiltered_hosts_dict.itervalues()
selected_hosts = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
hosts = self.host_manager.filter_hosts(hosts, filter_properties)
if not hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions, hosts,
filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(instance_properties)
selected_hosts.sort(key=operator.attrgetter('weight'))
return selected_hosts
def _schedule(self, context, topic, request_spec, filter_properties, instance_uuids=None):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
elevated = context.elevated()
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_properties = request_spec["instance_properties"]
instance_type = request_spec.get("instance_type", None)
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
# check retry policy. Rather ugly use of instance_uuids[0]...
# but if we've exceeded max retries... then we really only
# have a single instance.
properties = instance_properties.copy()
if instance_uuids:
properties["uuid"] = instance_uuids[0]
self._populate_retry(filter_properties, properties)
filter_properties.update(
{
"context": context,
"request_spec": request_spec,
"config_options": config_options,
"instance_type": instance_type,
}
)
self.populate_filter_properties(request_spec, filter_properties)
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.host_manager.get_all_host_states(elevated, topic)
# Note: remember, we are using an iterator here. So only
# traverse this list once. This can bite you if the hosts
# are being scanned in a filter or weighing function.
hosts = unfiltered_hosts_dict.itervalues()
selected_hosts = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get("num_instances", 1)
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
hosts = self.host_manager.filter_hosts(hosts, filter_properties)
if not hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions, hosts, filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(instance_properties)
selected_hosts.sort(key=operator.attrgetter("weight"))
return selected_hosts
def _schedule(self, context, topic, request_spec, filter_properties,
instance_uuids=None):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
elevated = context.elevated()
if topic != "compute":
msg = _("Scheduler only understands Compute nodes (for now)")
raise NotImplementedError(msg)
instance_properties = request_spec['instance_properties']
instance_type = request_spec.get("instance_type", None)
# Note(gtt): Make sure "extra_specs" in instance_type dict object.
# Since db.instance_type_get_by_flavor_id will change the type of
# extra_specs to dict, so no matter if "instance_type" is exists,
# update it with new from db.
if instance_type is not None:
try:
instance_type = db.instance_type_get_by_flavor_id(
context, instance_type.get('flavorid'))
except exception.FlavorNotFound:
LOG.warn("flavorid not in instance_type. instance_type: %s" %
instance_type)
else:
request_spec.update({"instance_type": instance_type})
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
# check retry policy. Rather ugly use of instance_uuids[0]...
# but if we've exceeded max retries... then we really only
# have a single instance.
properties = instance_properties.copy()
if instance_uuids:
properties['uuid'] = instance_uuids[0]
self._populate_retry(filter_properties, properties)
filter_properties.update({'context': context,
'request_spec': request_spec,
'config_options': config_options,
'instance_type': instance_type})
self.populate_filter_properties(request_spec,
filter_properties)
# Find our local list of acceptable hosts by repeatedly
# filtering and weighing our options. Each time we choose a
# host, we virtually consume resources on it so subsequent
# selections can adjust accordingly.
# unfiltered_hosts_dict is {host : ZoneManager.HostInfo()}
unfiltered_hosts_dict = self.host_manager.get_all_host_states(
elevated, topic)
# Note: remember, we are using an iterator here. So only
# traverse this list once. This can bite you if the hosts
# are being scanned in a filter or weighing function.
hosts = unfiltered_hosts_dict.itervalues()
selected_hosts = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
if FLAGS.use_network_qos:
if instance_uuids:
inst_uuid = instance_uuids[num]
else:
inst_uuid = None
qos_info = resource_tracker.get_instance_network_qos(
instance_type, inst_uuid)
instance_properties['private_network_bandwith'] =\
qos_info['private_qos']
instance_properties['public_network_bandwith'] =\
qos_info['public_qos']
filter_properties.update({'private_network_bandwith':
qos_info['private_qos'],
'public_network_bandwith':
qos_info['public_qos']
})
hosts = self.host_manager.filter_hosts(hosts,
filter_properties)
if not hosts:
# Can't get any more locally.
break
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions,
hosts, filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(
instance_properties, instance_type)
selected_hosts.sort(key=operator.attrgetter('weight'))
return selected_hosts