def test_most_matching_images_big(self, mock_get_meta_all):
candidates = ['node1_net', 'node2', 'node3', 'node4']
finalists = scheduler.Scheduler()._find_most_matching_images(
['req_image1'], candidates)
self.assertSetEqual(set(['node1_net']), set(finalists))
finalists = scheduler.Scheduler()._find_most_matching_images(
['req_image1', 'req_image2'], candidates)
self.assertSetEqual(set(['node1_net']), set(finalists))
finalists = scheduler.Scheduler()._find_most_matching_images(
['req_image2'], candidates)
self.assertSetEqual(set(['node1_net', 'node2', 'node3']), set(finalists))
def test_any_node_but_not_network_node(self, mock_get_image_meta):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance':
16,
'cpu_max':
4,
'memory_available':
22000,
'memory_max':
24000,
'disk_free':
2000 * 1024 * 1024 * 1024
})
fake_inst = FakeInstance()
fake_inst.db_setup(
cpus=1,
memory=1024,
block_devices={'devices': [{
'size': 8,
'base': 'some-os'
}]},
)
nets = [{'network_uuid': 'uuid-net2'}]
nodes = scheduler.Scheduler().place_instance(fake_inst, nets)
self.assertSetEqual(
set(self.fake_db.nodes) - {
'node1_net',
}, set(nodes))
def test_no_metrics(self):
fake_inst = FakeInstance(cpus=1)
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('No nodes with metrics', str(exc))
def test_not_enough_ram_on_node(self):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance':
16,
'cpu_max':
4,
'memory_available':
10000,
'memory_max':
10000,
'memory_total_instance_actual':
15001,
'disk_free':
2000 * 1024 * 1024 * 1024,
})
fake_inst = FakeInstance(
cpus=1,
memory=1,
block_devices={'devices': [{
'size': 21,
'base': 'some-os'
}]})
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('No nodes with enough idle RAM', str(exc))
def test_ok(self, mock_get_image_meta):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance':
16,
'cpu_max':
4,
'memory_available':
22000,
'memory_max':
24000,
'disk_free':
2000 * 1024 * 1024 * 1024
})
fake_inst = FakeInstance(
cpus=1,
memory=1024,
block_devices={'devices': [{
'size': 8,
'base': 'some-os'
}]})
nodes = scheduler.Scheduler().place_instance(fake_inst, [])
self.assertSetEqual(
set(self.fake_db.nodes) - {
'node1_net',
}, set(nodes))
def test_refresh(self):
self.mock_etcd.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'memory_available': 22000,
'memory_max': 24000,
'disk_free_instances': 2000 * GiB,
'cpu_total_instance_vcpus': 4,
'cpu_available': 12
})
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
s = scheduler.Scheduler()
s.place_instance(fake_inst, None)
self.assertEqual(22000, s.metrics['node1_net']['memory_available'])
self.mock_etcd.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'memory_available': 11000,
'memory_max': 24000,
'disk_free_instances': 2000 * GiB,
'cpu_total_instance_vcpus': 4,
'cpu_available': 12
})
s.metrics_updated = time.time() - 400
s.place_instance(fake_inst, None)
self.assertEqual(11000, s.metrics['node1_net']['memory_available'])
def test_single_node_that_has_network(self, mock_get_image_meta):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance':
16,
'cpu_max':
4,
'memory_available':
22000,
'memory_max':
24000,
'disk_free':
2000 * 1024 * 1024 * 1024
})
fake_inst = FakeInstance(
cpus=1,
memory=1024,
block_devices={'devices': [{
'size': 8,
'base': 'some-os'
}]})
nets = [{'network_uuid': 'uuid-net1'}]
nodes = scheduler.Scheduler().place_instance(fake_inst, nets)
self.assertSetEqual(set(['node3']), set(nodes))
def test_only_two(self):
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
nodes = scheduler.Scheduler().place_instance(
fake_inst, [], candidates=['node1_net', 'node2'])
self.assertSetEqual({
'node2',
}, set(nodes))
def instance_preflight(instance_uuid, network):
db.update_instance_state(instance_uuid, 'preflight')
s = scheduler.Scheduler()
instance = virt.from_db(instance_uuid)
try:
s.place_instance(instance, network, candidates=[config.NODE_NAME])
return None
except exceptions.LowResourceException as e:
db.add_event('instance', instance_uuid, 'schedule', 'retry', None,
'insufficient resources: ' + str(e))
if instance.db_entry.get('placement_attempts') > 3:
raise exceptions.AbortInstanceStartException('Too many start attempts')
try:
if instance.db_entry.get('requested_placement'):
candidates = [instance.db_entry.get('requested_placement')]
else:
candidates = []
for node in s.metrics.keys():
if node != config.NODE_NAME:
candidates.append(node)
candidates = s.place_instance(instance, network, candidates=candidates)
return candidates[0]
except exceptions.LowResourceException as e:
db.add_event('instance', instance_uuid, 'schedule', 'failed', None,
'insufficient resources: ' + str(e))
# This raise implies delete above
raise exceptions.AbortInstanceStartException(
'Unable to find suitable node')
def test_ok(self):
self.mock_etcd.set_node_metrics_same()
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
nodes = scheduler.Scheduler().place_instance(fake_inst, [])
self.assertSetEqual(
set(self.mock_etcd.node_names) - {
'node1_net',
}, set(nodes))
def test_requested_too_many_cpu(self):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance': 5,
})
fake_inst = FakeInstance(cpus=6)
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('Requested vCPUs exceeds vCPU limit', str(exc))
def test_requested_too_many_cpu(self):
self.mock_etcd.set_node_metrics_same({
'cpu_max_per_instance': 5,
'cpu_total_instance_vcpus': 4,
'cpu_available': 12
})
fake_inst = self.mock_etcd.create_instance('fake-inst',
'fakeuuid',
cpus=6)
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('Requested vCPUs exceeds vCPU limit', str(exc))
def test_any_node_but_not_network_node(self):
self.mock_etcd.create_instance('instance-1',
'uuid-inst-1',
place_on_node='node3')
self.mock_etcd.set_node_metrics_same()
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
nets = [{'network_uuid': 'uuid-net2'}]
nodes = scheduler.Scheduler().place_instance(fake_inst, nets)
self.assertSetEqual(
set(self.mock_etcd.node_names) - {
'node1_net',
}, set(nodes))
def test_not_enough_ram_on_node(self):
self.mock_etcd.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'memory_available': 10000,
'memory_max': 10000,
'memory_total_instance_actual': 15001,
'disk_free_instances': 2000 * GiB,
'cpu_total_instance_vcpus': 4,
'cpu_available': 12
})
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('No nodes with enough idle RAM', str(exc))
def test_not_enough_cpu(self):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'cpu_total_instance_vcpus': 4*16,
'memory_available': 5*1024+1024-1,
'memory_max': 24000,
'disk_free': 2000*1024*1024*1024
})
fake_inst = FakeInstance()
fake_inst.db_setup(cpus=1)
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst,
[])
self.assertEqual('No nodes with enough idle CPU', str(exc))
def instance_preflight(instance_uuid, network):
instance = virt.Instance.from_db(instance_uuid)
if not instance:
raise exceptions.InstanceNotInDBException(instance_uuid)
instance.update_instance_state('preflight')
# Try to place on this node
s = scheduler.Scheduler()
try:
s.place_instance(instance, network, candidates=[config.NODE_NAME])
return None
except exceptions.LowResourceException as e:
instance.add_event('schedule', 'retry', None,
'insufficient resources: ' + str(e))
# Unsuccessful placement, check if reached placement attempt limit
if instance.placement_attempts > 3:
raise exceptions.AbortInstanceStartException(
'Too many start attempts')
# Try placing on another node
try:
if instance.requested_placement:
# TODO(andy): Ask Mikal why this is not the current node?
candidates = [instance.requested_placement]
else:
candidates = []
for node in s.metrics.keys():
if node != config.NODE_NAME:
candidates.append(node)
candidates = s.place_instance(instance, network,
candidates=candidates)
instance.place_instance(candidates[0])
return candidates[0]
except exceptions.LowResourceException as e:
instance.add_event('schedule', 'failed', None,
'insufficient resources: ' + str(e))
# This raise implies delete above
raise exceptions.AbortInstanceStartException(
'Unable to find suitable node')
def test_anti_affinity_single_inst(self):
self.mock_etcd.create_instance('instance-1',
'uuid-inst-1',
place_on_node='node3',
metadata={'tags': ['nerd']})
# Start test
inst = self.mock_etcd.create_instance('instance-3',
'uuid-inst-3',
metadata={
"affinity": {
"cpu": {
"socialite": 2,
"nerd": -100,
}
},
})
nodes = scheduler.Scheduler().place_instance(inst, [])
self.assertSetEqual({'node2', 'node4'}, set(nodes))
def test_not_enough_disk(self):
self.fake_db.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'memory_available': 22000,
'memory_max': 24000,
'disk_free': 20*1024*1024*1024
})
fake_inst = FakeInstance()
fake_inst.db_setup(cpus=1, memory=1024,
block_devices={'devices': [
{'size': 21, 'base': 'some-os'}
]})
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst,
[])
self.assertEqual('No nodes with enough disk space', str(exc))
def test_not_enough_disk(self):
self.mock_etcd.set_node_metrics_same({
'cpu_max_per_instance': 16,
'cpu_max': 4,
'memory_available': 22000,
'memory_max': 24000,
'disk_free_instances': 20 * GiB,
'cpu_total_instance_vcpus': 4,
'cpu_available': 12
})
fake_inst = self.mock_etcd.create_instance('fake-inst',
'fakeuuid',
disk_spec=[{
'base': 'cirros',
'size': 21
}])
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('No nodes with enough disk space', str(exc))
def test_no_such_node(self):
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
self.assertRaises(exceptions.CandidateNodeNotFoundException,
scheduler.Scheduler().place_instance,
fake_inst, [],
candidates=['barry'])
def test_no_metrics(self):
fake_inst = self.mock_etcd.create_instance('fake-inst', 'fakeuuid')
exc = self.assertRaises(exceptions.LowResourceException,
scheduler.Scheduler().place_instance,
fake_inst, [])
self.assertEqual('No nodes with metrics', str(exc))
def post(self,
name=None,
cpus=None,
memory=None,
network=None,
disk=None,
ssh_key=None,
user_data=None,
placed_on=None,
namespace=None,
instance_uuid=None):
global SCHEDULER
# We need to sanitise the name so its safe for DNS
name = re.sub(r'([^a-zA-Z0-9_\-])', '', name)
if not namespace:
namespace = get_jwt_identity()
# If accessing a foreign namespace, we need to be an admin
if get_jwt_identity() not in [namespace, 'system']:
return error(
401,
'only admins can create resources in a different namespace')
# The instance needs to exist in the DB before network interfaces are created
if not instance_uuid:
instance_uuid = str(uuid.uuid4())
db.add_event('instance', instance_uuid, 'uuid allocated', None,
None, None)
# Create instance object
instance = virt.from_db(instance_uuid)
if instance:
if get_jwt_identity() not in [
instance.db_entry['namespace'], 'system'
]:
LOG.info('instance(%s): instance not found, ownership test' %
instance_uuid)
return error(404, 'instance not found')
if not instance:
instance = virt.from_definition(uuid=instance_uuid,
name=name,
disks=disk,
memory_mb=memory,
vcpus=cpus,
ssh_key=ssh_key,
user_data=user_data,
owner=namespace)
if not SCHEDULER:
SCHEDULER = scheduler.Scheduler()
# Have we been placed?
if not placed_on:
candidates = SCHEDULER.place_instance(instance, network)
if len(candidates) == 0:
db.add_event('instance', instance_uuid, 'schedule', 'failed',
None, 'insufficient resources')
db.update_instance_state(instance_uuid, 'error')
return error(507, 'insufficient capacity')
placed_on = candidates[0]
db.place_instance(instance_uuid, placed_on)
db.add_event('instance', instance_uuid, 'placement', None, None,
placed_on)
else:
try:
candidates = SCHEDULER.place_instance(instance,
network,
candidates=[placed_on])
if len(candidates) == 0:
db.add_event('instance', instance_uuid, 'schedule',
'failed', None, 'insufficient resources')
db.update_instance_state(instance_uuid, 'error')
return error(507, 'insufficient capacity')
except scheduler.CandidateNodeNotFoundException as e:
return error(404, 'node not found: %s' % e)
# Have we been placed on a different node?
if not placed_on == config.parsed.get('NODE_NAME'):
body = flask_get_post_body()
body['placed_on'] = placed_on
body['instance_uuid'] = instance_uuid
body['namespace'] = namespace
token = util.get_api_token(
'http://%s:%d' % (placed_on, config.parsed.get('API_PORT')),
namespace=namespace)
r = requests.request('POST',
'http://%s:%d/instances' %
(placed_on, config.parsed.get('API_PORT')),
data=json.dumps(body),
headers={
'Authorization': token,
'User-Agent': util.get_user_agent()
})
LOG.info('Returning proxied request: %d, %s' %
(r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
# Check we can get the required IPs
nets = {}
allocations = {}
def error_with_cleanup(status_code, message):
for network_uuid in allocations:
n = net.from_db(network_uuid)
for addr, _ in allocations[network_uuid]:
with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _:
ipm = db.get_ipmanager(n.uuid)
ipm.release(addr)
db.persist_ipmanager(n.uuid, ipm.save())
return error(status_code, message)
order = 0
if network:
for netdesc in network:
if 'network_uuid' not in netdesc or not netdesc['network_uuid']:
return error_with_cleanup(404, 'network not specified')
if netdesc['network_uuid'] not in nets:
n = net.from_db(netdesc['network_uuid'])
if not n:
return error_with_cleanup(
404,
'network %s not found' % netdesc['network_uuid'])
nets[netdesc['network_uuid']] = n
n.create()
with db.get_lock('sf/ipmanager/%s' % netdesc['network_uuid'],
ttl=120) as _:
db.add_event('network', netdesc['network_uuid'],
'allocate address', None, None, instance_uuid)
allocations.setdefault(netdesc['network_uuid'], [])
ipm = db.get_ipmanager(netdesc['network_uuid'])
if 'address' not in netdesc or not netdesc['address']:
netdesc['address'] = ipm.get_random_free_address()
else:
if not ipm.reserve(netdesc['address']):
return error_with_cleanup(
409, 'address %s in use' % netdesc['address'])
db.persist_ipmanager(netdesc['network_uuid'], ipm.save())
allocations[netdesc['network_uuid']].append(
(netdesc['address'], order))
if 'model' not in netdesc or not netdesc['model']:
netdesc['model'] = 'virtio'
db.create_network_interface(str(uuid.uuid4()), netdesc,
instance_uuid, order)
order += 1
# Initialise metadata
db.persist_metadata('instance', instance_uuid, {})
# Now we can start the instance
with db.get_lock('sf/instance/%s' % instance.db_entry['uuid'],
ttl=900) as lock:
with util.RecordedOperation('ensure networks exist',
instance) as _:
for network_uuid in nets:
n = nets[network_uuid]
n.ensure_mesh()
n.update_dhcp()
with util.RecordedOperation('instance creation', instance) as _:
instance.create(lock=lock)
for iface in db.get_instance_interfaces(instance.db_entry['uuid']):
db.update_network_interface_state(iface['uuid'], 'created')
return db.get_instance(instance_uuid)
def post(self,
name=None,
cpus=None,
memory=None,
network=None,
disk=None,
ssh_key=None,
user_data=None,
placed_on=None,
namespace=None,
instance_uuid=None,
video=None):
global SCHEDULER
# Check that the instance name is safe for use as a DNS host name
if name != re.sub(r'([^a-zA-Z0-9_\-])', '', name) or len(name) > 63:
return error(400,
'instance name must be useable as a DNS host name')
# Sanity check
if not disk:
return error(400, 'instance must specify at least one disk')
for d in disk:
if not isinstance(d, dict):
return error(400,
'disk specification should contain JSON objects')
if network:
for n in network:
if not isinstance(n, dict):
return error(
400,
'network specification should contain JSON objects')
if 'network_uuid' not in n:
return error(
400, 'network specification is missing network_uuid')
if not video:
video = {'model': 'cirrus', 'memory': 16384}
if not namespace:
namespace = get_jwt_identity()
# Only system can specify a uuid
if instance_uuid and get_jwt_identity() != 'system':
return error(401, 'only system can specify an instance uuid')
# If accessing a foreign namespace, we need to be an admin
if get_jwt_identity() not in [namespace, 'system']:
return error(
401,
'only admins can create resources in a different namespace')
# The instance needs to exist in the DB before network interfaces are created
if not instance_uuid:
instance_uuid = str(uuid.uuid4())
db.add_event('instance', instance_uuid, 'uuid allocated', None,
None, None)
# Create instance object
instance = virt.from_db(instance_uuid)
if instance:
if get_jwt_identity() not in [
instance.db_entry['namespace'], 'system'
]:
logutil.info([virt.ThinInstance(instance_uuid)],
'Instance not found, ownership test')
return error(404, 'instance not found')
if not instance:
instance = virt.from_definition(uuid=instance_uuid,
name=name,
disks=disk,
memory_mb=memory,
vcpus=cpus,
ssh_key=ssh_key,
user_data=user_data,
owner=namespace,
video=video,
requested_placement=placed_on)
# Initialise metadata
db.persist_metadata('instance', instance_uuid, {})
# Allocate IP addresses
order = 0
if network:
for netdesc in network:
n = net.from_db(netdesc['network_uuid'])
if not n:
db.enqueue_instance_delete(
config.parsed.get('NODE_NAME'), instance_uuid, 'error',
'missing network %s during IP allocation phase' %
netdesc['network_uuid'])
return error(
404, 'network %s not found' % netdesc['network_uuid'])
with db.get_lock('ipmanager',
None,
netdesc['network_uuid'],
ttl=120):
db.add_event('network', netdesc['network_uuid'],
'allocate address', None, None, instance_uuid)
ipm = db.get_ipmanager(netdesc['network_uuid'])
if 'address' not in netdesc or not netdesc['address']:
netdesc['address'] = ipm.get_random_free_address()
else:
if not ipm.reserve(netdesc['address']):
db.enqueue_instance_delete(
config.parsed.get('NODE_NAME'), instance_uuid,
'error',
'failed to reserve an IP on network %s' %
netdesc['network_uuid'])
return error(
409, 'address %s in use' % netdesc['address'])
db.persist_ipmanager(netdesc['network_uuid'], ipm.save())
if 'model' not in netdesc or not netdesc['model']:
netdesc['model'] = 'virtio'
db.create_network_interface(str(uuid.uuid4()), netdesc,
instance_uuid, order)
if not SCHEDULER:
SCHEDULER = scheduler.Scheduler()
try:
# Have we been placed?
if not placed_on:
candidates = SCHEDULER.place_instance(instance, network)
placement = candidates[0]
else:
SCHEDULER.place_instance(instance,
network,
candidates=[placed_on])
placement = placed_on
except exceptions.LowResourceException as e:
db.add_event('instance', instance_uuid, 'schedule', 'failed', None,
'insufficient resources: ' + str(e))
db.enqueue_instance_delete(config.parsed.get('NODE_NAME'),
instance_uuid, 'error',
'scheduling failed')
return error(507, str(e))
except exceptions.CandidateNodeNotFoundException as e:
db.add_event('instance', instance_uuid, 'schedule', 'failed', None,
'candidate node not found: ' + str(e))
db.enqueue_instance_delete(config.get.parsed('NODE_NAME'),
instance_uuid, 'error',
'scheduling failed')
return error(404, 'node not found: %s' % e)
# Record placement
db.place_instance(instance_uuid, placement)
db.add_event('instance', instance_uuid, 'placement', None, None,
placement)
# Create a queue entry for the instance start
tasks = [{
'type': 'instance_preflight',
'instance_uuid': instance_uuid,
'network': network
}]
for disk in instance.db_entry['block_devices']['devices']:
if 'base' in disk and disk['base']:
tasks.append({
'type': 'image_fetch',
'instance_uuid': instance_uuid,
'url': disk['base']
})
tasks.append({
'type': 'instance_start',
'instance_uuid': instance_uuid,
'network': network
})
# Enqueue creation tasks on desired node task queue
db.enqueue(placement, {'tasks': tasks})
db.add_event('instance', instance_uuid, 'create', 'enqueued', None,
None)
# Watch for a while and return results if things are fast, give up
# after a while and just return the current state
start_time = time.time()
while time.time() - start_time < config.parsed.get('API_ASYNC_WAIT'):
i = db.get_instance(instance_uuid)
if i['state'] in ['created', 'deleted', 'error']:
return i
time.sleep(0.5)
return i
