def test_node_timestamp_updated_only_by_agent(self):
node = self.env.create_node(api=False)
timestamp = node.timestamp
resp = self.app.put(
reverse('NodeCollectionHandler'),
jsonutils.dumps([
{'mac': node.mac, 'status': 'discover',
'manufacturer': 'old'}
]),
headers=self.default_headers)
self.assertEqual(resp.status_code, 200)
node = self.db.query(Node).get(node.id)
self.assertEqual(node.timestamp, timestamp)
resp = self.app.put(
reverse('NodeAgentHandler'),
jsonutils.dumps(
{'mac': node.mac, 'status': 'discover',
'manufacturer': 'new'}
),
headers=self.default_headers)
self.assertEqual(resp.status_code, 200)
node = self.db.query(Node).get(node.id)
self.assertNotEqual(node.timestamp, timestamp)
self.assertEqual('new', node.manufacturer)
def test_validate_fail_missing_image_source(self):
info = dict(INST_INFO_DICT)
del info["image_source"]
self.node.instance_info = json.dumps(info)
with task_manager.acquire(self.context, self.node.uuid, shared=True) as task:
task.node["instance_info"] = json.dumps(info)
self.assertRaises(exception.MissingParameterValue, task.driver.boot.validate, task)
def test_invalid_metadata_items_on_create(self, snapshot_get_by_id):
snapshot = {
'id': self.req_id,
'expected_attrs': ['metadata']
}
ctx = context.RequestContext('admin', 'fake', True)
snapshot_obj = fake_snapshot.fake_snapshot_obj(ctx, **snapshot)
snapshot_get_by_id.return_value = snapshot_obj
self.stubs.Set(cinder.db, 'snapshot_metadata_update',
return_create_snapshot_metadata)
req = fakes.HTTPRequest.blank(self.url)
req.method = 'POST'
req.headers["content-type"] = "application/json"
# test for long key
data = {"metadata": {"a" * 260: "value1"}}
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
self.controller.create, req, self.req_id, data)
# test for long value
data = {"metadata": {"key": "v" * 260}}
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
self.controller.create, req, self.req_id, data)
# test for empty key.
data = {"metadata": {"": "value1"}}
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.create, req, self.req_id, data)
def test_json_filter_basic_operators(self):
host = fakes.FakeHostState('host1', 'node1', {})
# (operator, arguments, expected_result)
ops_to_test = [
['=', [1, 1], True],
['=', [1, 2], False],
['<', [1, 2], True],
['<', [1, 1], False],
['<', [2, 1], False],
['>', [2, 1], True],
['>', [2, 2], False],
['>', [2, 3], False],
['<=', [1, 2], True],
['<=', [1, 1], True],
['<=', [2, 1], False],
['>=', [2, 1], True],
['>=', [2, 2], True],
['>=', [2, 3], False],
['in', [1, 1], True],
['in', [1, 1, 2, 3], True],
['in', [4, 1, 2, 3], False],
['not', [True], False],
['not', [False], True],
['or', [True, False], True],
['or', [False, False], False],
['and', [True, True], True],
['and', [False, False], False],
['and', [True, False], False],
# Nested ((True or False) and (2 > 1)) == Passes
['and', [['or', True, False], ['>', 2, 1]], True]]
for (op, args, expected) in ops_to_test:
raw = [op] + args
filter_properties = {
'scheduler_hints': {
'query': jsonutils.dumps(raw),
},
}
self.assertEqual(expected,
self.filt_cls.host_passes(host, filter_properties))
# This results in [False, True, False, True] and if any are True
# then it passes...
raw = ['not', True, False, True, False]
filter_properties = {
'scheduler_hints': {
'query': jsonutils.dumps(raw),
},
}
self.assertTrue(self.filt_cls.host_passes(host, filter_properties))
# This results in [False, False, False] and if any are True
# then it passes...which this doesn't
raw = ['not', True, True, True]
filter_properties = {
'scheduler_hints': {
'query': jsonutils.dumps(raw),
},
}
self.assertFalse(self.filt_cls.host_passes(host, filter_properties))
def test_302_location_no_endpoint(self):
fake1 = fakes.FakeHTTPResponse(
302,
'OK',
{'location': 'http://no.where/ishere'},
''
)
fake2 = fakes.FakeHTTPResponse(
200,
'OK',
{'content-type': 'application/json'},
jsonutils.dumps({'Mount': 'Fuji'})
)
self.request.side_effect = [
(fake1, ''), (fake2, jsonutils.dumps({'Mount': 'Fuji'}))]
client = http.SessionClient(session=mock.ANY,
auth=mock.ANY)
resp = client.request('', 'GET', redirect=True)
self.assertEqual(200, resp.status_code)
self.assertEqual({'Mount': 'Fuji'}, utils.get_response_body(resp))
self.assertEqual(('', 'GET'), self.request.call_args_list[0][0])
self.assertEqual(('http://no.where/ishere',
'GET'), self.request.call_args_list[1][0])
for call in self.request.call_args_list:
self.assertEqual({'user_agent': 'python-heatclient',
'raise_exc': False,
'redirect': True}, call[1])
def flavors(test, count):
"""A context manager for constructing flavors for use in testing.
Deletes the flavors after exiting the context.
:param test: Must expose simulate_* methods
:param count: Number of pools to create
:type count: int
:returns: (paths, pool_list capabilities)
:rtype: ([six.text_type], [six.text_type], [dict])
"""
pool_path_all = []
flavor_path_all = []
for i in range(count):
poolname = 'pool' + str(i)
pool_doc = {'weight': 100,
'uri': test.mongodb_url + '/test' + str(i)}
pool_path = test.url_prefix + '/pools/' + poolname
test.simulate_put(pool_path, body=jsonutils.dumps(pool_doc))
flavorname = str(i)
flavor_path = test.url_prefix + "/flavors/" + flavorname
flavor_doc = {'pool_list': [poolname]}
test.simulate_put(flavor_path, body=jsonutils.dumps(flavor_doc))
pool_path_all.append(pool_path)
flavor_path_all.append(flavor_path)
try:
yield flavor_path_all
finally:
for path in flavor_path_all:
test.simulate_delete(path)
for path in pool_path_all:
test.simulate_delete(path)
def get_available_resource(self, nodename):
LOG.debug('In get_available_resource')
local_cpu_info = self._get_cpuinfo()
cpu_topology = local_cpu_info['topology']
vcpus = (int(cpu_topology['cores']) *
int(cpu_topology['sockets']) *
int(cpu_topology['threads']))
local_memory_info = self._get_memory_mb_usage()
local_disk_info = self._get_fs_info(CONF.lxd.root_dir)
data = {
'vcpus': vcpus,
'memory_mb': local_memory_info['total'] / units.Mi,
'memory_mb_used': local_memory_info['used'] / units.Mi,
'local_gb': local_disk_info['total'] / units.Gi,
'local_gb_used': local_disk_info['used'] / units.Gi,
'vcpus_used': 0,
'hypervisor_type': 'lxd',
'hypervisor_version': '011',
'cpu_info': jsonutils.dumps(local_cpu_info),
'hypervisor_hostname': platform.node(),
'supported_instances': jsonutils.dumps(
[(arch.I686, hv_type.LXC, vm_mode.EXE),
(arch.X86_64, hv_type.LXC, vm_mode.EXE)]),
'numa_topology': None,
}
return data
def create(self, name, workflow_identifier, workflow_input=None,
workflow_params=None, pattern=None,
first_time=None, count=None):
self._ensure_not_empty(
name=name,
workflow_identifier=workflow_identifier
)
data = {
'name': name,
'pattern': pattern,
'first_execution_time': first_time,
'remaining_executions': count
}
if uuidutils.is_uuid_like(workflow_identifier):
data.update({'workflow_id': workflow_identifier})
else:
data.update({'workflow_name': workflow_identifier})
if workflow_input:
data.update({'workflow_input': jsonutils.dumps(workflow_input)})
if workflow_params:
data.update({'workflow_params': jsonutils.dumps(workflow_params)})
return self._create('/cron_triggers', data)
def test_invalid_metadata_items_on_update_item(self):
self.stubs.Set(patron.db, 'instance_metadata_update',
return_create_instance_metadata)
self.stubs.Set(patron.db, 'instance_metadata_update',
return_create_instance_metadata)
data = {"metadata": {}}
for num in range(CONF.quota_metadata_items + 1):
data['metadata']['key%i' % num] = "blah"
req = self._get_request()
req.method = 'PUT'
req.body = jsonutils.dumps(data)
req.headers["content-type"] = "application/json"
# test for long key
data = {"metadata": {"a" * 260: "value1"}}
req.body = jsonutils.dumps(data)
self.assertRaises(self.validation_ex_large,
self.controller.update_all, req, self.uuid,
body=data)
# test for long value
data = {"metadata": {"key": "v" * 260}}
req.body = jsonutils.dumps(data)
self.assertRaises(self.validation_ex_large,
self.controller.update_all, req, self.uuid,
body=data)
# test for empty key.
data = {"metadata": {"": "value1"}}
req.body = jsonutils.dumps(data)
self.assertRaises(self.validation_ex,
self.controller.update_all, req, self.uuid,
body=data)
def test_invalid_metadata_items_on_create(self):
self.stubs.Set(db, 'volume_metadata_update',
fake_create_volume_metadata)
req = fakes.HTTPRequest.blank('/v2/fake/volumes/1/action')
req.method = 'POST'
req.headers["content-type"] = "application/json"
data = {"os-set_image_metadata": {
"metadata": {"a" * 260: "value1"}}
}
# Test for long key
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
self.controller.create, req, 1, data)
# Test for long value
data = {"os-set_image_metadata": {
"metadata": {"key": "v" * 260}}
}
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
self.controller.create, req, 1, data)
# Test for empty key.
data = {"os-set_image_metadata": {
"metadata": {"": "value1"}}
}
req.body = jsonutils.dumps(data)
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.create, req, 1, data)
def authorize_console(self, context, token, console_type, host, port,
internal_access_path, instance_uuid,
access_url=None):
token_dict = {'token': token,
'instance_uuid': instance_uuid,
'console_type': console_type,
'host': host,
'port': port,
'internal_access_path': internal_access_path,
'access_url': access_url,
'last_activity_at': time.time()}
data = jsonutils.dumps(token_dict)
self.mc.set(token.encode('UTF-8'), data)
tokens = self._get_tokens_for_instance(instance_uuid)
# Remove the expired tokens from cache.
token_values = self.mc.get_multi(
[tok.encode('UTF-8') for tok in tokens])
tokens = [name for name, value in zip(tokens, token_values)
if value is not None]
tokens.append(token)
self.mc_instance.set(instance_uuid.encode('UTF-8'),
jsonutils.dumps(tokens))
LOG.info("Received Token: %(token)s, %(token_dict)s",
{'token': token, 'token_dict': token_dict})
def prepare_ports_for_replace(self):
if not self.is_using_neutron():
return
data = {'external_ports': [],
'internal_ports': []}
port_data = list(itertools.chain(
[('internal_ports', port) for port in self._data_get_ports()],
[('external_ports', port)
for port in self._data_get_ports('external_ports')]))
for port_type, port in port_data:
# store port fixed_ips for restoring after failed update
port_details = self.client('neutron').show_port(port['id'])['port']
fixed_ips = port_details.get('fixed_ips', [])
data[port_type].append({'id': port['id'], 'fixed_ips': fixed_ips})
if data.get('internal_ports'):
self.data_set('internal_ports',
jsonutils.dumps(data['internal_ports']))
if data.get('external_ports'):
self.data_set('external_ports',
jsonutils.dumps(data['external_ports']))
# reset fixed_ips for these ports by setting for each of them
# fixed_ips to []
for port_type, port in port_data:
self.client('neutron').update_port(
port['id'], {'port': {'fixed_ips': []}})
def upgrade_ubuntu_cobbler_profile_6_0_to_6_1(connection):
select_query = text("SELECT id, generated FROM attributes")
update_query = text(
"UPDATE attributes SET generated = :generated WHERE id = :attr_id")
for attr_id, generated in connection.execute(select_query):
attrs = jsonutils.loads(generated)
if attrs['cobbler']['profile'] == 'ubuntu_1204_x86_64':
attrs['cobbler']['profile'] = 'ubuntu_1404_x86_64'
connection.execute(
update_query,
generated=jsonutils.dumps(attrs),
attr_id=attr_id)
select_query = text("SELECT id, attributes_metadata FROM releases")
update_query = text(
"UPDATE releases SET attributes_metadata = :attrs_meta"
" WHERE id = :release_id")
for release_id, attributes_metadata in connection.execute(select_query):
attrs = jsonutils.loads(attributes_metadata)
if attrs['generated']['cobbler']['profile']['generator_arg'] == \
'ubuntu_1204_x86_64':
attrs['generated']['cobbler']['profile']['generator_arg'] = \
'ubuntu_1404_x86_64'
connection.execute(
update_query,
attrs_meta=jsonutils.dumps(attrs),
release_id=release_id)
def test_call(self, post, get, close):
nef_get = jsonrpc.NexentaJSONProxy(
'http', '1.1.1.1', '8080', 'user', 'pass', method='get')
nef_post = jsonrpc.NexentaJSONProxy(
'http', '1.1.1.1', '8080', 'user', 'pass', method='post')
data = {'key': 'value'}
get.return_value = requests.Response()
post.return_value = requests.Response()
get.return_value.__setstate__({
'status_code': 200, '_content': jsonutils.dumps(data)})
self.assertEqual({'key': 'value'}, nef_get('url'))
get.return_value.__setstate__({
'status_code': 201, '_content': ''})
self.assertEqual('Success', nef_get('url'))
data2 = {'links': [{'href': 'redirect_url'}]}
post.return_value.__setstate__({
'status_code': 202, '_content': jsonutils.dumps(data2)})
get.return_value.__setstate__({
'status_code': 200, '_content': jsonutils.dumps(data)})
self.assertEqual({'key': 'value'}, nef_post('url'))
get.return_value.__setstate__({
'status_code': 200, '_content': ''})
self.assertEqual('Success', nef_post('url', data))
get.return_value.__setstate__({
'status_code': 400,
'_content': jsonutils.dumps({'code': 'ENOENT'})})
self.assertRaises(exception.NexentaException, lambda: nef_get('url'))
def authorize_console(self, context, token, console_type, host, port,
internal_access_path, instance_uuid,
access_url=None):
token_dict = {'token': token,
'instance_uuid': instance_uuid,
'console_type': console_type,
'host': host,
'port': port,
'internal_access_path': internal_access_path,
'access_url': access_url,
'last_activity_at': time.time()}
data = jsonutils.dumps(token_dict)
# We need to log the warning message if the token is not cached
# successfully, because the failure will cause the console for
# instance to not be usable.
if not self.mc.set(token.encode('UTF-8'),
data, CONF.console_token_ttl):
LOG.warning(_LW("Token: %(token)s failed to save into memcached."),
{'token': token})
tokens = self._get_tokens_for_instance(instance_uuid)
# Remove the expired tokens from cache.
tokens = [tok for tok in tokens if self.mc.get(tok.encode('UTF-8'))]
tokens.append(token)
if not self.mc.set(instance_uuid.encode('UTF-8'),
jsonutils.dumps(tokens)):
LOG.warning(_LW("Instance: %(instance_uuid)s failed to save "
"into memcached"),
{'instance_uuid': instance_uuid})
LOG.info(_LI("Received Token: %(token)s, %(token_dict)s"),
{'token': token, 'token_dict': token_dict})
def check_task_created_only_on_new_opt_in(self, handler_method):
def get_settings_value(data, setting_name):
return data['settings']['statistics'][setting_name]['value']
def set_settings_value(data, setting_name, value):
data['settings']['statistics'][setting_name]['value'] = value
with mock.patch('nailgun.api.v1.handlers.master_node_settings.'
'MasterNodeSettingsHandler._handle_stats_opt_in'
) as task_creator:
# Checking called on enabling sending
data = self.get_current_settings()
self.assertFalse(get_settings_value(data, 'user_choice_saved'))
set_settings_value(data, 'user_choice_saved', True)
resp = handler_method(params=jsonutils.dumps(data))
self.assertEqual(200, resp.status_code)
self.assertEqual(1, task_creator.call_count)
# Checking not called on same value
data = self.get_current_settings()
self.assertTrue(get_settings_value(data, 'user_choice_saved'))
resp = handler_method(params=jsonutils.dumps(data))
self.assertEqual(200, resp.status_code)
self.assertEqual(1, task_creator.call_count)
# Checking called on another opt in value
data = self.get_current_settings()
self.assertTrue(get_settings_value(data, 'user_choice_saved'))
opt_in = get_settings_value(data, 'send_anonymous_statistic')
set_settings_value(data, 'send_anonymous_statistic', not opt_in)
resp = handler_method(params=jsonutils.dumps(data))
self.assertEqual(200, resp.status_code)
self.assertEqual(2, task_creator.call_count)
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 200 (task successfully executed)
* 202 (network checking task scheduled for execution)
* 400 (data validation failed)
* 404 (cluster not found in db)
"""
# TODO(pkaminski): this seems to be synchronous, no task needed here
data = jsonutils.loads(web.data())
cluster = self.get_object_or_404(objects.Cluster, cluster_id)
self.check_net_provider(cluster)
self.check_if_network_configuration_locked(cluster)
task_manager = CheckNetworksTaskManager(cluster_id=cluster.id)
task = task_manager.execute(data)
if task.status != consts.TASK_STATUSES.error:
try:
if "networks" in data:
self.validator.validate_networks_update(jsonutils.dumps(data))
if "networking_parameters" in data:
self.validator.validate_neutron_params(jsonutils.dumps(data), cluster_id=cluster_id)
objects.Cluster.get_network_manager(cluster).update(cluster, data)
except Exception as exc:
# set task status to error and update its corresponding data
data = {"status": "error", "progress": 100, "message": six.text_type(exc)}
objects.Task.update(task, data)
logger.error(traceback.format_exc())
self.raise_task(task)
def get(self):
"""Method of REST server to handle request get_notifications.
This method send an RPC call to configurator and returns Notification
data to config-agent
Returns: Dictionary that contains Notification data
"""
global notifications
try:
notification_data = jsonutils.dumps(notifications)
msg = ("NOTIFICATION_DATA sent to config_agent %s"
% notification_data)
LOG.info(msg)
notifications = []
return notification_data
except Exception as err:
pecan.response.status = 500
msg = ("Failed to get notification_data %s."
% str(err).capitalize())
LOG.error(msg)
error_data = self._format_description(msg)
return jsonutils.dumps(error_data)
def test_stats_sending_enabled(self):
self.assertEqual(objects.MasterNodeSettings.must_send_stats(), False)
resp = self.app.get(
reverse("MasterNodeSettingsHandler"),
headers=self.default_headers)
self.assertEqual(200, resp.status_code)
data = resp.json_body
# emulate user confirmed settings in UI
data["settings"]["statistics"]["user_choice_saved"]["value"] = True
resp = self.app.put(
reverse("MasterNodeSettingsHandler"),
headers=self.default_headers,
params=jsonutils.dumps(data)
)
self.assertEqual(200, resp.status_code)
self.assertTrue(objects.MasterNodeSettings.must_send_stats())
# emulate user disabled statistics sending
data["settings"]["statistics"]["send_anonymous_statistic"]["value"] = \
False
resp = self.app.put(
reverse("MasterNodeSettingsHandler"),
headers=self.default_headers,
params=jsonutils.dumps(data)
)
self.assertEqual(200, resp.status_code)
self.assertFalse(objects.MasterNodeSettings.must_send_stats())
def test_rpc_errors(self):
api = create_api()
req = webob.Request.blank('/rpc')
req.method = 'POST'
req.content_type = 'application/json'
# Body is not a list, it should fail
req.body = jsonutils.dumps({})
res = req.get_response(api)
self.assertEqual(400, res.status_int)
# cmd is not dict, it should fail.
req.body = jsonutils.dumps([None])
res = req.get_response(api)
self.assertEqual(400, res.status_int)
# No command key, it should fail.
req.body = jsonutils.dumps([{}])
res = req.get_response(api)
self.assertEqual(400, res.status_int)
# kwargs not dict, it should fail.
req.body = jsonutils.dumps([{"command": "test", "kwargs": 200}])
res = req.get_response(api)
self.assertEqual(400, res.status_int)
# Command does not exist, it should fail.
req.body = jsonutils.dumps([{"command": "test"}])
res = req.get_response(api)
self.assertEqual(404, res.status_int)
def test_update_node_with_wrong_ip(self):
node = self.env.create_node(
api=False, ip='10.20.0.2',
status=consts.NODE_STATUSES.deploying)
ipaddress = '192.168.0.10'
self.app.put(
reverse('NodeAgentHandler'),
jsonutils.dumps({'id': node.id,
'ip': ipaddress,
'status': consts.NODE_STATUSES.discover}),
headers=self.default_headers)
self.assertEqual(node.ip, ipaddress)
self.assertEqual(node.status, consts.NODE_STATUSES.error)
notif = self.db.query(Notification).filter_by(
node_id=node.id,
topic='error'
).first()
self.assertRegexpMatches(notif.message,
"that does not match any Admin network")
admin_ng = objects.NetworkGroup.get_admin_network_group(node)
ipaddress = str(netaddr.IPRange(admin_ng.ip_ranges[0].first,
admin_ng.ip_ranges[0].last)[1])
self.app.put(
reverse('NodeAgentHandler'),
jsonutils.dumps({'id': node.id,
'ip': ipaddress}),
headers=self.default_headers)
self.assertEqual(node.ip, ipaddress)
self.assertEqual(node.status, consts.NODE_STATUSES.discover)
def create(self, workflow_identifier='', namespace='',
workflow_input=None, description='', source_execution_id=None,
**params):
self._ensure_not_empty(
workflow_identifier=workflow_identifier or source_execution_id
)
data = {'description': description}
if uuidutils.is_uuid_like(source_execution_id):
data.update({'source_execution_id': source_execution_id})
if workflow_identifier:
if uuidutils.is_uuid_like(workflow_identifier):
data.update({'workflow_id': workflow_identifier})
else:
data.update({'workflow_name': workflow_identifier})
if namespace:
data.update({'workflow_namespace': namespace})
if workflow_input:
if isinstance(workflow_input, six.string_types):
data.update({'input': workflow_input})
else:
data.update({'input': jsonutils.dumps(workflow_input)})
if params:
data.update({'params': jsonutils.dumps(params)})
return self._create('/executions', data)
def test_parse_exception_http_exception_with_details(self):
details = jsonutils.dumps({
'error': {
'code': 404,
'message': 'Resource BAR is not found.'
}
})
raw = sdk.exc.ResourceNotFound('A message', details, http_status=404)
ex = self.assertRaises(bilean_exc.InternalError,
sdk.parse_exception, raw)
self.assertEqual(404, ex.code)
self.assertEqual('Resource BAR is not found.', six.text_type(ex))
# key name is not 'error' case
details = jsonutils.dumps({
'forbidden': {
'code': 403,
'message': 'Quota exceeded for instances.'
}
})
raw = sdk.exc.ResourceNotFound('A message', details, 403)
ex = self.assertRaises(bilean_exc.InternalError,
sdk.parse_exception, raw)
self.assertEqual(403, ex.code)
self.assertEqual('Quota exceeded for instances.', six.text_type(ex))
def _get_mitaka_db_build_request_no_instance_uuid(self):
fake_info_cache = objects.InstanceInfoCache(network_model=[])
fake_secgroups = objects.SecurityGroupList()
# This is more or less taken straight from bug 1633734.
db_req = {
'created_at': datetime.datetime(2016, 8, 2, 20, 26, 20),
'updated_at': None,
'project_id': self.context.project_id,
'user_id': self.context.user_id,
'display_name': 'admin-auth',
'instance_metadata': None,
'progress': 0,
'vm_state': 'building',
'task_state': 'scheduling',
'image_ref': None,
'access_ip_v4': None,
'access_ip_v6': None,
'info_cache': jsonutils.dumps(fake_info_cache.obj_to_primitive()),
'security_groups':
jsonutils.dumps(fake_secgroups.obj_to_primitive()),
'config_drive': 1,
'key_name': 'Turbo_Fredriksson',
'locked_by': None,
'instance_uuid': None,
'instance': None,
'block_device_mappings': None,
}
return db_req
def test_vif_migrate_data(self):
source_vif = network_model.VIF(
id=uuids.port_id,
network=network_model.Network(id=uuids.network_id),
type=network_model.VIF_TYPE_OVS,
vnic_type=network_model.VNIC_TYPE_NORMAL,
active=True,
profile={'migrating_to': 'dest-host'})
vif_details_dict = {'port_filter': True}
profile_dict = {'trusted': False}
vif_data = objects.VIFMigrateData(
port_id=uuids.port_id,
vnic_type=network_model.VNIC_TYPE_NORMAL,
vif_type=network_model.VIF_TYPE_BRIDGE,
vif_details=vif_details_dict, profile=profile_dict,
host='dest-host', source_vif=source_vif)
# Make sure the vif_details and profile fields are converted and
# stored properly.
self.assertEqual(
jsonutils.dumps(vif_details_dict), vif_data.vif_details_json)
self.assertEqual(
jsonutils.dumps(profile_dict), vif_data.profile_json)
self.assertDictEqual(vif_details_dict, vif_data.vif_details)
self.assertDictEqual(profile_dict, vif_data.profile)
obj = migrate_data.LibvirtLiveMigrateData(
file_backed_memory_discard=False)
obj.vifs = [vif_data]
manifest = ovo_base.obj_tree_get_versions(obj.obj_name())
primitive = obj.obj_to_primitive(target_version='1.8',
version_manifest=manifest)
self.assertIn(
'file_backed_memory_discard', primitive['nova_object.data'])
self.assertNotIn('vifs', primitive['nova_object.data'])
def test_attach_with_invalid_arguments(self):
# Invalid request to attach volume an invalid target
body = {"os-attach": {"mountpoint": "/dev/vdc"}}
req = webob.Request.blank("/v2/fake/volumes/1/action")
req.method = "POST"
req.headers["content-type"] = "application/json"
req.body = jsonutils.dumps(body)
res = req.get_response(fakes.wsgi_app())
self.assertEqual(res.status_int, 400)
# Invalid request to attach volume to an instance and a host
body = {"os-attach": {"instance_uuid": "fake", "host_name": "fake_host", "mountpoint": "/dev/vdc"}}
req = webob.Request.blank("/v2/fake/volumes/1/action")
req.method = "POST"
req.headers["content-type"] = "application/json"
req.body = jsonutils.dumps(body)
res = req.get_response(fakes.wsgi_app())
self.assertEqual(res.status_int, 400)
# Invalid request to attach volume with an invalid mode
body = {"os-attach": {"instance_uuid": "fake", "mountpoint": "/dev/vdc", "mode": "rr"}}
req = webob.Request.blank("/v2/fake/volumes/1/action")
req.method = "POST"
req.headers["content-type"] = "application/json"
req.body = jsonutils.dumps(body)
res = req.get_response(fakes.wsgi_app())
self.assertEqual(res.status_int, 400)
body = {"os-attach": {"host_name": "fake_host", "mountpoint": "/dev/vdc", "mode": "ww"}}
req = webob.Request.blank("/v2/fake/volumes/1/action")
req.method = "POST"
req.headers["content-type"] = "application/json"
req.body = jsonutils.dumps(body)
res = req.get_response(fakes.wsgi_app())
self.assertEqual(res.status_int, 400)
def test_openstack_config_execute_force(self, _):
# Turn node 2 into provisioned state
self.env.nodes[2].status = consts.NODE_STATUSES.provisioned
self.db.flush()
# Try to update OpenStack configuration for cluster
data = {"cluster_id": self.clusters[0].id}
resp = self.app.put(
reverse("OpenstackConfigExecuteHandler"),
jsonutils.dumps(data),
headers=self.default_headers,
expect_errors=True,
)
# Request shouldn't pass a validation
self.assertEqual(resp.status_code, 400)
self.assertEqual(
"Nodes '{0}' are not in status 'ready' and "
"can not be updated directly."
"".format(self.env.nodes[2].uid),
resp.json_body["message"],
)
# Try to update OpenStack configuration for cluster with 'force' key
data = {"cluster_id": self.clusters[0].id, "force": True}
resp = self.app.put(
reverse("OpenstackConfigExecuteHandler"), jsonutils.dumps(data), headers=self.default_headers
)
# Update OpenStack configuration executed successfully
self.assertEqual(resp.status_code, 202)
def test_cluster_node_list_update(self):
node1 = self.env.create_node(api=False, hostname='name1')
cluster = self.env.create_cluster(api=False)
resp = self.app.put(
reverse('ClusterHandler', kwargs={'obj_id': cluster.id}),
jsonutils.dumps({'nodes': [node1.id]}),
headers=self.default_headers,
expect_errors=True
)
self.assertEqual(resp.status_code, 200)
node2 = self.env.create_node(api=False, hostname='name1')
nodes = self.db.query(Node).filter(Node.cluster == cluster).all()
self.assertEqual(1, len(nodes))
self.assertEqual(nodes[0].id, node1.id)
resp = self.app.put(
reverse('ClusterHandler', kwargs={'obj_id': cluster.id}),
jsonutils.dumps({'nodes': [node2.id]}),
headers=self.default_headers
)
self.assertEqual(resp.status_code, 200)
self.assertEqual('node-{0}'.format(node1.id), node1.hostname)
nodes = self.db.query(Node).filter(Node.cluster == cluster)
self.assertEqual(1, nodes.count())
def _show(self, resource_type, response_file,
uuid1, uuid2=None, relations=None):
target_uuid = uuid2 or uuid1
if resource_type.endswith('attachment'):
resource_type = resource_type[:resource_type.index('attachment')]
with open("%s/%s" % (self.fake_files_path, response_file)) as f:
response_template = f.read()
res_dict = getattr(self, '_fake_%s_dict' % resource_type)
for item in res_dict.values():
if 'tags' in item:
item['tags_json'] = jsonutils.dumps(item['tags'])
# replace sec prof rules with their json dump
def jsonify_rules(rule_key):
if rule_key in item:
rules_json = jsonutils.dumps(item[rule_key])
item['%s_json' % rule_key] = rules_json
jsonify_rules('logical_port_egress_rules')
jsonify_rules('logical_port_ingress_rules')
items = [jsonutils.loads(response_template % res_dict[res_uuid])
for res_uuid in res_dict if res_uuid == target_uuid]
if items:
return jsonutils.dumps(items[0])
raise api_exc.ResourceNotFound()
def call(self, method_name, user_parameters):
parameters = {'retry': 'INFINITELY'} # Backend specific setting
if user_parameters:
parameters.update(user_parameters)
call_body = {'jsonrpc': '2.0',
'method': method_name,
'params': parameters,
'id': six.text_type(self._id)}
self.call_counter = 0
self._connection.connect() # prevents httplib timing issue
while self.call_counter < CONNECTION_RETRIES:
self.call_counter += 1
try:
self._id += 1
call_body['id'] = six.text_type(self._id)
LOG.debug("Posting to Quobyte backend: %s",
jsonutils.dumps(call_body))
self._connection.request(
"POST", self._url + '/', jsonutils.dumps(call_body),
dict(Authorization=(self._credentials.
get_authorization_header())))
response = self._connection.getresponse()
self._throw_on_http_error(response)
result = jsonutils.loads(response.read())
LOG.debug("Retrieved data from Quobyte backend: %s", result)
return self._checked_for_application_error(result)
except ssl.SSLError as e:
# Generic catch because OpenSSL does not return
# meaningful errors.
if (not self._disabled_cert_verification
and not self._require_cert_verify):
LOG.warning(_LW(
"Could not verify server certificate of "
"API service against CA."))
self._connection.close()
# Core HTTPSConnection does no certificate verification.
self._connection = httplib.HTTPSConnection(self._netloc)
self._disabled_cert_verification = True
else:
raise exception.QBException(_(
"Client SSL subsystem returned error: %s") % e)
except httplib.BadStatusLine as e:
raise exception.QBException(_(
"If SSL is enabled for the API service, the URL must"
" start with 'https://' for the URL. Failed to parse"
" status code from server response. Error was %s")
% e)
except (httplib.HTTPException, socket.error) as e:
if self._fail_fast:
raise exception.QBException(msg=six.text_type(e))
else:
LOG.warning(_LW("Encountered error, retrying: %s"),
six.text_type(e))
time.sleep(1)
raise exception.QBException("Unable to connect to backend after "
"%s retries" %
six.text_type(CONNECTION_RETRIES))
def _subports_action(self, action, trunk_id, subports):
uri = '%s/trunks/%s/%s' % (self.uri_prefix, trunk_id, action)
resp, body = self.put(uri, jsonutils.dumps({'sub_ports': subports}))
body = self.deserialize_single(body)
self.expected_success(200, resp.status)
return service_client.ResponseBody(resp, body)
def test_json_filter_happy_day(self):
# Test json filter more thoroughly.
raw = [
'and', '$capabilities.enabled',
['=', '$capabilities.opt1', 'match'],
[
'or',
[
'and', ['<', '$free_ram_mb', 30],
['<', '$free_disk_mb', 300]
],
[
'and', ['>', '$free_ram_mb', 30],
['>', '$free_disk_mb', 300]
]
]
]
filter_properties = {
'scheduler_hints': {
'query': jsonutils.dumps(raw),
},
}
# Passes
capabilities = {'opt1': 'match'}
service = {'disabled': False}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 10,
'free_disk_mb': 200,
'capabilities': capabilities,
'service': service
})
self.assertTrue(self.filt_cls.host_passes(host, filter_properties))
# Passes
capabilities = {'opt1': 'match'}
service = {'disabled': False}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 40,
'free_disk_mb': 400,
'capabilities': capabilities,
'service': service
})
self.assertTrue(self.filt_cls.host_passes(host, filter_properties))
# Fails due to capabilities being disabled
capabilities = {'enabled': False, 'opt1': 'match'}
service = {'disabled': False}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 40,
'free_disk_mb': 400,
'capabilities': capabilities,
'service': service
})
self.assertFalse(self.filt_cls.host_passes(host, filter_properties))
# Fails due to being exact memory/disk we don't want
capabilities = {'enabled': True, 'opt1': 'match'}
service = {'disabled': False}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 30,
'free_disk_mb': 300,
'capabilities': capabilities,
'service': service
})
self.assertFalse(self.filt_cls.host_passes(host, filter_properties))
# Fails due to memory lower but disk higher
capabilities = {'enabled': True, 'opt1': 'match'}
service = {'disabled': False}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 20,
'free_disk_mb': 400,
'capabilities': capabilities,
'service': service
})
self.assertFalse(self.filt_cls.host_passes(host, filter_properties))
# Fails due to capabilities 'opt1' not equal
capabilities = {'enabled': True, 'opt1': 'no-match'}
service = {'enabled': True}
host = fakes.FakeHostState(
'host1', 'node1', {
'free_ram_mb': 20,
'free_disk_mb': 400,
'capabilities': capabilities,
'service': service
})
self.assertFalse(self.filt_cls.host_passes(host, filter_properties))
def json_dumps_compact(data):
return json.dumps(data, separators=(',', ':'))
fake_vpmems = [
resource.LibvirtVPMEMDevice(label='4GB',
name='ns_0',
devpath='/dev/dax0.0',
size=4292870144,
align=2097152),
resource.LibvirtVPMEMDevice(label='4GB',
name='ns_1',
devpath='/dev/dax0.0',
size=4292870144,
align=2097152)
]
fake_instance_extras = {
'resources': jsonutils.dumps(fake_resources.obj_to_primitive())
}
class TestResourceObject(test_objects._LocalTest):
def _create_resource(self, metadata=None):
fake_resource = resource.Resource(provider_uuid=uuids.rp,
resource_class='bar',
identifier='foo')
if metadata:
fake_resource.metadata = metadata
return fake_resource
def _test_set_malformed_resource_class(self, rc):
try:
resource.Resource(provider_uuid=uuids.rp,
def _json_dumps(properties, attr):
prop = properties[attr]
if not isinstance(prop, six.string_types):
properties[attr] = jsonutils.dumps(prop)
def send_heartbeat(self):
context = {}
args = jsonutils.dumps(dict(when=time.ctime(), agent=thishost))
msg = self.clnt.make_msg('heartbeat', context, msg=args)
resp = self.clnt.cast(msg)
LOG.debug("send_heartbeat: resp = %s", resp)
def setUp(self):
super(TestRegistryURLVisibility, self).setUp()
self.cleanup()
self.registry_server.deployment_flavor = ''
self.req_body = jsonutils.dumps([{"command": "image_get_all"}])
def set_policy_rules(self, rules):
fap = open(self.policy_file, 'w')
fap.write(jsonutils.dumps(rules))
fap.close()
def test_update_devices_from_hypervisor_resources(self, _mock_dev_assign):
fake_pci_devs = [copy.deepcopy(fake_pci), copy.deepcopy(fake_pci_2)]
fake_pci_devs_json = jsonutils.dumps(fake_pci_devs)
tracker = manager.PciDevTracker(self.fake_context)
tracker.update_devices_from_hypervisor_resources(fake_pci_devs_json)
self.assertEqual(2, len(tracker.pci_devs))
def serialize(self, data):
return jsonutils.dumps(data)
def db_encrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50):
"""Encrypt parameters and properties for all templates in db.
:param ctxt: RPC context
:param encryption_key: key that will be used for parameter and property
encryption
:param batch_size: number of templates requested from db in each iteration.
50 means that heat requests 50 templates, encrypt them
and proceed with next 50 items.
"""
from heat.engine import template
session = get_session()
with session.begin():
query = session.query(models.RawTemplate)
for raw_template in _get_batch(
session=session, ctxt=ctxt, query=query,
model=models.RawTemplate, batch_size=batch_size):
tmpl = template.Template.load(ctxt, raw_template.id, raw_template)
env = raw_template.environment
if 'encrypted_param_names' in env:
encrypted_params = env['encrypted_param_names']
else:
encrypted_params = []
for param_name, param in tmpl.param_schemata().items():
if (param_name in encrypted_params) or (not param.hidden):
continue
try:
param_val = env['parameters'][param_name]
except KeyError:
param_val = param.default
encrypted_val = crypt.encrypt(param_val, encryption_key)
env['parameters'][param_name] = encrypted_val
encrypted_params.append(param_name)
if encrypted_params:
environment = env.copy()
environment['encrypted_param_names'] = encrypted_params
raw_template_update(ctxt, raw_template.id,
{'environment': environment})
query = session.query(models.Resource).filter(
~models.Resource.properties_data.is_(None),
~models.Resource.properties_data_encrypted.is_(True))
for resource in _get_batch(
session=session, ctxt=ctxt, query=query, model=models.Resource,
batch_size=batch_size):
result = {}
for prop_name, prop_value in resource.properties_data.items():
prop_string = jsonutils.dumps(prop_value)
encrypted_value = crypt.encrypt(prop_string,
encryption_key)
result[prop_name] = encrypted_value
resource.properties_data = result
resource.properties_data_encrypted = True
resource_update(ctxt, resource.id,
{'properties_data': result,
'properties_data_encrypted': True},
resource.atomic_key)
def notify(self, ctxt, message, priority, retry):
logger = logging.getLogger('%s.%s' %
(self.LOGGER_BASE, message['event_type']))
method = getattr(logger, priority.lower(), None)
if method:
method(strutils.mask_password(jsonutils.dumps(message)))
def force_delete_volume(self, volume_id):
"""Force Delete Volume."""
post_body = json.dumps({'os-force_delete': {}})
resp, body = self.post('volumes/%s/action' % volume_id, post_body)
self.validate_response(schema.force_delete_volume, resp, body)
return rest_client.ResponseBody(resp, body)
def _set_policy(self, new_policy):
with open(self.tmpfilename, "w") as policyfile:
policyfile.write(jsonutils.dumps(new_policy))
def assertJsonEqual(self, expected, observed):
"""Asserts that 2 complex data structures are json equivalent."""
self.assertEqual(jsonutils.dumps(expected, sort_keys=True),
jsonutils.dumps(observed, sort_keys=True))
def default(self, data):
def sanitizer(obj):
return six.text_type(obj)
return jsonutils.dumps(data, default=sanitizer)
def pretty_data(data):
data = jsonutils.dumps(jsonutils.loads(data), sort_keys=True,
indent=4)
return '\n'.join(line.rstrip() for line in data.split('\n')).strip()
def stub_instance(id=1,
user_id=None,
project_id=None,
host=None,
node=None,
vm_state=None,
task_state=None,
reservation_id="",
uuid=FAKE_UUID,
image_ref="10",
flavor_id="1",
name=None,
key_name='',
access_ipv4=None,
access_ipv6=None,
progress=0,
auto_disk_config=False,
display_name=None,
include_fake_metadata=True,
config_drive=None,
power_state=None,
nw_cache=None,
metadata=None,
security_groups=None,
root_device_name=None,
limit=None,
marker=None,
launched_at=timeutils.utcnow(),
terminated_at=timeutils.utcnow(),
availability_zone='',
locked_by=None,
cleaned=False,
memory_mb=0,
vcpus=0,
root_gb=0,
ephemeral_gb=0,
instance_type=None,
launch_index=0,
kernel_id="",
ramdisk_id="",
user_data=None):
if user_id is None:
user_id = 'fake_user'
if project_id is None:
project_id = 'fake_project'
if metadata:
metadata = [{'key': k, 'value': v} for k, v in metadata.items()]
elif include_fake_metadata:
metadata = [models.InstanceMetadata(key='seq', value=str(id))]
else:
metadata = []
inst_type = flavors.get_flavor_by_flavor_id(int(flavor_id))
sys_meta = flavors.save_flavor_info({}, inst_type)
if host is not None:
host = str(host)
if key_name:
key_data = 'FAKE'
else:
key_data = ''
if security_groups is None:
security_groups = [{
"id": 1,
"name": "test",
"description": "Foo:",
"project_id": "project",
"user_id": "user",
"created_at": None,
"updated_at": None,
"deleted_at": None,
"deleted": False
}]
# ReservationID isn't sent back, hack it in there.
server_name = name or "server%s" % id
if reservation_id != "":
server_name = "reservation_%s" % (reservation_id, )
info_cache = create_info_cache(nw_cache)
if instance_type is None:
instance_type = flavors.get_default_flavor()
flavorinfo = jsonutils.dumps({
'cur': instance_type.obj_to_primitive(),
'old': None,
'new': None,
})
instance = {
"id": int(id),
"created_at": datetime.datetime(2010, 10, 10, 12, 0, 0),
"updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0),
"deleted_at": datetime.datetime(2010, 12, 12, 10, 0, 0),
"deleted": None,
"user_id": user_id,
"project_id": project_id,
"image_ref": image_ref,
"kernel_id": kernel_id,
"ramdisk_id": ramdisk_id,
"launch_index": launch_index,
"key_name": key_name,
"key_data": key_data,
"config_drive": config_drive,
"vm_state": vm_state or vm_states.BUILDING,
"task_state": task_state,
"power_state": power_state,
"memory_mb": memory_mb,
"vcpus": vcpus,
"root_gb": root_gb,
"ephemeral_gb": ephemeral_gb,
"ephemeral_key_uuid": None,
"hostname": display_name or server_name,
"host": host,
"node": node,
"instance_type_id": 1,
"instance_type": inst_type,
"user_data": user_data,
"reservation_id": reservation_id,
"mac_address": "",
"launched_at": launched_at,
"terminated_at": terminated_at,
"availability_zone": availability_zone,
"display_name": display_name or server_name,
"display_description": "",
"locked": locked_by is not None,
"locked_by": locked_by,
"metadata": metadata,
"access_ip_v4": access_ipv4,
"access_ip_v6": access_ipv6,
"uuid": uuid,
"progress": progress,
"auto_disk_config": auto_disk_config,
"name": "instance-%s" % id,
"shutdown_terminate": True,
"disable_terminate": False,
"security_groups": security_groups,
"root_device_name": root_device_name,
"system_metadata": utils.dict_to_metadata(sys_meta),
"pci_devices": [],
"vm_mode": "",
"default_swap_device": "",
"default_ephemeral_device": "",
"launched_on": "",
"cell_name": "",
"architecture": "",
"os_type": "",
"extra": {
"numa_topology": None,
"pci_requests": None,
"flavor": flavorinfo,
},
"cleaned": cleaned
}
instance.update(info_cache)
instance['info_cache']['instance_uuid'] = instance['uuid']
return instance
def process_bind_param(self, value, dialect):
return zlib.compress(jsonutils.dumps(value).encode('utf-8'))
def _invoke_delete_request(self, network_name):
data = {'NetworkID': network_name}
response = self.app.post('/NetworkDriver.DeleteNetwork',
content_type='application/json',
data=jsonutils.dumps(data))
return response
def _call(self, action, resource, data, headers, binary=False):
if resource.startswith('http'):
uri = resource
else:
uri = self.base_uri + resource
if binary:
body = data
else:
body = jsonutils.dumps(data)
debug_data = 'binary' if binary else body
debug_data = debug_data if debug_data else 'EMPTY'
if not headers:
headers = {'Authorization': 'Basic %s' % self.auth}
else:
headers['Authorization'] = 'Basic %s' % self.auth
conn = None
if self.ssl:
if not self.ssl_verify_context:
# HTTPS certificate verification was cancelled in
# configuration. If python version has context attribute,
# switch the default context to unverified
try:
_create_unverified_https_context =\
ssl._create_unverified_context
except AttributeError:
# Legacy Python that doesn't verify HTTPS
# certificates by default
pass
else:
# Handle target environment that doesn't
# support HTTPS verification
ssl._create_default_https_context =\
_create_unverified_https_context
conn = http_client.HTTPSConnection(self.server,
self.port,
timeout=self.timeout)
if conn is None:
LOG.error('vdirectRESTClient: Could not establish HTTPS '
'connection')
return 0, None, None, None
else:
conn = http_client.HTTPConnection(self.server,
self.port,
timeout=self.timeout)
if conn is None:
LOG.error('vdirectRESTClient: Could not establish HTTP '
'connection')
return 0, None, None, None
try:
conn.request(action, uri, body, headers)
response = conn.getresponse()
respstr = response.read()
respdata = respstr
try:
respdata = jsonutils.loads(respstr)
except ValueError:
# response was not JSON, ignore the exception
pass
ret = (response.status, response.reason, respstr, respdata)
except Exception as e:
log_dict = {'action': action, 'e': e}
LOG.error('vdirectRESTClient: %(action)s failure, %(e)r', log_dict)
ret = -1, None, None, None
conn.close()
return ret
def process_bind_param(self, value, dialect):
return jsonutils.dumps(value)
def _execute(self):
# TODO(sbauza): Remove that once prep_resize() accepts a RequestSpec
# object in the signature and all the scheduler.utils methods too
legacy_spec = self.request_spec.to_legacy_request_spec_dict()
legacy_props = self.request_spec.to_legacy_filter_properties_dict()
scheduler_utils.setup_instance_group(self.context, self.request_spec)
# If a target host is set in a requested destination,
# 'populate_retry' need not be executed.
if not ('requested_destination' in self.request_spec
and self.request_spec.requested_destination
and 'host' in self.request_spec.requested_destination):
scheduler_utils.populate_retry(legacy_props, self.instance.uuid)
# NOTE(sbauza): Force_hosts/nodes needs to be reset
# if we want to make sure that the next destination
# is not forced to be the original host
self.request_spec.reset_forced_destinations()
# NOTE(danms): Right now we only support migrate to the same
# cell as the current instance, so request that the scheduler
# limit thusly.
instance_mapping = objects.InstanceMapping.get_by_instance_uuid(
self.context, self.instance.uuid)
LOG.debug('Requesting cell %(cell)s while migrating',
{'cell': instance_mapping.cell_mapping.identity},
instance=self.instance)
if ('requested_destination' in self.request_spec
and self.request_spec.requested_destination):
self.request_spec.requested_destination.cell = (
instance_mapping.cell_mapping)
# NOTE(takashin): In the case that the target host is specified,
# if the migration is failed, it is not necessary to retry
# the cold migration to the same host. So make sure that
# reschedule will not occur.
if 'host' in self.request_spec.requested_destination:
legacy_props.pop('retry', None)
self.request_spec.retry = None
else:
self.request_spec.requested_destination = objects.Destination(
cell=instance_mapping.cell_mapping)
# Once _preallocate_migration() is done, the source node allocation is
# moved from the instance consumer to the migration record consumer,
# and the instance consumer doesn't have any allocations. If this is
# the first time through here (not a reschedule), select_destinations
# below will allocate resources on the selected destination node for
# the instance consumer. If we're rescheduling, host_list is not None
# and we'll call claim_resources for the instance and the selected
# alternate. If we exhaust our alternates and raise MaxRetriesExceeded,
# the rollback() method should revert the allocation swaparoo and move
# the source node allocation from the migration record back to the
# instance record.
migration = self._preallocate_migration()
self.request_spec.ensure_project_and_user_id(self.instance)
# On an initial call to migrate, 'self.host_list' will be None, so we
# have to call the scheduler to get a list of acceptable hosts to
# migrate to. That list will consist of a selected host, along with
# zero or more alternates. On a reschedule, though, the alternates will
# be passed to this object and stored in 'self.host_list', so we can
# pop the first alternate from the list to use for the destination, and
# pass the remaining alternates to the compute.
if self.host_list is None:
selection_lists = self.scheduler_client.select_destinations(
self.context,
self.request_spec, [self.instance.uuid],
return_objects=True,
return_alternates=True)
# Since there is only ever one instance to migrate per call, we
# just need the first returned element.
selection_list = selection_lists[0]
# The selected host is the first item in the list, with the
# alternates being the remainder of the list.
selection, self.host_list = selection_list[0], selection_list[1:]
else:
# This is a reschedule that will use the supplied alternate hosts
# in the host_list as destinations. Since the resources on these
# alternates may have been consumed and might not be able to
# support the migrated instance, we need to first claim the
# resources to verify the host still has sufficient availabile
# resources.
elevated = self.context.elevated()
host_available = False
while self.host_list and not host_available:
selection = self.host_list.pop(0)
if selection.allocation_request:
alloc_req = jsonutils.loads(selection.allocation_request)
else:
alloc_req = None
if alloc_req:
# If this call succeeds, the resources on the destination
# host will be claimed by the instance.
host_available = scheduler_utils.claim_resources(
elevated, self.reportclient, self.request_spec,
self.instance.uuid, alloc_req,
selection.allocation_request_version)
else:
# Some deployments use different schedulers that do not
# use Placement, so they will not have an
# allocation_request to claim with. For those cases,
# there is no concept of claiming, so just assume that
# the host is valid.
host_available = True
# There are no more available hosts. Raise a MaxRetriesExceeded
# exception in that case.
if not host_available:
reason = ("Exhausted all hosts available for retrying build "
"failures for instance %(instance_uuid)s." % {
"instance_uuid": self.instance.uuid
})
raise exception.MaxRetriesExceeded(reason=reason)
scheduler_utils.populate_filter_properties(legacy_props, selection)
# context is not serializable
legacy_props.pop('context', None)
(host, node) = (selection.service_host, selection.nodename)
self.instance.availability_zone = (
availability_zones.get_host_availability_zone(self.context, host))
# FIXME(sbauza): Serialize/Unserialize the legacy dict because of
# oslo.messaging #1529084 to transform datetime values into strings.
# tl;dr: datetimes in dicts are not accepted as correct values by the
# rpc fake driver.
legacy_spec = jsonutils.loads(jsonutils.dumps(legacy_spec))
LOG.debug(
"Calling prep_resize with selected host: %s; "
"Selected node: %s; Alternates: %s",
host,
node,
self.host_list,
instance=self.instance)
# RPC cast to the destination host to start the migration process.
self.compute_rpcapi.prep_resize(self.context,
self.instance,
legacy_spec['image'],
self.flavor,
host,
migration,
request_spec=legacy_spec,
filter_properties=legacy_props,
node=node,
clean_shutdown=self.clean_shutdown,
host_list=self.host_list)
def connection_info(self, info):
self.connection_info_json = jsonutils.dumps(info)
def prepare():
meta = base.reflect_db_metadata()
releaseid = insert_table_row(
meta.tables['releases'], {
'name':
'test_name',
'version':
'2014.2.2-6.1',
'operating_system':
'ubuntu',
'state':
'available',
'networks_metadata':
jsonutils.dumps({'neutron': {
'networks': [],
'config': {}
}})
})
clusterid = insert_table_row(
meta.tables['clusters'], {
'name': 'test_env',
'release_id': releaseid,
'mode': 'ha_compact',
'status': 'new',
'net_provider': 'neutron',
'grouping': 'roles',
'fuel_version': '7.0',
})
db.execute(meta.tables['nodegroups'].insert(), [
{
'cluster_id': clusterid,
'name': 'test_nodegroup_a'
},
{
'cluster_id': clusterid,
'name': 'test_nodegroup_a'
},
{
'cluster_id': clusterid,
'name': 'test_nodegroup_b'
},
{
'cluster_id': clusterid,
'name': 'test_nodegroup_b'
},
{
'cluster_id': clusterid,
'name': consts.NODE_GROUPS.default
},
])
netconfigid = insert_table_row(
meta.tables['networking_configs'], {
'cluster_id': None,
'dns_nameservers': ['8.8.8.8'],
'floating_ranges': [],
'configuration_template': None,
})
db.execute(meta.tables['neutron_config'].insert(),
[{
'id': netconfigid,
'vlan_range': [],
'gre_id_range': [],
'base_mac': '00:00:00:00:00:00',
'internal_cidr': '10.10.10.00/24',
'internal_gateway': '10.10.10.01',
'segmentation_type': 'vlan',
'net_l23_provider': 'ovs'
}])
result = db.execute(meta.tables['plugins'].insert(), [{
'name':
'test_plugin_a',
'title':
'Test plugin A',
'version':
'1.0.0',
'description':
'Test plugin A for Fuel',
'homepage':
'http://fuel_plugins.test_plugin.com',
'package_version':
'3.0.0',
'groups':
jsonutils.dumps(['tgroup']),
'authors':
jsonutils.dumps(['tauthor']),
'licenses':
jsonutils.dumps(['tlicense']),
'releases':
jsonutils.dumps([{
'repository_path': 'repositories/ubuntu'
}]),
'fuel_version':
jsonutils.dumps(['6.1', '7.0']),
}])
pluginid_a = result.inserted_primary_key[0]
result = db.execute(meta.tables['plugins'].insert(), [{
'name':
'test_plugin_b',
'title':
'Test plugin B',
'version':
'1.0.0',
'description':
'Test plugin B for Fuel',
'homepage':
'http://fuel_plugins.test_plugin.com',
'package_version':
'3.0.0',
'groups':
jsonutils.dumps(['tgroup']),
'authors':
jsonutils.dumps(['tauthor']),
'licenses':
jsonutils.dumps(['tlicense']),
'releases':
jsonutils.dumps([{
'repository_path': 'repositories/ubuntu'
}]),
'fuel_version':
jsonutils.dumps(['6.1', '7.0']),
}])
pluginid_b = result.inserted_primary_key[0]
db.execute(meta.tables['cluster_plugins'].insert(),
[{
'cluster_id': clusterid,
'plugin_id': pluginid_a
}, {
'cluster_id': clusterid,
'plugin_id': pluginid_b
}])
db.execute(meta.tables['attributes'].insert(), [{
'cluster_id':
clusterid,
'editable':
jsonutils.dumps({
'test_plugin_a': {
'metadata': {
'plugin_id': pluginid_a,
'enabled': True,
'toggleable': True,
'weight': 70,
},
'attribute': {
'value': 'value',
'type': 'text',
'description': 'description',
'weight': 25,
'label': 'label'
}
},
'test_plugin_b': {
'metadata': {
'plugin_id': pluginid_b,
'enabled': False,
'toggleable': True,
'weight': 80,
}
}
}),
'generated':
jsonutils.dumps({}),
}])
db.commit()
def serialize(self, object_dict):
"""Serialize an Watcher object."""
return jsonutils.dumps(object_dict)
def ensure_images(glance_client,
image_specs,
base_url,
local_path=None,
is_package_public=False):
"""Ensure that images are available
Ensure that images from image_specs are available in glance. If not
attempts: instructs glance to download the images and sets murano-specific
metadata for it.
"""
def _image_valid(image, keys):
for key in keys:
if key not in image:
LOG.warning("Image specification invalid: "
"No {0} key in image ").format(key)
return False
return True
keys = [
'Name',
'DiskFormat',
'ContainerFormat',
]
installed_images = []
for image_spec in image_specs:
if not _image_valid(image_spec, keys):
continue
filters = {
'name': image_spec["Name"],
'disk_format': image_spec["DiskFormat"],
'container_format': image_spec["ContainerFormat"],
}
images = glance_client.images.list(filters=filters)
try:
img = next(images).to_dict()
except StopIteration:
img = None
update_metadata = False
if img:
LOG.info("Found desired image {0}, id {1}".format(
img['name'], img['id']))
# check for murano meta-data
if 'murano_image_info' in img.get('properties', {}):
LOG.info("Image {0} already has murano meta-data".format(
image_spec['Name']))
else:
update_metadata = True
else:
LOG.info("Desired image {0} not found attempting "
"to download".format(image_spec['Name']))
update_metadata = True
img_file = None
if local_path:
img_file = os.path.join(local_path, image_spec['Name'])
if img_file and not os.path.exists(img_file):
LOG.error("Image file {0} does not exist.".format(img_file))
if img_file and os.path.exists(img_file):
img = glance_client.images.create(
name=image_spec['Name'],
container_format=image_spec['ContainerFormat'],
disk_format=image_spec['DiskFormat'],
data=open(img_file, 'rb'),
)
img = img.to_dict()
else:
download_url = to_url(
image_spec.get("Url", image_spec['Name']),
base_url=base_url,
path='images/',
)
LOG.info("Instructing glance to download image {0}".format(
image_spec['Name']))
img = glance_client.images.create(
name=image_spec["Name"],
container_format=image_spec['ContainerFormat'],
disk_format=image_spec['DiskFormat'],
copy_from=download_url)
img = img.to_dict()
if is_package_public:
try:
glance_client.images.update(img['id'], is_public=True)
LOG.debug('Success update for image {0}'.format(img['id']))
except Exception as e:
LOG.exception(
_("Error {0} occurred while setting "
"image {1} public").format(e, img['id']))
installed_images.append(img)
if update_metadata and 'Meta' in image_spec:
LOG.info("Updating image {0} metadata".format(
image_spec['Name']))
murano_image_info = jsonutils.dumps(image_spec['Meta'])
glance_client.images.update(
img['id'],
properties={'murano_image_info': murano_image_info})
return installed_images
def default(self, data):
return six.b(jsonutils.dumps(data))
def main():
api_config.register_config()
cfg.CONF(args=[],
project='bagpipe-rest-attach',
default_config_files=['/etc/bagpipe-bgp/bgp.conf'])
usage = "usage: %prog [--attach|--detach] --network-type (ipvpn|evpn) "\
"--port (<port>|netns) --ip <ip>[/<mask>] [options] (see --help)"
parser = optparse.OptionParser(usage)
parser.add_option("--attach",
dest="operation",
action="store_const",
const="attach",
help="attach local port")
parser.add_option("--detach",
dest="operation",
action="store_const",
const="detach",
help="detach local port")
parser.add_option("--network-type",
dest="network_type",
help="network type (ipvpn or evpn)",
choices=[const.IPVPN, const.EVPN])
parser.add_option("--vpn-instance-id",
dest="vpn_instance_id",
help="UUID for the network instance "
"(default: %default-(ipvpn|evpn))",
default=DEFAULT_VPN_INSTANCE_ID)
parser.add_option("--port",
dest="port",
help="local port to attach/detach (use special port "
"'netns[:if]' to have an interface to a local network "
"namespace attached/detached "
"[with 'if' as the name of the interface to the netns]")
parser.add_option("--direction",
dest="direction",
choices=[const.TO_PORT, const.FROM_PORT, const.BOTH],
default=const.BOTH,
help=("local port direction (to-port|from-port|both) "
"in VPN (default: %default)"))
parser.add_option("--rt",
dest="route_targets",
help="route target [default: 64512:0] (can be "
"specified multiple times)",
default=[],
action="append")
parser.add_option("--import-rt",
dest="import_only_rts",
help="import-only route target (can be specified"
"multiple times)",
default=[],
action="append")
parser.add_option("--export-rt",
dest="export_only_rts",
help="export-only route target (can be specified"
"multiple times)",
default=[],
action="append")
parser.add_option("--ip",
dest="ip",
help="IP prefix / mask (mask defaults to /24)")
parser.add_option("--gateway-ip",
dest="gw_ip",
help="IP address of network gateway (optional, "
"defaults to last IP in range)")
parser.add_option("--mac",
dest="mac",
help="MAC address (required for evpn if port"
" is not 'netns')")
parser.set_defaults(advertise_subnet=False)
parser.add_option("--advertise-singleton",
action="store_false",
dest="advertise_subnet",
help="advertise IP address as a /32 (default)")
parser.add_option("--advertise-subnet",
action="store_true",
dest="advertise_subnet",
help="advertise the whole IP subnet")
parser.add_option("--ovs-preplug",
action="store_true",
dest="ovs_preplug",
default=False,
help="should we prealably plug the port "
"into an OVS bridge")
parser.add_option("--ovs-bridge",
dest="bridge",
default="br-int",
help="if preplug, specifies which OVS bridge to use"
" (default: %default)")
parser.add_option("--ovs-vlan",
dest="ovs_vlan",
type='int',
help="if specified, only this VLAN from the OVS "
"interface will be attached to the VPN instance "
"(optional)")
parser.add_option("--netns",
dest="netns",
help="name of network namespace (optional, for use with"
" --port netns)")
parser.add_option("--if2vpn",
dest="if2vpn",
default=NS2VPN_DEFAULT_IFNAME,
help="name of interface in netns toward VPN"
"defaults to %default "
"(optional, for use with --port netns)")
parser.add_option("--readv-from-rt",
dest="readv_from_rts",
help="enables route readvertisement from these RTs,"
" works in conjunction with --readv-to-rt",
default=[],
action="append")
parser.add_option("--readv-to-rt",
dest="readv_to_rts",
help="enables route readvertisement to these RTs,"
" works in conjunction with --readv-from-rt",
default=[],
action="append")
parser.add_option("--redirect-rts",
dest="redirect_rts",
help="Redirection Route Targets to attract traffic, "
"matching the traffic classifier, in specified VRF from "
"any VRF importing this route target",
default=[],
action="append")
parser.add_option("--source-prefix",
dest="sourcePrefix",
type="string",
help="Traffic classifier source prefix "
"filter",
action="callback",
callback=classifier_callback)
parser.add_option("--destination-prefix",
dest="destinationPrefix",
type="string",
help="Traffic classifier destination "
"prefix filter",
action="callback",
callback=classifier_callback)
parser.add_option("--source-port",
dest="sourcePort",
type="string",
help="Traffic classifier source port "
"number or range filter",
action="callback",
callback=classifier_callback)
parser.add_option("--destination-port",
dest="destinationPort",
type="string",
help="Traffic classifier destination port"
" number or range filter",
action="callback",
callback=classifier_callback)
parser.add_option("--protocol",
dest="protocol",
type="string",
help="Traffic classifier IP protocol "
"filter",
action="callback",
callback=classifier_callback)
parser.add_option("--attract-to-rt",
dest="attract_to_rts",
help="enables route advertisement to these RTs,"
" works in conjunction with "
"--static-destination-prefix",
default=[],
action="append")
parser.add_option("--static-destination-prefix",
dest="static_dest_prefixes",
help="static destination prefix to advertise,"
" works in conjunction with --attract-to-rts",
default=[],
action="append")
parser.add_option("--lb-consistent-hash-order",
dest="lb_consistent_hash_order",
default=0,
type="int",
help="Load Balancing consistent hash sort order")
parser.add_option("--vni",
dest="vni",
default=0,
type="int",
help="VXLAN VNI to use for this VPN instance (optional)")
parser.add_option("--local-pref",
dest="local_pref",
default=None,
type="int",
help="BGP LOCAL PREF attribute (optional)")
(options, _unused) = parser.parse_args()
if not (options.operation):
parser.error("Need to specify --attach or --detach")
if not (options.port):
parser.error("Need to specify --port <localport>")
if not (options.network_type):
parser.error("Need to specify --network-type")
if not (options.ip):
parser.error("Need to specify --ip")
if (len(options.route_targets) == 0
and not (options.import_only_rts or options.export_only_rts)):
if options.network_type == const.IPVPN:
options.route_targets = ["64512:512"]
else:
options.route_targets = ["64512:513"]
import_rts = copy.copy(options.route_targets or [])
for rt in options.import_only_rts:
import_rts.append(rt)
export_rts = copy.copy(options.route_targets or [])
for rt in options.export_only_rts:
export_rts.append(rt)
if not re.match('.*/[0-9]+$', options.ip):
options.ip = options.ip + "/24"
if not (options.gw_ip):
net = netaddr.IPNetwork(options.ip)
print("using %s as gateway address" % str(net[-2]))
options.gw_ip = str(net[-2])
if options.vpn_instance_id == DEFAULT_VPN_INSTANCE_ID:
options.vpn_instance_id = "%s-%s" % (options.network_type,
options.vpn_instance_id)
if options.port.startswith("netns"):
if not options.netns:
options.netns = options.vpn_instance_id
try:
(_unused, options.if2netns) = options.port.split(":")
except Exception:
options.if2netns = get_vpn2ns_if_name(options.netns)
if options.operation == "attach":
create_special_netns_port(options)
options.port = options.if2netns
if not options.mac:
options.mac = net_utils.get_device_mac(run_log_command,
options.if2vpn,
options.netns)
print("Local port: %s (%s)" % (options.port, options.mac))
run_log_command("ip link show %s" % options.port)
local_port = {}
if options.port[:5] == "evpn:":
if (options.network_type == const.IPVPN):
print("will plug evpn %s into the IPVPN" % options.port[5:])
local_port['evpn'] = {'id': options.port[5:]}
else:
raise Exception("Can only plug an evpn into an ipvpn")
else:
local_port['linuxif'] = options.port
# currently our only the MPLS OVS driver for ipvpn requires preplug
if (options.ovs_preplug and options.network_type == const.IPVPN):
print("pre-plugging %s into %s" % (options.port, options.bridge))
run_log_command("ovs-vsctl del-port %s %s" %
(options.bridge, options.port),
raise_on_error=False)
run_log_command("ovs-vsctl add-port %s %s" %
(options.bridge, options.port))
local_port['ovs'] = {'port_name': options.port, 'plugged': True}
if options.ovs_vlan:
local_port['ovs']['vlan'] = options.ovs_vlan
if not options.mac:
if options.network_type == const.IPVPN:
options.mac = "52:54:00:99:99:22"
else:
parser.error("Need to specify --mac for an EVPN network "
"attachment if port is not 'netns'")
readvertise = None
if options.readv_to_rts:
readvertise = {
"from_rt": options.readv_from_rts,
"to_rt": options.readv_to_rts
}
attract_traffic = dict()
if options.redirect_rts:
if options.classifier:
attract_traffic.update(
dict(redirect_rts=options.redirect_rts,
classifier=options.classifier))
else:
parser.error("Need to specify --redirect-rt and at least one "
"traffic classifier option")
if options.attract_to_rts:
if options.static_dest_prefixes:
attract_traffic.update(
dict(to=options.attract_to_rts,
static_destination_prefixes=options.
static_dest_prefixes))
else:
parser.error("Need to specify --attract-to-rt and at least "
"one static destination prefix option")
data = {
"import_rt": import_rts,
"export_rt": export_rts,
"local_port": local_port,
"vpn_instance_id": options.vpn_instance_id,
"vpn_type": options.network_type,
"gateway_ip": options.gw_ip,
"mac_address": options.mac,
"ip_address": options.ip,
"advertise_subnet": options.advertise_subnet,
"readvertise": readvertise,
"attract_traffic": attract_traffic,
"lb_consistent_hash_order": options.lb_consistent_hash_order,
"vni": options.vni
}
if options.local_pref:
data['local_pref'] = options.local_pref
if options.direction:
data['direction'] = options.direction
json_data = jsonutils.dumps(data)
print("request: %s" % json_data)
os.environ['NO_PROXY'] = "127.0.0.1"
req = urllib.request.Request(
"http://127.0.0.1:%d/%s_localport" %
(cfg.CONF.API.port, options.operation), json_data,
{'Content-Type': 'application/json'})
try:
response = urllib.request.urlopen(req)
response_content = response.read()
response.close()
print("response: %d %s" % (response.getcode(), response_content))
except urllib.error.HTTPError as e:
error_content = e.read()
print(" %s" % error_content)
sys.exit("error %d, reason: %s" % (e.code, e.reason))
def json_formatter(js):
return jsonutils.dumps(js, indent=2)
