def test_node_join_from_other_cluster(self, notify):
c = self.eng.cluster_create(self.ctx, 'c-1', 0, self.profile['id'])
cluster_id = c['id']
new_cluster = self.eng.cluster_create(self.ctx, 'c-new', 0,
self.profile['id'])
new_cluster_id = new_cluster['id']
node = self.eng.node_create(self.ctx,
'node1',
self.profile['id'],
cluster_id=cluster_id)
node_id = node['id']
result = self.eng.node_join(self.ctx, node_id, new_cluster_id)
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action,
'NODE_JOIN',
'node_join_%s' % node_id[:8],
node_id,
cause=action_mod.CAUSE_RPC,
inputs={'cluster_id': new_cluster_id})
notify.assert_called_with(action_id=mock.ANY)
# Three creations plus one join
self.assertEqual(4, notify.call_count)
def test_cluster_policy_detach(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
ex = self.assertRaises(rpc.ExpectedException,
self.eng.cluster_policy_detach, self.ctx,
self.cluster['id'], self.policy['id'])
self.assertEqual(exception.PolicyBindingNotFound, ex.exc_info[0])
self.assertEqual(
("The policy (%(policy)s) is not found attached to "
"the specified cluster (%(cluster)s)." %
dict(policy=self.policy['id'], cluster=self.cluster['id'])),
six.text_type(ex.exc_info[1]))
values = {'priority': 50, 'level': 50, 'cooldown': 0, 'enabled': True}
db_api.cluster_policy_attach(self.ctx, cluster_id, policy_id, values)
action = self.eng.cluster_policy_detach(self.ctx, cluster_id,
policy_id)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action,
'CLUSTER_DETACH_POLICY',
'detach_policy_%s' % cluster_id[:8],
cluster_id,
cause=action_mod.CAUSE_RPC,
inputs={'policy_id': policy_id})
notify.assert_called_with(action_id=action_id)
# called twice: attach and detach
self.assertEqual(1, notify.call_count)
def test_cluster_policy_attach(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
action = self.eng.cluster_policy_attach(self.ctx, cluster_id,
policy_id, 50, 50, 40, False)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
inputs = {
'policy_id': policy_id,
'priority': 50,
'level': 50,
'cooldown': 40,
'enabled': True
}
self._verify_action(action,
'CLUSTER_ATTACH_POLICY',
'attach_policy_%s' % cluster_id[:8],
cluster_id,
cause=action_mod.CAUSE_RPC,
inputs=inputs)
notify.assert_called_with(action_id=action_id)
self.assertEqual(1, notify.call_count)
def test_cluster_policy_update(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
values = {'priority': 50, 'level': 50, 'cooldown': 0, 'enabled': True}
db_api.cluster_policy_attach(self.ctx, cluster_id, policy_id, values)
action = self.eng.cluster_policy_update(self.ctx,
cluster_id,
policy_id,
priority=100,
level=10,
cooldown=60,
enabled=False)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action,
'CLUSTER_UPDATE_POLICY',
'update_policy_%s' % cluster_id[:8],
cluster_id,
cause=action_mod.CAUSE_RPC,
inputs={
'policy_id': policy_id,
'priority': 100,
'level': 10,
'cooldown': 60,
'enabled': False
})
notify.assert_called_once_with(action_id=action_id)
def test_cluster_policy_update(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
values = {
'priority': 50,
'level': 50,
'cooldown': 0,
'enabled': True
}
db_api.cluster_policy_attach(self.ctx, cluster_id, policy_id, values)
action = self.eng.cluster_policy_update(
self.ctx, cluster_id, policy_id,
priority=100, level=10, cooldown=60, enabled=False)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action, 'CLUSTER_UPDATE_POLICY',
'update_policy_%s' % cluster_id[:8],
cluster_id, cause=action_mod.CAUSE_RPC,
inputs={
'policy_id': policy_id,
'priority': 100,
'level': 10,
'cooldown': 60,
'enabled': False})
notify.assert_called_once_with(action_id=action_id)
def test_cluster_policy_detach(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
ex = self.assertRaises(rpc.ExpectedException,
self.eng.cluster_policy_detach,
self.ctx, self.cluster['id'], self.policy['id'])
self.assertEqual(exception.PolicyBindingNotFound, ex.exc_info[0])
self.assertEqual(("The policy (%(policy)s) is not found attached to "
"the specified cluster (%(cluster)s)." %
dict(policy=self.policy['id'],
cluster=self.cluster['id'])),
six.text_type(ex.exc_info[1]))
values = {
'priority': 50,
'level': 50,
'cooldown': 0,
'enabled': True
}
db_api.cluster_policy_attach(self.ctx, cluster_id, policy_id, values)
action = self.eng.cluster_policy_detach(self.ctx, cluster_id,
policy_id)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action, 'CLUSTER_DETACH_POLICY',
'detach_policy_%s' % cluster_id[:8],
cluster_id, cause=action_mod.CAUSE_RPC,
inputs={'policy_id': policy_id})
notify.assert_called_with(action_id=action_id)
# called twice: attach and detach
self.assertEqual(1, notify.call_count)
def node_lock_acquire(context, node_id, action_id, engine=None, forced=False):
"""Try to lock the specified node.
:param context: the context used for DB operations;
:param node_id: ID of the node to be locked.
:param action_id: ID of the action that attempts to lock the node.
:param engine: ID of the engine that attempts to lock the node.
:param forced: set to True to cancel current action that owns the lock,
if any.
:returns: True if lock is acquired, or False otherwise.
"""
# Step 1: try lock the node - if the returned owner_id is the
# action id, it was a success
owner = db_api.node_lock_acquire(node_id, action_id)
if action_id == owner:
return True
# Step 2: retry using global configuration options
retries = cfg.CONF.lock_retry_times
retry_interval = cfg.CONF.lock_retry_interval
while retries > 0:
scheduler.sleep(retry_interval)
LOG.debug('Acquire lock for node %s again' % node_id)
owner = db_api.node_lock_acquire(node_id, action_id)
if action_id == owner:
return True
retries = retries - 1
# Step 3: Last resort is 'forced locking', only needed when retry failed
if forced:
owner = db_api.node_lock_steal(node_id, action_id)
return action_id == owner
# if this node lock by dead engine
action = db_api.action_get(context, owner)
if (action and action.owner and action.owner != engine
and is_engine_dead(context, action.owner)):
LOG.info(
_LI('The node %(n)s is locked by dead action %(a)s, '
'try to steal the lock.'), {
'n': node_id,
'a': owner
})
reason = _('Engine died when executing this action.')
db_api.action_mark_failed(context,
action.id,
time.time(),
reason=reason)
db_api.node_lock_steal(node_id, action_id)
return True
LOG.error(
_LE('Node is already locked by action %(old)s, '
'action %(new)s failed grabbing the lock'), {
'old': owner,
'new': action_id
})
return False
def cluster_lock_acquire(context, cluster_id, action_id, engine=None,
scope=CLUSTER_SCOPE, forced=False):
"""Try to lock the specified cluster.
:param cluster_id: ID of the cluster to be locked.
:param action_id: ID of the action which wants to lock the cluster.
:param engine: ID of the engine which wants to lock the cluster.
:param scope: scope of lock, could be cluster wide lock, or node-wide
lock.
:param forced: set to True to cancel current action that owns the lock,
if any.
:returns: True if lock is acquired, or False otherwise.
"""
# Step 1: try lock the cluster - if the returned owner_id is the
# action id, it was a success
owners = db_api.cluster_lock_acquire(cluster_id, action_id, scope)
if action_id in owners:
return True
# Step 2: retry using global configuration options
retries = cfg.CONF.lock_retry_times
retry_interval = cfg.CONF.lock_retry_interval
while retries > 0:
scheduler.sleep(retry_interval)
LOG.debug('Acquire lock for cluster %s again' % cluster_id)
owners = db_api.cluster_lock_acquire(cluster_id, action_id, scope)
if action_id in owners:
return True
retries = retries - 1
# Step 3: Last resort is 'forced locking', only needed when retry failed
if forced:
owners = db_api.cluster_lock_steal(cluster_id, action_id)
return action_id in owners
# Will reach here only because scope == CLUSTER_SCOPE
action = db_api.action_get(context, owners[0])
if (action and action.owner and action.owner != engine and
is_engine_dead(context, action.owner)):
LOG.info(_LI('The cluster %(c)s is locked by dead action %(a)s, '
'try to steal the lock.'), {
'c': cluster_id,
'a': owners[0]
})
reason = _('Engine died when executing this action.')
db_api.action_mark_failed(context, action.id, time.time(),
reason=reason)
owners = db_api.cluster_lock_steal(cluster_id, action_id)
return action_id in owners
LOG.error(_LE('Cluster is already locked by action %(old)s, '
'action %(new)s failed grabbing the lock'),
{'old': str(owners), 'new': action_id})
return False
def load(cls, context, action_id=None, action=None):
'''Retrieve an action from database.'''
if action is None:
action = db_api.action_get(context, action_id)
if action is None:
raise exception.ActionNotFound(action=action_id)
return cls._from_db_record(action)
def load(cls, context, action_id=None, db_action=None, show_deleted=False):
'''Retrieve an action from database.'''
if db_action is None:
db_action = db_api.action_get(context, action_id,
show_deleted=show_deleted)
if db_action is None:
raise exception.ActionNotFound(action=action_id)
return cls._from_db_record(db_action)
def node_lock_acquire(context, node_id, action_id, engine=None,
forced=False):
"""Try to lock the specified node.
:param context: the context used for DB operations;
:param node_id: ID of the node to be locked.
:param action_id: ID of the action that attempts to lock the node.
:param engine: ID of the engine that attempts to lock the node.
:param forced: set to True to cancel current action that owns the lock,
if any.
:returns: True if lock is acquired, or False otherwise.
"""
# Step 1: try lock the node - if the returned owner_id is the
# action id, it was a success
owner = db_api.node_lock_acquire(node_id, action_id)
if action_id == owner:
return True
# Step 2: retry using global configuration options
retries = cfg.CONF.lock_retry_times
retry_interval = cfg.CONF.lock_retry_interval
while retries > 0:
scheduler.sleep(retry_interval)
LOG.debug('Acquire lock for node %s again' % node_id)
owner = db_api.node_lock_acquire(node_id, action_id)
if action_id == owner:
return True
retries = retries - 1
# Step 3: Last resort is 'forced locking', only needed when retry failed
if forced:
owner = db_api.node_lock_steal(node_id, action_id)
return action_id == owner
# if this node lock by dead engine
action = db_api.action_get(context, owner)
if (action and action.owner and action.owner != engine and
is_engine_dead(context, action.owner)):
LOG.info(_LI('The node %(n)s is locked by dead action %(a)s, '
'try to steal the lock.'), {
'n': node_id,
'a': owner
})
reason = _('Engine died when executing this action.')
db_api.action_mark_failed(context, action.id, time.time(),
reason=reason)
db_api.node_lock_steal(node_id, action_id)
return True
LOG.error(_LE('Node is already locked by action %(old)s, '
'action %(new)s failed grabbing the lock'),
{'old': owner, 'new': action_id})
return False
def load(cls, context, action_id=None, db_action=None):
"""Retrieve an action from database.
:param context: Instance of request context.
:param action_id: An UUID for the action to deserialize.
:param db_action: An action object for the action to deserialize.
:return: A `Action` object instance.
"""
if db_action is None:
db_action = db_api.action_get(context, action_id)
if db_action is None:
raise exception.ActionNotFound(action=action_id)
return cls._from_db_record(db_action)
def test_node_leave(self, notify):
c = self.eng.cluster_create(self.ctx, "c-1", 0, self.profile["id"])
cluster_id = c["id"]
node = self.eng.node_create(self.ctx, "node1", self.profile["id"], cluster_id=cluster_id)
node_id = node["id"]
result = self.eng.node_leave(self.ctx, node_id)
action = db_api.action_get(self.ctx, result["action"])
self.assertIsNotNone(action)
self._verify_action(action, "NODE_LEAVE", "node_leave_%s" % node_id[:8], node_id, cause=action_mod.CAUSE_RPC)
notify.assert_called_with(action_id=mock.ANY)
# Two creations plus one leave
self.assertEqual(3, notify.call_count)
def test_node_create_default(self, notify):
node = self.eng.node_create(self.ctx, "n-1", self.profile["id"])
self.assertIsNotNone(node)
self.assertEqual("n-1", node["name"])
self.assertEqual(-1, node["index"])
self.assertEqual(self.profile["id"], node["profile_id"])
self.assertIsNone(node["cluster_id"])
self.assertIsNone(node["role"])
self.assertEqual({}, node["metadata"])
action_id = node["action"]
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(
action, "NODE_CREATE", "node_create_%s" % node["id"][:8], node["id"], cause=action_mod.CAUSE_RPC
)
notify.assert_called_once_with(action_id=action_id)
def test_node_create_default(self, notify):
node = self.eng.node_create(self.ctx, 'n-1', self.profile['id'])
self.assertIsNotNone(node)
self.assertEqual('n-1', node['name'])
self.assertEqual(-1, node['index'])
self.assertEqual(self.profile['id'], node['profile_id'])
self.assertIsNone(node['cluster_id'])
self.assertIsNone(node['role'])
self.assertEqual({}, node['metadata'])
action_id = node['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action, 'NODE_CREATE',
'node_create_%s' % node['id'][:8],
node['id'],
cause=action_mod.CAUSE_RPC)
notify.assert_called_once_with(action_id=action_id)
def test_node_leave(self, notify):
c = self.eng.cluster_create(self.ctx, 'c-1', 0, self.profile['id'])
cluster_id = c['id']
node = self.eng.node_create(self.ctx, 'node1', self.profile['id'],
cluster_id=cluster_id)
node_id = node['id']
result = self.eng.node_leave(self.ctx, node_id)
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action, 'NODE_LEAVE',
'node_leave_%s' % node_id[:8], node_id,
cause=action_mod.CAUSE_RPC)
notify.assert_called_with(action_id=mock.ANY)
# Two creations plus one leave
self.assertEqual(3, notify.call_count)
def test_node_join(self, notify):
c = self.eng.cluster_create(self.ctx, 'c-1', 0, self.profile['id'])
cluster_id = c['id']
node = self.eng.node_create(self.ctx, 'node1', self.profile['id'])
node_id = node['id']
result = self.eng.node_join(self.ctx, node_id, cluster_id)
action_id = result['action']
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action, 'NODE_JOIN',
'node_join_%s' % node_id[:8], node_id,
cause=action_mod.CAUSE_RPC,
inputs={'cluster_id': cluster_id})
notify.assert_called_with(action_id=action_id)
# Two creations plus one join
self.assertEqual(3, notify.call_count)
def test_node_create_default(self, notify):
node = self.eng.node_create(self.ctx, 'n-1', self.profile['id'])
self.assertIsNotNone(node)
self.assertEqual('n-1', node['name'])
self.assertEqual(-1, node['index'])
self.assertEqual(self.profile['id'], node['profile_id'])
self.assertIsNone(node['cluster_id'])
self.assertIsNone(node['role'])
self.assertEqual({}, node['metadata'])
action_id = node['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
self._verify_action(action,
'NODE_CREATE',
'node_create_%s' % node['id'][:8],
node['id'],
cause=action_mod.CAUSE_RPC)
notify.assert_called_once_with(action_id=action_id)
def test_cluster_create_default(self, notify):
result = self.eng.cluster_create(self.ctx, 'c-1', 0,
self.profile['id'])
self.assertIsNotNone(result)
self.assertEqual('c-1', result['name'])
self.assertEqual(0, result['desired_capacity'])
self.assertEqual(self.profile['id'], result['profile_id'])
self.assertEqual(self.ctx.user, result['user'])
self.assertEqual('cluster_test_project', result['project'])
self.assertIsNone(result['parent'])
self.assertIsNone(result['timeout'])
self.assertIsNone(result['metadata'])
action_id = result['action']
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action, 'CLUSTER_CREATE',
'cluster_create_%s' % result['id'][:8],
result['id'],
cause=action_mod.CAUSE_RPC)
notify.assert_called_once_with(action_id=action_id)
def test_cluster_create_default(self, notify):
result = self.eng.cluster_create(self.ctx, 'c-1', 0,
self.profile['id'])
self.assertIsNotNone(result)
self.assertEqual('c-1', result['name'])
self.assertEqual(0, result['desired_capacity'])
self.assertEqual(self.profile['id'], result['profile_id'])
self.assertEqual(self.ctx.user, result['user'])
self.assertEqual('cluster_test_project', result['project'])
self.assertIsNone(result['parent'])
self.assertIsNone(result['timeout'])
self.assertIsNone(result['metadata'])
action_id = result['action']
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action,
'CLUSTER_CREATE',
'cluster_create_%s' % result['id'][:8],
result['id'],
cause=action_mod.CAUSE_RPC)
notify.assert_called_once_with(action_id=action_id)
def test_node_leave(self, notify):
c = self.eng.cluster_create(self.ctx, 'c-1', 0, self.profile['id'])
cluster_id = c['id']
node = self.eng.node_create(self.ctx,
'node1',
self.profile['id'],
cluster_id=cluster_id)
node_id = node['id']
result = self.eng.node_leave(self.ctx, node_id)
action = db_api.action_get(self.ctx, result['action'])
self.assertIsNotNone(action)
self._verify_action(action,
'NODE_LEAVE',
'node_leave_%s' % node_id[:8],
node_id,
cause=action_mod.CAUSE_RPC)
notify.assert_called_with(action_id=mock.ANY)
# Two creations plus one leave
self.assertEqual(3, notify.call_count)
def test_cluster_policy_attach_using_default(self, notify):
cluster_id = self.cluster['id']
policy_id = self.policy['id']
action = self.eng.cluster_policy_attach(self.ctx, cluster_id,
policy_id)
action_id = action['action']
action = db_api.action_get(self.ctx, action_id)
self.assertIsNotNone(action)
inputs = {
'policy_id': policy_id,
'priority': cfg.CONF.default_policy_priority,
'level': self.policy['level'],
'cooldown': self.policy['cooldown'],
'enabled': True
}
self._verify_action(action, 'CLUSTER_ATTACH_POLICY',
'attach_policy_%s' % cluster_id[:8],
cluster_id, cause=action_mod.CAUSE_RPC,
inputs=inputs)
notify.assert_called_with(action_id=action_id)
self.assertEqual(1, notify.call_count)
def test_node_join(self, notify):
c = self.eng.cluster_create(self.ctx, "c-1", 0, self.profile["id"])
cluster_id = c["id"]
node = self.eng.node_create(self.ctx, "node1", self.profile["id"])
node_id = node["id"]
result = self.eng.node_join(self.ctx, node_id, cluster_id)
action_id = result["action"]
action = db_api.action_get(self.ctx, result["action"])
self.assertIsNotNone(action)
self._verify_action(
action,
"NODE_JOIN",
"node_join_%s" % node_id[:8],
node_id,
cause=action_mod.CAUSE_RPC,
inputs={"cluster_id": cluster_id},
)
notify.assert_called_with(action_id=action_id)
# Two creations plus one join
self.assertEqual(3, notify.call_count)
def cluster_lock_acquire(context,
cluster_id,
action_id,
engine=None,
scope=CLUSTER_SCOPE,
forced=False):
"""Try to lock the specified cluster.
:param cluster_id: ID of the cluster to be locked.
:param action_id: ID of the action which wants to lock the cluster.
:param engine: ID of the engine which wants to lock the cluster.
:param scope: scope of lock, could be cluster wide lock, or node-wide
lock.
:param forced: set to True to cancel current action that owns the lock,
if any.
:returns: True if lock is acquired, or False otherwise.
"""
# Step 1: try lock the cluster - if the returned owner_id is the
# action id, it was a success
owners = db_api.cluster_lock_acquire(cluster_id, action_id, scope)
if action_id in owners:
return True
# Step 2: retry using global configuration options
retries = cfg.CONF.lock_retry_times
retry_interval = cfg.CONF.lock_retry_interval
while retries > 0:
scheduler.sleep(retry_interval)
LOG.debug('Acquire lock for cluster %s again' % cluster_id)
owners = db_api.cluster_lock_acquire(cluster_id, action_id, scope)
if action_id in owners:
return True
retries = retries - 1
# Step 3: Last resort is 'forced locking', only needed when retry failed
if forced:
owners = db_api.cluster_lock_steal(cluster_id, action_id)
return action_id in owners
# Will reach here only because scope == CLUSTER_SCOPE
action = db_api.action_get(context, owners[0])
if (action and action.owner and action.owner != engine
and is_engine_dead(context, action.owner)):
LOG.info(
_LI('The cluster %(c)s is locked by dead action %(a)s, '
'try to steal the lock.'), {
'c': cluster_id,
'a': owners[0]
})
reason = _('Engine died when executing this action.')
db_api.action_mark_failed(context,
action.id,
time.time(),
reason=reason)
owners = db_api.cluster_lock_steal(cluster_id, action_id)
return action_id in owners
LOG.error(
_LE('Cluster is already locked by action %(old)s, '
'action %(new)s failed grabbing the lock'), {
'old': str(owners),
'new': action_id
})
return False
def get(cls, context, action_id, **kwargs):
obj = db_api.action_get(context, action_id, **kwargs)
return cls._from_db_object(context, cls(), obj)
def get(cls, context, action_id, **kwargs):
obj = db_api.action_get(context, action_id, **kwargs)
return cls._from_db_object(context, cls(), obj)
def get_status(self):
action = db_api.action_get(self.context, self.id, refresh=True)
self.status = action.status
return action.status
def get_status(self):
action = db_api.action_get(self.context, self.id)
self.status = action.status
return action.status
