def _do_check_resource( self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data ): try: if is_update: try: check_resource_update(rsrc, tmpl.id, resource_data, self.engine_id, stack) except exception.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info("Replacing resource with new id %s", new_res_id) rpc_data = sync_point.serialize_input_data(resource_data) self._rpc_client.check_resource( cnxt, new_res_id, current_traversal, rpc_data, is_update, adopt_stack_data ) return False else: check_resource_cleanup(rsrc, tmpl.id, resource_data, self.engine_id, stack.time_remaining()) return True except exception.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): rpc_data = sync_point.serialize_input_data(resource_data) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) except exception.ResourceFailure as ex: reason = "Resource %s failed: %s" % (rsrc.action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: # reload the stack to verify current traversal stack = parser.Stack.load(cnxt, stack_id=stack.id) if stack.current_traversal != current_traversal: return self._handle_stack_timeout(cnxt, stack) return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data): try: if is_update: try: check_resource_update(rsrc, tmpl.id, resource_data, self.engine_id, stack, self.msg_queue) except resource.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info("Replacing resource with new id %s", new_res_id) rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, rpc_data, is_update, adopt_stack_data) return False else: check_resource_cleanup(rsrc, tmpl.id, resource_data, self.engine_id, stack.time_remaining(), self.msg_queue) return True except exception.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): rpc_data = sync_point.serialize_input_data(self.input_data) # set the resource state as failed status_reason = ('Worker went down ' 'during resource %s' % rsrc.action) rsrc.state_set(rsrc.action, rsrc.FAILED, six.text_type(status_reason)) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: # reload the stack to verify current traversal stack = parser.Stack.load(cnxt, stack_id=stack.id) if stack.current_traversal != current_traversal: return self._handle_stack_timeout(cnxt, stack) except CancelOperation: pass return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data): try: if is_update: try: check_resource_update(rsrc, tmpl.id, resource_data, self.engine_id, stack) except resource.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info(_LI("Replacing resource with new id %s"), new_res_id) rpc_data = sync_point.serialize_input_data(resource_data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, rpc_data, is_update, adopt_stack_data) return False else: check_resource_cleanup(rsrc, tmpl.id, resource_data, self.engine_id, stack.time_remaining()) return True except exception.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): rpc_data = sync_point.serialize_input_data(resource_data) # set the resource state as failed status_reason = ('Worker went down ' 'during resource %s' % rsrc.action) rsrc.state_set(rsrc.action, rsrc.FAILED, six.text_type(status_reason)) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: # reload the stack to verify current traversal stack = parser.Stack.load(cnxt, stack_id=stack.id) if stack.current_traversal != current_traversal: return self._handle_stack_timeout(cnxt, stack) return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, data, is_update, rsrc, stack_id): try: if is_update: try: check_resource_update(rsrc, tmpl.id, data, self.engine_id) except resource.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info("Replacing resource with new id %s", new_res_id) data = sync_point.serialize_input_data(data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, data, is_update) return False else: check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id) return True except resource.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, data, is_update) except exception.ResourceFailure as ex: reason = 'Resource %s failed: %s' % (rsrc.action, six.text_type(ex)) self._handle_resource_failure(cnxt, stack_id, current_traversal, reason) return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, data, is_update, rsrc, stack_id): try: if is_update: try: check_resource_update(rsrc, tmpl.id, data, self.engine_id) except resource.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info("Replacing resource with new id %s", new_res_id) data = sync_point.serialize_input_data(data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, data, is_update) return False else: check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id) return True except resource.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, data, is_update) except exception.ResourceFailure as ex: reason = 'Resource %s failed: %s' % (rsrc.action, six.text_type(ex)) self._handle_resource_failure( cnxt, stack_id, current_traversal, reason) return False
def test_requires(self, mock_load, mock_cru, mock_crc, mock_pcr, mock_csc): mock_load.return_value = self.resource, self.stack, self.stack res_data = {(1, True): {u'id': 5, u'name': 'A', 'attrs': {}}, (2, True): {u'id': 3, u'name': 'B', 'attrs': {}}} self.worker.check_resource(self.ctx, self.resource.id, self.stack.current_traversal, sync_point.serialize_input_data(res_data), self.is_update, {}) mock_cru.assert_called_once_with( self.resource, self.resource.stack.t.id, {5, 3}, self.worker.engine_id, self.stack, mock.ANY)
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data): try: if is_update: try: check_resource_update(rsrc, tmpl.id, resource_data, self.engine_id, stack, self.msg_queue) except resource.UpdateReplace: self._handle_resource_replacement(cnxt, current_traversal, tmpl.id, rsrc, stack, adopt_stack_data) return False else: check_resource_cleanup(rsrc, tmpl.id, resource_data, self.engine_id, stack.time_remaining(), self.msg_queue) return True except exception.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): rpc_data = sync_point.serialize_input_data(self.input_data) # set the resource state as failed status_reason = ('Worker went down ' 'during resource %s' % rsrc.action) rsrc.state_set(rsrc.action, rsrc.FAILED, six.text_type(status_reason)) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, u'Timed out') except CancelOperation as ex: # Stack is already marked FAILED, so we just need to retrigger # in case a new traversal has started and is waiting on us. self._retrigger_new_traversal(cnxt, current_traversal, is_update, stack.id, rsrc.id) return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data): prev_template_id = rsrc.current_template_id try: if is_update: requires = set(d.primary_key for d in resource_data.values() if d is not None) try: check_resource_update(rsrc, tmpl.id, requires, self.engine_id, stack, self.msg_queue) except resource.UpdateReplace: self._handle_resource_replacement(cnxt, current_traversal, tmpl.id, requires, rsrc, stack, adopt_stack_data) return False else: check_resource_cleanup(rsrc, tmpl.id, self.engine_id, stack.time_remaining(), self.msg_queue) return True except exception.UpdateInProgress: LOG.debug('Waiting for existing update to unlock resource %s', rsrc.id) if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id): rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) else: rsrc.handle_preempt() except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, u'Timed out') except CancelOperation as ex: # Stack is already marked FAILED, so we just need to retrigger # in case a new traversal has started and is waiting on us. self._retrigger_new_traversal(cnxt, current_traversal, is_update, stack.id, rsrc.id) return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data): prev_template_id = rsrc.current_template_id try: if is_update: requires = set(d.primary_key for d in resource_data.values() if d is not None) try: check_resource_update(rsrc, tmpl.id, requires, self.engine_id, stack, self.msg_queue) except resource.UpdateReplace: self._handle_resource_replacement(cnxt, current_traversal, tmpl.id, requires, rsrc, stack, adopt_stack_data) return False else: check_resource_cleanup(rsrc, tmpl.id, self.engine_id, stack.time_remaining(), self.msg_queue) return True except exception.UpdateInProgress: if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id): rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data) except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason) except scheduler.Timeout: self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, u'Timed out') except CancelOperation as ex: # Stack is already marked FAILED, so we just need to retrigger # in case a new traversal has started and is waiting on us. self._retrigger_new_traversal(cnxt, current_traversal, is_update, stack.id, rsrc.id) return False
def _handle_resource_replacement(self, cnxt, current_traversal, new_tmpl_id, rsrc, stack, adopt_stack_data): """Create a replacement resource and trigger a check on it.""" try: new_res_id = rsrc.make_replacement(new_tmpl_id) except exception.UpdateInProgress: LOG.info("No replacement created - " "resource already locked by new traversal") return if new_res_id is None: LOG.info("No replacement created - " "new traversal already in progress") self._retrigger_new_traversal(cnxt, current_traversal, True, stack.id, rsrc.id) return LOG.info("Replacing resource with new id %s", new_res_id) rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, rpc_data, True, adopt_stack_data)
def _handle_resource_replacement(self, cnxt, current_traversal, new_tmpl_id, requires, rsrc, stack, adopt_stack_data): """Create a replacement resource and trigger a check on it.""" try: new_res_id = rsrc.make_replacement(new_tmpl_id, requires) except exception.UpdateInProgress: LOG.info("No replacement created - " "resource already locked by new traversal") return if new_res_id is None: LOG.info("No replacement created - " "new traversal already in progress") self._retrigger_new_traversal(cnxt, current_traversal, True, stack.id, rsrc.id) return LOG.info("Replacing resource with new id %s", new_res_id) rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, rpc_data, True, adopt_stack_data)
def test_serialize_input_data(self): res = sync_point.serialize_input_data({(3, 8): None}) self.assertEqual({'input_data': [[[3, 8], None]]}, res)
def test_serialize_input_data(self): res = sync_point.serialize_input_data({(3, 8): None}) self.assertEqual({'input_data': {u'tuple:(3, 8)': None}}, res)
def test_serialize_input_data(self): res = sync_point.serialize_input_data({(3L, 8): None}) self.assertEqual({'input_data': [[[3L, 8], None]]}, res)
def check_resource(self, cnxt, resource_id, current_traversal, data, is_update): ''' Process a node in the dependency graph. The node may be associated with either an update or a cleanup of its associated resource. ''' adopt_data = data.get('adopt_stack_data') data = dict(sync_point.deserialize_input_data(data)) try: cache_data = {in_data.get( 'name'): in_data for in_data in data.values() if in_data is not None} cache_data['adopt_stack_data'] = adopt_data rsrc, stack = resource.Resource.load(cnxt, resource_id, is_update, cache_data) except (exception.ResourceNotFound, exception.NotFound): return tmpl = stack.t if current_traversal != rsrc.stack.current_traversal: LOG.debug('[%s] Traversal cancelled; stopping.', current_traversal) return current_deps = ([tuple(i), (tuple(j) if j is not None else None)] for i, j in rsrc.stack.current_deps['edges']) deps = dependencies.Dependencies(edges=current_deps) graph = deps.graph() if is_update: if (rsrc.replaced_by is not None and rsrc.current_template_id != tmpl.id): return try: check_resource_update(rsrc, tmpl.id, data, self.engine_id) except resource.UpdateReplace: new_res_id = rsrc.make_replacement(tmpl.id) LOG.info("Replacing resource with new id %s", new_res_id) data = sync_point.serialize_input_data(data) self._rpc_client.check_resource(cnxt, new_res_id, current_traversal, data, is_update) return except resource.UpdateInProgress: if self._try_steal_engine_lock(cnxt, resource_id): self._rpc_client.check_resource(cnxt, resource_id, current_traversal, data, is_update) return except exception.ResourceFailure as ex: reason = 'Resource %s failed: %s' % (stack.action, six.text_type(ex)) self._handle_resource_failure( cnxt, stack.id, current_traversal, reason) return input_data = construct_input_data(rsrc) else: try: check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id) except resource.UpdateInProgress: if self._try_steal_engine_lock(cnxt, resource_id): self._rpc_client.check_resource(cnxt, resource_id, current_traversal, data, is_update) return except exception.ResourceFailure as ex: reason = 'Resource %s failed: %s' % (stack.action, six.text_type(ex)) self._handle_resource_failure( cnxt, stack.id, current_traversal, reason) return graph_key = (resource_id, is_update) if graph_key not in graph and rsrc.replaces is not None: # If we are a replacement, impersonate the replaced resource for # the purposes of calculating whether subsequent resources are # ready, since everybody has to work from the same version of the # graph. Our real resource ID is sent in the input_data, so the # dependencies will get updated to point to this resource in time # for the next traversal. graph_key = (rsrc.replaces, is_update) try: for req, fwd in deps.required_by(graph_key): propagate_check_resource( cnxt, self._rpc_client, req, current_traversal, set(graph[(req, fwd)]), graph_key, input_data if fwd else None, fwd) check_stack_complete(cnxt, rsrc.stack, current_traversal, resource_id, deps, is_update) except sync_point.SyncPointNotFound: # Reload the stack to determine the current traversal, and check # the SyncPoint for the current node to determine if it is ready. # If it is, then retrigger the current node with the appropriate # data for the latest traversal. stack = parser.Stack.load(cnxt, stack_id=rsrc.stack.id) if current_traversal == rsrc.stack.current_traversal: LOG.debug('[%s] Traversal sync point missing.', current_traversal) return current_traversal = stack.current_traversal current_deps = ([tuple(i), (tuple(j) if j is not None else None)] for i, j in stack.current_deps['edges']) deps = dependencies.Dependencies(edges=current_deps) key = sync_point.make_key(resource_id, current_traversal, is_update) predecessors = deps.graph()[key] def do_check(target_key, data): self.check_resource(resource_id, current_traversal, data) try: sync_point.sync(cnxt, resource_id, current_traversal, is_update, do_check, predecessors, {key: None}) except sync_point.sync_points.NotFound: pass