def cleanup(self, task_uuid=None): """Cleans all platform in env. :param task_uuid: Cleans up only resources of specific task. :returns: Dict with status of all cleanups """ db.env_set_status(self.uuid, STATUS.READY, STATUS.CLEANING) result = {} for p in self._get_platforms(): try: cleanup_info = p.cleanup(task_uuid) cleanup_info.setdefault("message", "Succeeded") jsonschema.validate(cleanup_info, self._CLEANUP_FORMAT) except Exception as e: msg = "Plugin %s.cleanup() method is broken" % p.get_fullname() LOG.exception(msg) cleanup_info = { "message": "Failed", "discovered": 0, "deleted": 0, "failed": 0, "resources": {}, "errors": [{"message": msg}] } if isinstance(e, NotImplementedError): cleanup_info["message"] = "Not implemented" cleanup_info["errors"] = [] elif not isinstance(e, jsonschema.ValidationError): cleanup_info["errors"][0]["traceback"] = ( traceback.format_exc()) result[p.get_fullname()] = cleanup_info db.env_set_status(self.uuid, STATUS.CLEANING, STATUS.READY) return result
def cleanup(self, task_uuid=None): """Cleans all platform in env. :param task_uuid: Cleans up only resources of specific task. :returns: Dict with status of all cleanups """ db.env_set_status(self.uuid, STATUS.READY, STATUS.CLEANING) result = {} for p in self._get_platforms(): try: cleanup_info = p.cleanup(task_uuid) cleanup_info.setdefault("message", "Succeeded") jsonschema.validate(cleanup_info, self._CLEANUP_FORMAT) except Exception as e: msg = "Plugin %s.cleanup() method is broken" % p.get_fullname() LOG.exception(msg) cleanup_info = { "message": "Failed", "discovered": 0, "deleted": 0, "failed": 0, "resources": {}, "errors": [{"message": msg}] } if isinstance(e, NotImplementedError): cleanup_info["message"] = "Not implemented" cleanup_info["errors"] = [] elif not isinstance(e, jsonschema.ValidationError): cleanup_info["errors"][0]["traceback"] = ( traceback.format_exc()) result[p.get_fullname()] = cleanup_info db.env_set_status(self.uuid, STATUS.CLEANING, STATUS.READY) return result
def test_evn_set_status(self): env = db.env_create("name", "status", "descr", {}, {}, {}, []) self.assertRaises(exceptions.DBConflict, db.env_set_status, env["uuid"], "wrong_old_status", "new_status") env = db.env_get(env["uuid"]) self.assertEqual("status", env["status"]) self.assertTrue(db.env_set_status(env["uuid"], "status", "new_status")) env = db.env_get(env["uuid"]) self.assertEqual("new_status", env["status"])
def test_evn_set_status(self): env = db.env_create("name", "status", "descr", {}, {}, {}, []) self.assertRaises( exceptions.DBConflict, db.env_set_status, env["uuid"], "wrong_old_status", "new_status") env = db.env_get(env["uuid"]) self.assertEqual("status", env["status"]) self.assertTrue( db.env_set_status(env["uuid"], "status", "new_status")) env = db.env_get(env["uuid"]) self.assertEqual("new_status", env["status"])
def destroy(self, skip_cleanup=False): """Destroys all platforms related to env. :param skip_cleanup: Skip cleaning up platform resources """ cleanup_info = {"skipped": True} if not skip_cleanup: cleanup_info["info"] = self.cleanup() cleanup_info["skipped"] = False cleanup_info["failed"] = bool(any( v["errors"] for v in cleanup_info["info"].values())) if cleanup_info["failed"]: return { "cleanup_info": cleanup_info, "destroy_info": { "skipped": True, "platforms": {}, "message": "Skipped because cleanup has errors" } } result = { "cleanup_info": cleanup_info, "destroy_info": { "skipped": False, "platforms": {} } } db.env_set_status(self.uuid, STATUS.READY, STATUS.DESTROYING) platforms = result["destroy_info"]["platforms"] new_env_status = STATUS.DESTROYED for p in self._get_platforms(): name = p.get_fullname() platforms[name] = {"status": {"old": p.status}} if p.status == platform.STATUS.DESTROYED: platforms[name]["status"]["new"] = p.status platforms[name]["message"] = ( "Platform is already destroyed. Do nothing") continue db.platform_set_status( p.uuid, p.status, platform.STATUS.DESTROYING) try: p.destroy() except Exception: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.FAILED_TO_DESTROY) platforms[name]["message"] = "Failed to destroy" platforms[name]["status"]["new"] = ( platform.STATUS.FAILED_TO_DESTROY) platforms[name]["traceback"] = traceback.format_exc() new_env_status = STATUS.FAILED_TO_DESTROY else: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.DESTROYED) platforms[name]["message"] = "Successfully destroyed" platforms[name]["status"]["new"] = platform.STATUS.DESTROYED from rally.common import objects # TODO(boris-42): This is breaking all kinds of rules of good # architecture, and we should remove this thing from # here... for verifier in objects.Verifier.list(): verifier.set_env(self.uuid) verifier.manager.uninstall() db.env_set_status(self.uuid, STATUS.DESTROYING, new_env_status) return result
def _create_platforms(self): """Iterates over platform and creates them, storing results in DB. Do NOT use this method directly! Use create() instead. All platform statuses are going to be updated. - If everything is OK all platforms and env would have READY statuts. - If some of platforms failed, it will get status "FAILED TO CREATE" as well as Env, all following platforms would have "SKIPPED" state. - If there are issues with DB, and we can't store results to DB, platform will be destroyed so we won't keep messy env, everything will be logged. This is not ideal solution, but it's best that we can do at the moment """ new_env_status = STATUS.READY for p in self._get_platforms(): if new_env_status != STATUS.READY: db.platform_set_status( p.uuid, platform.STATUS.INIT, platform.STATUS.SKIPPED) continue try: platform_data, plugin_data = p.create() except Exception: new_env_status = STATUS.FAILED_TO_CREATE LOG.exception( "Failed to create platform (%(uuid)s): " "%(name)s with spec: %(spec)s" % {"uuid": p.uuid, "name": p.get_fullname(), "spec": p.spec}) try: db.platform_set_status(p.uuid, platform.STATUS.INIT, platform.STATUS.FAILED_TO_CREATE) except Exception: LOG.Exception( "Failed to set platform %(uuid)s status %(status)s" % {"uuid": p.uuid, "status": platform.STATUS.FAILED_TO_CREATE}) if new_env_status == STATUS.FAILED_TO_CREATE: continue try: db.platform_set_data( p.uuid, platform_data=platform_data, plugin_data=plugin_data) db.platform_set_status( p.uuid, platform.STATUS.INIT, platform.STATUS.READY) except Exception: new_env_status = STATUS.FAILED_TO_CREATE # NOTE(boris-42): We can't store platform data, because of # issues with DB, to keep env clean we must # destroy platform while we have complete data. p.status = platform.STATUS.FAILED_TO_CREATE p.platform_data, p.plugin_data = platform_data, plugin_data try: p.destroy() LOG.warning("Couldn't store platform %s data to DB." "Attempt to destroy it succeeded." % p.uuid) except Exception: LOG.exception( "Couldn't store data of platform(%(uuid)s): %(name)s " "with spec: %(spec)s. Attempt to destroy it failed. " "Sorry, but we can't do anything else for you. :(" % {"uuid": p.uuid, "name": p.get_fullname(), "spec": p.spec}) db.env_set_status(self.uuid, STATUS.INIT, new_env_status)
def destroy(self, skip_cleanup=False): """Destroys all platforms related to env. :param skip_cleanup: By default, before destroying plaform it's cleaned """ cleanup_info = {"skipped": True} if not skip_cleanup: cleanup_info = self.cleanup() cleanup_info["skipped"] = False if cleanup_info["errors"]: return { "cleanup_info": cleanup_info, "destroy_info": { "skipped": True, "platforms": {}, "message": "Skipped because cleanup failed" } } result = { "cleanup_info": cleanup_info, "destroy_info": { "skipped": False, "platforms": {} } } db.env_set_status(self.uuid, STATUS.READY, STATUS.DESTROYING) platforms = result["destroy_info"]["platforms"] new_env_status = STATUS.DESTROYED for p in self._get_platforms(): name = p.get_fullname() platforms[name] = {"status": {"old": p.status}} if p.status == platform.STATUS.DESTROYED: platforms[name]["status"]["new"] = p.status platforms[name]["message"] = ( "Platform is already destroyed. Do nothing") continue db.platform_set_status(p.uuid, p.status, platform.STATUS.DESTROYING) try: p.destroy() except Exception: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.FAILED_TO_DESTROY) platforms[name]["message"] = "Failed to destroy" platforms[name]["status"]["new"] = ( platform.STATUS.FAILED_TO_DESTROY) platforms[name]["traceback"] = sys.exc_info() new_env_status = STATUS.FAILED_TO_DESTROY else: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.DESTROYED) platforms[name]["message"] = "Successfully destroyed" platforms[name]["status"]["new"] = platform.STATUS.DESTROYED db.env_set_status(self.uuid, STATUS.DESTROYING, new_env_status) return result
def destroy(self, skip_cleanup=False): """Destroys all platforms related to env. :param skip_cleanup: Skip cleaning up platform resources """ cleanup_info = {"skipped": True} if not skip_cleanup: cleanup_info["info"] = self.cleanup() cleanup_info["skipped"] = False cleanup_info["failed"] = bool(any( v["errors"] for v in cleanup_info["info"].values())) if cleanup_info["failed"]: return { "cleanup_info": cleanup_info, "destroy_info": { "skipped": True, "platforms": {}, "message": "Skipped because cleanup has errors" } } result = { "cleanup_info": cleanup_info, "destroy_info": { "skipped": False, "platforms": {} } } db.env_set_status(self.uuid, STATUS.READY, STATUS.DESTROYING) platforms = result["destroy_info"]["platforms"] new_env_status = STATUS.DESTROYED for p in self._get_platforms(): name = p.get_fullname() platforms[name] = {"status": {"old": p.status}} if p.status == platform.STATUS.DESTROYED: platforms[name]["status"]["new"] = p.status platforms[name]["message"] = ( "Platform is already destroyed. Do nothing") continue db.platform_set_status( p.uuid, p.status, platform.STATUS.DESTROYING) try: p.destroy() except Exception: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.FAILED_TO_DESTROY) platforms[name]["message"] = "Failed to destroy" platforms[name]["status"]["new"] = ( platform.STATUS.FAILED_TO_DESTROY) platforms[name]["traceback"] = traceback.format_exc() new_env_status = STATUS.FAILED_TO_DESTROY else: db.platform_set_status(p.uuid, platform.STATUS.DESTROYING, platform.STATUS.DESTROYED) platforms[name]["message"] = "Successfully destroyed" platforms[name]["status"]["new"] = platform.STATUS.DESTROYED from rally.common import objects # TODO(boris-42): This is breaking all kinds of rules of good # architecture, and we should remove this thing from # here... for verifier in objects.Verifier.list(): verifier.set_env(self.uuid) verifier.manager.uninstall() db.env_set_status(self.uuid, STATUS.DESTROYING, new_env_status) return result
def _create_platforms(self): """Iterates over platform and creates them, storing results in DB. Do NOT use this method directly! Use create() instead. All platform statuses are going to be updated. - If everything is OK all platforms and env would have READY statuts. - If some of platforms failed, it will get status "FAILED TO CREATE" as well as Env, all following platforms would have "SKIPPED" state. - If there are issues with DB, and we can't store results to DB, platform will be destroyed so we won't keep messy env, everything will be logged. This is not ideal solution, but it's best that we can do at the moment """ new_env_status = STATUS.READY for p in self._get_platforms(): if new_env_status != STATUS.READY: db.platform_set_status( p.uuid, platform.STATUS.INIT, platform.STATUS.SKIPPED) continue try: platform_data, plugin_data = p.create() except Exception: new_env_status = STATUS.FAILED_TO_CREATE LOG.exception( "Failed to create platform (%(uuid)s): " "%(name)s with spec: %(spec)s" % {"uuid": p.uuid, "name": p.get_fullname(), "spec": p.spec}) try: db.platform_set_status(p.uuid, platform.STATUS.INIT, platform.STATUS.FAILED_TO_CREATE) except Exception: LOG.exception( "Failed to set platform %(uuid)s status %(status)s" % {"uuid": p.uuid, "status": platform.STATUS.FAILED_TO_CREATE}) if new_env_status == STATUS.FAILED_TO_CREATE: continue try: db.platform_set_data( p.uuid, platform_data=platform_data, plugin_data=plugin_data) db.platform_set_status( p.uuid, platform.STATUS.INIT, platform.STATUS.READY) except Exception: new_env_status = STATUS.FAILED_TO_CREATE # NOTE(boris-42): We can't store platform data, because of # issues with DB, to keep env clean we must # destroy platform while we have complete data. p.status = platform.STATUS.FAILED_TO_CREATE p.platform_data, p.plugin_data = platform_data, plugin_data try: p.destroy() LOG.warning("Couldn't store platform %s data to DB." "Attempt to destroy it succeeded." % p.uuid) except Exception: LOG.exception( "Couldn't store data of platform(%(uuid)s): %(name)s " "with spec: %(spec)s. Attempt to destroy it failed. " "Sorry, but we can't do anything else for you. :(" % {"uuid": p.uuid, "name": p.get_fullname(), "spec": p.spec}) db.env_set_status(self.uuid, STATUS.INIT, new_env_status)