def run(self):
"""
Run the job. Calls `startup`, does periodic `check`,
calls `shutdown` in any case
"""
self.log.info("Starting...")
exc_info = None
try:
self._startup()
self.post_startup_hook()
self._wait()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
self.stopping_reason = exc
exc_info = sys.exc_info()
finally:
self.log.warning("Please wait for graceful shutdown...")
try:
self.pre_shutdown_hook()
self._shutdown()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def post_process(self):
"""
Post-process executors
"""
exc_info = exc_value = None
for executor in self.executors:
if executor in self.engine.prepared:
self.log.debug("Post-process %s", executor)
try:
executor.post_process()
if executor in self.engine.started and not executor.has_results():
msg = "Empty results, most likely %s (%s) failed. " \
"Actual reason for this can be found in logs under %s"
message = msg % (executor.label, executor.__class__.__name__, self.engine.artifacts_dir)
diagnostics = None
if isinstance(executor, SelfDiagnosable):
diagnostics = executor.get_error_diagnostics()
raise ToolError(message, diagnostics)
except BaseException as exc:
msg = "Exception in post_process of %s: %s %s"
self.log.debug(msg, executor.__class__.__name__, exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if not exc_value:
exc_value = exc
if exc_info:
reraise(exc_info, exc_value)
def post_process(self):
"""
Do post-run analysis and processing for the results.
"""
self.log.info("Post-processing...")
# :type exception: BaseException
exc_info = None
modules = [self.provisioning, self.aggregator
] + self.reporters + self.services # order matters
# services are last because of shellexec which is "final-final" action
for module in modules:
if module in self.prepared:
try:
module.post_process()
except BaseException as exc:
if isinstance(exc, KeyboardInterrupt):
self.log.debug("post_process: %s", exc)
else:
self.log.debug("post_process: %s\n%s", exc,
traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_info:
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info)
def post_process(self):
"""
Do post-run analysis and processing for the results.
"""
self.log.info("Post-processing...")
# :type exception: BaseException
exc_info = exc_value = None
modules = [self.provisioning, self.aggregator] + self.reporters + self.services # order matters
# services are last because of shellexec which is "final-final" action
for module in modules:
if module in self.prepared:
try:
module.post_process()
except BaseException as exc:
if isinstance(exc, KeyboardInterrupt):
self.log.debug("post_process: %s", exc)
else:
self.log.debug("post_process: %s\n%s", exc, traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_value:
exc_value = exc
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info, exc_value)
def _shutdown(self):
"""
Shutdown modules
:return:
"""
self.log.info("Shutting down...")
self.log.debug("Current stop reason: %s", self.stopping_reason)
exc_info = exc_value = None
modules = [self.provisioning, self.aggregator
] + self.reporters + self.services # order matters
for module in modules:
try:
if module in self.started:
module.shutdown()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_value:
exc_value = exc
exc_info = sys.exc_info()
self.config.dump()
if exc_value:
reraise(exc_info, exc_value)
def post_process(self):
"""
Do post-run analysis and processing for the results.
"""
self.log.info("Post-processing...")
# :type exception: BaseException
exc_info = None
modules = [self.provisioning, self.aggregator] + self.reporters + self.services
for module in modules:
if module in self.prepared:
try:
module.post_process()
except BaseException as exc:
if isinstance(exc, KeyboardInterrupt):
self.log.debug("Shutdown: %s", exc)
else:
self.log.debug("Error while post-processing: %s", exc)
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_info:
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info)
def run(self):
"""
Run the job. Calls `startup`, does periodic `check`,
calls `shutdown` in any case
"""
self.log.info("Starting...")
exc_info = exc_value = None
try:
self._startup()
self.logging_level_down()
self._wait()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
exc_value = exc
exc_info = sys.exc_info()
finally:
self.log.warning("Please wait for graceful shutdown...")
try:
self.logging_level_up()
self._shutdown()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not self.stopping_reason:
self.stopping_reason = exc
if not exc_value:
exc_value = exc
exc_info = sys.exc_info()
if exc_value:
reraise(exc_info, exc_value)
def post_process(self):
"""
Post-process executors
"""
exc_info = exc_value = None
for executor in self.executors:
self.log.debug("Post-process %s", executor)
try:
executor.post_process()
if executor in self.started_modules and not executor.has_results():
msg = "Empty results, most likely %s (%s) failed. " \
"Actual reason for this can be found in logs under %s"
message = msg % (executor.label, executor.__class__.__name__, self.engine.artifacts_dir)
diagnostics = None
if isinstance(executor, SelfDiagnosable):
diagnostics = executor.get_error_diagnostics()
raise ToolError(message, diagnostics)
except BaseException as exc:
msg = "Exception in post_process of %s: %s %s"
self.log.debug(msg, executor.__class__.__name__, exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if not exc_value:
exc_value = exc
if exc_info:
reraise(exc_info, exc_value)
def shutdown(self):
"""
Call shutdown on executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.started:
self.log.debug("Shutdown %s", executor)
try:
executor.shutdown()
except BaseException as exc:
self.log.error("Exception in shutdown of %s: %s" % (executor.__class__.__name__, exc))
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def shutdown(self):
"""
Call shutdown on executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.started:
self.log.debug("Shutdown %s", executor)
try:
executor.shutdown()
except BaseException as exc:
self.log.error("Exception in shutdown of %s: %s" % (executor.__class__.__name__, exc))
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def shutdown(self):
"""
Call shutdown on executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.started:
self.log.debug("Shutdown %s", executor)
try:
executor.shutdown()
except BaseException as exc:
msg = "Exception in shutdown of %s: %s %s"
self.log.debug(msg, executor.__class__.__name__, exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def shutdown(self):
"""
Call shutdown on executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.started:
self.log.debug("Shutdown %s", executor)
try:
executor.shutdown()
except BaseException as exc:
msg = "Exception in shutdown of %s: %s %s"
self.log.debug(msg, executor.__class__.__name__, exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def post_process(self):
"""
Post-process executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.prepared:
self.log.debug("Post-process %s", executor)
try:
executor.post_process()
if executor in self.engine.started and not executor.has_results():
raise RuntimeWarning("Empty results, most likely %s failed" % executor.__class__.__name__)
except BaseException as exc:
self.log.error("Exception in post_process of %s: %s" % (executor.__class__.__name__, exc))
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def post_process(self):
"""
Post-process executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.prepared:
self.log.debug("Post-process %s", executor)
try:
executor.post_process()
if executor in self.engine.started and not executor.has_results():
msg = "Empty results, most likely %s (%s) failed"
raise RuntimeWarning(msg % (executor.label, executor.__class__.__name__))
except BaseException as exc:
self.log.debug("Exception in post_process of %s: %s" % (executor.__class__.__name__, exc))
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def _shutdown(self):
"""
Shutdown modules
:return:
"""
self.log.info("Shutting down...")
exc_info = None
modules = [self.provisioning, self.aggregator] + self.reporters + self.services
for module in modules:
try:
if module in self.started:
module.shutdown()
except BaseException:
self.log.debug("Error while shutting down: %s", traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info)
def _shutdown(self):
"""
Shutdown modules
:return:
"""
self.log.info("Shutting down...")
exc_info = None
modules = [self.provisioning] + self.services # order matters
for module in modules:
try:
if module in self.started:
module.shutdown()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info)
def post_process(self):
"""
Post-process executors
"""
exc_info = None
for executor in self.executors:
if executor in self.engine.prepared:
self.log.debug("Post-process %s", executor)
try:
executor.post_process()
if executor in self.engine.started and not executor.has_results():
msg = "Empty results, most likely %s (%s) failed. " \
"Actual reason for this can be found in logs under %s"
raise ToolError(msg % (executor.label, executor.__class__.__name__, self.engine.artifacts_dir))
except BaseException as exc:
msg = "Exception in post_process of %s: %s %s"
self.log.debug(msg, executor.__class__.__name__, exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if exc_info:
reraise(exc_info)
def _shutdown(self):
"""
Shutdown modules
:return:
"""
self.log.info("Shutting down...")
exc_info = None
modules = [self.provisioning, self.aggregator] + self.reporters + self.services
for module in modules:
try:
if module in self.started:
module.shutdown()
except BaseException as exc:
self.log.error("Error while shutting down: %s", traceback.format_exc())
self.stopping_reason = exc if not self.stopping_reason else self.stopping_reason
if not exc_info:
exc_info = sys.exc_info()
self.config.dump()
if exc_info:
reraise(exc_info)
def _shutdown(self):
"""
Shutdown modules
:return:
"""
self.log.info("Shutting down...")
self.log.debug("Current stop reason: %s", self.stopping_reason)
exc_info = exc_value = None
modules = [self.provisioning, self.aggregator] + self.reporters + self.services # order matters
for module in modules:
try:
if module in self.started:
module.shutdown()
except BaseException as exc:
self.log.debug("%s:\n%s", exc, traceback.format_exc())
if not exc_info:
exc_info = sys.exc_info()
if not exc_value:
exc_value = exc
self.config.dump()
if exc_info:
reraise(exc_info, exc_value)