def tearDown(self): # You can't import this gobally because DJANGO_SETTINGS_MODULE is not initialized yet for some # reason, but maybe by the time the code meanders its way to here it will work. from chroma_core.services.rpc import RpcClientFactory test_failed = (sys.exc_info() != (None, None, None)) if test_failed: log.info(self._xmlrpc.system.listMethods()) log_chunk = self._xmlrpc.supervisor.readLog(0, 4096) log.error("Supervisor log: %s" % log_chunk) # Shutdown any RPC Threads if they were started. Bit of horrible insider knowledge here. if RpcClientFactory._lightweight is False: RpcClientFactory.shutdown_threads() RpcClientFactory._lightweight = True RpcClientFactory._available = True RpcClientFactory._instances = {} if self._supervisor is not None: try: self._xmlrpc.supervisor.shutdown() stdout, stderr = self._supervisor.communicate() # Echo these at the end: by outputting using sys.std* rather than # letting the subprocess write directly, this verbose output can be # captured by nose and only output on failure. sys.stdout.write(stdout) sys.stdout.write(stderr) except: self._supervisor.kill() finally: self._supervisor = None if self._tmp_conf and os.path.exists(self._tmp_conf.name): os.unlink(self._tmp_conf.name)
def tearDown(self): # You can't import this gobally because DJANGO_SETTINGS_MODULE is not initialized yet for some # reason, but maybe by the time the code meanders its way to here it will work. from chroma_core.services.rpc import RpcClientFactory # Shutdown any RPC Threads if they were started. Bit of horrible insider knowledge here. if RpcClientFactory._lightweight is False: RpcClientFactory.shutdown_threads() RpcClientFactory._lightweight = True RpcClientFactory._available = True RpcClientFactory._instances = {} for service in self.SERVICES: log.info("Stopping service '%s'" % service) ServiceControlEL7(service).stop(retry_time=0.1, validate_time=0)
def signal_handler(*args, **kwargs): if not setup_complete.is_set(): log.warning("Terminated during setup, exiting hard") os._exit(0) if not options['lightweight_rpc']: RpcClientFactory.shutdown_threads() for service_thread in service_mains: log.info("Stopping %s" % service_thread.service.name) service_thread.service.stop() for service_thread in service_mains: log.info("Joining %s" % service_thread.service.name) service_thread.join() stopped.set()
def _execute_inner(self, *args, **options): if options['verbose']: log_enable_stdout() if options['gevent']: from gevent.monkey import patch_all patch_all(thread=True) # Gevent's implementation of select removes 'poll' import subprocess subprocess._has_poll = False import django.db django.db.connections._connections = threading.local() if options['trace']: class Trace(object): def __init__(self): self.tracefile = open('trace.log', 'w', buffering=0) self.tracefile.write("Started at %s: %s %s\n" % (IMLDateTime.utcnow(), args, options)) def __call__(self, frame, event, arg): if event == "line": try: pyfile = frame.f_globals['__file__'].strip('co') line = linecache.getline(pyfile, frame.f_lineno) except KeyError: pass else: if line is not None: self.tracefile.write( "%s:%s %s" % (pyfile, frame.f_lineno, line)) return self chroma_core.services.log.trace = Trace() sys.settrace(chroma_core.services.log.trace) from chroma_core.lib.service_config import ServiceConfig if not ServiceConfig().configured(): # FIXME: return an error code which will prevent supervisord from restarting this service # (using the 'exitcodes' option for the programs in the supervisord conf) sys.stderr.write( "Chroma is not configured, please run chroma-config setup first\n" ) sys.exit(-1) if not options['lightweight_rpc']: RpcClientFactory.initialize_threads() log_set_filename("%s.log" % options['name']) # Respond to Ctrl+C stopped = threading.Event() # Ensure that threads are .start()ed before we possibly try to .join() them setup_complete = threading.Event() def signal_handler(*args, **kwargs): """Params undefined because gevent vs. threading pass different things to handler """ if not setup_complete.is_set(): log.warning("Terminated during setup, exiting hard") os._exit(0) if not options['lightweight_rpc']: RpcClientFactory.shutdown_threads() for service_thread in service_mains: log.info("Stopping %s" % service_thread.service.name) service_thread.service.stop() for service_thread in service_mains: log.info("Joining %s" % service_thread.service.name) service_thread.join() stopped.set() if options['gevent']: import gevent gevent.signal(signal.SIGINT, signal_handler) gevent.signal(signal.SIGTERM, signal_handler) else: signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) service_mains = [] for service_name in args: module_path = "chroma_core.services.%s" % service_name # Load the module mod = __import__(module_path) components = module_path.split('.') for comp in components[1:]: mod = getattr(mod, comp) service = getattr(mod, 'Service')() service.log = log_register(service.name) service_thread = ServiceThread(service) service_thread.start() service_mains.append(service_thread) setup_complete.set() while not stopped.is_set(): # Using a timeout changes the behaviour of CPython's waiting so that it will # receive signals (like ctrl-c SIGINT) immediately -- logically we don't want # any timeout here, but a pure wait() breaks ctrl-c. stopped.wait(10) if len(threading.enumerate()) > 1 and not options['gevent']: log.error("Rogue thread still running, exiting hard") log.error([t.name for t in threading.enumerate()]) os._exit(-1)