def mock_death(self): # try to keep Control sockets open after process death by spawning a # subprocess and then commit suicide, using SIGKILL. the observable # behavior (when this works) is that the call to mock_death() never # returns to the client because no answer is sent (the process is dead) # while its open sockets keep lingering (is this due to a GC detail in # Python that keeps sockets open as long as a child process is still # around?). from the client's point of view, the call to mock_death # hangs indefinitely. # TODO: there is a race condition if the subprocess has not been fully # started when the parent gets SIGKILL'ed. should try to remove the call # to sync_ping(). however, this should not be a problem in real life # because all processes except the root broker process are governed by # their parents, which never receive SIGKILL. only the broker will ever # get SIGKILL'ed, and even that should never happen in real deployment. # a simple protection against that eventuality would be to respawn it # immediately on startup and register the spawn for PDEATHSIG, before # it starts accepting client connections, so that everything after this # point is guaranteed to be insulated from the race condition. sock, port = find_free_port() c = MockControl(port, 'authkey', sock) c.start() r = RemoteControl(('',port), 'authkey', timeout=5) try: r.connect(2) except Exception, e: traceback.print_exc() os.kill(self.pid, signal.SIGKILL) # suicide! return
def t04(pretty, factory): c1 = factory.make_control(timeout=None) c2 = RemoteControl(c1.address, None, timeout=None) c2.connect(timeout=None) c1.run_external('sleep 2', __async__=True) try: o = c2.sync_ping(__async__=False) except Exception, e: print('FAIL %s: call failed: %s' % (pretty, e)) return False
def t03(pretty, factory): c1 = factory.make_control(timeout=None) c2 = RemoteControl(c1.address, None, timeout=None) c2.connect(timeout=None) c1.run_external('sleep 2', __async__=True) try: # should not time out because the connection is already established o = c2.sync_ping(__async__=True) except Exception, e: print('FAIL %s: call failed: %s' % (pretty, e)) return False
class MockControl(Control): pipe = None # used to shortcut communication in the tests proxee = None def __init__(self, port, authkey, socket, alt_keys=[], pipe=None, interval=None, proxee=None, home=None, max_tx=-1): Control.__init__(self, port, authkey, socket, alt_keys, interval, home) self.pipe = pipe self.proxee = proxee self.max_tx = max_tx def run(self): Control.run(self) def shutdown(self, details=None): Control.shutdown(self, details) def close_fds(self, exclude): if self.pipe: exclude.append(self.pipe.w) Control.close_fds(self, exclude) def lost_connection(self, connection, authkey): if self.pipe: self.pipe.put(connection.address) def idle(self): if self.pipe: self.pipe.put('idle') def shutdown(self): if self.pipe: self.pipe.put('shutdown') Control.shutdown(self) @Control.rpc def sync_ping(self, alt=None): return alt or 'pong' @Control.rpc def async_ping(self): if self.pipe: self.pipe.put('pong') @Control.rpc def stop(self): Control.stop(self) @Control.rpc def raise_plain_exception(self, message): raise Exception(message) @Control.rpc def raise_ave_exception(self, details, depth=0): if self.proxee: details['message'] = 'proxied: ' + details['message'] self.proxee.raise_ave_exception(details, depth) if depth > 0: self.raise_ave_exception(details, depth-1) raise AveException(details) @Control.rpc def raise_exit(self, msg='passed to client'): raise Exit(msg) @Control.rpc def raise_timeout(self): ave.cmd.run('sleep 10', timeout=1) @Control.rpc def raise_run_error(self): ave.cmd.run(['no_such_exe']) @Control.rpc def run_external(self, cmd): return ave.cmd.run(cmd) @Control.rpc def connect_remote(self, address, password): if type(password) == unicode: password = str(password) # bind control to self to prevent garbage collection self.outbound = RemoteControl(tuple(address), password, 1) self.add_connection(self.outbound.connect(1), password) return self.outbound.sync_ping() @Control.rpc def mock_death(self): # try to keep Control sockets open after process death by spawning a # subprocess and then commit suicide, using SIGKILL. the observable # behavior (when this works) is that the call to mock_death() never # returns to the client because no answer is sent (the process is dead) # while its open sockets keep lingering (is this due to a GC detail in # Python that keeps sockets open as long as a child process is still # around?). from the client's point of view, the call to mock_death # hangs indefinitely. # TODO: there is a race condition if the subprocess has not been fully # started when the parent gets SIGKILL'ed. should try to remove the call # to sync_ping(). however, this should not be a problem in real life # because all processes except the root broker process are governed by # their parents, which never receive SIGKILL. only the broker will ever # get SIGKILL'ed, and even that should never happen in real deployment. # a simple protection against that eventuality would be to respawn it # immediately on startup and register the spawn for PDEATHSIG, before # it starts accepting client connections, so that everything after this # point is guaranteed to be insulated from the race condition. sock, port = find_free_port() c = MockControl(port, 'authkey', sock) c.start() r = RemoteControl(('',port), 'authkey', timeout=5) try: r.connect(2) except Exception, e: traceback.print_exc() os.kill(self.pid, signal.SIGKILL) # suicide! return os.kill(self.pid, signal.SIGKILL) # suicide!