def client_01(port, prefix):
c = RemoteControl(('', port), None, timeout=10)
while True:
if random.random() > 0.9:
c = RemoteControl(('', port), None, timeout=10)
try:
c.sync_ping()
except Exit:
os._exit(0)
except Exception, e:
print('client PID=%d got exception: %s' % (os.getpid(), e))
os._exit(1)
def t02(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
c2.sync_ping(__async__=True)
print('FAIL %s: implicit connection attempt did not time out' % pretty)
return False
except ConnectionTimeout:
pass # good
except Exception, e:
print('FAIL %s: wrong exception: %s' % (pretty, e))
return False
def t10():
pretty = '%s t10' % __file__
print(pretty)
s,p = find_free_port()
q = Pipe()
c = MockControl(p, None, s, [], q)
c.start()
r = RemoteControl(('',p), None, timeout=5)
r.sync_ping()
c.terminate()
try:
q.get(timeout=1)
except Exception, e:
print('FAIL %s: never got oob message: %s' % (pretty, str(e)))
c.kill()
return False
def t04(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c2.connect(timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
o = c2.sync_ping(__async__=False)
except Exception, e:
print('FAIL %s: call failed: %s' % (pretty, e))
return False
def t03(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c2.connect(timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
# should not time out because the connection is already established
o = c2.sync_ping(__async__=True)
except Exception, e:
print('FAIL %s: call failed: %s' % (pretty, e))
return False
try:
c1.start()
c2.start()
except Exception, e:
print('FAIL %s: could not start both controls: %s' % (pretty, str(e)))
return False
try:
r1 = RemoteControl(('',p1), 'password', 1)
r2 = RemoteControl(('',p2), 'password', 1)
except Exception, e:
print('FAIL %s: could not create two remotes: %s' % (pretty, str(e)))
return False
try:
pong1 = r1.sync_ping()
pong2 = r2.sync_ping()
except Exception, e:
print('FAIL %s: could not ping both controls: %s' % (pretty, str(e)))
return False
if pong1 != pong2 != 'pong':
print('FAIL %s: wrong pong: %s %s' % (pretty, pong1, pong2))
return False
c1.terminate()
c2.terminate()
c1.join()
c2.join()
return True
class MockControl(Control):
pipe = None # used to shortcut communication in the tests
proxee = None
def __init__(self, port, authkey, socket, alt_keys=[], pipe=None,
interval=None, proxee=None, home=None, max_tx=-1):
Control.__init__(self, port, authkey, socket, alt_keys, interval, home)
self.pipe = pipe
self.proxee = proxee
self.max_tx = max_tx
def run(self):
Control.run(self)
def shutdown(self, details=None):
Control.shutdown(self, details)
def close_fds(self, exclude):
if self.pipe:
exclude.append(self.pipe.w)
Control.close_fds(self, exclude)
def lost_connection(self, connection, authkey):
if self.pipe:
self.pipe.put(connection.address)
def idle(self):
if self.pipe:
self.pipe.put('idle')
def shutdown(self):
if self.pipe:
self.pipe.put('shutdown')
Control.shutdown(self)
@Control.rpc
def sync_ping(self, alt=None):
return alt or 'pong'
@Control.rpc
def async_ping(self):
if self.pipe:
self.pipe.put('pong')
@Control.rpc
def stop(self):
Control.stop(self)
@Control.rpc
def raise_plain_exception(self, message):
raise Exception(message)
@Control.rpc
def raise_ave_exception(self, details, depth=0):
if self.proxee:
details['message'] = 'proxied: ' + details['message']
self.proxee.raise_ave_exception(details, depth)
if depth > 0:
self.raise_ave_exception(details, depth-1)
raise AveException(details)
@Control.rpc
def raise_exit(self, msg='passed to client'):
raise Exit(msg)
@Control.rpc
def raise_timeout(self):
ave.cmd.run('sleep 10', timeout=1)
@Control.rpc
def raise_run_error(self):
ave.cmd.run(['no_such_exe'])
@Control.rpc
def run_external(self, cmd):
return ave.cmd.run(cmd)
@Control.rpc
def connect_remote(self, address, password):
if type(password) == unicode:
password = str(password)
# bind control to self to prevent garbage collection
self.outbound = RemoteControl(tuple(address), password, 1)
self.add_connection(self.outbound.connect(1), password)
return self.outbound.sync_ping()
@Control.rpc
def mock_death(self):
# try to keep Control sockets open after process death by spawning a
# subprocess and then commit suicide, using SIGKILL. the observable
# behavior (when this works) is that the call to mock_death() never
# returns to the client because no answer is sent (the process is dead)
# while its open sockets keep lingering (is this due to a GC detail in
# Python that keeps sockets open as long as a child process is still
# around?). from the client's point of view, the call to mock_death
# hangs indefinitely.
# TODO: there is a race condition if the subprocess has not been fully
# started when the parent gets SIGKILL'ed. should try to remove the call
# to sync_ping(). however, this should not be a problem in real life
# because all processes except the root broker process are governed by
# their parents, which never receive SIGKILL. only the broker will ever
# get SIGKILL'ed, and even that should never happen in real deployment.
# a simple protection against that eventuality would be to respawn it
# immediately on startup and register the spawn for PDEATHSIG, before
# it starts accepting client connections, so that everything after this
# point is guaranteed to be insulated from the race condition.
sock, port = find_free_port()
c = MockControl(port, 'authkey', sock)
c.start()
r = RemoteControl(('',port), 'authkey', timeout=5)
try:
r.connect(2)
except Exception, e:
traceback.print_exc()
os.kill(self.pid, signal.SIGKILL) # suicide!
return
os.kill(self.pid, signal.SIGKILL) # suicide!