def connect_remote(self, address, password):
if type(password) == unicode:
password = str(password)
# bind control to self to prevent garbage collection
self.outbound = RemoteControl(tuple(address), password, 1)
self.add_connection(self.outbound.connect(1), password)
return self.outbound.sync_ping()
def mock_death(self):
# try to keep Control sockets open after process death by spawning a
# subprocess and then commit suicide, using SIGKILL. the observable
# behavior (when this works) is that the call to mock_death() never
# returns to the client because no answer is sent (the process is dead)
# while its open sockets keep lingering (is this due to a GC detail in
# Python that keeps sockets open as long as a child process is still
# around?). from the client's point of view, the call to mock_death
# hangs indefinitely.
# TODO: there is a race condition if the subprocess has not been fully
# started when the parent gets SIGKILL'ed. should try to remove the call
# to sync_ping(). however, this should not be a problem in real life
# because all processes except the root broker process are governed by
# their parents, which never receive SIGKILL. only the broker will ever
# get SIGKILL'ed, and even that should never happen in real deployment.
# a simple protection against that eventuality would be to respawn it
# immediately on startup and register the spawn for PDEATHSIG, before
# it starts accepting client connections, so that everything after this
# point is guaranteed to be insulated from the race condition.
sock, port = find_free_port()
c = MockControl(port, 'authkey', sock)
c.start()
r = RemoteControl(('',port), 'authkey', timeout=5)
try:
r.connect(2)
except Exception, e:
traceback.print_exc()
os.kill(self.pid, signal.SIGKILL) # suicide!
return
def t11():
pretty = '%s t11' % __file__
print(pretty)
s,p1 = find_free_port()
q1 = Pipe()
ctr1 = MockControl(p1, 'password', s, [], q1)
ctr1.start()
s,p2 = find_free_port()
q2 = Pipe()
ctr2 = MockControl(p2, None, s, [], q2)
ctr2.start()
# ask ctr1 to create a connection to ctr2
rem1 = RemoteControl(('',p1), 'password', timeout=5)
msg = rem1.connect_remote(('',p2), None)
if msg != 'pong':
print('FAIL %s: bad connection established: %s' % (pretty, msg))
ctr1.terminate()
ctr2.terminate()
ctr1.join()
ctr2.join()
return False
# tell ctr1 to shut down and wait for out of band ack
rem1.stop(__async__=True)
try:
msg = q1.get(1)
except Exception, e:
print('FAIL %s: first oob not received: %s' % (pretty, str(e)))
ctr1.terminate()
ctr2.terminate()
return False
def t1(factory):
pretty = '%s t1' % __file__
print(pretty)
master = factory.make_master('master', hsl_paths=[])
port = master.address[1]
b = RemoteControl(('', port), 'share_key', timeout=5)
profile = HandsetProfile({
'type': 'handset',
'serial': 'ABC321',
'sysfs_path': '/sys/devices/pci0000:00/0000:00:1a.7/usb1/1-99/',
'vendor': '0fce',
'product': '0xyz',
'pretty': 'Zphone',
'power_state': 'boot_completed',
'platform': 'android'
})
# add profile to broker
b.add_equipment('local', [profile])
if not profile in master.list_equipment():
print('FAIL %s: failed to add handset to broker: %s, %s' %
(pretty, profile, master.list_equipment()))
return False
return True
def t04(pretty, factory):
ctrl = factory.make_control(home=factory.HOME.path)
pid = ctrl.get_pid()
cport, cpid = ctrl.make_child()
remote = RemoteControl(('',cport), None, None)
exe = os.path.join(os.path.dirname(__file__),'hickup_catch_sigusr1')
remote.run_external([exe, factory.HOME.path], __async__=True)
time.sleep(1)
os.kill(pid, signal.SIGUSR1)
path = os.path.join(factory.HOME.path, '.ave', 'hickup')
wait_hickup_dir(path, 3)
time.sleep(1)
files = glob.glob(os.path.join(path, '*'))
for f in files:
if f.endswith('hickup_catch_sigusr1'):
print('FAIL %s: external child got SIGUSR1' % pretty)
return False
if len(files) != 2:
print('FAIL %s: wrong number of files: %s' % (pretty, files))
return False
return True
def t35(control, remote, pipe):
pretty = '%s t35' % __file__
print(pretty)
remote = RemoteControl(remote.address, remote.authkey, None)
try:
remote.sleep(2)
except ConnectionTimeout, e:
print('FAIL %s: could not sleep for 2 seconds: %s' % (pretty, e))
return False
def t37(control, remote, pipe):
pretty = '%s t37' % __file__
print(pretty)
remote = RemoteControl(remote.address, remote.authkey, 0.5)
try:
garbage = remote.make_garbage()
except Exception, e:
print('FAIL %s: could not handle garbage: %s' % (pretty, e))
return False
def t36(control, remote, pipe):
pretty = '%s t36' % __file__
print(pretty)
remote = RemoteControl(remote.address, remote.authkey, 0.5)
try:
remote.sleep(2)
print('FAIL %s: could sleep for 2 seconds' % pretty)
return False
except ConnectionTimeout, e:
pass
def t2(pretty, factory):
class Defiant(Process):
def handle_SIGTERM(self, signum, frame):
time.sleep(5) # very poor behavior
Process.handle_SIGTERM(self, signum, frame)
def run(self):
while True:
time.sleep(1)
class Spawner(Control):
@Control.rpc
def spawn(self):
pids = []
for i in range(500):
d = Defiant()
d.start()
self.join_later(d)
pids.append(d.pid)
return pids
@Control.rpc
def list_orphans(self):
return [p._pid for p in self.deferred_joins]
sock, port = find_free_port()
spawner = Spawner(port, None, sock)
spawner.start()
factory.processes.append(spawner)
master = RemoteControl(('', port), None, 5)
defiers = master.spawn()
for pid in defiers:
os.kill(pid, signal.SIGTERM)
orphans = master.list_orphans()
if len(orphans) != 500:
print('FAIL %s: there are not 500 orphans: %s' % (pretty, orphans))
return False
for pid in defiers:
os.kill(pid, signal.SIGKILL)
for i in range(10):
orphans = master.list_orphans()
if not orphans:
break
time.sleep(0.3)
if orphans:
print('FAIL %s: some orphans survived: %d' % (pretty, len(orphans)))
return False
return True
def t04(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c2.connect(timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
o = c2.sync_ping(__async__=False)
except Exception, e:
print('FAIL %s: call failed: %s' % (pretty, e))
return False
def t03(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c2.connect(timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
# should not time out because the connection is already established
o = c2.sync_ping(__async__=True)
except Exception, e:
print('FAIL %s: call failed: %s' % (pretty, e))
return False
def t09():
pretty = '%s t9' % __file__
print(pretty)
s,p1 = find_free_port()
q1 = Pipe()
ctr1 = MockControl(p1, 'password', s, [], q1)
ctr1.start()
s,p2 = find_free_port()
q2 = Pipe()
ctr2 = MockControl(p2, None, s, [], q2)
ctr2.start()
rem1 = RemoteControl(('',p1), 'password', timeout=5)
rem1.connect_remote(('',p2), None)
rem2 = RemoteControl(('',p2), None, timeout=5)
rem2.stop(__async__=True)
result = True
try:
msg = q1.get(timeout=1)
if msg != ['',p2]:
print('FAIL %s: wrong connection lost: %s' % (pretty, msg))
result = False
except Exception, e:
print('FAIL %s: connection not lost: %s' % (pretty, str(e)))
result = False
ctr2.terminate()
def t02(pretty, factory):
c1 = factory.make_control(timeout=None)
c2 = RemoteControl(c1.address, None, timeout=None)
c1.run_external('sleep 2', __async__=True)
try:
c2.sync_ping(__async__=True)
print('FAIL %s: implicit connection attempt did not time out' % pretty)
return False
except ConnectionTimeout:
pass # good
except Exception, e:
print('FAIL %s: wrong exception: %s' % (pretty, e))
return False
def make_panotti_server(self, home):
sock,port = find_free_port()
c = PanottiControl(port, sock, home=home)
r = RemoteControl(('',port), None, 1)
c.start()
self.processes.append(c)
return r
def t07():
pretty = '%s t7' % __file__
print(pretty)
conn, ctrl, pipe = make_peers()
step_connect(conn, ctrl)
step_authenticate(conn, ctrl, 'password')
# send the message in multiple pieces to cause partial reads by the server.
# perform control steps in between partial reads to check that the server
# does not hang.
payload = RemoteControl.make_rpc_blob('sync_ping', None)
header = Connection.make_header(payload)
conn.write(header[:2]) # send half the header
ctrl.step_main() # should be a noop
conn.write(header[2:]) # send the rest of the header
ctrl.step_main() # should be a noop
conn.write(payload[:5]) # send the first couple of bytes from the payload
ctrl.step_main() # should be a noop
conn.write(payload[5:]) # send the rest of the payload
ctrl.step_main() # validate and evaluate rpc
ctrl.step_main() # send response to client
msg = conn.get(timeout=2)
expected = {'result': 'pong'}
if json.loads(msg) != expected:
print('FAIL %s: wrong response: %s' % (pretty, msg))
return False
return True
def __init__(self,
host=None,
port=0,
user=None,
pipe=None,
mailbox=None,
home=None):
if not home:
self.home = ave.config.load_etc()['home']
else:
self.home = home
# load the default configuration file
config = ave.gerrit.config.load(self.home)
# override configuration file entries
if host is not None:
config['host'] = host
if port != 0:
config['port'] = port
if user is not None:
config['user'] = user
if pipe and type(pipe) != Pipe:
raise Exception('pipe must be an ave.networking.pipe.Pipe')
if mailbox and (type(mailbox) != tuple or len(mailbox) != 2):
raise Exception('mailbox must be a (host,port) tuple')
# validate the final configuration
ave.gerrit.config.validate(config)
self.config = config
self.pipe = pipe
if mailbox:
self.mailbox = RemoteControl(mailbox, None, None)
# superclass initialization
Process.__init__(self,
target=self._run,
proc_name='ave-gerrit-event-stream')
def make_control(self, home=None, authkey=None, max_tx=-1, timeout=1):
sock,port = find_free_port()
c = MockControl(port, authkey, sock, home=home, max_tx=max_tx)
r = RemoteControl(('',port), None, timeout, home=home)
c.start()
self.processes.append(c)
return r
def make_proxy(self, target):
sock,port = find_free_port()
c = MockControl(port, None, sock, proxee=target)
r = RemoteControl(('',port), None, 1)
c.start()
self.processes.append(c)
return r
def t21():
pretty = '%s t21' % __file__
print(pretty)
sock, port = find_free_port()
accounts = {'admin':'key2', 'root':'key3', 'nobody':'key4'}
c = MockControl(port, 'key1', sock, accounts)
c.start()
r = RemoteControl(('',port), 'key4', timeout=5)
who = r.whoami()
if who != 'key4':
print('FAIL %s: wrong who: %s' % (pretty, who))
c.terminate()
c.join()
return False
r = RemoteControl(('',port), 'key4', timeout=5)
try:
r.whoami_preauth()
except Exception, e:
if str(e) != 'not authorized to make this call':
print('FAIL %s: wrong error: %s' % (pretty, str(e)))
c.terminate()
c.join()
return False
def t10():
pretty = '%s t10' % __file__
print(pretty)
s,p = find_free_port()
q = Pipe()
c = MockControl(p, None, s, [], q)
c.start()
r = RemoteControl(('',p), None, timeout=5)
r.sync_ping()
c.terminate()
try:
q.get(timeout=1)
except Exception, e:
print('FAIL %s: never got oob message: %s' % (pretty, str(e)))
c.kill()
return False
def t23():
pretty = '%s t23' % __file__
print(pretty)
sock, port = find_free_port()
c = MockControl(port, None, sock, {'admin':None, 'root':None})
c.start()
r = RemoteControl(('',port), 'foo bar', timeout=5)
who = r.whoami_preauth()
if who != None:
print('FAIL %s: wrong who: %s' % (pretty, who))
c.terminate()
c.join()
return False
c.terminate()
c.join()
return True
def t07():
pretty = '%s t7' % __file__
print(pretty)
s,p = find_free_port()
ctrl = MockControl(p, 'password', s)
ctrl.start()
con1 = RemoteControl(('',p), 'password', 1)
try:
con1.raise_exit()
except Exit:
pass # good
ctrl.join(2)
con2 = BlockingConnection(('',p), 'password')
try:
con2.connect(timeout=1)
except ConnectionRefused, e:
pass # good
def t17():
pretty = '%s t17' % __file__
print(pretty)
sock, port = find_free_port()
c = MockControl(port, 'key1', sock, {'alt':'key'})
c.start()
r = RemoteControl(('',port), 'key', timeout=5)
who = r.whoami()
result = True
if who != 'key':
print('FAIL %s: wrong who: %s' % (pretty, who))
result = False
c.terminate()
c.join()
return result
def t14():
pretty = '%s t14' % __file__
print(pretty)
s,p = find_free_port()
c = MockControl(p, 'pass', s)
c.start()
r = RemoteControl(('',p), 'pass', timeout=5)
# any call to a non-function will do
result = True
try:
r.bla_bla_bla()
print('FAIL %s: call did not fail' % pretty)
result = False
except Exception, e:
if str(e) != 'no such RPC: bla_bla_bla':
print('FAIL %s: wrong error message: %s' % (pretty, str(e)))
result = False
def t18():
pretty = '%s t18' % __file__
print(pretty)
sock, port = find_free_port()
c = MockControl(port, 'key1', sock, {'alt2':'key2', 'alt3':'key3'})
c.start()
result = True
for key in ['key1', 'key2', 'key3']:
r = RemoteControl(('',port), key, timeout=5)
who = r.whoami()
if who != key:
print('FAIL %s: wrong who: %s' % (pretty, who))
result = False
break
c.terminate()
c.join()
return result
def t20():
pretty = '%s t20' % __file__
print(pretty)
sock, port = find_free_port()
accounts = {'admin':'key2', 'root':'key3', 'nobody':'key4'}
c = MockControl(port, 'key1', sock, accounts)
c.start()
r = RemoteControl(('',port), 'key2', timeout=5)
who = r.whoami_preauth()
if who != 'key2':
print('FAIL %s: wrong who: %s' % (pretty, who))
c.terminate()
c.join()
return False
c.terminate()
c.join()
return True
def t08():
pretty = '%s t8' % __file__
print(pretty)
conn, ctrl, pipe = make_peers()
step_connect(conn, ctrl)
step_authenticate(conn, ctrl, 'password')
# set the recv/send buffer sizes for all control connections to 1 byte.
# this will normally result in buffer sizes of about 2 kilobytes.
payload = RemoteControl.make_rpc_blob('set_connection_buf_sizes', None,
1024 * 64, 1)
response = step_call(conn, ctrl, payload)
recv_size, send_size = tuple(response['result'])
# craft a message that is just too big to send in one chunk
payload = RemoteControl.make_rpc_blob('sync_ping', None,
'.' * send_size * 30)
conn.put(payload)
ctrl.step_main() # read message from client
# read the response back to the client. the server will need to perform a
# number of steps in its main loop to get all the bytes out on the socket.
response = ''
while True:
ctrl.step_main() # send response to client
try:
# reading non-blocking should never fail under these conditions,
# except that data may not be available (raises ConnectionAgain).
response += Connection.read(conn, send_size)
except ConnectionAgain:
continue
except Exception, e:
print('FAIL %s: wrong error: %s' % (pretty, e))
return False
try:
size, payload = Connection.validate_message(response)
result = json.loads(payload)
break
except Exception, e:
pass # simply not done yet. do another step in the loop
def t19():
pretty = '%s t19' % __file__
print(pretty)
sock, port = find_free_port()
c = MockControl(port, 'key1', sock, {'alt2':'key2', 'alt3':'key3'})
c.start()
r = RemoteControl(('',port), 'no such key', timeout=5)
who = r.whoami()
if who != None:
print('FAIL %s: wrong who 1: %s' % (pretty, who))
c.terminate()
c.join()
return False
r = RemoteControl(('',port), 'key3', timeout=5)
who = r.whoami()
if who != 'key3':
print('FAIL %s: wrong who 2: %s' % (pretty, who))
c.terminate()
c.join()
return False
c.terminate()
c.join()
return True
def t13():
pretty = '%s t13' % __file__
print(pretty)
s,p1 = find_free_port()
q1 = Pipe()
ctr1 = MockControl(p1, 'pass', s, [], q1)
ctr1.start()
s,p2 = find_free_port()
q2 = Pipe()
ctr2 = MockControl(p2, 'secret', s, [], q2)
ctr2.start()
# ask ctr1 to create a connection to ctr2
rem1 = RemoteControl(('',p1), 'pass', timeout=5)
msg = rem1.connect_remote(('',p2), 'secret')
if msg != 'pong':
print('FAIL %s: bad connection established: %s' % (pretty, msg))
ctr1.terminate()
ctr2.terminate()
ctr1.join()
ctr2.join()
return False
# stop ctr1 but don't drop the connection to it (rem1 is still in scope and
# doesn't get garbage collected or expressely closed)
rem1.stop(__async__=True)
ctr1.join()
# check if ctr2 discovered the connection loss from ctr1
result = True
try:
msg = q2.get(1)
if type(msg) != list: # no way of knowing the exact content beforehand
print('FAIL %s: wrong oob received: %s' % (pretty, msg))
result = False
except Exception, e:
print('FAIL %s: oob not received: %s' % (pretty, str(e)))
result = False
def t08():
pretty = '%s t8' % __file__
print(pretty)
s,p1 = find_free_port()
q = Pipe()
ctr1 = MockControl(p1, 'password', s, [], q)
ctr1.start()
s,p2 = find_free_port()
ctr2 = MockControl(p2, None, s)
ctr2.start()
rem1 = RemoteControl(('',p1), 'password', timeout=5)
rem1.connect_remote(('',p2), None)
result = True
ctr2.kill()
try:
msg = q.get(timeout=1)
except Exception, e:
print('FAIL %s: connection not lost: %s' % (pretty, str(e)))
result = False
