def t9(factory):
pretty = '%s t9' % __file__
print(pretty)
handover = factory.make_master('master')
client = RemoteBroker(handover.address, home=factory.HOME.path)
# make first allocation
h, w = client.get_resources({'type': 'handset'}, {'type': 'workspace'})
# hand over
adoption, config, fdtx_path = handover.begin_handover()
takeover = factory.make_takeover('master', adoption, config, fdtx_path)
handover.end_handover(1)
# make seconc allocation
try:
client.get({'type': 'handset'})
print('FAIL %s: second allocation did not fail' % pretty)
return False
except Restarting:
pass # good
except Exception, e:
print('FAIL %s: wrong exception: %s' % (pretty, e))
return False
def t3(factory):
pretty = '%s t3' % __file__
print(pretty)
master = factory.make_master('master', [])
slave = factory.make_share(master, 'slave', True)
time.sleep(1)
r1 = master.list_equipment()
r2 = slave.list_equipment()
wanted = r2.pop()
client = RemoteBroker(master.address, home=factory.HOME.path)
handset = client.get_resources(wanted)
del client # drop connection to "crash" client. note that this would not
# work if we had made the allocation with the master because it would not
# get garbage collected until t3() returns.
# check that wanted is available again from the master
found = False
for i in range(10):
if master.list_available(wanted):
found = True
break
time.sleep(0.3)
if not found:
print('FAIL %s: resource not available after job crash' % pretty)
return False
if wanted not in slave.list_available():
print('FAIL %s: resource not available from slave' % pretty)
return False
return True
def t14(factory):
pretty = '%s t14' % __file__
print(pretty)
PROFILE = {'type': 'handset'}
MAX_ALLOWED_ITERATIONS = 10
try:
master = factory.make_master('master')
slave_a = factory.make_share(master, 'slave_a', True)
slave_b = factory.make_share(slave_a, 'slave_b', True)
if not verify_allocators_on_broker(pretty, master, 2):
return False
# create clients for manipulation
master_client = RemoteBroker(master.address, home=factory.HOME.path)
slave_a_client = RemoteBroker(slave_a.address, home=factory.HOME.path)
slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path)
handsets_master = []
handsets_slave_a = []
handsets_slave_b = []
except Exception, e:
print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message))
return False
def run(self, address, q1, q2, home):
r2 = RemoteBroker(address, home=home)
try: # take whatever handset is available
w, d = r2.get_resource({'type':'workspace'}, {'type':'handset'})
except Exception, e:
print('FAIL %s: got no handset: %s' % (pretty, str(e)))
return
def t22(HOME):
pretty = '%s t22' % __file__
print(pretty)
ave.config.create_default(HOME.path)
path = os.path.join(HOME.path, '.ave', 'config', 'broker.json')
with open(path, 'w') as f:
json.dump({'logging': False}, f)
sock, port = find_free_port()
b = Broker(socket=sock, authkeys={'admin': 'key'}, home=HOME.path)
b.start()
r = RemoteBroker(('', port), home=HOME.path)
try:
r.stop()
print('FAIL %s: stop() did not fail' % pretty)
b.join()
return False
except Exception, e:
if 'not authorized to make this call' not in str(e):
print('FAIL %s: wrong error message: %s' % (pretty, str(e)))
b.terminate()
b.join()
return False
def restart(self, authkey):
# find the already running broker. the port number mustn't have changed
handover = RemoteBroker(timeout=3, authkey=authkey)
try:
adoption, config, fdtx_path = handover.begin_handover()
except Exception, e:
self.log('ERROR: handover failed: %s' % str(e))
return
def t15(factory):
pretty = '%s t15' % __file__
print(pretty)
ws_config = {
'root' : factory.HOME.path,
'env' : [],
'tools' : {},
'flocker': {
"ftp": {
"password": "******",
"store": "/srv/www/flocker",
"port": 21,
"timeout": 30,
"user": "******"
},
"host": "cnbjlx20050",
"enable" : True,
"http": {
"doc-root": "/srv/www",
"port": 80
}
}
}
factory.write_config('workspace.json', json.dumps(ws_config))
sock, port = find_free_port()
# set remote explicitly to avoid reading config from disk
broker = Broker(
('',port), sock, remote={}, authkeys={'admin':None}, hsl_paths=[],
home=factory.HOME.path
)
broker.start()
proc = psutil.Process(broker.pid)
remote = RemoteBroker(address=('',port), home=factory.HOME.path)
l = remote.list_available() # just to make sure the connection is up
del remote # client disconnects itself
# check the CPU utilization of the broker through it's PID
result = True
for i in range(10):
if 'get_cpu_percent' in dir(proc):
load = proc.get_cpu_percent() * psutil.NUM_CPUS
else:
load = proc.cpu_percent() * psutil.cpu_count()
if load > 90.0:
print('FAIL %s: runaway CPU load: %f' % (pretty, load))
result = False
break
time.sleep(0.3)
broker.terminate()
broker.join()
return result
def run(self, address, q1, q2, home):
# get own interface to the broker. expect all client's resources to
# get reclaimed when the interface gets garbage collected
r2 = RemoteBroker(address, home=home)
# take whatever handset is available
try:
w, d = r2.get_resource({'type':'workspace'}, {'type':'handset'})
except Exception, e:
print('FAIL %s: got no handset: %s' % (pretty, str(e)))
return
def t16(HOME, r):
pretty = '%s t16' % __file__
print(pretty)
result = True
for i in range(20):
r2 = RemoteBroker(r.address, home=HOME.path)
try:
h = r2.get_resources({'type':'handset'})
except Exception, e:
print('FAIL %s: iteration %d failed: %s' % (pretty, i, str(e)))
result = False
break
def t10(factory):
pretty = '%s t10' % __file__
print(pretty)
master = factory.make_master('master')
share = factory.make_share(master, 'share')
share.start_sharing()
time.sleep(1)
client = RemoteBroker(address=master.address, home=factory.HOME.path)
h = client.get_resources({'type': 'handset', 'serial': 'share-1'})
a1 = master.list_available()
# restart the share
adoption, config, fdtx_path = share.begin_handover()
takeover = factory.make_takeover('share', adoption, config, fdtx_path)
a2 = master.list_available()
if len(a1) == len(a2):
print('FAIL %s: shared resources still visible: %s' % (pretty, a2))
return False
# finish the handover so that takeover can start accepting RPC's. then
# check that the master sees all equipment except the one allocated
share.end_handover(1)
ok = False
for i in range(10):
a3 = master.list_available()
if len(a3) == len(a1):
ok = True
break
time.sleep(0.3)
if not ok:
print('FAIL %s: wrong availability: %s' % (pretty, a3))
return False
for profile in a3:
if 'serial' in profile and profile['serial'] == 'share-1':
print('FAIL %s: busy equipment shared' % pretty)
return False
# finally check that the resource can still be manipulated
try:
p = h.get_profile()
if p['serial'] != 'share-1':
print('FAIL %s: wrong profile: %s' % (pretty, p))
return False
except Exception, e:
print('FAIL %s: unexpected error: %s' % (pretty, e))
return False
def make_forwarder(self, master, prefix, HOME):
sock, port = find_free_port()
handsets = make_handsets(prefix)
config = {'host': '', 'port': master.address[1], 'policy': 'forward'}
ws_cfg = {
'root': os.path.join(HOME.path, prefix),
'env': [],
'tools': {
'ls': '/bin/ls'
},
'pretty': prefix
}
broker = MockBroker(('', port),
sock,
config,
handsets, [],
None,
None,
ws_cfg,
None,
None, [],
True,
home=HOME.path)
remote = RemoteBroker(address=('', port), home=HOME.path)
return broker, remote
def t6(HOME, b):
pretty = '%s t6' % __file__
print(pretty)
for i in range(3):
# the broker connection will be garbage collected between iterations,
# causing the broker to free all resources allocated to the client.
b2 = RemoteBroker(b.address, home=HOME.path)
try:
w,h,r = b2.get_resource(
{'type':'workspace'},
{'type':'handset'},
{'type':'relay'}
)
except Exception, e:
print('FAIL %s: allocation %d failed: %s' % (pretty, i, str(e)))
return False
def make_broker(self):
sock, port = find_free_port()
self.write_config('broker.json', {'port': port, 'logging': False})
broker = self._make_broker(sock, port)
broker.start()
remote = RemoteBroker(('', port), 5, 'admin', self.HOME.path)
self.processes.append(broker)
return remote
def t16(factory):
pretty = '%s t16' % __file__
print(pretty)
PROFILE = {'type': 'handset', 'pretty': 'jane'}
try:
master = factory.make_master('master')
slave_a = factory.make_share(master, 'slave_a', True, slow_down=0.1)
slave_b = factory.make_share(slave_a, 'slave_b', True)
if not verify_allocators_on_broker(pretty, master, 2):
raise Exception('Failing allocator setup')
# create clients for manipulation
master_client = RemoteBroker(master.address, home=factory.HOME.path)
slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path)
# Allocate three handsets
h1 = slave_b_client.get(PROFILE)
time.sleep(0.15)
h2 = master_client.get(PROFILE)
time.sleep(0.15)
h3 = master_client.get(PROFILE)
time.sleep(0.15)
except Exception, e:
print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message))
return False
def t1(factory):
pretty = '%s t1' % __file__
print(pretty)
sock, port = find_free_port()
sock.shutdown(socket.SHUT_RDWR)
sock.close()
for i in range(10):
try:
broker = Broker(('',port), home=factory.HOME.path)
broker.start()
remote = RemoteBroker(('',port), 5, 'admin_key', factory.HOME.path)
remote.stop(__async__=True)
broker.join()
except Exception, e:
print('FAIL %s: stopping broker %d failed: %s' % (pretty,i,str(e)))
return False
def t4(factory):
pretty = '%s t4' % __file__
print(pretty)
# add testdrive and stack configurations to a broker
factory.write_config('spirent.json', json.dumps(cfg, indent=4))
b = factory.make_master('master')
handsets = b.list_equipment({'type': 'handset'})
testdrives = b.list_equipment({'type': 'testdrive'})
s = RemoteBroker(b.address, 3, 'share_key')
s.set_stacks('local', [[handsets[0], testdrives[0]]])
# allocate a stack
try:
h, t = b.get_resource({'type': 'handset'}, {'type': 'testdrive'})
except Exception, e:
print('FAIL %s: could not allocate: %s' % (pretty, str(e)))
return False
def t14(HOME, b):
pretty = '%s t14' % __file__
print(pretty)
for i in range(3):
b2 = RemoteBroker(b.address, home=HOME.path)
result = b2.get_resource(
{'type':'relay', 'uid':'master-a'},
{'type':'handset', 'serial':'master-1'},
{'type':'workspace'}
)
if len(result) != 3:
print('FAIL %s: wrong number of resources: %s' % (pretty, result))
return False
b2.yield_resources(*result)
return True
def t3(factory):
pretty = '%s t3' % __file__
print(pretty)
# add testdrive and stack configurations to a broker
factory.write_config('spirent.json', json.dumps(cfg, indent=4))
b = factory.make_master('master')
h = b.list_equipment({'type': 'handset'})[0]
t = b.list_equipment({'type': 'testdrive'})[0]
s = RemoteBroker(b.address, 3, 'share_key')
s.set_stacks('local', [[h, t]])
# check that the stack exists
stacks = b.list_stacks()
if [h, t] not in stacks:
print('FAIL %s: wrong stacks: %s' % (pretty, stacks))
return False
return True
def t6(factory, h1, h2, r1):
pretty = '%s t6' % __file__
print(pretty)
h1p = h1.get_profile()
h2p = h2.get_profile()
# create a master and a share (only the share will see real equipment)
hsl_paths = [h1p['sysfs_path'], h2p['sysfs_path']]
handover = factory.make_master('master', hsl_paths=[])
# droppable connection:
client = RemoteBroker(handover.address, home=factory.HOME.path)
share = factory.make_share(handover,
'share',
autoshare=True,
hsl_paths=hsl_paths)
try:
wait_equipment(handover, [h1p, h2p])
except Timeout:
print('FAIL %s: handsets not found in handover' % pretty)
return False
# allocate handset and save a profile
handset = client.get_resources({
'type': 'handset',
'serial': h1p['serial']
})
profile = HandsetProfile(handset.get_profile())
# stop the handover
adoption, config, fdtx_path = handover.begin_handover() # stops listening
# disconnect the handset while neither broker is listening for updates,
# then drop the allocation entirely (disconnect from the handover). the
# handset should show up as available in power state "boot_completed" in
# the takeover
r1.set_circuit('usb.pc.vcc', False)
try:
handset.wait_power_state('offline', 5)
except Exception, e:
print('FAIL %s: wait for offline failed: %s' % (pretty, e))
return False
def t13(factory):
pretty = '%s t13' % __file__
print(pretty)
try:
master = factory.make_master('master')
slave_a = factory.make_share(master, 'slave_a', True)
slave_b = factory.make_share(slave_a, 'slave_b', True)
if not verify_allocators_on_broker(pretty, master, 2):
return False
# make a new slave connection so that we can close it by garbage
# collecting it. the 'slave' parameter is held by the setup,
# so del(slave) won't work
master_client = RemoteBroker(master.address, home=factory.HOME.path)
slave_a_client = RemoteBroker(slave_a.address, home=factory.HOME.path)
slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path)
except Exception, e:
print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message))
return False
def t13(HOME, r):
pretty = '%s t13' % __file__
print(pretty)
r = RemoteBroker(r.address, home=HOME.path)
h = r.get_resources({'type':'handset'})
p = h.get_profile()
s = RemoteSession(h.address, h.authkey)
s.stop(__async__=True)
ok = False
for i in range(10):
if p not in r.list_allocations_all():
ok = True
break
time.sleep(1)
if not ok:
print('FAIL %s: handset still allocated' % pretty)
return False
return True
def t5(factory):
pretty = '%s t5' % __file__
print(pretty)
handover = factory.make_master('master')
avail_1 = handover.list_available()
# make some allocations
c1 = RemoteBroker(handover.address, home=factory.HOME.path)
h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'})
avail_2 = handover.list_available()
c2 = RemoteBroker(handover.address, home=factory.HOME.path)
h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'})
avail_3 = handover.list_available()
adoption, config, fdtx_path = handover.begin_handover()
session = RemoteSession(h2.address, h2.authkey)
try:
session.crash() # kill the second session during the handover
except ConnectionClosed:
pass
takeover = factory.make_takeover('master', adoption, config, fdtx_path)
handover.end_handover(1)
result = takeover.list_available()
if len(result) != len(avail_2):
print('FAIL %s: wrong avail: %s != %s' % (pretty, result, avail_2))
return False
return True
def t2(HOME, master):
pretty = '%s t2' % __file__
print(pretty)
c1 = RemoteBroker(master.address, authkey=master.authkey, home=HOME.path)
c1.get_resources({'type': 'handset'}, {'type': 'workspace'})
c2 = RemoteBroker(master.address, authkey=master.authkey, home=HOME.path)
c2.get_resources({'type': 'handset'}, {'type': 'relay'})
try:
s = master.serialize()
except Exception, e:
print('FAIL %s: trivial serialization failed: %s' % (pretty, str(e)))
return False
def t23(HOME):
pretty = '%s t23' % __file__
print(pretty)
ave.config.create_default(HOME.path)
path = os.path.join(HOME.path, '.ave', 'config', 'broker.json')
with open(path, 'w') as f:
json.dump({'logging': False}, f)
sock, port = find_free_port()
b = Broker(socket=sock, authkeys={'admin': 'key'}, home=HOME.path)
b.start()
r = RemoteBroker(('', port), authkey='key', home=HOME.path)
try:
r.stop()
except ConnectionClosed:
pass # good
except Exception, e:
print('FAIL %s: admin authkey not accepted: %s' % (pretty, str(e)))
b.terminate()
b.join()
return False
def make_takeover(self,
prefix,
adoption,
config,
fdtx_path,
hsl_paths=None):
if hsl_paths != None: # use real equipment
handsets = []
relays = []
stacks = []
alloc = False
else: # use mocked equipment
hsl_paths = []
handsets = make_handsets(prefix)
relays = make_relays(prefix)
stacks = make_stacks(prefix, 'messy')
alloc = True
broker = MockBroker(address=(config['host'], config['port']),
socket=None,
remote=config['remote'],
handsets=handsets,
relays=relays,
stacks=stacks,
authkeys={
'admin': None,
'share': 'share_key'
},
ws_cfg={
'root': os.path.join(self.HOME.path, prefix),
'tools': {},
'pretty': prefix,
'flocker': FLOCKER
},
adoption=adoption,
fdtx_path=fdtx_path,
hsl_paths=hsl_paths,
alloc=alloc,
autoshare=True,
home=self.HOME.path)
broker.start()
self.processes.append(broker)
remote = RemoteBroker(address=(config['host'], config['port']),
home=self.HOME.path)
return remote
def t4(factory):
pretty = '%s t4' % __file__
print(pretty)
handover = factory.make_master('master')
avail_1 = handover.list_available()
# make some allocations
c1 = RemoteBroker(handover.address, home=factory.HOME.path)
h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'})
avail_2 = handover.list_available()
c2 = RemoteBroker(handover.address, home=factory.HOME.path)
h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'})
avail_3 = handover.list_available()
# hand over
adoption, config, fdtx_path = handover.begin_handover()
takeover = factory.make_takeover('master', adoption, config, fdtx_path)
handover.end_handover(1)
# check that availability is correct. stop the sessions started against the
# handover and check that the resources become availabe in the takeover
result = takeover.list_available()
if len(result) != len(avail_3):
print('FAIL %s: wrong avail 3: %s != %s' % (pretty, result, avail_3))
return False
ok = False
del (c2)
for i in range(10): # allow some time for brokers to detect session death
result = takeover.list_available()
if len(result) == len(avail_2):
ok = True
break
time.sleep(0.3)
if not ok:
print('FAIL %s: wrong avail 2: %s != %s' % (pretty, result, avail_2))
return False
ok = False
del (c1)
for i in range(10): # allow some time for brokers to detect session death
result = takeover.list_available()
if len(result) == len(avail_1):
ok = True
break
time.sleep(0.3)
if not ok:
print('FAIL %s: wrong avail 1: %s != %s' % (pretty, result, avail_2))
return False
return True
def make_master(self, prefix, hsl_paths=None):
sock, port = find_free_port()
if hsl_paths != None: # use real equipment
handsets = []
relays = []
stacks = []
alloc = False
else: # use mocked equipment
hsl_paths = []
handsets = make_handsets(prefix)
relays = make_relays(prefix)
stacks = make_stacks(prefix, 'messy')
alloc = True
broker = MockBroker(address=('', port),
socket=sock,
remote=None,
handsets=handsets,
relays=relays,
stacks=stacks,
authkeys={
'admin': None,
'share': 'share_key'
},
ws_cfg={
'root': os.path.join(self.HOME.path, prefix),
'env': [],
'tools': {
'sleep': '/bin/sleep'
},
'pretty': prefix,
'flocker': FLOCKER
},
adoption=None,
fdtx_path=None,
hsl_paths=hsl_paths,
alloc=alloc,
autoshare=False,
home=self.HOME.path)
broker.start()
remote = RemoteBroker(address=('', port), home=self.HOME.path)
self.processes.append(broker)
return remote
def make_master(self, HOME):
prefix = self.master
sock, port = find_free_port()
if self.pure:
handsets = []
relays = []
stacks = []
else:
handsets = make_handsets(prefix)
relays = make_relays(prefix)
stacks = make_stacks(prefix, self.stacking)
authkeys = {'admin': None, 'share': 'share_key'}
ws_cfg = {
'root': os.path.join(HOME.path, prefix),
'env': [],
'tools': {
'sleep': '/bin/sleep'
},
'pretty': prefix,
'flocker': FLOCKER,
'wifi-capable': True,
'wlan': {
'ssid': 'ssid',
'auth': 'password'
}
}
broker = MockBroker(('', port),
sock,
None,
handsets,
relays,
stacks,
authkeys,
ws_cfg,
None,
None, [],
True,
home=HOME.path)
remote = RemoteBroker(address=('', port), authkey=None, home=HOME.path)
return broker, remote
def t13(factory):
pretty = '%s t13' % __file__
print(pretty)
handover = factory.make_master('master')
# make some sessions
c1 = RemoteBroker(handover.address, home=factory.HOME.path)
h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'})
avail_2 = handover.list_available()
c2 = RemoteBroker(handover.address, home=factory.HOME.path)
h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'})
avail_3 = handover.list_available()
adoption, config, fdtx_path = handover.begin_handover()
takeover = factory.make_takeover('master', adoption, config, fdtx_path)
handover.end_handover(1)
# crash the sessions
session = RemoteSession(h1.address, h1.authkey)
try:
session.crash()
except ConnectionClosed:
pass
session = RemoteSession(h2.address, h2.authkey)
try:
session.crash()
except ConnectionClosed:
pass
for i in range(10): # wait until only one session remains, then close it
authkeys = handover.get_session_authkeys()
if len(authkeys) == 1:
break
time.sleep(0.3)
# check that the handover sends its exit message when the last session is
# closed
try:
handover.close_session(authkeys[0])
except Exit, e:
if str(e) != 'broker restarted. please reconnect':
print('FAIL %s: wrong exit message: %s' % (pretty, str(e)))
return False
def t11(factory):
pretty = '%s t11' % __file__
print(pretty)
master = factory.make_master('master')
share = factory.make_share(master, 'share')
share.start_sharing()
time.sleep(1)
client = RemoteBroker(address=master.address, home=factory.HOME.path)
h1 = client.get_resources({'type': 'handset', 'serial': 'share-1'})
h2 = client.get_resources({'type': 'handset', 'serial': 'master-1'})
a1 = master.list_available()
# restart the master
adoption, config, fdtx_path = master.begin_handover()
takeover = factory.make_takeover('master', adoption, config, fdtx_path)
master.end_handover(1)
# connect to the new master and check the availability again
master = RemoteBroker(address=master.address, home=factory.HOME.path)
ok = False
for i in range(10):
a2 = master.list_available()
if len(a2) == len(a1):
ok = True
break
time.sleep(0.3)
if not ok:
print('FAIL %s: wrong availability: %s' % (pretty, a2))
return False
for profile in a2:
if 'serial' in profile and profile['serial'] == 'share-1':
print('FAIL %s: busy equipment shared' % pretty)
return False
return True