def t16(factory): pretty = '%s t16' % __file__ print(pretty) PROFILE = {'type': 'handset', 'pretty': 'jane'} try: master = factory.make_master('master') slave_a = factory.make_share(master, 'slave_a', True, slow_down=0.1) slave_b = factory.make_share(slave_a, 'slave_b', True) if not verify_allocators_on_broker(pretty, master, 2): raise Exception('Failing allocator setup') # create clients for manipulation master_client = RemoteBroker(master.address, home=factory.HOME.path) slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path) # Allocate three handsets h1 = slave_b_client.get(PROFILE) time.sleep(0.15) h2 = master_client.get(PROFILE) time.sleep(0.15) h3 = master_client.get(PROFILE) time.sleep(0.15) except Exception, e: print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message)) return False
def t14(factory): pretty = '%s t14' % __file__ print(pretty) PROFILE = {'type': 'handset'} MAX_ALLOWED_ITERATIONS = 10 try: master = factory.make_master('master') slave_a = factory.make_share(master, 'slave_a', True) slave_b = factory.make_share(slave_a, 'slave_b', True) if not verify_allocators_on_broker(pretty, master, 2): return False # create clients for manipulation master_client = RemoteBroker(master.address, home=factory.HOME.path) slave_a_client = RemoteBroker(slave_a.address, home=factory.HOME.path) slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path) handsets_master = [] handsets_slave_a = [] handsets_slave_b = [] except Exception, e: print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message)) return False
def t5(factory): pretty = '%s t5' % __file__ print(pretty) handover = factory.make_master('master') avail_1 = handover.list_available() # make some allocations c1 = RemoteBroker(handover.address, home=factory.HOME.path) h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'}) avail_2 = handover.list_available() c2 = RemoteBroker(handover.address, home=factory.HOME.path) h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'}) avail_3 = handover.list_available() adoption, config, fdtx_path = handover.begin_handover() session = RemoteSession(h2.address, h2.authkey) try: session.crash() # kill the second session during the handover except ConnectionClosed: pass takeover = factory.make_takeover('master', adoption, config, fdtx_path) handover.end_handover(1) result = takeover.list_available() if len(result) != len(avail_2): print('FAIL %s: wrong avail: %s != %s' % (pretty, result, avail_2)) return False return True
def t4(factory): pretty = '%s t4' % __file__ print(pretty) handover = factory.make_master('master') avail_1 = handover.list_available() # make some allocations c1 = RemoteBroker(handover.address, home=factory.HOME.path) h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'}) avail_2 = handover.list_available() c2 = RemoteBroker(handover.address, home=factory.HOME.path) h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'}) avail_3 = handover.list_available() # hand over adoption, config, fdtx_path = handover.begin_handover() takeover = factory.make_takeover('master', adoption, config, fdtx_path) handover.end_handover(1) # check that availability is correct. stop the sessions started against the # handover and check that the resources become availabe in the takeover result = takeover.list_available() if len(result) != len(avail_3): print('FAIL %s: wrong avail 3: %s != %s' % (pretty, result, avail_3)) return False ok = False del (c2) for i in range(10): # allow some time for brokers to detect session death result = takeover.list_available() if len(result) == len(avail_2): ok = True break time.sleep(0.3) if not ok: print('FAIL %s: wrong avail 2: %s != %s' % (pretty, result, avail_2)) return False ok = False del (c1) for i in range(10): # allow some time for brokers to detect session death result = takeover.list_available() if len(result) == len(avail_1): ok = True break time.sleep(0.3) if not ok: print('FAIL %s: wrong avail 1: %s != %s' % (pretty, result, avail_2)) return False return True
def t2(HOME, master): pretty = '%s t2' % __file__ print(pretty) c1 = RemoteBroker(master.address, authkey=master.authkey, home=HOME.path) c1.get_resources({'type': 'handset'}, {'type': 'workspace'}) c2 = RemoteBroker(master.address, authkey=master.authkey, home=HOME.path) c2.get_resources({'type': 'handset'}, {'type': 'relay'}) try: s = master.serialize() except Exception, e: print('FAIL %s: trivial serialization failed: %s' % (pretty, str(e))) return False
def t22(HOME): pretty = '%s t22' % __file__ print(pretty) ave.config.create_default(HOME.path) path = os.path.join(HOME.path, '.ave', 'config', 'broker.json') with open(path, 'w') as f: json.dump({'logging': False}, f) sock, port = find_free_port() b = Broker(socket=sock, authkeys={'admin': 'key'}, home=HOME.path) b.start() r = RemoteBroker(('', port), home=HOME.path) try: r.stop() print('FAIL %s: stop() did not fail' % pretty) b.join() return False except Exception, e: if 'not authorized to make this call' not in str(e): print('FAIL %s: wrong error message: %s' % (pretty, str(e))) b.terminate() b.join() return False
def t3(factory): pretty = '%s t3' % __file__ print(pretty) master = factory.make_master('master', []) slave = factory.make_share(master, 'slave', True) time.sleep(1) r1 = master.list_equipment() r2 = slave.list_equipment() wanted = r2.pop() client = RemoteBroker(master.address, home=factory.HOME.path) handset = client.get_resources(wanted) del client # drop connection to "crash" client. note that this would not # work if we had made the allocation with the master because it would not # get garbage collected until t3() returns. # check that wanted is available again from the master found = False for i in range(10): if master.list_available(wanted): found = True break time.sleep(0.3) if not found: print('FAIL %s: resource not available after job crash' % pretty) return False if wanted not in slave.list_available(): print('FAIL %s: resource not available from slave' % pretty) return False return True
def t9(factory): pretty = '%s t9' % __file__ print(pretty) handover = factory.make_master('master') client = RemoteBroker(handover.address, home=factory.HOME.path) # make first allocation h, w = client.get_resources({'type': 'handset'}, {'type': 'workspace'}) # hand over adoption, config, fdtx_path = handover.begin_handover() takeover = factory.make_takeover('master', adoption, config, fdtx_path) handover.end_handover(1) # make seconc allocation try: client.get({'type': 'handset'}) print('FAIL %s: second allocation did not fail' % pretty) return False except Restarting: pass # good except Exception, e: print('FAIL %s: wrong exception: %s' % (pretty, e)) return False
def run(self, address, q1, q2, home): r2 = RemoteBroker(address, home=home) try: # take whatever handset is available w, d = r2.get_resource({'type':'workspace'}, {'type':'handset'}) except Exception, e: print('FAIL %s: got no handset: %s' % (pretty, str(e))) return
def make_forwarder(self, master, prefix, HOME): sock, port = find_free_port() handsets = make_handsets(prefix) config = {'host': '', 'port': master.address[1], 'policy': 'forward'} ws_cfg = { 'root': os.path.join(HOME.path, prefix), 'env': [], 'tools': { 'ls': '/bin/ls' }, 'pretty': prefix } broker = MockBroker(('', port), sock, config, handsets, [], None, None, ws_cfg, None, None, [], True, home=HOME.path) remote = RemoteBroker(address=('', port), home=HOME.path) return broker, remote
def restart(self, authkey): # find the already running broker. the port number mustn't have changed handover = RemoteBroker(timeout=3, authkey=authkey) try: adoption, config, fdtx_path = handover.begin_handover() except Exception, e: self.log('ERROR: handover failed: %s' % str(e)) return
def make_broker(self): sock, port = find_free_port() self.write_config('broker.json', {'port': port, 'logging': False}) broker = self._make_broker(sock, port) broker.start() remote = RemoteBroker(('', port), 5, 'admin', self.HOME.path) self.processes.append(broker) return remote
def t13(factory): pretty = '%s t13' % __file__ print(pretty) handover = factory.make_master('master') # make some sessions c1 = RemoteBroker(handover.address, home=factory.HOME.path) h1, w1 = c1.get_resources({'type': 'handset'}, {'type': 'workspace'}) avail_2 = handover.list_available() c2 = RemoteBroker(handover.address, home=factory.HOME.path) h2, r2 = c2.get_resources({'type': 'handset'}, {'type': 'relay'}) avail_3 = handover.list_available() adoption, config, fdtx_path = handover.begin_handover() takeover = factory.make_takeover('master', adoption, config, fdtx_path) handover.end_handover(1) # crash the sessions session = RemoteSession(h1.address, h1.authkey) try: session.crash() except ConnectionClosed: pass session = RemoteSession(h2.address, h2.authkey) try: session.crash() except ConnectionClosed: pass for i in range(10): # wait until only one session remains, then close it authkeys = handover.get_session_authkeys() if len(authkeys) == 1: break time.sleep(0.3) # check that the handover sends its exit message when the last session is # closed try: handover.close_session(authkeys[0]) except Exit, e: if str(e) != 'broker restarted. please reconnect': print('FAIL %s: wrong exit message: %s' % (pretty, str(e))) return False
def t15(factory): pretty = '%s t15' % __file__ print(pretty) ws_config = { 'root' : factory.HOME.path, 'env' : [], 'tools' : {}, 'flocker': { "ftp": { "password": "******", "store": "/srv/www/flocker", "port": 21, "timeout": 30, "user": "******" }, "host": "cnbjlx20050", "enable" : True, "http": { "doc-root": "/srv/www", "port": 80 } } } factory.write_config('workspace.json', json.dumps(ws_config)) sock, port = find_free_port() # set remote explicitly to avoid reading config from disk broker = Broker( ('',port), sock, remote={}, authkeys={'admin':None}, hsl_paths=[], home=factory.HOME.path ) broker.start() proc = psutil.Process(broker.pid) remote = RemoteBroker(address=('',port), home=factory.HOME.path) l = remote.list_available() # just to make sure the connection is up del remote # client disconnects itself # check the CPU utilization of the broker through it's PID result = True for i in range(10): if 'get_cpu_percent' in dir(proc): load = proc.get_cpu_percent() * psutil.NUM_CPUS else: load = proc.cpu_percent() * psutil.cpu_count() if load > 90.0: print('FAIL %s: runaway CPU load: %f' % (pretty, load)) result = False break time.sleep(0.3) broker.terminate() broker.join() return result
def run(self, address, q1, q2, home): # get own interface to the broker. expect all client's resources to # get reclaimed when the interface gets garbage collected r2 = RemoteBroker(address, home=home) # take whatever handset is available try: w, d = r2.get_resource({'type':'workspace'}, {'type':'handset'}) except Exception, e: print('FAIL %s: got no handset: %s' % (pretty, str(e))) return
def t3(pretty, factory): factory.write_config('authkeys.json', setup.AUTHKEYS) b = factory.make_broker() client = RemoteBroker(b.address) admin = RemoteBroker(b.address,authkey='admin') # create a server with a couple of boards handover = factory.make_server() handover.set_boards([setup.BOARD_1,setup.BOARD_2,setup.BOARD_3]) # wait for added boards to become available in the broker ok = False wanted = RelayProfile(setup.RELAY_3) for i in range(10): if wanted in admin.list_available(): ok = True break time.sleep(0.3) if not ok: print('FAIL %s: boards not available' % pretty) return False # let the client allocate a relay and manipulate it to make sure it has # connected to the relay server relay = client.get({ 'type':'relay','circuits':['usb.pc.vcc','handset.battery'] }) relay.set_circuit('usb.pc.vcc', False) # hand over to the replacement server state = handover.begin_handover() takeover = factory.make_server(state['boards'], make_socket=False) takeover.set_boards([setup.BOARD_1,setup.BOARD_2,setup.BOARD_3]) handover.end_handover(__async__=True) try: relay.set_circuit('handset.battery', False) except AveException, e: print e.format_trace() print('FAIL %s: could not set battery circuit: %s' % (pretty, e)) return False
def t13(factory): pretty = '%s t13' % __file__ print(pretty) try: master = factory.make_master('master') slave_a = factory.make_share(master, 'slave_a', True) slave_b = factory.make_share(slave_a, 'slave_b', True) if not verify_allocators_on_broker(pretty, master, 2): return False # make a new slave connection so that we can close it by garbage # collecting it. the 'slave' parameter is held by the setup, # so del(slave) won't work master_client = RemoteBroker(master.address, home=factory.HOME.path) slave_a_client = RemoteBroker(slave_a.address, home=factory.HOME.path) slave_b_client = RemoteBroker(slave_b.address, home=factory.HOME.path) except Exception, e: print('FAIL %s: Setup failed, Exception: %s' % (pretty, e.message)) return False
def t16(HOME, r): pretty = '%s t16' % __file__ print(pretty) result = True for i in range(20): r2 = RemoteBroker(r.address, home=HOME.path) try: h = r2.get_resources({'type':'handset'}) except Exception, e: print('FAIL %s: iteration %d failed: %s' % (pretty, i, str(e))) result = False break
def t11(factory): pretty = '%s t11' % __file__ print(pretty) master = factory.make_master('master') share = factory.make_share(master, 'share') share.start_sharing() time.sleep(1) client = RemoteBroker(address=master.address, home=factory.HOME.path) h1 = client.get_resources({'type': 'handset', 'serial': 'share-1'}) h2 = client.get_resources({'type': 'handset', 'serial': 'master-1'}) a1 = master.list_available() # restart the master adoption, config, fdtx_path = master.begin_handover() takeover = factory.make_takeover('master', adoption, config, fdtx_path) master.end_handover(1) # connect to the new master and check the availability again master = RemoteBroker(address=master.address, home=factory.HOME.path) ok = False for i in range(10): a2 = master.list_available() if len(a2) == len(a1): ok = True break time.sleep(0.3) if not ok: print('FAIL %s: wrong availability: %s' % (pretty, a2)) return False for profile in a2: if 'serial' in profile and profile['serial'] == 'share-1': print('FAIL %s: busy equipment shared' % pretty) return False return True
def t10(factory): pretty = '%s t10' % __file__ print(pretty) master = factory.make_master('master') share = factory.make_share(master, 'share') share.start_sharing() time.sleep(1) client = RemoteBroker(address=master.address, home=factory.HOME.path) h = client.get_resources({'type': 'handset', 'serial': 'share-1'}) a1 = master.list_available() # restart the share adoption, config, fdtx_path = share.begin_handover() takeover = factory.make_takeover('share', adoption, config, fdtx_path) a2 = master.list_available() if len(a1) == len(a2): print('FAIL %s: shared resources still visible: %s' % (pretty, a2)) return False # finish the handover so that takeover can start accepting RPC's. then # check that the master sees all equipment except the one allocated share.end_handover(1) ok = False for i in range(10): a3 = master.list_available() if len(a3) == len(a1): ok = True break time.sleep(0.3) if not ok: print('FAIL %s: wrong availability: %s' % (pretty, a3)) return False for profile in a3: if 'serial' in profile and profile['serial'] == 'share-1': print('FAIL %s: busy equipment shared' % pretty) return False # finally check that the resource can still be manipulated try: p = h.get_profile() if p['serial'] != 'share-1': print('FAIL %s: wrong profile: %s' % (pretty, p)) return False except Exception, e: print('FAIL %s: unexpected error: %s' % (pretty, e)) return False
def t6(HOME, b): pretty = '%s t6' % __file__ print(pretty) for i in range(3): # the broker connection will be garbage collected between iterations, # causing the broker to free all resources allocated to the client. b2 = RemoteBroker(b.address, home=HOME.path) try: w,h,r = b2.get_resource( {'type':'workspace'}, {'type':'handset'}, {'type':'relay'} ) except Exception, e: print('FAIL %s: allocation %d failed: %s' % (pretty, i, str(e))) return False
def make_takeover(self, prefix, adoption, config, fdtx_path, hsl_paths=None): if hsl_paths != None: # use real equipment handsets = [] relays = [] stacks = [] alloc = False else: # use mocked equipment hsl_paths = [] handsets = make_handsets(prefix) relays = make_relays(prefix) stacks = make_stacks(prefix, 'messy') alloc = True broker = MockBroker(address=(config['host'], config['port']), socket=None, remote=config['remote'], handsets=handsets, relays=relays, stacks=stacks, authkeys={ 'admin': None, 'share': 'share_key' }, ws_cfg={ 'root': os.path.join(self.HOME.path, prefix), 'tools': {}, 'pretty': prefix, 'flocker': FLOCKER }, adoption=adoption, fdtx_path=fdtx_path, hsl_paths=hsl_paths, alloc=alloc, autoshare=True, home=self.HOME.path) broker.start() self.processes.append(broker) remote = RemoteBroker(address=(config['host'], config['port']), home=self.HOME.path) return remote
def t4(factory): pretty = '%s t4' % __file__ print(pretty) # add testdrive and stack configurations to a broker factory.write_config('spirent.json', json.dumps(cfg, indent=4)) b = factory.make_master('master') handsets = b.list_equipment({'type': 'handset'}) testdrives = b.list_equipment({'type': 'testdrive'}) s = RemoteBroker(b.address, 3, 'share_key') s.set_stacks('local', [[handsets[0], testdrives[0]]]) # allocate a stack try: h, t = b.get_resource({'type': 'handset'}, {'type': 'testdrive'}) except Exception, e: print('FAIL %s: could not allocate: %s' % (pretty, str(e))) return False
def t1(factory): pretty = '%s t1' % __file__ print(pretty) sock, port = find_free_port() sock.shutdown(socket.SHUT_RDWR) sock.close() for i in range(10): try: broker = Broker(('',port), home=factory.HOME.path) broker.start() remote = RemoteBroker(('',port), 5, 'admin_key', factory.HOME.path) remote.stop(__async__=True) broker.join() except Exception, e: print('FAIL %s: stopping broker %d failed: %s' % (pretty,i,str(e))) return False
def make_master(self, prefix, hsl_paths=None): sock, port = find_free_port() if hsl_paths != None: # use real equipment handsets = [] relays = [] stacks = [] alloc = False else: # use mocked equipment hsl_paths = [] handsets = make_handsets(prefix) relays = make_relays(prefix) stacks = make_stacks(prefix, 'messy') alloc = True broker = MockBroker(address=('', port), socket=sock, remote=None, handsets=handsets, relays=relays, stacks=stacks, authkeys={ 'admin': None, 'share': 'share_key' }, ws_cfg={ 'root': os.path.join(self.HOME.path, prefix), 'env': [], 'tools': { 'sleep': '/bin/sleep' }, 'pretty': prefix, 'flocker': FLOCKER }, adoption=None, fdtx_path=None, hsl_paths=hsl_paths, alloc=alloc, autoshare=False, home=self.HOME.path) broker.start() remote = RemoteBroker(address=('', port), home=self.HOME.path) self.processes.append(broker) return remote
def t14(HOME, b): pretty = '%s t14' % __file__ print(pretty) for i in range(3): b2 = RemoteBroker(b.address, home=HOME.path) result = b2.get_resource( {'type':'relay', 'uid':'master-a'}, {'type':'handset', 'serial':'master-1'}, {'type':'workspace'} ) if len(result) != 3: print('FAIL %s: wrong number of resources: %s' % (pretty, result)) return False b2.yield_resources(*result) return True
def t6(factory, h1, h2, r1): pretty = '%s t6' % __file__ print(pretty) h1p = h1.get_profile() h2p = h2.get_profile() # create a master and a share (only the share will see real equipment) hsl_paths = [h1p['sysfs_path'], h2p['sysfs_path']] handover = factory.make_master('master', hsl_paths=[]) # droppable connection: client = RemoteBroker(handover.address, home=factory.HOME.path) share = factory.make_share(handover, 'share', autoshare=True, hsl_paths=hsl_paths) try: wait_equipment(handover, [h1p, h2p]) except Timeout: print('FAIL %s: handsets not found in handover' % pretty) return False # allocate handset and save a profile handset = client.get_resources({ 'type': 'handset', 'serial': h1p['serial'] }) profile = HandsetProfile(handset.get_profile()) # stop the handover adoption, config, fdtx_path = handover.begin_handover() # stops listening # disconnect the handset while neither broker is listening for updates, # then drop the allocation entirely (disconnect from the handover). the # handset should show up as available in power state "boot_completed" in # the takeover r1.set_circuit('usb.pc.vcc', False) try: handset.wait_power_state('offline', 5) except Exception, e: print('FAIL %s: wait for offline failed: %s' % (pretty, e)) return False
def t3(factory): pretty = '%s t3' % __file__ print(pretty) # add testdrive and stack configurations to a broker factory.write_config('spirent.json', json.dumps(cfg, indent=4)) b = factory.make_master('master') h = b.list_equipment({'type': 'handset'})[0] t = b.list_equipment({'type': 'testdrive'})[0] s = RemoteBroker(b.address, 3, 'share_key') s.set_stacks('local', [[h, t]]) # check that the stack exists stacks = b.list_stacks() if [h, t] not in stacks: print('FAIL %s: wrong stacks: %s' % (pretty, stacks)) return False return True
def t13(HOME, r): pretty = '%s t13' % __file__ print(pretty) r = RemoteBroker(r.address, home=HOME.path) h = r.get_resources({'type':'handset'}) p = h.get_profile() s = RemoteSession(h.address, h.authkey) s.stop(__async__=True) ok = False for i in range(10): if p not in r.list_allocations_all(): ok = True break time.sleep(1) if not ok: print('FAIL %s: handset still allocated' % pretty) return False return True
def make_master(self, HOME): prefix = self.master sock, port = find_free_port() if self.pure: handsets = [] relays = [] stacks = [] else: handsets = make_handsets(prefix) relays = make_relays(prefix) stacks = make_stacks(prefix, self.stacking) authkeys = {'admin': None, 'share': 'share_key'} ws_cfg = { 'root': os.path.join(HOME.path, prefix), 'env': [], 'tools': { 'sleep': '/bin/sleep' }, 'pretty': prefix, 'flocker': FLOCKER, 'wifi-capable': True, 'wlan': { 'ssid': 'ssid', 'auth': 'password' } } broker = MockBroker(('', port), sock, None, handsets, relays, stacks, authkeys, ws_cfg, None, None, [], True, home=HOME.path) remote = RemoteBroker(address=('', port), authkey=None, home=HOME.path) return broker, remote