def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): print 'adding producer/consumer for agent: ' + agent count += 1 ext = str(count) data_product_name = 'test-stream-' + ext producer_component_name = 'pro-'+ext consumer_component_name = 'con-'+ext producer_config = InstrumentConfiguration(data_product_name, 1, instrument_configuration=100, easy_registration=True, sleep_even_zero=False, persist_product=False, report_timing=True, timing_rate=5000) producer = InstrumentSimulator(producer_component_name, None, producer_config) consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False)) # consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration('jon.ion.xs.ioncore.xp.science_data', log_value=False)) print 'creating producer and consumer' m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) # print 'next: register producer (creates publisher)' # wait(2) # print 'registering producer' # m.send_request(RegisterWithContainer(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'next: start consumer process' wait(2) print 'starting consumer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) print 'next: starting producer' wait(2) print 'starting producer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'now all components are running' sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) sleep(5)
def _get_controller_agent(self, manager): if not self.controller_agent: self.controller_agent = self.get_agents()[0] log.debug('starting instrument controller component on %s' % self.controller_agent) ims = InstrumentController('device_controller', None, None) manager.send_request(AddComponent(ims), agent_filter=agent_id(self.controller_agent), component_filter=component_id('AGENT')) return self.controller_agent
def start_devices(self, config, manager, nrange=None, catch_up_frequency=1, catch_up_time=2400, timeout=300): """ start all devices defined in config file """ range_str = config.get("start-devices.range").split('-') template = config.get("start-devices.devices") delay = config.get("start-devices.sleep-time") if not template: # log.info("starting devices using template: %s", template) # else: log.warn("no devices indicated for starting") return some_agent = self._get_controller_agent(manager) gateway = self.find_gateway() gw_config = GatewayConfiguration(hostname=gateway) manager.send_request(ChangeConfiguration(gw_config), agent_filter=agent_id(some_agent), component_filter=component_id('device_controller')) time.sleep(5) if not nrange: nrange = xrange(int(range_str[0]), int(range_str[1])+1) for n in nrange: name = template % n # log.info("starting instrument %d at %s", n, time.ctime()) future = self.answer_listener.expect_message() manager.send_request(GetInstrumentId(name), agent_filter=agent_id(some_agent), component_filter=component_id('device_controller')) future.wait(timeout=60) msg = future.get() if msg.exception: raise msg.exception device_id = msg.result future = self.answer_listener.expect_message() manager.send_request(StartDevice(device_id, timeout=timeout), component_filter=component_id('device_controller')) future.wait(timeout=timeout) msg = future.get() if msg.exception: raise msg.exception # else: # log.info('result: %r',msg.result) # give some time for things to stabilize time.sleep(delay) if catch_up_frequency and n%catch_up_frequency==0: catch_up_start = time.time() self._wait_for_messages(n, catch_up_time) elapsed = time.time() - catch_up_start log.info('took %f sec for all devices to send message rates',elapsed)
def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) # start component on each agent and let it add documents to db components = [] initial_config = PotatoConfiguration() initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_config.create_count = 1000 initial_config.threads = 1 for agent in l1.inventory.keys(): print 'adding couch potato for agent: ' + agent component_name = 'chip-'+agent components.append(component_name) component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name)) # while len(l2.latest_data)<len(components): # sleep(1) print 'initialization ops/sec: %f create, %f read, %f update, %f delete' % l2.get_rates() cycle_config = PotatoConfiguration() cycle_config.threads=1 for component_name in components: print 'starting db operations on: ' + component_name m.send_request(ChangeConfiguration(cycle_config), component_filter=component_id(component_name)) m.send_request(StartRequest(), component_filter=component_id(component_name)) # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(Potato)) sleep(5)
def find_node_max(manager, listener, agent, report_rate): """ send messages at max rate on one node """ config = InstrumentConfiguration('node-max-data-product', 0, instrument_configuration=GRANULE_SIZE_FLOATS, report_timing=True, log_timing=True, timing_rate=report_rate) producer = InstrumentSimulator('node-max-producer', None, config) consumer = DataProductConsumer('node-max-consumer', None, ConsumerConfiguration('node-max-data-product', log_value=False)) manager.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) manager.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) manager.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id('node-max-consumer')) manager.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id('node-max-producer')) log.debug('started producer/consumer on %s, waiting for performance measurement', agent) listener.clear() max_wait=300 # 5min for n in xrange(max_wait/5): sleep(5) if listener.get_rate()>0: break # manager.send_request(StopRequest(), agent_filter=agent_id(agent), component_filter=component_id('node-max-consumer')) # manager.send_request(StopRequest(), agent_filter=agent_id(agent), component_filter=component_id('node-max-producer')) return listener.get_rate()
def add_node_traffic(manager, listener, agent, rate, name): """ add producer/consumer on node and make sure message rate is close to target """ # start producer/consumer config = InstrumentConfiguration(name + '-data-product', rate, instrument_configuration=GRANULE_SIZE_FLOATS, log_timing=False, timing_rate=rate*10) producer = InstrumentSimulator(name + '-producer', None, config) consumer = DataProductConsumer(name + '-consumer', None, ConsumerConfiguration(name + '-data-product', log_value=False)) manager.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) manager.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) manager.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(name + '-consumer')) manager.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(name + '-producer')) log.debug('started producer/consumer on %s, waiting for message rate', agent) # measure performance max_wait=600 # 10min measured_rate=0 for n in xrange(max_wait/5): sleep(5) measured_rate = listener.get_node_rate(name + '-producer') if measured_rate>0: break log.debug('agent %s message rate is %.2e', agent, measured_rate) return measured_rate
def main(): start_time = time.time() log = logging.getLogger('test') config = read_test_configuration() # _step('config:\n' + pformat(config.as_dict())) _step('read config file') test = SystemTest(config) try: _step('connecting to agent') test.reconnect_system(cloudinitd=False, filter=agent_id('server')) test.system = None _step('taking inventory') inventory = test.get_inventory() _step('inventory: ' + ', '.join(inventory.keys())) _step('performing base preload') preload_start_time = time.time() test.init_system() preload_time = time.time() _step("preload took %.2f seconds" % (preload_time-preload_start_time)) config = test.get_preload_template() nrange = test.get_preload_range(config) for n in nrange: _step("starting device %d"%n) device_begin = time.time() test.init_device(config,n, catch_up_frequency=1000) # some tests do not run logging_transform elapsed = time.time() - device_begin _step("completed device %d launch in %f seconds" % (n,elapsed)) time.sleep(120) _rates(test.get_message_rates()) _step('performing test') results = test.perform_test() _step('*** test completed! ***') if results: _step('results: ' + str(results)) except Exception,e: log.error('test failed', exc_info=True)
def preload_system(self, config): """ perform DB preload as defined in config file should only be called once -- it starts a component called 'loader' have to refactor to make sure it doesn't create a second loader if we want to be able to call multiple times """ preload_configs = config.get("preload") if not preload_configs: log.warn('no preload config found') return some_agent = self.get_agents()[0] log.info('starting preload component on %s' % some_agent) self.manager.send_request(AddComponent(Preloader('loader', None, None)), agent_filter=agent_id(some_agent), component_filter=component_id('AGENT')) for preload_config in preload_configs: if preload_config.get("path") or preload_config.get("scenarios"): self._preload_path(preload_config)
def main(): l1 = InventoryListener() m = SimpleManager() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): # producer --> transform --> consumer print 'adding components for agent: ' + agent count += 1 ext = str(count) instrument1_dataproduct_name = 'test-stream1-' + ext instrument2_dataproduct_name = 'test-stream2-' + ext transform_dataproduct_name = 'test-stream3-' + ext producer1_component_name = 'pro1-'+ext producer2_component_name = 'pro2-'+ext transform_component_name = 'tran-'+ext print 'adding producer' producer1_config = InstrumentConfiguration(instrument1_dataproduct_name, 2, instrument_configuration=1, persist_product=False, report_timing=True, timing_rate=5000) producer1 = InstrumentSimulator(producer1_component_name, None, producer1_config) m.send_request(AddComponent(producer1), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) producer2_config = InstrumentConfiguration(instrument2_dataproduct_name, 2, instrument_configuration=5, persist_product=False, report_timing=True, timing_rate=5000) producer2 = InstrumentSimulator(producer2_component_name, None, producer2_config) m.send_request(AddComponent(producer2), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding transform' transform_config = TransformConfiguration(transform_function=sum_values) #(lambda input: dict(value=sum(input['value'])))) transform_config.add_input(instrument1_dataproduct_name) transform_config.add_input(instrument2_dataproduct_name) transform_config.add_output(transform_dataproduct_name) transform_config.add_output_field('value') transform_config.stats_interval = 10 transform_config.report_stats = True transform = TransformComponent(transform_component_name, None, transform_config) # print 'outputs: ' + repr(transform.configuration.outputs) m.send_request(AddComponent(transform), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding consumer' consumer_component_name = 'con-'+ext consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(transform_dataproduct_name, log_value=True)) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'starting data flow' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(transform_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer1_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer2_component_name)) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for a while wait_time=90 end_time = time.time()+wait_time wait_interval=5 while time.time()<end_time: print 'messages per sec: %f'%l2.get_rate() sleep(wait_interval) # then stop traffic m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) print 'data producer stopped' sleep(5)
def main(): if len(argv): update_count=int(argv[1]) nodes=int(argv[2]) print 'parsed arguments -- update count: ' + str(update_count) + ' expected nodes: ' + str(nodes) else: raise Exception('missing arguments') print 'defining a launch plan' t = ScriptedTroop(clobber=True) t.configure('resources/multiple-containers-ec2.trp') t.create_launch_plan() print 'created a launch plan with %d containers' % t.get_container_count() print '-------\nstarting launch (this will take a while)' # t.start_nodes() print 'launch completed!\n-------' m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) while nodes and len(l1.inventory)<nodes: print 'only have %d nodes so far, waiting for more...'%len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) # start component on each agent and let it add documents to db components = [] initial_config = PotatoConfiguration() initial_config.bulk_count = 100 initial_config.bulk_frequency = 1 initial_config.threads=12 initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_document_count = 0 initial_config.create_count = int(initial_document_count/(nodes*initial_config.bulk_count)) initial_config.id_salt = None # initial_config.create_count = 5 agent_list = [id for id in l1.inventory.keys()] salt = {} count = 0 for agent in agent_list: print 'adding couch potato for agent: ' + agent component_name = 'chip-'+agent components.append(component_name) # give each agent unique salt for id generation salt[agent] = _CHARSET[count] count+=1 # initial_config.id_salt = ['-','-',salt[agent]] component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition if initial_document_count>0: m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name)) if initial_document_count>0: start_time = time() print 'waiting for containers to finish creating initial documents in db' while len(l2.latest_data)<len(components): sleep(1) elapsed = time() - start_time print 'initialization ops/sec: %f create, %f read, %f update, %f delete, %d nodes' % l2.get_rates() print 'created %d docs in %f secs' % (initial_document_count, elapsed) # initialize DB only -- do not perform load test if update_count < 0: exit() cycle_config = PotatoConfiguration() cycle_config.threads = 2 cycle_config.read_count = 100 cycle_config.create_count = update_count cycle_config.update_count = update_count print 'starting db operations -- initializing first' for agent in agent_list: print 'updating configuration of agent ' + agent cycle_config.id_salt = ['-','-',salt[agent]] m.send_request(ChangeConfiguration(cycle_config), agent_filter=agent_id(agent), component_filter=component_type(Potato)) m.send_request(StartRequest(), component_filter=component_type(Potato)) l2.latest_data.clear() wait_active(l2, components) print 'agents have all ready initial document list -- now performing load test' # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(Potato)) sleep(5)
def main(): if len(argv): bytes=int(argv[1]) nodes=int(argv[2]) print 'parsed arguments -- sending messages size: ' + str(bytes) + ' expecting nodes: ' + str(nodes) else: bytes=1 nodes=None print 'defining a launch plan' t = ScriptedTroop(clobber=True) t.configure('resources/multiple-containers-ec2.trp') t.create_launch_plan() print 'created a launch plan with %d containers' % t.get_container_count() print '-------\nstarting launch (this will take a while)' # t.start_nodes() print 'launch completed!\n-------' m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) while nodes and len(l1.inventory)<nodes: print 'only have %d nodes so far, waiting for more...'%len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) pubsub_same_host = False count = 0 producer_names = [] previous_consumer = None agents = sorted(l1.inventory.keys()) for agent in agents: print 'adding producer/consumer for agent: ' + agent count += 1 ext = str(count) data_product_name = 'test-stream-' + ext producer_component_name = 'pro-'+ext consumer_component_name = 'con-'+ext producer_names.append((agent,producer_component_name)) producer_config = InstrumentConfiguration(data_product_name, 0, instrument_configuration=bytes, sleep_even_zero=False, persist_product=False, report_timing=True, timing_rate=1000) producer = InstrumentSimulator(producer_component_name, None, producer_config) consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False)) m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) wait('wait for producer to register before starting consumer (press ENTER): ') if pubsub_same_host: m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) else: ### if agents are A, B, C, ... # put producer on A and consumer on B; then producer on B and consumer on C; etc if previous_consumer: m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(previous_component_name)) previous_consumer = consumer previous_component_name = consumer_component_name if not pubsub_same_host: first_agent = agents[0] m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(first_agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(first_agent), component_filter=component_id(previous_component_name)) for agent,component in producer_names: m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(component)) # log results as they arrive for 5 min then stop traffic print 'producers/consumers running for 5 min' sleep(300) print 'shutting down' m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) print 'stopped producers' sleep(5) m.send_request(StopRequest(), component_filter=component_type(DataProductConsumer)) print 'stopped consumers' sleep(5) m.close() print 'closed manager'
def main(): if len(argv): update_count = int(argv[1]) nodes = int(argv[2]) print "parsed arguments -- update count: " + str(update_count) + " expected nodes: " + str(nodes) else: raise Exception("missing arguments") print "defining a launch plan" t = ScriptedTroop(clobber=True) t.configure("resources/multiple-containers-ec2.trp") t.create_launch_plan() print "created a launch plan with %d containers" % t.get_container_count() print "-------\nstarting launch (this will take a while)" # t.start_nodes() print "launch completed!\n-------" m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id("AGENT")) sleep(5) while nodes and len(l1.inventory) < nodes: print "only have %d nodes so far, waiting for more..." % len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id("AGENT")) sleep(5) # start component on each agent and let it add documents to db total_documents_target = 1000000 documents_per_iteration = 20000 components = [] initial_config = PotatoConfiguration() initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_config.create_count = int(documents_per_iteration / nodes) agent_list = [id for id in l1.inventory.keys()] salt = {} count = 0 for agent in agent_list: print "adding couch potato for agent: " + agent component_name = "chip-" + agent components.append(component_name) # give each agent unique salt for id generation salt[agent] = _CHARSET[count] count += 1 initial_config.id_salt = ["-", "-", salt[agent]] component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id("AGENT")) sleep( 2 ) # need at least a little time to let first component register name or second may fail due to race condition total_start = time() for x in xrange(total_documents_target / documents_per_iteration): l2.latest_data.clear() m.send_request(PerformOneCycle(), component_filter=component_type(Potato)) iteration_start = time() print "waiting for containers to finish creating initial documents in db" while len(l2.latest_data) < len(components): sleep(5) elapsed = time() - iteration_start print "created %d docs in %f secs: %f ops/sec" % ( documents_per_iteration, elapsed, documents_per_iteration / elapsed, ) elapsed = time() - total_start print "DONE: created %d docs in %f secs" % (total_documents_target, elapsed)