def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): print 'adding producer/consumer for agent: ' + agent count += 1 ext = str(count) data_product_name = 'test-stream-' + ext producer_component_name = 'pro-'+ext consumer_component_name = 'con-'+ext producer_config = InstrumentConfiguration(data_product_name, 1, instrument_configuration=100, easy_registration=True, sleep_even_zero=False, persist_product=False, report_timing=True, timing_rate=5000) producer = InstrumentSimulator(producer_component_name, None, producer_config) consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False)) # consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration('jon.ion.xs.ioncore.xp.science_data', log_value=False)) print 'creating producer and consumer' m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) # print 'next: register producer (creates publisher)' # wait(2) # print 'registering producer' # m.send_request(RegisterWithContainer(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'next: start consumer process' wait(2) print 'starting consumer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) print 'next: starting producer' wait(2) print 'starting producer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'now all components are running' sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) sleep(5)
def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) # start component on each agent and let it add documents to db components = [] initial_config = PotatoConfiguration() initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_config.create_count = 1000 initial_config.threads = 1 for agent in l1.inventory.keys(): print 'adding couch potato for agent: ' + agent component_name = 'chip-'+agent components.append(component_name) component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name)) # while len(l2.latest_data)<len(components): # sleep(1) print 'initialization ops/sec: %f create, %f read, %f update, %f delete' % l2.get_rates() cycle_config = PotatoConfiguration() cycle_config.threads=1 for component_name in components: print 'starting db operations on: ' + component_name m.send_request(ChangeConfiguration(cycle_config), component_filter=component_id(component_name)) m.send_request(StartRequest(), component_filter=component_id(component_name)) # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(Potato)) sleep(5)
def main(): l1 = InventoryListener() m = SimpleManager() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): # producer --> transform --> consumer print 'adding components for agent: ' + agent count += 1 ext = str(count) instrument1_dataproduct_name = 'test-stream1-' + ext instrument2_dataproduct_name = 'test-stream2-' + ext transform_dataproduct_name = 'test-stream3-' + ext producer1_component_name = 'pro1-'+ext producer2_component_name = 'pro2-'+ext transform_component_name = 'tran-'+ext print 'adding producer' producer1_config = InstrumentConfiguration(instrument1_dataproduct_name, 2, instrument_configuration=1, persist_product=False, report_timing=True, timing_rate=5000) producer1 = InstrumentSimulator(producer1_component_name, None, producer1_config) m.send_request(AddComponent(producer1), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) producer2_config = InstrumentConfiguration(instrument2_dataproduct_name, 2, instrument_configuration=5, persist_product=False, report_timing=True, timing_rate=5000) producer2 = InstrumentSimulator(producer2_component_name, None, producer2_config) m.send_request(AddComponent(producer2), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding transform' transform_config = TransformConfiguration(transform_function=sum_values) #(lambda input: dict(value=sum(input['value'])))) transform_config.add_input(instrument1_dataproduct_name) transform_config.add_input(instrument2_dataproduct_name) transform_config.add_output(transform_dataproduct_name) transform_config.add_output_field('value') transform_config.stats_interval = 10 transform_config.report_stats = True transform = TransformComponent(transform_component_name, None, transform_config) # print 'outputs: ' + repr(transform.configuration.outputs) m.send_request(AddComponent(transform), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding consumer' consumer_component_name = 'con-'+ext consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(transform_dataproduct_name, log_value=True)) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'starting data flow' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(transform_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer1_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer2_component_name)) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for a while wait_time=90 end_time = time.time()+wait_time wait_interval=5 while time.time()<end_time: print 'messages per sec: %f'%l2.get_rate() sleep(wait_interval) # then stop traffic m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) print 'data producer stopped' sleep(5)
def main(): if len(argv): update_count=int(argv[1]) nodes=int(argv[2]) print 'parsed arguments -- update count: ' + str(update_count) + ' expected nodes: ' + str(nodes) else: raise Exception('missing arguments') print 'defining a launch plan' t = ScriptedTroop(clobber=True) t.configure('resources/multiple-containers-ec2.trp') t.create_launch_plan() print 'created a launch plan with %d containers' % t.get_container_count() print '-------\nstarting launch (this will take a while)' # t.start_nodes() print 'launch completed!\n-------' m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) while nodes and len(l1.inventory)<nodes: print 'only have %d nodes so far, waiting for more...'%len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) # start component on each agent and let it add documents to db components = [] initial_config = PotatoConfiguration() initial_config.bulk_count = 100 initial_config.bulk_frequency = 1 initial_config.threads=12 initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_document_count = 0 initial_config.create_count = int(initial_document_count/(nodes*initial_config.bulk_count)) initial_config.id_salt = None # initial_config.create_count = 5 agent_list = [id for id in l1.inventory.keys()] salt = {} count = 0 for agent in agent_list: print 'adding couch potato for agent: ' + agent component_name = 'chip-'+agent components.append(component_name) # give each agent unique salt for id generation salt[agent] = _CHARSET[count] count+=1 # initial_config.id_salt = ['-','-',salt[agent]] component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition if initial_document_count>0: m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name)) if initial_document_count>0: start_time = time() print 'waiting for containers to finish creating initial documents in db' while len(l2.latest_data)<len(components): sleep(1) elapsed = time() - start_time print 'initialization ops/sec: %f create, %f read, %f update, %f delete, %d nodes' % l2.get_rates() print 'created %d docs in %f secs' % (initial_document_count, elapsed) # initialize DB only -- do not perform load test if update_count < 0: exit() cycle_config = PotatoConfiguration() cycle_config.threads = 2 cycle_config.read_count = 100 cycle_config.create_count = update_count cycle_config.update_count = update_count print 'starting db operations -- initializing first' for agent in agent_list: print 'updating configuration of agent ' + agent cycle_config.id_salt = ['-','-',salt[agent]] m.send_request(ChangeConfiguration(cycle_config), agent_filter=agent_id(agent), component_filter=component_type(Potato)) m.send_request(StartRequest(), component_filter=component_type(Potato)) l2.latest_data.clear() wait_active(l2, components) print 'agents have all ready initial document list -- now performing load test' # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(Potato)) sleep(5)
def main(): if len(argv): bytes=int(argv[1]) nodes=int(argv[2]) print 'parsed arguments -- sending messages size: ' + str(bytes) + ' expecting nodes: ' + str(nodes) else: bytes=1 nodes=None print 'defining a launch plan' t = ScriptedTroop(clobber=True) t.configure('resources/multiple-containers-ec2.trp') t.create_launch_plan() print 'created a launch plan with %d containers' % t.get_container_count() print '-------\nstarting launch (this will take a while)' # t.start_nodes() print 'launch completed!\n-------' m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) while nodes and len(l1.inventory)<nodes: print 'only have %d nodes so far, waiting for more...'%len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(5) pubsub_same_host = False count = 0 producer_names = [] previous_consumer = None agents = sorted(l1.inventory.keys()) for agent in agents: print 'adding producer/consumer for agent: ' + agent count += 1 ext = str(count) data_product_name = 'test-stream-' + ext producer_component_name = 'pro-'+ext consumer_component_name = 'con-'+ext producer_names.append((agent,producer_component_name)) producer_config = InstrumentConfiguration(data_product_name, 0, instrument_configuration=bytes, sleep_even_zero=False, persist_product=False, report_timing=True, timing_rate=1000) producer = InstrumentSimulator(producer_component_name, None, producer_config) consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False)) m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) wait('wait for producer to register before starting consumer (press ENTER): ') if pubsub_same_host: m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) else: ### if agents are A, B, C, ... # put producer on A and consumer on B; then producer on B and consumer on C; etc if previous_consumer: m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(previous_component_name)) previous_consumer = consumer previous_component_name = consumer_component_name if not pubsub_same_host: first_agent = agents[0] m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(first_agent), component_filter=component_id('AGENT')) m.send_request(StartRequest(), agent_filter=agent_id(first_agent), component_filter=component_id(previous_component_name)) for agent,component in producer_names: m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(component)) # log results as they arrive for 5 min then stop traffic print 'producers/consumers running for 5 min' sleep(300) print 'shutting down' m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) print 'stopped producers' sleep(5) m.send_request(StopRequest(), component_filter=component_type(DataProductConsumer)) print 'stopped consumers' sleep(5) m.close() print 'closed manager'
def main(): #log.setLevel(DEBUG) t = Troop(clobber=True) t.configure('resources/one-container-ec2.trp') t.set_name('load-test') print 'before:\nconfig: %s\ntypes: %s' % (repr(t.configuration), repr(t.node_types)) t.change_count('container-with-services', AGENT_CONTAINERS) print 'after:\nconfig: %s\ntypes: %s' % (repr(t.configuration), repr(t.node_types)) t.create_launch_plan() m = None try: log.info('now starting nodes\n\n-----------------------------------------------') # bracket STDOUT of cloudinitd t.start_nodes() log.info('\n-----------------------------------------------\n\n') i_listener = InventoryListener() p_listener = PerformanceListener() m = t.get_manager() m.add_listener(i_listener) m.add_listener(p_listener) # TODO: troop should start agent with configuration setting for name of container type, # so here we call can target only the non-service containers: # m.send_request(InventoryRequest(), agent_filter=container_type('container-without-services')) # without this ability, we run producer/consumer on container with services which has other overhead log.info('requested inventory -- waiting for reply messages') max_wait=9999999999 #sec for n in xrange(max_wait/30): m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) sleep(30) reply_count = len(i_listener.inventory) log.info('requested %d containers, have replies from %d containers' % (AGENT_CONTAINERS, reply_count)) if AGENT_CONTAINERS<reply_count: log.warn('have more containers than requested?') break elif AGENT_CONTAINERS==reply_count: break log.debug('found %d containers', len(i_listener.inventory)) if len(i_listener.inventory)<AGENT_CONTAINERS: raise ApeException('failed to start requested number of containers') # determine reasonable rate for nodes any_agent = i_listener.inventory.keys()[0] max_node_rate = find_node_max(m, p_listener, any_agent, 1000) log.debug('max rate %.2e', max_node_rate) if not max_node_rate: raise ApeException('failed to get max rate from first VM') target_node_rate = max_node_rate * TARGET_RATE_PORTION_OF_MAX log.debug('max rate for one node is %.2f msgs/sec, target rate is %.2f msgs/sec', max_node_rate, target_node_rate) # create producers/consumers on remaining agents index=2 max_broker_rate = max_node_rate max_index=1 for agent in i_listener.inventory.keys()[1:]: add_node_traffic(m, p_listener, agent, target_node_rate, 'agent%d'%index) # make sure have fresh measurement from each node # b/c node that was going faster may have slowed down # now that latest node has pushed broker to saturation sleep(10) p_listener.clear() for n in xrange(60/5): if len(p_listener.latest_data)>=index: break sleep(5) measured_broker_rate = p_listener.get_rate() log.debug('broker passing %.2e msgs/sec with %d producers', measured_broker_rate, index) if measured_broker_rate>max_broker_rate: max_broker_rate = measured_broker_rate max_index = index index+=1 log.info('max rate was %.2e msgs/sec with %d producers', max_broker_rate, max_index) finally: log.info('now stopping nodes') if m: m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) m.send_request(StopRequest(), component_filter=component_type(DataProductConsumer)) m.close()
def main(): if len(argv): update_count = int(argv[1]) nodes = int(argv[2]) print "parsed arguments -- update count: " + str(update_count) + " expected nodes: " + str(nodes) else: raise Exception("missing arguments") print "defining a launch plan" t = ScriptedTroop(clobber=True) t.configure("resources/multiple-containers-ec2.trp") t.create_launch_plan() print "created a launch plan with %d containers" % t.get_container_count() print "-------\nstarting launch (this will take a while)" # t.start_nodes() print "launch completed!\n-------" m = t.get_manager() l1 = InventoryListener() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest(), component_filter=component_id("AGENT")) sleep(5) while nodes and len(l1.inventory) < nodes: print "only have %d nodes so far, waiting for more..." % len(l1.inventory) m.send_request(InventoryRequest(), component_filter=component_id("AGENT")) sleep(5) # start component on each agent and let it add documents to db total_documents_target = 1000000 documents_per_iteration = 20000 components = [] initial_config = PotatoConfiguration() initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_config.create_count = int(documents_per_iteration / nodes) agent_list = [id for id in l1.inventory.keys()] salt = {} count = 0 for agent in agent_list: print "adding couch potato for agent: " + agent component_name = "chip-" + agent components.append(component_name) # give each agent unique salt for id generation salt[agent] = _CHARSET[count] count += 1 initial_config.id_salt = ["-", "-", salt[agent]] component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id("AGENT")) sleep( 2 ) # need at least a little time to let first component register name or second may fail due to race condition total_start = time() for x in xrange(total_documents_target / documents_per_iteration): l2.latest_data.clear() m.send_request(PerformOneCycle(), component_filter=component_type(Potato)) iteration_start = time() print "waiting for containers to finish creating initial documents in db" while len(l2.latest_data) < len(components): sleep(5) elapsed = time() - iteration_start print "created %d docs in %f secs: %f ops/sec" % ( documents_per_iteration, elapsed, documents_per_iteration / elapsed, ) elapsed = time() - total_start print "DONE: created %d docs in %f secs" % (total_documents_target, elapsed)