def main(): print 'defining a launch plan' t = Troop(clobber=True) t.configure('resources/three-containers.trp') t.create_launch_plan() print 'created a launch plan with %d containers' % t.get_container_count() print 'now starting nodes\n\n-----------------------------------------------' try: t.start_nodes() broker_config = t.get_nodes_broker() # broker_config = { 'broker_hostname': 'ec2-50-17-108-226.compute-1.amazonaws.com', # 'broker_username': '******', # 'broker_password': '******' } m = SimpleManager(**broker_config) l = InventoryListener() m.add_listener(l) while True: # get inventory -- see what agents we have running m.send_request(InventoryRequest()) print '-----------------------------------------------\n\nrequested inventory -- waiting for reply messages' sleep(5) print 'inventory after nodes have started: ' show_inventory(l.inventory) wait(5) finally: print 'now stopping nodes'
def main(): p = PingRequest() m = SimpleManager(broker_hostname='localhost', broker_username='******', broker_password='******') print 'sending 100 ping requests to all components on all agents, one every 5 seconds' for n in xrange(100): m.send_request(p) print 'sent a ping' sleep(5) print 'sending complete' wait(2)
def main(): m = SimpleManager() print 'performance with no sandbags' sleep(20) print 'starting 1gb memory usage, 10% cpu' config = SandbagConfiguration(ram_used_bytes=1000000000, cpu_used_portion=.1) m.send_request(AddComponent(ResourceWaster('sandbag', None, config))) sleep(30) print 'starting 100k memory usage, 70% cpu' config = SandbagConfiguration(ram_used_bytes=1000000, cpu_used_portion=.7) m.send_request(ChangeConfiguration(config), component_filter=component_id('sandbag')) sleep(30)
def main(): m = SimpleManager() l = PreloadListener() m.add_listener(l) _step('have manger') # first preload a scenario from the normal google doc # doc = PathConfiguration(path=TESTED_DOC, scenarios=["BASE,BETA"]) req = PerformPreload("goog", doc) m.send_request(AddComponent(Preloader('loader', None, None)), component_filter=component_id('AGENT')) m.send_request(req, component_filter=component_id('loader')) _step('started doc preload') result = l.wait_for_result('goog') _step('preload finished. success? %r' % result.success) # now use a preload template: # create 10 users where each field is either a value string # or a template expression with %d substituted with values from 1 to 10 # rows = { 'row_type': 'User', 'ID': 'TEMPLATE_USER_%d', 'subject': 'TBD', 'name': 'Full Name %d', 'description': 'Very %d descriptive', 'contact_id': '', 'role_ids': '' } template = TemplateConfiguration(range=xrange(1,10), templates=[rows]) req = PerformPreload("rowz", template) m.send_request(req, component_filter=component_id('loader')) _step('started template preload') result = l.wait_for_result('rowz') _step('preload finished. success? %r' % result.success)
def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) # start component on each agent and let it add documents to db components = [] initial_config = PotatoConfiguration() initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0 initial_config.create_count = 1000 initial_config.threads = 1 for agent in l1.inventory.keys(): print 'adding couch potato for agent: ' + agent component_name = 'chip-'+agent components.append(component_name) component = Potato(component_name, None, initial_config) m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name)) # while len(l2.latest_data)<len(components): # sleep(1) print 'initialization ops/sec: %f create, %f read, %f update, %f delete' % l2.get_rates() cycle_config = PotatoConfiguration() cycle_config.threads=1 for component_name in components: print 'starting db operations on: ' + component_name m.send_request(ChangeConfiguration(cycle_config), component_filter=component_id(component_name)) m.send_request(StartRequest(), component_filter=component_id(component_name)) # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(Potato)) sleep(5)
def main(): l1 = InventoryListener() m = SimpleManager() l2 = PerformanceListener() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): # producer --> transform --> consumer print 'adding components for agent: ' + agent count += 1 ext = str(count) instrument1_dataproduct_name = 'test-stream1-' + ext instrument2_dataproduct_name = 'test-stream2-' + ext transform_dataproduct_name = 'test-stream3-' + ext producer1_component_name = 'pro1-'+ext producer2_component_name = 'pro2-'+ext transform_component_name = 'tran-'+ext print 'adding producer' producer1_config = InstrumentConfiguration(instrument1_dataproduct_name, 2, instrument_configuration=1, persist_product=False, report_timing=True, timing_rate=5000) producer1 = InstrumentSimulator(producer1_component_name, None, producer1_config) m.send_request(AddComponent(producer1), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) producer2_config = InstrumentConfiguration(instrument2_dataproduct_name, 2, instrument_configuration=5, persist_product=False, report_timing=True, timing_rate=5000) producer2 = InstrumentSimulator(producer2_component_name, None, producer2_config) m.send_request(AddComponent(producer2), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding transform' transform_config = TransformConfiguration(transform_function=sum_values) #(lambda input: dict(value=sum(input['value'])))) transform_config.add_input(instrument1_dataproduct_name) transform_config.add_input(instrument2_dataproduct_name) transform_config.add_output(transform_dataproduct_name) transform_config.add_output_field('value') transform_config.stats_interval = 10 transform_config.report_stats = True transform = TransformComponent(transform_component_name, None, transform_config) # print 'outputs: ' + repr(transform.configuration.outputs) m.send_request(AddComponent(transform), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'adding consumer' consumer_component_name = 'con-'+ext consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(transform_dataproduct_name, log_value=True)) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) print 'starting data flow' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(transform_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer1_component_name)) m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer2_component_name)) sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for a while wait_time=90 end_time = time.time()+wait_time wait_interval=5 while time.time()<end_time: print 'messages per sec: %f'%l2.get_rate() sleep(wait_interval) # then stop traffic m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) print 'data producer stopped' sleep(5)
def main(): WASTE_RAM_PERCENT=.00001 WASTE_CPU_PERCENT=.00001 QUICK_LOAD_DEVICES=12 ans = AnswerListener() perf = PerformanceListener() # m = SimpleManager() m = SimpleManager(broker_hostname='localhost') m.add_listener(ans) m.add_listener(perf) # waste_ram_bytes=int(512*1024*1024*WASTE_RAM_PERCENT/100) # gumstix with 512mb # config = SandbagConfiguration(ram_used_bytes=waste_ram_bytes, cpu_used_portion=WASTE_CPU_PERCENT/100, cycle_size=10000) # m.send_request(AddComponent(ResourceWaster('sandbag', None, config))) # creating instrument controller ims = InstrumentController('device_controller', None, None) m.send_request(AddComponent(ims), component_filter=component_id('AGENT')) # create consumer of data products # transform_config = TransformConfiguration(transform_function=None) #(lambda input: dict(value=sum(input['value'])))) # transform_config.stats_interval = 60 # transform_config.report_stats = True # transform = TransformComponent('consumer', None, transform_config) # get device IDs count=0 # for x in xrange(50): # for dev in DEVICE_INFO: # count+=1 # future = ans.expect_message() # m.send_request(GetInstrumentId(dev[0]), component_filter=component_id('device_controller')) # future.wait(timeout=30) # msg = future.get() # if msg.exception: # raise msg.exception # device_id = msg.result # dev.append(device_id) # transform_config.add_input(dev[1]) # transform_config.add_input(dev[2]) # print 'have id for device %s: %s' % (dev[0],device_id) # m.send_request(AddComponent(transform), component_filter=component_id('AGENT')) # # # start devices # count=0 future = ans.expect_message() m.send_request(GetInstrumentId("CTD Simulator end to end R2 testing"), component_filter=component_id('device_controller')) msg = future.get() if msg.exception: raise msg.exception id = msg.result print 'device id %s' % id m.send_request(StartDevice(id), component_filter=component_id('device_controller'))
def main(): l1 = InventoryListener() l2 = PerformanceListener() m = SimpleManager() m.add_listener(l1) m.add_listener(l2) # get inventory -- see what agents we have running m.send_request(InventoryRequest()) sleep(5) count = 0 for agent in l1.inventory.keys(): print 'adding producer/consumer for agent: ' + agent count += 1 ext = str(count) data_product_name = 'test-stream-' + ext producer_component_name = 'pro-'+ext consumer_component_name = 'con-'+ext producer_config = InstrumentConfiguration(data_product_name, 1, instrument_configuration=100, easy_registration=True, sleep_even_zero=False, persist_product=False, report_timing=True, timing_rate=5000) producer = InstrumentSimulator(producer_component_name, None, producer_config) consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False)) # consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration('jon.ion.xs.ioncore.xp.science_data', log_value=False)) print 'creating producer and consumer' m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT')) # print 'next: register producer (creates publisher)' # wait(2) # print 'registering producer' # m.send_request(RegisterWithContainer(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'next: start consumer process' wait(2) print 'starting consumer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name)) print 'next: starting producer' wait(2) print 'starting producer' m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name)) print 'now all components are running' sleep(2) # need at least a little time to let first component register name or second may fail due to race condition # log results as they arrive for 5 min then stop traffic sleep(300) m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator)) sleep(5)
def main(): l = InventoryListener() m = SimpleManager() m.add_listener(l) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) print '---> sent inventory request' sleep(2) print '---> inventory now:\n' show_inventory(l.inventory) wait(10) sim1 = InstrumentSimulator('ins1', None, InstrumentConfiguration('str1', 0.05, easy_registration=False)) r1 = AddComponent(sim1) m.send_request(r1, component_filter=component_id('AGENT')) print 'creating sim1' # sleep(5) sim2 = InstrumentSimulator('ins2', None, InstrumentConfiguration('str2', 0)) r2 = AddComponent(sim2) m.send_request(r2, component_filter=component_id('AGENT')) print 'creating sim2' print '---> created 2 instruments' sleep(5) m.send_request(InventoryRequest(), component_filter=component_id('AGENT')) print '---> sent inventory request' sleep(2) print '---> inventory now:\n' show_inventory(l.inventory) wait(10) show_inventory(l.inventory) wait(10)