def main():
    if len(argv):
        bytes=int(argv[1])
        nodes=int(argv[2])
        print 'parsed arguments -- sending messages size: ' + str(bytes) + ' expecting nodes: ' + str(nodes)
    else:
        bytes=1
        nodes=None

    print 'defining a launch plan'
    t = ScriptedTroop(clobber=True)
    t.configure('resources/multiple-containers-ec2.trp')
    t.create_launch_plan()
    print 'created a launch plan with %d containers' % t.get_container_count()

    print '-------\nstarting launch (this will take a while)'
#    t.start_nodes()
    print 'launch completed!\n-------'

    m = t.get_manager()

    l1 = InventoryListener()
    l2 = PerformanceListener()
    m.add_listener(l1)
    m.add_listener(l2)

    # get inventory -- see what agents we have running
    m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
    sleep(5)
    while nodes and len(l1.inventory)<nodes:
        print 'only have %d nodes so far, waiting for more...'%len(l1.inventory)
        m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
        sleep(5)

    pubsub_same_host = False
    count = 0
    producer_names = []
    previous_consumer = None
    agents = sorted(l1.inventory.keys())
    for agent in agents:
        print 'adding producer/consumer for agent: ' + agent
        count += 1
        ext = str(count)
        data_product_name = 'test-stream-' + ext
        producer_component_name = 'pro-'+ext
        consumer_component_name = 'con-'+ext
        producer_names.append((agent,producer_component_name))

        producer_config = InstrumentConfiguration(data_product_name, 0, instrument_configuration=bytes,
            sleep_even_zero=False,
            persist_product=False, report_timing=True, timing_rate=1000)
        producer = InstrumentSimulator(producer_component_name, None, producer_config)
        consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False))

        m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
        wait('wait for producer to register before starting consumer (press ENTER): ')

        if pubsub_same_host:
            m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
            m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name))
        else:
            ### if agents are A, B, C, ...
            # put producer on A and consumer on B; then producer on B and consumer on C; etc
            if previous_consumer:
                m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
                m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(previous_component_name))
            previous_consumer = consumer
            previous_component_name = consumer_component_name

    if not pubsub_same_host:
        first_agent = agents[0]
        m.send_request(AddComponent(previous_consumer), agent_filter=agent_id(first_agent), component_filter=component_id('AGENT'))
        m.send_request(StartRequest(), agent_filter=agent_id(first_agent), component_filter=component_id(previous_component_name))


    for agent,component in producer_names:
        m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(component))

    # log results as they arrive for 5 min then stop traffic
    print 'producers/consumers running for 5 min'
    sleep(300)

    print 'shutting down'
    m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator))
    print 'stopped producers'
    sleep(5)
    m.send_request(StopRequest(), component_filter=component_type(DataProductConsumer))
    print 'stopped consumers'
    sleep(5)
    m.close()
    print 'closed manager'
def main():
    if len(argv):
        update_count=int(argv[1])
        nodes=int(argv[2])
        print 'parsed arguments -- update count: ' + str(update_count) + ' expected nodes: ' + str(nodes)
    else:
        raise Exception('missing arguments')

    print 'defining a launch plan'
    t = ScriptedTroop(clobber=True)
    t.configure('resources/multiple-containers-ec2.trp')
    t.create_launch_plan()
    print 'created a launch plan with %d containers' % t.get_container_count()

    print '-------\nstarting launch (this will take a while)'
    #    t.start_nodes()
    print 'launch completed!\n-------'

    m = t.get_manager()

    l1 = InventoryListener()
    l2 = PerformanceListener()
    m.add_listener(l1)
    m.add_listener(l2)

    # get inventory -- see what agents we have running
    m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
    sleep(5)
    while nodes and len(l1.inventory)<nodes:
        print 'only have %d nodes so far, waiting for more...'%len(l1.inventory)
        m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
        sleep(5)

    # start component on each agent and let it add documents to db
    components = []
    initial_config = PotatoConfiguration()
    initial_config.bulk_count = 100
    initial_config.bulk_frequency = 1
    initial_config.threads=12
    initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0
    initial_document_count = 0
    initial_config.create_count = int(initial_document_count/(nodes*initial_config.bulk_count))
    initial_config.id_salt = None

#    initial_config.create_count = 5

    agent_list = [id for id in l1.inventory.keys()]
    salt = {}
    count = 0
    for agent in agent_list:
        print 'adding couch potato for agent: ' + agent
        component_name = 'chip-'+agent
        components.append(component_name)

        # give each agent unique salt for id generation
        salt[agent] = _CHARSET[count]
        count+=1
#        initial_config.id_salt = ['-','-',salt[agent]]

        component = Potato(component_name, None, initial_config)
        m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
        sleep(2) # need at least a little time to let first component register name or second may fail due to race condition
        if initial_document_count>0:
            m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name))

    if initial_document_count>0:
        start_time = time()
        print 'waiting for containers to finish creating initial documents in db'
        while len(l2.latest_data)<len(components):
            sleep(1)
        elapsed = time() - start_time
        print 'initialization ops/sec: %f create, %f read, %f update, %f delete, %d nodes' % l2.get_rates()
        print 'created %d docs in %f secs' % (initial_document_count, elapsed)

    # initialize DB only -- do not perform load test
    if update_count < 0:
        exit()

    cycle_config = PotatoConfiguration()
    cycle_config.threads = 2
    cycle_config.read_count = 100
    cycle_config.create_count = update_count
    cycle_config.update_count = update_count

    print 'starting db operations -- initializing first'
    for agent in agent_list:
        print 'updating configuration of agent ' + agent
        cycle_config.id_salt = ['-','-',salt[agent]]
        m.send_request(ChangeConfiguration(cycle_config), agent_filter=agent_id(agent), component_filter=component_type(Potato))
    m.send_request(StartRequest(), component_filter=component_type(Potato))

    l2.latest_data.clear()
    wait_active(l2, components)
    print 'agents have all ready initial document list -- now performing load test'

    # log results as they arrive for 5 min then stop traffic
    sleep(300)
    m.send_request(StopRequest(), component_filter=component_type(Potato))
    sleep(5)
Exemple #3
0
def main():
    if len(argv):
        update_count = int(argv[1])
        nodes = int(argv[2])
        print "parsed arguments -- update count: " + str(update_count) + " expected nodes: " + str(nodes)
    else:
        raise Exception("missing arguments")

    print "defining a launch plan"
    t = ScriptedTroop(clobber=True)
    t.configure("resources/multiple-containers-ec2.trp")
    t.create_launch_plan()
    print "created a launch plan with %d containers" % t.get_container_count()

    print "-------\nstarting launch (this will take a while)"
    #    t.start_nodes()
    print "launch completed!\n-------"

    m = t.get_manager()

    l1 = InventoryListener()
    l2 = PerformanceListener()
    m.add_listener(l1)
    m.add_listener(l2)

    # get inventory -- see what agents we have running
    m.send_request(InventoryRequest(), component_filter=component_id("AGENT"))
    sleep(5)
    while nodes and len(l1.inventory) < nodes:
        print "only have %d nodes so far, waiting for more..." % len(l1.inventory)
        m.send_request(InventoryRequest(), component_filter=component_id("AGENT"))
        sleep(5)

    # start component on each agent and let it add documents to db
    total_documents_target = 1000000
    documents_per_iteration = 20000
    components = []
    initial_config = PotatoConfiguration()
    initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0
    initial_config.create_count = int(documents_per_iteration / nodes)
    agent_list = [id for id in l1.inventory.keys()]
    salt = {}
    count = 0
    for agent in agent_list:
        print "adding couch potato for agent: " + agent
        component_name = "chip-" + agent
        components.append(component_name)

        # give each agent unique salt for id generation
        salt[agent] = _CHARSET[count]
        count += 1
        initial_config.id_salt = ["-", "-", salt[agent]]

        component = Potato(component_name, None, initial_config)
        m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id("AGENT"))
        sleep(
            2
        )  # need at least a little time to let first component register name or second may fail due to race condition

    total_start = time()
    for x in xrange(total_documents_target / documents_per_iteration):
        l2.latest_data.clear()
        m.send_request(PerformOneCycle(), component_filter=component_type(Potato))
        iteration_start = time()
        print "waiting for containers to finish creating initial documents in db"
        while len(l2.latest_data) < len(components):
            sleep(5)
        elapsed = time() - iteration_start
        print "created %d docs in %f secs: %f ops/sec" % (
            documents_per_iteration,
            elapsed,
            documents_per_iteration / elapsed,
        )
    elapsed = time() - total_start
    print "DONE: created %d docs in %f secs" % (total_documents_target, elapsed)