def main():
print 'defining a launch plan'
t = Troop(clobber=True)
t.configure('resources/three-containers.trp')
t.create_launch_plan()
print 'created a launch plan with %d containers' % t.get_container_count()
print 'now starting nodes\n\n-----------------------------------------------'
try:
t.start_nodes()
broker_config = t.get_nodes_broker()
# broker_config = { 'broker_hostname': 'ec2-50-17-108-226.compute-1.amazonaws.com',
# 'broker_username': '******',
# 'broker_password': '******' }
m = SimpleManager(**broker_config)
l = InventoryListener()
m.add_listener(l)
while True:
# get inventory -- see what agents we have running
m.send_request(InventoryRequest())
print '-----------------------------------------------\n\nrequested inventory -- waiting for reply messages'
sleep(5)
print 'inventory after nodes have started: '
show_inventory(l.inventory)
wait(5)
finally:
print 'now stopping nodes'
def main():
p = PingRequest()
m = SimpleManager(broker_hostname='localhost', broker_username='******', broker_password='******')
print 'sending 100 ping requests to all components on all agents, one every 5 seconds'
for n in xrange(100):
m.send_request(p)
print 'sent a ping'
sleep(5)
print 'sending complete'
wait(2)
def main():
m = SimpleManager()
print 'performance with no sandbags'
sleep(20)
print 'starting 1gb memory usage, 10% cpu'
config = SandbagConfiguration(ram_used_bytes=1000000000, cpu_used_portion=.1)
m.send_request(AddComponent(ResourceWaster('sandbag', None, config)))
sleep(30)
print 'starting 100k memory usage, 70% cpu'
config = SandbagConfiguration(ram_used_bytes=1000000, cpu_used_portion=.7)
m.send_request(ChangeConfiguration(config), component_filter=component_id('sandbag'))
sleep(30)
def main():
m = SimpleManager()
l = PreloadListener()
m.add_listener(l)
_step('have manger')
# first preload a scenario from the normal google doc
#
doc = PathConfiguration(path=TESTED_DOC, scenarios=["BASE,BETA"])
req = PerformPreload("goog", doc)
m.send_request(AddComponent(Preloader('loader', None, None)), component_filter=component_id('AGENT'))
m.send_request(req, component_filter=component_id('loader'))
_step('started doc preload')
result = l.wait_for_result('goog')
_step('preload finished. success? %r' % result.success)
# now use a preload template:
# create 10 users where each field is either a value string
# or a template expression with %d substituted with values from 1 to 10
#
rows = { 'row_type': 'User',
'ID': 'TEMPLATE_USER_%d',
'subject': 'TBD',
'name': 'Full Name %d',
'description': 'Very %d descriptive',
'contact_id': '',
'role_ids': ''
}
template = TemplateConfiguration(range=xrange(1,10), templates=[rows])
req = PerformPreload("rowz", template)
m.send_request(req, component_filter=component_id('loader'))
_step('started template preload')
result = l.wait_for_result('rowz')
_step('preload finished. success? %r' % result.success)
def main():
l1 = InventoryListener()
l2 = PerformanceListener()
m = SimpleManager()
m.add_listener(l1)
m.add_listener(l2)
# get inventory -- see what agents we have running
m.send_request(InventoryRequest())
sleep(5)
# start component on each agent and let it add documents to db
components = []
initial_config = PotatoConfiguration()
initial_config.read_count = initial_config.update_count = initial_config.delete_count = 0
initial_config.create_count = 1000
initial_config.threads = 1
for agent in l1.inventory.keys():
print 'adding couch potato for agent: ' + agent
component_name = 'chip-'+agent
components.append(component_name)
component = Potato(component_name, None, initial_config)
m.send_request(AddComponent(component), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
sleep(2) # need at least a little time to let first component register name or second may fail due to race condition
m.send_request(PerformOneIteration(), agent_filter=agent_id(agent), component_filter=component_id(component_name))
# while len(l2.latest_data)<len(components):
# sleep(1)
print 'initialization ops/sec: %f create, %f read, %f update, %f delete' % l2.get_rates()
cycle_config = PotatoConfiguration()
cycle_config.threads=1
for component_name in components:
print 'starting db operations on: ' + component_name
m.send_request(ChangeConfiguration(cycle_config), component_filter=component_id(component_name))
m.send_request(StartRequest(), component_filter=component_id(component_name))
# log results as they arrive for 5 min then stop traffic
sleep(300)
m.send_request(StopRequest(), component_filter=component_type(Potato))
sleep(5)
def main():
l1 = InventoryListener()
m = SimpleManager()
l2 = PerformanceListener()
m.add_listener(l1)
m.add_listener(l2)
# get inventory -- see what agents we have running
m.send_request(InventoryRequest())
sleep(5)
count = 0
for agent in l1.inventory.keys():
# producer --> transform --> consumer
print 'adding components for agent: ' + agent
count += 1
ext = str(count)
instrument1_dataproduct_name = 'test-stream1-' + ext
instrument2_dataproduct_name = 'test-stream2-' + ext
transform_dataproduct_name = 'test-stream3-' + ext
producer1_component_name = 'pro1-'+ext
producer2_component_name = 'pro2-'+ext
transform_component_name = 'tran-'+ext
print 'adding producer'
producer1_config = InstrumentConfiguration(instrument1_dataproduct_name, 2, instrument_configuration=1,
persist_product=False, report_timing=True, timing_rate=5000)
producer1 = InstrumentSimulator(producer1_component_name, None, producer1_config)
m.send_request(AddComponent(producer1), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
producer2_config = InstrumentConfiguration(instrument2_dataproduct_name, 2, instrument_configuration=5,
persist_product=False, report_timing=True, timing_rate=5000)
producer2 = InstrumentSimulator(producer2_component_name, None, producer2_config)
m.send_request(AddComponent(producer2), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
print 'adding transform'
transform_config = TransformConfiguration(transform_function=sum_values) #(lambda input: dict(value=sum(input['value']))))
transform_config.add_input(instrument1_dataproduct_name)
transform_config.add_input(instrument2_dataproduct_name)
transform_config.add_output(transform_dataproduct_name)
transform_config.add_output_field('value')
transform_config.stats_interval = 10
transform_config.report_stats = True
transform = TransformComponent(transform_component_name, None, transform_config)
# print 'outputs: ' + repr(transform.configuration.outputs)
m.send_request(AddComponent(transform), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
print 'adding consumer'
consumer_component_name = 'con-'+ext
consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(transform_dataproduct_name, log_value=True))
m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
print 'starting data flow'
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name))
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(transform_component_name))
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer1_component_name))
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer2_component_name))
sleep(2) # need at least a little time to let first component register name or second may fail due to race condition
# log results as they arrive for a while
wait_time=90
end_time = time.time()+wait_time
wait_interval=5
while time.time()<end_time:
print 'messages per sec: %f'%l2.get_rate()
sleep(wait_interval)
# then stop traffic
m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator))
print 'data producer stopped'
sleep(5)
def main():
WASTE_RAM_PERCENT=.00001
WASTE_CPU_PERCENT=.00001
QUICK_LOAD_DEVICES=12
ans = AnswerListener()
perf = PerformanceListener()
# m = SimpleManager()
m = SimpleManager(broker_hostname='localhost')
m.add_listener(ans)
m.add_listener(perf)
# waste_ram_bytes=int(512*1024*1024*WASTE_RAM_PERCENT/100) # gumstix with 512mb
# config = SandbagConfiguration(ram_used_bytes=waste_ram_bytes, cpu_used_portion=WASTE_CPU_PERCENT/100, cycle_size=10000)
# m.send_request(AddComponent(ResourceWaster('sandbag', None, config)))
# creating instrument controller
ims = InstrumentController('device_controller', None, None)
m.send_request(AddComponent(ims), component_filter=component_id('AGENT'))
# create consumer of data products
# transform_config = TransformConfiguration(transform_function=None) #(lambda input: dict(value=sum(input['value']))))
# transform_config.stats_interval = 60
# transform_config.report_stats = True
# transform = TransformComponent('consumer', None, transform_config)
# get device IDs
count=0
# for x in xrange(50):
# for dev in DEVICE_INFO:
# count+=1
# future = ans.expect_message()
# m.send_request(GetInstrumentId(dev[0]), component_filter=component_id('device_controller'))
# future.wait(timeout=30)
# msg = future.get()
# if msg.exception:
# raise msg.exception
# device_id = msg.result
# dev.append(device_id)
# transform_config.add_input(dev[1])
# transform_config.add_input(dev[2])
# print 'have id for device %s: %s' % (dev[0],device_id)
# m.send_request(AddComponent(transform), component_filter=component_id('AGENT'))
#
# # start devices
# count=0
future = ans.expect_message()
m.send_request(GetInstrumentId("CTD Simulator end to end R2 testing"), component_filter=component_id('device_controller'))
msg = future.get()
if msg.exception:
raise msg.exception
id = msg.result
print 'device id %s' % id
m.send_request(StartDevice(id), component_filter=component_id('device_controller'))
def main():
l1 = InventoryListener()
l2 = PerformanceListener()
m = SimpleManager()
m.add_listener(l1)
m.add_listener(l2)
# get inventory -- see what agents we have running
m.send_request(InventoryRequest())
sleep(5)
count = 0
for agent in l1.inventory.keys():
print 'adding producer/consumer for agent: ' + agent
count += 1
ext = str(count)
data_product_name = 'test-stream-' + ext
producer_component_name = 'pro-'+ext
consumer_component_name = 'con-'+ext
producer_config = InstrumentConfiguration(data_product_name, 1, instrument_configuration=100, easy_registration=True,
sleep_even_zero=False,
persist_product=False, report_timing=True, timing_rate=5000)
producer = InstrumentSimulator(producer_component_name, None, producer_config)
consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration(data_product_name, log_value=False))
# consumer = DataProductConsumer(consumer_component_name, None, ConsumerConfiguration('jon.ion.xs.ioncore.xp.science_data', log_value=False))
print 'creating producer and consumer'
m.send_request(AddComponent(producer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
m.send_request(AddComponent(consumer), agent_filter=agent_id(agent), component_filter=component_id('AGENT'))
# print 'next: register producer (creates publisher)'
# wait(2)
# print 'registering producer'
# m.send_request(RegisterWithContainer(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name))
print 'next: start consumer process'
wait(2)
print 'starting consumer'
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(consumer_component_name))
print 'next: starting producer'
wait(2)
print 'starting producer'
m.send_request(StartRequest(), agent_filter=agent_id(agent), component_filter=component_id(producer_component_name))
print 'now all components are running'
sleep(2) # need at least a little time to let first component register name or second may fail due to race condition
# log results as they arrive for 5 min then stop traffic
sleep(300)
m.send_request(StopRequest(), component_filter=component_type(InstrumentSimulator))
sleep(5)
def main():
l = InventoryListener()
m = SimpleManager()
m.add_listener(l)
m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
print '---> sent inventory request'
sleep(2)
print '---> inventory now:\n'
show_inventory(l.inventory)
wait(10)
sim1 = InstrumentSimulator('ins1', None, InstrumentConfiguration('str1', 0.05, easy_registration=False))
r1 = AddComponent(sim1)
m.send_request(r1, component_filter=component_id('AGENT'))
print 'creating sim1'
# sleep(5)
sim2 = InstrumentSimulator('ins2', None, InstrumentConfiguration('str2', 0))
r2 = AddComponent(sim2)
m.send_request(r2, component_filter=component_id('AGENT'))
print 'creating sim2'
print '---> created 2 instruments'
sleep(5)
m.send_request(InventoryRequest(), component_filter=component_id('AGENT'))
print '---> sent inventory request'
sleep(2)
print '---> inventory now:\n'
show_inventory(l.inventory)
wait(10)
show_inventory(l.inventory)
wait(10)
