def _create_schema(self, fail):
fail.trap(sqlite3.OperationalError)
commands = [
text_helper.format_block("""
CREATE TABLE processed_requests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
method VARCHAR(10) NOT NULL,
viewname VARCHAR(1000) NOT NULL,
started DATETIME NOT NULL,
elapsed FLOAT NOT NULL
)
"""),
text_helper.format_block("""
CREATE TABLE waiting_times (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created DATETIME NOT NULL,
elapsed FLOAT NOT NULL
)
"""),
("CREATE INDEX processed_requestsx "
"ON processed_requests(method, viewname)"),
]
def run_all(connection, commands):
for command in commands:
connection.execute(command)
return self._db.runWithConnection(run_all, commands)
def testInclude(self):
tmpfile = tempfile.mktemp()
included = format_block("""
[manhole]
public_key: /etc/ssl/ssh/public_key
private_key: /etc/ssl/ssh/private_key
authorized_keys: /etc/ssl/ssh/authorized_keys
port: 6000
""")
self.addCleanup(os.remove, tmpfile)
with file(tmpfile, 'w') as f:
f.write(included)
test_config = format_block("""
[include]
to_include: %s""" % (tmpfile, ))
f = StringIO(test_config)
configfile.parse_file(self.parser, f)
v = self.parser.values
self.assertEqual('/etc/ssl/ssh/public_key', v.manhole_public_key)
self.assertEqual('/etc/ssl/ssh/private_key', v.manhole_private_key)
self.assertEqual('/etc/ssl/ssh/authorized_keys',
v.manhole_authorized_keys)
self.assertEqual(6000,
v.manhole_port)
def prolog(self):
hostdef = host.HostDef(categories=dict(address=Address.fixed),
ports_ranges=dict(dns=(8000, 8000)))
self.set_local('hostdef', hostdef)
setup = text_helper.format_block("""
a = spawn_agency()
a.disable_protocol('setup-monitoring', 'Task')
medium = a.start_agent(descriptor_factory('host_agent'),\
hostdef=hostdef)
host = medium.get_agent()
wait_for_idle()
""")
dns = text_helper.format_block("""
host.start_agent(descriptor_factory('dns_agent'))
""")
api = text_helper.format_block("""
host.start_agent(descriptor_factory('api_agent'))
wait_for_idle()
""")
yield self.process(setup)
yield self.process(dns)
yield self.process(api)
self.dns = first(self.driver.iter_agents('dns_agent')).get_agent()
self.api = first(self.driver.iter_agents('api_agent')).get_agent()
def prolog(self):
setup1 = text_helper.format_block(
"""
agency = spawn_agency()
host1 = agency.get_host_agent()
wait_for_idle()
agency = spawn_agency()
host1.start_agent(descriptor_factory('test_exportable_agent'))
wait_for_idle()
agency = spawn_agency()
agency = spawn_agency()
host2 = agency.get_host_agent()
host2.start_agent(descriptor_factory('export_agent'))
wait_for_idle()
"""
)
yield self.process(self.drivers[0], setup1)
setup2 = text_helper.format_block(
"""
agency = spawn_agency(hostdef=hostdef1)
host = agency.get_host_agent()
host.start_agent(descriptor_factory('alert_agent'))
host.start_agent(descriptor_factory('migration_agent'))
wait_for_idle()
"""
)
hd1 = host.HostDef()
hd1.resources["epu"] = self.epu_in_new_cluster
self.drivers[1].set_local("hostdef1", hd1)
hd2 = host.HostDef()
hd2.resources["epu"] = 500
self.drivers[1].set_local("hostdef2", hd2)
yield self.process(self.drivers[1], setup2)
self.export = first(self.drivers[0].iter_agents("export_agent")).get_agent()
self.migration = first(self.drivers[1].iter_agents("migration_agent")).get_agent()
self.host1 = self.drivers[0].get_local("host1")
self.host2 = self.drivers[0].get_local("host2")
self.alert = first(self.drivers[1].iter_agents("alert_agent")).get_agent()
recp = yield self.export.get_own_address()
self.assertIsInstance(recp, recipient.Recipient)
url = self.export.get_tunneling_url()
yield self.migration.add_tunneling_route(recp, url)
yield self.migration.handshake(recp)
yield self.migration.set_current("testing_site")
def prolog(self):
setup = format_block("""
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
host_desc = descriptor_factory('host_agent')
req_desc = descriptor_factory('requesting_agent_mod')
host_medium = agency.start_agent(host_desc, hostdef=hostdef, \
run_startup=False)
host_agent = host_medium.get_agent()
host_agent.start_agent(req_desc)
""")
hostdef = host.HostDef()
hostdef.resources = {"a": 5, "epu": 5}
self.set_local("hostdef", hostdef)
yield self.process(setup)
yield self.wait_for_idle(10)
self.host_agent = self.get_local('host_agent')
medium = yield self.driver.find_agent(self.get_local('req_desc'))
self.req_agent = medium.get_agent()
def __init__(self, window, builder, driver):
self.builder = builder
self.window = window
self.editor = self.builder.get_object('command_editor')
textbuffer = self.editor.get_buffer()
textbuffer.set_text(format_block("""
agency = spawn_agency()
host_desc = descriptor_factory('host_agent')
agency.start_agent(host_desc)
"""))
textbuffer.connect('changed', self.on_text_changed)
self.run_button = self.builder.get_object('run_button')
self.run_button.connect('clicked', self.on_run_clicked)
self.clear_button = self.builder.get_object('clear_button')
self.clear_button.connect('clicked', self.on_clear_clicked)
self.history_button = self.builder.get_object('history_button')
self.history_button.connect('clicked', self.on_history_clicked)
self.driver = driver
self.driver.on_processed_callback(self._on_script_processed)
self.history = history.HistoryLogger()
def prolog(self):
setup = format_block("""
agency = spawn_agency()
medium = agency.start_agent(descriptor_factory('task-agent'))
agent = medium.get_agent()
""")
return self.process(setup)
def _get_history_id(self, connection, agent_id, instance_id):
'''
Checks own cache for history_id for agent_id and instance_id.
If information is missing fetch it from database. If it is not there
create the new record.
BEWARE: This method runs in a thread.
'''
cache_key = (agent_id, instance_id, )
if cache_key in self._history_id_cache:
history_id = self._history_id_cache[cache_key]
return history_id
else:
command = text_helper.format_block("""
SELECT id FROM histories WHERE agent_id = ? AND instance_id = ?
""")
cursor = connection.cursor()
cursor.execute(command, (agent_id, instance_id, ))
res = cursor.fetchall()
if res:
history_id = res[0][0]
self._history_id_cache[cache_key] = history_id
return history_id
else:
command = 'INSERT INTO histories VALUES (NULL, ?, ?)'
cursor.execute(command, (agent_id, instance_id, ))
history_id = cursor.lastrowid
self._history_id_cache[cache_key] = history_id
return history_id
def testSubstitutePartner(self):
'''
Three agents, all being partners. Than check the termination of
two of them.
'''
yield self.process(format_block("""
third = agency.start_agent(descriptor_factory('partner-agent'))
"""))
agents = [self.initiator, self.receiver, self.get_local('third')]
alloc1 = yield agents[2].get_agent().allocate_resource(foo=1)
alloc2 = yield agents[2].get_agent().allocate_resource(foo=1)
yield self._establish_partnership('initiator', 'receiver')
self.assert_partners(agents, [1, 1, 0])
yield self.initiator.get_agent().substitute_partner(
recipient.IRecipient(self.receiver),
recipient.IRecipient(agents[2]),
alloc1.id)
self.assert_partners(agents, [1, 1, 1])
yield self.receiver.get_agent().substitute_partner(
recipient.IRecipient(self.initiator),
recipient.IRecipient(agents[2]),
alloc2.id)
self.assert_partners(agents, [1, 1, 2])
def _establish_partnership(self, initiator='initiator',
receiver='receiver'):
script = format_block("""
agent = %s.get_agent()
agent.propose_to(%s)
""" % (initiator, receiver, ))
return self.process(script)
def prolog(self):
setup = format_block("""
agency1 = spawn_agency()
agency1.disable_protocol('setup-monitoring', 'Task')
desc1 = descriptor_factory('host_agent')
medium1 = agency1.start_agent(desc1, hostdef=hostdef, \
run_startup=False)
agent1 = medium1.get_agent()
agency2 = spawn_agency()
agency2.disable_protocol('setup-monitoring', 'Task')
desc2 = descriptor_factory('host_agent')
medium2 = agency2.start_agent(desc2, hostdef=hostdef_id, \
run_startup=False)
agent2 = medium2.get_agent()
""")
hostdef = host.HostDef()
hostdef.doc_id = "someid"
hostdef.resources = {"spam": 999, "bacon": 42, "eggs": 3, "epu": 10}
self.driver.save_document(hostdef)
self.set_local("hostdef", hostdef)
self.set_local("hostdef_id", "someid")
return self.process(setup)
def prolog(self):
setup = format_block("""
test_desc = descriptor_factory('contract-running-agent')
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('host_agent', shard='s1'), \
run_startup=False)
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('host_agent', shard='s1'), \
run_startup=False)
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('host_agent', shard='s1'), \
run_startup=False)
agent = _.get_agent()
agent.wait_for_ready()
agent.start_agent(test_desc)
""")
yield self.process(setup)
medium = first(self.driver.iter_agents('contract-running-agent'))
self.agent = medium.get_agent()
def prolog(self):
script = format_block("""
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('shard_agent', shard=uuid()), \
run_startup=False)
agent1 = _.get_agent()
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('shard_agent', shard=uuid()), \
run_startup=False)
agent2 = _.get_agent()
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
agency.start_agent(descriptor_factory('shard_agent', shard=uuid()), \
run_startup=False)
agent3 = _.get_agent()
""")
yield self.process(script)
self.agent1, self.agent2, self.agent3 =\
self.get_local('agent1', 'agent2', 'agent3')
self.alloc = list()
for x in range(2):
alloc = yield self.agent3.allocate_resource(neighbours=1)
self.alloc.append(alloc.id)
def prolog(self):
setup = format_block(
"""
agency = spawn_agency(start_host=False)
cdesc1 = descriptor_factory('collector_test_agent')
cdesc2 = descriptor_factory('collector_test_agent')
pdesc1 = descriptor_factory('poster_test_agent')
pdesc2 = descriptor_factory('poster_test_agent')
pdesc3 = descriptor_factory('poster_test_agent')
cmedium1 = agency.start_agent(cdesc1)
cmedium2 = agency.start_agent(cdesc2)
pmedium1 = agency.start_agent(pdesc1, desc=cdesc1)
pmedium2 = agency.start_agent(pdesc2, desc=cdesc2)
pmedium3 = agency.start_agent(pdesc3, desc=broadcast)
collector1 = cmedium1.get_agent()
collector2 = cmedium2.get_agent()
poster1 = pmedium1.get_agent()
poster2 = pmedium2.get_agent()
poster3 = pmedium3.get_agent()
"""
)
recip = recipient.Broadcast("dummy-notification", "lobby")
self.set_local("broadcast", recip)
return self.process(setup)
def prolog(self):
setup = format_block("""
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
host_desc = descriptor_factory('host_agent')
test_desc = descriptor_factory('condition-agent')
error_desc = descriptor_factory('conditionerror-agent')
host_medium = agency.start_agent(host_desc, hostdef=hostdef, \
run_startup=False)
host_agent = host_medium.get_agent()
host_agent.start_agent(test_desc)
host_agent.start_agent(error_desc)
""")
hostdef = host.HostDef()
hostdef.doc_id = "someid"
hostdef.categories = {"access": Access.private,
"address": Address.fixed,
"storage": Storage.static}
hostdef.resources = {"epu": 10}
self.driver.save_document(hostdef)
self.set_local("hostdef", hostdef)
return self.process(setup)
def prolog(self):
setup = text_helper.format_block(
"""
agency = spawn_agency()
host1 = agency.get_host_agent()
agency = spawn_agency()
host1.start_agent(descriptor_factory('test_exportable_agent'))
wait_for_idle()
agency = spawn_agency()
agency = spawn_agency()
host2 = agency.get_host_agent()
host2.start_agent(descriptor_factory('export_agent'))
wait_for_idle()
host2.start_agent(descriptor_factory('migration_agent'))
wait_for_idle()
"""
)
yield self.process(setup)
self.export = first(self.driver.iter_agents("export_agent")).get_agent()
self.migration = first(self.driver.iter_agents("migration_agent")).get_agent()
self.assertEqual(1, self.count_agents("test_exportable_agent"))
self.assertEqual(2, self.count_agents("test_child_agent"))
self.host1 = self.get_local("host1")
self.host2 = self.get_local("host2")
def testThreeAgents(self):
'''
Three agents, all being partners. Than check the termination of
two of them.
'''
yield self.process(format_block("""
third = agency.start_agent(descriptor_factory('partner-agent'))
"""))
agents = [self.initiator, self.receiver, self.get_local('third')]
yield self._establish_partnership('initiator', 'receiver')
self.assert_partners(agents, [1, 1, 0])
yield self._establish_partnership('receiver', 'third')
self.assert_partners(agents, [1, 2, 1])
yield self._establish_partnership('third', 'initiator')
self.assert_partners(agents, [2, 2, 2])
yield self.initiator._terminate()
yield agents[1].wait_for_protocols_finish()
yield agents[2].wait_for_protocols_finish()
self.assert_partners(agents, [2, 1, 1])
yield self.receiver._terminate()
yield agents[2].wait_for_protocols_finish()
self.assert_partners(agents, [2, 1, 0])
def prolog(self):
setup = format_block("""
load('feat.test.integration.resource')
agency = spawn_agency(start_host=False)
host_desc = descriptor_factory('host_agent')
req_desc = descriptor_factory('requesting_agent')
host_medium = agency.start_agent(host_desc, hostdef=hostdef)
host_agent = host_medium.get_agent()
host_agent.wait_for_ready()
host_agent.start_agent(req_desc)
""")
hostdef = host.HostDef()
hostdef.resources = {"host": 1, "epu": 10}
hostdef.categories = {"access": Access.private,
"address": Address.dynamic,
"storage": Storage.static}
hostdef.ports_ranges = {"port": (1000, 1050)}
self.set_local("hostdef", hostdef)
yield self.process(setup)
yield self.wait_for_idle(10)
raage_medium = list(self.driver.iter_agents('raage_agent'))[0]
self.raage_agent = raage_medium.get_agent()
self.host_medium = self.get_local('host_medium')
self.host_agent = self.get_local('host_agent')
medium = yield self.driver.find_agent(self.get_local('req_desc'))
self.req_agent = medium.get_agent()
def prolog(self):
setup = format_block("""
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
cdesc1 = descriptor_factory('collector_test_agent')
cdesc2 = descriptor_factory('collector_test_agent')
pdesc1 = descriptor_factory('poster_test_agent')
pdesc2 = descriptor_factory('poster_test_agent')
pdesc3 = descriptor_factory('poster_test_agent')
cmedium1 = agency.start_agent(cdesc1)
cmedium2 = agency.start_agent(cdesc2)
pmedium1 = agency.start_agent(pdesc1, desc=cdesc1)
pmedium2 = agency.start_agent(pdesc2, desc=cdesc2)
pmedium3 = agency.start_agent(pdesc3, desc=broadcast)
collector1 = cmedium1.get_agent()
collector2 = cmedium2.get_agent()
poster1 = pmedium1.get_agent()
poster2 = pmedium2.get_agent()
poster3 = pmedium3.get_agent()
""")
recip = recipient.Broadcast('dummy-notification', 'lobby')
self.set_local("broadcast", recip)
return self.process(setup)
def prolog(self):
setup = format_block("""
load('feat.test.integration.resource')
req_desc = descriptor_factory('requesting_agent')
# First agency will eventually run Host, Shard, Raage and
# Requesting agent
agency = spawn_agency(hostdef=hostdef)
host = agency.get_host_agent()
wait_for_idle()
host.start_agent(req_desc)
# Second agency runs the host agent
agency = spawn_agency(hostdef=hostdef)
# Third is like second
agency = spawn_agency(hostdef=hostdef)
""")
hostdef = host.HostDef()
hostdef.resources = {"host": 1, "epu": 10}
hostdef.categories = {"access": Access.private,
"address": Address.dynamic,
"storage": Storage.static}
self.set_local("hostdef", hostdef)
yield self.process(setup)
yield self.wait_for_idle(20)
self.agents = [x.get_agent() \
for x in self.driver.iter_agents('host_agent')]
req_medium = list(self.driver.iter_agents('requesting_agent'))[0]
self.req_agent = req_medium.get_agent()
def _start_agent(self):
yield self.process(format_block("""
agency.start_agent(descriptor_factory('partner-agent'))
_.get_agent()
"""))
defer.returnValue(self.get_local('_'))
def testParsingAgent(self):
test_config = format_block("""
[application:feat]
import: feat.agents.application
name: feat
[agent:dns_0]
agent_type: dns_agent
descriptor.suffix: "test.lan"
initiate.slaves: [["1.2.3.4", 53], ["5.6.7.8", 1000]]""")
f = StringIO(test_config)
configfile.parse_file(self.parser, f)
v = self.parser.values
self.assertEqual(1, len(v.agents))
d = v.agents[0][0]
self.assertIsInstance(d, dns_agent.Descriptor)
self.assertIsInstance(d.suffix, unicode)
self.assertEqual('test.lan', d.suffix)
i = v.agents[0][1]
self.assertIsInstance(i, dict)
self.assertIn('slaves', i)
self.assertEqual([["1.2.3.4", 53], ["5.6.7.8", 1000]], i['slaves'])
self.assertEquals('dns_0', v.agents[0][2])
def get_entries(self, history, start_date=0, limit=None):
'''
Returns a list of journal entries for the given history_id.
'''
if not isinstance(history, History):
raise AttributeError(
'First paremeter is expected to be History instance, got %r'
% history)
command = text_helper.format_block("""
SELECT histories.agent_id,
histories.instance_id,
entries.journal_id,
entries.function_id,
entries.fiber_id,
entries.fiber_depth,
entries.args,
entries.kwargs,
entries.side_effects,
entries.result,
entries.timestamp
FROM entries
LEFT JOIN histories ON histories.id = entries.history_id
WHERE entries.history_id = ?""")
if start_date:
command += " AND entries.timestamp >= %s" % (start_date, )
command += " ORDER BY entries.rowid ASC"
if limit:
command += " LIMIT %s" % (limit, )
d = self._db.runQuery(command, (history.history_id, ))
d.addCallback(self._decode)
return d
def prolog(self):
setup = text_helper.format_block("""
agency1 = spawn_agency()
host1 = agency1.get_host_agent()
agency2 = spawn_agency()
host2 = agency2.get_host_agent()
host2.start_agent(descriptor_factory('export_agent'))
host2.start_agent(descriptor_factory('alert_agent'))
wait_for_idle()
agency3 = spawn_agency()
host3 = agency3.get_host_agent()
host3.wait_for_ready()
signal_desc = descriptor_factory('test_signal_agent')
signal = host3.start_agent(signal_desc)
host3.start_agent(descriptor_factory('test_worker_agent'),\
dependency=signal)
agency4 = spawn_agency()
host4 = agency4.get_host_agent()
host4.wait_for_ready()
host4.start_agent(descriptor_factory('test_worker_agent'),\
dependency=signal)
wait_for_idle()
""")
yield self.process(setup)
self.export_agent = self._get_agent('export_agent')
self.host1 = self.get_local('host1')
self.host2 = self.get_local('host2')
self.host3 = self.get_local('host3')
self.host4 = self.get_local('host4')
def prolog(self):
setup = format_block(
"""
agency = spawn_agency(hostname='test.host.lan')
agent = agency.get_host_agent()
"""
)
return self.process(setup)
def testKeywords(self):
test = format_block("""
echo(echo='some text')
""")
d = self.cb_after(None, self.parser, 'on_finish')
self.parser.dataReceived(test)
yield d
self.assertEqual('some text\n', self.output.getvalue())
def prolog(self):
setup = format_block("""
agency = spawn_agency()
agent = agency.get_host_agent()
medium = find_agent(agent)
""")
yield self.process(setup)
yield self.wait_for_idle(10)
def prolog(self):
setup = text_helper.format_block("""
agency = spawn_agency()
agency.start_agent(descriptor_factory('notifier-agent'), \
run_startup=False)
""")
yield self.process(setup)
self.agent = self.get_local('_').get_agent()
def do_insert_log(connection, data):
command = text_helper.format_block("""
INSERT INTO logs VALUES (?, ?, ?, ?, ?, ?, strftime('%s', 'now'))
""")
connection.execute(
command, (data['message'], int(data['level']),
data['category'], data['log_name'],
data['file_path'], data['line_num']))
def prolog(self):
setup = format_block("""
agency = spawn_agency()
agency.disable_protocol('setup-monitoring', 'Task')
medium = agency.start_agent(descriptor_factory('task-agent'))
agent = medium.get_agent()
""")
return self.process(setup)
def prolog(self):
setup = format_block("""
agency = spawn_agency(start_host=False)
desc = descriptor_factory('test_prop_agent')
medium = agency.start_agent(desc)
agent = medium.get_agent()
""")
yield self.process(setup)
def render_resource(self, request, response, location):
response.set_mime_type('text/html')
response.set_status(http.Status[404])
response.write(
format_block('''
<html>
<body>
<h1>This server is configured to run under /%(prefix)s prefix.</h1>
You should use <a href="/%(url)s">this url</a>.
</body>
</html>''') % dict(prefix='/'.join(self._prefix),
url="/".join(self._prefix + location[1:])))
def to_gnuplot(self):
script = base_settings + format_block('''
set xlabel "%(xlabel)s"
box(time)=(int(time / %(barwidth)s) * %(barwidth)s)
set boxwidth %(barwidth)s
set ylabel "Events in range"
set title "%(title)s"
plot "-" using (box($2)):(1.0) smooth freq with boxes
''' % dict(title=self.title,
barwidth=self.barwidth,
xlabel=self.xlabel,
ylabel=self.ylabel))
return format_data(script, self.data)
def to_gnuplot(self):
script = base_settings + format_block('''
set xlabel "%(xlabel)s"
set xdata time
set timefmt "%%s"
set xtics rotate by 90 offset 0,-5 out nomirror
set format x "%%m/%%d %%H:%%M"
set ylabel "%(ylabel)s"
set title "%(title)s"
plot "-" using 1:2 w point title "%(plotlabel)s"
''' % dict(title=self.title,
plotlabel=self.plotlabel,
xlabel=self.xlabel,
ylabel=self.ylabel))
return format_data(script, self.data)
class IGraph(Interface):
data = Attribute("Raw data the graph is constructed from")
def to_gnuplot():
'''
@returns: C{str} script to use to generate a plot.
'''
base_settings = format_block('''
set term png transparent truecolor enhanced font "Tahoma,12"
set style fill transparent solid 0.5
set border lc rgb "%(base_color)s"
set xlabel tc rgb "%(base_color)s"
set ylabel tc rgb "%(base_color)s"
set key tc rgb "%(base_color)s"
''' % dict(base_color="#aaa190"))
class TimelineGraph(object):
implements(IGraph)
def __init__(self,
data,
title='',
plotlabel="Waiting time",
xlabel="Date",