def sendMessage(bridgeNode, bridgePort, nodes, docks, method, args):
# Join the overlay at the specified bridge node.
messaging = api.ClientConnection(CLIENT_NAME, bridgeNode, bridgePort)
# Create a ping message and send on the overlay
# All node on the overlay will receive it and the daemon will respond with a pong message
msg = MAGIMessage(nodes=nodes,
docks=docks,
contenttype=MAGIMessage.YAML,
data=yaml.safe_dump({
'method': method,
'args': args,
'version': 1.0
}))
log.debug("Sending msg: %s" % (msg))
messaging.send(msg)
return messaging
def doMainLoop():
# create the message handler
handler = MessageHandler()
handler.loadRoutes(options.routeFile)
# Connect to network as node "watcheri"
conn = api.ClientConnection("watcheri", options.host, 18808)
# Request to receive messages to group "data"
conn.join("data")
# Run forever
done = False
while not done:
# block until next message
msg = conn.nextMessage(True)
# if the destination dock indicates a worm message
if 'worm' in msg.dstdocks:
# do something with the lines of text here
textlines = msg.data
if options.verbose:
print "incoming data:"
print textlines
lines = textlines.split('\n')
for l in lines:
data = parseLine(l)
if len(data) != 4:
print "Parse error on line: %s" % l
continue
handler.handleMessage(data[2], data[0], data[1], data[3])
def getMessenger(node, port):
global msgrCache
if msgrCache.get(node + str(port)) == None:
msgrCache[node + str(port)] = api.ClientConnection(srcdock, node, port)
return msgrCache[node + str(port)]
optparser.add_option(
"-c",
"--control",
dest="control",
help="The control node to connect to (i.e. control.exp.proj)")
optparser.add_option(
"-l",
"--level",
dest="level",
help="The log level to use, 10=debug, 20=info, 30=warning, 40=error",
default=20)
#optparser.add_option("-t", "--thread", dest="thread", help="The thread to restrict to", default=None)
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, handler)
messaging = api.ClientConnection("pylogs", options.control, 18808)
messaging.join("loglis") # same group as we ask log agent to stream to
doStartup(level=int(options.level)) #, thread=options.thread)
while not done:
msg = messaging.nextMessage(
True, sys.maxint
) # without at least some timeout main thread stops receiving signals
if msg is not None and type(
msg) is not str and 'loglis' in msg.dstdocks:
logdata = yaml.load(msg.data)
for nodename in logdata:
for record in logdata[nodename]:
logdisplay(nodename, record)
doShutdown()
if not options.trigger:
log.critical('Must give the trigger via -t or --trigger.')
sys.exit(1)
# Connect to backend
log.info("Connecting to %s" % options.control)
p = None
if options.tunnel:
localport = 18803
p = subprocess.Popen("ssh users.deterlab.net -L " + str(localport) +
":" + options.control + ":18808 -N",
shell=True)
time.sleep(2)
connection = api.ClientConnection("sendTrigger", "127.0.0.1",
localport)
else:
connection = api.ClientConnection("sendTrigger", options.control,
18808)
# get/send trigger events
connection.join('trigger')
# Should wait for some type of confirmation here instead
# of assuming it worked.
time.sleep(2)
args = dict()
if options.args:
args = dict([pair.split('=') for pair in options.args.split(',')])
# Give the ability to just display the events file
if options.justparse:
exit(0)
try:
tunnel_cmd = None
if options.tunnel:
tunnel_cmd = create_tunnel('users.deterlab.net', 18808,
options.control, 18808)
bridge = '127.0.0.1'
logging.info('Tunnel setup done')
else:
bridge = options.control
try:
messaging = api.ClientConnection(options.name, bridge, 18808)
except gaierror as e:
logging.critical('Error connecting to %s: %s', options.control,
str(e))
exit(3)
signal.signal(signal.SIGUSR1, sigusr1_handler)
orch = Orchestrator(messaging,
aal,
dagdisplay=options.display,
verbose=options.verbose,
exitOnFailure=options.exitOnFailure,
useColor=(not options.nocolor))
orch.run()
import time
from magi.orchestrator import *
from magi.messaging import api
logging.basicConfig(level=logging.INFO)
if __name__ == '__main__':
optparser = optparse.OptionParser()
optparser.add_option("-c", "--control", dest="control", help="The control node to connect to (i.e. control.exp.proj)", default="127.0.0.1")
optparser.add_option("-s", "--size", dest="size", type="int", help="The size of the message packet. Default 1000Bytes ", default="1000")
optparser.add_option("-r", "--rate", dest="rate", type="float", help="The rate of the packets in packets per second. Default 100messages/sec", default="0.01")
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, signal.SIG_DFL)
messaging = api.ClientConnection("pystresser", options.control, 18808)
#messaging = api.SSLClientConnection("pystresser", options.control, 18810, "DeterTest", "bwilson-orch")
messaging.join("data")
time.sleep(2)
options.rate = 1/options.rate
print options.size, options.rate
req = {
'version': 1.0,
'method': 'loadAgent',
'args': {
'name':'stress1',
'code':'stress',
'dock':'stress1',
'args':[],
"-n",
"--tunnel",
dest="tunnel",
help=
"Tell orchestrator to tunnel data through Deter Ops (users.deterlab.net).",
default=False,
action="store_true")
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, handler)
if options.tunnel:
localport = 18802
p = subprocess.Popen("ssh users.deterlab.net -L " + str(localport) +
":" + options.control + ":18808 -N",
shell=True)
messaging = api.ClientConnection("pypassive", "127.0.0.1", localport)
else:
messaging = api.ClientConnection("pypassive", options.control, 18808)
if ':' in options.group:
for grp in options.group.split(':'):
print "Joining group:", grp
messaging.join(grp)
else:
print "Joining group:", options.group
messaging.join(options.group)
engine = create_engine('sqlite:////tmp/magi.db')
metadata = MetaData()
countertable = Table('pktdata', metadata, Column('timestamp', Integer),
Column('eth0_pkts', Integer),
optparser.add_option(
"-C",
"--columns",
dest="columns",
help=
"If given, only stream the data from the given columns. Requires knowledge of counter data. If not given all columns are streamed. Columns must be given as comma separated string."
)
(options, args) = optparser.parse_args()
clientId = 'listener_' + ''.join(
choice(ascii_lowercase) for x in range(10))
agentId = options.agent
streamId = 'stream_' + options.agent
signal.signal(signal.SIGINT, handler)
messaging = api.ClientConnection(clientId, options.control, 18808)
cols = None if not options.columns else options.columns.split(',')
doStartup(options.agent, cols)
messaging.join(streamId)
while not done:
msg = messaging.nextMessage(
True, sys.maxint
) # without at least some timeout main thread stops receiving signals
if msg is not None and type(
msg) is not str and streamId in msg.dstdocks:
streamData = yaml.load(msg.data)
for nodename in streamData:
for record in streamData[nodename]:
print "%s: %s" % (nodename, record)
doShutdown(options.unload)
optparser.add_option(
"-c",
"--control",
dest="control",
help="The control node to connect to (i.e. control.exp.proj)")
optparser.add_option(
"-g",
"--group",
dest="group",
help=
"List one of more groups to listen to (ie \"control:data:trigger\") ",
default="control")
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, handler)
messaging = api.ClientConnection("grcat", options.control, 18808)
sleep(3)
groups = options.group.split(':')
for group in groups:
messaging.join(group)
sleep(20)
while not done:
line = sys.stdin.readline()
if not line:
break
msg = MAGIMessage(groups=groups,
contenttype=MAGIMessage.YAML,
done = False
messaging = None
def handler(signum, frame):
global done
print "shutting down ..."
done = True
messaging.poisinPill()
if __name__ == '__main__':
optparser = optparse.OptionParser()
optparser.add_option("-f", "--file", dest="file", help="file with events to send")
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, handler)
messaging = api.ClientConnection("orchTesterClient", "128.9.168.95", 18808)
messaging.join('control')
print "messaging up and ready"
while not done:
print "waiting for a message"
try:
msg = messaging.nextMessage(block=True, timeout=1)
if msg:
print "HELLO GOT", msg
except Queue.Empty:
pass
nmsg = MAGIMessage(groups='control', contenttype=MAGIMessage.YAML, data=yaml.safe_dump({'data': 'lovely sent from client'}))
messaging.send(nmsg)
"--nodes",
dest="nodes",
help=
"One or more nodes that should should join the group (node0:node1:control)."
)
optparser.add_option(
"-x",
"--exit",
action='store_true',
help=
"If given, tell everyone to leave the group when this script exits.",
default=False)
(options, args) = optparser.parse_args()
signal.signal(signal.SIGINT, handler)
messaging = api.ClientConnection("group_joiner", options.control, 18808)
time.sleep(3)
# This sets up the negihtbors for the clientConnection transport
if ':' in options.group:
for grp in options.group.split(':'):
print "Joining group:", grp
messaging.join(grp)
else:
print "Joining group:", options.group
messaging.join(options.group)
nodes = options.nodes.split(':')
# This sets up the negihtbors for the experiment network transport
sys.exit(2)
nodeset = set()
if options.file is not None:
aaldata = yaml.load(open(options.file, 'r'))
for name, nodes in aaldata['groups'].iteritems():
log.info("Adding nodes from group %s", name)
nodeset.update(nodes)
# Terminate if the user presses ctrl+c
signal.signal(signal.SIGINT, signal.SIG_DFL)
# Join the overlay at the specified bridge node.
#Port is hard coded :(
#TODO: read the msdl to find a bridge
messaging = api.ClientConnection("ping", options.bridge, 18808)
# Create a ping message and send on the overlay
# All node on the overlay will receive it and the daemon will respond with a pong message
msg = MAGIMessage(groups=['__ALL__'],
docks='daemon',
contenttype=MAGIMessage.YAML,
data=yaml.safe_dump({
'method': 'ping',
'version': 1.0
}))
messaging.send(msg)
# Wait for timeout seconds before stoppping
start = time.time()
stop = start + int(options.timeout)
nodeSet=nodeSet,
timeout=10)
if status:
break
log.info("Magi daemon on one or more nodes not up")
log.info("Did not receive reply from %s",
sorted(list(nodeSet - set(result.keys()))))
nodeSet = nodeSet - set(result.keys())
except:
log.info("Magi daemon on one or more nodes not up")
time.sleep(5)
log.info("All magi daemons are up and listening")
try:
messaging = api.ClientConnection(options.name, bridgeNode,
bridgePort)
except gaierror as e:
logging.error("Error connecting to %s: %d" %
(bridgeNode, bridgePort))
exit(3)
signal.signal(signal.SIGINT, signal_handler)
orch = Orchestrator(messaging,
aal,
dagdisplay=options.display,
verbose=options.verbose,
exitOnFailure=options.exitOnFailure,
useColor=(not options.nocolor),
dbHost=dbHost,
dbPort=dbPort)