def __init__(self, context, dir, ext="pkt"):
SThread.__init__(self)
self.context = context
self.dir = dir
self.ext = ext
PmkShared._ensure_dir(dir)
pass
def __init__(self, context, dir, ext="pkt"):
SThread.__init__(self)
self.context = context
self.dir = dir
self.ext = ext
PmkShared._ensure_dir(dir)
pass
def move_data_file(self, src, dst, postfix=None):
PmkShared._ensure_dir(self.context.getWorkingDir()+dst)
if postfix:
dst_file = self.context.getWorkingDir()+"/"+dst+"/"+src+"-"+postfix
else:
dst_file = self.context.getWorkingDir()+"/"+dst+"/"+src
dst_file = dst_file.replace("//","/")
src_file = self.context.getWorkingDir()+src
shutil.move(src_file,dst_file)
return dst_file
def move_data_file(self, src, dst, postfix=None):
PmkShared._ensure_dir(self.context.getWorkingDir() + dst)
if postfix:
dst_file = self.context.getWorkingDir(
) + "/" + dst + "/" + src + "-" + postfix
else:
dst_file = self.context.getWorkingDir() + "/" + dst + "/" + src
dst_file = dst_file.replace("//", "/")
src_file = self.context.getWorkingDir() + src
shutil.move(src_file, dst_file)
return dst_file
def startContext(self):
context = self.context
logging.info("Node assigned UID: "+context.getUuid())
logging.info("Exec context: "+context.getExecContext())
logging.info("Node bound to IP: "+context.get_local_ip())
home = expanduser("~")
wd = home+"/.pumpkin/"+context.getUuid()+"/"
context.working_dir = wd
PmkShared._ensure_dir(wd)
PmkShared._ensure_dir(wd+"logs/")
PmkShared._ensure_dir(wd+"seeds/")
context.log_to_file()
context.startPktShelve("PktStore")
context.startPktShelve2("RemotePktStore")
if context.is_speedy():
logging.info("Running in speedy gonzales mode")
logging.debug("Working directory: "+context.getWorkingDir())
#context.openfiles.append(context.getWorkingDir())
context = self.context
zmq_context = self.zmq_context
context.zmq_context = zmq_context
#PmkShared.ZMQ_PUB_PORT = PmkShared._get_nextport(ZMQ_PUB_PORT, "TCP")
PmkShared.ZMQ_ENDPOINT_PORT = PmkShared._get_nextport(ZMQ_ENDPOINT_PORT, "TCP")
logging.debug("ZMQ endpoint port: "+str(PmkShared.ZMQ_ENDPOINT_PORT))
PmkShared.TFTP_FILE_SERVER_PORT = PmkShared._get_nextport(TFTP_FILE_SERVER_PORT, "UDP")
context.setEndpoints()
if not context.is_ghost():
#zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
#zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
#zmqbc.start()
#context.addThread(zmqbc)
pass
if context.with_broadcast():
#Listen for UDP broadcasts on LAN
udplisten = BroadcastListener(context, int(context.getAttributeValue().bcport), zmq_context)
udplisten.start()
context.addThread(udplisten)
udpbc = Broadcaster(context, int(context.getAttributeValue().bcport), rate = context.get_broadcast_rate())
udpbc.start()
context.addThread(udpbc)
zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
zmqbc.start()
context.addThread(zmqbc)
pass
#Local stuff to exploit multi-cores still needs testing
#context.peers[context.getUuid()] = "/tmp/"+context.getUuid()+"-bcast"
#local_peers = self._shelve_safe_open("/tmp/pumpkin")
#local_peers[context.getUuid()] = "ipc:///tmp/"+context.getUuid()+"-bcast"
#local_peers.close()
# zmqbc = ZMQBroadcaster(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bcast")
# context.openfiles.append("/tmp/"+context.getUuid()+"-bcast")
# #context.openfiles.append("/tmp/pumpkin-bus")
# zmqbc.start()
# context.addThread(zmqbc)
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bus")
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/pumpkin-bus")
#zmqsub.start()
#context.addThread(zmqsub)
ftpdir = wd + 'tx/'
if not context.is_ghost():
#TFTP for sending files between pumpkins
tftpserver = TftpServer(context, ftpdir, PmkShared.TFTP_FILE_SERVER_PORT)
tftpserver.start()
context.setFileDir(ftpdir)
context.addThread(tftpserver)
#FTP for user access of working directory
ftpserver = FtpServer(context, context.getWorkingDir())
ftpserver.start()
context.addThread(ftpserver)
edispatch = ExternalDispatch(context)
context.setExternalDispatch(edispatch)
edispatch.start()
context.addThread(edispatch)
if context.broadcast_rabbitmq():
host, port, username, password, vhost = self.context.get_rabbitmq_cred()
credentials = pika.PlainCredentials(username, password)
connection = pika.BlockingConnection(pika.ConnectionParameters(host=host, credentials=credentials, virtual_host=vhost))
rabbitmq = RabbitMQMonitor(context, connection)
context.set_rabbitmq(rabbitmq)
q = context.get_group()+":info"
bunny = RabbitMQBroadcaster(context, exchange=q)
bunny.start()
context.addThread(bunny)
bunnylistener = RabbitMQBroadcastSubscriber(context, exchange=q)
bunnylistener.start()
context.addThread(bunnylistener)
logging.debug("Adding RabbitMQ queue: "+context.getUuid())
rabbitmq.add_monitor_queue(context.getUuid())
rabbitmq.add_monitor_queue(context.getUuid()+"ack")
qm = context.get_group()+":track"
monitor = RabbitMqLog(self.context)
monitor.connect(qm)
mon_dispatcher = LogDisptacher(self.context)
mon_dispatcher.add_monitor(monitor)
mon_dispatcher.start()
context.addThread(mon_dispatcher)
if context.hasRx():
rxdir = context.hasRx()
pfm = PacketFileMonitor(context, rxdir)
pfm.start()
context.addThread(pfm)
else:
rxdir = context.getWorkingDir()+"/rx"
pfm = PacketFileMonitor(context, rxdir)
pfm.start()
context.addThread(pfm)
if context.isSupernode() and not context.is_ghost():
logging.debug("In supernode mode")
http = HttpServer(context)
http.start()
context.addThread(http)
for fd in listdir(context.getTaskDir()):
src = context.getTaskDir()+"/"+fd
if( src[-3:] == "pyc"):
continue
if( src[-2:] == "py"):
dst = context.getWorkingDir()+"/seeds/"+fd
else:
dst = context.getWorkingDir()+"/"+fd
try:
shutil.copytree(src, dst)
except OSError as exc: # python >2.5
if exc.errno == errno.ENOTDIR:
shutil.copy(src, dst)
else: raise
if not context.isWithNoPlugins():# and not context.isSupernode():
for ep in context.getEndpoints():
if context.isZMQEndpoint(ep):
#ep[0] = PmkShared._get_nextport(ep[0], "TCP")
tcpm = ZMQPacketMonitor(context, zmq_context, ep[0])
tcpm.start()
context.addThread(tcpm)
for sn in get_zmq_supernodes(PmkShared.SUPERNODES):
if not str(sn).__contains__("127.0.0.1"):
#logging.debug("Subscribing to: "+sn)
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, sn)
#zmqsub.start()
#context.addThread(zmqsub)
pass
pass
try:
if not context.singleSeed():
onlyfiles = [ f for f in listdir(context.getTaskDir()) if isfile(join(context.getTaskDir(),f)) ]
for fl in onlyfiles:
fullpath = context.getTaskDir()+"/"+fl
modname = fl[:-3]
#ext = fl[-2:]
self.load_seed(fullpath)
else:
seedfp = context.singleSeed()
self.load_seed(seedfp)
except Exception as e:
logging.error("Import error "+ str(e))
pass
for x in PmkSeed.iplugins.keys():
klass = PmkSeed.iplugins[x]
js = klass.getConfEntry()
#logging.debug(js)
context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
#context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
#context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
logging.debug("Registry dump: "+context.getProcGraph().dumpExternalRegistry())
seedmonitor = PacketFileMonitor(context, context.getWorkingDir()+"seeds/", ext="py")
seedmonitor.start()
context.addThread(seedmonitor)
for i in range(1,context.get_cores()+1):
logging.info("Starting internal dispatch thread "+str(i))
idispatch = InternalDispatch(context)
idispatch.start()
context.addThread(idispatch)
#context.startDisplay()
self._checkLocalPeers(zmq_context)
##############START TASKS WITH NO INPUTS#############################
inj = Injector(context)
inj.start()
context.addThread(inj)
#####################################################################
if context.hasShell():
cmdp = Shell(context)
cmdp.cmdloop()
def run(self):
PmkShared._ensure_dir(self.dir)
tftp_server = tftpy.TftpServer(self.dir)
tftp_server.listen('0.0.0.0', self.port)
def run(self):
PmkShared._ensure_dir(self.dir)
tftp_server = tftpy.TftpServer(self.dir)
tftp_server.listen('0.0.0.0', self.port)
def startContext(self):
context = self.context
logging.info("Node assigned UID: " + context.getUuid())
logging.info("Exec context: " + context.getExecContext())
logging.info("Node bound to IP: " + context.get_local_ip())
home = expanduser("~")
wd = home + "/.pumpkin/" + context.getUuid() + "/"
context.working_dir = wd
PmkShared._ensure_dir(wd)
PmkShared._ensure_dir(wd + "logs/")
PmkShared._ensure_dir(wd + "seeds/")
context.log_to_file()
context.startPktShelve("PktStore")
context.startPktShelve2("RemotePktStore")
if context.is_speedy():
logging.info("Running in speedy gonzales mode")
logging.debug("Working directory: " + context.getWorkingDir())
#context.openfiles.append(context.getWorkingDir())
context = self.context
zmq_context = self.zmq_context
context.zmq_context = zmq_context
#PmkShared.ZMQ_PUB_PORT = PmkShared._get_nextport(ZMQ_PUB_PORT, "TCP")
PmkShared.ZMQ_ENDPOINT_PORT = PmkShared._get_nextport(
ZMQ_ENDPOINT_PORT, "TCP")
logging.debug("ZMQ endpoint port: " + str(PmkShared.ZMQ_ENDPOINT_PORT))
PmkShared.TFTP_FILE_SERVER_PORT = PmkShared._get_nextport(
TFTP_FILE_SERVER_PORT, "UDP")
context.setEndpoints()
if not context.is_ghost():
#zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
#zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
#zmqbc.start()
#context.addThread(zmqbc)
pass
if context.with_broadcast():
#Listen for UDP broadcasts on LAN
udplisten = BroadcastListener(
context, int(context.getAttributeValue().bcport), zmq_context)
udplisten.start()
context.addThread(udplisten)
udpbc = Broadcaster(context,
int(context.getAttributeValue().bcport),
rate=context.get_broadcast_rate())
udpbc.start()
context.addThread(udpbc)
zmqbc = ZMQBroadcaster(context, zmq_context,
"tcp://*:" + str(PmkShared.ZMQ_PUB_PORT))
zmqbc = ZMQBroadcaster(context, zmq_context,
context.get_our_pub_ep("tcp"))
zmqbc.start()
context.addThread(zmqbc)
pass
#Local stuff to exploit multi-cores still needs testing
#context.peers[context.getUuid()] = "/tmp/"+context.getUuid()+"-bcast"
#local_peers = self._shelve_safe_open("/tmp/pumpkin")
#local_peers[context.getUuid()] = "ipc:///tmp/"+context.getUuid()+"-bcast"
#local_peers.close()
# zmqbc = ZMQBroadcaster(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bcast")
# context.openfiles.append("/tmp/"+context.getUuid()+"-bcast")
# #context.openfiles.append("/tmp/pumpkin-bus")
# zmqbc.start()
# context.addThread(zmqbc)
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bus")
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/pumpkin-bus")
#zmqsub.start()
#context.addThread(zmqsub)
ftpdir = wd + 'tx/'
if not context.is_ghost():
#TFTP for sending files between pumpkins
tftpserver = TftpServer(context, ftpdir,
PmkShared.TFTP_FILE_SERVER_PORT)
tftpserver.start()
context.setFileDir(ftpdir)
context.addThread(tftpserver)
#FTP for user access of working directory
ftpserver = FtpServer(context, context.getWorkingDir())
ftpserver.start()
context.addThread(ftpserver)
edispatch = ExternalDispatch(context)
context.setExternalDispatch(edispatch)
edispatch.start()
context.addThread(edispatch)
if context.broadcast_rabbitmq():
host, port, username, password, vhost = self.context.get_rabbitmq_cred(
)
credentials = pika.PlainCredentials(username, password)
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=host,
credentials=credentials,
virtual_host=vhost))
rabbitmq = RabbitMQMonitor(context, connection)
context.set_rabbitmq(rabbitmq)
q = context.get_group() + ":info"
bunny = RabbitMQBroadcaster(context, exchange=q)
bunny.start()
context.addThread(bunny)
bunnylistener = RabbitMQBroadcastSubscriber(context, exchange=q)
bunnylistener.start()
context.addThread(bunnylistener)
logging.debug("Adding RabbitMQ queue: " + context.getUuid())
rabbitmq.add_monitor_queue(context.getUuid())
rabbitmq.add_monitor_queue(context.getUuid() + "ack")
qm = context.get_group() + ":track"
monitor = RabbitMqLog(self.context)
monitor.connect(qm)
mon_dispatcher = LogDisptacher(self.context)
mon_dispatcher.add_monitor(monitor)
mon_dispatcher.start()
context.addThread(mon_dispatcher)
if context.hasRx():
rxdir = context.hasRx()
pfm = PacketFileMonitor(context, rxdir)
pfm.start()
context.addThread(pfm)
else:
rxdir = context.getWorkingDir() + "/rx"
pfm = PacketFileMonitor(context, rxdir)
pfm.start()
context.addThread(pfm)
if context.isSupernode() and not context.is_ghost():
logging.debug("In supernode mode")
http = HttpServer(context)
http.start()
context.addThread(http)
for fd in listdir(context.getTaskDir()):
src = context.getTaskDir() + "/" + fd
if (src[-3:] == "pyc"):
continue
if (src[-2:] == "py"):
dst = context.getWorkingDir() + "/seeds/" + fd
else:
dst = context.getWorkingDir() + "/" + fd
try:
shutil.copytree(src, dst)
except OSError as exc: # python >2.5
if exc.errno == errno.ENOTDIR:
shutil.copy(src, dst)
else:
raise
if not context.isWithNoPlugins(): # and not context.isSupernode():
for ep in context.getEndpoints():
if context.isZMQEndpoint(ep):
#ep[0] = PmkShared._get_nextport(ep[0], "TCP")
tcpm = ZMQPacketMonitor(context, zmq_context, ep[0])
tcpm.start()
context.addThread(tcpm)
for sn in get_zmq_supernodes(PmkShared.SUPERNODES):
if not str(sn).__contains__("127.0.0.1"):
#logging.debug("Subscribing to: "+sn)
#zmqsub = ZMQBroadcastSubscriber(context, zmq_context, sn)
#zmqsub.start()
#context.addThread(zmqsub)
pass
pass
try:
if not context.singleSeed():
onlyfiles = [
f for f in listdir(context.getTaskDir())
if isfile(join(context.getTaskDir(), f))
]
for fl in onlyfiles:
fullpath = context.getTaskDir() + "/" + fl
modname = fl[:-3]
#ext = fl[-2:]
self.load_seed(fullpath)
else:
seedfp = context.singleSeed()
self.load_seed(seedfp)
except Exception as e:
logging.error("Import error " + str(e))
pass
for x in PmkSeed.iplugins.keys():
klass = PmkSeed.iplugins[x]
js = klass.getConfEntry()
#logging.debug(js)
context.getProcGraph().updateRegistry(json.loads(js),
loc="locallocal")
#context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
#context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
logging.debug("Registry dump: " +
context.getProcGraph().dumpExternalRegistry())
seedmonitor = PacketFileMonitor(context,
context.getWorkingDir() + "seeds/",
ext="py")
seedmonitor.start()
context.addThread(seedmonitor)
for i in range(1, context.get_cores() + 1):
logging.info("Starting internal dispatch thread " + str(i))
idispatch = InternalDispatch(context)
idispatch.start()
context.addThread(idispatch)
#context.startDisplay()
self._checkLocalPeers(zmq_context)
##############START TASKS WITH NO INPUTS#############################
inj = Injector(context)
inj.start()
context.addThread(inj)
#####################################################################
if context.hasShell():
cmdp = Shell(context)
cmdp.cmdloop()
def run(self):
PmkShared._ensure_dir(self.dir)
self.server.serve_forever()
def run(self):
PmkShared._ensure_dir(self.dir)
self.server.serve_forever()