Exemplo n.º 1
0
 def __init__(self, context, dir, ext="pkt"):
     SThread.__init__(self)
     self.context = context
     self.dir = dir
     self.ext = ext
     PmkShared._ensure_dir(dir)
     pass
Exemplo n.º 2
0
 def __init__(self, context, dir, ext="pkt"):
     SThread.__init__(self)
     self.context = context
     self.dir = dir
     self.ext = ext
     PmkShared._ensure_dir(dir)
     pass
Exemplo n.º 3
0
    def move_data_file(self, src, dst, postfix=None):

        PmkShared._ensure_dir(self.context.getWorkingDir()+dst)
        if postfix:
            dst_file = self.context.getWorkingDir()+"/"+dst+"/"+src+"-"+postfix
        else:
            dst_file = self.context.getWorkingDir()+"/"+dst+"/"+src
        dst_file = dst_file.replace("//","/")
        src_file = self.context.getWorkingDir()+src

        shutil.move(src_file,dst_file)
        return dst_file
Exemplo n.º 4
0
    def move_data_file(self, src, dst, postfix=None):

        PmkShared._ensure_dir(self.context.getWorkingDir() + dst)
        if postfix:
            dst_file = self.context.getWorkingDir(
            ) + "/" + dst + "/" + src + "-" + postfix
        else:
            dst_file = self.context.getWorkingDir() + "/" + dst + "/" + src
        dst_file = dst_file.replace("//", "/")
        src_file = self.context.getWorkingDir() + src

        shutil.move(src_file, dst_file)
        return dst_file
Exemplo n.º 5
0
    def startContext(self):
        context = self.context
        logging.info("Node assigned UID: "+context.getUuid())
        logging.info("Exec context: "+context.getExecContext())
        logging.info("Node bound to IP: "+context.get_local_ip())
        home = expanduser("~")
        wd = home+"/.pumpkin/"+context.getUuid()+"/"
        context.working_dir = wd

        PmkShared._ensure_dir(wd)
        PmkShared._ensure_dir(wd+"logs/")
        PmkShared._ensure_dir(wd+"seeds/")
        context.log_to_file()

        context.startPktShelve("PktStore")
        context.startPktShelve2("RemotePktStore")
        if context.is_speedy():
            logging.info("Running in speedy gonzales mode")

        logging.debug("Working directory: "+context.getWorkingDir())

        #context.openfiles.append(context.getWorkingDir())

        context = self.context
        zmq_context = self.zmq_context
        context.zmq_context = zmq_context
        #PmkShared.ZMQ_PUB_PORT  = PmkShared._get_nextport(ZMQ_PUB_PORT, "TCP")
        PmkShared.ZMQ_ENDPOINT_PORT = PmkShared._get_nextport(ZMQ_ENDPOINT_PORT, "TCP")
        logging.debug("ZMQ endpoint port: "+str(PmkShared.ZMQ_ENDPOINT_PORT))
        PmkShared.TFTP_FILE_SERVER_PORT  = PmkShared._get_nextport(TFTP_FILE_SERVER_PORT, "UDP")

        context.setEndpoints()

        if not context.is_ghost():
            #zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
            #zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
            #zmqbc.start()
            #context.addThread(zmqbc)
            pass

        if context.with_broadcast():
            #Listen for UDP broadcasts on LAN
            udplisten = BroadcastListener(context, int(context.getAttributeValue().bcport), zmq_context)
            udplisten.start()
            context.addThread(udplisten)


            udpbc = Broadcaster(context, int(context.getAttributeValue().bcport), rate = context.get_broadcast_rate())
            udpbc.start()
            context.addThread(udpbc)

            zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
            zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
            zmqbc.start()
            context.addThread(zmqbc)
            pass

        #Local stuff to exploit multi-cores still needs testing

        #context.peers[context.getUuid()] = "/tmp/"+context.getUuid()+"-bcast"
        #local_peers = self._shelve_safe_open("/tmp/pumpkin")
        #local_peers[context.getUuid()] = "ipc:///tmp/"+context.getUuid()+"-bcast"
        #local_peers.close()
        # zmqbc = ZMQBroadcaster(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bcast")
        # context.openfiles.append("/tmp/"+context.getUuid()+"-bcast")
        # #context.openfiles.append("/tmp/pumpkin-bus")
        # zmqbc.start()
        # context.addThread(zmqbc)

        #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bus")
        #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/pumpkin-bus")
        #zmqsub.start()
        #context.addThread(zmqsub)

        ftpdir = wd + 'tx/'

        if not context.is_ghost():
            #TFTP for sending files between pumpkins
            tftpserver = TftpServer(context, ftpdir, PmkShared.TFTP_FILE_SERVER_PORT)
            tftpserver.start()
            context.setFileDir(ftpdir)
            context.addThread(tftpserver)

            #FTP for user access of working directory
            ftpserver = FtpServer(context, context.getWorkingDir())
            ftpserver.start()
            context.addThread(ftpserver)

        edispatch = ExternalDispatch(context)
        context.setExternalDispatch(edispatch)
        edispatch.start()
        context.addThread(edispatch)

        if context.broadcast_rabbitmq():
            host, port, username, password, vhost = self.context.get_rabbitmq_cred()
            credentials = pika.PlainCredentials(username, password)

            connection = pika.BlockingConnection(pika.ConnectionParameters(host=host,  credentials=credentials, virtual_host=vhost))
            rabbitmq = RabbitMQMonitor(context, connection)
            context.set_rabbitmq(rabbitmq)

            q = context.get_group()+":info"
            bunny = RabbitMQBroadcaster(context, exchange=q)
            bunny.start()
            context.addThread(bunny)

            bunnylistener = RabbitMQBroadcastSubscriber(context, exchange=q)
            bunnylistener.start()
            context.addThread(bunnylistener)

            logging.debug("Adding RabbitMQ queue: "+context.getUuid())
            rabbitmq.add_monitor_queue(context.getUuid())
            rabbitmq.add_monitor_queue(context.getUuid()+"ack")

            qm = context.get_group()+":track"
            monitor = RabbitMqLog(self.context)
            monitor.connect(qm)

            mon_dispatcher = LogDisptacher(self.context)
            mon_dispatcher.add_monitor(monitor)
            mon_dispatcher.start()
            context.addThread(mon_dispatcher)




        if context.hasRx():
            rxdir = context.hasRx()
            pfm = PacketFileMonitor(context, rxdir)
            pfm.start()
            context.addThread(pfm)
        else:
            rxdir = context.getWorkingDir()+"/rx"
            pfm = PacketFileMonitor(context, rxdir)
            pfm.start()
            context.addThread(pfm)


        if context.isSupernode() and not context.is_ghost():
            logging.debug("In supernode mode")

            http = HttpServer(context)
            http.start()
            context.addThread(http)

        for fd in listdir(context.getTaskDir()):
                src = context.getTaskDir()+"/"+fd
                if( src[-3:] == "pyc"):
                    continue

                if( src[-2:] == "py"):
                    dst = context.getWorkingDir()+"/seeds/"+fd
                else:
                    dst = context.getWorkingDir()+"/"+fd
                try:
                    shutil.copytree(src, dst)
                except OSError as exc: # python >2.5
                    if exc.errno == errno.ENOTDIR:
                        shutil.copy(src, dst)
                    else: raise


        if not context.isWithNoPlugins():# and not context.isSupernode():
            for ep in context.getEndpoints():
                if context.isZMQEndpoint(ep):
                    #ep[0] = PmkShared._get_nextport(ep[0], "TCP")
                    tcpm = ZMQPacketMonitor(context, zmq_context, ep[0])
                    tcpm.start()
                    context.addThread(tcpm)


            for sn in get_zmq_supernodes(PmkShared.SUPERNODES):
                if not str(sn).__contains__("127.0.0.1"):
                    #logging.debug("Subscribing to: "+sn)
                    #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, sn)
                    #zmqsub.start()
                    #context.addThread(zmqsub)
                    pass
                pass

            try:
                if not context.singleSeed():
                    onlyfiles = [ f for f in listdir(context.getTaskDir()) if isfile(join(context.getTaskDir(),f)) ]
                    for fl in onlyfiles:
                        fullpath = context.getTaskDir()+"/"+fl
                        modname = fl[:-3]
                        #ext = fl[-2:]
                        self.load_seed(fullpath)

                else:
                    seedfp = context.singleSeed()
                    self.load_seed(seedfp)


            except Exception as e:
                logging.error("Import error "+ str(e))
                pass




            for x in PmkSeed.iplugins.keys():
               klass = PmkSeed.iplugins[x]
               js = klass.getConfEntry()
               #logging.debug(js)
               context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
               #context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
               #context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")

            logging.debug("Registry dump: "+context.getProcGraph().dumpExternalRegistry())

            seedmonitor = PacketFileMonitor(context, context.getWorkingDir()+"seeds/", ext="py")
            seedmonitor.start()
            context.addThread(seedmonitor)

            for i in range(1,context.get_cores()+1):
                logging.info("Starting internal dispatch thread "+str(i))
                idispatch = InternalDispatch(context)
                idispatch.start()
                context.addThread(idispatch)


            #context.startDisplay()

            self._checkLocalPeers(zmq_context)

            ##############START TASKS WITH NO INPUTS#############################
            inj = Injector(context)
            inj.start()
            context.addThread(inj)
            #####################################################################

            if context.hasShell():
                cmdp = Shell(context)
                cmdp.cmdloop()
Exemplo n.º 6
0
 def run(self):
     PmkShared._ensure_dir(self.dir)
     tftp_server = tftpy.TftpServer(self.dir)
     tftp_server.listen('0.0.0.0', self.port)
Exemplo n.º 7
0
 def run(self):
     PmkShared._ensure_dir(self.dir)
     tftp_server = tftpy.TftpServer(self.dir)
     tftp_server.listen('0.0.0.0', self.port)
Exemplo n.º 8
0
    def startContext(self):
        context = self.context
        logging.info("Node assigned UID: " + context.getUuid())
        logging.info("Exec context: " + context.getExecContext())
        logging.info("Node bound to IP: " + context.get_local_ip())
        home = expanduser("~")
        wd = home + "/.pumpkin/" + context.getUuid() + "/"
        context.working_dir = wd

        PmkShared._ensure_dir(wd)
        PmkShared._ensure_dir(wd + "logs/")
        PmkShared._ensure_dir(wd + "seeds/")
        context.log_to_file()

        context.startPktShelve("PktStore")
        context.startPktShelve2("RemotePktStore")
        if context.is_speedy():
            logging.info("Running in speedy gonzales mode")

        logging.debug("Working directory: " + context.getWorkingDir())

        #context.openfiles.append(context.getWorkingDir())

        context = self.context
        zmq_context = self.zmq_context
        context.zmq_context = zmq_context
        #PmkShared.ZMQ_PUB_PORT  = PmkShared._get_nextport(ZMQ_PUB_PORT, "TCP")
        PmkShared.ZMQ_ENDPOINT_PORT = PmkShared._get_nextport(
            ZMQ_ENDPOINT_PORT, "TCP")
        logging.debug("ZMQ endpoint port: " + str(PmkShared.ZMQ_ENDPOINT_PORT))
        PmkShared.TFTP_FILE_SERVER_PORT = PmkShared._get_nextport(
            TFTP_FILE_SERVER_PORT, "UDP")

        context.setEndpoints()

        if not context.is_ghost():
            #zmqbc = ZMQBroadcaster(context, zmq_context, "tcp://*:"+str(PmkShared.ZMQ_PUB_PORT))
            #zmqbc = ZMQBroadcaster(context, zmq_context, context.get_our_pub_ep("tcp"))
            #zmqbc.start()
            #context.addThread(zmqbc)
            pass

        if context.with_broadcast():
            #Listen for UDP broadcasts on LAN
            udplisten = BroadcastListener(
                context, int(context.getAttributeValue().bcport), zmq_context)
            udplisten.start()
            context.addThread(udplisten)

            udpbc = Broadcaster(context,
                                int(context.getAttributeValue().bcport),
                                rate=context.get_broadcast_rate())
            udpbc.start()
            context.addThread(udpbc)

            zmqbc = ZMQBroadcaster(context, zmq_context,
                                   "tcp://*:" + str(PmkShared.ZMQ_PUB_PORT))
            zmqbc = ZMQBroadcaster(context, zmq_context,
                                   context.get_our_pub_ep("tcp"))
            zmqbc.start()
            context.addThread(zmqbc)
            pass

        #Local stuff to exploit multi-cores still needs testing

        #context.peers[context.getUuid()] = "/tmp/"+context.getUuid()+"-bcast"
        #local_peers = self._shelve_safe_open("/tmp/pumpkin")
        #local_peers[context.getUuid()] = "ipc:///tmp/"+context.getUuid()+"-bcast"
        #local_peers.close()
        # zmqbc = ZMQBroadcaster(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bcast")
        # context.openfiles.append("/tmp/"+context.getUuid()+"-bcast")
        # #context.openfiles.append("/tmp/pumpkin-bus")
        # zmqbc.start()
        # context.addThread(zmqbc)

        #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/"+context.getUuid()+"-bus")
        #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, "ipc:///tmp/pumpkin-bus")
        #zmqsub.start()
        #context.addThread(zmqsub)

        ftpdir = wd + 'tx/'

        if not context.is_ghost():
            #TFTP for sending files between pumpkins
            tftpserver = TftpServer(context, ftpdir,
                                    PmkShared.TFTP_FILE_SERVER_PORT)
            tftpserver.start()
            context.setFileDir(ftpdir)
            context.addThread(tftpserver)

            #FTP for user access of working directory
            ftpserver = FtpServer(context, context.getWorkingDir())
            ftpserver.start()
            context.addThread(ftpserver)

        edispatch = ExternalDispatch(context)
        context.setExternalDispatch(edispatch)
        edispatch.start()
        context.addThread(edispatch)

        if context.broadcast_rabbitmq():
            host, port, username, password, vhost = self.context.get_rabbitmq_cred(
            )
            credentials = pika.PlainCredentials(username, password)

            connection = pika.BlockingConnection(
                pika.ConnectionParameters(host=host,
                                          credentials=credentials,
                                          virtual_host=vhost))
            rabbitmq = RabbitMQMonitor(context, connection)
            context.set_rabbitmq(rabbitmq)

            q = context.get_group() + ":info"
            bunny = RabbitMQBroadcaster(context, exchange=q)
            bunny.start()
            context.addThread(bunny)

            bunnylistener = RabbitMQBroadcastSubscriber(context, exchange=q)
            bunnylistener.start()
            context.addThread(bunnylistener)

            logging.debug("Adding RabbitMQ queue: " + context.getUuid())
            rabbitmq.add_monitor_queue(context.getUuid())
            rabbitmq.add_monitor_queue(context.getUuid() + "ack")

            qm = context.get_group() + ":track"
            monitor = RabbitMqLog(self.context)
            monitor.connect(qm)

            mon_dispatcher = LogDisptacher(self.context)
            mon_dispatcher.add_monitor(monitor)
            mon_dispatcher.start()
            context.addThread(mon_dispatcher)

        if context.hasRx():
            rxdir = context.hasRx()
            pfm = PacketFileMonitor(context, rxdir)
            pfm.start()
            context.addThread(pfm)
        else:
            rxdir = context.getWorkingDir() + "/rx"
            pfm = PacketFileMonitor(context, rxdir)
            pfm.start()
            context.addThread(pfm)

        if context.isSupernode() and not context.is_ghost():
            logging.debug("In supernode mode")

            http = HttpServer(context)
            http.start()
            context.addThread(http)

        for fd in listdir(context.getTaskDir()):
            src = context.getTaskDir() + "/" + fd
            if (src[-3:] == "pyc"):
                continue

            if (src[-2:] == "py"):
                dst = context.getWorkingDir() + "/seeds/" + fd
            else:
                dst = context.getWorkingDir() + "/" + fd
            try:
                shutil.copytree(src, dst)
            except OSError as exc:  # python >2.5
                if exc.errno == errno.ENOTDIR:
                    shutil.copy(src, dst)
                else:
                    raise

        if not context.isWithNoPlugins():  # and not context.isSupernode():
            for ep in context.getEndpoints():
                if context.isZMQEndpoint(ep):
                    #ep[0] = PmkShared._get_nextport(ep[0], "TCP")
                    tcpm = ZMQPacketMonitor(context, zmq_context, ep[0])
                    tcpm.start()
                    context.addThread(tcpm)

            for sn in get_zmq_supernodes(PmkShared.SUPERNODES):
                if not str(sn).__contains__("127.0.0.1"):
                    #logging.debug("Subscribing to: "+sn)
                    #zmqsub = ZMQBroadcastSubscriber(context, zmq_context, sn)
                    #zmqsub.start()
                    #context.addThread(zmqsub)
                    pass
                pass

            try:
                if not context.singleSeed():
                    onlyfiles = [
                        f for f in listdir(context.getTaskDir())
                        if isfile(join(context.getTaskDir(), f))
                    ]
                    for fl in onlyfiles:
                        fullpath = context.getTaskDir() + "/" + fl
                        modname = fl[:-3]
                        #ext = fl[-2:]
                        self.load_seed(fullpath)

                else:
                    seedfp = context.singleSeed()
                    self.load_seed(seedfp)

            except Exception as e:
                logging.error("Import error " + str(e))
                pass

            for x in PmkSeed.iplugins.keys():
                klass = PmkSeed.iplugins[x]
                js = klass.getConfEntry()
                #logging.debug(js)
                context.getProcGraph().updateRegistry(json.loads(js),
                                                      loc="locallocal")
                #context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")
                #context.getProcGraph().updateRegistry(json.loads(js), loc="locallocal")

            logging.debug("Registry dump: " +
                          context.getProcGraph().dumpExternalRegistry())

            seedmonitor = PacketFileMonitor(context,
                                            context.getWorkingDir() + "seeds/",
                                            ext="py")
            seedmonitor.start()
            context.addThread(seedmonitor)

            for i in range(1, context.get_cores() + 1):
                logging.info("Starting internal dispatch thread " + str(i))
                idispatch = InternalDispatch(context)
                idispatch.start()
                context.addThread(idispatch)

            #context.startDisplay()

            self._checkLocalPeers(zmq_context)

            ##############START TASKS WITH NO INPUTS#############################
            inj = Injector(context)
            inj.start()
            context.addThread(inj)
            #####################################################################

            if context.hasShell():
                cmdp = Shell(context)
                cmdp.cmdloop()
Exemplo n.º 9
0
 def run(self):
     PmkShared._ensure_dir(self.dir)
     self.server.serve_forever()
Exemplo n.º 10
0
 def run(self):
     PmkShared._ensure_dir(self.dir)
     self.server.serve_forever()