Exemple #1
0
class WorkerServer(object):
    """Manages the Worker

    The WorkerServer class connects to the nameserver and registers
    itself with the MaxiNetManager instance. It is used by the Cluster instances
    to start mininet instances, manage the ssh daemon and run commands etc.

    Attributes:
        logger: logging instance
        mnManager: instance of class MininetManager which is used to create mininet
            instances
        sshManager: instance of class SSH_Manager which is used to manage the ssh
            daemon.
        ssh_folder: folder which holds configuration files for the ssh daemon.
        ip: ip address of Worker
    """
    def __init__(self):
        self._ns = None
        self._pyrodaemon = None
        self.logger = logging.getLogger(__name__)
        self._manager = None
        self.mnManager = MininetManager()
        self.sshManager = None
        self.ssh_folder = tempfile.mkdtemp()
        atexit.register(subprocess.call, ["rm", "-rf", self.ssh_folder])
        logging.basicConfig(level=logging.DEBUG)
        self.ip = None
        self._shutdown = False
        #Pyro4.config.COMMTIMEOUT = 2

        #for frontend
        self._ip = None
        self._port = None
        self._password = None

        self._looping_thread = None

    def exit_handler(self, signal, frame):
        # I have absolutely no clue why but without this print atexit sometimes
        # doesn't seem to wait for called functions to finish...
        print "exiting..."
        self._shutdown = True
        sys.exit()

    @Pyro4.expose
    def monitorFrontend(self):
        """ function to monitor if the frontend is still alive.
            if not, try to reconnect.
        """
        while (not self._shutdown):
            try:
                self._manager.getStatus()
            except:
                if self._ip != None:
                    #self.ip as an indicator that this worker was connected to the frontend once.
                    print "Trying to reconnect to FrontendServer..."
                    try:
                        try:
                            self._pyrodaemon.unregister(self)
                        except:
                            pass
                        try:
                            self._pyrodaemon.unregister(self.mnManager)
                        except:
                            pass
                        try:
                            self._pyrodaemon.unregister(self.sshManager)
                        except:
                            pass
                        try:
                            self._pyrodaemon.shutdown()
                        except:
                            pass
                        try:
                            self._pyrodaemon.close()
                        except:
                            pass
                        self.start(self._ip, self._port, self._password)
                    except Exception as e:
                        traceback.print_exc(e)
                        pass
                pass
            time.sleep(5)

    @Pyro4.expose
    def start(self, ip, port, password, retry=float("inf")):
        """Start WorkerServer and ssh daemon and connect to nameserver."""
        self.logger.info("starting up and connecting to  %s:%d" % (ip, port))

        #store for reconnection attempts
        self._ip = ip
        self._port = port
        self._password = password

        #Pyro4.config.HMAC_KEY = password
        tries = 1
        self._ns = None
        while not self._ns:
            try:
                self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
            except Pyro4.errors.NamingError:
                if tries < retry:
                    self.logger.warn(
                        "Unable to locate Nameserver. Trying again in 5 seconds..."
                    )
                    time.sleep(5)
                    tries += 1
                else:
                    self.logger.error("Unable to locate Nameserver.")
                    sys.exit()
        self.config = Pyro4.Proxy(self._ns.lookup("config"))
        self.config._pyroHmacKey = password
        self.ip = self.config.get_worker_ip(self.get_hostname())
        if (not self.ip):
            self.ip = Tools.guess_ip()
            if not self.config.has_section(self.get_hostname()):
                self.config.add_section(self.get_hostname())
            self.config.set(self.get_hostname(), "ip", self.ip)
            self.logger.warn(
                """FrontendServer did not know IP of this host (check configuration for hostname).
                             Guessed: %s""" % self.ip)
        self.logger.info("configuring and starting ssh daemon...")
        self.sshManager = SSH_Manager(folder=self.ssh_folder,
                                      ip=self.ip,
                                      port=self.config.get_sshd_port(),
                                      user=self.config.get("all", "sshuser"))
        self.sshManager.start_sshd()
        self._pyrodaemon = Pyro4.Daemon(host=self.ip)
        self._pyrodaemon._pyroHmacKey = password
        uri = self._pyrodaemon.register(self)
        self._ns.register(self._get_pyroname(), uri)
        uri = self._pyrodaemon.register(self.mnManager)
        self._ns.register(self._get_pyroname() + ".mnManager", uri)
        uri = self._pyrodaemon.register(self.sshManager)
        self._ns.register(self._get_pyroname() + ".sshManager", uri)
        atexit.register(self._stop)
        self.logger.info("looking for manager application...")
        manager_uri = self._ns.lookup("MaxiNetManager")
        if (manager_uri):
            self._manager = Pyro4.Proxy(manager_uri)
            self._manager._pyroHmacKey = self._password
            self.logger.info("signing in...")
            if (self._manager.worker_signin(self._get_pyroname(),
                                            self.get_hostname())):
                self.logger.info("done. Entering requestloop.")
                self._started = True
                self._looping_thread = threading.Thread(
                    target=self._pyrodaemon.requestLoop)
                self._looping_thread.daemon = True
                self._looping_thread.start()
            else:
                self.logger.error("signin failed.")
        else:
            self.logger.error("no manager found.")

    def _get_pyroname(self):
        return "MaxiNetWorker_%s" % self.get_hostname()

    @Pyro4.expose
    def get_hostname(self):
        return subprocess.check_output(["hostname"]).strip()

    def _stop(self):
        self.logger.info("signing out...")
        if (self._manager):
            self._manager.worker_signout(self.get_hostname())
        self.logger.info("shutting down...")
        self._ns.remove(self._get_pyroname())
        self._ns.remove(self._get_pyroname() + ".mnManager")
        self._pyrodaemon.unregister(self)
        self._pyrodaemon.unregister(self.mnManager)
        self._pyrodaemon.unregister(self.sshManager)
        self._pyrodaemon.shutdown()
        self._pyrodaemon.close()

    @Pyro4.expose
    def remoteShutdown(self):
        self._pyrodaemon.shutdown()

    @Pyro4.expose
    def stop(self):
        (signedin,
         assigned) = self._manager.get_worker_status(self.get_hostname())
        if (assigned):
            self.logger.warn(
                "can't shut down as worker is still assigned to id %d" %
                assigned)
            return False
        else:
            self._stop()
            return True

    @Pyro4.expose
    def check_output(self, cmd):
        """Run cmd on Worker and return output

        Args:
            cmd: command to call with optional parameters

        Returns:
            Shell output of command
        """
        self.logger.debug("Executing %s" % cmd)
        return subprocess.check_output(cmd,
                                       shell=True,
                                       stderr=subprocess.STDOUT).strip()

    @Pyro4.expose
    def script_check_output(self, cmd):
        """Call MaxiNet Script and return output

        Args:
            cmd: name of script to call
        Returns:
            Shell output of script
        """
        # Prefix command by our worker directory
        cmd = Tools.get_script_dir() + cmd
        return self.check_output(cmd)

    @Pyro4.expose
    def run_cmd(self, command):
        """Call command (blocking)

        Args:
            command: command to call with optional parameters
        """
        subprocess.call(command, shell=True)

    @Pyro4.expose
    def daemonize(self, cmd):
        """Call command (non-blocking)

        Args:
            command: command to call with optional parameters
        """
        p = subprocess.Popen(cmd, shell=True)
        atexit.register(p.terminate)

    @Pyro4.expose
    def daemonize_script(self, script, args):
        """Call MaxiNet Script (non-blocking)

        Args:
            cmd: name of script to call
        """
        cmd = Tools.get_script_dir() + script + " " + args
        p = subprocess.Popen(cmd, shell=True)
        atexit.register(p.terminate)
Exemple #2
0
class WorkerServer(object):
    """Manages the Worker

    The WorkerServer class connects to the nameserver and registers
    itself with the MaxiNetManager instance. It is used by the Cluster instances
    to start mininet instances, manage the ssh daemon and run commands etc.

    Attributes:
        logger: logging instance
        mnManager: instance of class MininetManager which is used to create mininet
            instances
        sshManager: instance of class SSH_Manager which is used to manage the ssh
            daemon.
        ssh_folder: folder which holds configuration files for the ssh daemon.
        ip: ip address of Worker
    """
    def __init__(self):
        self._ns = None
        self._pyrodaemon = None
        self.logger = logging.getLogger(__name__)
        self._manager = None
        self.mnManager = MininetManager()
        self.sshManager = None
        self.ssh_folder = tempfile.mkdtemp()
        atexit.register(subprocess.call, ["rm", "-rf", self.ssh_folder])
        logging.basicConfig(level=logging.DEBUG)
        self.ip = None
        self._shutdown = False
        #Pyro4.config.COMMTIMEOUT = 2

        #for frontend
        self._ip = None
        self._port = None
        self._password = None

        self._looping_thread = None


    def exit_handler(self, signal, frame):
        # I have absolutely no clue why but without this print atexit sometimes
        # doesn't seem to wait for called functions to finish...
        print "exiting..."
        self._shutdown = True
        sys.exit()

    @Pyro4.expose
    def monitorFrontend(self):
        """ function to monitor if the frontend is still alive.
            if not, try to reconnect.
        """
        while(not self._shutdown):
            try:
                self._manager.getStatus()
            except:
                if self._ip != None:
                    #self.ip as an indicator that this worker was connected to the frontend once.
                    print "Trying to reconnect to FrontendServer..."
                    try:
                        try:
                            self._pyrodaemon.unregister(self)
                        except:
                            pass
                        try:
                            self._pyrodaemon.unregister(self.mnManager)
                        except:
                            pass
                        try:
                            self._pyrodaemon.unregister(self.sshManager)
                        except:
                            pass
                        try:
                            self._pyrodaemon.shutdown()
                        except:
                            pass
                        try:
                            self._pyrodaemon.close()
                        except:
                            pass
                        self.start(self._ip, self._port, self._password)
                    except Exception as e:
                        traceback.print_exc(e)
                        pass
                pass
            time.sleep(5)

    @Pyro4.expose
    def start(self, ip, port, password, retry=float("inf")):
        """Start WorkerServer and ssh daemon and connect to nameserver."""
        self.logger.info("starting up and connecting to  %s:%d"
                         % (ip, port))

        #store for reconnection attempts
        self._ip = ip
        self._port = port
        self._password = password

        #Pyro4.config.HMAC_KEY = password
        tries=1
        self._ns = None
        while not self._ns:
            try:
                self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
            except Pyro4.errors.NamingError:
                if tries < retry:
                    self.logger.warn("Unable to locate Nameserver. Trying again in 5 seconds...")
                    time.sleep(5)
                    tries += 1
                else:
                    self.logger.error("Unable to locate Nameserver.")
                    sys.exit()
        self.config = Pyro4.Proxy(self._ns.lookup("config"))
        self.config._pyroHmacKey=password
        self.ip = self.config.get_worker_ip(self.get_hostname())
        if(not self.ip):
            self.ip = Tools.guess_ip()
            if not self.config.has_section(self.get_hostname()):
                self.config.add_section(self.get_hostname())
            self.config.set(self.get_hostname(), "ip", self.ip)
            self.logger.warn("""FrontendServer did not know IP of this host.
                             Guessed: %s""" % self.ip)
        self.logger.info("configuring and starting ssh daemon...")
        self.sshManager = SSH_Manager(folder=self.ssh_folder, ip=self.ip, port=self.config.get_sshd_port(), user=self.config.get("all", "sshuser"))
        self.sshManager.start_sshd()
        self._pyrodaemon = Pyro4.Daemon(host=self.ip)
        self._pyrodaemon._pyroHmacKey=password
        uri = self._pyrodaemon.register(self)
        self._ns.register(self._get_pyroname(), uri)
        uri = self._pyrodaemon.register(self.mnManager)
        self._ns.register(self._get_pyroname()+".mnManager", uri)
        uri = self._pyrodaemon.register(self.sshManager)
        self._ns.register(self._get_pyroname()+".sshManager", uri)
        atexit.register(self._stop)
        self.logger.info("looking for manager application...")
        manager_uri = self._ns.lookup("MaxiNetManager")
        if(manager_uri):
            self._manager = Pyro4.Proxy(manager_uri)
            self._manager._pyroHmacKey=self._password
            self.logger.info("signing in...")
            if(self._manager.worker_signin(self._get_pyroname(), self.get_hostname())):
                self.logger.info("done. Entering requestloop.")
                self._started = True
                self._looping_thread = threading.Thread(target=self._pyrodaemon.requestLoop)
                self._looping_thread.daemon = True
                self._looping_thread.start()
            else:
                self.logger.error("signin failed.")
        else:
            self.logger.error("no manager found.")

    def _get_pyroname(self):
        return "MaxiNetWorker_%s" % self.get_hostname()

    @Pyro4.expose
    def get_hostname(self):
        return subprocess.check_output(["hostname"]).strip()

    def _stop(self):
        self.logger.info("signing out...")
        if(self._manager):
            self._manager.worker_signout(self.get_hostname())
        self.logger.info("shutting down...")
        self._ns.remove(self._get_pyroname())
        self._ns.remove(self._get_pyroname()+".mnManager")
        self._pyrodaemon.unregister(self)
        self._pyrodaemon.unregister(self.mnManager)
        self._pyrodaemon.unregister(self.sshManager)
        self._pyrodaemon.shutdown()
        self._pyrodaemon.close()

    @Pyro4.expose
    def remoteShutdown(self):
        self._pyrodaemon.shutdown()


    @Pyro4.expose
    def stop(self):
        (signedin, assigned) = self._manager.get_worker_status(self.get_hostname())
        if(assigned):
            self.logger.warn("can't shut down as worker is still assigned to id %d" % assigned)
            return False
        else:
            self._stop()
            return True

    @Pyro4.expose
    def check_output(self, cmd):
        """Run cmd on Worker and return output

        Args:
            cmd: command to call with optional parameters

        Returns:
            Shell output of command
        """
        self.logger.debug("Executing %s" % cmd)
        return subprocess.check_output(cmd, shell=True,
                                       stderr=subprocess.STDOUT).strip()

    @Pyro4.expose
    def script_check_output(self, cmd):
        """Call MaxiNet Script and return output

        Args:
            cmd: name of script to call
        Returns:
            Shell output of script
        """
        # Prefix command by our worker directory
        cmd = Tools.get_script_dir() + cmd
        return self.check_output(cmd)

    @Pyro4.expose
    def run_cmd(self, command):
        """Call command (blocking)

        Args:
            command: command to call with optional parameters
        """
        subprocess.call(command, shell=True)

    @Pyro4.expose
    def daemonize(self, cmd):
        """Call command (non-blocking)

        Args:
            command: command to call with optional parameters
        """
        p = subprocess.Popen(cmd, shell=True)
        atexit.register(p.terminate)

    @Pyro4.expose
    def daemonize_script(self, script, args):
        """Call MaxiNet Script (non-blocking)

        Args:
            cmd: name of script to call
        """
        cmd = Tools.get_script_dir()+script+" "+args
        p = subprocess.Popen(cmd, shell=True)
        atexit.register(p.terminate)
Exemple #3
0
    def start(self, ip, port, password, retry=float("inf")):
        """Start WorkerServer and ssh daemon and connect to nameserver."""
        self.logger.info("starting up and connecting to  %s:%d" % (ip, port))

        #store for reconnection attempts
        self._ip = ip
        self._port = port
        self._password = password

        #Pyro4.config.HMAC_KEY = password
        tries = 1
        self._ns = None
        while not self._ns:
            try:
                self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
            except Pyro4.errors.NamingError:
                if tries < retry:
                    self.logger.warn(
                        "Unable to locate Nameserver. Trying again in 5 seconds..."
                    )
                    time.sleep(5)
                    tries += 1
                else:
                    self.logger.error("Unable to locate Nameserver.")
                    sys.exit()
        self.config = Pyro4.Proxy(self._ns.lookup("config"))
        self.config._pyroHmacKey = password
        self.ip = self.config.get_worker_ip(self.get_hostname())
        if (not self.ip):
            self.ip = Tools.guess_ip()
            if not self.config.has_section(self.get_hostname()):
                self.config.add_section(self.get_hostname())
            self.config.set(self.get_hostname(), "ip", self.ip)
            self.logger.warn(
                """FrontendServer did not know IP of this host (check configuration for hostname).
                             Guessed: %s""" % self.ip)
        self.logger.info("configuring and starting ssh daemon...")
        self.sshManager = SSH_Manager(folder=self.ssh_folder,
                                      ip=self.ip,
                                      port=self.config.get_sshd_port(),
                                      user=self.config.get("all", "sshuser"))
        self.sshManager.start_sshd()
        self._pyrodaemon = Pyro4.Daemon(host=self.ip)
        self._pyrodaemon._pyroHmacKey = password
        uri = self._pyrodaemon.register(self)
        self._ns.register(self._get_pyroname(), uri)
        uri = self._pyrodaemon.register(self.mnManager)
        self._ns.register(self._get_pyroname() + ".mnManager", uri)
        uri = self._pyrodaemon.register(self.sshManager)
        self._ns.register(self._get_pyroname() + ".sshManager", uri)
        atexit.register(self._stop)
        self.logger.info("looking for manager application...")
        manager_uri = self._ns.lookup("MaxiNetManager")
        if (manager_uri):
            self._manager = Pyro4.Proxy(manager_uri)
            self._manager._pyroHmacKey = self._password
            self.logger.info("signing in...")
            if (self._manager.worker_signin(self._get_pyroname(),
                                            self.get_hostname())):
                self.logger.info("done. Entering requestloop.")
                self._started = True
                self._looping_thread = threading.Thread(
                    target=self._pyrodaemon.requestLoop)
                self._looping_thread.daemon = True
                self._looping_thread.start()
            else:
                self.logger.error("signin failed.")
        else:
            self.logger.error("no manager found.")
Exemple #4
0
    def start(self, ip, port, password, retry=float("inf")):
        """Start WorkerServer and ssh daemon and connect to nameserver."""
        self.logger.info("starting up and connecting to  %s:%d"
                         % (ip, port))

        #store for reconnection attempts
        self._ip = ip
        self._port = port
        self._password = password

        #Pyro4.config.HMAC_KEY = password
        tries=1
        self._ns = None
        while not self._ns:
            try:
                self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
            except Pyro4.errors.NamingError:
                if tries < retry:
                    self.logger.warn("Unable to locate Nameserver. Trying again in 5 seconds...")
                    time.sleep(5)
                    tries += 1
                else:
                    self.logger.error("Unable to locate Nameserver.")
                    sys.exit()
        self.config = Pyro4.Proxy(self._ns.lookup("config"))
        self.config._pyroHmacKey=password
        self.ip = self.config.get_worker_ip(self.get_hostname())
        if(not self.ip):
            self.ip = Tools.guess_ip()
            if not self.config.has_section(self.get_hostname()):
                self.config.add_section(self.get_hostname())
            self.config.set(self.get_hostname(), "ip", self.ip)
            self.logger.warn("""FrontendServer did not know IP of this host.
                             Guessed: %s""" % self.ip)
        self.logger.info("configuring and starting ssh daemon...")
        self.sshManager = SSH_Manager(folder=self.ssh_folder, ip=self.ip, port=self.config.get_sshd_port(), user=self.config.get("all", "sshuser"))
        self.sshManager.start_sshd()
        self._pyrodaemon = Pyro4.Daemon(host=self.ip)
        self._pyrodaemon._pyroHmacKey=password
        uri = self._pyrodaemon.register(self)
        self._ns.register(self._get_pyroname(), uri)
        uri = self._pyrodaemon.register(self.mnManager)
        self._ns.register(self._get_pyroname()+".mnManager", uri)
        uri = self._pyrodaemon.register(self.sshManager)
        self._ns.register(self._get_pyroname()+".sshManager", uri)
        atexit.register(self._stop)
        self.logger.info("looking for manager application...")
        manager_uri = self._ns.lookup("MaxiNetManager")
        if(manager_uri):
            self._manager = Pyro4.Proxy(manager_uri)
            self._manager._pyroHmacKey=self._password
            self.logger.info("signing in...")
            if(self._manager.worker_signin(self._get_pyroname(), self.get_hostname())):
                self.logger.info("done. Entering requestloop.")
                self._started = True
                self._looping_thread = threading.Thread(target=self._pyrodaemon.requestLoop)
                self._looping_thread.daemon = True
                self._looping_thread.start()
            else:
                self.logger.error("signin failed.")
        else:
            self.logger.error("no manager found.")