class WorkerServer(object):
"""Manages the Worker
The WorkerServer class connects to the nameserver and registers
itself with the MaxiNetManager instance. It is used by the Cluster instances
to start mininet instances, manage the ssh daemon and run commands etc.
Attributes:
logger: logging instance
mnManager: instance of class MininetManager which is used to create mininet
instances
sshManager: instance of class SSH_Manager which is used to manage the ssh
daemon.
ssh_folder: folder which holds configuration files for the ssh daemon.
ip: ip address of Worker
"""
def __init__(self):
self._ns = None
self._pyrodaemon = None
self.logger = logging.getLogger(__name__)
self._manager = None
self.mnManager = MininetManager()
self.sshManager = None
self.ssh_folder = tempfile.mkdtemp()
atexit.register(subprocess.call, ["rm", "-rf", self.ssh_folder])
logging.basicConfig(level=logging.DEBUG)
self.ip = None
self._shutdown = False
#Pyro4.config.COMMTIMEOUT = 2
#for frontend
self._ip = None
self._port = None
self._password = None
self._looping_thread = None
def exit_handler(self, signal, frame):
# I have absolutely no clue why but without this print atexit sometimes
# doesn't seem to wait for called functions to finish...
print "exiting..."
self._shutdown = True
sys.exit()
@Pyro4.expose
def monitorFrontend(self):
""" function to monitor if the frontend is still alive.
if not, try to reconnect.
"""
while (not self._shutdown):
try:
self._manager.getStatus()
except:
if self._ip != None:
#self.ip as an indicator that this worker was connected to the frontend once.
print "Trying to reconnect to FrontendServer..."
try:
try:
self._pyrodaemon.unregister(self)
except:
pass
try:
self._pyrodaemon.unregister(self.mnManager)
except:
pass
try:
self._pyrodaemon.unregister(self.sshManager)
except:
pass
try:
self._pyrodaemon.shutdown()
except:
pass
try:
self._pyrodaemon.close()
except:
pass
self.start(self._ip, self._port, self._password)
except Exception as e:
traceback.print_exc(e)
pass
pass
time.sleep(5)
@Pyro4.expose
def start(self, ip, port, password, retry=float("inf")):
"""Start WorkerServer and ssh daemon and connect to nameserver."""
self.logger.info("starting up and connecting to %s:%d" % (ip, port))
#store for reconnection attempts
self._ip = ip
self._port = port
self._password = password
#Pyro4.config.HMAC_KEY = password
tries = 1
self._ns = None
while not self._ns:
try:
self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
except Pyro4.errors.NamingError:
if tries < retry:
self.logger.warn(
"Unable to locate Nameserver. Trying again in 5 seconds..."
)
time.sleep(5)
tries += 1
else:
self.logger.error("Unable to locate Nameserver.")
sys.exit()
self.config = Pyro4.Proxy(self._ns.lookup("config"))
self.config._pyroHmacKey = password
self.ip = self.config.get_worker_ip(self.get_hostname())
if (not self.ip):
self.ip = Tools.guess_ip()
if not self.config.has_section(self.get_hostname()):
self.config.add_section(self.get_hostname())
self.config.set(self.get_hostname(), "ip", self.ip)
self.logger.warn(
"""FrontendServer did not know IP of this host (check configuration for hostname).
Guessed: %s""" % self.ip)
self.logger.info("configuring and starting ssh daemon...")
self.sshManager = SSH_Manager(folder=self.ssh_folder,
ip=self.ip,
port=self.config.get_sshd_port(),
user=self.config.get("all", "sshuser"))
self.sshManager.start_sshd()
self._pyrodaemon = Pyro4.Daemon(host=self.ip)
self._pyrodaemon._pyroHmacKey = password
uri = self._pyrodaemon.register(self)
self._ns.register(self._get_pyroname(), uri)
uri = self._pyrodaemon.register(self.mnManager)
self._ns.register(self._get_pyroname() + ".mnManager", uri)
uri = self._pyrodaemon.register(self.sshManager)
self._ns.register(self._get_pyroname() + ".sshManager", uri)
atexit.register(self._stop)
self.logger.info("looking for manager application...")
manager_uri = self._ns.lookup("MaxiNetManager")
if (manager_uri):
self._manager = Pyro4.Proxy(manager_uri)
self._manager._pyroHmacKey = self._password
self.logger.info("signing in...")
if (self._manager.worker_signin(self._get_pyroname(),
self.get_hostname())):
self.logger.info("done. Entering requestloop.")
self._started = True
self._looping_thread = threading.Thread(
target=self._pyrodaemon.requestLoop)
self._looping_thread.daemon = True
self._looping_thread.start()
else:
self.logger.error("signin failed.")
else:
self.logger.error("no manager found.")
def _get_pyroname(self):
return "MaxiNetWorker_%s" % self.get_hostname()
@Pyro4.expose
def get_hostname(self):
return subprocess.check_output(["hostname"]).strip()
def _stop(self):
self.logger.info("signing out...")
if (self._manager):
self._manager.worker_signout(self.get_hostname())
self.logger.info("shutting down...")
self._ns.remove(self._get_pyroname())
self._ns.remove(self._get_pyroname() + ".mnManager")
self._pyrodaemon.unregister(self)
self._pyrodaemon.unregister(self.mnManager)
self._pyrodaemon.unregister(self.sshManager)
self._pyrodaemon.shutdown()
self._pyrodaemon.close()
@Pyro4.expose
def remoteShutdown(self):
self._pyrodaemon.shutdown()
@Pyro4.expose
def stop(self):
(signedin,
assigned) = self._manager.get_worker_status(self.get_hostname())
if (assigned):
self.logger.warn(
"can't shut down as worker is still assigned to id %d" %
assigned)
return False
else:
self._stop()
return True
@Pyro4.expose
def check_output(self, cmd):
"""Run cmd on Worker and return output
Args:
cmd: command to call with optional parameters
Returns:
Shell output of command
"""
self.logger.debug("Executing %s" % cmd)
return subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT).strip()
@Pyro4.expose
def script_check_output(self, cmd):
"""Call MaxiNet Script and return output
Args:
cmd: name of script to call
Returns:
Shell output of script
"""
# Prefix command by our worker directory
cmd = Tools.get_script_dir() + cmd
return self.check_output(cmd)
@Pyro4.expose
def run_cmd(self, command):
"""Call command (blocking)
Args:
command: command to call with optional parameters
"""
subprocess.call(command, shell=True)
@Pyro4.expose
def daemonize(self, cmd):
"""Call command (non-blocking)
Args:
command: command to call with optional parameters
"""
p = subprocess.Popen(cmd, shell=True)
atexit.register(p.terminate)
@Pyro4.expose
def daemonize_script(self, script, args):
"""Call MaxiNet Script (non-blocking)
Args:
cmd: name of script to call
"""
cmd = Tools.get_script_dir() + script + " " + args
p = subprocess.Popen(cmd, shell=True)
atexit.register(p.terminate)
class WorkerServer(object):
"""Manages the Worker
The WorkerServer class connects to the nameserver and registers
itself with the MaxiNetManager instance. It is used by the Cluster instances
to start mininet instances, manage the ssh daemon and run commands etc.
Attributes:
logger: logging instance
mnManager: instance of class MininetManager which is used to create mininet
instances
sshManager: instance of class SSH_Manager which is used to manage the ssh
daemon.
ssh_folder: folder which holds configuration files for the ssh daemon.
ip: ip address of Worker
"""
def __init__(self):
self._ns = None
self._pyrodaemon = None
self.logger = logging.getLogger(__name__)
self._manager = None
self.mnManager = MininetManager()
self.sshManager = None
self.ssh_folder = tempfile.mkdtemp()
atexit.register(subprocess.call, ["rm", "-rf", self.ssh_folder])
logging.basicConfig(level=logging.DEBUG)
self.ip = None
self._shutdown = False
#Pyro4.config.COMMTIMEOUT = 2
#for frontend
self._ip = None
self._port = None
self._password = None
self._looping_thread = None
def exit_handler(self, signal, frame):
# I have absolutely no clue why but without this print atexit sometimes
# doesn't seem to wait for called functions to finish...
print "exiting..."
self._shutdown = True
sys.exit()
@Pyro4.expose
def monitorFrontend(self):
""" function to monitor if the frontend is still alive.
if not, try to reconnect.
"""
while(not self._shutdown):
try:
self._manager.getStatus()
except:
if self._ip != None:
#self.ip as an indicator that this worker was connected to the frontend once.
print "Trying to reconnect to FrontendServer..."
try:
try:
self._pyrodaemon.unregister(self)
except:
pass
try:
self._pyrodaemon.unregister(self.mnManager)
except:
pass
try:
self._pyrodaemon.unregister(self.sshManager)
except:
pass
try:
self._pyrodaemon.shutdown()
except:
pass
try:
self._pyrodaemon.close()
except:
pass
self.start(self._ip, self._port, self._password)
except Exception as e:
traceback.print_exc(e)
pass
pass
time.sleep(5)
@Pyro4.expose
def start(self, ip, port, password, retry=float("inf")):
"""Start WorkerServer and ssh daemon and connect to nameserver."""
self.logger.info("starting up and connecting to %s:%d"
% (ip, port))
#store for reconnection attempts
self._ip = ip
self._port = port
self._password = password
#Pyro4.config.HMAC_KEY = password
tries=1
self._ns = None
while not self._ns:
try:
self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
except Pyro4.errors.NamingError:
if tries < retry:
self.logger.warn("Unable to locate Nameserver. Trying again in 5 seconds...")
time.sleep(5)
tries += 1
else:
self.logger.error("Unable to locate Nameserver.")
sys.exit()
self.config = Pyro4.Proxy(self._ns.lookup("config"))
self.config._pyroHmacKey=password
self.ip = self.config.get_worker_ip(self.get_hostname())
if(not self.ip):
self.ip = Tools.guess_ip()
if not self.config.has_section(self.get_hostname()):
self.config.add_section(self.get_hostname())
self.config.set(self.get_hostname(), "ip", self.ip)
self.logger.warn("""FrontendServer did not know IP of this host.
Guessed: %s""" % self.ip)
self.logger.info("configuring and starting ssh daemon...")
self.sshManager = SSH_Manager(folder=self.ssh_folder, ip=self.ip, port=self.config.get_sshd_port(), user=self.config.get("all", "sshuser"))
self.sshManager.start_sshd()
self._pyrodaemon = Pyro4.Daemon(host=self.ip)
self._pyrodaemon._pyroHmacKey=password
uri = self._pyrodaemon.register(self)
self._ns.register(self._get_pyroname(), uri)
uri = self._pyrodaemon.register(self.mnManager)
self._ns.register(self._get_pyroname()+".mnManager", uri)
uri = self._pyrodaemon.register(self.sshManager)
self._ns.register(self._get_pyroname()+".sshManager", uri)
atexit.register(self._stop)
self.logger.info("looking for manager application...")
manager_uri = self._ns.lookup("MaxiNetManager")
if(manager_uri):
self._manager = Pyro4.Proxy(manager_uri)
self._manager._pyroHmacKey=self._password
self.logger.info("signing in...")
if(self._manager.worker_signin(self._get_pyroname(), self.get_hostname())):
self.logger.info("done. Entering requestloop.")
self._started = True
self._looping_thread = threading.Thread(target=self._pyrodaemon.requestLoop)
self._looping_thread.daemon = True
self._looping_thread.start()
else:
self.logger.error("signin failed.")
else:
self.logger.error("no manager found.")
def _get_pyroname(self):
return "MaxiNetWorker_%s" % self.get_hostname()
@Pyro4.expose
def get_hostname(self):
return subprocess.check_output(["hostname"]).strip()
def _stop(self):
self.logger.info("signing out...")
if(self._manager):
self._manager.worker_signout(self.get_hostname())
self.logger.info("shutting down...")
self._ns.remove(self._get_pyroname())
self._ns.remove(self._get_pyroname()+".mnManager")
self._pyrodaemon.unregister(self)
self._pyrodaemon.unregister(self.mnManager)
self._pyrodaemon.unregister(self.sshManager)
self._pyrodaemon.shutdown()
self._pyrodaemon.close()
@Pyro4.expose
def remoteShutdown(self):
self._pyrodaemon.shutdown()
@Pyro4.expose
def stop(self):
(signedin, assigned) = self._manager.get_worker_status(self.get_hostname())
if(assigned):
self.logger.warn("can't shut down as worker is still assigned to id %d" % assigned)
return False
else:
self._stop()
return True
@Pyro4.expose
def check_output(self, cmd):
"""Run cmd on Worker and return output
Args:
cmd: command to call with optional parameters
Returns:
Shell output of command
"""
self.logger.debug("Executing %s" % cmd)
return subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT).strip()
@Pyro4.expose
def script_check_output(self, cmd):
"""Call MaxiNet Script and return output
Args:
cmd: name of script to call
Returns:
Shell output of script
"""
# Prefix command by our worker directory
cmd = Tools.get_script_dir() + cmd
return self.check_output(cmd)
@Pyro4.expose
def run_cmd(self, command):
"""Call command (blocking)
Args:
command: command to call with optional parameters
"""
subprocess.call(command, shell=True)
@Pyro4.expose
def daemonize(self, cmd):
"""Call command (non-blocking)
Args:
command: command to call with optional parameters
"""
p = subprocess.Popen(cmd, shell=True)
atexit.register(p.terminate)
@Pyro4.expose
def daemonize_script(self, script, args):
"""Call MaxiNet Script (non-blocking)
Args:
cmd: name of script to call
"""
cmd = Tools.get_script_dir()+script+" "+args
p = subprocess.Popen(cmd, shell=True)
atexit.register(p.terminate)
def start(self, ip, port, password, retry=float("inf")):
"""Start WorkerServer and ssh daemon and connect to nameserver."""
self.logger.info("starting up and connecting to %s:%d" % (ip, port))
#store for reconnection attempts
self._ip = ip
self._port = port
self._password = password
#Pyro4.config.HMAC_KEY = password
tries = 1
self._ns = None
while not self._ns:
try:
self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
except Pyro4.errors.NamingError:
if tries < retry:
self.logger.warn(
"Unable to locate Nameserver. Trying again in 5 seconds..."
)
time.sleep(5)
tries += 1
else:
self.logger.error("Unable to locate Nameserver.")
sys.exit()
self.config = Pyro4.Proxy(self._ns.lookup("config"))
self.config._pyroHmacKey = password
self.ip = self.config.get_worker_ip(self.get_hostname())
if (not self.ip):
self.ip = Tools.guess_ip()
if not self.config.has_section(self.get_hostname()):
self.config.add_section(self.get_hostname())
self.config.set(self.get_hostname(), "ip", self.ip)
self.logger.warn(
"""FrontendServer did not know IP of this host (check configuration for hostname).
Guessed: %s""" % self.ip)
self.logger.info("configuring and starting ssh daemon...")
self.sshManager = SSH_Manager(folder=self.ssh_folder,
ip=self.ip,
port=self.config.get_sshd_port(),
user=self.config.get("all", "sshuser"))
self.sshManager.start_sshd()
self._pyrodaemon = Pyro4.Daemon(host=self.ip)
self._pyrodaemon._pyroHmacKey = password
uri = self._pyrodaemon.register(self)
self._ns.register(self._get_pyroname(), uri)
uri = self._pyrodaemon.register(self.mnManager)
self._ns.register(self._get_pyroname() + ".mnManager", uri)
uri = self._pyrodaemon.register(self.sshManager)
self._ns.register(self._get_pyroname() + ".sshManager", uri)
atexit.register(self._stop)
self.logger.info("looking for manager application...")
manager_uri = self._ns.lookup("MaxiNetManager")
if (manager_uri):
self._manager = Pyro4.Proxy(manager_uri)
self._manager._pyroHmacKey = self._password
self.logger.info("signing in...")
if (self._manager.worker_signin(self._get_pyroname(),
self.get_hostname())):
self.logger.info("done. Entering requestloop.")
self._started = True
self._looping_thread = threading.Thread(
target=self._pyrodaemon.requestLoop)
self._looping_thread.daemon = True
self._looping_thread.start()
else:
self.logger.error("signin failed.")
else:
self.logger.error("no manager found.")
def start(self, ip, port, password, retry=float("inf")):
"""Start WorkerServer and ssh daemon and connect to nameserver."""
self.logger.info("starting up and connecting to %s:%d"
% (ip, port))
#store for reconnection attempts
self._ip = ip
self._port = port
self._password = password
#Pyro4.config.HMAC_KEY = password
tries=1
self._ns = None
while not self._ns:
try:
self._ns = Pyro4.locateNS(ip, port, hmac_key=password)
except Pyro4.errors.NamingError:
if tries < retry:
self.logger.warn("Unable to locate Nameserver. Trying again in 5 seconds...")
time.sleep(5)
tries += 1
else:
self.logger.error("Unable to locate Nameserver.")
sys.exit()
self.config = Pyro4.Proxy(self._ns.lookup("config"))
self.config._pyroHmacKey=password
self.ip = self.config.get_worker_ip(self.get_hostname())
if(not self.ip):
self.ip = Tools.guess_ip()
if not self.config.has_section(self.get_hostname()):
self.config.add_section(self.get_hostname())
self.config.set(self.get_hostname(), "ip", self.ip)
self.logger.warn("""FrontendServer did not know IP of this host.
Guessed: %s""" % self.ip)
self.logger.info("configuring and starting ssh daemon...")
self.sshManager = SSH_Manager(folder=self.ssh_folder, ip=self.ip, port=self.config.get_sshd_port(), user=self.config.get("all", "sshuser"))
self.sshManager.start_sshd()
self._pyrodaemon = Pyro4.Daemon(host=self.ip)
self._pyrodaemon._pyroHmacKey=password
uri = self._pyrodaemon.register(self)
self._ns.register(self._get_pyroname(), uri)
uri = self._pyrodaemon.register(self.mnManager)
self._ns.register(self._get_pyroname()+".mnManager", uri)
uri = self._pyrodaemon.register(self.sshManager)
self._ns.register(self._get_pyroname()+".sshManager", uri)
atexit.register(self._stop)
self.logger.info("looking for manager application...")
manager_uri = self._ns.lookup("MaxiNetManager")
if(manager_uri):
self._manager = Pyro4.Proxy(manager_uri)
self._manager._pyroHmacKey=self._password
self.logger.info("signing in...")
if(self._manager.worker_signin(self._get_pyroname(), self.get_hostname())):
self.logger.info("done. Entering requestloop.")
self._started = True
self._looping_thread = threading.Thread(target=self._pyrodaemon.requestLoop)
self._looping_thread.daemon = True
self._looping_thread.start()
else:
self.logger.error("signin failed.")
else:
self.logger.error("no manager found.")