def wait(self): local = self.local() # Wait for the connection to be established retrytime = 2.0 for spin in xrange(30): if self.status() != rspawn.RUNNING: self._logger.warn("FAILED TO CONNECT! %s", self) break # Connected? (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "cd %(home)s ; grep -a -c Connected capture" % dict(home=server.shell_escape(self.home_path)), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) proc.wait() if out.strip() == '1': break # At least listening? (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "cd %(home)s ; grep -a -c Listening capture" % dict(home=server.shell_escape(self.home_path)), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) proc.wait() time.sleep(min(30.0, retrytime)) retrytime *= 1.1 else: (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "cat %(home)s/capture" % dict(home=server.shell_escape(self.home_path)), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, retry=3, err_on_timeout=False) proc.wait() raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self, out, err)
def _make_home(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Make sure all the paths are created where # they have to be created for deployment # Also remove pidfile, if there is one. # Old pidfiles from previous runs can be troublesome. cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % { 'home' : server.shell_escape(self.home_path) } (out,err),proc = server.eintr_retry(server.popen_ssh_command)( cmd, host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, retry = 3 ) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)
def if_alive(self): name = self.if_name if name: local = self.local() for i in xrange(30): (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "ip link show %s >/dev/null 2>&1 && echo ALIVE || echo DEAD" % (name,), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) if proc.wait(): time.sleep(1) continue if out.strip() == 'DEAD': return False elif out.strip() == 'ALIVE': return True return False
def _make_home(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Make sure all the paths are created where # they have to be created for deployment # Also remove pidfile, if there is one. # Old pidfiles from previous runs can be troublesome. cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % { 'home': server.shell_escape(self.home_path) } (out, err), proc = server.eintr_retry(server.popen_ssh_command)( cmd, host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, retry=3) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % ( out, err, )
def is_alive(self, verbose = False): # Make sure all the paths are created where # they have to be created for deployment (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "echo 'ALIVE'", host = self.hostip, port = None, user = self.slicename, agent = None, ident_key = self.ident_path, server_key = self.server_key, timeout = 60, err_on_timeout = False, persistent = False ) if proc.wait(): if verbose: self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err) return False elif not err and out.strip() == 'ALIVE': return True else: if verbose: self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err) return False
def is_alive(self, verbose=False): # Make sure all the paths are created where # they have to be created for deployment (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "echo 'ALIVE'", host=self.hostip, port=None, user=self.slicename, agent=None, ident_key=self.ident_path, server_key=self.server_key, timeout=60, err_on_timeout=False, persistent=False) if proc.wait(): if verbose: self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err) return False elif not err and out.strip() == 'ALIVE': return True else: if verbose: self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err) return False
def if_alive(self): name = self.if_name if name: local = self.local() for i in xrange(30): (out, err), proc = server.eintr_retry( server.popen_ssh_command )("ip link show %s >/dev/null 2>&1 && echo ALIVE || echo DEAD" % (name, ), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) if proc.wait(): time.sleep(1) continue if out.strip() == 'DEAD': return False elif out.strip() == 'ALIVE': return True return False
def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None): (out,err),proc = server.popen_ssh_command( command, host = self.node.hostname, port = None, user = self.node.slicename, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key, timeout = timeout, retry = retry ) if server.eintr_retry(proc.wait)(): if not noerrors: raise RuntimeError, (out, err) return (out, err), proc
def if_up(self): # TODO!!! need to set the vif up with vsys/vif_up.in ... which # doesn't currently work. local = self.local() if local: (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "sudo -S bash -c 'kill -s USR2 %d'" % (self._pid, ), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) proc.wait()
def if_up(self): # TODO!!! need to set the vif up with vsys/vif_up.in ... which # doesn't currently work. local = self.local() if local: (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "sudo -S bash -c 'kill -s USR2 %d'" % (self._pid,), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) proc.wait()
def _popen_scp(self, src, dst, retry=3): while 1: try: (out, err), proc = server.popen_scp(src, dst, port=None, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key) if server.eintr_retry(proc.wait)(): raise RuntimeError, (out, err) return (out, err), proc except: if retry <= 0: raise else: retry -= 1
def if_name(self): if not self._if_name: # Inspect the trace to check the assigned iface local = self.local() if local: cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict( home=server.shell_escape(self.home_path)) for spin in xrange(30): (out, err), proc = server.eintr_retry(server.popen_ssh_command)( cmd, host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) if proc.wait(): self._logger.debug("if_name: failed cmd %s", cmd) time.sleep(1) continue out = out.strip() match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*", out) if match: self._if_name = match.group(1) break elif out: self._logger.debug( "if_name: %r does not match expected pattern from cmd %s", out, cmd) else: self._logger.debug("if_name: empty output from cmd %s", cmd) time.sleep(3) else: self._logger.warn("if_name: Could not get interface name") return self._if_name
def _popen_scp(self, src, dst, retry = 3): while 1: try: (out,err),proc = server.popen_scp( src, dst, port = None, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key ) if server.eintr_retry(proc.wait)(): raise RuntimeError, (out, err) return (out, err), proc except: if retry <= 0: raise else: retry -= 1
def _popen_ssh_command(self, command, retry=0, noerrors=False, timeout=None): (out, err), proc = server.popen_ssh_command(command, host=self.node.hostname, port=None, user=self.node.slicename, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key, timeout=timeout, retry=retry) if server.eintr_retry(proc.wait)(): if not noerrors: raise RuntimeError, (out, err) return (out, err), proc
def if_name(self): if not self._if_name: # Inspect the trace to check the assigned iface local = self.local() if local: cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict( home = server.shell_escape(self.home_path)) for spin in xrange(30): (out,err),proc = server.eintr_retry(server.popen_ssh_command)( cmd, host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) if proc.wait(): self._logger.debug("if_name: failed cmd %s", cmd) time.sleep(1) continue out = out.strip() match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*",out) if match: self._if_name = match.group(1) break elif out: self._logger.debug("if_name: %r does not match expected pattern from cmd %s", out, cmd) else: self._logger.debug("if_name: empty output from cmd %s", cmd) time.sleep(3) else: self._logger.warn("if_name: Could not get interface name") return self._if_name
def waitkill(self): interval = 1.0 for i in xrange(30): status = self.status() if status != rspawn.RUNNING: self._logger.info("Stopped %s", self) break time.sleep(interval) interval = min(30.0, interval * 1.1) else: self.kill(nowait=False) if self.if_name: for i in xrange(30): if not self.if_alive(): self._logger.info("Device down %s", self) break time.sleep(interval) interval = min(30.0, interval * 1.1) else: local = self.local() if local: # Forcibly shut down interface (out, err), proc = server.eintr_retry(server.popen_ssh_command)( "sudo -S bash -c 'echo %s > /vsys/vif_down.in'" % (self.if_name, ), host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=60, err_on_timeout=False) proc.wait()
def waitkill(self): interval = 1.0 for i in xrange(30): status = self.status() if status != rspawn.RUNNING: self._logger.info("Stopped %s", self) break time.sleep(interval) interval = min(30.0, interval * 1.1) else: self.kill(nowait=False) if self.if_name: for i in xrange(30): if not self.if_alive(): self._logger.info("Device down %s", self) break time.sleep(interval) interval = min(30.0, interval * 1.1) else: local = self.local() if local: # Forcibly shut down interface (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "sudo -S bash -c 'echo %s > /vsys/vif_down.in'" % (self.if_name,), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) proc.wait()
def wait(self): local = self.local() # Wait for the connection to be established retrytime = 2.0 for spin in xrange(30): if self.status() != rspawn.RUNNING: self._logger.warn("FAILED TO CONNECT! %s", self) break # Connected? (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "cd %(home)s ; grep -a -c Connected capture" % dict( home = server.shell_escape(self.home_path)), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) proc.wait() if out.strip() == '1': break # At least listening? (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "cd %(home)s ; grep -a -c Listening capture" % dict( home = server.shell_escape(self.home_path)), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, err_on_timeout = False ) proc.wait() time.sleep(min(30.0, retrytime)) retrytime *= 1.1 else: (out,err),proc = server.eintr_retry(server.popen_ssh_command)( "cat %(home)s/capture" % dict( home = server.shell_escape(self.home_path)), host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 60, retry = 3, err_on_timeout = False ) proc.wait() raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self,out,err)
def _install_scripts(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Install the tun_connect script and tunalloc utility from nepi.util import tunchannel from nepi.util import ipaddr2 sources = [ os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'), os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'), re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific ] if local.filter_module: filter_sources = filter( bool, map(str.strip, local.filter_module.module.split())) filter_module = filter_sources[0] # Translate paths to builtin sources for i, source in enumerate(filter_sources): if not os.path.exists(source): # Um... try the builtin folder source = os.path.join(os.path.dirname(__file__), "scripts", source) if os.path.exists(source): # Yep... replace filter_sources[i] = source sources.extend(set(filter_sources)) else: filter_module = None filter_sources = None dest = "%s@%s:%s" % ( local.node.slicename, local.node.hostname, os.path.join(self.home_path, '.'), ) (out, err), proc = server.eintr_retry(server.popen_scp)( sources, dest, ident_key=local.node.ident_path, server_key=local.node.server_key) if proc.wait(): raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % ( sources, out, err, ) # Make sure all dependencies are satisfied local.node.wait_dependencies() cmd = ( ("cd %(home)s && " "gcc -fPIC -shared tunalloc.c -o tunalloc.so && " "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && " "mkdir -p python-iovec && " "cd python-iovec && " "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. && " "cd .. " + (" && " "gcc -fPIC -shared %(sources)s -o %(module)s.so " % { 'module': os.path.basename(filter_module).rsplit('.', 1)[0], 'sources': ' '.join(map(os.path.basename, filter_sources)) } if filter_module is not None and filter_module.endswith('.c') else "") + (" && " "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && " "mkdir -p python-passfd && " "cd python-passfd && " "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. " if local.tun_proto == "fd" else "")) % { 'home': server.shell_escape(self.home_path), 'passfd_url': "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz", 'iovec_url': "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz", }) (out, err), proc = server.popen_ssh_command( cmd, host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=300) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % ( out, err, )
def _install_scripts(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Install the tun_connect script and tunalloc utility from nepi.util import tunchannel from nepi.util import ipaddr2 sources = [ os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'), os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'), re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific ] if local.filter_module: filter_sources = filter(bool,map(str.strip,local.filter_module.module.split())) filter_module = filter_sources[0] # Translate paths to builtin sources for i,source in enumerate(filter_sources): if not os.path.exists(source): # Um... try the builtin folder source = os.path.join(os.path.dirname(__file__), "scripts", source) if os.path.exists(source): # Yep... replace filter_sources[i] = source sources.extend(set(filter_sources)) else: filter_module = None filter_sources = None dest = "%s@%s:%s" % ( local.node.slicename, local.node.hostname, os.path.join(self.home_path,'.'),) (out,err),proc = server.eintr_retry(server.popen_scp)( sources, dest, ident_key = local.node.ident_path, server_key = local.node.server_key ) if proc.wait(): raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (sources, out,err,) # Make sure all dependencies are satisfied local.node.wait_dependencies() cmd = ( ( "cd %(home)s && " "gcc -fPIC -shared tunalloc.c -o tunalloc.so && " "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && " "mkdir -p python-iovec && " "cd python-iovec && " "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. && " "cd .. " + ( " && " "gcc -fPIC -shared %(sources)s -o %(module)s.so " % { 'module' : os.path.basename(filter_module).rsplit('.',1)[0], 'sources' : ' '.join(map(os.path.basename,filter_sources)) } if filter_module is not None and filter_module.endswith('.c') else "" ) + ( " && " "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && " "mkdir -p python-passfd && " "cd python-passfd && " "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. " if local.tun_proto == "fd" else "" ) ) % { 'home' : server.shell_escape(self.home_path), 'passfd_url' : "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz", 'iovec_url' : "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz", } ) (out,err),proc = server.popen_ssh_command( cmd, host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 300 ) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)