Example #1
0
    def wait(self):
        local = self.local()

        # Wait for the connection to be established
        retrytime = 2.0
        for spin in xrange(30):
            if self.status() != rspawn.RUNNING:
                self._logger.warn("FAILED TO CONNECT! %s", self)
                break

            # Connected?
            (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
                "cd %(home)s ; grep -a -c Connected capture" %
                dict(home=server.shell_escape(self.home_path)),
                host=local.node.hostname,
                port=None,
                user=local.node.slicename,
                agent=None,
                ident_key=local.node.ident_path,
                server_key=local.node.server_key,
                timeout=60,
                err_on_timeout=False)
            proc.wait()

            if out.strip() == '1':
                break

            # At least listening?
            (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
                "cd %(home)s ; grep -a -c Listening capture" %
                dict(home=server.shell_escape(self.home_path)),
                host=local.node.hostname,
                port=None,
                user=local.node.slicename,
                agent=None,
                ident_key=local.node.ident_path,
                server_key=local.node.server_key,
                timeout=60,
                err_on_timeout=False)
            proc.wait()

            time.sleep(min(30.0, retrytime))
            retrytime *= 1.1
        else:
            (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
                "cat %(home)s/capture" %
                dict(home=server.shell_escape(self.home_path)),
                host=local.node.hostname,
                port=None,
                user=local.node.slicename,
                agent=None,
                ident_key=local.node.ident_path,
                server_key=local.node.server_key,
                timeout=60,
                retry=3,
                err_on_timeout=False)
            proc.wait()

            raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self, out, err)
Example #2
0
 def _make_home(self):
     local = self.local()
     
     if not local:
         raise RuntimeError, "Lost reference to peering interfaces before launching"
     if not local.node:
         raise RuntimeError, "Unconnected TUN - missing node"
     
     # Make sure all the paths are created where 
     # they have to be created for deployment
     # Also remove pidfile, if there is one.
     # Old pidfiles from previous runs can be troublesome.
     cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % {
         'home' : server.shell_escape(self.home_path)
     }
     (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
         cmd,
         host = local.node.hostname,
         port = None,
         user = local.node.slicename,
         agent = None,
         ident_key = local.node.ident_path,
         server_key = local.node.server_key,
         timeout = 60,
         retry = 3
         )
     
     if proc.wait():
         raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)
Example #3
0
 def if_alive(self):
     name = self.if_name
     if name:
         local = self.local()
         for i in xrange(30):
             (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                 "ip link show %s >/dev/null 2>&1 && echo ALIVE || echo DEAD" % (name,),
                 host = local.node.hostname,
                 port = None,
                 user = local.node.slicename,
                 agent = None,
                 ident_key = local.node.ident_path,
                 server_key = local.node.server_key,
                 timeout = 60,
                 err_on_timeout = False
                 )
             
             if proc.wait():
                 time.sleep(1)
                 continue
             
             if out.strip() == 'DEAD':
                 return False
             elif out.strip() == 'ALIVE':
                 return True
     return False
Example #4
0
    def _make_home(self):
        local = self.local()

        if not local:
            raise RuntimeError, "Lost reference to peering interfaces before launching"
        if not local.node:
            raise RuntimeError, "Unconnected TUN - missing node"

        # Make sure all the paths are created where
        # they have to be created for deployment
        # Also remove pidfile, if there is one.
        # Old pidfiles from previous runs can be troublesome.
        cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % {
            'home': server.shell_escape(self.home_path)
        }
        (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
            cmd,
            host=local.node.hostname,
            port=None,
            user=local.node.slicename,
            agent=None,
            ident_key=local.node.ident_path,
            server_key=local.node.server_key,
            timeout=60,
            retry=3)

        if proc.wait():
            raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (
                out,
                err,
            )
Example #5
0
 def is_alive(self, verbose = False):
     # Make sure all the paths are created where 
     # they have to be created for deployment
     (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
         "echo 'ALIVE'",
         host = self.hostip,
         port = None,
         user = self.slicename,
         agent = None,
         ident_key = self.ident_path,
         server_key = self.server_key,
         timeout = 60,
         err_on_timeout = False,
         persistent = False
         )
     
     if proc.wait():
         if verbose:
             self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err)
         return False
     elif not err and out.strip() == 'ALIVE':
         return True
     else:
         if verbose:
             self._logger.warn("Unresponsive node %s got:\n%s%s", self.hostname, out, err)
         return False
Example #6
0
    def is_alive(self, verbose=False):
        # Make sure all the paths are created where
        # they have to be created for deployment
        (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
            "echo 'ALIVE'",
            host=self.hostip,
            port=None,
            user=self.slicename,
            agent=None,
            ident_key=self.ident_path,
            server_key=self.server_key,
            timeout=60,
            err_on_timeout=False,
            persistent=False)

        if proc.wait():
            if verbose:
                self._logger.warn("Unresponsive node %s got:\n%s%s",
                                  self.hostname, out, err)
            return False
        elif not err and out.strip() == 'ALIVE':
            return True
        else:
            if verbose:
                self._logger.warn("Unresponsive node %s got:\n%s%s",
                                  self.hostname, out, err)
            return False
Example #7
0
    def if_alive(self):
        name = self.if_name
        if name:
            local = self.local()
            for i in xrange(30):
                (out, err), proc = server.eintr_retry(
                    server.popen_ssh_command
                )("ip link show %s >/dev/null 2>&1 && echo ALIVE || echo DEAD"
                  % (name, ),
                  host=local.node.hostname,
                  port=None,
                  user=local.node.slicename,
                  agent=None,
                  ident_key=local.node.ident_path,
                  server_key=local.node.server_key,
                  timeout=60,
                  err_on_timeout=False)

                if proc.wait():
                    time.sleep(1)
                    continue

                if out.strip() == 'DEAD':
                    return False
                elif out.strip() == 'ALIVE':
                    return True
        return False
Example #8
0
    def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
        (out,err),proc = server.popen_ssh_command(
            command,
            host = self.node.hostname,
            port = None,
            user = self.node.slicename,
            agent = None,
            ident_key = self.node.ident_path,
            server_key = self.node.server_key,
            timeout = timeout,
            retry = retry
            )

        if server.eintr_retry(proc.wait)():
            if not noerrors:
                raise RuntimeError, (out, err)
        return (out, err), proc
Example #9
0
    def if_up(self):
        # TODO!!! need to set the vif up with vsys/vif_up.in ... which
        # doesn't currently work.
        local = self.local()

        if local:
            (out, err), proc = server.eintr_retry(server.popen_ssh_command)(
                "sudo -S bash -c 'kill -s USR2 %d'" % (self._pid, ),
                host=local.node.hostname,
                port=None,
                user=local.node.slicename,
                agent=None,
                ident_key=local.node.ident_path,
                server_key=local.node.server_key,
                timeout=60,
                err_on_timeout=False)
            proc.wait()
Example #10
0
 def if_up(self):
     # TODO!!! need to set the vif up with vsys/vif_up.in ... which 
     # doesn't currently work.
     local = self.local()
     
     if local:
         (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
             "sudo -S bash -c 'kill -s USR2 %d'" % (self._pid,),
             host = local.node.hostname,
             port = None,
             user = local.node.slicename,
             agent = None,
             ident_key = local.node.ident_path,
             server_key = local.node.server_key,
             timeout = 60,
             err_on_timeout = False
             )
         proc.wait()    
Example #11
0
    def _popen_scp(self, src, dst, retry=3):
        while 1:
            try:
                (out,
                 err), proc = server.popen_scp(src,
                                               dst,
                                               port=None,
                                               agent=None,
                                               ident_key=self.node.ident_path,
                                               server_key=self.node.server_key)

                if server.eintr_retry(proc.wait)():
                    raise RuntimeError, (out, err)
                return (out, err), proc
            except:
                if retry <= 0:
                    raise
                else:
                    retry -= 1
Example #12
0
    def if_name(self):
        if not self._if_name:
            # Inspect the trace to check the assigned iface
            local = self.local()
            if local:
                cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict(
                    home=server.shell_escape(self.home_path))
                for spin in xrange(30):
                    (out,
                     err), proc = server.eintr_retry(server.popen_ssh_command)(
                         cmd,
                         host=local.node.hostname,
                         port=None,
                         user=local.node.slicename,
                         agent=None,
                         ident_key=local.node.ident_path,
                         server_key=local.node.server_key,
                         timeout=60,
                         err_on_timeout=False)

                    if proc.wait():
                        self._logger.debug("if_name: failed cmd %s", cmd)
                        time.sleep(1)
                        continue

                    out = out.strip()

                    match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*", out)
                    if match:
                        self._if_name = match.group(1)
                        break
                    elif out:
                        self._logger.debug(
                            "if_name: %r does not match expected pattern from cmd %s",
                            out, cmd)
                    else:
                        self._logger.debug("if_name: empty output from cmd %s",
                                           cmd)
                    time.sleep(3)
                else:
                    self._logger.warn("if_name: Could not get interface name")
        return self._if_name
Example #13
0
    def _popen_scp(self, src, dst, retry = 3):
        while 1:
            try:
                (out,err),proc = server.popen_scp(
                    src,
                    dst, 
                    port = None,
                    agent = None,
                    ident_key = self.node.ident_path,
                    server_key = self.node.server_key
                    )

                if server.eintr_retry(proc.wait)():
                    raise RuntimeError, (out, err)
                return (out, err), proc
            except:
                if retry <= 0:
                    raise
                else:
                    retry -= 1
Example #14
0
    def _popen_ssh_command(self,
                           command,
                           retry=0,
                           noerrors=False,
                           timeout=None):
        (out,
         err), proc = server.popen_ssh_command(command,
                                               host=self.node.hostname,
                                               port=None,
                                               user=self.node.slicename,
                                               agent=None,
                                               ident_key=self.node.ident_path,
                                               server_key=self.node.server_key,
                                               timeout=timeout,
                                               retry=retry)

        if server.eintr_retry(proc.wait)():
            if not noerrors:
                raise RuntimeError, (out, err)
        return (out, err), proc
Example #15
0
 def if_name(self):
     if not self._if_name:
         # Inspect the trace to check the assigned iface
         local = self.local()
         if local:
             cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict(
                         home = server.shell_escape(self.home_path))
             for spin in xrange(30):
                 (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                     cmd,
                     host = local.node.hostname,
                     port = None,
                     user = local.node.slicename,
                     agent = None,
                     ident_key = local.node.ident_path,
                     server_key = local.node.server_key,
                     timeout = 60,
                     err_on_timeout = False
                     )
                 
                 if proc.wait():
                     self._logger.debug("if_name: failed cmd %s", cmd)
                     time.sleep(1)
                     continue
                 
                 out = out.strip()
                 
                 match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*",out)
                 if match:
                     self._if_name = match.group(1)
                     break
                 elif out:
                     self._logger.debug("if_name: %r does not match expected pattern from cmd %s", out, cmd)
                 else:
                     self._logger.debug("if_name: empty output from cmd %s", cmd)
                 time.sleep(3)
             else:
                 self._logger.warn("if_name: Could not get interface name")
     return self._if_name
Example #16
0
    def waitkill(self):
        interval = 1.0
        for i in xrange(30):
            status = self.status()
            if status != rspawn.RUNNING:
                self._logger.info("Stopped %s", self)
                break
            time.sleep(interval)
            interval = min(30.0, interval * 1.1)
        else:
            self.kill(nowait=False)

        if self.if_name:
            for i in xrange(30):
                if not self.if_alive():
                    self._logger.info("Device down %s", self)
                    break
                time.sleep(interval)
                interval = min(30.0, interval * 1.1)
            else:
                local = self.local()

                if local:
                    # Forcibly shut down interface
                    (out,
                     err), proc = server.eintr_retry(server.popen_ssh_command)(
                         "sudo -S bash -c 'echo %s > /vsys/vif_down.in'" %
                         (self.if_name, ),
                         host=local.node.hostname,
                         port=None,
                         user=local.node.slicename,
                         agent=None,
                         ident_key=local.node.ident_path,
                         server_key=local.node.server_key,
                         timeout=60,
                         err_on_timeout=False)
                    proc.wait()
Example #17
0
    def waitkill(self):
        interval = 1.0
        for i in xrange(30):
            status = self.status()
            if status != rspawn.RUNNING:
                self._logger.info("Stopped %s", self)
                break
            time.sleep(interval)
            interval = min(30.0, interval * 1.1)
        else:
            self.kill(nowait=False)

        if self.if_name:
            for i in xrange(30):
                if not self.if_alive():
                    self._logger.info("Device down %s", self)
                    break
                time.sleep(interval)
                interval = min(30.0, interval * 1.1)
            else:
                local = self.local()
                
                if local:
                    # Forcibly shut down interface
                    (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                        "sudo -S bash -c 'echo %s > /vsys/vif_down.in'" % (self.if_name,),
                        host = local.node.hostname,
                        port = None,
                        user = local.node.slicename,
                        agent = None,
                        ident_key = local.node.ident_path,
                        server_key = local.node.server_key,
                        timeout = 60,
                        err_on_timeout = False
                        )
                    proc.wait()
Example #18
0
    def wait(self):
        local = self.local()
        
        # Wait for the connection to be established
        retrytime = 2.0
        for spin in xrange(30):
            if self.status() != rspawn.RUNNING:
                self._logger.warn("FAILED TO CONNECT! %s", self)
                break
            
            # Connected?
            (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                "cd %(home)s ; grep -a -c Connected capture" % dict(
                    home = server.shell_escape(self.home_path)),
                host = local.node.hostname,
                port = None,
                user = local.node.slicename,
                agent = None,
                ident_key = local.node.ident_path,
                server_key = local.node.server_key,
                timeout = 60,
                err_on_timeout = False
                )
            proc.wait()

            if out.strip() == '1':
                break

            # At least listening?
            (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                "cd %(home)s ; grep -a -c Listening capture" % dict(
                    home = server.shell_escape(self.home_path)),
                host = local.node.hostname,
                port = None,
                user = local.node.slicename,
                agent = None,
                ident_key = local.node.ident_path,
                server_key = local.node.server_key,
                timeout = 60,
                err_on_timeout = False
                )
            proc.wait()

            time.sleep(min(30.0, retrytime))
            retrytime *= 1.1
        else:
            (out,err),proc = server.eintr_retry(server.popen_ssh_command)(
                "cat %(home)s/capture" % dict(
                    home = server.shell_escape(self.home_path)),
                host = local.node.hostname,
                port = None,
                user = local.node.slicename,
                agent = None,
                ident_key = local.node.ident_path,
                server_key = local.node.server_key,
                timeout = 60,
                retry = 3,
                err_on_timeout = False
                )
            proc.wait()

            raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self,out,err)
Example #19
0
    def _install_scripts(self):
        local = self.local()

        if not local:
            raise RuntimeError, "Lost reference to peering interfaces before launching"
        if not local.node:
            raise RuntimeError, "Unconnected TUN - missing node"

        # Install the tun_connect script and tunalloc utility
        from nepi.util import tunchannel
        from nepi.util import ipaddr2
        sources = [
            os.path.join(os.path.dirname(__file__), 'scripts',
                         'tun_connect.py'),
            os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'),
            re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__,
                   1),  # pyc/o files are version-specific
            re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__,
                   1),  # pyc/o files are version-specific
        ]
        if local.filter_module:
            filter_sources = filter(
                bool, map(str.strip, local.filter_module.module.split()))
            filter_module = filter_sources[0]

            # Translate paths to builtin sources
            for i, source in enumerate(filter_sources):
                if not os.path.exists(source):
                    # Um... try the builtin folder
                    source = os.path.join(os.path.dirname(__file__), "scripts",
                                          source)
                    if os.path.exists(source):
                        # Yep... replace
                        filter_sources[i] = source

            sources.extend(set(filter_sources))

        else:
            filter_module = None
            filter_sources = None
        dest = "%s@%s:%s" % (
            local.node.slicename,
            local.node.hostname,
            os.path.join(self.home_path, '.'),
        )
        (out, err), proc = server.eintr_retry(server.popen_scp)(
            sources,
            dest,
            ident_key=local.node.ident_path,
            server_key=local.node.server_key)

        if proc.wait():
            raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (
                sources,
                out,
                err,
            )

        # Make sure all dependencies are satisfied
        local.node.wait_dependencies()

        cmd = (
            ("cd %(home)s && "
             "gcc -fPIC -shared tunalloc.c -o tunalloc.so && "
             "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && "
             "mkdir -p python-iovec && "
             "cd python-iovec && "
             "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && "
             "python setup.py build && "
             "python setup.py install --install-lib .. && "
             "cd .. " +
             (" && "
              "gcc -fPIC -shared %(sources)s -o %(module)s.so " % {
                  'module': os.path.basename(filter_module).rsplit('.', 1)[0],
                  'sources': ' '.join(map(os.path.basename, filter_sources))
              } if filter_module is not None and filter_module.endswith('.c')
              else "") +
             (" && "
              "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && "
              "mkdir -p python-passfd && "
              "cd python-passfd && "
              "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && "
              "python setup.py build && "
              "python setup.py install --install-lib .. "
              if local.tun_proto == "fd" else "")) %
            {
                'home':
                server.shell_escape(self.home_path),
                'passfd_url':
                "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz",
                'iovec_url':
                "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz",
            })
        (out, err), proc = server.popen_ssh_command(
            cmd,
            host=local.node.hostname,
            port=None,
            user=local.node.slicename,
            agent=None,
            ident_key=local.node.ident_path,
            server_key=local.node.server_key,
            timeout=300)

        if proc.wait():
            raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (
                out,
                err,
            )
Example #20
0
    def _install_scripts(self):
        local = self.local()
        
        if not local:
            raise RuntimeError, "Lost reference to peering interfaces before launching"
        if not local.node:
            raise RuntimeError, "Unconnected TUN - missing node"
        
        # Install the tun_connect script and tunalloc utility
        from nepi.util import tunchannel
        from nepi.util import ipaddr2
        sources = [
            os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'),
            os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'),
            re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific
            re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific
        ]
        if local.filter_module:
            filter_sources = filter(bool,map(str.strip,local.filter_module.module.split()))
            filter_module = filter_sources[0]
            
            # Translate paths to builtin sources
            for i,source in enumerate(filter_sources):
                if not os.path.exists(source):
                    # Um... try the builtin folder
                    source = os.path.join(os.path.dirname(__file__), "scripts", source)
                    if os.path.exists(source):
                        # Yep... replace
                        filter_sources[i] = source

            sources.extend(set(filter_sources))
                
        else:
            filter_module = None
            filter_sources = None
        dest = "%s@%s:%s" % (
            local.node.slicename, local.node.hostname, 
            os.path.join(self.home_path,'.'),)
        (out,err),proc = server.eintr_retry(server.popen_scp)(
            sources,
            dest,
            ident_key = local.node.ident_path,
            server_key = local.node.server_key
            )
    
        if proc.wait():
            raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (sources, out,err,)
        
        # Make sure all dependencies are satisfied
        local.node.wait_dependencies()

        cmd = ( (
            "cd %(home)s && "
            "gcc -fPIC -shared tunalloc.c -o tunalloc.so && "
            
            "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && "
            "mkdir -p python-iovec && "
            "cd python-iovec && "
            "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && "
            "python setup.py build && "
            "python setup.py install --install-lib .. && "
            "cd .. "
            
            + ( " && "
                "gcc -fPIC -shared %(sources)s -o %(module)s.so " % {
                   'module' : os.path.basename(filter_module).rsplit('.',1)[0],
                   'sources' : ' '.join(map(os.path.basename,filter_sources))
                }
                
                if filter_module is not None and filter_module.endswith('.c')
                else ""
            )
            
            + ( " && "
                "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && "
                "mkdir -p python-passfd && "
                "cd python-passfd && "
                "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && "
                "python setup.py build && "
                "python setup.py install --install-lib .. "
                
                if local.tun_proto == "fd" 
                else ""
            ) 
          )
        % {
            'home' : server.shell_escape(self.home_path),
            'passfd_url' : "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz",
            'iovec_url' : "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz",
        } )
        (out,err),proc = server.popen_ssh_command(
            cmd,
            host = local.node.hostname,
            port = None,
            user = local.node.slicename,
            agent = None,
            ident_key = local.node.ident_path,
            server_key = local.node.server_key,
            timeout = 300
            )
        
        if proc.wait():
            raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)