Exemple #1
0
    def _do_launch_build(self):
        script = "bash ./nepi-build.sh"
        if self._master_passphrase:
            script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
                server.shell_escape(self._master_passphrase),
                script
            )
        (out,err),proc = rspawn.remote_spawn(
            script,
            pidfile = 'build-pid',
            home = self.home_path,
            stdin = '/dev/null',
            stdout = 'buildlog',
            stderr = rspawn.STDOUT,
            
            host = self.node.hostname,
            port = None,
            user = self.node.slicename,
            agent = None,
            ident_key = self.node.ident_path,
            server_key = self.node.server_key,
            hostip = self.node.hostip,
            )
        
        if proc.wait():
            if self.check_bad_host(out, err):
                self.node.blacklist()
            raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
        
        
        pid = ppid = None
        delay = 1.0
        for i in xrange(5):
            pidtuple = rspawn.remote_check_pid(
                os.path.join(self.home_path,'build-pid'),
                host = self.node.hostip,
                port = None,
                user = self.node.slicename,
                agent = None,
                ident_key = self.node.ident_path,
                server_key = self.node.server_key,
                hostip = self.node.hostip
                )
            
            if pidtuple:
                pid, ppid = pidtuple
                self._build_pid, self._build_ppid = pidtuple
                break
            else:
                time.sleep(delay)
                delay = min(30,delay*1.2)
        else:
            raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)

        self._logger.info("Deploying %s at %s", self, self.node.hostname)
Exemple #2
0
    def _do_launch_build(self):
        script = "bash ./nepi-build.sh"
        if self._master_passphrase:
            script = "NEPI_MASTER_PASSPHRASE=%s %s" % (server.shell_escape(
                self._master_passphrase), script)
        (out, err), proc = rspawn.remote_spawn(
            script,
            pidfile='build-pid',
            home=self.home_path,
            stdin='/dev/null',
            stdout='buildlog',
            stderr=rspawn.STDOUT,
            host=self.node.hostname,
            port=None,
            user=self.node.slicename,
            agent=None,
            ident_key=self.node.ident_path,
            server_key=self.node.server_key,
            hostip=self.node.hostip,
        )

        if proc.wait():
            if self.check_bad_host(out, err):
                self.node.blacklist()
            raise RuntimeError, "Failed to set up build slave %s: %s %s" % (
                self.home_path,
                out,
                err,
            )

        pid = ppid = None
        delay = 1.0
        for i in xrange(5):
            pidtuple = rspawn.remote_check_pid(os.path.join(
                self.home_path, 'build-pid'),
                                               host=self.node.hostip,
                                               port=None,
                                               user=self.node.slicename,
                                               agent=None,
                                               ident_key=self.node.ident_path,
                                               server_key=self.node.server_key,
                                               hostip=self.node.hostip)

            if pidtuple:
                pid, ppid = pidtuple
                self._build_pid, self._build_ppid = pidtuple
                break
            else:
                time.sleep(delay)
                delay = min(30, delay * 1.2)
        else:
            raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (
                self.home_path, )

        self._logger.info("Deploying %s at %s", self, self.node.hostname)
Exemple #3
0
class Application(Dependency):
    """
    An application also has dependencies, but also a command to be ran and monitored.
    
    It adds the output of that command as traces.
    """

    TRACES = ('stdout', 'stderr', 'buildlog', 'output')

    def __init__(self, api=None):
        super(Application, self).__init__(api)

        # Attributes
        self.command = None
        self.sudo = False

        self.stdin = None
        self.stdout = None
        self.stderr = None
        self.output = None

        # Those are filled when the app is started
        #   Having both pid and ppid makes it harder
        #   for pid rollover to induce tracking mistakes
        self._started = False
        self._pid = None
        self._ppid = None

        # Do not add to the python path of nodes
        self.add_to_path = False

    def __str__(self):
        return "%s<command:%s%s>" % (
            self.__class__.__name__,
            "sudo " if self.sudo else "",
            self.command,
        )

    def start(self):
        self._logger.info("Starting %s", self)

        # Create shell script with the command
        # This way, complex commands and scripts can be ran seamlessly
        # sync files
        command = cStringIO.StringIO()
        command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (':'.join([
            "${HOME}/" + server.shell_escape(s) for s in self.node.pythonpath
        ])))
        command.write('export PATH=$PATH:%s\n' % (':'.join([
            "${HOME}/" + server.shell_escape(s) for s in self.node.pythonpath
        ])))
        if self.node.env:
            for envkey, envvals in self.node.env.iteritems():
                for envval in envvals:
                    command.write('export %s=%s\n' % (envkey, envval))
        command.write(self.command)
        command.seek(0)

        try:
            self._popen_scp(
                command, '%s@%s:%s' % (self.node.slicename, self.node.hostname,
                                       os.path.join(self.home_path, "app.sh")))
        except RuntimeError, e:
            raise RuntimeError, "Failed to set up application: %s %s" \
                    % (e.args[0], e.args[1],)

        # Start process in a "daemonized" way, using nohup and heavy
        # stdin/out redirection to avoid connection issues
        (out, err), proc = rspawn.remote_spawn(
            self._replace_paths("bash ./app.sh"),
            pidfile='./pid',
            home=self.home_path,
            stdin='stdin' if self.stdin is not None else '/dev/null',
            stdout='stdout' if self.stdout else '/dev/null',
            stderr='stderr' if self.stderr else '/dev/null',
            sudo=self.sudo,
            host=self.node.hostname,
            port=None,
            user=self.node.slicename,
            agent=None,
            ident_key=self.node.ident_path,
            server_key=self.node.server_key)

        if proc.wait():
            if self.check_bad_host(out, err):
                self.node.blacklist()
            raise RuntimeError, "Failed to set up application: %s %s" % (
                out,
                err,
            )

        self._started = True
Exemple #4
0
class CCNxDaemon(Application):
    """
    An application also has dependencies, but also a command to be ran and monitored.
    
    It adds the output of that command as traces.
    """
    def __init__(self, api=None):
        super(CCNxDaemon, self).__init__(api)

        # Attributes
        self.ccnLocalPort = None
        self.ccnRoutes = None
        self.ccnxVersion = "0.7.1"
        self.repository = False

        #self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
        self.ccnx_sources = "http://www.ccnx.org/releases/ccnx-%s.tar.gz"
        self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'

        self.ccnx_build = (
            " ( "
            "  cd .. && "
            "  test -d ccnx-src/build/bin "
            " ) || ( "
            # Not working, rebuild
            "("
            " mkdir -p ccnx-src && "
            " wget -q -c -O ccnx-src.tar.gz %(ccnx_source_url)s &&"
            " tar xf ccnx-src.tar.gz --strip-components=1 -C ccnx-src "
            ") && "
            "cd ccnx-src && "
            "./configure && make"
            " )") % dict(ccnx_source_url=server.shell_escape(
                self.ccnx_sources % self.ccnxVersion), )

        self.ccnx_install = (" ( "
                             "  test -d ${BUILD}/ccnx-src/bin && "
                             "  cp -r ${BUILD}/ccnx-src/bin ${SOURCES}"
                             " )")

        self.env['PATH'] = "$PATH:${SOURCES}/bin"

    def setup(self):
        # setting ccn sources
        if not self.build:
            self.build = self.ccnx_build

        if not self.install:
            self.install = self.ccnx_install

        super(CCNxDaemon, self).setup()

    def start(self):
        self.command = ""
        if self.ccnLocalPort:
            self.command = "export CCN_LOCAL_PORT=%s ; " % self.ccnLocalPort
        self.command += " ccndstart "

        # configure ccn routes
        if self.ccnRoutes:
            routes = self.ccnRoutes.split("|")

            if self.ccnLocalPort:
                routes = map(
                    lambda route: "%s %s" % (route, self.ccnLocalPort)
                    if _ccnre.match(route) else route, routes)

            routes = map(lambda route: "ccndc add %s" % route, routes)

            routescmd = " ; ".join(routes)
            self.command += " ; "
            self.command += routescmd

            if self.repository:
                self.command += " ; ccnr "

        # Start will be invoked in prestart step
        super(CCNxDaemon, self).start()

    def kill(self):
        self._logger.info("Killing %s", self)

        command = "${SOURCES}/bin/ccndstop"

        if self.ccnLocalPort:
            self.command = "export CCN_LOCAL_PORT=%s; %s" % (self.ccnLocalPort,
                                                             command)

        cmd = self._replace_paths(command)
        command = cStringIO.StringIO()
        command.write(cmd)
        command.seek(0)

        try:
            self._popen_scp(
                command,
                '%s@%s:%s' % (self.node.slicename, self.node.hostname,
                              os.path.join(self.home_path, "kill.sh")))
        except RuntimeError, e:
            raise RuntimeError, "Failed to kill ccndxdaemon: %s %s" \
                    % (e.args[0], e.args[1],)

        script = "bash ./kill.sh"
        (out, err), proc = rspawn.remote_spawn(
            script,
            pidfile='kill-pid',
            home=self.home_path,
            stdin='/dev/null',
            stdout='killlog',
            stderr=rspawn.STDOUT,
            host=self.node.hostname,
            port=None,
            user=self.node.slicename,
            agent=None,
            ident_key=self.node.ident_path,
            server_key=self.node.server_key,
            hostip=self.node.hostip,
        )

        if proc.wait():
            raise RuntimeError, "Failed to kill cnnxdaemon: %s %s" % (
                out,
                err,
            )

        super(CCNxDaemon, self).kill()
Exemple #5
0
    def launch(self, check_proto):
        peer = self.peer()
        local = self.local()

        if not peer or not local:
            raise RuntimeError, "Lost reference to peering interfaces before launching"

        peer_port = peer.tun_port
        peer_addr = peer.tun_addr
        peer_proto = peer.tun_proto
        peer_cipher = peer.tun_cipher

        local_port = self.port
        local_cap = local.capture
        self._address = local_addr = local.address
        self._netprefix = local_mask = local.netprefix
        local_snat = local.snat
        local_txq = local.txqueuelen
        self._pointopoint = local_p2p = local.pointopoint
        local_cipher = local.tun_cipher
        local_mcast = local.multicast
        local_bwlim = local.bwlimit
        local_mcastfwd = local.multicast_forwarder

        if not local_p2p and hasattr(peer, 'address'):
            self._pointopoint = local_p2p = peer.address

        if check_proto != peer_proto:
            raise RuntimeError, "Peering protocol mismatch: %s != %s" % (
                check_proto, peer_proto)

        if local_cipher != peer_cipher:
            raise RuntimeError, "Peering cipher mismatch: %s != %s" % (
                local_cipher, peer_cipher)

        if check_proto == 'gre' and local_cipher.lower() != 'plain':
            raise RuntimeError, "Misconfigured TUN: %s - GRE tunnels do not support encryption. Got %s, you MUST use PLAIN" % (
                local,
                local_cipher,
            )

        if local.filter_module:
            if check_proto not in ('udp', 'tcp'):
                raise RuntimeError, "Miscofnigured TUN: %s - filtered tunnels only work with udp or tcp links" % (
                    local, )
            filter_module = filter(
                bool, map(str.strip, local.filter_module.module.split()))
            filter_module = os.path.join('.',
                                         os.path.basename(filter_module[0]))
            if filter_module.endswith('.c'):
                filter_module = filter_module.rsplit('.', 1)[0] + '.so'
            filter_args = local.filter_module.args
        else:
            filter_module = None
            filter_args = None

        args = [
            "python",
            "tun_connect.py",
            "-m",
            str(self.mode),
            "-t",
            str(check_proto),
            "-A",
            str(local_addr),
            "-M",
            str(local_mask),
            "-C",
            str(local_cipher),
        ]

        if check_proto == 'fd':
            passfd_arg = str(peer_addr)
            if passfd_arg.startswith('\x00'):
                # cannot shell_encode null characters :(
                passfd_arg = "base64:" + base64.b64encode(passfd_arg)
            else:
                passfd_arg = '$HOME/' + server.shell_escape(passfd_arg)
            args.extend(["--pass-fd", passfd_arg])
        elif check_proto == 'gre':
            if self.cross_slice:
                args.extend(["-K", str(self.key.strip('='))])

            args.extend([
                "-a",
                str(peer_addr),
            ])
        # both udp and tcp
        else:
            args.extend([
                "-P",
                str(local_port), "-p",
                str(peer_port), "-a",
                str(peer_addr), "-k",
                str(self.key)
            ])

        if local_snat:
            args.append("-S")
        if local_p2p:
            args.extend(("-Z", str(local_p2p)))
        if local_txq:
            args.extend(("-Q", str(local_txq)))
        if not local_cap:
            args.append("-N")
        elif local_cap == 'pcap':
            args.extend(('-c', 'pcap'))
        if local_bwlim:
            args.extend(("-b", str(local_bwlim * 1024)))
        if filter_module:
            args.extend(("--filter", filter_module))
        if filter_args:
            args.extend(("--filter-args", filter_args))
        if local_mcast and local_mcastfwd:
            args.extend(("--multicast-forwarder", local_mcastfwd))

        self._logger.info("Starting %s", self)

        self._make_home()
        self._install_scripts()

        # Start process in a "daemonized" way, using nohup and heavy
        # stdin/out redirection to avoid connection issues
        (out,
         err), proc = rspawn.remote_spawn(" ".join(args),
                                          pidfile='./pid',
                                          home=self.home_path,
                                          stdin='/dev/null',
                                          stdout='capture',
                                          stderr=rspawn.STDOUT,
                                          sudo=True,
                                          host=local.node.hostname,
                                          port=None,
                                          user=local.node.slicename,
                                          agent=None,
                                          ident_key=local.node.ident_path,
                                          server_key=local.node.server_key)

        if proc.wait():
            raise RuntimeError, "Failed to set up TUN: %s %s" % (
                out,
                err,
            )

        self._started = True
Exemple #6
0
    def launch(self, check_proto):
        peer = self.peer()
        local = self.local()
        
        if not peer or not local:
            raise RuntimeError, "Lost reference to peering interfaces before launching"
        
        peer_port = peer.tun_port
        peer_addr = peer.tun_addr
        peer_proto = peer.tun_proto
        peer_cipher = peer.tun_cipher
        
        local_port = self.port
        local_cap  = local.capture
        self._address = local_addr = local.address
        self._netprefix = local_mask = local.netprefix
        local_snat = local.snat
        local_txq  = local.txqueuelen
        self._pointopoint = local_p2p  = local.pointopoint
        local_cipher=local.tun_cipher
        local_mcast= local.multicast
        local_bwlim= local.bwlimit
        local_mcastfwd = local.multicast_forwarder
        
        if not local_p2p and hasattr(peer, 'address'):
            self._pointopoint = local_p2p = peer.address

        if check_proto != peer_proto:
            raise RuntimeError, "Peering protocol mismatch: %s != %s" % (check_proto, peer_proto)
        
        if local_cipher != peer_cipher:
            raise RuntimeError, "Peering cipher mismatch: %s != %s" % (local_cipher, peer_cipher)
        
        if check_proto == 'gre' and local_cipher.lower() != 'plain':
            raise RuntimeError, "Misconfigured TUN: %s - GRE tunnels do not support encryption. Got %s, you MUST use PLAIN" % (local, local_cipher,)

        if local.filter_module:
            if check_proto not in ('udp', 'tcp'):
                raise RuntimeError, "Miscofnigured TUN: %s - filtered tunnels only work with udp or tcp links" % (local,)
            filter_module = filter(bool,map(str.strip,local.filter_module.module.split()))
            filter_module = os.path.join('.',os.path.basename(filter_module[0]))
            if filter_module.endswith('.c'):
                filter_module = filter_module.rsplit('.',1)[0] + '.so'
            filter_args = local.filter_module.args
        else:
            filter_module = None
            filter_args = None
        
        args = ["python", "tun_connect.py", 
            "-m", str(self.mode),
            "-t", str(check_proto),
            "-A", str(local_addr),
            "-M", str(local_mask),
            "-C", str(local_cipher),
            ]
        
        if check_proto == 'fd':
            passfd_arg = str(peer_addr)
            if passfd_arg.startswith('\x00'):
                # cannot shell_encode null characters :(
                passfd_arg = "base64:"+base64.b64encode(passfd_arg)
            else:
                passfd_arg = '$HOME/'+server.shell_escape(passfd_arg)
            args.extend([
                "--pass-fd", passfd_arg
            ])
        elif check_proto == 'gre':
            if self.cross_slice:
                args.extend([
                    "-K", str(self.key.strip('='))
                ])

            args.extend([
                "-a", str(peer_addr),
            ])
        # both udp and tcp
        else:
            args.extend([
                "-P", str(local_port),
                "-p", str(peer_port),
                "-a", str(peer_addr),
                "-k", str(self.key)
            ])
        
        if local_snat:
            args.append("-S")
        if local_p2p:
            args.extend(("-Z",str(local_p2p)))
        if local_txq:
            args.extend(("-Q",str(local_txq)))
        if not local_cap:
            args.append("-N")
        elif local_cap == 'pcap':
            args.extend(('-c','pcap'))
        if local_bwlim:
            args.extend(("-b",str(local_bwlim*1024)))
        if filter_module:
            args.extend(("--filter", filter_module))
        if filter_args:
            args.extend(("--filter-args", filter_args))
        if local_mcast and local_mcastfwd:
            args.extend(("--multicast-forwarder", local_mcastfwd))

        self._logger.info("Starting %s", self)
        
        self._make_home()
        self._install_scripts()

        # Start process in a "daemonized" way, using nohup and heavy
        # stdin/out redirection to avoid connection issues
        (out,err),proc = rspawn.remote_spawn(
            " ".join(args),
            
            pidfile = './pid',
            home = self.home_path,
            stdin = '/dev/null',
            stdout = 'capture',
            stderr = rspawn.STDOUT,
            sudo = True,
            
            host = local.node.hostname,
            port = None,
            user = local.node.slicename,
            agent = None,
            ident_key = local.node.ident_path,
            server_key = local.node.server_key
            )
        
        if proc.wait():
            raise RuntimeError, "Failed to set up TUN: %s %s" % (out,err,)
       
        self._started = True