Пример #1
0
    def body(self, args, cwd):

        #
        # - simply execute the snippet from the temporary directory
        # - any file uploaded in the process will be found in there as well
        #
        return shell(args, cwd=cwd)
Пример #2
0
        def probe(self, cluster):

            #
            # - check each pod and issue a MNTR command
            # - the goal is to make sure they are all part of the ensemble
            #
            leader = None
            for key, pod in cluster.pods.items():

                ip = pod['ip'] if key != cluster.key else 'localhost'
                port = pod['ports']['2181'] if key != cluster.key else '2181'
                code, lines = shell('echo mntr | nc -w 5 %s %s' % (ip, port))
                assert code == 0, 'failed to connect to pod #%d (is it dead ?)' % pod['seq']

                props = {}
                for line in lines:
                    if line:
                        tokens = line.split('\t')
                        props[tokens[0]] = ' '.join(tokens[1:])

                assert 'zk_server_state' in props, 'pod #%d -> not serving requests (is zk down ?)' % pod['seq']

                state = props['zk_server_state']
                assert state in ['leader', 'follower'], 'pod #%d -> <%s>' % (pod['seq'], state)
                if state == 'leader':
                    leader = props

            assert leader, 'no leader found ?'
            assert int(leader['zk_synced_followers']) == cluster.size - 1, '1+ follower not synced'
            return '%s zk nodes / ~ %s KB' % (leader['zk_znode_count'], leader['zk_approximate_data_size'])
Пример #3
0
        def configure(self, _):

            #
            # - dig master.mesos
            # - this should give us a list of internal master IPs
            # - please note this will only work if mesos-dns has been setup (and is running)
            #
            _, lines = shell("dig master.mesos +short")
            if lines:
                logger.debug("retrieved %d ips via dig" % len(lines))
                masters = ",".join(["%s:8080" % line for line in lines])

            #
            # - no mesos-dns running ?
            # - if so $MARATHON_MASTER must be defined (legacy behavior)
            #
            else:
                assert "MARATHON_MASTER" in os.environ, "failed to look mesos-dns up and no $MARATHON_MASTER defined"
                masters = os.environ["MARATHON_MASTER"]

            #
            # - run the webserver
            # - don't forget to pass the secret token as an environment variable
            #
            return "python portal.py", {"token": token, "MARATHON_MASTER": masters}
Пример #4
0
        def body(self, args, cwd):

            #
            # - force the output to be formatted in JSON
            # - add any variable defined on the command line using -v
            #
            headers = {"Accept": "application/json"}
            if args.variables:
                for value in args.variables:
                    tokens = value.split(":")
                    headers["X-Var-%s" % tokens[0]] = tokens[1]

            #
            # - fetch the uploaded TGZ archive in our temp. directory
            # - compute its SHA1 digest
            # - format the corresponding X-Signature header
            #
            tgz = join(cwd, args.tgz[0])
            code, lines = shell('openssl dgst -sha1 -hmac "%s" %s' % (token, tgz))
            assert code == 0, "failed to sign the archive"
            headers["X-Signature"] = "sha1=%s" % lines[0].split(" ")[1]

            #
            # - fire a POST /run request to ourselves
            # - pass the response back to the CLI
            #
            with open(tgz, "rb") as f:

                files = {"tgz": f.read()}
                reply = requests.post(
                    "http://localhost:5000/run/%s" % "+".join(args.scripts), files=files, headers=headers
                )
                assert reply.status_code < 300, "invalid response (HTTP %d)" % reply.status_code
                js = json.loads(reply.text)
                return 0 if js["ok"] else 1, js["log"]
Пример #5
0
        def body(self, args, _):

            #
            # - simply exec the echo
            # - please note we don't need any command line parsing
            # - the output will be returned back to the caller
            #
            return shell('echo "your command was %s"' % args)
 def rs_status():
     """
     Executes rs.status() against the localhost mongod
     """
     logger.debug("getting replicaset status")
     code, lines = shell("echo 'JSON.stringify(rs.status())' | mongo localhost:27018 --quiet")
     assert code == 0, 'failed to connect to local pod (is it dead ?)'
     return json.loads(' '.join(lines))
Пример #7
0
                def _2():

                    #
                    # - same as above except for slightly older DCOS releases
                    # - $MESOS_MASTER is located in /opt/mesosphere/etc/mesos-slave
                    #
                    logger.debug('checking /opt/mesosphere/etc/mesos-slave...')
                    _, lines = shell("grep MESOS_MASTER /opt/mesosphere/etc/mesos-slave")
                    return lines[0][18:].split('/')[0]
Пример #8
0
                def _install_from_package():

                    #
                    # - a regular package install will write the slave settings under /etc/mesos/zk
                    # - the snippet in there looks like zk://10.0.0.56:2181/mesos
                    #
                    code, lines = shell("cat /etc/mesos/zk")
                    assert code is 0 and lines[0], 'unable to retrieve the zk connection string'
                    return lines[0][5:].split('/')[0]
 def rs_add(pod):
     """
     Add the pod as a replicaset member using rs.add({_id: pod['seq'], host: pod['ip']:pod['ports']['27018']})
     """
     hoststr = "%s:%d" % (pod['ip'], pod['ports']['27018'])
     doc = {'_id': pod['seq'], 'host': hoststr}
     jsonstr = json.dumps(doc)
     code, _ = shell("echo 'rs.add(%s)' | mongo localhost:27018 --quiet" % jsonstr)
     assert code == 0, 'Unable to do rs.add(%s)' % jsonstr
Пример #10
0
                def _3():

                    #
                    # - a regular package install will write the slave settings under /etc/mesos/zk (the snippet in
                    #   there looks like zk://10.0.0.56:2181/mesos)
                    #
                    logger.debug('checking /etc/mesos/zk...')
                    _, lines = shell("cat /etc/mesos/zk")
                    return lines[0][5:].split('/')[0]
Пример #11
0
                def _1():

                    #
                    # - most recent DCOS release
                    # - $MESOS_MASTER is located in /opt/mesosphere/etc/mesos-slave-common
                    # - the snippet in there is prefixed by MESOS_ZK=zk://<ip:port>/mesos
                    #
                    logger.debug('checking /opt/mesosphere/etc/mesos-slave-common...')
                    _, lines = shell("grep MESOS_MASTER /opt/mesosphere/etc/mesos-slave-common")
                    return lines[0][18:].split('/')[0]
Пример #12
0
                def _install_from_package():

                    #
                    # - a regular package install will write the slave settings under /etc/mesos/zk
                    # - This is mesos-slave /etc/mesos which contains zookeeper
                    # config
                    #
                    code, lines = shell("cat /etc/mesos/zk")
                    assert code is 0 and lines[0], 'unable to retrieve the zk connection string'
                    return lines[0][5:].split('/')[0]
Пример #13
0
                def _dcos_deployment():

                    #
                    # - a DCOS slave is setup slightly differently with the settings being environment
                    #   variables set in /opt/mesosphere/etc/mesos-slave
                    # - the snippet in there is prefixed by MESOS_MASTER= and uses an alias
                    # - it looks like MESOS_MASTER=zk://leader.mesos:2181/mesos
                    #
                    code, lines = shell("grep MASTER /opt/mesosphere/etc/mesos-slave")
                    assert code is 0 and lines[0], 'unable to retrieve the zk connection string'
                    return lines[0][18:].split('/')[0]
Пример #14
0
 def get_pid(self, hints, hints_ignore = None):
     if hints_ignore:
         hints_ignore.append("grep")
     else:
         hints_ignore = ["grep"]
     assert hints, "need at least one hint"
     try:
         _, lines = shell("ps -ef | grep -v %s | grep %s | awk '{print $2}'" % (" | grep -v ".join(hints_ignore),  " | grep ".join(hints)))
         if lines:
             return [int(x) for x in lines]
         else:
             return None
     except Exception:
         return None
Пример #15
0
def servo(strict=True, verbose=False):
    try:

        #
        # - retrieve the portal coordinates from /opt/servo/.portal
        # - this file is rendered by the pod script upon boot
        #
        _, lines = shell('cat .portal', cwd='/opt/servo')
        portal = lines[0]
        assert portal, '/opt/servo/.portal not found (pod not yet configured ?)'

        def _proxy(cmdline):

            #
            # - this block is taken from cli.py in ochothon
            # - in debug mode the verbatim response from the portal is dumped on stdout
            # - slight modification : we force the json output (-j)
            #
            tokens = cmdline.split(' ') + ['-j']
            files = ['-F %[email protected]%s' % (basename(token), expanduser(token)) for token in tokens if isfile(expanduser(token))]
            line = ' '.join([basename(token) if isfile(expanduser(token)) else token for token in tokens])
            snippet = 'curl -X POST -H "X-Shell:%s" %s %s/shell' % (line, ' '.join(files), portal)
            code, lines = shell(snippet)
            assert code is 0, 'is the portal @ %s down ?' % portal
            js = json.loads(lines[0])
            ok = js['ok']
            if verbose:
                print '[%s] "%s"' % ('passed' if ok else 'failed', cmdline)
            assert not strict or ok, '"%s" failed' % cmdline
            return json.loads(js['out']) if ok else None

        yield _proxy

        #
        # - all clear, return 0 to signal a success
        #
        sys.exit(0)

    except AssertionError as failure:

        print 'failure -> %s' % failure

    except Exception as failure:

        print 'unexpected failure -> %s' % diagnostic(failure)

    sys.exit(1)
 def rs_initiate(pods):
     """
     Executes rs.initiate(...) against the local mongod. The _id of members is pod['seq'] and
     host is pod['ip']:pod['ports']['27018']
     """
     rs_name = os.getenv('REPLSET_NAME', 'rs0')
     rs_config_doc = {'_id': rs_name, 'members': []}
     for pod in pods:
         rs_config_doc['members'].append({
             '_id': pod['seq'],
             'host': "%s:%d" % (pod['ip'], pod['ports']['27018'])
         })
     # configure
     jsonstr = json.dumps(rs_config_doc)
     logger.info("initializing replicaset %s", rs_config_doc)
     code, _ = shell("echo 'rs.initiate(%s)' | mongo localhost:27018 --quiet" % jsonstr)
     assert code == 0, 'Unable to do rs.initiate(%s)' % rs_config_doc
Пример #17
0
        def _remote(cmdline):

            #
            # - this block is taken from cli.py in ochothon
            # - in debug mode the verbatim response from the portal is dumped on stdout
            #
            now = time.time()
            tokens = cmdline.split(' ')
            files = ['-F %[email protected]%s' % (basename(token), expanduser(token)) for token in tokens if isfile(expanduser(token))]
            line = ' '.join([basename(token) if isfile(expanduser(token)) else token for token in tokens])
            logger.debug('"%s" -> %s' % (line, portal))
            snippet = 'curl -X POST -H "X-Shell:%s" %s %s/shell' % (line, ' '.join(files), portal)
            code, lines = shell(snippet)
            assert code is 0, 'i/o failure (is the proxy portal down ?)'
            js = json.loads(lines[0])
            elapsed = time.time() - now
            logger.debug('<- %s (took %.2f seconds) ->\n\t%s' % (portal, elapsed, '\n\t'.join(js['out'].split('\n'))))
            return js
Пример #18
0
        def _proxy(cmdline):

            #
            # - this block is taken from cli.py in ochothon
            # - in debug mode the verbatim response from the portal is dumped on stdout
            # - slight modification : we force the json output (-j)
            #
            tokens = cmdline.split(' ') + ['-j']
            files = ['-F %[email protected]%s' % (basename(token), expanduser(token)) for token in tokens if isfile(expanduser(token))]
            line = ' '.join([basename(token) if isfile(expanduser(token)) else token for token in tokens])
            snippet = 'curl -X POST -H "X-Shell:%s" %s %s/shell' % (line, ' '.join(files), portal)
            code, lines = shell(snippet)
            assert code is 0, 'is the portal @ %s down ?' % portal
            js = json.loads(lines[0])
            ok = js['ok']
            if verbose:
                print '[%s] "%s"' % ('passed' if ok else 'failed', cmdline)
            assert not strict or ok, '"%s" failed' % cmdline
            return json.loads(js['out']) if ok else None
Пример #19
0
        def sanity_check(self, pid):

            #
            # - simply use the provided process ID to start counting time
            # - this is a cheap way to measure the sub-process up-time
            #
            now = time.time()
            if pid != self.pid:
                self.pid = pid
                self.since = now

            lapse = (now - self.since) / 3600.0

            #
            # - include the build tag in the pod's metrics
            # - this tag is added during integration prior to building the docker image
            #
            _, lines = shell('cat BUILD', cwd=self.cwd)

            return \
                {
                    'build': lines[0],
                    'uptime': '%.2f hours (pid %s)' % (lapse, pid)
                }
Пример #20
0
        def initialize(self):

            splunk = cfg['splunk']

            env = Environment(loader=FileSystemLoader('/opt/watcher/templates'))
            template = env.get_template('props.conf')
            
            with open('/opt/splunkforwarder/etc/system/local/props.conf', 'wb') as f:
                f.write(template.render(
                    {
                        'sourcetype': splunk['sourcetype']
                    }))

            shell('splunk start --accept-license && splunk edit user admin -password foo -auth admin:changeme')
            
            for url in splunk['forward'].split(','):
                shell('splunk add forward-server %s' % url)

            shell('touch /var/log/watcher.log && splunk add monitor /var/log/watcher.log -index service -sourcetype %s' % splunk['sourcetype'])
Пример #21
0
        def body(self, args):

            #
            # - setup a temp directory
            # - use it to store a tar of the current folder
            #
            stated = time.time()
            tmp = tempfile.mkdtemp()
            try:

                #
                # - tar the whole thing
                # - loop over the specified image tags
                #
                code, _ = shell("tar zcf %s/bundle.tgz *" % tmp)
                assert code == 0, "failed to tar"
                for tag in args.tags.split(","):

                    #
                    # - send the archive over to the underlying docker daemon
                    # - make sure to remove the intermediate containers
                    # - by design our container runs a socat on TCP 9001
                    #
                    tick = time.time()
                    _, lines = shell(
                        'curl -H "Content-Type:application/octet-stream" '
                        "--data-binary @bundle.tgz "
                        '"http://localhost:9001/build?forcerm=1\&t=%s:%s"' % (args.repo[0], tag),
                        cwd=tmp,
                    )
                    assert len(lines) > 1, "empty docker output (failed to build or docker error ?)"
                    last = json.loads(lines[-1])

                    #
                    # - the only way to test out for failure is to peek at the end of the docker output
                    #
                    lapse = time.time() - tick
                    assert "error" not in last, last["error"]
                    logger.debug("built tag %s in %d seconds" % (tag, lapse))

                    #
                    # - cat our .dockercfg (which is mounted)
                    # - craft the authentication header required for the push
                    # - push the image using the specified tag
                    #
                    _, lines = shell("cat /host/.docker/config.json")
                    assert lines, "was docker login run (no config.json found) ?"
                    js = json.loads(" ".join(lines))
                    assert "auths" in js, "invalid config.json setup (unsupported docker install ?)"
                    for url, payload in js["auths"].items():
                        tokens = base64.b64decode(payload["auth"]).split(":")
                        host = urlparse(url).hostname
                        credentials = {
                            "serveraddress": host,
                            "username": tokens[0],
                            "password": tokens[1],
                            "email": payload["email"],
                            "auth": "",
                        }

                        tick = time.time()
                        auth = base64.b64encode(json.dumps(credentials))
                        shell(
                            'curl -X POST -H "X-Registry-Auth:%s" '
                            '"http://localhost:9001/images/%s/push?tag=%s"' % (auth, args.repo[0], tag)
                        )
                        lapse = time.time() - tick
                        logger.debug("pushed tag %s to %s in %d seconds" % (tag, host, lapse))

                    #
                    # - remove the image we just built if not latest
                    # - this is done to avoid keeping around too many tagged images
                    #
                    if tag != "latest":
                        shell('curl -X DELETE "http://localhost:9001/images/%s:%s?force=true"' % (args.repo[0], tag))

                #
                # - clean up and remove any untagged image
                # - this is important otherwise the number of images will slowly creep up
                #
                _, lines = shell('curl "http://localhost:9001/images/json?all=0"')
                js = json.loads(lines[0])
                victims = [item["Id"] for item in js if item["RepoTags"] == ["<none>:<none>"]]
                for victim in victims:
                    logger.debug("removing untagged image %s" % victim)
                    shell('curl -X DELETE "http://localhost:9001/images/%s?force=true"' % victim)

            finally:

                #
                # - make sure to cleanup our temporary directory
                #
                shutil.rmtree(tmp)

            lapse = int(time.time() - stated)
            logger.info("%s built and pushed in %d seconds" % (args.repo[0], lapse))
            return 0
Пример #22
0
 def _peek(token):
     _, lines = shell('curl --max-time 1 -f http://169.254.169.254/latest/meta-data/%s' % token)
     return lines[0] if lines else ''
Пример #23
0
 def _peek(snippet):
     _, lines = shell(snippet)
     return lines[0] if lines else ''
Пример #24
0
        # - pass down the ZK ensemble coordinate
        #
        env = environ
        hints = json.loads(env['ochopod'])
        env['OCHOPOD_ZK'] = hints['zk']

        #
        # - Check for passed set of clusters to be watched in deployment yaml
        #
        watching = env['DAYCARE'].split(',') if 'DAYCARE' in env else ['*'] 
        period = float(env['PERIOD']) if 'PERIOD' in env else 60

        #
        # - Get the portal that we found during cluster configuration (see pod/pod.py)
        #
        _, lines = shell('cat /opt/watcher/.portal')
        portal = lines[0]
        assert portal, '/opt/watcher/.portal not found (pod not yet configured ?)'
        logger.debug('using proxy @ %s' % portal)
        
        #
        # - Prepare message logging
        #
        from logging import INFO, Formatter
        from logging.config import fileConfig
        from logging.handlers import RotatingFileHandler
        #
        # - the location on disk used for logging watcher messages
        #
        message_file = '/var/log/watcher.log'
Пример #25
0
        def body(self, args):

            #
            # - setup a temp directory
            # - use it to store a tar of the current folder
            #
            stated = time.time()
            tmp = tempfile.mkdtemp()
            try:

                #
                # - tar the whole thing
                # - loop over the specified image tags
                #
                code, _ = shell('tar zcf %s/bundle.tgz *' % tmp)
                assert code == 0, 'failed to tar'
                for tag in args.tags.split(','):

                    image = '%s:%s' % (args.repo[0], tag)

                    #
                    # - send the archive over to the underlying docker daemon
                    # - make sure to remove the intermediate containers
                    #
                    tick = time.time()
                    built, output = docker.build(path='%s/bundle.tgz' % tmp, pull=True, forcerm=True, tag=image)
                    assert built, 'empty docker output (failed to build or docker error ?)'
                    logger.debug('built image %s in %d seconds' % (image, time.time() - tick))

                    #
                    # - cat our .dockercfg (which is mounted)
                    # - craft the authentication header required for the push
                    # - push the image using the specified tag
                    #
                    auth = docker.login('autodeskcloud','/host/.docker/config.json')

                    tick = time.time()
                    docker.push(image)
                    logger.debug('pushed image %s to %s in %d seconds' % (image, auth['serveraddress'], time.time() - tick))

                    #
                    # - remove the image we just built if not latest
                    # - this is done to avoid keeping around too many tagged images
                    #
                    if tag != 'latest':
                        docker.remove_image(image, force=True)

                #
                # - clean up and remove any untagged image
                # - this is important otherwise the number of images will slowly creep up
                #
                images = docker.images(quiet=True, all=True)
                victims = [item['Id'] for item in images if item['RepoTags'] == ['<none>:<none>']]
                for victim in victims:
                    logger.debug('removing untagged image %s' % victim)
                    docker.remove_image(victim, force=True)

            finally:

                #
                # - make sure to cleanup our temporary directory
                #
                shutil.rmtree(tmp)

            lapse = int(time.time() - stated)
            logger.info('%s built and pushed in %d seconds' % (args.repo[0], lapse))
            return 0
Пример #26
0
                                shutil.rmtree(tmp)
                                logger.info('wiped out %s' % tmp)
                            except IOError:
                                pass

                        repo = path.join(tmp, cfg['name'])
                        if not path.exists(repo):

                            #
                            # - the repo is not in our cache
                            # - git clone it
                            #
                            os.makedirs(tmp)
                            logger.info('cloning %s' % tag)
                            url = 'https://%s' % cfg['git_url'][6:]
                            code, _ = shell('git clone -b master --single-branch %s' % url, cwd=tmp)
                            assert code == 0, 'unable to clone %s' % url
                        else:

                            #
                            # - the repo is already in there
                            # - git pull
                            #
                            shell('git pull', cwd=repo)

                        #
                        # - checkout the specified commit hash
                        #
                        logger.info('checkout @ %s' % sha[0:10])
                        code, _ = shell('git checkout %s' % sha, cwd=repo)
                        assert code == 0, 'unable to checkout %s (wrong credentials and/or git issue ?)' % sha[0:10]
Пример #27
0
    def run(self):
        try:

            #
            # - we need to pass the framework master IPs around (ugly)
            #
            assert 'MARATHON_MASTER' in os.environ, '$MARATHON_MASTER not specified (check your portal pod)'
            master = choice(os.environ['MARATHON_MASTER'].split(','))
            headers = \
                {
                    'content-type': 'application/json',
                    'accept': 'application/json'
                }

            with open(self.template, 'r') as f:

                #
                # - parse the template yaml file (e.g container definition)
                #
                raw = yaml.load(f)
                assert raw, 'empty YAML input (user error ?)'

                #
                # - merge with our defaults
                # - we want at least the cluster & image settings
                # - TCP 8080 is added by default to the port list
                #
                defaults = \
                    {
                        'start': True,
                        'debug': False,
                        'settings': {},
                        'ports': [8080],
                        'verbatim': {}
                    }

                cfg = merge(defaults, raw)
                assert 'cluster' in cfg, 'cluster identifier undefined (user error ?)'
                assert 'image' in cfg, 'docker image undefined (user error ?)'

                #
                # - if a suffix is specified append it to the cluster identifier
                #
                if self.suffix:
                    cfg['cluster'] = '%s-%s' % (cfg['cluster'], self.suffix)

                #
                # - timestamp the application (we really want a new uniquely identified application)
                # - lookup the optional overrides and merge with our pod settings if specified
                # - this is what happens when the -o option is used
                #
                stamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
                qualified = '%s.%s' % (self.namespace, cfg['cluster'])
                application = 'ochopod.%s-%s' % (qualified, stamp)
                if qualified in self.overrides:

                    blk = self.overrides[qualified]
                    logger.debug('%s : overriding %d settings (%s)' % (self.template, len(blk), qualified))
                    cfg['settings'] = merge(cfg['settings'], blk)

                def _nullcheck(cfg, prefix):

                    #
                    # - walk through the settings and flag any null value
                    #
                    missing = []
                    if cfg is not None:
                        for key, value in cfg.items():
                            if value is None:
                                missing += ['%s.%s' % ('.'.join(prefix), key)]
                            elif isinstance(value, dict):
                                missing += _nullcheck(value, prefix + [key])

                    return missing

                missing = _nullcheck(cfg['settings'], ['pod'])
                assert not missing, '%d setting(s) missing ->\n\t - %s' % (len(missing), '\n\t - '.join(missing))

                #
                # - lookup our all the pods for that identifier
                # - get their sequence indices (we'll use it to phase out them out)
                # - if the target # of pods we want is not specified default to 1 unless we are cycling
                # - set it to the current # of pods in that case
                #
                def _query(zk):
                    replies = fire(zk, qualified, 'info')
                    return [(hints['process'], seq) for _, (seq, hints, code) in replies.items() if code == 200]

                prev = run(self.proxy, _query)
                if self.cycle and not self.pods:
                    self.pods = sum(1 if state != 'dead' else 0 for state, _ in prev)

                #
                # - if we still have no target default it to 1 single pod
                #
                if not self.pods:
                    self.pods = 1

                #
                # - setup our port list
                # - the port binding is specified either by an integer (container port -> dynamic mesos port), by
                #   two integers (container port -> host port) or by an integer followed by a * (container port ->
                #   same port on the host)
                # - the marathon pods must by design map /etc/mesos
                #
                def _parse_port(token):
                    if isinstance(token, int):
                        return {'containerPort': token}
                    elif isinstance(token, str) and token.endswith(' *'):
                        port = int(token[:-2])
                        return {'containerPort': port, 'hostPort': port}
                    elif isinstance(token, str):
                        ports = token.split(' ')
                        assert len(ports) == 2, 'invalid port syntax (must be two integers separated by 1+ spaces)'
                        return {'containerPort': int(ports[0]), 'hostPort': int(ports[1])}
                    else:
                        assert 0, 'invalid port syntax ("%s")' % token

                #
                # - note the marathon-ec2 ochopod bindings will set the application hint automatically
                #   via environment variable (e.g no need to specify it here)
                # - make sure to mount /etc/mesos and /opt/mesosphere to account for various mesos installs
                #
                ports = [_parse_port(token) for token in cfg['ports']] if 'ports' in cfg else []
                spec = \
                    {
                        'id': application,
                        'instances': self.pods,
                        'env':
                            {
                                'ochopod_cluster': cfg['cluster'],
                                'ochopod_debug': str(cfg['debug']).lower(),
                                'ochopod_start': str(cfg['start']).lower(),
                                'ochopod_namespace': self.namespace,
                                'pod': json.dumps(cfg['settings'])
                            },
                        'container':
                            {
                                'type': 'DOCKER',
                                'docker':
                                    {
                                        'forcePullImage': True,
                                        'image': cfg['image'],
                                        'network': 'BRIDGE',
                                        'portMappings': ports
                                    },
                                'volumes':
                                    [
                                        {
                                            'containerPath': '/etc/mesos',
                                            'hostPath': '/etc/mesos',
                                            'mode': 'RO'
                                        },
                                        {
                                            'containerPath': '/opt/mesosphere',
                                            'hostPath': '/opt/mesosphere',
                                            'mode': 'RO'
                                        }
                                    ]
                            }
                    }

                #
                # - if we have a 'verbatim' block in our image definition yaml, merge it now
                #
                if 'verbatim' in cfg:
                    spec = merge(cfg['verbatim'], spec)

                #
                # - pick a marathon master at random
                # - fire the POST /v2/apps to create our application
                # - this will indirectly spawn our pods
                #
                url = 'http://%s/v2/apps' % master
                reply = post(url, data=json.dumps(spec), headers=headers)
                code = reply.status_code
                logger.debug('-> %s (HTTP %d)' % (url, code))
                assert code == 200 or code == 201, 'submission failed (HTTP %d)' % code

                #
                # - wait for all the pods to be in the 'running' mode
                # - the 'application' hint is set by design to the marathon application identifier
                # - the sequence counters allocated to our new pods are returned as well
                #
                target = ['dead', 'running'] if self.strict else ['dead', 'stopped', 'running']
                @retry(timeout=self.timeout, pause=3, default={})
                def _spin():
                    def _query(zk):
                        replies = fire(zk, qualified, 'info')
                        return [(hints['process'], seq) for seq, hints, _ in replies.values()
                                if hints['application'] == application and hints['process'] in target]

                    js = run(self.proxy, _query)
                    assert len(js) == self.pods, 'not all pods running yet'
                    return js

                js = _spin()
                running = sum(1 for state, _ in js if state is not 'dead')
                up = [seq for _, seq in js]
                self.out['up'] = up
                self.out['ok'] = self.pods == running
                logger.debug('%s : %d/%d pods are running ' % (self.template, running, self.pods))

                if not up:

                    #
                    # - nothing is running (typically because the image has an issue and is not
                    #   not booting the ochopod script for instance, which happens often)
                    # - in that case fire a HTTP DELETE against the marathon application to clean it up
                    #
                    url = 'http://%s/v2/apps/%s' % (master, application)
                    reply = delete(url, headers=headers)
                    code = reply.status_code
                    logger.debug('-> %s (HTTP %d)' % (url, code))
                    assert code == 200 or code == 204, 'application deletion failed (HTTP %d)' % code

                elif self.cycle:

                    #
                    # - phase out & clean-up the pods that were previously running
                    # - simply exec() the kill tool for this
                    #
                    time.sleep(self.cycle)
                    down = [seq for _, seq in prev]
                    code, _ = shell('toolset kill %s -i %s -d' % (qualified, ' '.join(['%d' % seq for seq in down])))
                    assert code == 0, 'failed to phase out %d pods' % len(prev)
                    self.out['down'] = down

        except AssertionError as failure:

            logger.debug('%s : failed to deploy -> %s' % (self.template, failure))

        except YAMLError as failure:

            if hasattr(failure, 'problem_mark'):
                mark = failure.problem_mark
                logger.debug('%s : invalid deploy.yml (line %s, column %s)' % (self.template, mark.line+1, mark.column+1))

        except Exception as failure:

            logger.debug('%s : failed to deploy -> %s' % (self.template, diagnostic(failure)))
Пример #28
0
        def _from_curl(scripts):

            #
            # - retrieve the X-Signature header
            # - fast-fail on a HTTP 403 if not there or if there is a mismatch
            #
            if not "X-Signature" in request.headers:
                return "", 403

            #
            # - force a json output if the Accept header matches 'application/json'
            # - otherwise default to a text/plain response
            # - create a temporary directory to run from
            #
            ok = 0
            log = []
            alphabet = string.letters + string.digits
            token = "".join(alphabet[ord(c) % len(alphabet)] for c in os.urandom(8))
            raw = request.accept_mimetypes.best_match(["application/json"]) is None
            tmp = tempfile.mkdtemp()
            try:

                #
                # - any request header in the form X-Var-* will be kept around and passed as
                #   an environment variable when executing the script
                # - make sure the variable is spelled in uppercase
                #
                local = {key[6:].upper(): value for key, value in request.headers.items() if key.startswith("X-Var-")}

                #
                # - craft a unique callback URL that points to this pod
                # - this will be passed down to the script to enable transient testing jobs
                #
                cwd = path.join(tmp, "uploaded")
                local["CALLBACK"] = "http://%s/callback/%s" % (env["local"], token)
                blocked[token] = cwd
                for key, value in local.items():
                    log += ["$%s = %s" % (key, value)]

                #
                # - download the archive
                # - compute the HMAC and compare (use our pod token as the key)
                # - fail on a 403 if mismatch
                #
                where = path.join(tmp, "bundle.tgz")
                request.files["tgz"].save(where)
                with open(where, "rb") as f:
                    bytes = f.read()
                    digest = "sha1=" + hmac.new(env["token"], bytes, hashlib.sha1).hexdigest()
                    if digest != request.headers["X-Signature"]:
                        return "", 403

                #
                # - extract it into its own folder
                # - make sure the requested script is there
                #
                code, _ = shell("mkdir uploaded && tar zxf bundle.tgz -C uploaded", cwd=tmp)
                assert code == 0, "unable to open the archive (bogus payload ?)"

                #
                # - decrypt any file whose extension is .aes
                # - just run openssl directly and dump the output in the working directory
                # - note: at this point we just look for .aes file in the top level directory
                #
                for file in os.listdir(cwd):
                    bare, ext = path.splitext(file)
                    if ext != ".aes":
                        continue

                    code, _ = shell(
                        "openssl enc -d -base64 -aes-256-cbc -k %s -in %s -out %s" % (env["token"], file, bare), cwd=cwd
                    )
                    if code == 0:
                        log += ["decrypted %s" % file]

                #
                # - run each script in order
                # - abort immediately if the script exit code is not zero
                # - keep the script output as a json array
                #
                for script in scripts.split("+"):
                    now = time.time()
                    assert path.exists(path.join(cwd, script)), "unable to find %s (check your scripts)" % script
                    code, lines = shell("python %s 2>&1" % script, cwd=cwd, env=local)
                    log += lines + ["%s ran in %d seconds" % (script, int(time.time() - now))]
                    assert code == 0, "%s failed on exit code %d" % (script, code)

                ok = 1

            except AssertionError as failure:

                log += ["failure (%s)" % failure]

            except Exception as failure:

                log += ["unexpected failure (%s)" % diagnostic(failure)]

            finally:

                #
                # - make sure to cleanup our temporary directory
                #
                del blocked[token]
                shutil.rmtree(tmp)

            if raw:

                #
                # - if 'application/json' was not requested simply dump the log as is
                # - force the response code to be HTTP 412 upon failure and HTTP 200 otherwise
                #
                code = 200 if ok else 412
                return "\n".join(log), code, {"Content-Type": "text/plain; charset=utf-8"}

            else:

                #
                # - if 'application/json' was requested always respond with a HTTP 200
                # - the response body then contains our serialized JSON output
                #
                js = {"ok": ok, "log": log}

                return json.dumps(js), 200, {"Content-Type": "application/json; charset=utf-8"}
Пример #29
0
                                shutil.rmtree(cached)
                                logger.info('wiped out %s' % cached)
                            except IOError:
                                pass

                        repo = path.join(cached, cfg['name'])
                        if not path.exists(repo):

                            #
                            # - the repo is not in our cache
                            # - git clone it
                            #
                            os.makedirs(cached)
                            logger.info('cloning %s [%s]' % (tag, branch))
                            url = 'https://%s' % cfg['git_url'][6:]
                            code, _ = shell('git clone -b %s --single-branch %s' % (branch, url), cwd=cached)
                            assert code == 0, 'unable to clone %s' % url
                        else:

                            #
                            # - the repo is already in there
                            # - git pull
                            #
                            shell('git pull', cwd=repo)

                        #
                        # - checkout the specified commit hash
                        #
                        logger.info('checkout @ %s' % sha[0:10])
                        code, _ = shell('git checkout %s' % sha, cwd=repo)
                        assert code == 0, 'unable to checkout %s (wrong credentials and/or git issue ?)' % sha[0:10]
Пример #30
0
 def _peek(token, strict=True):
     code, lines = shell('curl -f http://169.254.169.254/latest/meta-data/%s' % token)
     assert not strict or code is 0, 'unable to lookup EC2 metadata for %s (are you running on EC2 ?)' % token
     return lines[0]