コード例 #1
0
ファイル: core.py プロジェクト: ashouc/substance
class Core(object):
    def __init__(self, configFile=None, basePath=None):
        self.basePath = os.path.abspath(
            basePath) if basePath else os.path.expanduser(
                os.path.join('~', '.substance'))
        self.enginesPath = os.path.join(self.basePath, "engines")
        self.boxesPath = os.path.join(self.basePath, "boxes")
        self.dbFile = os.path.join(self.basePath, "db.json")

        configFile = configFile if configFile else "substance.yml"
        configFile = os.path.join(self.basePath, configFile)
        self.config = Config(configFile)

        self.insecureKey = None
        self.insecurePubKey = None

        self.assumeYes = False
        self.initialized = False

    def getBasePath(self):
        return self.basePath

    def getEnginesPath(self):
        return self.enginesPath

    def getBoxesPath(self):
        return self.boxesPath

    def getDbFile(self):
        return self.dbFile

    def initialize(self):
        if self.initialized:
            return OK(None)
        return self.assertPaths().then(self.assertConfig).then(
            self.initializeDB).then(defer(self.setInitialized, b=True))

    def setInitialized(self, b):
        self.initialized = b

    def assertPaths(self):
        return OK([self.basePath, self.enginesPath,
                   self.boxesPath]).mapM(Shell.makeDirectory)

    def assertConfig(self):
        return self.config.loadConfigFile()  \
            .catchError(FileDoesNotExist, self.makeDefaultConfig)

    def getDefaultConfig(self):
        defaults = OrderedDict()
        defaults['assumeYes'] = False
        defaults['drivers'] = ['virtualbox']
        defaults['tld'] = '.dev'
        defaults['devroot'] = os.path.join('~', 'substance')
        defaults['current'] = OrderedDict()
        defaults['engine'] = None
        defaults['subenv'] = None
        return defaults

    def makeDefaultConfig(self, data=None):
        logger.info("Generating default substance configuration in %s",
                    self.config.getConfigFile())
        defaults = self.getDefaultConfig()
        for kkk, vvv in defaults.iteritems():
            self.config.set(kkk, vvv)
        self.config.set("basePath", self.basePath)
        return self.config.saveConfig()

    # -- Use

    def setUse(self, engine, subenvName=None):
        ops = [self.setCurrentEngine(engine)]
        if subenvName:
            ops.append(engine.envSwitch(subenvName))
        return Try.sequence(ops)

    def setCurrentEngine(self, engine):
        current = self.config.get('current')
        current.update({'engine': engine.name})
        self.config.set('current', current)
        return OK(self)

    def readCurrentEngineName(self):
        current = self.config.get('current', {})
        name = current.get('engine', None)
        if not name:
            return Fail(
                EngineNotFoundError(
                    "No current engine is specified. Check the 'use' command for details."
                ))
        return OK(name)

    def loadCurrentEngine(self, name=None):
        current = self.config.get('current', {})
        engineName = name
        if not engineName:
            engineName = current.get('engine', None)

        if not engineName:
            return Fail(
                EngineNotFoundError(
                    "No current engine is specified. Check the 'use' command for details."
                ))

        engine = self.loadEngine(engineName) \
            .bind(Engine.loadConfigFile) \
            .bind(Engine.loadState)
        if engine.isFail():
            return engine

        engine = engine.getOK()

        if engine.state is not EngineStates.RUNNING:
            return Fail(
                EngineNotRunning("Engine '%s' is not running." % engine.name))

        return OK(engine)

    # -- Runtime

    def setAssumeYes(self, ay):
        self.assumeYes = True
        return True

    def getAssumeYes(self):
        if self.config.get('assumeYes', False):
            return True
        elif self.assumeYes:
            return True
        return False

    def getDefaultBoxString(self):
        return DefaultEngineBox

    # -- Engine library management

    def getEngines(self):
        ddebug("getEngines()")
        dirs = [
            d for d in os.listdir(self.enginesPath)
            if os.path.isdir(os.path.join(self.enginesPath, d))
        ]
        return OK(dirs)

    def loadEngines(self, engines=[]):
        return OK([self.loadEngine(x) for x in engines])

    def loadEngine(self, name):
        enginePath = os.path.join(self.enginesPath, name)
        if not os.path.isdir(enginePath):
            return Fail(
                EngineNotFoundError("Engine \"%s\" does not exist." % name))
        else:
            return OK(Engine(name, enginePath=enginePath, core=self))

    def createEngine(self, name, config=None, profile=None):
        enginePath = os.path.join(self.enginesPath, name)
        newEngine = Engine(name, enginePath=enginePath, core=self)
        return newEngine.create(config=config, profile=profile)

    def removeEngine(self, name):
        return self.loadEngine(name) \
            >> Engine.remove

    # -- Driver handling

    def getDrivers(self):
        return self.config.get('drivers', [])

    def validateDriver(self, driver):
        if driver in self.getDrivers():
            return OK(driver)
        return Fail(ValueError("Driver '%s' is not a valid driver."))

    def getDriver(self, name):
        cls = {'virtualbox': VirtualBoxDriver}.get(name, 'virtualbox')
        driver = cls(core=self)
        return driver

    # -- Link handling

    def getLink(self, type="ssh"):
        link = Link(keyFile=self.getInsecureKeyFile(), keyFormat='RSA')
        return link

    # -- Database

    def getDB(self):
        return self.db

    def initializeDB(self):
        db = DB(self.dbFile)
        db = db.initialize()
        if db.isFail():
            return db
        self.db = db.getOK()
        return OK(self.db)

    # -- Box handling

    def readBox(self, boxstring):
        return Box.parseBoxString(boxstring) \
            .map(lambda p: Box(core=self, **p))

    def pullBox(self, box):
        return box.fetch()

    def removeBox(self, box):
        return box.delete()

    def getBoxes(self):
        return self.getDB().getBoxRecords() \
            .mapM(lambda r: OK(Box(self, r.get('name'), r.get('version'), r.get('namespace'), r.get('registry'), r.get('boxstring'), r.get('archiveSHA1'))))

    def getInsecureKeyFile(self):
        return getSupportFile('support/substance_insecure')

    def getInsecurePubKeyFile(self):
        return getSupportFile('support/substance_insecure.pub')
コード例 #2
0
class Engine(object):
    def __init__(self, name, enginePath, core):
        self.name = name
        self.enginePath = enginePath
        self.core = core
        self.link = None
        configFile = os.path.join(self.enginePath, "engine.yml")
        self.config = Config(configFile)
        self.logAdapter = EngineLogAdapter(logger, self.__dict__)
        self.currentEnv = None
        self.subenvConfig = None

    def getDefaults(self):
        defaults = OrderedDict()
        defaults['name'] = 'default'
        defaults['driver'] = 'virtualbox'
        defaults['id'] = None
        defaults['box'] = DefaultEngineBox
        defaults['profile'] = EngineProfile().__dict__
        defaults['docker'] = OrderedDict()
        defaults['docker']['port'] = 2375
        defaults['docker']['tls'] = False
        defaults['docker']['certificateFile'] = None
        defaults['network'] = OrderedDict()
        defaults['network']['privateIP'] = None
        defaults['network']['publicIP'] = None
        defaults['network']['sshIP'] = None
        defaults['network']['sshPort'] = 4500
        defaults['devroot'] = OrderedDict()
        defaults['devroot']['path'] = getHomeDirectory('devroot')
        defaults['devroot']['mode'] = 'unison'
        defaults['devroot']['syncArgs'] = [
            '-ignore', 'Path */var', '-ignore', 'Path */data'
        ]
        defaults['devroot']['excludes'] = [
            '{.*,*}.sw[pon]', '.bash*', '.composer', '.git', '.idea', '.npm',
            '.ssh', '.viminfo'
        ]
        aliases = ['make', 'composer', 'npm', 'heap', 'watch']
        defaults['aliases'] = {
            alias: {
                'container': 'web',
                'user': '******',
                'cwd': '/vol/website',
                'args': [alias]
            }
            for alias in aliases
        }
        defaults['aliases']['watch']['container'] = 'devs'
        defaults['aliases']['heap']['args'][0] = 'vendor/bin/heap'

        return defaults

    def validateConfig(self, config):
        fields = [
            'name', 'driver', 'profile', 'docker', 'network', 'devroot', 'box'
        ]

        ops = []
        ops.append(
            self.config.validateFieldsPresent(config.getConfig(), fields))
        ops.append(self.confValidateName(config.get('name', None)))
        ops.append(self.core.validateDriver(config.get('driver', None)))
        ops.append(self.confValidateDevroot(config.get('devroot', {})))

        def dd(err):
            print(("%s" % err))
            return ConfigValidationError(err.message)

        return Try.sequence(ops) \
            .catch(dd) \
            .then(lambda: OK(config.getConfig()))

    def confValidateName(self, name):
        if name != self.name:
            return Fail(
                ConfigValidationError(
                    "Invalid name property in configuration (got %s expected %s)"
                    % (config.get('name'), self.name)))
        return OK(name)

    def confValidateDevroot(self, devroot):
        path = devroot.get('path', '')
        path = os.path.expanduser(path)
        if not path:
            return Fail(
                ConfigValidationError(
                    "devroot path configuration is missing."))
        elif not os.path.isdir(path):
            self.logAdapter.info(
                "WARNING: devroot '%s' does not exist locally." % path)

        mode = devroot.get('mode', None)
        if not mode:
            return Fail(ConfigValidationError("devroot mode is not set."))
        elif mode not in ['sharedfolder', 'rsync', 'unison']:
            # XXX Fix hardcoded values.
            return Fail(
                ConfigValidationError("devroot mode '%s' is not valid." %
                                      mode))

        return OK(devroot)

    def getName(self):
        return self.name

    def getID(self):
        return self.config.id

    def getEnginePath(self):
        return self.enginePath

    def getEngineFilePath(self, fileName):
        return os.path.join(self.enginePath, fileName)

    def getDockerURL(self):
        if self.getDriverID():
            return "tcp://%s:%s" % (self.getPublicIP(), self.getDockerPort())

    def getDockerPort(self):
        return self.config.get('docker', {}).get('port', 2375)

    def getDockerEnv(self):
        env = OrderedDict()
        # env['DOCKER_API_VERSION'] = '1.19'
        env['DOCKER_HOST'] = self.getDockerURL()
        env['DOCKER_TLS_VERIFY'] = ''
        return env

    def getPublicIP(self):
        return self.config.get('network').get('publicIP', None)

    def getPrivateIP(self):
        return self.config.get('network').get('privateIP', None)

    def getDriver(self):
        return self.core.getDriver(self.config.get('driver'))

    def getEngineProfile(self):
        profile = self.config.get('profile', {})
        return EngineProfile(**profile)

    def getDriverID(self):
        return self.config.get('id', None)

    def setDriverID(self, driverID):
        self.config.set('id', driverID)
        return OK(self)

    def getSSHPort(self):
        return self.config.getBlockKey('network', 'sshPort', 22)

    def getSSHIP(self):
        return self.config.getBlockKey('network', 'sshIP', 'localhost')

    def getConnectInfo(self):
        info = {'hostname': self.getSSHIP(), 'port': self.getSSHPort()}
        return info

    def getDNSName(self):
        tld = self.core.config.get('tld')
        return self.name + tld

    def getSyncher(self):
        syncMode = self.config.get('devroot').get('mode')
        keyfile = self.core.getInsecureKeyFile()
        return UnisonSyncher(engine=self, keyfile=keyfile)

    def clearDriverID(self):
        self.config.set('id', None)
        return OK(self)

    def loadConfigFile(self):
        return self.config.loadConfigFile().bind(self.validateConfig).map(
            self.chainSelf)

    def loadSubenvConfigFile(self):
        basePath = os.path.expanduser(self.config.get('devroot').get('path'))
        self.subenvConfig = Config(
            os.path.join(basePath,
                         '%s/.substance/subenv.yml' % self.currentEnv))
        return self.subenvConfig.loadConfigFile().map(self.chainSelf)

    def loadState(self):
        ddebug("Engine load state %s" % self.name)
        return self.fetchState().bind(self.setState).map(self.chainSelf)

    def setState(self, state):
        self.state = state
        return OK(None)

    def create(self, config=None, profile=None):
        if os.path.isdir(self.enginePath):
            return Fail(
                EngineExistsError("Engine \"%s\" already exists." % self.name))

        return Shell.makeDirectory(self.enginePath) \
            .then(defer(self.makeDefaultConfig, config=config, profile=profile)) \
            .bind(self.__ensureDevroot) \
            .bind(self.validateConfig) \
            .then(self.config.saveConfig) \
            .bind(dinfo("Generated default engine configuration in %s", self.config.configFile)) \
            .map(self.chainSelf)

    def __ensureDevroot(self, config):
        devroot = os.path.expanduser(config.get('devroot', {}).get('path'))
        if not os.path.isdir(devroot):
            self.logAdapter.info("Creating devroot at %s" % devroot)
            return Shell.makeDirectory(devroot).then(lambda: OK(config))
        return OK(config)

    def remove(self):
        if not os.path.isdir(self.enginePath):
            return Fail(
                FileSystemError("Engine \"%s\" does not exist." % self.name))
        if self.isProvisioned():
            return Fail(
                EngineProvisioned("Engine \"%s\" is provisioned." % self.name))
        return Shell.nukeDirectory(self.enginePath)

    def makeDefaultConfig(self, config=None, profile=None):
        default = self.getDefaults()
        default["name"] = self.name
        default["devroot"]["path"] = os.path.join(
            self.core.config.get('devroot'), self.name)

        cf = {}
        mergeDict(cf, default)
        mergeDict(cf, config)

        if profile:
            cf['profile']['cpus'] = profile.cpus
            cf['profile']['memory'] = profile.memory

        for ck, cv in cf.items():
            self.config.set(ck, cv)

        return OK(self.config)

    def getProfile(self):
        prof = self.config.get('profile', {})
        return EngineProfile(prof.get('cpus'), prof.get('memory'))

    def isProvisioned(self):
        '''
        Check that this engine has an attached Virtual Machine.
        '''
        return True if self.getDriverID() else False

    def validateProvision(self):
        '''
        Check that the provision attached to this engine is valid in the driver.
        '''
        if not self.isProvisioned():
            return OK(False)

        machID = self.getDriverID()
        return self.getDriver().exists(machID)

    def isRunning(self):
        return self.fetchState().map(lambda state:
                                     (state is EngineStates.RUNNING))

    def isSuspended(self):
        return self.fetchState().map(lambda state:
                                     (state is EngineStates.SUSPENDED))

    def fetchState(self):
        if not self.isProvisioned():
            return OK(EngineStates.INEXISTENT)
        return self.getDriver().getMachineState(self.getDriverID())

    def launch(self):
        '''
        # 1. Check that we know about a provisioned machined for this engine. Provision if not.
        # 2. Validate that the machine we know about is still provisioned. Provision if not.
        # 3. Check the machine state, boot accordingly
        # 4. Setup guest networking
        # 4. Fetch the guest IP from the machine and store it in the machine state.
        '''
        return self.provision() \
            .then(self.start) \
            .then(self.updateNetworkInfo) \
            .then(self.addToHostsFile) \
            .then(self.envLoadCurrent) \
            .catchError(EnvNotDefinedError, lambda e: OK(None)) \
            .then(dinfo("Engine \"%s\" has been launched.", self.name)) \
            .then(self.envStart if self.currentEnv else lambda: OK(None))

    def addToHostsFile(self):
        self.logAdapter.info("Registering engine as '%s' in local hosts file" %
                             self.getDNSName())
        return SubHosts.register(self.getDNSName(), self.getPublicIP())

    def removeFromHostsFile(self):
        self.logAdapter.info("Removing engine from local hosts file")
        return SubHosts.unregister(self.getDNSName())

    def updateNetworkInfo(self):
        self.logAdapter.info("Updating network info from driver")
        return self.__readDriverNetworkInfo().bind(self.saveDriverNetworkInfo)

    def postLaunch(self):
        return self.setHostname() \
            .then(self.mountFolders)

    def saveDriverNetworkInfo(self, info):
        self.logAdapter.debug("Network information for machine: %s" % info)
        net = self.config.get('network', OrderedDict())
        net.update(info)
        self.config.set('network', net)
        return self.config.saveConfig()

    def start(self):

        state = self.fetchState()
        if state.isFail():
            return state

        if state.getOK() is EngineStates.RUNNING:
            return EngineAlreadyRunning("Engine \"%s\" is already running" %
                                        self.name)

        if state.getOK() is EngineStates.SUSPENDED:
            return self.__start().then(self.__waitForReady)
        else:
            return self.__configure() \
                .then(self.updateNetworkInfo) \
                .then(self.__start) \
                .then(self.__waitForReady) \
                .then(self.__waitForNetwork) \
                .then(self.postLaunch)

    def readBox(self):
        return self.core.readBox(self.config.get('box'))

    def provision(self):

        logger.info("Provisioning engine \"%s\" with driver \"%s\"", self.name,
                    self.config.get('driver'))

        prov = self.validateProvision()
        if prov.isFail():
            return prov
        elif prov.getOK():
            return OK(self)

        box = self.readBox().bind(Box.fetch)
        if box.isFail():
            return box
        box = box.getOK()

        return self.getDriver().importMachine(self.name, box.getOVFFile(), self.getEngineProfile()) \
            .bind(self.setDriverID) \
            .then(self.config.saveConfig) \
            .map(self.chainSelf)
        # .then(self.__configure) \
        # .then(self.updateNetworkInfo) \

    def deprovision(self):

        if not self.isProvisioned():
            return Fail(
                EngineNotProvisioned(
                    "Engine \"%s\" is not provisioned." % self.name, self))

        self.__cacheLink(None)

        return self.validateProvision() \
            .thenIfTrue(self.isRunning) \
            .thenIfTrue(self.__terminate) \
            .then(self.__delete) \
            .then(self.clearDriverID)  \
            .then(self.config.saveConfig)  \
            .then(self.removeFromHostsFile) \
            .then(dinfo("Engine \"%s\" has been deprovisioned.", self.name)) \
            .map(self.chainSelf)

    def suspend(self):
        if not self.isProvisioned():
            return Fail(
                EngineNotProvisioned("Engine \"%s\" is not provisioned." %
                                     self.name))
        self.__cacheLink(None)

        return self.validateProvision() \
            .thenIfTrue(self.isRunning) \
            .thenIfFalse(failWith(EngineNotRunning("Engine \"%s\" is not running." % self.name))) \
            .then(self.__suspend) \
            .then(dinfo("Engine \"%s\" has been suspended.", self.name)) \
            .map(chainSelf)

        # XXX Insert wait for suspension

    def stop(self, force=False):

        self.__cacheLink(None)

        operation = self.validateProvision() \
            .thenIfTrue(self.isRunning) \
            .thenIfFalse(failWith(EngineNotRunning("Engine \"%s\" is not running." % self.name)))

        if (force):
            operation >> self.__terminate >> dinfo(
                "Engine \"%s\" has been terminated.", self.name)
        else:
            operation >> self.__halt >> dinfo("Engine \"%s\" has been halted.",
                                              self.name)

        # XXX insert wait for stopped state
        return operation.map(self.chainSelf)

    def readLink(self):
        if self.link is not None:
            return OK(self.link)
        link = self.core.getLink()
        return link.connectEngine(self).map(self.__cacheLink)

    def __cacheCurrentEnv(self, lr):
        env = lr.stdout.strip()
        self.logAdapter.debug("Current environment: '%s'" % (env))
        self.currentEnv = env
        return lr

    def __cacheLink(self, link):
        self.link = link
        return link

    def __start(self, *args):
        return self.getDriver().startMachine(self.getDriverID()).map(
            self.chainSelf)

    def __suspend(self, *args):
        return self.getDriver().suspendMachine(self.getDriverID()).map(
            self.chainSelf)

    def __terminate(self, *args):
        return self.getDriver().terminateMachine(self.getDriverID()).map(
            self.chainSelf)

    def __halt(self, *args):
        return self.getDriver().haltMachine(self.getDriverID()).map(
            self.chainSelf)

    def __delete(self, *args):
        return self.getDriver().deleteMachine(self.getDriverID()).map(
            self.chainSelf)

    def __readDriverNetworkInfo(self):
        return self.getDriver().readMachineNetworkInfo(self.getDriverID())

    def __configure(self):
        return self.getDriver().configureMachine(self.getDriverID(), self)

    def shell(self):
        return self.readLink() >> Link.interactive

    def __waitForNetwork(self):
        self.logAdapter.info("Waiting for machine network...")
        return self.getDriver().readMachineWaitForNetwork(self.getDriverID())

    def __waitForReady(self):
        self.logAdapter.info("Waiting for machine to boot...")
        return self.readLink()

    def setHostname(self):
        self.logAdapter.info("Configuring machine hostname")
        dns = self.getDNSName()
        echoCmd = "echo %s > /etc/hostname" % dns
        hostCmd = "hostname %s" % dns
        hostsCmd = "sed -i \"s/127.0.1.1.*/127.0.1.1\t%s/g\" /etc/hosts" % self.name
        # hostsCmd = "hostnamectl set-hostname %s" % dns
        serviceCmd = "service hostname restart"
        cmd = "sudo -- /bin/sh -c '%s && %s && %s && %s'" % (
            echoCmd, hostsCmd, hostCmd, serviceCmd)
        cmds = list(
            map(defer(self.link.runCommand, stream=True, sudo=False), [cmd]))
        return Try.sequence(cmds)

    def exposePort(self, local_port, public_port, scheme):
        keyfile = self.core.getInsecureKeyFile()
        keyfile = inner(keyfile)
        forward_descr = "0.0.0.0:%s:127.0.0.1:%s" % (public_port, local_port)
        engineIP = self.getSSHIP()
        enginePort = str(self.getSSHPort())
        cmdPath = "ssh"
        cmdArgs = [
            "ssh", "-N", "-L", forward_descr, engineIP, "-l", "substance",
            "-p", enginePort, "-o", "StrictHostKeyChecking=no", "-o",
            "UserKnownHostsFile=/dev/null", "-i", keyfile
        ]
        sudo = False
        if public_port < 1024 and not isWithinWindowsSubsystem():
            sudo = True
        self.logAdapter.info(
            "Exposing port %s as %s; kill the process (CTRL-C) to un-expose.",
            local_port, public_port)
        with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as s:
            s.connect(("8.8.8.8", 80))
            ip = s.getsockname()[0]
            if scheme:
                self.logAdapter.info("%s://%s:%s is now available.", scheme,
                                     ip, public_port)
            else:
                self.logAdapter.info(
                    "Others can now connect to port %s via IP %s.",
                    public_port, ip)
        Shell.execvp(cmdPath, cmdArgs, {}, sudo=sudo)

    def getOrchestrator(self):
        if self.subenvConfig:
            return self._getOrchestrator()

        return self.envLoadCurrent() \
            .then(self.loadSubenvConfigFile) \
            .catch(lambda err: OK(None)) \
            .then(self._getOrchestrator)

    def _getOrchestrator(self):
        orchestrator = self.subenvConfig.get(
            'orchestrator') if self.subenvConfig else None
        logger.debug("Orchestrator: %s" % orchestrator)
        if orchestrator and orchestrator == Orchestrators.COMPOSE:
            return OK(Compose(self))
        elif orchestrator and orchestrator != Orchestrators.DOCKWRKR:
            return Fail(
                InvalidOrchestrator('%s is not a known orchestrator' %
                                    orchestrator))
        else:
            return OK(Dockwrkr(self))

    def getOrchestratorCommand(self, command, *args):
        orch = self.getOrchestrator()
        if orch.isFail():
            raise orch.value
        else:
            orch = orch.getOK()

        if orch:
            return getattr(orch, command)(*args)
        else:
            return ''

    def orchestrate(self, command, *args):
        commands = self.getOrchestratorCommand(command, *args)
        if len(commands) > 0:
            return list(map(lambda x: "(subenv run %s)" % x, commands))

    def envSwitch(self, subenvName, restart=False):
        self.logAdapter.info("Switch engine '%s' to subenv '%s'" %
                             (self.name, subenvName))

        self.currentEnv = subenvName
        self.loadSubenvConfigFile()

        cmds = CommandList([
            "subenv init '/substance/devroot/%s'" % (subenvName),
            "subenv use '%s'" % (subenvName),
        ])

        if restart:
            cmds.append(self.orchestrate('resetAll'))
            cmds.append(self.orchestrate('login'))
            cmds.append(self.orchestrate('startAll'))

        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False) \
            .then(self.envRegister)

    def envRegister(self):
        return self.readLink() \
            .bind(Link.runCommand, 'subenv vars', stream=False, capture=True) \
            .bind(self.__envRegister)

    def __envRegister(self, lr):
        vars = lr.stdout.split("\n")
        env = Try.attempt(parseDotEnv, vars)
        if env.isFail():
            return env
        env = env.getOK()
        if 'SUBENV_FQDN' in env:
            return SubHosts.register(env['SUBENV_FQDN'], self.getPublicIP())
        elif 'SUBENV_NAME' in env:
            return SubHosts.register(
                env['SUBENV_NAME'] + self.core.config.get('tld'),
                self.getPublicIP())

    def envLoadCurrent(self):
        cmds = CommandList(["subenv current"])
        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=False, sudo=False, capture=True) \
            .map(self.__cacheCurrentEnv) \
            .catch(lambda err: Fail(EnvNotDefinedError("No current subenv is set. Check 'switch' for detais."))) \
            .then(lambda: self)

    def envStart(self, reset=False, containers=[]):
        cmds = CommandList()

        if reset:
            cmds.append(self.getOrchestratorCommand('resetAll'))

        if len(containers) > 0:
            self.logAdapter.info("Starting %s container(s)" %
                                 (' '.join(containers)))
            cmds.append(self.orchestrate('start', containers))
        else:
            self.logAdapter.info("Starting all containers...")
            cmds.append(self.orchestrate('startAll'))

        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)

    def envRestart(self, time=10, containers=[]):
        cmds = CommandList()

        if len(containers) > 0:
            self.logAdapter.info("Restarting %s containers" %
                                 (' '.join(containers)))
            cmds.append(self.orchestrate('restart', containers, time))
        else:
            self.logAdapter.info("Restarting all containers...")
            cmds.append(self.orchestrate('restartAll', time))

        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)

    def envRecreate(self, time=10, containers=[]):
        cmds = CommandList()

        if len(containers) > 0:
            self.logAdapter.info("Recreating %s containers" %
                                 (' '.join(containers)))
            cmds.append(self.orchestrate('recreate', containers, time))
        else:
            self.logAdapter.info("Recreating all containers...")
            cmds.append(self.orchestrate('recreateAll', time))

        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)

    def envShell(self, container=None, user=None, cwd=None):
        if container:
            return self.envEnter(container, user, cwd)
        else:
            return self.readLink().bind(Link.interactive)

    def envEnter(self, container, user=None, cwd=None):
        self.logAdapter.info("Entering %s container..." % (container))
        return self.envExec(container, ["exec /bin/bash"], cwd, user)

    def envExec(self, container, cmd, cwd=None, user=None):
        cmd = CommandList(self.orchestrate('exec', container, cmd, cwd, user))
        return self.readLink().bind(Link.runCommand,
                                    cmd=cmd.logicAnd(),
                                    interactive=True,
                                    stream=True,
                                    shell=False,
                                    capture=False)

    def envRun(self, task, args):
        cmd = CommandList(self.orchestrate('run', task, args))
        return self.readLink().bind(Link.runCommand,
                                    cmd=cmd.logicAnd(),
                                    interactive=True,
                                    stream=True,
                                    shell=False,
                                    capture=False)

    def envExecAlias(self, alias, args):
        aliases = self.config.get('aliases') if self.config.get(
            'aliases') else {}
        subenvAliases = self.subenvConfig.get(
            'aliases') if self.subenvConfig.get('aliases') else {}
        if subenvAliases:
            aliases = mergeDictOverwrite(aliases, subenvAliases)

        if aliases and alias in aliases:
            cmd = aliases[alias]
            args = cmd['args'] + args
            return self.envExec(container=cmd['container'],
                                cmd=args,
                                cwd=cmd['cwd'],
                                user=cmd['user'])
        return Fail(
            InvalidCommandError(
                "Invalid command '%s' specified for '%s'.\n\nUse 'substance help' for available commands."
                % (alias, 'substance')))

    def envStop(self, time=10, containers=[]):
        cmds = CommandList()

        if len(containers) > 0:
            self.logAdapter.info("Stopping %s container(s)" %
                                 (' '.join(containers)))

            cmds.append(self.orchestrate('stop', containers, time))
        else:
            self.logAdapter.info("Stopping containers...")
            cmds.append(self.orchestrate('stopAll', time))

        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)

    def envStatus(self, full=False):
        if full:
            cmds = CommandList([self.orchestrate('status')])
            return self.readLink() \
                .bind(Link.runCommand, cmds.logicAnd(), stream=False, sudo=False, capture=True) \
                .map(self.__envStatus)
        else:
            return OK(self.__envStatus())

    def envCleanup(self):
        cmds = CommandList()
        cmds.append('docker system prune -a')
        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), interactive=True, stream=True, shell=False, capture=False)

    def envLogs(self, parts=[], pattern=None, follow=True, lines=None):

        if not pattern:
            pattern = "%s*.log" % ('-'.join(parts))

        cmds = CommandList()
        cmd = "tail"
        if follow:
            cmd += " -f"
        if lines:
            cmd += " -n %s" % int(lines)

        cmd += " \"logs/%s\"" % pattern

        cmds.append('subenv run %s' % cmd)
        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, capture=False, sudo=False)

    def envListLogs(self, parts=[], pattern=None):

        if not pattern:
            pattern = "%s*.log" % ('-'.join(parts))

        cmds = CommandList()
        cmd = "ls -1 \"logs/%s\" | xargs -n1 basename" % pattern

        cmds.append('subenv run %s' % cmd)
        return self.readLink() \
            .bind(Link.runCommand, cmds.logicAnd(), stream=True, capture=False, sudo=False)

    def __envStatus(self, containers=None):
        return {
            'engine':
            self,
            'containers':
            containers.stdout + containers.stderr if containers else None
        }

    def envDocker(self, command):
        cmd = "docker %s" % command
        logger.debug("DOCKER: %s" % cmd)
        return self.readLink().bind(Link.runCommand,
                                    cmd,
                                    stream=True,
                                    interactive=True,
                                    shell=False)

    def getEngineFolders(self):
        # XXX Dynamic mounts / remove hardcoded values.
        devroot = self.config.get('devroot')
        pfolder = EngineFolder(name='devroot',
                               mode=devroot.get('mode'),
                               hostPath=os.path.expanduser(
                                   devroot.get('path')),
                               guestPath='/substance/devroot',
                               uid=1000,
                               gid=1000,
                               umask="0022",
                               excludes=devroot.get('excludes', []),
                               syncArgs=devroot.get('syncArgs', []))
        return [pfolder]

    def mountFolders(self):
        self.logAdapter.info("Mounting engine folders")
        folders = self.getEngineFolders()
        return Try.of(list(map(self.mountFolder, folders)))

    def mountFolder(self, folder):
        mountCmd = "mount -t vboxsf -o umask=%(umask)s,gid=%(gid)s,uid=%(uid)s %(name)s %(guestPath)s" % folder.__dict__
        mkdirCmd = "mkdir -p %(guestPath)s && chown -R %(uid)s:%(gid)s %(guestPath)s" % folder.__dict__
        # XXX Make this non VBOX specific
        chain = self.link.runCommand(mkdirCmd, sudo=True)
        if folder.mode is 'sharedfolder':
            chain = chain.then(defer(self.link.runCommand, mountCmd,
                                     sudo=True))
        return chain

    chainSelf = chainSelf

    def __repr__(self):
        return "Engine(%s)" % self.name
コード例 #3
0
ファイル: __init__.py プロジェクト: savardc/substance
class VirtualBoxDriver(Driver):
    '''
    Substance VirtualBox driver class. Interface to virtual box manager.
    '''
    def __init__(self, core):
        super(self.__class__, self).__init__(core)
        self.config = Config(
            os.path.join(self.core.getBasePath(), "virtualbox.yml"))

    def assertSetup(self):
        return self.assertConfig().then(self.assertNetworking)

    # -- Configuration

    def getDefaultConfig(self):
        defaults = OrderedDict()
        defaults['network'] = "172.21.21.0/24"
        defaults['interface'] = None
        return defaults

    def makeDefaultConfig(self):
        self.logAdapter.info("Generating default virtualbox config in %s" %
                             self.config.getConfigFile())
        defaults = self.getDefaultConfig()
        for kkk, vvv in defaults.items():
            self.config.set(kkk, vvv)
        return self.config.saveConfig()

    def assertConfig(self):
        return self.config.loadConfigFile() \
            .catchError(FileDoesNotExist, lambda err: self.makeDefaultConfig())

    # ---- Networking setup

    def getNetworkInterface(self):
        return self.config.get('interface', None)

    def readNetworkConfig(self):
        netrange = IPNetwork(self.config.get('network'))
        netconfig = {
            'gateway': netrange[1].format(),
            'netmask': netrange.netmask.format(),
            'lowerIP': netrange[5].format(),
            'upperIP': netrange[-5].format()
        }

        return OK(netconfig)

    def assertNetworking(self):
        interface = self.getNetworkInterface()
        netconfig = self.readNetworkConfig()

        if interface:
            hoif = network.readHostOnlyInterface(interface) \
                .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))
            dhcp = network.readDHCP(interface) \
                .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))

            return Try.sequence((netconfig, hoif, dhcp)) \
                .bind(self.assertNetworkConfiguration)
        else:
            return netconfig.bind(self.provisionNetworking)

    def assertNetworkConfiguration(self, netparts):

        (netconfig, hoif, dhcp) = netparts

        if not hoif:
            return self.provisionNetworking(netconfig)
        elif hoif.v4ip.format() != netconfig['gateway']:
            self.logAdapter.warning(
                "VirtualBox interface \"%s\" is not properly configured. Creating a new host-only network."
                % hoif.name)
            return self.provisionNetworking(netconfig)
        elif dhcp is None:
            self.logAdapter.warning(
                "VirtualBox interface \"%s\" does not have DHCP enabled. Re-Establishing now."
                % hoif.name)
            return self.provisionDHCP(hoif.name, netconfig)

        return OK(hoif)

    def findInterface(self, netconfig, interfaces):
        matches = list(
            filter(lambda x: x.v4ip == netconfig['gateway'], interfaces))
        return OK(None) if len(matches) <= 0 else OK(matches[0])

    def provisionNetworking(self, netconfig):
        self.logAdapter.info(
            "Provisioning VirtualBox networking for substance")

        hoif = network.readHostOnlyInterfaces() \
            .bind(defer(self.findInterface, netconfig))
        if hoif.isFail():
            return hoif
        hoif = hoif.getOK()

        if not hoif:
            ifm = network.addHostOnlyInterface()
            if ifm.isFail():
                return ifm
            iface = ifm.getOK()
        else:
            iface = hoif.name

        return network.configureHostOnlyInterface(iface, ip=netconfig['gateway'], netmask=netconfig['netmask']) \
            .then(defer(self.provisionDHCP, interface=iface, netconfig=netconfig)) \
            .then(defer(self.saveInterface, iface=iface)) \
            .then(defer(network.readHostOnlyInterface, name=iface))

    def provisionDHCP(self, interface, netconfig):
        self.logAdapter.info(
            "Provisioning DHCP service for host only interface")
        network.removeDHCP(interface).catch(lambda x: OK(interface)) \
            .then(defer(network.addDHCP, interface, **netconfig))

    def saveInterface(self, iface):
        self.config.set('interface', iface)
        return self.config.saveConfig()

    # ---- Machine API

    def importMachine(self, name, ovfFile, engineProfile=None):
        return self.assertSetup() \
            .then(defer(machine.inspectOVF, ovfFile)) \
            .bind(defer(machine.makeImportParams, name=name, engineProfile=engineProfile)) \
            .bind(defer(machine.importOVF, ovfFile=ovfFile, name=name))

    def startMachine(self, uuid):
        '''
        Start the machine by driver identifier.
        '''
        state = machine.readMachineState(uuid)
        if state.isFail():
            return state

        if state.getOK() in [machine.MachineStates.PAUSED]:
            return machine.resume(uuid)
        else:
            return machine.start(uuid)

    def readMachineWaitForNetwork(self, uuid, timeout=5000):
        return machine.readWaitGuestProperty(uuid, "/VirtualBox/GuestInfo/Net",
                                             timeout)

    def readMachineNetworkInfo(self, uuid):
        def format(res):
            info = OrderedDict()
            info['sshPort'] = res[0].hostPort if res[0] else None
            info['sshIP'] = '127.0.0.1'
            info['privateIP'] = res[1]
            info['publicIP'] = res[2]
            return info

        return Try.sequence([
            network.readPortForward(uuid, name="substance-ssh"),
            machine.readGuestProperty(uuid,
                                      "/VirtualBox/GuestInfo/Net/0/V4/IP"),
            machine.readGuestProperty(uuid,
                                      "/VirtualBox/GuestInfo/Net/1/V4/IP")
        ]).map(format)

    def configureMachine(self, uuid, engine):
        def engineFolderToVBoxFolder(acc, x):
            if x.mode == 'sharedfolder':
                acc.append(
                    machine.SharedFolder(name=x.name, hostPath=x.hostPath))
            return acc

        folders = reduce(engineFolderToVBoxFolder, engine.getEngineFolders(),
                         [])
        desiredPort = engine.getSSHPort()

        return self.assertSetup() \
            .then(defer(self.resolvePortConflict, uuid=uuid, desiredPort=desiredPort)) \
            .then(defer(self.configureMachineAdapters, uuid=uuid)) \
            .then(defer(self.configureMachineFolders, uuid=uuid, folders=folders)) \
            .then(defer(self.configureMachineProfile, uuid=uuid, engine=engine))

    def configureMachineProfile(self, uuid, engine):
        self.logAdapter.info("Configure machine profile")
        profile = engine.getProfile()
        return machine.configureProfile(uuid, profile.cpus, profile.memory)

    def configureMachineFolders(self, folders, uuid):
        self.logAdapter.info("Configure machine shared folders")
        return machine.clearSharedFolders(uuid) \
            .then(defer(machine.addSharedFolders, folders=folders, uuid=uuid))

    def resolvePortConflict(self, uuid, desiredPort):
        self.logAdapter.info("Checking for port conflicts")
        return network.readAllPortForwards(ignoreUUIDs=[uuid]) \
            .bind(defer(self.determinePort, desiredPort=desiredPort)) \
            .bind(defer(self.configurePort, uuid=uuid))

    def determinePort(self, usedPorts, desiredPort):
        basePort = 4500
        self.logAdapter.debug("Base port: %s" % basePort)
        self.logAdapter.debug("Desired port: %s" % desiredPort)
        unavailable = [x.hostPort for x in usedPorts]
        port = desiredPort if desiredPort >= basePort else basePort
        while port in unavailable or port < basePort:
            self.logAdapter.debug("Port %s is in use." % port)
            port += 1
        self.logAdapter.info("Determined SSH port as %s" % port)
        return OK(port)

    def configurePort(self, port, uuid):
        self.logAdapter.debug("Configure substance-ssh port on port %s" % port)
        pf = network.PortForward("substance-ssh", 1, "tcp", None, port, None,
                                 22)
        return network.removePortForwards([pf], uuid) \
            .catch(lambda err: OK(None)) \
            .then(defer(network.addPortForwards, [pf], uuid))

    def configureMachineAdapters(self, uuid):
        interface = self.getNetworkInterface()
        adapters = [
            machine.AdapterSettings(machine.AdapterTypes.NAT, 'default', None,
                                    False),
            machine.AdapterSettings(machine.AdapterTypes.HOSTONLY, interface,
                                    None, False),
            machine.AdapterSettings(machine.AdapterTypes.NONE, None, None,
                                    False),
            machine.AdapterSettings(machine.AdapterTypes.NONE, None, None,
                                    False)
        ]
        return Try.sequence([
            machine.configureAdapter(uuid, i + 1, x)
            for i, x in enumerate(adapters)
        ])

    def suspendMachine(self, uuid):
        '''
        Suspend the machine.
        '''
        return machine.suspend(uuid)

    def haltMachine(self, uuid):
        '''
        Halt the machine.
        '''
        return machine.halt(uuid)

    def terminateMachine(self, uuid):
        '''
        Terminate the machine forcefully.
        '''
        return machine.terminate(uuid)

    def deleteMachine(self, uuid):
        '''
        Delete the machine by driver identifier.
        '''
        return machine.delete(uuid)

    def exportMachine(self, uuid):
        # XXX To be implemented
        pass

    def getMachines(self):
        '''
        Retrieve the list of machines and their driver identifiers.
        '''
        return machine.readMachines()

    # -- Parse results from Virtual Box

    def getMachineID(self, name):
        '''
        Retrieve the driver specific machine ID for a machine name.
        '''
        return machine.findMachineID(name)

    def exists(self, uuid):
        '''
        Check in the driver that the specified identifier exists.
        '''
        return machine.readMachineExists(uuid)

    def isRunning(self, uuid):
        if self.getMachineState(uuid) is EngineStates.RUNNING:
            return True

    def isStopped(self, uuid):
        if self.getMachineState(uuid) is not EngineStates.RUNNING:
            return True

    def isSuspended(self, uuid):
        if self.getMachineState(uuid) is EngineStates.SUSPENDED:
            return True

    def getMachineState(self, uuid):
        '''
        Retrieve the Substance machine state for this driver id
        '''
        return machine.readMachineState(uuid) \
            .bind(self.machineStateToEngineState)

    def machineStateToEngineState(self, vboxState):
        '''
        Resolve a vbox machine state to a substance engine state.
        '''

        mapping = {
            machine.MachineStates.POWEROFF: EngineStates.STOPPED,
            machine.MachineStates.SAVED: EngineStates.SUSPENDED,
            machine.MachineStates.PAUSED: EngineStates.SUSPENDED,
            machine.MachineStates.ABORTED: EngineStates.STOPPED,
            machine.MachineStates.STUCK: EngineStates.STOPPED,
            machine.MachineStates.RESTORING: EngineStates.STOPPED,
            machine.MachineStates.SNAPSHOTTING: EngineStates.STOPPED,
            machine.MachineStates.SETTING_UP: EngineStates.STOPPED,
            machine.MachineStates.ONLINE_SNAPSHOTTING: EngineStates.STOPPED,
            machine.MachineStates.RESTORING_SNAPSHOT: EngineStates.STOPPED,
            machine.MachineStates.DELETING_SNAPSHOT: EngineStates.STOPPED,
            machine.MachineStates.LIVE_SNAPSHOTTING: EngineStates.RUNNING,
            machine.MachineStates.RUNNING: EngineStates.RUNNING,
            machine.MachineStates.STARTING: EngineStates.RUNNING,
            machine.MachineStates.STOPPING: EngineStates.RUNNING,
            machine.MachineStates.SAVING: EngineStates.RUNNING,
            machine.MachineStates.UNKNOWN: EngineStates.UNKNOWN,
            machine.MachineStates.INACCESSIBLE: EngineStates.INEXISTENT,
            machine.MachineStates.INEXISTENT: EngineStates.INEXISTENT
        }
        state = mapping.get(vboxState, EngineStates.UNKNOWN)
        ddebug("Machine state: %s : %s", vboxState, state)
        return OK(state)