class Core(object):
def __init__(self, configFile=None, basePath=None):
self.basePath = os.path.abspath(
basePath) if basePath else os.path.expanduser(
os.path.join('~', '.substance'))
self.enginesPath = os.path.join(self.basePath, "engines")
self.boxesPath = os.path.join(self.basePath, "boxes")
self.dbFile = os.path.join(self.basePath, "db.json")
configFile = configFile if configFile else "substance.yml"
configFile = os.path.join(self.basePath, configFile)
self.config = Config(configFile)
self.insecureKey = None
self.insecurePubKey = None
self.assumeYes = False
self.initialized = False
def getBasePath(self):
return self.basePath
def getEnginesPath(self):
return self.enginesPath
def getBoxesPath(self):
return self.boxesPath
def getDbFile(self):
return self.dbFile
def initialize(self):
if self.initialized:
return OK(None)
return self.assertPaths().then(self.assertConfig).then(
self.initializeDB).then(defer(self.setInitialized, b=True))
def setInitialized(self, b):
self.initialized = b
def assertPaths(self):
return OK([self.basePath, self.enginesPath,
self.boxesPath]).mapM(Shell.makeDirectory)
def assertConfig(self):
return self.config.loadConfigFile() \
.catchError(FileDoesNotExist, self.makeDefaultConfig)
def getDefaultConfig(self):
defaults = OrderedDict()
defaults['assumeYes'] = False
defaults['drivers'] = ['virtualbox']
defaults['tld'] = '.dev'
defaults['devroot'] = os.path.join('~', 'substance')
defaults['current'] = OrderedDict()
defaults['engine'] = None
defaults['subenv'] = None
return defaults
def makeDefaultConfig(self, data=None):
logger.info("Generating default substance configuration in %s",
self.config.getConfigFile())
defaults = self.getDefaultConfig()
for kkk, vvv in defaults.iteritems():
self.config.set(kkk, vvv)
self.config.set("basePath", self.basePath)
return self.config.saveConfig()
# -- Use
def setUse(self, engine, subenvName=None):
ops = [self.setCurrentEngine(engine)]
if subenvName:
ops.append(engine.envSwitch(subenvName))
return Try.sequence(ops)
def setCurrentEngine(self, engine):
current = self.config.get('current')
current.update({'engine': engine.name})
self.config.set('current', current)
return OK(self)
def readCurrentEngineName(self):
current = self.config.get('current', {})
name = current.get('engine', None)
if not name:
return Fail(
EngineNotFoundError(
"No current engine is specified. Check the 'use' command for details."
))
return OK(name)
def loadCurrentEngine(self, name=None):
current = self.config.get('current', {})
engineName = name
if not engineName:
engineName = current.get('engine', None)
if not engineName:
return Fail(
EngineNotFoundError(
"No current engine is specified. Check the 'use' command for details."
))
engine = self.loadEngine(engineName) \
.bind(Engine.loadConfigFile) \
.bind(Engine.loadState)
if engine.isFail():
return engine
engine = engine.getOK()
if engine.state is not EngineStates.RUNNING:
return Fail(
EngineNotRunning("Engine '%s' is not running." % engine.name))
return OK(engine)
# -- Runtime
def setAssumeYes(self, ay):
self.assumeYes = True
return True
def getAssumeYes(self):
if self.config.get('assumeYes', False):
return True
elif self.assumeYes:
return True
return False
def getDefaultBoxString(self):
return DefaultEngineBox
# -- Engine library management
def getEngines(self):
ddebug("getEngines()")
dirs = [
d for d in os.listdir(self.enginesPath)
if os.path.isdir(os.path.join(self.enginesPath, d))
]
return OK(dirs)
def loadEngines(self, engines=[]):
return OK([self.loadEngine(x) for x in engines])
def loadEngine(self, name):
enginePath = os.path.join(self.enginesPath, name)
if not os.path.isdir(enginePath):
return Fail(
EngineNotFoundError("Engine \"%s\" does not exist." % name))
else:
return OK(Engine(name, enginePath=enginePath, core=self))
def createEngine(self, name, config=None, profile=None):
enginePath = os.path.join(self.enginesPath, name)
newEngine = Engine(name, enginePath=enginePath, core=self)
return newEngine.create(config=config, profile=profile)
def removeEngine(self, name):
return self.loadEngine(name) \
>> Engine.remove
# -- Driver handling
def getDrivers(self):
return self.config.get('drivers', [])
def validateDriver(self, driver):
if driver in self.getDrivers():
return OK(driver)
return Fail(ValueError("Driver '%s' is not a valid driver."))
def getDriver(self, name):
cls = {'virtualbox': VirtualBoxDriver}.get(name, 'virtualbox')
driver = cls(core=self)
return driver
# -- Link handling
def getLink(self, type="ssh"):
link = Link(keyFile=self.getInsecureKeyFile(), keyFormat='RSA')
return link
# -- Database
def getDB(self):
return self.db
def initializeDB(self):
db = DB(self.dbFile)
db = db.initialize()
if db.isFail():
return db
self.db = db.getOK()
return OK(self.db)
# -- Box handling
def readBox(self, boxstring):
return Box.parseBoxString(boxstring) \
.map(lambda p: Box(core=self, **p))
def pullBox(self, box):
return box.fetch()
def removeBox(self, box):
return box.delete()
def getBoxes(self):
return self.getDB().getBoxRecords() \
.mapM(lambda r: OK(Box(self, r.get('name'), r.get('version'), r.get('namespace'), r.get('registry'), r.get('boxstring'), r.get('archiveSHA1'))))
def getInsecureKeyFile(self):
return getSupportFile('support/substance_insecure')
def getInsecurePubKeyFile(self):
return getSupportFile('support/substance_insecure.pub')
class Engine(object):
def __init__(self, name, enginePath, core):
self.name = name
self.enginePath = enginePath
self.core = core
self.link = None
configFile = os.path.join(self.enginePath, "engine.yml")
self.config = Config(configFile)
self.logAdapter = EngineLogAdapter(logger, self.__dict__)
self.currentEnv = None
self.subenvConfig = None
def getDefaults(self):
defaults = OrderedDict()
defaults['name'] = 'default'
defaults['driver'] = 'virtualbox'
defaults['id'] = None
defaults['box'] = DefaultEngineBox
defaults['profile'] = EngineProfile().__dict__
defaults['docker'] = OrderedDict()
defaults['docker']['port'] = 2375
defaults['docker']['tls'] = False
defaults['docker']['certificateFile'] = None
defaults['network'] = OrderedDict()
defaults['network']['privateIP'] = None
defaults['network']['publicIP'] = None
defaults['network']['sshIP'] = None
defaults['network']['sshPort'] = 4500
defaults['devroot'] = OrderedDict()
defaults['devroot']['path'] = getHomeDirectory('devroot')
defaults['devroot']['mode'] = 'unison'
defaults['devroot']['syncArgs'] = [
'-ignore', 'Path */var', '-ignore', 'Path */data'
]
defaults['devroot']['excludes'] = [
'{.*,*}.sw[pon]', '.bash*', '.composer', '.git', '.idea', '.npm',
'.ssh', '.viminfo'
]
aliases = ['make', 'composer', 'npm', 'heap', 'watch']
defaults['aliases'] = {
alias: {
'container': 'web',
'user': '******',
'cwd': '/vol/website',
'args': [alias]
}
for alias in aliases
}
defaults['aliases']['watch']['container'] = 'devs'
defaults['aliases']['heap']['args'][0] = 'vendor/bin/heap'
return defaults
def validateConfig(self, config):
fields = [
'name', 'driver', 'profile', 'docker', 'network', 'devroot', 'box'
]
ops = []
ops.append(
self.config.validateFieldsPresent(config.getConfig(), fields))
ops.append(self.confValidateName(config.get('name', None)))
ops.append(self.core.validateDriver(config.get('driver', None)))
ops.append(self.confValidateDevroot(config.get('devroot', {})))
def dd(err):
print(("%s" % err))
return ConfigValidationError(err.message)
return Try.sequence(ops) \
.catch(dd) \
.then(lambda: OK(config.getConfig()))
def confValidateName(self, name):
if name != self.name:
return Fail(
ConfigValidationError(
"Invalid name property in configuration (got %s expected %s)"
% (config.get('name'), self.name)))
return OK(name)
def confValidateDevroot(self, devroot):
path = devroot.get('path', '')
path = os.path.expanduser(path)
if not path:
return Fail(
ConfigValidationError(
"devroot path configuration is missing."))
elif not os.path.isdir(path):
self.logAdapter.info(
"WARNING: devroot '%s' does not exist locally." % path)
mode = devroot.get('mode', None)
if not mode:
return Fail(ConfigValidationError("devroot mode is not set."))
elif mode not in ['sharedfolder', 'rsync', 'unison']:
# XXX Fix hardcoded values.
return Fail(
ConfigValidationError("devroot mode '%s' is not valid." %
mode))
return OK(devroot)
def getName(self):
return self.name
def getID(self):
return self.config.id
def getEnginePath(self):
return self.enginePath
def getEngineFilePath(self, fileName):
return os.path.join(self.enginePath, fileName)
def getDockerURL(self):
if self.getDriverID():
return "tcp://%s:%s" % (self.getPublicIP(), self.getDockerPort())
def getDockerPort(self):
return self.config.get('docker', {}).get('port', 2375)
def getDockerEnv(self):
env = OrderedDict()
# env['DOCKER_API_VERSION'] = '1.19'
env['DOCKER_HOST'] = self.getDockerURL()
env['DOCKER_TLS_VERIFY'] = ''
return env
def getPublicIP(self):
return self.config.get('network').get('publicIP', None)
def getPrivateIP(self):
return self.config.get('network').get('privateIP', None)
def getDriver(self):
return self.core.getDriver(self.config.get('driver'))
def getEngineProfile(self):
profile = self.config.get('profile', {})
return EngineProfile(**profile)
def getDriverID(self):
return self.config.get('id', None)
def setDriverID(self, driverID):
self.config.set('id', driverID)
return OK(self)
def getSSHPort(self):
return self.config.getBlockKey('network', 'sshPort', 22)
def getSSHIP(self):
return self.config.getBlockKey('network', 'sshIP', 'localhost')
def getConnectInfo(self):
info = {'hostname': self.getSSHIP(), 'port': self.getSSHPort()}
return info
def getDNSName(self):
tld = self.core.config.get('tld')
return self.name + tld
def getSyncher(self):
syncMode = self.config.get('devroot').get('mode')
keyfile = self.core.getInsecureKeyFile()
return UnisonSyncher(engine=self, keyfile=keyfile)
def clearDriverID(self):
self.config.set('id', None)
return OK(self)
def loadConfigFile(self):
return self.config.loadConfigFile().bind(self.validateConfig).map(
self.chainSelf)
def loadSubenvConfigFile(self):
basePath = os.path.expanduser(self.config.get('devroot').get('path'))
self.subenvConfig = Config(
os.path.join(basePath,
'%s/.substance/subenv.yml' % self.currentEnv))
return self.subenvConfig.loadConfigFile().map(self.chainSelf)
def loadState(self):
ddebug("Engine load state %s" % self.name)
return self.fetchState().bind(self.setState).map(self.chainSelf)
def setState(self, state):
self.state = state
return OK(None)
def create(self, config=None, profile=None):
if os.path.isdir(self.enginePath):
return Fail(
EngineExistsError("Engine \"%s\" already exists." % self.name))
return Shell.makeDirectory(self.enginePath) \
.then(defer(self.makeDefaultConfig, config=config, profile=profile)) \
.bind(self.__ensureDevroot) \
.bind(self.validateConfig) \
.then(self.config.saveConfig) \
.bind(dinfo("Generated default engine configuration in %s", self.config.configFile)) \
.map(self.chainSelf)
def __ensureDevroot(self, config):
devroot = os.path.expanduser(config.get('devroot', {}).get('path'))
if not os.path.isdir(devroot):
self.logAdapter.info("Creating devroot at %s" % devroot)
return Shell.makeDirectory(devroot).then(lambda: OK(config))
return OK(config)
def remove(self):
if not os.path.isdir(self.enginePath):
return Fail(
FileSystemError("Engine \"%s\" does not exist." % self.name))
if self.isProvisioned():
return Fail(
EngineProvisioned("Engine \"%s\" is provisioned." % self.name))
return Shell.nukeDirectory(self.enginePath)
def makeDefaultConfig(self, config=None, profile=None):
default = self.getDefaults()
default["name"] = self.name
default["devroot"]["path"] = os.path.join(
self.core.config.get('devroot'), self.name)
cf = {}
mergeDict(cf, default)
mergeDict(cf, config)
if profile:
cf['profile']['cpus'] = profile.cpus
cf['profile']['memory'] = profile.memory
for ck, cv in cf.items():
self.config.set(ck, cv)
return OK(self.config)
def getProfile(self):
prof = self.config.get('profile', {})
return EngineProfile(prof.get('cpus'), prof.get('memory'))
def isProvisioned(self):
'''
Check that this engine has an attached Virtual Machine.
'''
return True if self.getDriverID() else False
def validateProvision(self):
'''
Check that the provision attached to this engine is valid in the driver.
'''
if not self.isProvisioned():
return OK(False)
machID = self.getDriverID()
return self.getDriver().exists(machID)
def isRunning(self):
return self.fetchState().map(lambda state:
(state is EngineStates.RUNNING))
def isSuspended(self):
return self.fetchState().map(lambda state:
(state is EngineStates.SUSPENDED))
def fetchState(self):
if not self.isProvisioned():
return OK(EngineStates.INEXISTENT)
return self.getDriver().getMachineState(self.getDriverID())
def launch(self):
'''
# 1. Check that we know about a provisioned machined for this engine. Provision if not.
# 2. Validate that the machine we know about is still provisioned. Provision if not.
# 3. Check the machine state, boot accordingly
# 4. Setup guest networking
# 4. Fetch the guest IP from the machine and store it in the machine state.
'''
return self.provision() \
.then(self.start) \
.then(self.updateNetworkInfo) \
.then(self.addToHostsFile) \
.then(self.envLoadCurrent) \
.catchError(EnvNotDefinedError, lambda e: OK(None)) \
.then(dinfo("Engine \"%s\" has been launched.", self.name)) \
.then(self.envStart if self.currentEnv else lambda: OK(None))
def addToHostsFile(self):
self.logAdapter.info("Registering engine as '%s' in local hosts file" %
self.getDNSName())
return SubHosts.register(self.getDNSName(), self.getPublicIP())
def removeFromHostsFile(self):
self.logAdapter.info("Removing engine from local hosts file")
return SubHosts.unregister(self.getDNSName())
def updateNetworkInfo(self):
self.logAdapter.info("Updating network info from driver")
return self.__readDriverNetworkInfo().bind(self.saveDriverNetworkInfo)
def postLaunch(self):
return self.setHostname() \
.then(self.mountFolders)
def saveDriverNetworkInfo(self, info):
self.logAdapter.debug("Network information for machine: %s" % info)
net = self.config.get('network', OrderedDict())
net.update(info)
self.config.set('network', net)
return self.config.saveConfig()
def start(self):
state = self.fetchState()
if state.isFail():
return state
if state.getOK() is EngineStates.RUNNING:
return EngineAlreadyRunning("Engine \"%s\" is already running" %
self.name)
if state.getOK() is EngineStates.SUSPENDED:
return self.__start().then(self.__waitForReady)
else:
return self.__configure() \
.then(self.updateNetworkInfo) \
.then(self.__start) \
.then(self.__waitForReady) \
.then(self.__waitForNetwork) \
.then(self.postLaunch)
def readBox(self):
return self.core.readBox(self.config.get('box'))
def provision(self):
logger.info("Provisioning engine \"%s\" with driver \"%s\"", self.name,
self.config.get('driver'))
prov = self.validateProvision()
if prov.isFail():
return prov
elif prov.getOK():
return OK(self)
box = self.readBox().bind(Box.fetch)
if box.isFail():
return box
box = box.getOK()
return self.getDriver().importMachine(self.name, box.getOVFFile(), self.getEngineProfile()) \
.bind(self.setDriverID) \
.then(self.config.saveConfig) \
.map(self.chainSelf)
# .then(self.__configure) \
# .then(self.updateNetworkInfo) \
def deprovision(self):
if not self.isProvisioned():
return Fail(
EngineNotProvisioned(
"Engine \"%s\" is not provisioned." % self.name, self))
self.__cacheLink(None)
return self.validateProvision() \
.thenIfTrue(self.isRunning) \
.thenIfTrue(self.__terminate) \
.then(self.__delete) \
.then(self.clearDriverID) \
.then(self.config.saveConfig) \
.then(self.removeFromHostsFile) \
.then(dinfo("Engine \"%s\" has been deprovisioned.", self.name)) \
.map(self.chainSelf)
def suspend(self):
if not self.isProvisioned():
return Fail(
EngineNotProvisioned("Engine \"%s\" is not provisioned." %
self.name))
self.__cacheLink(None)
return self.validateProvision() \
.thenIfTrue(self.isRunning) \
.thenIfFalse(failWith(EngineNotRunning("Engine \"%s\" is not running." % self.name))) \
.then(self.__suspend) \
.then(dinfo("Engine \"%s\" has been suspended.", self.name)) \
.map(chainSelf)
# XXX Insert wait for suspension
def stop(self, force=False):
self.__cacheLink(None)
operation = self.validateProvision() \
.thenIfTrue(self.isRunning) \
.thenIfFalse(failWith(EngineNotRunning("Engine \"%s\" is not running." % self.name)))
if (force):
operation >> self.__terminate >> dinfo(
"Engine \"%s\" has been terminated.", self.name)
else:
operation >> self.__halt >> dinfo("Engine \"%s\" has been halted.",
self.name)
# XXX insert wait for stopped state
return operation.map(self.chainSelf)
def readLink(self):
if self.link is not None:
return OK(self.link)
link = self.core.getLink()
return link.connectEngine(self).map(self.__cacheLink)
def __cacheCurrentEnv(self, lr):
env = lr.stdout.strip()
self.logAdapter.debug("Current environment: '%s'" % (env))
self.currentEnv = env
return lr
def __cacheLink(self, link):
self.link = link
return link
def __start(self, *args):
return self.getDriver().startMachine(self.getDriverID()).map(
self.chainSelf)
def __suspend(self, *args):
return self.getDriver().suspendMachine(self.getDriverID()).map(
self.chainSelf)
def __terminate(self, *args):
return self.getDriver().terminateMachine(self.getDriverID()).map(
self.chainSelf)
def __halt(self, *args):
return self.getDriver().haltMachine(self.getDriverID()).map(
self.chainSelf)
def __delete(self, *args):
return self.getDriver().deleteMachine(self.getDriverID()).map(
self.chainSelf)
def __readDriverNetworkInfo(self):
return self.getDriver().readMachineNetworkInfo(self.getDriverID())
def __configure(self):
return self.getDriver().configureMachine(self.getDriverID(), self)
def shell(self):
return self.readLink() >> Link.interactive
def __waitForNetwork(self):
self.logAdapter.info("Waiting for machine network...")
return self.getDriver().readMachineWaitForNetwork(self.getDriverID())
def __waitForReady(self):
self.logAdapter.info("Waiting for machine to boot...")
return self.readLink()
def setHostname(self):
self.logAdapter.info("Configuring machine hostname")
dns = self.getDNSName()
echoCmd = "echo %s > /etc/hostname" % dns
hostCmd = "hostname %s" % dns
hostsCmd = "sed -i \"s/127.0.1.1.*/127.0.1.1\t%s/g\" /etc/hosts" % self.name
# hostsCmd = "hostnamectl set-hostname %s" % dns
serviceCmd = "service hostname restart"
cmd = "sudo -- /bin/sh -c '%s && %s && %s && %s'" % (
echoCmd, hostsCmd, hostCmd, serviceCmd)
cmds = list(
map(defer(self.link.runCommand, stream=True, sudo=False), [cmd]))
return Try.sequence(cmds)
def exposePort(self, local_port, public_port, scheme):
keyfile = self.core.getInsecureKeyFile()
keyfile = inner(keyfile)
forward_descr = "0.0.0.0:%s:127.0.0.1:%s" % (public_port, local_port)
engineIP = self.getSSHIP()
enginePort = str(self.getSSHPort())
cmdPath = "ssh"
cmdArgs = [
"ssh", "-N", "-L", forward_descr, engineIP, "-l", "substance",
"-p", enginePort, "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", keyfile
]
sudo = False
if public_port < 1024 and not isWithinWindowsSubsystem():
sudo = True
self.logAdapter.info(
"Exposing port %s as %s; kill the process (CTRL-C) to un-expose.",
local_port, public_port)
with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as s:
s.connect(("8.8.8.8", 80))
ip = s.getsockname()[0]
if scheme:
self.logAdapter.info("%s://%s:%s is now available.", scheme,
ip, public_port)
else:
self.logAdapter.info(
"Others can now connect to port %s via IP %s.",
public_port, ip)
Shell.execvp(cmdPath, cmdArgs, {}, sudo=sudo)
def getOrchestrator(self):
if self.subenvConfig:
return self._getOrchestrator()
return self.envLoadCurrent() \
.then(self.loadSubenvConfigFile) \
.catch(lambda err: OK(None)) \
.then(self._getOrchestrator)
def _getOrchestrator(self):
orchestrator = self.subenvConfig.get(
'orchestrator') if self.subenvConfig else None
logger.debug("Orchestrator: %s" % orchestrator)
if orchestrator and orchestrator == Orchestrators.COMPOSE:
return OK(Compose(self))
elif orchestrator and orchestrator != Orchestrators.DOCKWRKR:
return Fail(
InvalidOrchestrator('%s is not a known orchestrator' %
orchestrator))
else:
return OK(Dockwrkr(self))
def getOrchestratorCommand(self, command, *args):
orch = self.getOrchestrator()
if orch.isFail():
raise orch.value
else:
orch = orch.getOK()
if orch:
return getattr(orch, command)(*args)
else:
return ''
def orchestrate(self, command, *args):
commands = self.getOrchestratorCommand(command, *args)
if len(commands) > 0:
return list(map(lambda x: "(subenv run %s)" % x, commands))
def envSwitch(self, subenvName, restart=False):
self.logAdapter.info("Switch engine '%s' to subenv '%s'" %
(self.name, subenvName))
self.currentEnv = subenvName
self.loadSubenvConfigFile()
cmds = CommandList([
"subenv init '/substance/devroot/%s'" % (subenvName),
"subenv use '%s'" % (subenvName),
])
if restart:
cmds.append(self.orchestrate('resetAll'))
cmds.append(self.orchestrate('login'))
cmds.append(self.orchestrate('startAll'))
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False) \
.then(self.envRegister)
def envRegister(self):
return self.readLink() \
.bind(Link.runCommand, 'subenv vars', stream=False, capture=True) \
.bind(self.__envRegister)
def __envRegister(self, lr):
vars = lr.stdout.split("\n")
env = Try.attempt(parseDotEnv, vars)
if env.isFail():
return env
env = env.getOK()
if 'SUBENV_FQDN' in env:
return SubHosts.register(env['SUBENV_FQDN'], self.getPublicIP())
elif 'SUBENV_NAME' in env:
return SubHosts.register(
env['SUBENV_NAME'] + self.core.config.get('tld'),
self.getPublicIP())
def envLoadCurrent(self):
cmds = CommandList(["subenv current"])
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=False, sudo=False, capture=True) \
.map(self.__cacheCurrentEnv) \
.catch(lambda err: Fail(EnvNotDefinedError("No current subenv is set. Check 'switch' for detais."))) \
.then(lambda: self)
def envStart(self, reset=False, containers=[]):
cmds = CommandList()
if reset:
cmds.append(self.getOrchestratorCommand('resetAll'))
if len(containers) > 0:
self.logAdapter.info("Starting %s container(s)" %
(' '.join(containers)))
cmds.append(self.orchestrate('start', containers))
else:
self.logAdapter.info("Starting all containers...")
cmds.append(self.orchestrate('startAll'))
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)
def envRestart(self, time=10, containers=[]):
cmds = CommandList()
if len(containers) > 0:
self.logAdapter.info("Restarting %s containers" %
(' '.join(containers)))
cmds.append(self.orchestrate('restart', containers, time))
else:
self.logAdapter.info("Restarting all containers...")
cmds.append(self.orchestrate('restartAll', time))
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)
def envRecreate(self, time=10, containers=[]):
cmds = CommandList()
if len(containers) > 0:
self.logAdapter.info("Recreating %s containers" %
(' '.join(containers)))
cmds.append(self.orchestrate('recreate', containers, time))
else:
self.logAdapter.info("Recreating all containers...")
cmds.append(self.orchestrate('recreateAll', time))
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)
def envShell(self, container=None, user=None, cwd=None):
if container:
return self.envEnter(container, user, cwd)
else:
return self.readLink().bind(Link.interactive)
def envEnter(self, container, user=None, cwd=None):
self.logAdapter.info("Entering %s container..." % (container))
return self.envExec(container, ["exec /bin/bash"], cwd, user)
def envExec(self, container, cmd, cwd=None, user=None):
cmd = CommandList(self.orchestrate('exec', container, cmd, cwd, user))
return self.readLink().bind(Link.runCommand,
cmd=cmd.logicAnd(),
interactive=True,
stream=True,
shell=False,
capture=False)
def envRun(self, task, args):
cmd = CommandList(self.orchestrate('run', task, args))
return self.readLink().bind(Link.runCommand,
cmd=cmd.logicAnd(),
interactive=True,
stream=True,
shell=False,
capture=False)
def envExecAlias(self, alias, args):
aliases = self.config.get('aliases') if self.config.get(
'aliases') else {}
subenvAliases = self.subenvConfig.get(
'aliases') if self.subenvConfig.get('aliases') else {}
if subenvAliases:
aliases = mergeDictOverwrite(aliases, subenvAliases)
if aliases and alias in aliases:
cmd = aliases[alias]
args = cmd['args'] + args
return self.envExec(container=cmd['container'],
cmd=args,
cwd=cmd['cwd'],
user=cmd['user'])
return Fail(
InvalidCommandError(
"Invalid command '%s' specified for '%s'.\n\nUse 'substance help' for available commands."
% (alias, 'substance')))
def envStop(self, time=10, containers=[]):
cmds = CommandList()
if len(containers) > 0:
self.logAdapter.info("Stopping %s container(s)" %
(' '.join(containers)))
cmds.append(self.orchestrate('stop', containers, time))
else:
self.logAdapter.info("Stopping containers...")
cmds.append(self.orchestrate('stopAll', time))
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, sudo=False)
def envStatus(self, full=False):
if full:
cmds = CommandList([self.orchestrate('status')])
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=False, sudo=False, capture=True) \
.map(self.__envStatus)
else:
return OK(self.__envStatus())
def envCleanup(self):
cmds = CommandList()
cmds.append('docker system prune -a')
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), interactive=True, stream=True, shell=False, capture=False)
def envLogs(self, parts=[], pattern=None, follow=True, lines=None):
if not pattern:
pattern = "%s*.log" % ('-'.join(parts))
cmds = CommandList()
cmd = "tail"
if follow:
cmd += " -f"
if lines:
cmd += " -n %s" % int(lines)
cmd += " \"logs/%s\"" % pattern
cmds.append('subenv run %s' % cmd)
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, capture=False, sudo=False)
def envListLogs(self, parts=[], pattern=None):
if not pattern:
pattern = "%s*.log" % ('-'.join(parts))
cmds = CommandList()
cmd = "ls -1 \"logs/%s\" | xargs -n1 basename" % pattern
cmds.append('subenv run %s' % cmd)
return self.readLink() \
.bind(Link.runCommand, cmds.logicAnd(), stream=True, capture=False, sudo=False)
def __envStatus(self, containers=None):
return {
'engine':
self,
'containers':
containers.stdout + containers.stderr if containers else None
}
def envDocker(self, command):
cmd = "docker %s" % command
logger.debug("DOCKER: %s" % cmd)
return self.readLink().bind(Link.runCommand,
cmd,
stream=True,
interactive=True,
shell=False)
def getEngineFolders(self):
# XXX Dynamic mounts / remove hardcoded values.
devroot = self.config.get('devroot')
pfolder = EngineFolder(name='devroot',
mode=devroot.get('mode'),
hostPath=os.path.expanduser(
devroot.get('path')),
guestPath='/substance/devroot',
uid=1000,
gid=1000,
umask="0022",
excludes=devroot.get('excludes', []),
syncArgs=devroot.get('syncArgs', []))
return [pfolder]
def mountFolders(self):
self.logAdapter.info("Mounting engine folders")
folders = self.getEngineFolders()
return Try.of(list(map(self.mountFolder, folders)))
def mountFolder(self, folder):
mountCmd = "mount -t vboxsf -o umask=%(umask)s,gid=%(gid)s,uid=%(uid)s %(name)s %(guestPath)s" % folder.__dict__
mkdirCmd = "mkdir -p %(guestPath)s && chown -R %(uid)s:%(gid)s %(guestPath)s" % folder.__dict__
# XXX Make this non VBOX specific
chain = self.link.runCommand(mkdirCmd, sudo=True)
if folder.mode is 'sharedfolder':
chain = chain.then(defer(self.link.runCommand, mountCmd,
sudo=True))
return chain
chainSelf = chainSelf
def __repr__(self):
return "Engine(%s)" % self.name
class VirtualBoxDriver(Driver):
'''
Substance VirtualBox driver class. Interface to virtual box manager.
'''
def __init__(self, core):
super(self.__class__, self).__init__(core)
self.config = Config(
os.path.join(self.core.getBasePath(), "virtualbox.yml"))
def assertSetup(self):
return self.assertConfig().then(self.assertNetworking)
# -- Configuration
def getDefaultConfig(self):
defaults = OrderedDict()
defaults['network'] = "172.21.21.0/24"
defaults['interface'] = None
return defaults
def makeDefaultConfig(self):
self.logAdapter.info("Generating default virtualbox config in %s" %
self.config.getConfigFile())
defaults = self.getDefaultConfig()
for kkk, vvv in defaults.items():
self.config.set(kkk, vvv)
return self.config.saveConfig()
def assertConfig(self):
return self.config.loadConfigFile() \
.catchError(FileDoesNotExist, lambda err: self.makeDefaultConfig())
# ---- Networking setup
def getNetworkInterface(self):
return self.config.get('interface', None)
def readNetworkConfig(self):
netrange = IPNetwork(self.config.get('network'))
netconfig = {
'gateway': netrange[1].format(),
'netmask': netrange.netmask.format(),
'lowerIP': netrange[5].format(),
'upperIP': netrange[-5].format()
}
return OK(netconfig)
def assertNetworking(self):
interface = self.getNetworkInterface()
netconfig = self.readNetworkConfig()
if interface:
hoif = network.readHostOnlyInterface(interface) \
.catchError(VirtualBoxObjectNotFound, lambda err: OK(None))
dhcp = network.readDHCP(interface) \
.catchError(VirtualBoxObjectNotFound, lambda err: OK(None))
return Try.sequence((netconfig, hoif, dhcp)) \
.bind(self.assertNetworkConfiguration)
else:
return netconfig.bind(self.provisionNetworking)
def assertNetworkConfiguration(self, netparts):
(netconfig, hoif, dhcp) = netparts
if not hoif:
return self.provisionNetworking(netconfig)
elif hoif.v4ip.format() != netconfig['gateway']:
self.logAdapter.warning(
"VirtualBox interface \"%s\" is not properly configured. Creating a new host-only network."
% hoif.name)
return self.provisionNetworking(netconfig)
elif dhcp is None:
self.logAdapter.warning(
"VirtualBox interface \"%s\" does not have DHCP enabled. Re-Establishing now."
% hoif.name)
return self.provisionDHCP(hoif.name, netconfig)
return OK(hoif)
def findInterface(self, netconfig, interfaces):
matches = list(
filter(lambda x: x.v4ip == netconfig['gateway'], interfaces))
return OK(None) if len(matches) <= 0 else OK(matches[0])
def provisionNetworking(self, netconfig):
self.logAdapter.info(
"Provisioning VirtualBox networking for substance")
hoif = network.readHostOnlyInterfaces() \
.bind(defer(self.findInterface, netconfig))
if hoif.isFail():
return hoif
hoif = hoif.getOK()
if not hoif:
ifm = network.addHostOnlyInterface()
if ifm.isFail():
return ifm
iface = ifm.getOK()
else:
iface = hoif.name
return network.configureHostOnlyInterface(iface, ip=netconfig['gateway'], netmask=netconfig['netmask']) \
.then(defer(self.provisionDHCP, interface=iface, netconfig=netconfig)) \
.then(defer(self.saveInterface, iface=iface)) \
.then(defer(network.readHostOnlyInterface, name=iface))
def provisionDHCP(self, interface, netconfig):
self.logAdapter.info(
"Provisioning DHCP service for host only interface")
network.removeDHCP(interface).catch(lambda x: OK(interface)) \
.then(defer(network.addDHCP, interface, **netconfig))
def saveInterface(self, iface):
self.config.set('interface', iface)
return self.config.saveConfig()
# ---- Machine API
def importMachine(self, name, ovfFile, engineProfile=None):
return self.assertSetup() \
.then(defer(machine.inspectOVF, ovfFile)) \
.bind(defer(machine.makeImportParams, name=name, engineProfile=engineProfile)) \
.bind(defer(machine.importOVF, ovfFile=ovfFile, name=name))
def startMachine(self, uuid):
'''
Start the machine by driver identifier.
'''
state = machine.readMachineState(uuid)
if state.isFail():
return state
if state.getOK() in [machine.MachineStates.PAUSED]:
return machine.resume(uuid)
else:
return machine.start(uuid)
def readMachineWaitForNetwork(self, uuid, timeout=5000):
return machine.readWaitGuestProperty(uuid, "/VirtualBox/GuestInfo/Net",
timeout)
def readMachineNetworkInfo(self, uuid):
def format(res):
info = OrderedDict()
info['sshPort'] = res[0].hostPort if res[0] else None
info['sshIP'] = '127.0.0.1'
info['privateIP'] = res[1]
info['publicIP'] = res[2]
return info
return Try.sequence([
network.readPortForward(uuid, name="substance-ssh"),
machine.readGuestProperty(uuid,
"/VirtualBox/GuestInfo/Net/0/V4/IP"),
machine.readGuestProperty(uuid,
"/VirtualBox/GuestInfo/Net/1/V4/IP")
]).map(format)
def configureMachine(self, uuid, engine):
def engineFolderToVBoxFolder(acc, x):
if x.mode == 'sharedfolder':
acc.append(
machine.SharedFolder(name=x.name, hostPath=x.hostPath))
return acc
folders = reduce(engineFolderToVBoxFolder, engine.getEngineFolders(),
[])
desiredPort = engine.getSSHPort()
return self.assertSetup() \
.then(defer(self.resolvePortConflict, uuid=uuid, desiredPort=desiredPort)) \
.then(defer(self.configureMachineAdapters, uuid=uuid)) \
.then(defer(self.configureMachineFolders, uuid=uuid, folders=folders)) \
.then(defer(self.configureMachineProfile, uuid=uuid, engine=engine))
def configureMachineProfile(self, uuid, engine):
self.logAdapter.info("Configure machine profile")
profile = engine.getProfile()
return machine.configureProfile(uuid, profile.cpus, profile.memory)
def configureMachineFolders(self, folders, uuid):
self.logAdapter.info("Configure machine shared folders")
return machine.clearSharedFolders(uuid) \
.then(defer(machine.addSharedFolders, folders=folders, uuid=uuid))
def resolvePortConflict(self, uuid, desiredPort):
self.logAdapter.info("Checking for port conflicts")
return network.readAllPortForwards(ignoreUUIDs=[uuid]) \
.bind(defer(self.determinePort, desiredPort=desiredPort)) \
.bind(defer(self.configurePort, uuid=uuid))
def determinePort(self, usedPorts, desiredPort):
basePort = 4500
self.logAdapter.debug("Base port: %s" % basePort)
self.logAdapter.debug("Desired port: %s" % desiredPort)
unavailable = [x.hostPort for x in usedPorts]
port = desiredPort if desiredPort >= basePort else basePort
while port in unavailable or port < basePort:
self.logAdapter.debug("Port %s is in use." % port)
port += 1
self.logAdapter.info("Determined SSH port as %s" % port)
return OK(port)
def configurePort(self, port, uuid):
self.logAdapter.debug("Configure substance-ssh port on port %s" % port)
pf = network.PortForward("substance-ssh", 1, "tcp", None, port, None,
22)
return network.removePortForwards([pf], uuid) \
.catch(lambda err: OK(None)) \
.then(defer(network.addPortForwards, [pf], uuid))
def configureMachineAdapters(self, uuid):
interface = self.getNetworkInterface()
adapters = [
machine.AdapterSettings(machine.AdapterTypes.NAT, 'default', None,
False),
machine.AdapterSettings(machine.AdapterTypes.HOSTONLY, interface,
None, False),
machine.AdapterSettings(machine.AdapterTypes.NONE, None, None,
False),
machine.AdapterSettings(machine.AdapterTypes.NONE, None, None,
False)
]
return Try.sequence([
machine.configureAdapter(uuid, i + 1, x)
for i, x in enumerate(adapters)
])
def suspendMachine(self, uuid):
'''
Suspend the machine.
'''
return machine.suspend(uuid)
def haltMachine(self, uuid):
'''
Halt the machine.
'''
return machine.halt(uuid)
def terminateMachine(self, uuid):
'''
Terminate the machine forcefully.
'''
return machine.terminate(uuid)
def deleteMachine(self, uuid):
'''
Delete the machine by driver identifier.
'''
return machine.delete(uuid)
def exportMachine(self, uuid):
# XXX To be implemented
pass
def getMachines(self):
'''
Retrieve the list of machines and their driver identifiers.
'''
return machine.readMachines()
# -- Parse results from Virtual Box
def getMachineID(self, name):
'''
Retrieve the driver specific machine ID for a machine name.
'''
return machine.findMachineID(name)
def exists(self, uuid):
'''
Check in the driver that the specified identifier exists.
'''
return machine.readMachineExists(uuid)
def isRunning(self, uuid):
if self.getMachineState(uuid) is EngineStates.RUNNING:
return True
def isStopped(self, uuid):
if self.getMachineState(uuid) is not EngineStates.RUNNING:
return True
def isSuspended(self, uuid):
if self.getMachineState(uuid) is EngineStates.SUSPENDED:
return True
def getMachineState(self, uuid):
'''
Retrieve the Substance machine state for this driver id
'''
return machine.readMachineState(uuid) \
.bind(self.machineStateToEngineState)
def machineStateToEngineState(self, vboxState):
'''
Resolve a vbox machine state to a substance engine state.
'''
mapping = {
machine.MachineStates.POWEROFF: EngineStates.STOPPED,
machine.MachineStates.SAVED: EngineStates.SUSPENDED,
machine.MachineStates.PAUSED: EngineStates.SUSPENDED,
machine.MachineStates.ABORTED: EngineStates.STOPPED,
machine.MachineStates.STUCK: EngineStates.STOPPED,
machine.MachineStates.RESTORING: EngineStates.STOPPED,
machine.MachineStates.SNAPSHOTTING: EngineStates.STOPPED,
machine.MachineStates.SETTING_UP: EngineStates.STOPPED,
machine.MachineStates.ONLINE_SNAPSHOTTING: EngineStates.STOPPED,
machine.MachineStates.RESTORING_SNAPSHOT: EngineStates.STOPPED,
machine.MachineStates.DELETING_SNAPSHOT: EngineStates.STOPPED,
machine.MachineStates.LIVE_SNAPSHOTTING: EngineStates.RUNNING,
machine.MachineStates.RUNNING: EngineStates.RUNNING,
machine.MachineStates.STARTING: EngineStates.RUNNING,
machine.MachineStates.STOPPING: EngineStates.RUNNING,
machine.MachineStates.SAVING: EngineStates.RUNNING,
machine.MachineStates.UNKNOWN: EngineStates.UNKNOWN,
machine.MachineStates.INACCESSIBLE: EngineStates.INEXISTENT,
machine.MachineStates.INEXISTENT: EngineStates.INEXISTENT
}
state = mapping.get(vboxState, EngineStates.UNKNOWN)
ddebug("Machine state: %s : %s", vboxState, state)
return OK(state)