def __init__(self, conf):
self.conf = conf
self.persistManager = persist.ClusterPersistManager()
self.machineConf = config.configFromStream(open(conf('config.machine_conf')),
base=config.configFromEnv())
self.clusterLocks = lock_manager.LockManager()
self.unresponsiveClusters = {}
def __init__(self, conf):
self.conf = conf
self.pipelinePersist = persist.PipelinePersistManager()
self.tagNotify = tag_notify_listener.TagNotifyListener()
self.machineconf = config.configFromStream(open(conf('config.machine_conf')),
base=config.configFromEnv())
self.pipelinesCache = pipelines_cache.PipelinesCache(self.machineconf, self.pipelinePersist, self.tagNotify)
self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def __init__(self, conf):
self.conf = conf
self.persistManager = persist.ClusterPersistManager()
self.machineConf = config.configFromStream(open(
conf('config.machine_conf')),
base=config.configFromEnv())
self.clusterLocks = lock_manager.LockManager()
self.unresponsiveClusters = {}
def handleWWWListAddCredentials(request):
if 'credential_name' in request.body and core.keysInDict(
['credential_name', 'description', 'ctype', 'metadata'], request.body):
# Users can provide a file name or the actual contents of the
# certificate.
if 'cert_file' in request.body:
cert = open(request.body['cert_file']).read()
else:
cert = request.body['cert']
if 'pkey_file' in request.body:
pkey = open(request.body['pkey_file']).read()
else:
pkey = request.body['pkey']
conf = config.configFromMap(request.body.get('conf', {}),
base=config.configFromEnv())
cred = persist.createCredential(name=request.body['credential_name'],
desc=request.body['description'],
ctype=request.body['ctype'],
cert=cert,
pkey=pkey,
active=True,
metadata=request.body['metadata'],
conf=conf)
taskName = yield tasks_tx.createTaskAndSave('addCredential', 1)
instantiateAndSaveCredential(taskName, cred,
request.state.credentialPersist)
queue.returnQueueSuccess(request.mq, request.body['return_queue'],
taskName)
defer.returnValue(request)
elif 'credential_name' not in request.body:
credentials = request.state.credentialsCache.getAllCredentials()
credentialsDicts = [{
'name': name,
'description': c['cred_instance'].credential.desc,
'num_instances': len(c['instances']),
'ctype': c['cred_instance'].credential.getCType()
} for name, c in credentials.iteritems()
if ('credential_names' in request.body
and name in request.body['credential_names'])
or 'credential_names' not in request.body]
queue.returnQueueSuccess(request.mq, request.body['return_queue'],
credentialsDicts)
defer.returnValue(request)
else:
queue.returnQueueError(request.mq, request.body['return_queue'],
'Unknown credential query')
raise UnknownRequestError(str(request.body))
def __init__(self, conf):
self.conf = conf
self.pipelinePersist = persist.PipelinePersistManager()
self.tagNotify = tag_notify_listener.TagNotifyListener()
self.machineconf = config.configFromStream(open(
conf('config.machine_conf')),
base=config.configFromEnv())
self.pipelinesCache = pipelines_cache.PipelinesCache(
self.machineconf, self.pipelinePersist, self.tagNotify)
self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config')
options = list(pipeline.OPTIONS)
options.append(
('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)',
const('/tmp/machine.conf')))
options.append((
'CONFIG_FILE', '-c', '--CONFIG_FILE',
'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line',
const(tmpConfigName)))
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(conf,
config.configFromStream(
open('/tmp/machine.conf'),
config.configFromEnv()),
lazy=True)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v)
for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, 'w')
for s, d in confVals.iteritems():
if s not in ['', 'env']:
fout.write('[' + s + ']\n')
for k, v in d.iteritems():
fout.write('%s=%s\n' % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config")
options = list(pipeline.OPTIONS)
options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf")))
options.append(
(
"CONFIG_FILE",
"-c",
"--CONFIG_FILE",
"Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line",
const(tmpConfigName),
)
)
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(
conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True
)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, "w")
for s, d in confVals.iteritems():
if s not in ["", "env"]:
fout.write("[" + s + "]\n")
for k, v in d.iteritems():
fout.write("%s=%s\n" % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def handleWWWListAddCredentials(request):
if 'credential_name' in request.body and core.keysInDict(['credential_name',
'description',
'ctype',
'metadata'],
request.body):
# Users can provide a file name or the actual contents of the
# certificate.
if 'cert_file' in request.body:
cert = open(request.body['cert_file']).read()
else:
cert = request.body['cert']
if 'pkey_file' in request.body:
pkey = open(request.body['pkey_file']).read()
else:
pkey = request.body['pkey']
conf = config.configFromMap(request.body.get('conf', {}),
base=config.configFromEnv())
cred = persist.createCredential(name=request.body['credential_name'],
desc=request.body['description'],
ctype=request.body['ctype'],
cert=cert,
pkey=pkey,
active=True,
metadata=request.body['metadata'],
conf=conf)
taskName = yield tasks_tx.createTaskAndSave('addCredential', 1)
instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist)
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
taskName)
defer.returnValue(request)
elif 'credential_name' not in request.body:
credentials = request.state.credentialsCache.getAllCredentials()
credentialsDicts = [{'name': name,
'description': c['cred_instance'].credential.desc,
'num_instances': len(c['instances']),
'ctype': c['cred_instance'].credential.getCType()}
for name, c in credentials.iteritems()
if ('credential_names' in request.body and name in request.body['credential_names']) or
'credential_names' not in request.body]
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
credentialsDicts)
defer.returnValue(request)
else:
queue.returnQueueError(request.mq,
request.body['return_queue'],
'Unknown credential query')
raise UnknownRequestError(str(request.body))
def buildConfigN(options,
args=None,
usage=None,
baseConf=None,
putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(
open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
if (cluster.credName == 'local' and
conf('MASTER_IP') not in [cluster.master['public_dns'],
cluster.master['private_dns']]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(credential['name'],
credential['desc'],
credential['ctype'],
credential['cert'],
credential['pkey'],
credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost',
'local',
credTaskName)
credClient = cred_client.CredentialClient('local',
mq,
state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
cluster = persist.Cluster('local',
None,
'local',
conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(startTaskName,
lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def configFromStream(stream):
return fixVariables(config.configFromStream(stream, config.configFromEnv()))
import os
from igs.utils.logging import errorPrintS
from igs.utils.commands import runSingleProgram, ProgramRunError
from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr
##
# These are default config options, these will be moved to a config file eventually
conf = configFromStream(open('/tmp/machine.conf'), configFromMap({
'stow': {'package_dir': '/usr/local/stow',
'base_dir': '/usr/local'},
'opt': {'package_dir': '/opt/opt-packages',
'base_dir': '/opt'},
'config': {'filename': '/tmp/machine.conf'},
}, configFromEnv()))
##
# Exceptions
class PolicyError(Exception):
pass
##
# A little helper function
def runSystemEx(cmd):
"""This just ignores all stdout"""
code = runSingleProgram(cmd, None, errorPrintS)
if code != 0:
conf = configFromStream(
open('/tmp/machine.conf'),
configFromMap(
{
'stow': {
'package_dir': '/usr/local/stow',
'base_dir': '/usr/local'
},
'opt': {
'package_dir': '/opt/opt-packages',
'base_dir': '/opt'
},
'config': {
'filename': '/tmp/machine.conf'
},
}, configFromEnv()))
##
# Exceptions
class PolicyError(Exception):
pass
##
# A little helper function
def runSystemEx(cmd):
"""This just ignores all stdout"""
code = runSingleProgram(cmd, None, errorPrintS)
if code != 0:
raise ProgramRunError(cmd, code)
def configFromStream(stream):
return fixVariables(config.configFromStream(stream,
config.configFromEnv()))
