def __init__(self, conf): self.conf = conf self.persistManager = persist.ClusterPersistManager() self.machineConf = config.configFromStream(open(conf('config.machine_conf')), base=config.configFromEnv()) self.clusterLocks = lock_manager.LockManager() self.unresponsiveClusters = {}
def __init__(self, conf): self.conf = conf self.pipelinePersist = persist.PipelinePersistManager() self.tagNotify = tag_notify_listener.TagNotifyListener() self.machineconf = config.configFromStream(open(conf('config.machine_conf')), base=config.configFromEnv()) self.pipelinesCache = pipelines_cache.PipelinesCache(self.machineconf, self.pipelinePersist, self.tagNotify) self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def __init__(self, conf): self.conf = conf self.persistManager = persist.ClusterPersistManager() self.machineConf = config.configFromStream(open( conf('config.machine_conf')), base=config.configFromEnv()) self.clusterLocks = lock_manager.LockManager() self.unresponsiveClusters = {}
def handleWWWListAddCredentials(request): if 'credential_name' in request.body and core.keysInDict( ['credential_name', 'description', 'ctype', 'metadata'], request.body): # Users can provide a file name or the actual contents of the # certificate. if 'cert_file' in request.body: cert = open(request.body['cert_file']).read() else: cert = request.body['cert'] if 'pkey_file' in request.body: pkey = open(request.body['pkey_file']).read() else: pkey = request.body['pkey'] conf = config.configFromMap(request.body.get('conf', {}), base=config.configFromEnv()) cred = persist.createCredential(name=request.body['credential_name'], desc=request.body['description'], ctype=request.body['ctype'], cert=cert, pkey=pkey, active=True, metadata=request.body['metadata'], conf=conf) taskName = yield tasks_tx.createTaskAndSave('addCredential', 1) instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist) queue.returnQueueSuccess(request.mq, request.body['return_queue'], taskName) defer.returnValue(request) elif 'credential_name' not in request.body: credentials = request.state.credentialsCache.getAllCredentials() credentialsDicts = [{ 'name': name, 'description': c['cred_instance'].credential.desc, 'num_instances': len(c['instances']), 'ctype': c['cred_instance'].credential.getCType() } for name, c in credentials.iteritems() if ('credential_names' in request.body and name in request.body['credential_names']) or 'credential_names' not in request.body] queue.returnQueueSuccess(request.mq, request.body['return_queue'], credentialsDicts) defer.returnValue(request) else: queue.returnQueueError(request.mq, request.body['return_queue'], 'Unknown credential query') raise UnknownRequestError(str(request.body))
def __init__(self, conf): self.conf = conf self.pipelinePersist = persist.PipelinePersistManager() self.tagNotify = tag_notify_listener.TagNotifyListener() self.machineconf = config.configFromStream(open( conf('config.machine_conf')), base=config.configFromEnv()) self.pipelinesCache = pipelines_cache.PipelinesCache( self.machineconf, self.pipelinePersist, self.tagNotify) self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config') options = list(pipeline.OPTIONS) options.append( ('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)', const('/tmp/machine.conf'))) options.append(( 'CONFIG_FILE', '-c', '--CONFIG_FILE', 'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line', const(tmpConfigName))) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig(conf, config.configFromStream( open('/tmp/machine.conf'), config.configFromEnv()), lazy=True) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, 'w') for s, d in confVals.iteritems(): if s not in ['', 'env']: fout.write('[' + s + ']\n') for k, v in d.iteritems(): fout.write('%s=%s\n' % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config") options = list(pipeline.OPTIONS) options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf"))) options.append( ( "CONFIG_FILE", "-c", "--CONFIG_FILE", "Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line", const(tmpConfigName), ) ) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig( conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True ) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, "w") for s, d in confVals.iteritems(): if s not in ["", "env"]: fout.write("[" + s + "]\n") for k, v in d.iteritems(): fout.write("%s=%s\n" % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def handleWWWListAddCredentials(request): if 'credential_name' in request.body and core.keysInDict(['credential_name', 'description', 'ctype', 'metadata'], request.body): # Users can provide a file name or the actual contents of the # certificate. if 'cert_file' in request.body: cert = open(request.body['cert_file']).read() else: cert = request.body['cert'] if 'pkey_file' in request.body: pkey = open(request.body['pkey_file']).read() else: pkey = request.body['pkey'] conf = config.configFromMap(request.body.get('conf', {}), base=config.configFromEnv()) cred = persist.createCredential(name=request.body['credential_name'], desc=request.body['description'], ctype=request.body['ctype'], cert=cert, pkey=pkey, active=True, metadata=request.body['metadata'], conf=conf) taskName = yield tasks_tx.createTaskAndSave('addCredential', 1) instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist) queue.returnQueueSuccess(request.mq, request.body['return_queue'], taskName) defer.returnValue(request) elif 'credential_name' not in request.body: credentials = request.state.credentialsCache.getAllCredentials() credentialsDicts = [{'name': name, 'description': c['cred_instance'].credential.desc, 'num_instances': len(c['instances']), 'ctype': c['cred_instance'].credential.getCType()} for name, c in credentials.iteritems() if ('credential_names' in request.body and name in request.body['credential_names']) or 'credential_names' not in request.body] queue.returnQueueSuccess(request.mq, request.body['return_queue'], credentialsDicts) defer.returnValue(request) else: queue.returnQueueError(request.mq, request.body['return_queue'], 'Unknown credential query') raise UnknownRequestError(str(request.body))
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream( open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t') return {'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)} else: return {'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({})} try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [cluster.master['public_dns'], cluster.master['private_dns']]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential(credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask(startTaskName, lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split( '\t') return { 'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True) } else: return { 'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({}) } try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [ cluster.master['public_dns'], cluster.master['private_dns'] ]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential( credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask( startTaskName, lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find( lambda i: masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)
def configFromStream(stream): return fixVariables(config.configFromStream(stream, config.configFromEnv()))
import os from igs.utils.logging import errorPrintS from igs.utils.commands import runSingleProgram, ProgramRunError from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr ## # These are default config options, these will be moved to a config file eventually conf = configFromStream(open('/tmp/machine.conf'), configFromMap({ 'stow': {'package_dir': '/usr/local/stow', 'base_dir': '/usr/local'}, 'opt': {'package_dir': '/opt/opt-packages', 'base_dir': '/opt'}, 'config': {'filename': '/tmp/machine.conf'}, }, configFromEnv())) ## # Exceptions class PolicyError(Exception): pass ## # A little helper function def runSystemEx(cmd): """This just ignores all stdout""" code = runSingleProgram(cmd, None, errorPrintS) if code != 0:
conf = configFromStream( open('/tmp/machine.conf'), configFromMap( { 'stow': { 'package_dir': '/usr/local/stow', 'base_dir': '/usr/local' }, 'opt': { 'package_dir': '/opt/opt-packages', 'base_dir': '/opt' }, 'config': { 'filename': '/tmp/machine.conf' }, }, configFromEnv())) ## # Exceptions class PolicyError(Exception): pass ## # A little helper function def runSystemEx(cmd): """This just ignores all stdout""" code = runSingleProgram(cmd, None, errorPrintS) if code != 0: raise ProgramRunError(cmd, code)