def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config') options = list(pipeline.OPTIONS) options.append( ('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)', const('/tmp/machine.conf'))) options.append(( 'CONFIG_FILE', '-c', '--CONFIG_FILE', 'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line', const(tmpConfigName))) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig(conf, config.configFromStream( open('/tmp/machine.conf'), config.configFromEnv()), lazy=True) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, 'w') for s, d in confVals.iteritems(): if s not in ['', 'env']: fout.write('[' + s + ']\n') for k, v in d.iteritems(): fout.write('%s=%s\n' % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config") options = list(pipeline.OPTIONS) options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf"))) options.append( ( "CONFIG_FILE", "-c", "--CONFIG_FILE", "Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line", const(tmpConfigName), ) ) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig( conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True ) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, "w") for s, d in confVals.iteritems(): if s not in ["", "env"]: fout.write("[" + s + "]\n") for k, v in d.iteritems(): fout.write("%s=%s\n" % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipeline(taskName, name, pipeline, args=None, queue=None): """ Runes a pipeline with command line arguments in args taskName - the name of the task to update as the pipeline runs name is the name of this pipeline args are any CLI arguments you want to use instead of whatever is in sys.argv This should be refactored to not rely on args in this way Takes a pipeline which is some sort of object which has: TEMPLATE_DIR - where the template lives OPTIONS - list of options needed for config file """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future options = list(pipeline.OPTIONS) options.append( ('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)', const('/tmp/machine.conf'))) options.append(( 'CONFIG_FILE', '-c', '--CONFIG_FILE', 'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line', identity)) conf, _args = cli.buildConfigN(options, args, putInGeneral=False) ## # If they specified a pipeline_conf, load that and set the values conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS) return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import downloadTag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')), ('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity), ('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY), ('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY), ('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, files):
from igs.utils import logging from igs.config_manage.policy import installPkg, installOptPkg from vappio.instance.init_instance import runInit, executePolicyDirWEx from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, configFromStream ## # I'm using lambdas here just because I want to define the functions lower in the file # and they need to exist before I can reference them, unless I wrap them in a lambda NODE_TYPE_MAP = { 'pre': lambda c: startUpAllNodes(c), DEV_NODE: lambda c: startUpDevNode(c), MASTER_NODE: lambda c: startUpMasterNode(c), EXEC_NODE: lambda c: startUpExecNode(c), 'post': const(None) } def startUp(conf): runInit(conf, NODE_TYPE_MAP) def startUpFromConfigFile(fname): """Run startup from a config file""" return startUp(configFromStream(open(fname))) def startPolicy(m): """ Calls startup on the policy
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import downloadTag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')), ('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity), ('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY), ('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY), ('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, files): taskName = downloadTag(options('general.host'), options('general.tag_name'), options('general.src_cluster'), options('general.dst_cluster'), options('general.output_dir'), options('general.expand'), options('general.compress'))
# How we want to bring down the node import os from igs.utils.errors import TryError from igs.utils.functional import const from igs.config_manage.policy import uninstallPkg, uninstallOptPkg from vappio.instance.init_instance import runInit, executePolicyDirWEx from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, RELEASE_CUT, configFromStream ## # I'm using lambdas here just because I want to define the functions lower in the file # and they need to exist before I can reference them, unless I wrap them in a lambda NODE_TYPE_MAP = { "pre": const(None), DEV_NODE: lambda c: shutdownDevNode(c), MASTER_NODE: lambda c: shutdownMasterNode(c), EXEC_NODE: lambda c: shutdownExecNode(c), "post": lambda c: shutdownAllNodes(c), } def shutdown(conf): """ Does everything necessary for shutdown an instance and cleaning everything up """ runInit(conf, NODE_TYPE_MAP) def shutdownFromConfigFile(fname):
from vappio.instance.init_instance import runInit, executePolicyDirWEx from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, configFromStream ## # I'm using lambdas here just because I want to define the functions lower in the file # and they need to exist before I can reference them, unless I wrap them in a lambda NODE_TYPE_MAP = { 'pre': lambda c : startUpAllNodes(c), DEV_NODE: lambda c : startUpDevNode(c), MASTER_NODE: lambda c : startUpMasterNode(c), EXEC_NODE: lambda c : startUpExecNode(c), 'post': const(None) } def startUp(conf): runInit(conf, NODE_TYPE_MAP) def startUpFromConfigFile(fname): """Run startup from a config file""" return startUp(configFromStream(open(fname))) def startPolicy(m): """ Calls startup on the policy """
# How we want to bring down the node import os from igs.utils.errors import TryError from igs.utils.functional import const from igs.config_manage.policy import uninstallPkg, uninstallOptPkg from vappio.instance.init_instance import runInit, executePolicyDirWEx from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, RELEASE_CUT, configFromStream ## # I'm using lambdas here just because I want to define the functions lower in the file # and they need to exist before I can reference them, unless I wrap them in a lambda NODE_TYPE_MAP = { 'pre': const(None), DEV_NODE: lambda c: shutdownDevNode(c), MASTER_NODE: lambda c: shutdownMasterNode(c), EXEC_NODE: lambda c: shutdownExecNode(c), 'post': lambda c: shutdownAllNodes(c) } def shutdown(conf): """ Does everything necessary for shutdown an instance and cleaning everything up """ runInit(conf, NODE_TYPE_MAP) def shutdownFromConfigFile(fname):
def runPipeline(taskName, name, pipeline, args=None, queue=None): """ Runes a pipeline with command line arguments in args taskName - the name of the task to update as the pipeline runs name is the name of this pipeline args are any CLI arguments you want to use instead of whatever is in sys.argv This should be refactored to not rely on args in this way Takes a pipeline which is some sort of object which has: TEMPLATE_DIR - where the template lives OPTIONS - list of options needed for config file """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future options = list(pipeline.OPTIONS) options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf"))) options.append( ( "CONFIG_FILE", "-c", "--CONFIG_FILE", "Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line", identity, ) ) conf, _args = cli.buildConfigN(options, args, putInGeneral=False) ## # If they specified a pipeline_conf, load that and set the values conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS) return runPipelineWithConfig(taskName, name, pipeline, conf, queue)