def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config')
options = list(pipeline.OPTIONS)
options.append(
('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)',
const('/tmp/machine.conf')))
options.append((
'CONFIG_FILE', '-c', '--CONFIG_FILE',
'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line',
const(tmpConfigName)))
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(conf,
config.configFromStream(
open('/tmp/machine.conf'),
config.configFromEnv()),
lazy=True)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v)
for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, 'w')
for s, d in confVals.iteritems():
if s not in ['', 'env']:
fout.write('[' + s + ']\n')
for k, v in d.iteritems():
fout.write('%s=%s\n' % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config")
options = list(pipeline.OPTIONS)
options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf")))
options.append(
(
"CONFIG_FILE",
"-c",
"--CONFIG_FILE",
"Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line",
const(tmpConfigName),
)
)
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(
conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True
)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, "w")
for s, d in confVals.iteritems():
if s not in ["", "env"]:
fout.write("[" + s + "]\n")
for k, v in d.iteritems():
fout.write("%s=%s\n" % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipeline(taskName, name, pipeline, args=None, queue=None):
"""
Runes a pipeline with command line arguments in args
taskName - the name of the task to update as the pipeline runs
name is the name of this pipeline
args are any CLI arguments you want to use instead of
whatever is in sys.argv
This should be refactored to not rely on args in this way
Takes a pipeline which is some sort of object
which has:
TEMPLATE_DIR - where the template lives
OPTIONS - list of options needed for config file
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
options = list(pipeline.OPTIONS)
options.append(
('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)',
const('/tmp/machine.conf')))
options.append((
'CONFIG_FILE', '-c', '--CONFIG_FILE',
'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line',
identity))
conf, _args = cli.buildConfigN(options, args, putInGeneral=False)
##
# If they specified a pipeline_conf, load that and set the values
conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS)
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import downloadTag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host',
'Host of web services to connect to, defaults to local host',
cli.defaultIfNone('localhost')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster',
cli.notNone),
('dst_cluster', '', '--dst-cluster',
'Name of dest cluster, hardcoded to local for now', func.const('local')),
('output_dir', '', '--output-dir', 'Name of directory to download to',
func.identity),
('block', '-b', '--block', 'Block until download is complete',
func.identity, cli.BINARY),
('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False),
cli.BINARY),
('compress', '', '--compress', 'Compress files', func.identity,
cli.BINARY),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
]
def main(options, files):
from igs.utils import logging
from igs.config_manage.policy import installPkg, installOptPkg
from vappio.instance.init_instance import runInit, executePolicyDirWEx
from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, configFromStream
##
# I'm using lambdas here just because I want to define the functions lower in the file
# and they need to exist before I can reference them, unless I wrap them in a lambda
NODE_TYPE_MAP = {
'pre': lambda c: startUpAllNodes(c),
DEV_NODE: lambda c: startUpDevNode(c),
MASTER_NODE: lambda c: startUpMasterNode(c),
EXEC_NODE: lambda c: startUpExecNode(c),
'post': const(None)
}
def startUp(conf):
runInit(conf, NODE_TYPE_MAP)
def startUpFromConfigFile(fname):
"""Run startup from a config file"""
return startUp(configFromStream(open(fname)))
def startPolicy(m):
"""
Calls startup on the policy
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import downloadTag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')),
('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity),
('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY),
('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY),
('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
]
def main(options, files):
taskName = downloadTag(options('general.host'),
options('general.tag_name'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.output_dir'),
options('general.expand'),
options('general.compress'))
# How we want to bring down the node
import os
from igs.utils.errors import TryError
from igs.utils.functional import const
from igs.config_manage.policy import uninstallPkg, uninstallOptPkg
from vappio.instance.init_instance import runInit, executePolicyDirWEx
from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, RELEASE_CUT, configFromStream
##
# I'm using lambdas here just because I want to define the functions lower in the file
# and they need to exist before I can reference them, unless I wrap them in a lambda
NODE_TYPE_MAP = {
"pre": const(None),
DEV_NODE: lambda c: shutdownDevNode(c),
MASTER_NODE: lambda c: shutdownMasterNode(c),
EXEC_NODE: lambda c: shutdownExecNode(c),
"post": lambda c: shutdownAllNodes(c),
}
def shutdown(conf):
"""
Does everything necessary for shutdown an instance and cleaning everything up
"""
runInit(conf, NODE_TYPE_MAP)
def shutdownFromConfigFile(fname):
from vappio.instance.init_instance import runInit, executePolicyDirWEx
from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, configFromStream
##
# I'm using lambdas here just because I want to define the functions lower in the file
# and they need to exist before I can reference them, unless I wrap them in a lambda
NODE_TYPE_MAP = {
'pre': lambda c : startUpAllNodes(c),
DEV_NODE: lambda c : startUpDevNode(c),
MASTER_NODE: lambda c : startUpMasterNode(c),
EXEC_NODE: lambda c : startUpExecNode(c),
'post': const(None)
}
def startUp(conf):
runInit(conf, NODE_TYPE_MAP)
def startUpFromConfigFile(fname):
"""Run startup from a config file"""
return startUp(configFromStream(open(fname)))
def startPolicy(m):
"""
Calls startup on the policy
"""
# How we want to bring down the node
import os
from igs.utils.errors import TryError
from igs.utils.functional import const
from igs.config_manage.policy import uninstallPkg, uninstallOptPkg
from vappio.instance.init_instance import runInit, executePolicyDirWEx
from vappio.instance.config import DEV_NODE, MASTER_NODE, EXEC_NODE, RELEASE_CUT, configFromStream
##
# I'm using lambdas here just because I want to define the functions lower in the file
# and they need to exist before I can reference them, unless I wrap them in a lambda
NODE_TYPE_MAP = {
'pre': const(None),
DEV_NODE: lambda c: shutdownDevNode(c),
MASTER_NODE: lambda c: shutdownMasterNode(c),
EXEC_NODE: lambda c: shutdownExecNode(c),
'post': lambda c: shutdownAllNodes(c)
}
def shutdown(conf):
"""
Does everything necessary for shutdown an instance and cleaning everything up
"""
runInit(conf, NODE_TYPE_MAP)
def shutdownFromConfigFile(fname):
def runPipeline(taskName, name, pipeline, args=None, queue=None):
"""
Runes a pipeline with command line arguments in args
taskName - the name of the task to update as the pipeline runs
name is the name of this pipeline
args are any CLI arguments you want to use instead of
whatever is in sys.argv
This should be refactored to not rely on args in this way
Takes a pipeline which is some sort of object
which has:
TEMPLATE_DIR - where the template lives
OPTIONS - list of options needed for config file
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
options = list(pipeline.OPTIONS)
options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf")))
options.append(
(
"CONFIG_FILE",
"-c",
"--CONFIG_FILE",
"Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line",
identity,
)
)
conf, _args = cli.buildConfigN(options, args, putInGeneral=False)
##
# If they specified a pipeline_conf, load that and set the values
conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS)
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
