def runPipeline(taskName, name, pipeline, args=None, queue=None):
"""
Runes a pipeline with command line arguments in args
taskName - the name of the task to update as the pipeline runs
name is the name of this pipeline
args are any CLI arguments you want to use instead of
whatever is in sys.argv
This should be refactored to not rely on args in this way
Takes a pipeline which is some sort of object
which has:
TEMPLATE_DIR - where the template lives
OPTIONS - list of options needed for config file
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
options = list(pipeline.OPTIONS)
options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf")))
options.append(
(
"CONFIG_FILE",
"-c",
"--CONFIG_FILE",
"Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line",
identity,
)
)
conf, _args = cli.buildConfigN(options, args, putInGeneral=False)
##
# If they specified a pipeline_conf, load that and set the values
conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS)
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipeline(taskName, name, pipeline, args=None, queue=None):
"""
Runes a pipeline with command line arguments in args
taskName - the name of the task to update as the pipeline runs
name is the name of this pipeline
args are any CLI arguments you want to use instead of
whatever is in sys.argv
This should be refactored to not rely on args in this way
Takes a pipeline which is some sort of object
which has:
TEMPLATE_DIR - where the template lives
OPTIONS - list of options needed for config file
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
options = list(pipeline.OPTIONS)
options.append(
('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)',
const('/tmp/machine.conf')))
options.append((
'CONFIG_FILE', '-c', '--CONFIG_FILE',
'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line',
identity))
conf, _args = cli.buildConfigN(options, args, putInGeneral=False)
##
# If they specified a pipeline_conf, load that and set the values
conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS)
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
##
# If there are tasks, find the largest, otherwise just return 0
maxTaskNameLen = tasks and max([len(t.name) for t in tasks]) or 0
printSpacing = False
for t in tasks:
if not printSpacing:
printSpacing = True
else:
print
print
printTask(options, t, maxTaskNameLen)
if options('general.exit_code'):
debugPrint(
lambda:
'Exiting with non-zero state if any tasks are not in a completed state'
)
notCompleted = [t for t in tasks if t.state != task.TASK_COMPLETED]
if notCompleted:
return 1
else:
return 0
return 0
if __name__ == '__main__':
sys.exit(main(*buildConfigN(OPTIONS)))
##
import sys
from twisted.python.reflect import namedModule
from igs.utils.cli import buildConfigN, notNone
from igs.utils.logging import errorPrint
from vappio.ergatis.pipeline import runPipeline
OPTIONS = [
('pipeline', '-p', '--pipeline',
'Pipeline for which you want help displayed', notNone),
]
def main(options, _args):
##
# Incredible hack right now
sys.argv = [sys.argv[0]] + ['--help']
try:
pipeline = namedModule('vappio.pipelines.' +
options('general.pipeline'))
runPipeline(None, None, pipeline)
except ImportError:
errorPrint('The requested pipeline could not be found')
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
('id', '', '--ID', 'ID', func.identity),
('props', '', '--props', 'Props', func.identity),
('host', '', '--host', 'Host', func.identity),
('message', '', '--message', 'Message', func.identity)
]
def main(options, _args):
try:
d = dict((k, options(k))
for k in ['event',
'name',
'retval',
'time',
'file',
'id',
'props',
'host',
'message'])
request.performQuery('localhost',
'/vappio/observer_ws.py',
d)
except Exception, err:
open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
if __name__ == '__main__':
try:
main(*cli.buildConfigN(OPTIONS, putInGeneral=False))
except Exception, err:
open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
def main(options, args):
logging.DEBUG = options('general.debug')
if len(args) != 3:
raise cli.MissingOptionError('Must specify PROJECT REPOPATH OUTPUTPATH, see --help')
project, repoPath, outputPath = args
if options('general.checkout') and options('general.export'):
raise cli.InvalidOptionError('You cannot specify both checkout and export')
logging.debugPrint(lambda : 'Loading repositories information...')
repositories = loadRepositories(options('general.config_dir'))
logging.debugPrint(lambda : 'Loaded repositories: %s' % (' '.join(repositories.keys()),))
if project not in repositories:
raise cli.InvalidOptionError('%s is not a valid project name' % project)
repo = repositories[project]
branch = options('general.branch') or repo.branch
if not options('general.export') and (repo.exportType == CHECKOUT or options('general.checkout')):
exportFunc = repo.rType.checkout
else:
exportFunc = repo.rType.export
exportFunc(options, repo, repoPath, outputPath, branch)
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS, usage='usage: %prog [options] PROJECT REPOPATH OUTPUTPATH\nDO NOT USE LEADING SLASHES FOR REPOPATH'))
print 'CHECKSUM\t%s' % pipeline['checksum']
print 'PROTOCOL\t%s' % pipeline['protocol']
for i in pipeline['input_tags']:
print 'INPUT_TAG\t%s' % i
for i in pipeline['output_tags']:
print 'OUTUPUT_TAG\t%s' % i
keys = pipeline['config'].keys()
keys.sort()
for k in keys:
print 'CONFIG\t%s\t%s' % (k, pipeline['config'][k])
else:
pipelines = pipeline_client.pipelineList(options('general.host'),
options('general.cluster'),
{})
pipelines.sort(lambda x, y: cmp(
int(x['pipeline_id']) if x['pipeline_id'] else x['pipeline_id'],
int(y['pipeline_id']) if y['pipeline_id'] else y['pipeline_id']))
for p in pipelines:
print '\t'.join([
'PIPELINE', p['pipeline_id'] if p['pipeline_id'] else '',
p['pipeline_name'], p['state'], p['task_name'],
'wrapper' if p['wrapper'] else '', p['protocol']
])
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS))
cli.defaultIfNone('.md5')),
('debug', '', '--debug', 'Add debug logging', func.identity,
cli.BINARY)]
def main(options, args):
logging.DEBUG = options('general.debug')
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
urls.append(line.strip() + options('general.suffix'))
else:
urls.extend([u + options('general.suffix') for u in args])
for url in urls:
reliableDownloader.deleteDownloadedFiles('/tmp', url)
commands.runSystemEx(' '.join(['wget', '-P', '/tmp', '-nv', url]),
log=logging.DEBUG)
for f in reliableDownloader.getDownloadFilenames('/tmp', url):
sys.stdout.write('\n'.join([
'%s %s' % (l.strip().split()[0],
os.path.join(os.path.dirname(url),
l.strip().split()[1])) for l in open(f)
]) + '\n')
reliableDownloader.deleteDownloadedFiles('/tmp', url)
if __name__ == '__main__':
sys.exit(main(*cli.buildConfigN(OPTIONS, usage='%prog [options] URLs')))
from igs.cgi import request
OPTIONS = [('event', '', '--event', 'Event', func.identity),
('name', '', '--name', 'Name', func.identity),
('retval', '', '--retval', 'Retval', func.identity),
('time', '', '--time', 'Time', func.identity),
('file', '', '--file', 'File', func.identity),
('id', '', '--ID', 'ID', func.identity),
('props', '', '--props', 'Props', func.identity),
('host', '', '--host', 'Host', func.identity),
('message', '', '--message', 'Message', func.identity)]
def main(options, _args):
try:
d = dict((k, options(k)) for k in [
'event', 'name', 'retval', 'time', 'file', 'id', 'props', 'host',
'message'
])
request.performQuery('localhost', '/vappio/observer_ws.py', d)
except Exception, err:
open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
if __name__ == '__main__':
try:
main(*cli.buildConfigN(OPTIONS, putInGeneral=False))
except Exception, err:
open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
('in_key', '', '--in-key', 'Nimbus key file', identity),
('out_key', '', '--out-key', 'Converted EC2 key file', identity),
('java_cert_dir', '', '--java-cert-dir', 'Directory for java certificate', identity),
('java_cert_host', '', '--java-cert-host', 'Host for java cert', identity),
('java_cert_port', '', '--java-cert-port', 'Port for java cert', identity)
]
def promptIfNone(value, prompt):
if value is None:
return raw_input(prompt + ': ')
else:
return value
def main(options, _args):
values = {}
for v, p in [(v, p) for v, _s, _l, p, _f in OPTIONS]:
values[v] = promptIfNone(options(v), p)
convertCert(open(values['in_cert']), open(values['out_cert'], 'w'))
convertKey(values['in_key'], values['out_key'])
addJavaCert(values['java_cert_dir'], values['java_cert_host'], int(values['java_cert_port']))
print 'The certificates have been successfully convert, please set the following variables:'
print 'export EC2_JVM_ARGS="-Djavax.net.ssl.trustStore=%s"' % os.path.join(values['java_cert_dir'], 'jssecacerts')
print 'export EC2_CERT=' + values['out_cert']
print 'export EC2_PRIVATE_KEY=' + values['out_key']
if __name__ == '__main__':
main(*buildConfigN(OPTIONS, usage='usage: %prog [options]\n%prog will prompt for any variables not passed on the command line', putInGeneral=False))
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster to run on', cli.defaultIfNone('local')),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
('pipeline', '', '--pipeline-name', 'Name of pipeline to run against', func.identity),
('config', '-c', '', 'Add config options, multiple allowed in style -c key=value -c key=value',
cli.defaultIfNone([]), cli.LIST)
]
def main(options, args):
tasklet = args[0]
conf = dict([v.split('=', 1) for v in options('general.config', default=[])])
if options('general.pipeline'):
if tasklet:
tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline'))
else:
tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline'))
taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet)
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS, usage='%prog --name=cluster [options] "tasklet1 | tasklet2 | .. | taskletn"'))
##
# If any URLs failed, exit with fail
sys.exit(1)
else:
if options("general.join_name"):
logging.debugPrint(lambda: "Joining files into: " + options("general.join_name"))
files = []
for url in urls:
files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url)))
fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb")
for f in files:
logging.debugPrint(lambda: "Reading: " + f)
fin = open(f, "rb")
d = fin.read(1000000)
while d:
fout.write(d)
d = fin.read(1000000)
fin.close()
fout.close()
logging.debugPrint(lambda: "Deleting downloaded files after join")
for f in files:
logging.debugPrint(lambda: "Deleting: " + f)
deleteFile(f)
if __name__ == "__main__":
sys.exit(main(*cli.buildConfigN(OPTIONS)))
conf)
if ret['errors']:
for e in ret['errors']:
print '\t'.join(['ERROR', ','.join(e['keys']), e['message']])
# Exit with an error
sys.exit(1)
if not options('general.validate'):
p = pipeline.runPipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_parent'),
options('general.bare_run'),
conf,
options('general.pipeline_queue'),
options('general.overwrite'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
else:
p = pipeline.resumePipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_resume'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS, usage='usage: %prog --name x --pipeline y --pipeline-config=CONFIG_FILE'))
if options('general.append') and not options('general.overwrite'):
action = 'append'
elif not options('general.append') and options('general.overwrite'):
action = 'overwrite'
elif not options('general.append') and not options('general.overwrite'):
action = 'create'
else:
raise Exception('--append and --overwrite are mutually exclusive')
if options('general.stdin'):
files = [f.strip() for f in sys.stdin.readlines()]
tag = tagData(
options('general.host'), options('general.cluster'), action,
options('general.tag_name'), [_makeAbsolute(f) for f in files],
options('general.urls'), metadata, options('general.recursive'),
options('general.expand'),
options('general.compress')
and _makeAbsolute(options('general.compress')) or None)
if options('general.print_task_name'):
print tag['task_name']
else:
runTaskStatus(tag['task_name'], options('general.cluster'))
if __name__ == '__main__':
main(*cli.buildConfigN(
OPTIONS,
usage='--name=cluster --tag-name=name [options] file_1 .. file_n'))
raise cli.InvalidOptionError(
'You cannot specify both checkout and export')
logging.debugPrint(lambda: 'Loading repositories information...')
repositories = loadRepositories(options('general.config_dir'))
logging.debugPrint(lambda: 'Loaded repositories: %s' %
(' '.join(repositories.keys()), ))
if project not in repositories:
raise cli.InvalidOptionError('%s is not a valid project name' %
project)
repo = repositories[project]
branch = options('general.branch') or repo.branch
if not options('general.export') and (repo.exportType == CHECKOUT
or options('general.checkout')):
exportFunc = repo.rType.checkout
else:
exportFunc = repo.rType.export
exportFunc(options, repo, repoPath, outputPath, branch)
if __name__ == '__main__':
main(*cli.buildConfigN(
OPTIONS,
usage=
'usage: %prog [options] PROJECT REPOPATH OUTPUTPATH\nDO NOT USE LEADING SLASHES FOR REPOPATH'
))
('block',
'-b',
'--block',
'Block until cluster is up',
func.identity,
cli.BINARY),
('print_task_name',
'-t',
'--print-task-name',
'Print the name of the task at the end',
cli.defaultIfNone(False),
cli.BINARY),
]
def main(options, _args):
taskName = addInstances(options('general.host'),
options('general.cluster'),
options('general.num_exec'),
0,
options('general.exec_type'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName, clusterName=options('general.cluster'))
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python
##
# Uploads files to a cluster
from igs.utils.cli import buildConfigN, notNone, MissingOptionError
from vappio.instance.transfer import uploadAndTag
from vappio.cluster.persist_mongo import load
OPTIONS = [
('name', '', '--name', 'Name of cluster', notNone),
('tag', '', '--tag', 'The uploaded files will be put into a file list tagged by this name', notNone)
]
def main(options, args):
if len(args) < 1:
raise MissingOptionError('Must provide input files')
srcFiles = args
cluster = load(options('general.name'))
uploadAndTag(cluster.master, cluster.config, options('general.tag'), srcFiles, options('general.tag'), log=True)
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
elif not options('general.append') and not options('general.overwrite'):
action = 'create'
else:
raise Exception('--append and --overwrite are mutually exclusive')
if options('general.stdin'):
files = [f.strip() for f in sys.stdin.readlines()]
tag = tagData(options('general.host'),
options('general.cluster'),
action,
options('general.tag_name'),
[_makeAbsolute(f) for f in files],
options('general.urls'),
metadata,
options('general.recursive'),
options('general.expand'),
options('general.compress') and _makeAbsolute(options('general.compress')) or None)
if options('general.print_task_name'):
print tag['task_name']
else:
runTaskStatus(tag['task_name'], options('general.cluster'))
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS, usage='--name=cluster --tag-name=name [options] file_1 .. file_n'))
