def runPipeline(taskName, name, pipeline, args=None, queue=None): """ Runes a pipeline with command line arguments in args taskName - the name of the task to update as the pipeline runs name is the name of this pipeline args are any CLI arguments you want to use instead of whatever is in sys.argv This should be refactored to not rely on args in this way Takes a pipeline which is some sort of object which has: TEMPLATE_DIR - where the template lives OPTIONS - list of options needed for config file """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future options = list(pipeline.OPTIONS) options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf"))) options.append( ( "CONFIG_FILE", "-c", "--CONFIG_FILE", "Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line", identity, ) ) conf, _args = cli.buildConfigN(options, args, putInGeneral=False) ## # If they specified a pipeline_conf, load that and set the values conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS) return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipeline(taskName, name, pipeline, args=None, queue=None): """ Runes a pipeline with command line arguments in args taskName - the name of the task to update as the pipeline runs name is the name of this pipeline args are any CLI arguments you want to use instead of whatever is in sys.argv This should be refactored to not rely on args in this way Takes a pipeline which is some sort of object which has: TEMPLATE_DIR - where the template lives OPTIONS - list of options needed for config file """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future options = list(pipeline.OPTIONS) options.append( ('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)', const('/tmp/machine.conf'))) options.append(( 'CONFIG_FILE', '-c', '--CONFIG_FILE', 'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line', identity)) conf, _args = cli.buildConfigN(options, args, putInGeneral=False) ## # If they specified a pipeline_conf, load that and set the values conf = confIfPipelineConfigSet(conf, pipeline.OPTIONS) return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
## # If there are tasks, find the largest, otherwise just return 0 maxTaskNameLen = tasks and max([len(t.name) for t in tasks]) or 0 printSpacing = False for t in tasks: if not printSpacing: printSpacing = True else: print print printTask(options, t, maxTaskNameLen) if options('general.exit_code'): debugPrint( lambda: 'Exiting with non-zero state if any tasks are not in a completed state' ) notCompleted = [t for t in tasks if t.state != task.TASK_COMPLETED] if notCompleted: return 1 else: return 0 return 0 if __name__ == '__main__': sys.exit(main(*buildConfigN(OPTIONS)))
## import sys from twisted.python.reflect import namedModule from igs.utils.cli import buildConfigN, notNone from igs.utils.logging import errorPrint from vappio.ergatis.pipeline import runPipeline OPTIONS = [ ('pipeline', '-p', '--pipeline', 'Pipeline for which you want help displayed', notNone), ] def main(options, _args): ## # Incredible hack right now sys.argv = [sys.argv[0]] + ['--help'] try: pipeline = namedModule('vappio.pipelines.' + options('general.pipeline')) runPipeline(None, None, pipeline) except ImportError: errorPrint('The requested pipeline could not be found') if __name__ == '__main__': main(*buildConfigN(OPTIONS))
('id', '', '--ID', 'ID', func.identity), ('props', '', '--props', 'Props', func.identity), ('host', '', '--host', 'Host', func.identity), ('message', '', '--message', 'Message', func.identity) ] def main(options, _args): try: d = dict((k, options(k)) for k in ['event', 'name', 'retval', 'time', 'file', 'id', 'props', 'host', 'message']) request.performQuery('localhost', '/vappio/observer_ws.py', d) except Exception, err: open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n') if __name__ == '__main__': try: main(*cli.buildConfigN(OPTIONS, putInGeneral=False)) except Exception, err: open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
def main(options, args): logging.DEBUG = options('general.debug') if len(args) != 3: raise cli.MissingOptionError('Must specify PROJECT REPOPATH OUTPUTPATH, see --help') project, repoPath, outputPath = args if options('general.checkout') and options('general.export'): raise cli.InvalidOptionError('You cannot specify both checkout and export') logging.debugPrint(lambda : 'Loading repositories information...') repositories = loadRepositories(options('general.config_dir')) logging.debugPrint(lambda : 'Loaded repositories: %s' % (' '.join(repositories.keys()),)) if project not in repositories: raise cli.InvalidOptionError('%s is not a valid project name' % project) repo = repositories[project] branch = options('general.branch') or repo.branch if not options('general.export') and (repo.exportType == CHECKOUT or options('general.checkout')): exportFunc = repo.rType.checkout else: exportFunc = repo.rType.export exportFunc(options, repo, repoPath, outputPath, branch) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS, usage='usage: %prog [options] PROJECT REPOPATH OUTPUTPATH\nDO NOT USE LEADING SLASHES FOR REPOPATH'))
print 'CHECKSUM\t%s' % pipeline['checksum'] print 'PROTOCOL\t%s' % pipeline['protocol'] for i in pipeline['input_tags']: print 'INPUT_TAG\t%s' % i for i in pipeline['output_tags']: print 'OUTUPUT_TAG\t%s' % i keys = pipeline['config'].keys() keys.sort() for k in keys: print 'CONFIG\t%s\t%s' % (k, pipeline['config'][k]) else: pipelines = pipeline_client.pipelineList(options('general.host'), options('general.cluster'), {}) pipelines.sort(lambda x, y: cmp( int(x['pipeline_id']) if x['pipeline_id'] else x['pipeline_id'], int(y['pipeline_id']) if y['pipeline_id'] else y['pipeline_id'])) for p in pipelines: print '\t'.join([ 'PIPELINE', p['pipeline_id'] if p['pipeline_id'] else '', p['pipeline_name'], p['state'], p['task_name'], 'wrapper' if p['wrapper'] else '', p['protocol'] ]) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS))
cli.defaultIfNone('.md5')), ('debug', '', '--debug', 'Add debug logging', func.identity, cli.BINARY)] def main(options, args): logging.DEBUG = options('general.debug') urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: urls.append(line.strip() + options('general.suffix')) else: urls.extend([u + options('general.suffix') for u in args]) for url in urls: reliableDownloader.deleteDownloadedFiles('/tmp', url) commands.runSystemEx(' '.join(['wget', '-P', '/tmp', '-nv', url]), log=logging.DEBUG) for f in reliableDownloader.getDownloadFilenames('/tmp', url): sys.stdout.write('\n'.join([ '%s %s' % (l.strip().split()[0], os.path.join(os.path.dirname(url), l.strip().split()[1])) for l in open(f) ]) + '\n') reliableDownloader.deleteDownloadedFiles('/tmp', url) if __name__ == '__main__': sys.exit(main(*cli.buildConfigN(OPTIONS, usage='%prog [options] URLs')))
from igs.cgi import request OPTIONS = [('event', '', '--event', 'Event', func.identity), ('name', '', '--name', 'Name', func.identity), ('retval', '', '--retval', 'Retval', func.identity), ('time', '', '--time', 'Time', func.identity), ('file', '', '--file', 'File', func.identity), ('id', '', '--ID', 'ID', func.identity), ('props', '', '--props', 'Props', func.identity), ('host', '', '--host', 'Host', func.identity), ('message', '', '--message', 'Message', func.identity)] def main(options, _args): try: d = dict((k, options(k)) for k in [ 'event', 'name', 'retval', 'time', 'file', 'id', 'props', 'host', 'message' ]) request.performQuery('localhost', '/vappio/observer_ws.py', d) except Exception, err: open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n') if __name__ == '__main__': try: main(*cli.buildConfigN(OPTIONS, putInGeneral=False)) except Exception, err: open('/tmp/ergatisObserver.log', 'a').write(str(err) + '\n')
('in_key', '', '--in-key', 'Nimbus key file', identity), ('out_key', '', '--out-key', 'Converted EC2 key file', identity), ('java_cert_dir', '', '--java-cert-dir', 'Directory for java certificate', identity), ('java_cert_host', '', '--java-cert-host', 'Host for java cert', identity), ('java_cert_port', '', '--java-cert-port', 'Port for java cert', identity) ] def promptIfNone(value, prompt): if value is None: return raw_input(prompt + ': ') else: return value def main(options, _args): values = {} for v, p in [(v, p) for v, _s, _l, p, _f in OPTIONS]: values[v] = promptIfNone(options(v), p) convertCert(open(values['in_cert']), open(values['out_cert'], 'w')) convertKey(values['in_key'], values['out_key']) addJavaCert(values['java_cert_dir'], values['java_cert_host'], int(values['java_cert_port'])) print 'The certificates have been successfully convert, please set the following variables:' print 'export EC2_JVM_ARGS="-Djavax.net.ssl.trustStore=%s"' % os.path.join(values['java_cert_dir'], 'jssecacerts') print 'export EC2_CERT=' + values['out_cert'] print 'export EC2_PRIVATE_KEY=' + values['out_key'] if __name__ == '__main__': main(*buildConfigN(OPTIONS, usage='usage: %prog [options]\n%prog will prompt for any variables not passed on the command line', putInGeneral=False))
OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster to run on', cli.defaultIfNone('local')), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ('pipeline', '', '--pipeline-name', 'Name of pipeline to run against', func.identity), ('config', '-c', '', 'Add config options, multiple allowed in style -c key=value -c key=value', cli.defaultIfNone([]), cli.LIST) ] def main(options, args): tasklet = args[0] conf = dict([v.split('=', 1) for v in options('general.config', default=[])]) if options('general.pipeline'): if tasklet: tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline')) else: tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline')) taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS, usage='%prog --name=cluster [options] "tasklet1 | tasklet2 | .. | taskletn"'))
## # If any URLs failed, exit with fail sys.exit(1) else: if options("general.join_name"): logging.debugPrint(lambda: "Joining files into: " + options("general.join_name")) files = [] for url in urls: files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url))) fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb") for f in files: logging.debugPrint(lambda: "Reading: " + f) fin = open(f, "rb") d = fin.read(1000000) while d: fout.write(d) d = fin.read(1000000) fin.close() fout.close() logging.debugPrint(lambda: "Deleting downloaded files after join") for f in files: logging.debugPrint(lambda: "Deleting: " + f) deleteFile(f) if __name__ == "__main__": sys.exit(main(*cli.buildConfigN(OPTIONS)))
conf) if ret['errors']: for e in ret['errors']: print '\t'.join(['ERROR', ','.join(e['keys']), e['message']]) # Exit with an error sys.exit(1) if not options('general.validate'): p = pipeline.runPipeline(options('general.host'), options('general.cluster'), options('general.pipeline_parent'), options('general.bare_run'), conf, options('general.pipeline_queue'), options('general.overwrite')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name']) else: p = pipeline.resumePipeline(options('general.host'), options('general.cluster'), options('general.pipeline_resume')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name']) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS, usage='usage: %prog --name x --pipeline y --pipeline-config=CONFIG_FILE'))
if options('general.append') and not options('general.overwrite'): action = 'append' elif not options('general.append') and options('general.overwrite'): action = 'overwrite' elif not options('general.append') and not options('general.overwrite'): action = 'create' else: raise Exception('--append and --overwrite are mutually exclusive') if options('general.stdin'): files = [f.strip() for f in sys.stdin.readlines()] tag = tagData( options('general.host'), options('general.cluster'), action, options('general.tag_name'), [_makeAbsolute(f) for f in files], options('general.urls'), metadata, options('general.recursive'), options('general.expand'), options('general.compress') and _makeAbsolute(options('general.compress')) or None) if options('general.print_task_name'): print tag['task_name'] else: runTaskStatus(tag['task_name'], options('general.cluster')) if __name__ == '__main__': main(*cli.buildConfigN( OPTIONS, usage='--name=cluster --tag-name=name [options] file_1 .. file_n'))
raise cli.InvalidOptionError( 'You cannot specify both checkout and export') logging.debugPrint(lambda: 'Loading repositories information...') repositories = loadRepositories(options('general.config_dir')) logging.debugPrint(lambda: 'Loaded repositories: %s' % (' '.join(repositories.keys()), )) if project not in repositories: raise cli.InvalidOptionError('%s is not a valid project name' % project) repo = repositories[project] branch = options('general.branch') or repo.branch if not options('general.export') and (repo.exportType == CHECKOUT or options('general.checkout')): exportFunc = repo.rType.checkout else: exportFunc = repo.rType.export exportFunc(options, repo, repoPath, outputPath, branch) if __name__ == '__main__': main(*cli.buildConfigN( OPTIONS, usage= 'usage: %prog [options] PROJECT REPOPATH OUTPUTPATH\nDO NOT USE LEADING SLASHES FOR REPOPATH' ))
('block', '-b', '--block', 'Block until cluster is up', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, _args): taskName = addInstances(options('general.host'), options('general.cluster'), options('general.num_exec'), 0, options('general.exec_type')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName, clusterName=options('general.cluster')) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python ## # Uploads files to a cluster from igs.utils.cli import buildConfigN, notNone, MissingOptionError from vappio.instance.transfer import uploadAndTag from vappio.cluster.persist_mongo import load OPTIONS = [ ('name', '', '--name', 'Name of cluster', notNone), ('tag', '', '--tag', 'The uploaded files will be put into a file list tagged by this name', notNone) ] def main(options, args): if len(args) < 1: raise MissingOptionError('Must provide input files') srcFiles = args cluster = load(options('general.name')) uploadAndTag(cluster.master, cluster.config, options('general.tag'), srcFiles, options('general.tag'), log=True) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
elif not options('general.append') and not options('general.overwrite'): action = 'create' else: raise Exception('--append and --overwrite are mutually exclusive') if options('general.stdin'): files = [f.strip() for f in sys.stdin.readlines()] tag = tagData(options('general.host'), options('general.cluster'), action, options('general.tag_name'), [_makeAbsolute(f) for f in files], options('general.urls'), metadata, options('general.recursive'), options('general.expand'), options('general.compress') and _makeAbsolute(options('general.compress')) or None) if options('general.print_task_name'): print tag['task_name'] else: runTaskStatus(tag['task_name'], options('general.cluster')) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS, usage='--name=cluster --tag-name=name [options] file_1 .. file_n'))