#!/usr/bin/env python ## # Checks the status of all, or a specific pipeline, running on the cluster from igs.utils import cli from igs.utils import functional as func from vappio.webservice import pipeline as pipeline_client OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')), ('pipeline_name', '-p', '--pipeline-name', 'Name of pipeline to show detauls on', func.identity) ] def main(options, _args): if options('general.pipeline_name'): pipeline = pipeline_client.pipelineList(options('general.host'), options('general.cluster'), {'pipeline_name': options('general.pipeline_name')}, True)[0] print 'PIPELINE_NAME\t%s' % pipeline['pipeline_name'] print 'PIPELINE_ID\t%s' % pipeline['pipeline_id'] print 'PIPELINE_TYPE\t%s' % pipeline['protocol'] print 'PIPELINE_DESC\t%s' % pipeline['pipeline_desc'] if pipeline['wrapper']: print 'WRAPPER' print 'STATE\t%s' % pipeline['state'] for cl, pName in pipeline['children']: print 'CHILD\t%s\t%s' % (cl, pName)
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import downloadTag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')), ('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity), ('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY), ('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY), ('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ]
#!/usr/bin/env python from igs.utils.cli import buildConfigN, notNone, defaultIfNone from igs.utils.functional import identity from vappio.webservice.pipeline import downloadPipelineOutput from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster', notNone), ## # Want to make sure this is an int but we want it as a string later in the program ('pipeline', '-p', '--pipeline-name', 'Name of pipeline', notNone), ('output_dir', '-o', '--output-dir', 'Directory the output file should go to', notNone), ('overwrite', '', '--overwrite', 'Do you want to overwrite a local file if it already exists?', defaultIfNone(False), True), ('block', '-b', '--block', 'Block until download is complete', identity, True), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', defaultIfNone(False), True), ] def main(options, _args): taskName = downloadPipelineOutput(options('general.host'), options('general.name'), options('general.pipeline'),
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.cluster import startCluster from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.notNone), ('num_exec', '', '--num-exec', 'Number of exec nodes to start', int), ('cred', '', '--cred', 'Credential to use', cli.notNone), ('master_instance_type', '', '--master-instance-type', 'Which instance type to use, defaults to "default"', cli.defaultIfNone('default')), ('exec_instance_type', '', '--exec-instance-type', 'Which instance type to use, defaults to "default"', cli.defaultIfNone('default')), ('master_bid_price', '', '--master-bid-price', 'Bid price for master, defaults to nothing (on demand instance)', cli.defaultIfNone('')), ('exec_bid_price', '', '--exec-bid-price',
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice import tag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective', cli.defaultIfNone('local')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster, hardcoded to local for now', cli.defaultIfNone('local')), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster', cli.defaultIfNone('local')), ('transfer_type', '', '--transfer-type', 'Type of transfer to do (cluster, s3) default is cluster', func.compose(cli.restrictValues(['cluster', 's3']), cli.defaultIfNone('cluster'))), ('block', '-b', '--block', 'Block until cluster is up (no longer used)', func.identity, cli.BINARY), ('dst_type', '', '--dst-type', 'Set type of destination. (stage,local) default is stage', cli.defaultIfNone('stage')),
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice import tag OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')), ('tag_name', '', '--tag-name', 'Name of tag', func.identity), ] def main(options, files): if options('general.tag_name'): tagData = tag.listTags(options('general.host'), options('general.cluster'), {'tag_name': options('general.tag_name')}, detail=True)[0] if tagData['phantom']: print 'PHANTOM' for f in tagData['files']: print '\t'.join(['FILE', f]) for url in tagData['metadata'].get('urls', []): print '\t'.join(['URL', url]) for k, v in tagData['metadata'].iteritems(): if k not in ['urls']:
#!/usr/bin/env python ## # Runs commands on the cluster from igs.utils.cli import buildConfigN, notNone, defaultIfNone from vappio.cluster.control import runCommandOnCluster from vappio.webservice.cluster import loadCluster OPTIONS = [('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster', notNone), ('just_master', '-m', '--just_master', 'Just run the command on the master', defaultIfNone(False), True), ('command', '', '--cmd', 'Command to run', notNone)] def main(options, _args): cluster = loadCluster(options('general.host'), options('general.name')) runCommandOnCluster(cluster, options('general.command'), options('general.just_master')) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
#!/usr/bin/env python from igs.utils.cli import buildConfigN, notNone, defaultIfNone from igs.utils.functional import identity from vappio.webservice.pipeline import downloadPipelineOutput from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster', notNone), ## # Want to make sure this is an int but we want it as a string later in the program ('pipeline', '-p', '--pipeline-name', 'Name of pipeline', notNone), ('output_dir', '-o', '--output-dir', 'Directory the output file should go to', notNone), ('overwrite', '', '--overwrite', 'Do you want to overwrite a local file if it already exists?', defaultIfNone(False), True), ('block', '-b', '--block', 'Block until download is complete', identity, True), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', defaultIfNone(False), True), ] def main(options, _args): taskName = downloadPipelineOutput(options('general.host'), options('general.name'), options('general.pipeline'), options('general.output_dir'), options('general.overwrite')) if options('general.print_task_name'): print taskName
#!/usr/bin/env python ## # Runs commands on the cluster from igs.utils.cli import buildConfigN, notNone, defaultIfNone from vappio.cluster.control import runCommandOnCluster from vappio.webservice.cluster import loadCluster OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster', notNone), ('just_master', '-m', '--just_master', 'Just run the command on the master', defaultIfNone(False), True), ('command', '', '--cmd', 'Command to run', notNone) ] def main(options, _args): cluster = loadCluster(options('general.host'), options('general.name')) runCommandOnCluster(cluster, options('general.command'), options('general.just_master')) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
import os from igs.utils.cli import buildConfigN, notNone, defaultIfNone from igs.utils.functional import identity from igs.utils import config from igs.utils.commands import runSystemEx, runSingleProgramEx from igs.utils.logging import logPrint, errorPrint from igs.threading import threads OPTIONS = [ ('version', '', '--version', 'Version of this release', notNone), ('remote_name', '', '--remote-name', 'Name of remote machine the image lives on', notNone), ('image', '-i', '--image', 'Image to bundle', identity), ('cert', '-c', '--cert', 'Certifiate to use, default $EC2_CERT', defaultIfNone(os.getenv('EC2_CERT'))), ('key', '-k', '--key', 'Key to use, default $EC2_PRIVATE_KEY', defaultIfNone(os.getenv('EC2_PRIVATE_KEY'))), ('user', '-u', '--user', 'AWS account number, defaults to $EC2_ACCOUNT_ID', defaultIfNone(os.getenv('EC2_ACCOUNT_ID'))), ('access_key', '', '--access_key', 'AWS access key, defaults to $EC2_ACCESS_KEY', defaultIfNone(os.getenv('EC2_ACCESS_KEY'))), ('secret_access_key', '', '--secret_access_key', 'AWS secret access key, defaults to $EC2_SECRET_ACCESS_KEY', defaultIfNone(os.getenv('EC2_SECRET_ACCESS_KEY'))), ('dest', '-d', '--dest', 'Destination', notNone), ('ec2cert', '', '--ec2cert', 'EC2 cert to use', identity), ('kernel', '', '--kernel', 'What AKI to use', defaultIfNone('aki-fd15f694')), ('arch', '-r', '--arch', 'Architecture (i386 or x86_64)', identity),
# This script provides some insight into tasks import sys import time from igs.utils.cli import buildConfigN, defaultIfNone from igs.utils.functional import identity from igs.utils.logging import debugPrint from igs.utils import logging from igs.utils import commands from vappio.webservice.task import loadAllTasks, loadTask from vappio.tasks import task OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster, defaults to local', defaultIfNone('local')), ('show_msgs', '', '--show', 'Print out any messages present', identity, True), ('show_error_msgs', '', '--show-error', 'Print out only error messages', identity, True), ('show_debug_msgs', '', '--show-debug', 'Print out any debug messages', identity, True), ('show_all_msgs', '', '--show-all', 'Print out all messages', identity, True), ('debug', '', '--debug', 'Print debugging messages', identity, True), ('no_completed', '', '--nc', 'Filtered out completed tasks', identity, True), ('exit_code', '', '--exit-code', 'Exit with a non zero value if any tasks are not in a completed state', identity, True), ('block', '-b', '--block', 'Block on the tasks until they have completed or errored', identity, True), ('no_print_polling', '', '--no-print-polling', 'If block is true, do not print while polling to see what is happening in real time', identity, True), ] def timestampToStr(ts): return time.strftime('%Y/%m/%d %H:%M:%S UTC', time.gmtime(ts))
#!/usr/bin/env python from twisted.python import reflect from igs.utils import cli from igs.utils import functional as func from vappio.webservice import credential from vappio.tasks.utils import runTaskStatus from vappio.credentials import manager OPTIONS = [ ('cred_name', '', '--cred-name', 'Name of the credential', cli.notNone), ('cred_desc', '', '--desc', 'Description of the credential', cli.defaultIfNone('')), ('ctype', '', '--ctype', 'Cluster type this cert is for (ec2, nimbus, diag, adhoc, local)', cli.restrictValues(['diag', 'nimbus', 'ec2', 'local', 'adhoc'])), ('cert', '-c', '--cert', 'Path to cert', func.identity), ('pkey', '-p', '--pkey', 'path to private key', func.identity), ('devel', '-d', '--devel', 'Path to devel1.pem', func.identity), ('metadata', '-m', '', ('Add metadata in a key=value notation. Multiple options are valid. ' 'Ex: -m ec2_url=http://foo/bar/zoom -m comment="This works"'), cli.defaultIfNone([]), cli.LIST), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY) ]
#!/usr/bin/env python import sys import os from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import tagData from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster, defaults to local', cli.defaultIfNone('local')), ('tag_name', '', '--tag-name', 'Name of tag', cli.notNone), ('urls', '', '--url', 'URLs to download, multiple --url flags can be specified', cli.defaultIfNone([]), cli.LIST), ('stdin', '', '--stdin', 'Take list of files from stdin rather than arguments on command line', cli.defaultIfNone(False), cli.BINARY), ('tag_base_dir', '', '--tag-base-dir', 'Base directory of the tag', func.identity), ('recursive', '-r', '--recursive', 'Recursively include directories', cli.defaultIfNone(False), cli.BINARY), ('expand', '-e', '--expand', 'Expand archives', cli.defaultIfNone(False), cli.BINARY), ('compress', '-c', '--compress',
import os from igs.utils import cli from igs.utils import logging from igs.utils import commands from igs.utils import functional as func # # A bit unorthodoxed but it is useful that every CLI app # is designed to also work as a library if needed from vappio.cli import reliableDownloader OPTIONS = [( 'suffix', '-s', '--suffix', 'The suffice that the md5 version of the files are expected to have. Defaults to ".md5" (not the ".")', cli.defaultIfNone('.md5')), ('debug', '', '--debug', 'Add debug logging', func.identity, cli.BINARY)] def main(options, args): logging.DEBUG = options('general.debug') urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: urls.append(line.strip() + options('general.suffix')) else: urls.extend([u + options('general.suffix') for u in args]) for url in urls: reliableDownloader.deleteDownloadedFiles('/tmp', url)
import sys import os import time import signal import Queue import urlparse import glob from igs.utils import logging from igs.utils import cli from igs.utils import commands from igs.utils import functional as func from igs.threading import threads OPTIONS = [ ('base_dir', '-b', '--base-dir', 'Base directory to download into', cli.defaultIfNone('.')), ('join_name', '-j', '--join-name', 'If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.', func.identity), ('join_md5', '', '--join-md5', 'The md5sum of the joined file', func.identity), ('min_rate', '-m', '--min-rate', 'Minimum download rate in kilobytes per second', func.compose(int, cli.notNone)), ('tries', '-t', '--tries', 'Number of download attempts to make', func.compose(int, cli.defaultIfNone('3'))), ('max_threads', '', '--max-threads', 'Maximum number of threads to download at once', func.compose(int, cli.defaultIfNone('3'))), ('continue_download', '-c', '--continue', 'Continue the download. Default is to download everything from scratch', cli.defaultIfNone(False), True), ('debug', '-d', '--debug', 'Turn debug on', cli.defaultIfNone(False), True), ] ## # How many seconds each sample should be SAMPLE_RATE = 5 MAX_SAMPLE_SIZE = 10
# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state # 2) Someone could check out a whole directory and then check out a file into of that directory # # To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single # files an import os from igs.utils import commands from igs.utils import cli from igs.utils import functional as func from igs.utils import logging OPTIONS = [ ('codir', '', '--co-dir', 'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir', func.compose(cli.defaultIfNone('/opt/co-dir'), cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))), ('branch', '-b', '--branch', 'Override the branch specified in config files', func.identity), ('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY), ('export', '', '--export', 'Force an export, mutually exclusive with --checkout', func.identity, cli.BINARY), ('config_dir', '-c', '--config-dir', 'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout', func.compose(cli.defaultIfNone('/etc/vappio-checkout'), cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))), ('debug', '-d', '--debug', 'Turn debugging information on', func.identity, cli.BINARY), ] class CheckoutModifiedError(Exception): """ Trying to do another check out when modifications exist """
#!/usr/bin/env python from igs.utils import functional as func from igs.utils.cli import buildConfigN, notNone, defaultIfNone from vappio.webservice.cluster import terminateCluster from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', notNone), ('auth_token', '', '--auth-token', 'Authorization token to pass along to target cluster', func.identity), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', defaultIfNone(False), True), ] def main(options, _args): taskName = terminateCluster(options('general.host'), options('general.cluster'), options('general.auth_token')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName, clusterName='local', hostName=options('general.host')) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state # 2) Someone could check out a whole directory and then check out a file into of that directory # # To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single # files an import os from igs.utils import commands from igs.utils import cli from igs.utils import functional as func from igs.utils import logging OPTIONS = [ ('codir', '', '--co-dir', 'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir', func.compose(cli.defaultIfNone('/opt/co-dir'), cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))), ('branch', '-b', '--branch', 'Override the branch specified in config files', func.identity), ('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY), ('export', '', '--export', 'Force an export, mutually exclusive with --checkout', func.identity, cli.BINARY), ('config_dir', '-c', '--config-dir', 'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout', func.compose(cli.defaultIfNone('/etc/vappio-checkout'), cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))), ('debug', '-d', '--debug', 'Turn debugging information on', func.identity, cli.BINARY), ]
#!/usr/bin/env python import sys import os from igs.utils import commands from igs.utils import logging from igs.utils import functional as func from igs.utils import cli OPTIONS = [('prefix', '-p', '--prefix', 'Prefix directory to install everything into', func.compose(os.path.expanduser, os.path.expandvars, cli.notNone)), ('tmpdir', '', '--tmpdir', 'Temporary directory to use, defaults to $TMPDIR and then to /tmp', func.compose( cli.defaultIfNone('/tmp'), cli.defaultIfNone(os.getenv('TMPDIR')))), ('debug', '-d', '--debug', 'Turn debugging output on', func.identity, cli.BINARY)] URLS = [ ('ocaml', 'http://caml.inria.fr/pub/distrib/ocaml-3.12/ocaml-3.12.0.tar.gz'), ('findlib', 'http://download.camlcity.org/download/findlib-1.2.6.tar.gz'), ('ounit', 'http://www.xs4all.nl/~mmzeeman/ocaml/ounit-1.0.3.tar.gz'), ('res', 'http://hg.ocaml.info/release/res/archive/release-3.2.0.tar.gz'), ('pcre-ocaml', 'http://hg.ocaml.info/release/pcre-ocaml/archive/release-6.1.0.tar.gz'), ('type-conv', 'http://hg.ocaml.info/release/type-conv/archive/release-2.0.1.tar.gz'), ('fieldslib', 'http://www.janestreet.com/ocaml/fieldslib-0.1.0.tgz'),
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice import pipeline from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster to run on', cli.defaultIfNone('local')), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ('pipeline', '', '--pipeline-name', 'Name of pipeline to run against', func.identity), ('config', '-c', '', 'Add config options, multiple allowed in style -c key=value -c key=value', cli.defaultIfNone([]), cli.LIST) ] def main(options, args): tasklet = args[0] conf = dict([v.split('=', 1) for v in options('general.config', default=[])]) if options('general.pipeline'): if tasklet: tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline')) else: tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline')) taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet) if options('general.print_task_name'):
#!/usr/bin/env python import sys import os from igs.utils import commands from igs.utils import logging from igs.utils import functional as func from igs.utils import cli OPTIONS = [ ('prefix', '-p', '--prefix', 'Prefix directory to install everything into', func.compose(os.path.expanduser, os.path.expandvars, cli.notNone)), ('tmpdir', '', '--tmpdir', 'Temporary directory to use, defaults to $TMPDIR and then to /tmp', func.compose(cli.defaultIfNone('/tmp'), cli.defaultIfNone(os.getenv('TMPDIR')))), ('debug', '-d', '--debug', 'Turn debugging output on', func.identity, cli.BINARY) ] URLS = [ ('ocaml', 'http://caml.inria.fr/pub/distrib/ocaml-3.12/ocaml-3.12.0.tar.gz'), ('findlib', 'http://download.camlcity.org/download/findlib-1.2.6.tar.gz'), ('ounit', 'http://www.xs4all.nl/~mmzeeman/ocaml/ounit-1.0.3.tar.gz'), ('res', 'http://hg.ocaml.info/release/res/archive/release-3.2.0.tar.gz'), ('pcre-ocaml', 'http://hg.ocaml.info/release/pcre-ocaml/archive/release-6.1.0.tar.gz'), ('type-conv', 'http://hg.ocaml.info/release/type-conv/archive/release-2.0.1.tar.gz'), ('fieldslib', 'http://www.janestreet.com/ocaml/fieldslib-0.1.0.tgz'), ('sexplib', 'http://www.janestreet.com/ocaml/sexplib310-release-4.2.15.tar.gz'), ('bin-prot', 'http://hg.ocaml.info/release/bin-prot/archive/release-1.2.23.tar.gz'), ('core', 'http://www.janestreet.com/ocaml/core-0.6.0.tgz'), ]
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.cluster import addInstances from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster (in this case public host name of master)', cli.defaultIfNone('local')), ('num_exec', '', '--num-exec', 'Number of exec nodes to create', func.compose(int, cli.notNone)), ('exec_type', '', '--type', 'Specify a type for exec if you want', func.identity), ('block', '-b', '--block', 'Block until cluster is up', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, _args): taskName = addInstances(options('general.host'), options('general.cluster'), options('general.num_exec'), 0, options('general.exec_type'))
#!/usr/bin/env python ## # Runs a pipeline throuhg the webservice call import sys from igs.utils import cli from igs.utils import config from igs.utils import functional as func from vappio.webservice import pipeline from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact, defaults to localhost', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')), ('bare_run', '', '--bare', 'Do not run with a wrapper', cli.defaultIfNone(False), cli.BINARY), ('overwrite', '-o', '--overwrite', 'Overwrite the pipeline if it exists', cli.defaultIfNone(False), cli.BINARY), ('pipeline_parent', '', '--pipeline-parent', 'Name of parent pipeline', func.identity), ('pipeline_config', '', '--pipeline-config', 'Config file to use for the pipeline', func.identity), ('pipeline_resume', '', '--resume', 'Name of pipeline to be resumed', func.identity), ('pipeline_queue', '-q', '--pipeline-queue', 'Queue to use, not required', func.identity), ('validate', '-v', '--validate', 'Do a simple type check on the config file and exit without running', cli.defaultIfNone(False), cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ]
#!/usr/bin/env python ## # This updates every directory with the latest from SVN. from igs.utils.cli import buildConfigN, defaultIfNone from igs.utils.commands import runSystemEx, runSingleProgramEx from igs.utils.logging import errorPrint OPTIONS = [ ('stow', '', '--stow', 'Update stow', defaultIfNone(False), True), ('opt_packages', '', '--opt-packages', 'Update opt-packages', defaultIfNone(False), True), ('config_policies', '', '--config_policies', 'Update config_policies', defaultIfNone(False), True), ('vappio_py', '', '--vappio-py', 'Update vappio-py', defaultIfNone(False), True), ('vappio_apps', '', '--vappio-apps', 'Update vappio-apps', defaultIfNone(False), True), ('vappio_twisted', '', '--vappio-twisted', 'Update vappio-twisted', defaultIfNone(False), True), ('vappio_scripts', '', '--vappio-scripts', 'Update vappio-scripts', defaultIfNone(False), True), ('clovr_pipelines', '', '--clovr_pipelines', 'Update clovr_pipelines', defaultIfNone(False), True), ('vappio_py_www', '', '--vappio-py-www', 'Update vappio-www/py-ww', defaultIfNone(False), True), #('vappio_conf', '', '--vappio-conf', 'Update vappio/conf', defaultIfNone(False), True), ('hudson', '', '--hudson', 'Update hudson', defaultIfNone(False), True), ('clovr_www', '', '--clovr-www', 'CloVR web gui', defaultIfNone(False), True), ('co', '', '--co', 'Check out rather than export', defaultIfNone(False), True),
#! /usr/bin/env python ## # This loads the local cluster and if it is an ec2 cluster it updates its information # The main thing it updates right now is the state of the child instances from igs.utils import cli from igs.utils import logging from vappio.cluster import control as cluster_ctl OPTIONS = [ ('debug', '', '--debug', 'Debugging', cli.defaultIfNone(False), True), ] def main(options, _args): logging.DEBUG = options('general.debug') cluster = cluster_ctl.loadCluster('local') if cluster.ctype.NAME in ['EC2', 'Nimbus', 'DIAG']: cluster = cluster.update(dataNodes=cluster.ctype.updateInstances(cluster.credInst, cluster.dataNodes), execNodes=cluster.ctype.updateInstances(cluster.credInst, cluster.execNodes)) logging.debugPrint(lambda : 'Dumping new cluster') cluster_ctl.saveCluster(cluster) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice import tag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective', cli.defaultIfNone('local')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster, hardcoded to local for now', cli.defaultIfNone('local')), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster', cli.defaultIfNone('local')), ('transfer_type', '', '--transfer-type', 'Type of transfer to do (cluster, s3) default is cluster', func.compose(cli.restrictValues(['cluster', 's3']), cli.defaultIfNone('cluster'))), ('block', '-b', '--block', 'Block until cluster is up (no longer used)', func.identity, cli.BINARY), ('dst_type', '', '--dst-type', 'Set type of destination. (stage,local) default is stage', cli.defaultIfNone('stage')), ('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY), ('compress_dir', '', '--compress-dir', 'Compress files into the specified directory', cli.defaultIfNone(None)), ('expand', '', '--expand', 'Expand files (always on regardless of this right now )', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ]
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice import credential OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster', cli.defaultIfNone('local')), ('cred_name', '', '--cred-name', 'Name of credential, all credentials are listed if not provided', func.identity) ] def main(options, _args): if options('general.cred_name'): cred_names = [options('general.cred_name')] else: cred_names = [] cred = credential.loadCredentials(options('general.host'), options('general.name'), cred_names) print '\n'.join(['\t'.join(['CRED', c['name'], str(c['num_instances'])]) for c in cred]) if __name__ == '__main__': main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python ## # Provides useful URLs and other information about the cluster from igs.utils import cli from igs.utils.functional import identity from vappio.webservice.cluster import listClusters OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')), ('list', '-l', '--list', 'List all clusters', cli.defaultIfNone(False), cli.BINARY) ] URL = '/vappio/clusterInfo_ws.py' def instanceToList(i): if i: return [i['instance_id'] or i['spot_request_id'] or 'Undefined', i['public_dns'] or 'Undefined', i['state'] or 'Undefined'] else: return ['None'] def returnEmptyDictIfNone(d, k): if d[k] is None: return {} else: return d[k]
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.cluster import addInstances from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster (in this case public host name of master)', cli.defaultIfNone('local')), ('num_exec', '', '--num-exec', 'Number of exec nodes to create', func.compose(int, cli.notNone)), ('exec_type', '', '--type', 'Specify a type for exec if you want', func.identity), ('block',
#!/usr/bin/env python import os from igs.utils.cli import buildConfigN, notNone, defaultIfNone from igs.utils.config import replaceStr from igs.utils.commands import runSystemEx from igs.utils.functional import identity OPTIONS = [ ('image', '-i', '--image', 'Image to bundle', identity), ('cert', '-c', '--cert', 'Certifiate to use, default $EC2_CERT', defaultIfNone(os.getenv('EC2_CERT'))), ('key', '-k', '--key', 'Key to use, default $EC2_PRIVATE_KEY', defaultIfNone(os.getenv('EC2_PRIVATE_KEY'))), ('user', '-u', '--user', 'AWS account number, defaults to $EC2_ACCOUNT_ID', defaultIfNone(os.getenv('EC2_ACCOUNT_ID'))), ('access_key', '', '--access_key', 'AWS access key, defaults to $EC2_ACCESS_KEY', defaultIfNone(os.getenv('EC2_ACCESS_KEY'))), ('secret_access_key', '', '--secret_access_key', 'AWS secret access key, defaults to $EC2_SECRET_ACCESS_KEY', defaultIfNone(os.getenv('EC2_SECRET_ACCESS_KEY'))), ('dest', '-d', '--dest', 'Destination', notNone), ('ec2cert', '', '--ec2cert', 'EC2 cert to use', identity), ('kernel', '', '--kernel', 'What AKI to use', defaultIfNone('aki-fd15f694')), ('arch', '-r', '--arch', 'Architecture (i386 or x86_64)', identity), ('debug', '', '--debug', 'Display debugging information', identity, True), ] def main(options, _args): cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}',
#!/usr/bin/env python ## # Checks the status of all, or a specific pipeline, running on the cluster from igs.utils import cli from igs.utils import functional as func from vappio.webservice import pipeline as pipeline_client OPTIONS = [('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')), ('pipeline_name', '-p', '--pipeline-name', 'Name of pipeline to show detauls on', func.identity)] def main(options, _args): if options('general.pipeline_name'): pipeline = pipeline_client.pipelineList( options('general.host'), options('general.cluster'), {'pipeline_name': options('general.pipeline_name')}, True)[0] print 'PIPELINE_NAME\t%s' % pipeline['pipeline_name'] print 'PIPELINE_ID\t%s' % pipeline['pipeline_id'] print 'PIPELINE_TYPE\t%s' % pipeline['protocol'] print 'PIPELINE_DESC\t%s' % pipeline['pipeline_desc'] if pipeline['wrapper']: print 'WRAPPER' print 'STATE\t%s' % pipeline['state'] for cl, pName in pipeline['children']: print 'CHILD\t%s\t%s' % (cl, pName)
#!/usr/bin/env python from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import downloadTag from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone), ('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')), ('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity), ('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY), ('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY), ('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, files): taskName = downloadTag(options('general.host'), options('general.tag_name'), options('general.src_cluster'), options('general.dst_cluster'), options('general.output_dir'), options('general.expand'), options('general.compress'))
#!/usr/bin/env python ## # This creates a machine.conf file for the local machine. # This is useful if you need to create a machine.conf for something coming up in Xen or VMWare from igs.utils.cli import buildConfigN, defaultIfNone, notNone from igs.utils.commands import runSingleProgramEx from igs.utils.logging import errorPrintS from vappio.instance.config import createDataFile OPTIONS = [ ('conf', '', '--conf', 'Config file to load', notNone), ('mode', '-m', '--mode', 'The mode of this node type, defaults to MASTER', defaultIfNone('MASTER')), ## # if f is sys.stdout just return that, otherwise try to open f ('output', '-o', '--output', 'Output filename', notNone) ] def main(options, _args): res = [] runSingleProgramEx('hostname -f', res.append, errorPrintS) localHost = ''.join(res).strip() outFile = createDataFile(options, [options('general.mode')], options('general.output')) open(outFile, 'a').write('MASTER_IP=' + localHost + '\n') if __name__ == '__main__': main(*buildConfigN(OPTIONS))
import sys import os import time import signal import Queue import urlparse import glob from igs.utils import logging from igs.utils import cli from igs.utils import commands from igs.utils import functional as func from igs.threading import threads OPTIONS = [ ("base_dir", "-b", "--base-dir", "Base directory to download into", cli.defaultIfNone(".")), ( "join_name", "-j", "--join-name", "If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.", func.identity, ), ("join_md5", "", "--join-md5", "The md5sum of the joined file", func.identity), ("min_rate", "-m", "--min-rate", "Minimum download rate in kilobytes per second", func.compose(int, cli.notNone)), ("tries", "-t", "--tries", "Number of download attempts to make", func.compose(int, cli.defaultIfNone("3"))), ( "max_threads", "", "--max-threads", "Maximum number of threads to download at once",
import sys import time from igs.utils.cli import buildConfigN, defaultIfNone from igs.utils.functional import identity from igs.utils.logging import debugPrint from igs.utils import logging from igs.utils import commands from vappio.webservice.task import loadAllTasks, loadTask from vappio.tasks import task OPTIONS = [ ('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')), ('name', '', '--name', 'Name of cluster, defaults to local', defaultIfNone('local')), ('show_msgs', '', '--show', 'Print out any messages present', identity, True), ('show_error_msgs', '', '--show-error', 'Print out only error messages', identity, True), ('show_debug_msgs', '', '--show-debug', 'Print out any debug messages', identity, True), ('show_all_msgs', '', '--show-all', 'Print out all messages', identity, True), ('debug', '', '--debug', 'Print debugging messages', identity, True), ('no_completed', '', '--nc', 'Filtered out completed tasks', identity, True), ('exit_code', '', '--exit-code', 'Exit with a non zero value if any tasks are not in a completed state',
#!/usr/bin/env python from igs.utils import cli from vappio.webservice.cluster import importCluster from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host from which cluster will be imported', cli.notNone), ('src_cluster', '', '--src-cluster', 'Name of remote cluster to import', cli.defaultIfNone('local')), ('dst_cluster', '', '--dst-cluster', 'Name of cluster on importing VM', cli.notNone), ('cred', '', '--cred', 'Credential to use', cli.notNone), ('print_task_name', '-t', '--print-task-name', 'Print name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ] def main(options, _args): taskName = importCluster(options('general.host'), options('general.src_cluster'), options('general.dst_cluster'), options('general.cred')) if options('general.print_task_name'): print taskName else:
#!/usr/bin/env python import sys import os from igs.utils import cli from igs.utils import functional as func from vappio.webservice.tag import tagData from vappio.tasks.utils import runTaskStatus OPTIONS = [ ('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')), ('cluster', '', '--cluster', 'Name of cluster, defaults to local', cli.defaultIfNone('local')), ('tag_name', '', '--tag-name', 'Name of tag', cli.notNone), ('urls', '', '--url', 'URLs to download, multiple --url flags can be specified', cli.defaultIfNone([]), cli.LIST), ('stdin', '', '--stdin', 'Take list of files from stdin rather than arguments on command line', cli.defaultIfNone(False), cli.BINARY), ('tag_base_dir', '', '--tag-base-dir', 'Base directory of the tag', func.identity), ('recursive', '-r', '--recursive', 'Recursively include directories', cli.defaultIfNone(False), cli.BINARY), ('expand', '-e', '--expand', 'Expand archives', cli.defaultIfNone(False), cli.BINARY), ('compress', '-c', '--compress', 'Make a tarball of the tagged results. This should be the directory to put the tarball. This is not mutually exclusive with --expand', func.identity), ('append', '-a', '--append', 'Append listed files to tag name, ignoring duplicate files', cli.defaultIfNone(False), cli.BINARY), ('overwrite', '-o', '--overwrite', 'Overwrite file list if it exists', cli.defaultIfNone(False), cli.BINARY), ('block', '-b', '--block', 'Block on the tagging', cli.defaultIfNone(False), cli.BINARY), ('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY), ('metadata', '-m', '', 'Add metadata in a key=value notation. Multiple options are valid. Ex: -m filetype=fasta -m usage=referencedb', cli.defaultIfNone([]), cli.LIST)