#!/usr/bin/env python
##
# Checks the status of all, or a specific pipeline, running on the cluster
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import pipeline as pipeline_client
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')),
('pipeline_name', '-p', '--pipeline-name', 'Name of pipeline to show detauls on', func.identity)
]
def main(options, _args):
if options('general.pipeline_name'):
pipeline = pipeline_client.pipelineList(options('general.host'),
options('general.cluster'),
{'pipeline_name': options('general.pipeline_name')},
True)[0]
print 'PIPELINE_NAME\t%s' % pipeline['pipeline_name']
print 'PIPELINE_ID\t%s' % pipeline['pipeline_id']
print 'PIPELINE_TYPE\t%s' % pipeline['protocol']
print 'PIPELINE_DESC\t%s' % pipeline['pipeline_desc']
if pipeline['wrapper']:
print 'WRAPPER'
print 'STATE\t%s' % pipeline['state']
for cl, pName in pipeline['children']:
print 'CHILD\t%s\t%s' % (cl, pName)
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import downloadTag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host',
'Host of web services to connect to, defaults to local host',
cli.defaultIfNone('localhost')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster',
cli.notNone),
('dst_cluster', '', '--dst-cluster',
'Name of dest cluster, hardcoded to local for now', func.const('local')),
('output_dir', '', '--output-dir', 'Name of directory to download to',
func.identity),
('block', '-b', '--block', 'Block until download is complete',
func.identity, cli.BINARY),
('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False),
cli.BINARY),
('compress', '', '--compress', 'Compress files', func.identity,
cli.BINARY),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
]
#!/usr/bin/env python
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from igs.utils.functional import identity
from vappio.webservice.pipeline import downloadPipelineOutput
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact',
defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster', notNone),
##
# Want to make sure this is an int but we want it as a string later in the program
('pipeline', '-p', '--pipeline-name', 'Name of pipeline', notNone),
('output_dir', '-o', '--output-dir',
'Directory the output file should go to', notNone),
('overwrite', '', '--overwrite',
'Do you want to overwrite a local file if it already exists?',
defaultIfNone(False), True),
('block', '-b', '--block', 'Block until download is complete', identity,
True),
('print_task_name', '-t', '--print-task-name',
'Print the name of the task at the end', defaultIfNone(False), True),
]
def main(options, _args):
taskName = downloadPipelineOutput(options('general.host'),
options('general.name'),
options('general.pipeline'),
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.cluster import startCluster
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', cli.notNone),
('num_exec', '', '--num-exec', 'Number of exec nodes to start', int),
('cred', '', '--cred', 'Credential to use', cli.notNone),
('master_instance_type',
'',
'--master-instance-type',
'Which instance type to use, defaults to "default"',
cli.defaultIfNone('default')),
('exec_instance_type',
'',
'--exec-instance-type',
'Which instance type to use, defaults to "default"',
cli.defaultIfNone('default')),
('master_bid_price',
'',
'--master-bid-price',
'Bid price for master, defaults to nothing (on demand instance)',
cli.defaultIfNone('')),
('exec_bid_price',
'',
'--exec-bid-price',
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import tag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host',
'Host of web services to connect to, defaults to local host',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster',
'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective',
cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster',
'Name of source cluster, hardcoded to local for now',
cli.defaultIfNone('local')),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster',
cli.defaultIfNone('local')),
('transfer_type', '', '--transfer-type',
'Type of transfer to do (cluster, s3) default is cluster',
func.compose(cli.restrictValues(['cluster', 's3']),
cli.defaultIfNone('cluster'))),
('block', '-b', '--block', 'Block until cluster is up (no longer used)',
func.identity, cli.BINARY),
('dst_type', '', '--dst-type',
'Set type of destination. (stage,local) default is stage',
cli.defaultIfNone('stage')),
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import tag
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag', func.identity),
]
def main(options, files):
if options('general.tag_name'):
tagData = tag.listTags(options('general.host'),
options('general.cluster'),
{'tag_name': options('general.tag_name')},
detail=True)[0]
if tagData['phantom']:
print 'PHANTOM'
for f in tagData['files']:
print '\t'.join(['FILE', f])
for url in tagData['metadata'].get('urls', []):
print '\t'.join(['URL', url])
for k, v in tagData['metadata'].iteritems():
if k not in ['urls']:
#!/usr/bin/env python
##
# Runs commands on the cluster
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from vappio.cluster.control import runCommandOnCluster
from vappio.webservice.cluster import loadCluster
OPTIONS = [('host', '', '--host', 'Host of webservice to contact',
defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster', notNone),
('just_master', '-m', '--just_master',
'Just run the command on the master', defaultIfNone(False), True),
('command', '', '--cmd', 'Command to run', notNone)]
def main(options, _args):
cluster = loadCluster(options('general.host'), options('general.name'))
runCommandOnCluster(cluster, options('general.command'),
options('general.just_master'))
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
#!/usr/bin/env python
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from igs.utils.functional import identity
from vappio.webservice.pipeline import downloadPipelineOutput
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster', notNone),
##
# Want to make sure this is an int but we want it as a string later in the program
('pipeline', '-p', '--pipeline-name', 'Name of pipeline', notNone),
('output_dir', '-o', '--output-dir', 'Directory the output file should go to', notNone),
('overwrite', '', '--overwrite', 'Do you want to overwrite a local file if it already exists?', defaultIfNone(False), True),
('block', '-b', '--block', 'Block until download is complete', identity, True),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', defaultIfNone(False), True),
]
def main(options, _args):
taskName = downloadPipelineOutput(options('general.host'),
options('general.name'),
options('general.pipeline'),
options('general.output_dir'),
options('general.overwrite'))
if options('general.print_task_name'):
print taskName
#!/usr/bin/env python
##
# Runs commands on the cluster
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from vappio.cluster.control import runCommandOnCluster
from vappio.webservice.cluster import loadCluster
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster', notNone),
('just_master', '-m', '--just_master', 'Just run the command on the master', defaultIfNone(False), True),
('command', '', '--cmd', 'Command to run', notNone)
]
def main(options, _args):
cluster = loadCluster(options('general.host'), options('general.name'))
runCommandOnCluster(cluster, options('general.command'), options('general.just_master'))
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
import os
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from igs.utils.functional import identity
from igs.utils import config
from igs.utils.commands import runSystemEx, runSingleProgramEx
from igs.utils.logging import logPrint, errorPrint
from igs.threading import threads
OPTIONS = [
('version', '', '--version', 'Version of this release', notNone),
('remote_name', '', '--remote-name',
'Name of remote machine the image lives on', notNone),
('image', '-i', '--image', 'Image to bundle', identity),
('cert', '-c', '--cert', 'Certifiate to use, default $EC2_CERT',
defaultIfNone(os.getenv('EC2_CERT'))),
('key', '-k', '--key', 'Key to use, default $EC2_PRIVATE_KEY',
defaultIfNone(os.getenv('EC2_PRIVATE_KEY'))),
('user', '-u', '--user', 'AWS account number, defaults to $EC2_ACCOUNT_ID',
defaultIfNone(os.getenv('EC2_ACCOUNT_ID'))),
('access_key', '', '--access_key',
'AWS access key, defaults to $EC2_ACCESS_KEY',
defaultIfNone(os.getenv('EC2_ACCESS_KEY'))),
('secret_access_key', '', '--secret_access_key',
'AWS secret access key, defaults to $EC2_SECRET_ACCESS_KEY',
defaultIfNone(os.getenv('EC2_SECRET_ACCESS_KEY'))),
('dest', '-d', '--dest', 'Destination', notNone),
('ec2cert', '', '--ec2cert', 'EC2 cert to use', identity),
('kernel', '', '--kernel', 'What AKI to use',
defaultIfNone('aki-fd15f694')),
('arch', '-r', '--arch', 'Architecture (i386 or x86_64)', identity),
# This script provides some insight into tasks
import sys
import time
from igs.utils.cli import buildConfigN, defaultIfNone
from igs.utils.functional import identity
from igs.utils.logging import debugPrint
from igs.utils import logging
from igs.utils import commands
from vappio.webservice.task import loadAllTasks, loadTask
from vappio.tasks import task
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster, defaults to local', defaultIfNone('local')),
('show_msgs', '', '--show', 'Print out any messages present', identity, True),
('show_error_msgs', '', '--show-error', 'Print out only error messages', identity, True),
('show_debug_msgs', '', '--show-debug', 'Print out any debug messages', identity, True),
('show_all_msgs', '', '--show-all', 'Print out all messages', identity, True),
('debug', '', '--debug', 'Print debugging messages', identity, True),
('no_completed', '', '--nc', 'Filtered out completed tasks', identity, True),
('exit_code', '', '--exit-code', 'Exit with a non zero value if any tasks are not in a completed state', identity, True),
('block', '-b', '--block', 'Block on the tasks until they have completed or errored', identity, True),
('no_print_polling', '', '--no-print-polling', 'If block is true, do not print while polling to see what is happening in real time', identity, True),
]
def timestampToStr(ts):
return time.strftime('%Y/%m/%d %H:%M:%S UTC', time.gmtime(ts))
#!/usr/bin/env python
from twisted.python import reflect
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import credential
from vappio.tasks.utils import runTaskStatus
from vappio.credentials import manager
OPTIONS = [
('cred_name', '', '--cred-name', 'Name of the credential', cli.notNone),
('cred_desc', '', '--desc', 'Description of the credential',
cli.defaultIfNone('')),
('ctype', '', '--ctype',
'Cluster type this cert is for (ec2, nimbus, diag, adhoc, local)',
cli.restrictValues(['diag', 'nimbus', 'ec2', 'local', 'adhoc'])),
('cert', '-c', '--cert', 'Path to cert', func.identity),
('pkey', '-p', '--pkey', 'path to private key', func.identity),
('devel', '-d', '--devel', 'Path to devel1.pem', func.identity),
('metadata', '-m', '',
('Add metadata in a key=value notation. Multiple options are valid. '
'Ex: -m ec2_url=http://foo/bar/zoom -m comment="This works"'),
cli.defaultIfNone([]), cli.LIST),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY)
]
#!/usr/bin/env python
import sys
import os
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import tagData
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host',
'Host of web services to connect to, defaults to local host',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster, defaults to local',
cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag', cli.notNone),
('urls', '', '--url',
'URLs to download, multiple --url flags can be specified',
cli.defaultIfNone([]), cli.LIST),
('stdin', '', '--stdin',
'Take list of files from stdin rather than arguments on command line',
cli.defaultIfNone(False), cli.BINARY),
('tag_base_dir', '', '--tag-base-dir', 'Base directory of the tag',
func.identity),
('recursive', '-r', '--recursive', 'Recursively include directories',
cli.defaultIfNone(False), cli.BINARY),
('expand', '-e', '--expand', 'Expand archives', cli.defaultIfNone(False),
cli.BINARY),
('compress', '-c', '--compress',
import os
from igs.utils import cli
from igs.utils import logging
from igs.utils import commands
from igs.utils import functional as func
#
# A bit unorthodoxed but it is useful that every CLI app
# is designed to also work as a library if needed
from vappio.cli import reliableDownloader
OPTIONS = [(
'suffix', '-s', '--suffix',
'The suffice that the md5 version of the files are expected to have. Defaults to ".md5" (not the ".")',
cli.defaultIfNone('.md5')),
('debug', '', '--debug', 'Add debug logging', func.identity,
cli.BINARY)]
def main(options, args):
logging.DEBUG = options('general.debug')
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
urls.append(line.strip() + options('general.suffix'))
else:
urls.extend([u + options('general.suffix') for u in args])
for url in urls:
reliableDownloader.deleteDownloadedFiles('/tmp', url)
import sys
import os
import time
import signal
import Queue
import urlparse
import glob
from igs.utils import logging
from igs.utils import cli
from igs.utils import commands
from igs.utils import functional as func
from igs.threading import threads
OPTIONS = [
('base_dir', '-b', '--base-dir', 'Base directory to download into', cli.defaultIfNone('.')),
('join_name', '-j', '--join-name', 'If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.', func.identity),
('join_md5', '', '--join-md5', 'The md5sum of the joined file', func.identity),
('min_rate', '-m', '--min-rate', 'Minimum download rate in kilobytes per second', func.compose(int, cli.notNone)),
('tries', '-t', '--tries', 'Number of download attempts to make', func.compose(int, cli.defaultIfNone('3'))),
('max_threads', '', '--max-threads', 'Maximum number of threads to download at once', func.compose(int, cli.defaultIfNone('3'))),
('continue_download', '-c', '--continue', 'Continue the download. Default is to download everything from scratch', cli.defaultIfNone(False), True),
('debug', '-d', '--debug', 'Turn debug on', cli.defaultIfNone(False), True),
]
##
# How many seconds each sample should be
SAMPLE_RATE = 5
MAX_SAMPLE_SIZE = 10
# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state
# 2) Someone could check out a whole directory and then check out a file into of that directory
#
# To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single
# files an
import os
from igs.utils import commands
from igs.utils import cli
from igs.utils import functional as func
from igs.utils import logging
OPTIONS = [
('codir', '', '--co-dir',
'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir',
func.compose(cli.defaultIfNone('/opt/co-dir'), cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))),
('branch', '-b', '--branch',
'Override the branch specified in config files',
func.identity),
('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY),
('export', '', '--export', 'Force an export, mutually exclusive with --checkout', func.identity, cli.BINARY),
('config_dir', '-c', '--config-dir', 'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout',
func.compose(cli.defaultIfNone('/etc/vappio-checkout'), cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))),
('debug', '-d', '--debug', 'Turn debugging information on', func.identity, cli.BINARY),
]
class CheckoutModifiedError(Exception):
"""
Trying to do another check out when modifications exist
"""
#!/usr/bin/env python
from igs.utils import functional as func
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from vappio.webservice.cluster import terminateCluster
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', notNone),
('auth_token', '', '--auth-token', 'Authorization token to pass along to target cluster', func.identity),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', defaultIfNone(False), True),
]
def main(options, _args):
taskName = terminateCluster(options('general.host'),
options('general.cluster'),
options('general.auth_token'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName,
clusterName='local',
hostName=options('general.host'))
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state
# 2) Someone could check out a whole directory and then check out a file into of that directory
#
# To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single
# files an
import os
from igs.utils import commands
from igs.utils import cli
from igs.utils import functional as func
from igs.utils import logging
OPTIONS = [
('codir', '', '--co-dir',
'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir',
func.compose(cli.defaultIfNone('/opt/co-dir'),
cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))),
('branch', '-b', '--branch',
'Override the branch specified in config files', func.identity),
('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY),
('export', '', '--export',
'Force an export, mutually exclusive with --checkout', func.identity,
cli.BINARY),
('config_dir', '-c', '--config-dir',
'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout',
func.compose(cli.defaultIfNone('/etc/vappio-checkout'),
cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))),
('debug', '-d', '--debug', 'Turn debugging information on', func.identity,
cli.BINARY),
]
#!/usr/bin/env python
import sys
import os
from igs.utils import commands
from igs.utils import logging
from igs.utils import functional as func
from igs.utils import cli
OPTIONS = [('prefix', '-p', '--prefix',
'Prefix directory to install everything into',
func.compose(os.path.expanduser, os.path.expandvars, cli.notNone)),
('tmpdir', '', '--tmpdir',
'Temporary directory to use, defaults to $TMPDIR and then to /tmp',
func.compose(
cli.defaultIfNone('/tmp'),
cli.defaultIfNone(os.getenv('TMPDIR')))),
('debug', '-d', '--debug', 'Turn debugging output on',
func.identity, cli.BINARY)]
URLS = [
('ocaml',
'http://caml.inria.fr/pub/distrib/ocaml-3.12/ocaml-3.12.0.tar.gz'),
('findlib', 'http://download.camlcity.org/download/findlib-1.2.6.tar.gz'),
('ounit', 'http://www.xs4all.nl/~mmzeeman/ocaml/ounit-1.0.3.tar.gz'),
('res', 'http://hg.ocaml.info/release/res/archive/release-3.2.0.tar.gz'),
('pcre-ocaml',
'http://hg.ocaml.info/release/pcre-ocaml/archive/release-6.1.0.tar.gz'),
('type-conv',
'http://hg.ocaml.info/release/type-conv/archive/release-2.0.1.tar.gz'),
('fieldslib', 'http://www.janestreet.com/ocaml/fieldslib-0.1.0.tgz'),
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import pipeline
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster to run on', cli.defaultIfNone('local')),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
('pipeline', '', '--pipeline-name', 'Name of pipeline to run against', func.identity),
('config', '-c', '', 'Add config options, multiple allowed in style -c key=value -c key=value',
cli.defaultIfNone([]), cli.LIST)
]
def main(options, args):
tasklet = args[0]
conf = dict([v.split('=', 1) for v in options('general.config', default=[])])
if options('general.pipeline'):
if tasklet:
tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline'))
else:
tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline'))
taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet)
if options('general.print_task_name'):
#!/usr/bin/env python
import sys
import os
from igs.utils import commands
from igs.utils import logging
from igs.utils import functional as func
from igs.utils import cli
OPTIONS = [
('prefix', '-p', '--prefix', 'Prefix directory to install everything into',
func.compose(os.path.expanduser, os.path.expandvars, cli.notNone)),
('tmpdir', '', '--tmpdir', 'Temporary directory to use, defaults to $TMPDIR and then to /tmp',
func.compose(cli.defaultIfNone('/tmp'), cli.defaultIfNone(os.getenv('TMPDIR')))),
('debug', '-d', '--debug', 'Turn debugging output on', func.identity, cli.BINARY)
]
URLS = [
('ocaml', 'http://caml.inria.fr/pub/distrib/ocaml-3.12/ocaml-3.12.0.tar.gz'),
('findlib', 'http://download.camlcity.org/download/findlib-1.2.6.tar.gz'),
('ounit', 'http://www.xs4all.nl/~mmzeeman/ocaml/ounit-1.0.3.tar.gz'),
('res', 'http://hg.ocaml.info/release/res/archive/release-3.2.0.tar.gz'),
('pcre-ocaml', 'http://hg.ocaml.info/release/pcre-ocaml/archive/release-6.1.0.tar.gz'),
('type-conv', 'http://hg.ocaml.info/release/type-conv/archive/release-2.0.1.tar.gz'),
('fieldslib', 'http://www.janestreet.com/ocaml/fieldslib-0.1.0.tgz'),
('sexplib', 'http://www.janestreet.com/ocaml/sexplib310-release-4.2.15.tar.gz'),
('bin-prot', 'http://hg.ocaml.info/release/bin-prot/archive/release-1.2.23.tar.gz'),
('core', 'http://www.janestreet.com/ocaml/core-0.6.0.tgz'),
]
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.cluster import addInstances
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster',
'Name of cluster (in this case public host name of master)',
cli.defaultIfNone('local')),
('num_exec', '', '--num-exec', 'Number of exec nodes to create',
func.compose(int, cli.notNone)),
('exec_type', '', '--type', 'Specify a type for exec if you want',
func.identity),
('block', '-b', '--block', 'Block until cluster is up', func.identity,
cli.BINARY),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
]
def main(options, _args):
taskName = addInstances(options('general.host'),
options('general.cluster'),
options('general.num_exec'), 0,
options('general.exec_type'))
#!/usr/bin/env python
##
# Runs a pipeline throuhg the webservice call
import sys
from igs.utils import cli
from igs.utils import config
from igs.utils import functional as func
from vappio.webservice import pipeline
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact, defaults to localhost', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')),
('bare_run', '', '--bare', 'Do not run with a wrapper', cli.defaultIfNone(False), cli.BINARY),
('overwrite', '-o', '--overwrite', 'Overwrite the pipeline if it exists', cli.defaultIfNone(False), cli.BINARY),
('pipeline_parent', '', '--pipeline-parent', 'Name of parent pipeline', func.identity),
('pipeline_config', '', '--pipeline-config', 'Config file to use for the pipeline', func.identity),
('pipeline_resume', '', '--resume', 'Name of pipeline to be resumed', func.identity),
('pipeline_queue', '-q', '--pipeline-queue', 'Queue to use, not required', func.identity),
('validate',
'-v',
'--validate',
'Do a simple type check on the config file and exit without running',
cli.defaultIfNone(False),
cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
]
#!/usr/bin/env python
##
# This updates every directory with the latest from SVN.
from igs.utils.cli import buildConfigN, defaultIfNone
from igs.utils.commands import runSystemEx, runSingleProgramEx
from igs.utils.logging import errorPrint
OPTIONS = [
('stow', '', '--stow', 'Update stow', defaultIfNone(False), True),
('opt_packages', '', '--opt-packages', 'Update opt-packages',
defaultIfNone(False), True),
('config_policies', '', '--config_policies', 'Update config_policies',
defaultIfNone(False), True),
('vappio_py', '', '--vappio-py', 'Update vappio-py', defaultIfNone(False),
True),
('vappio_apps', '', '--vappio-apps', 'Update vappio-apps',
defaultIfNone(False), True),
('vappio_twisted', '', '--vappio-twisted', 'Update vappio-twisted',
defaultIfNone(False), True),
('vappio_scripts', '', '--vappio-scripts', 'Update vappio-scripts',
defaultIfNone(False), True),
('clovr_pipelines', '', '--clovr_pipelines', 'Update clovr_pipelines',
defaultIfNone(False), True),
('vappio_py_www', '', '--vappio-py-www', 'Update vappio-www/py-ww',
defaultIfNone(False), True),
#('vappio_conf', '', '--vappio-conf', 'Update vappio/conf', defaultIfNone(False), True),
('hudson', '', '--hudson', 'Update hudson', defaultIfNone(False), True),
('clovr_www', '', '--clovr-www', 'CloVR web gui', defaultIfNone(False),
True),
('co', '', '--co', 'Check out rather than export', defaultIfNone(False),
True),
#! /usr/bin/env python
##
# This loads the local cluster and if it is an ec2 cluster it updates its information
# The main thing it updates right now is the state of the child instances
from igs.utils import cli
from igs.utils import logging
from vappio.cluster import control as cluster_ctl
OPTIONS = [
('debug', '', '--debug', 'Debugging', cli.defaultIfNone(False), True),
]
def main(options, _args):
logging.DEBUG = options('general.debug')
cluster = cluster_ctl.loadCluster('local')
if cluster.ctype.NAME in ['EC2', 'Nimbus', 'DIAG']:
cluster = cluster.update(dataNodes=cluster.ctype.updateInstances(cluster.credInst, cluster.dataNodes),
execNodes=cluster.ctype.updateInstances(cluster.credInst, cluster.execNodes))
logging.debugPrint(lambda : 'Dumping new cluster')
cluster_ctl.saveCluster(cluster)
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import tag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster',
'',
'--cluster',
'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective',
cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster, hardcoded to local for now', cli.defaultIfNone('local')),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster', cli.defaultIfNone('local')),
('transfer_type', '', '--transfer-type', 'Type of transfer to do (cluster, s3) default is cluster',
func.compose(cli.restrictValues(['cluster', 's3']), cli.defaultIfNone('cluster'))),
('block', '-b', '--block', 'Block until cluster is up (no longer used)', func.identity, cli.BINARY),
('dst_type', '', '--dst-type', 'Set type of destination. (stage,local) default is stage', cli.defaultIfNone('stage')),
('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY),
('compress_dir', '', '--compress-dir', 'Compress files into the specified directory', cli.defaultIfNone(None)),
('expand', '', '--expand', 'Expand files (always on regardless of this right now )', func.identity, cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
]
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import credential
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact', cli.defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster', cli.defaultIfNone('local')),
('cred_name', '', '--cred-name', 'Name of credential, all credentials are listed if not provided', func.identity)
]
def main(options, _args):
if options('general.cred_name'):
cred_names = [options('general.cred_name')]
else:
cred_names = []
cred = credential.loadCredentials(options('general.host'), options('general.name'), cred_names)
print '\n'.join(['\t'.join(['CRED', c['name'], str(c['num_instances'])]) for c in cred])
if __name__ == '__main__':
main(*cli.buildConfigN(OPTIONS))
#!/usr/bin/env python
from twisted.python import reflect
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import credential
from vappio.tasks.utils import runTaskStatus
from vappio.credentials import manager
OPTIONS = [
('cred_name', '', '--cred-name', 'Name of the credential', cli.notNone),
('cred_desc', '', '--desc', 'Description of the credential', cli.defaultIfNone('')),
('ctype', '', '--ctype', 'Cluster type this cert is for (ec2, nimbus, diag, adhoc, local)',
cli.restrictValues(['diag', 'nimbus', 'ec2', 'local', 'adhoc'])),
('cert', '-c', '--cert', 'Path to cert', func.identity),
('pkey', '-p', '--pkey', 'path to private key', func.identity),
('devel', '-d', '--devel', 'Path to devel1.pem', func.identity),
('metadata', '-m', '',
('Add metadata in a key=value notation. Multiple options are valid. '
'Ex: -m ec2_url=http://foo/bar/zoom -m comment="This works"'),
cli.defaultIfNone([]),
cli.LIST),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY)
]
#!/usr/bin/env python
##
# Provides useful URLs and other information about the cluster
from igs.utils import cli
from igs.utils.functional import identity
from vappio.webservice.cluster import listClusters
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster', cli.defaultIfNone('local')),
('list', '-l', '--list', 'List all clusters', cli.defaultIfNone(False), cli.BINARY)
]
URL = '/vappio/clusterInfo_ws.py'
def instanceToList(i):
if i:
return [i['instance_id'] or i['spot_request_id'] or 'Undefined',
i['public_dns'] or 'Undefined',
i['state'] or 'Undefined']
else:
return ['None']
def returnEmptyDictIfNone(d, k):
if d[k] is None:
return {}
else:
return d[k]
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.cluster import addInstances
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host',
'',
'--host',
'Host of webservice to contact',
cli.defaultIfNone('localhost')),
('cluster',
'',
'--cluster',
'Name of cluster (in this case public host name of master)',
cli.defaultIfNone('local')),
('num_exec',
'',
'--num-exec',
'Number of exec nodes to create',
func.compose(int, cli.notNone)),
('exec_type',
'',
'--type',
'Specify a type for exec if you want',
func.identity),
('block',
#!/usr/bin/env python
import os
from igs.utils.cli import buildConfigN, notNone, defaultIfNone
from igs.utils.config import replaceStr
from igs.utils.commands import runSystemEx
from igs.utils.functional import identity
OPTIONS = [
('image', '-i', '--image', 'Image to bundle', identity),
('cert', '-c', '--cert', 'Certifiate to use, default $EC2_CERT', defaultIfNone(os.getenv('EC2_CERT'))),
('key', '-k', '--key', 'Key to use, default $EC2_PRIVATE_KEY', defaultIfNone(os.getenv('EC2_PRIVATE_KEY'))),
('user', '-u', '--user', 'AWS account number, defaults to $EC2_ACCOUNT_ID', defaultIfNone(os.getenv('EC2_ACCOUNT_ID'))),
('access_key', '', '--access_key', 'AWS access key, defaults to $EC2_ACCESS_KEY', defaultIfNone(os.getenv('EC2_ACCESS_KEY'))),
('secret_access_key', '', '--secret_access_key', 'AWS secret access key, defaults to $EC2_SECRET_ACCESS_KEY',
defaultIfNone(os.getenv('EC2_SECRET_ACCESS_KEY'))),
('dest', '-d', '--dest', 'Destination', notNone),
('ec2cert', '', '--ec2cert', 'EC2 cert to use', identity),
('kernel', '', '--kernel', 'What AKI to use', defaultIfNone('aki-fd15f694')),
('arch', '-r', '--arch', 'Architecture (i386 or x86_64)', identity),
('debug', '', '--debug', 'Display debugging information', identity, True),
]
def main(options, _args):
cmd = ['ec2-bundle-image',
'-c ${general.cert}',
'-k ${general.key}',
'-u ${general.user}',
'--kernel ${general.kernel}',
#!/usr/bin/env python
##
# Checks the status of all, or a specific pipeline, running on the cluster
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice import pipeline as pipeline_client
OPTIONS = [('host', '', '--host', 'Host of webservice to contact',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster',
cli.defaultIfNone('local')),
('pipeline_name', '-p', '--pipeline-name',
'Name of pipeline to show detauls on', func.identity)]
def main(options, _args):
if options('general.pipeline_name'):
pipeline = pipeline_client.pipelineList(
options('general.host'), options('general.cluster'),
{'pipeline_name': options('general.pipeline_name')}, True)[0]
print 'PIPELINE_NAME\t%s' % pipeline['pipeline_name']
print 'PIPELINE_ID\t%s' % pipeline['pipeline_id']
print 'PIPELINE_TYPE\t%s' % pipeline['protocol']
print 'PIPELINE_DESC\t%s' % pipeline['pipeline_desc']
if pipeline['wrapper']:
print 'WRAPPER'
print 'STATE\t%s' % pipeline['state']
for cl, pName in pipeline['children']:
print 'CHILD\t%s\t%s' % (cl, pName)
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import downloadTag
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster', cli.notNone),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster, hardcoded to local for now', func.const('local')),
('output_dir', '', '--output-dir', 'Name of directory to download to', func.identity),
('block', '-b', '--block', 'Block until download is complete', func.identity, cli.BINARY),
('expand', '', '--expand', 'Expand files', cli.defaultIfNone(False), cli.BINARY),
('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
]
def main(options, files):
taskName = downloadTag(options('general.host'),
options('general.tag_name'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.output_dir'),
options('general.expand'),
options('general.compress'))
#!/usr/bin/env python
##
# This creates a machine.conf file for the local machine.
# This is useful if you need to create a machine.conf for something coming up in Xen or VMWare
from igs.utils.cli import buildConfigN, defaultIfNone, notNone
from igs.utils.commands import runSingleProgramEx
from igs.utils.logging import errorPrintS
from vappio.instance.config import createDataFile
OPTIONS = [
('conf', '', '--conf', 'Config file to load', notNone),
('mode', '-m', '--mode', 'The mode of this node type, defaults to MASTER',
defaultIfNone('MASTER')),
##
# if f is sys.stdout just return that, otherwise try to open f
('output', '-o', '--output', 'Output filename', notNone)
]
def main(options, _args):
res = []
runSingleProgramEx('hostname -f', res.append, errorPrintS)
localHost = ''.join(res).strip()
outFile = createDataFile(options, [options('general.mode')],
options('general.output'))
open(outFile, 'a').write('MASTER_IP=' + localHost + '\n')
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
import sys
import os
import time
import signal
import Queue
import urlparse
import glob
from igs.utils import logging
from igs.utils import cli
from igs.utils import commands
from igs.utils import functional as func
from igs.threading import threads
OPTIONS = [
("base_dir", "-b", "--base-dir", "Base directory to download into", cli.defaultIfNone(".")),
(
"join_name",
"-j",
"--join-name",
"If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.",
func.identity,
),
("join_md5", "", "--join-md5", "The md5sum of the joined file", func.identity),
("min_rate", "-m", "--min-rate", "Minimum download rate in kilobytes per second", func.compose(int, cli.notNone)),
("tries", "-t", "--tries", "Number of download attempts to make", func.compose(int, cli.defaultIfNone("3"))),
(
"max_threads",
"",
"--max-threads",
"Maximum number of threads to download at once",
import sys
import time
from igs.utils.cli import buildConfigN, defaultIfNone
from igs.utils.functional import identity
from igs.utils.logging import debugPrint
from igs.utils import logging
from igs.utils import commands
from vappio.webservice.task import loadAllTasks, loadTask
from vappio.tasks import task
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact',
defaultIfNone('localhost')),
('name', '', '--name', 'Name of cluster, defaults to local',
defaultIfNone('local')),
('show_msgs', '', '--show', 'Print out any messages present', identity,
True),
('show_error_msgs', '', '--show-error', 'Print out only error messages',
identity, True),
('show_debug_msgs', '', '--show-debug', 'Print out any debug messages',
identity, True),
('show_all_msgs', '', '--show-all', 'Print out all messages', identity,
True),
('debug', '', '--debug', 'Print debugging messages', identity, True),
('no_completed', '', '--nc', 'Filtered out completed tasks', identity,
True),
('exit_code', '', '--exit-code',
'Exit with a non zero value if any tasks are not in a completed state',
#!/usr/bin/env python
from igs.utils import cli
from vappio.webservice.cluster import importCluster
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host from which cluster will be imported', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of remote cluster to import', cli.defaultIfNone('local')),
('dst_cluster', '', '--dst-cluster', 'Name of cluster on importing VM', cli.notNone),
('cred', '', '--cred', 'Credential to use', cli.notNone),
('print_task_name',
'-t',
'--print-task-name',
'Print name of the task at the end',
cli.defaultIfNone(False),
cli.BINARY),
]
def main(options, _args):
taskName = importCluster(options('general.host'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.cred'))
if options('general.print_task_name'):
print taskName
else:
#!/usr/bin/env python
##
# This creates a machine.conf file for the local machine.
# This is useful if you need to create a machine.conf for something coming up in Xen or VMWare
from igs.utils.cli import buildConfigN, defaultIfNone, notNone
from igs.utils.commands import runSingleProgramEx
from igs.utils.logging import errorPrintS
from vappio.instance.config import createDataFile
OPTIONS = [
('conf', '', '--conf', 'Config file to load', notNone),
('mode', '-m', '--mode', 'The mode of this node type, defaults to MASTER', defaultIfNone('MASTER')),
##
# if f is sys.stdout just return that, otherwise try to open f
('output', '-o', '--output', 'Output filename', notNone)
]
def main(options, _args):
res = []
runSingleProgramEx('hostname -f', res.append, errorPrintS)
localHost = ''.join(res).strip()
outFile = createDataFile(options, [options('general.mode')], options('general.output'))
open(outFile, 'a').write('MASTER_IP=' + localHost + '\n')
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
#!/usr/bin/env python
import sys
import os
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.tag import tagData
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster', '', '--cluster', 'Name of cluster, defaults to local', cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag', cli.notNone),
('urls', '', '--url', 'URLs to download, multiple --url flags can be specified', cli.defaultIfNone([]), cli.LIST),
('stdin', '', '--stdin', 'Take list of files from stdin rather than arguments on command line', cli.defaultIfNone(False), cli.BINARY),
('tag_base_dir', '', '--tag-base-dir', 'Base directory of the tag', func.identity),
('recursive', '-r', '--recursive', 'Recursively include directories', cli.defaultIfNone(False), cli.BINARY),
('expand', '-e', '--expand', 'Expand archives', cli.defaultIfNone(False), cli.BINARY),
('compress', '-c', '--compress',
'Make a tarball of the tagged results. This should be the directory to put the tarball. This is not mutually exclusive with --expand',
func.identity),
('append', '-a', '--append', 'Append listed files to tag name, ignoring duplicate files', cli.defaultIfNone(False), cli.BINARY),
('overwrite', '-o', '--overwrite', 'Overwrite file list if it exists', cli.defaultIfNone(False), cli.BINARY),
('block', '-b', '--block', 'Block on the tagging', cli.defaultIfNone(False), cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
('metadata', '-m', '',
'Add metadata in a key=value notation. Multiple options are valid. Ex: -m filetype=fasta -m usage=referencedb',
cli.defaultIfNone([]),
cli.LIST)
