# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state
# 2) Someone could check out a whole directory and then check out a file into of that directory
#
# To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single
# files an
import os
from igs.utils import commands
from igs.utils import cli
from igs.utils import functional as func
from igs.utils import logging
OPTIONS = [
('codir', '', '--co-dir',
'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir',
func.compose(cli.defaultIfNone('/opt/co-dir'), cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))),
('branch', '-b', '--branch',
'Override the branch specified in config files',
func.identity),
('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY),
('export', '', '--export', 'Force an export, mutually exclusive with --checkout', func.identity, cli.BINARY),
('config_dir', '-c', '--config-dir', 'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout',
func.compose(cli.defaultIfNone('/etc/vappio-checkout'), cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))),
('debug', '-d', '--debug', 'Turn debugging information on', func.identity, cli.BINARY),
]
class CheckoutModifiedError(Exception):
"""
Trying to do another check out when modifications exist
"""
# 1) You cannot checkout a single file (in SVN) because a single file cannot keep track of its state
# 2) Someone could check out a whole directory and then check out a file into of that directory
#
# To solve (1), when something is checked out, it is checked out into a secondary location and then symlinked in. This happens for single
# files an
import os
from igs.utils import commands
from igs.utils import cli
from igs.utils import functional as func
from igs.utils import logging
OPTIONS = [
('codir', '', '--co-dir',
'Directory to put files in case of a check out. If not set, defaults to $VAPPIO_CO_DIR, if that is not set defaults to /opt/co-dir',
func.compose(cli.defaultIfNone('/opt/co-dir'),
cli.defaultIfNone(os.getenv('VAPPIO_CO_DIR')))),
('branch', '-b', '--branch',
'Override the branch specified in config files', func.identity),
('checkout', '', '--co', 'Force a checkout', func.identity, cli.BINARY),
('export', '', '--export',
'Force an export, mutually exclusive with --checkout', func.identity,
cli.BINARY),
('config_dir', '-c', '--config-dir',
'Directory to look for config files, defaults to $VAPPIO_REPO_CONF then /etc/vappio-checkout',
func.compose(cli.defaultIfNone('/etc/vappio-checkout'),
cli.defaultIfNone(os.getenv('VAPPIO_REPO_CONF')))),
('debug', '-d', '--debug', 'Turn debugging information on', func.identity,
cli.BINARY),
]
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of web services to connect to, defaults to local host', cli.defaultIfNone('localhost')),
('cluster',
'',
'--cluster',
'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective',
cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster', 'Name of source cluster, hardcoded to local for now', cli.defaultIfNone('local')),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster', cli.defaultIfNone('local')),
('transfer_type', '', '--transfer-type', 'Type of transfer to do (cluster, s3) default is cluster',
func.compose(cli.restrictValues(['cluster', 's3']), cli.defaultIfNone('cluster'))),
('block', '-b', '--block', 'Block until cluster is up (no longer used)', func.identity, cli.BINARY),
('dst_type', '', '--dst-type', 'Set type of destination. (stage,local) default is stage', cli.defaultIfNone('stage')),
('compress', '', '--compress', 'Compress files', func.identity, cli.BINARY),
('compress_dir', '', '--compress-dir', 'Compress files into the specified directory', cli.defaultIfNone(None)),
('expand', '', '--expand', 'Expand files (always on regardless of this right now )', func.identity, cli.BINARY),
('print_task_name', '-t', '--print-task-name', 'Print the name of the task at the end', cli.defaultIfNone(False), cli.BINARY),
]
def transferBetweenClusters(options):
return tag.transferTag(options('general.host'),
options('general.cluster'),
options('general.tag_name'),
options('general.src_cluster'),
options('general.dst_cluster'),
OPTIONS = [
('host',
'',
'--host',
'Host of webservice to contact',
cli.defaultIfNone('localhost')),
('cluster',
'',
'--cluster',
'Name of cluster (in this case public host name of master)',
cli.defaultIfNone('local')),
('num_exec',
'',
'--num-exec',
'Number of exec nodes to create',
func.compose(int, cli.notNone)),
('exec_type',
'',
'--type',
'Specify a type for exec if you want',
func.identity),
('block',
'-b',
'--block',
'Block until cluster is up',
func.identity,
cli.BINARY),
('print_task_name',
'-t',
'--print-task-name',
'Print the name of the task at the end',
#!/usr/bin/env python
from igs.utils import cli
from igs.utils import functional as func
from vappio.webservice.cluster import addInstances
from vappio.tasks.utils import runTaskStatus
OPTIONS = [
('host', '', '--host', 'Host of webservice to contact',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster',
'Name of cluster (in this case public host name of master)',
cli.defaultIfNone('local')),
('num_exec', '', '--num-exec', 'Number of exec nodes to create',
func.compose(int, cli.notNone)),
('exec_type', '', '--type', 'Specify a type for exec if you want',
func.identity),
('block', '-b', '--block', 'Block until cluster is up', func.identity,
cli.BINARY),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),
]
def main(options, _args):
taskName = addInstances(options('general.host'),
options('general.cluster'),
options('general.num_exec'), 0,
options('general.exec_type'))
from igs.utils import cli
from igs.utils import commands
from igs.utils import functional as func
from igs.threading import threads
OPTIONS = [
("base_dir", "-b", "--base-dir", "Base directory to download into", cli.defaultIfNone(".")),
(
"join_name",
"-j",
"--join-name",
"If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.",
func.identity,
),
("join_md5", "", "--join-md5", "The md5sum of the joined file", func.identity),
("min_rate", "-m", "--min-rate", "Minimum download rate in kilobytes per second", func.compose(int, cli.notNone)),
("tries", "-t", "--tries", "Number of download attempts to make", func.compose(int, cli.defaultIfNone("3"))),
(
"max_threads",
"",
"--max-threads",
"Maximum number of threads to download at once",
func.compose(int, cli.defaultIfNone("3")),
),
(
"continue_download",
"-c",
"--continue",
"Continue the download. Default is to download everything from scratch",
cli.defaultIfNone(False),
True,
#!/usr/bin/env python
import sys
import os
from igs.utils import commands
from igs.utils import logging
from igs.utils import functional as func
from igs.utils import cli
OPTIONS = [
('prefix', '-p', '--prefix', 'Prefix directory to install everything into',
func.compose(os.path.expanduser, os.path.expandvars, cli.notNone)),
('tmpdir', '', '--tmpdir', 'Temporary directory to use, defaults to $TMPDIR and then to /tmp',
func.compose(cli.defaultIfNone('/tmp'), cli.defaultIfNone(os.getenv('TMPDIR')))),
('debug', '-d', '--debug', 'Turn debugging output on', func.identity, cli.BINARY)
]
URLS = [
('ocaml', 'http://caml.inria.fr/pub/distrib/ocaml-3.12/ocaml-3.12.0.tar.gz'),
('findlib', 'http://download.camlcity.org/download/findlib-1.2.6.tar.gz'),
('ounit', 'http://www.xs4all.nl/~mmzeeman/ocaml/ounit-1.0.3.tar.gz'),
('res', 'http://hg.ocaml.info/release/res/archive/release-3.2.0.tar.gz'),
('pcre-ocaml', 'http://hg.ocaml.info/release/pcre-ocaml/archive/release-6.1.0.tar.gz'),
('type-conv', 'http://hg.ocaml.info/release/type-conv/archive/release-2.0.1.tar.gz'),
('fieldslib', 'http://www.janestreet.com/ocaml/fieldslib-0.1.0.tgz'),
('sexplib', 'http://www.janestreet.com/ocaml/sexplib310-release-4.2.15.tar.gz'),
('bin-prot', 'http://hg.ocaml.info/release/bin-prot/archive/release-1.2.23.tar.gz'),
('core', 'http://www.janestreet.com/ocaml/core-0.6.0.tgz'),
]
import signal
import Queue
import urlparse
import glob
from igs.utils import logging
from igs.utils import cli
from igs.utils import commands
from igs.utils import functional as func
from igs.threading import threads
OPTIONS = [
('base_dir', '-b', '--base-dir', 'Base directory to download into', cli.defaultIfNone('.')),
('join_name', '-j', '--join-name', 'If multiple files are downloaded, join them into the specified file name in the base directory and delete the other files. They are joined in the order they were specified.', func.identity),
('join_md5', '', '--join-md5', 'The md5sum of the joined file', func.identity),
('min_rate', '-m', '--min-rate', 'Minimum download rate in kilobytes per second', func.compose(int, cli.notNone)),
('tries', '-t', '--tries', 'Number of download attempts to make', func.compose(int, cli.defaultIfNone('3'))),
('max_threads', '', '--max-threads', 'Maximum number of threads to download at once', func.compose(int, cli.defaultIfNone('3'))),
('continue_download', '-c', '--continue', 'Continue the download. Default is to download everything from scratch', cli.defaultIfNone(False), True),
('debug', '-d', '--debug', 'Turn debug on', cli.defaultIfNone(False), True),
]
##
# How many seconds each sample should be
SAMPLE_RATE = 5
MAX_SAMPLE_SIZE = 10
def deleteFile(fname):
OPTIONS = [
('host', '', '--host',
'Host of web services to connect to, defaults to local host',
cli.defaultIfNone('localhost')),
('cluster', '', '--cluster',
'Name of cluster to run this transfer on, --src-cluster and --dst-cluster must be in terms of this clusters perspective',
cli.defaultIfNone('local')),
('tag_name', '', '--tag-name', 'Name of tag to upload', cli.notNone),
('src_cluster', '', '--src-cluster',
'Name of source cluster, hardcoded to local for now',
cli.defaultIfNone('local')),
('dst_cluster', '', '--dst-cluster', 'Name of dest cluster',
cli.defaultIfNone('local')),
('transfer_type', '', '--transfer-type',
'Type of transfer to do (cluster, s3) default is cluster',
func.compose(cli.restrictValues(['cluster', 's3']),
cli.defaultIfNone('cluster'))),
('block', '-b', '--block', 'Block until cluster is up (no longer used)',
func.identity, cli.BINARY),
('dst_type', '', '--dst-type',
'Set type of destination. (stage,local) default is stage',
cli.defaultIfNone('stage')),
('compress', '', '--compress', 'Compress files', func.identity,
cli.BINARY),
('compress_dir', '', '--compress-dir',
'Compress files into the specified directory', cli.defaultIfNone(None)),
('expand', '', '--expand',
'Expand files (always on regardless of this right now )', func.identity,
cli.BINARY),
('print_task_name', '-t',
'--print-task-name', 'Print the name of the task at the end',
cli.defaultIfNone(False), cli.BINARY),