def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
config = parseBasicOptions(parser)
cluster = Cluster(provisioner=config.provisioner,
clusterName=config.clusterName, zone=config.zone)
cluster.destroyCluster()
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
cluster = Cluster(provisioner=config.provisioner, clusterName=config.clusterName)
cluster.destroyCluster()
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
cluster = Cluster(provisioner=config.provisioner,
clusterName=config.clusterName)
cluster.destroyCluster()
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument('args', nargs=argparse.REMAINDER)
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
cluster = Cluster(provisioner=config.provisioner,
clusterName=config.clusterName)
cluster.sshCluster(args=config.args)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--insecure", dest='insecure', action='store_true', required=False,
help="Temporarily disable strict host key checking.")
parser.add_argument('args', nargs=argparse.REMAINDER)
config = parseBasicOptions(parser)
cluster = Cluster(provisioner=config.provisioner,
clusterName=config.clusterName, zone=config.zone)
cluster.sshCluster(args=config.args, strict=not config.insecure)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--nodeType", dest='nodeType', required=True,
help="Node type for {non-|}preemptable nodes. The syntax depends on the "
"provisioner used. For the aws provisioner this is the name of an "
"EC2 instance type followed by a colon and the price in dollar to "
"bid for a spot instance, for example 'c3.8xlarge:0.42'.")
parser.add_argument("--keyPairName", dest='keyPairName', required=True,
help="The name of the AWS key pair to include on the instance")
parser.add_argument("-t", "--tag", metavar='NAME=VALUE', dest='tags', default=[], action='append',
help="Tags are added to the AWS cluster for this node and all of its"
"children. Tags are of the form: "
" -t key1=value1 --tag key2=value2 "
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {"
" \"Name\": clusterName,"
" \"Owner\": IAM username"
" }. ")
parser.add_argument("--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not specified."
"This subnet needs to have auto assign IPs turned on.")
parser.add_argument("-w", "--workers", dest='workers', default=0, type=int,
help="Specify a number of workers to launch alongside the leader when the "
"cluster is created. This can be useful if running toil without "
"auto-scaling but with need of more hardware support")
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
tagsDict = None if config.tags is None else createTagsDict(config.tags)
spotBid = None
if config.provisioner == 'aws':
logger.info('Using aws provisioner.')
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
except ImportError:
raise RuntimeError('The aws extra must be installed to use this provisioner')
provisioner = AWSProvisioner()
parsedBid = config.nodeType.split(':', 1)
if len(config.nodeType) != len(parsedBid[0]):
# there is a bid
spotBid = float(parsedBid[1])
config.nodeType = parsedBid[0]
else:
assert False
provisioner.launchCluster(instanceType=config.nodeType,
keyName=config.keyPairName,
clusterName=config.clusterName,
workers=config.workers,
spotBid=spotBid,
userTags=tagsDict,
zone=config.zone,
vpcSubnet=config.vpcSubnet)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--insecure", dest='insecure', action='store_true', required=False,
help="Temporarily disable strict host key checking.")
parser.add_argument('args', nargs=argparse.REMAINDER)
config = parseBasicOptions(parser)
cluster = clusterFactory(provisioner=config.provisioner,
clusterName=config.clusterName,
zone=config.zone)
command = config.args if config.args else ['bash']
cluster.getLeader().sshAppliance(*command, strict=not config.insecure, tty=sys.stdin.isatty())
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--insecure", dest='insecure', action='store_true', required=False,
help="Temporarily disable strict host key checking.")
parser.add_argument("args", nargs=argparse.REMAINDER, help="Arguments to pass to"
"`rsync`. Takes any arguments that rsync accepts. Specify the"
" remote with a colon. For example, to upload `example.py`,"
" specify `toil rsync-cluster -p aws test-cluster example.py :`."
"\nOr, to download a file from the remote:, `toil rsync-cluster"
" -p aws test-cluster :example.py .`")
config = parseBasicOptions(parser)
cluster = Cluster(provisioner=config.provisioner, clusterName=config.clusterName, zone=config.zone)
cluster.rsyncCluster(args=config.args, strict=not config.insecure)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument(
"args",
nargs=argparse.REMAINDER,
help="Arguments to pass to"
"`rsync`. Takes any arguments that rsync accepts. Specify the"
" remote with a colon. For example, to upload `example.py`,"
" specify `toil rsync-cluster -p aws test-cluster example.py :`."
"\nOr, to download a file from the remote:, `toil rsync-cluster"
" -p aws test-cluster :example.py .`")
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
cluster = Cluster(provisioner=config.provisioner,
clusterName=config.clusterName,
zone=config.zone)
cluster.rsyncCluster(args=config.args)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--insecure",
action='store_true',
help="Temporarily disable strict host key checking.")
parser.add_argument("--sshOption",
dest='sshOptions',
default=[],
action='append',
help="Pass an additional option to the SSH command.")
parser.add_argument('args', nargs=argparse.REMAINDER)
config = parseBasicOptions(parser)
cluster = clusterFactory(provisioner=config.provisioner,
clusterName=config.clusterName,
zone=config.zone)
command = config.args if config.args else ['bash']
cluster.getLeader().sshAppliance(*command,
strict=not config.insecure,
tty=sys.stdin.isatty(),
sshOptions=config.sshOptions)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument(
"--nodeType",
dest='nodeType',
required=True,
help="Node type for {non-|}preemptable nodes. The syntax depends on the "
"provisioner used. For the aws provisioner this is the name of an "
"EC2 instance type followed by a colon and the price in dollar to "
"bid for a spot instance, for example 'c3.8xlarge:0.42'.")
parser.add_argument(
"--keyPairName",
dest='keyPairName',
required=True,
help="The name of the AWS key pair to include on the instance")
config = parseBasicOptions(parser)
setLoggingFromOptions(config)
spotBid = None
if config.provisioner == 'aws':
logger.info('Using aws provisioner.')
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
except ImportError:
raise RuntimeError(
'The aws extra must be installed to use this provisioner')
provisioner = AWSProvisioner
parsedBid = config.nodeType.split(':', 1)
if len(config.nodeType) != len(parsedBid[0]):
# there is a bid
spotBid = float(parsedBid[1])
config.nodeType = parsedBid[0]
else:
assert False
provisioner.launchCluster(instanceType=config.nodeType,
clusterName=config.clusterName,
keyName=config.keyPairName,
spotBid=spotBid)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument(
"--leaderNodeType",
dest="leaderNodeType",
required=True,
help="Non-preemptable node type to use for the cluster leader.")
parser.add_argument(
"--keyPairName",
dest='keyPairName',
help="On AWS, the name of the AWS key pair to include on the instance."
" On Google/GCE, this is the ssh key pair.")
parser.add_argument(
"--owner",
dest='owner',
help="The owner tag for all instances. If not given, the value in"
" --keyPairName will be used if given.")
parser.add_argument(
"--boto",
dest='botoPath',
help="The path to the boto credentials directory. This is transferred "
"to all nodes in order to access the AWS jobStore from non-AWS instances."
)
parser.add_argument(
"-t",
"--tag",
metavar='NAME=VALUE',
dest='tags',
default=[],
action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" -t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument(
"--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not "
"specified. This subnet needs to have auto assign IPs turned on.")
parser.add_argument(
"--nodeTypes",
dest='nodeTypes',
default=None,
type=str,
help="Comma-separated list of node types to create while launching the "
"leader. The syntax for each node type depends on the provisioner "
"used. For the aws provisioner this is the name of an EC2 instance "
"type followed by a colon and the price in dollar to bid for a spot "
"instance, for example 'c3.8xlarge:0.42'. Must also provide the "
"--workers argument to specify how many workers of each node type "
"to create.")
parser.add_argument(
"-w",
"--workers",
dest='workers',
default=None,
type=str,
help=
"Comma-separated list of the number of workers of each node type to "
"launch alongside the leader when the cluster is created. This can be "
"useful if running toil without auto-scaling but with need of more "
"hardware support")
parser.add_argument(
"--leaderStorage",
dest='leaderStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for the leader "
"instance. This is an EBS volume.")
parser.add_argument(
"--nodeStorage",
dest='nodeStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for any worker "
"instances created when using the -w flag. This is an EBS volume.")
parser.add_argument(
'--forceDockerAppliance',
dest='forceDockerAppliance',
action='store_true',
default=False,
help=
"Disables sanity checking the existence of the docker image specified "
"by TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for "
"autoscaling.")
parser.add_argument(
'--awsEc2ProfileArn',
dest='awsEc2ProfileArn',
default=None,
type=str,
help=
"If provided, the specified ARN is used as the instance profile for EC2 instances."
"Useful for setting custom IAM profiles. If not specified, a new IAM role is created "
"by default with sufficient access to perform basic cluster operations."
)
config = parseBasicOptions(parser)
tagsDict = None if config.tags is None else createTagsDict(config.tags)
checkValidNodeTypes(config.provisioner, config.nodeTypes)
checkValidNodeTypes(config.provisioner, config.leaderNodeType)
# checks the validity of TOIL_APPLIANCE_SELF before proceeding
applianceSelf(forceDockerAppliance=config.forceDockerAppliance)
spotBids = []
nodeTypes = []
preemptableNodeTypes = []
numNodes = []
numPreemptableNodes = []
if (config.nodeTypes
or config.workers) and not (config.nodeTypes and config.workers):
raise RuntimeError(
"The --nodeTypes and --workers options must be specified together,"
)
if config.nodeTypes:
nodeTypesList = config.nodeTypes.split(",")
numWorkersList = config.workers.split(",")
if not len(nodeTypesList) == len(numWorkersList):
raise RuntimeError(
"List of node types must be the same length as the list of workers."
)
for nodeTypeStr, num in zip(nodeTypesList, numWorkersList):
parsedBid = nodeTypeStr.split(':', 1)
if len(nodeTypeStr) != len(parsedBid[0]):
#Is a preemptable node
preemptableNodeTypes.append(parsedBid[0])
spotBids.append(float(parsedBid[1]))
numPreemptableNodes.append(int(num))
else:
nodeTypes.append(nodeTypeStr)
numNodes.append(int(num))
# set owner (default to keyPairName if not given)
owner = 'toil'
if config.owner:
owner = config.owner
elif config.keyPairName:
owner = config.keyPairName
# Check to see if the user specified a zone. If not, see if one is stored in an environment variable.
config.zone = config.zone or getZoneFromEnv(config.provisioner)
if not config.zone:
raise RuntimeError(
'Please provide a value for --zone or set a default in the TOIL_' +
config.provisioner.upper() + '_ZONE enviroment variable.')
cluster = clusterFactory(provisioner=config.provisioner,
clusterName=config.clusterName,
zone=config.zone,
nodeStorage=config.nodeStorage)
cluster.launchCluster(leaderNodeType=config.leaderNodeType,
leaderStorage=config.leaderStorage,
owner=owner,
keyName=config.keyPairName,
botoPath=config.botoPath,
userTags=tagsDict,
vpcSubnet=config.vpcSubnet,
awsEc2ProfileArn=config.awsEc2ProfileArn)
for nodeType, workers in zip(nodeTypes, numNodes):
cluster.addNodes(nodeType=nodeType,
numNodes=workers,
preemptable=False)
for nodeType, workers, spotBid in zip(preemptableNodeTypes,
numPreemptableNodes, spotBids):
cluster.addNodes(nodeType=nodeType,
numNodes=workers,
preemptable=True,
spotBid=spotBid)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--leaderNodeType", dest="leaderNodeType", required=True,
help="Non-preemptable node type to use for the cluster leader.")
parser.add_argument("--keyPairName", dest='keyPairName', required=True,
help="The name of the AWS or ssh key pair to include on the instance")
parser.add_argument("--boto", dest='botoPath',
help="The path to the boto credentials directory. This is transferred to all "
" nodes in order to access the AWS jobStore from non-AWS instances.")
parser.add_argument("-t", "--tag", metavar='NAME=VALUE', dest='tags', default=[], action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" -t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument("--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not specified. "
"This subnet needs to have auto assign IPs turned on.")
parser.add_argument("--nodeTypes", dest='nodeTypes', default=None, type=str,
help="Comma-separated list of node types to create while launching the leader. The "
"syntax for each node type depends on the "
"provisioner used. For the aws provisioner this is the name of an "
"EC2 instance type followed by a colon and the price in dollar to "
"bid for a spot instance, for example 'c3.8xlarge:0.42'. Must also provide "
"the --workers argument to specify how many workers of each node type to create")
parser.add_argument("-w", "--workers", dest='workers', default=None, type=str,
help="Comma-separated list of the number of workers of each node type to launch "
"alongside the leader when the "
"cluster is created. This can be useful if running toil without "
"auto-scaling but with need of more hardware support")
parser.add_argument("--leaderStorage", dest='leaderStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for the leader instance. "
"This is an EBS volume.")
parser.add_argument("--nodeStorage", dest='nodeStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for any worker instances "
"created when using the -w flag. This is an EBS volume.")
config = parseBasicOptions(parser)
tagsDict = None if config.tags is None else createTagsDict(config.tags)
spotBids = []
nodeTypes = []
preemptableNodeTypes = []
numNodes = []
numPreemptableNodes = []
leaderSpotBid = None
if config.provisioner == 'aws':
logger.info('Using aws provisioner.')
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
except ImportError:
logger.error('The aws extra must be installed to use this provisioner')
raise
provisioner = AWSProvisioner()
elif config.provisioner == 'gce':
logger.info('Using a gce provisioner.')
try:
from toil.provisioners.gceProvisioner import GCEProvisioner
except ImportError:
logger.error('The google extra must be installed to use this provisioner')
raise
provisioner = GCEProvisioner()
else:
assert False
#Parse leader node type and spot bid
parsedBid = config.leaderNodeType.split(':', 1)
if len(config.leaderNodeType) != len(parsedBid[0]):
leaderSpotBid = float(parsedBid[1])
config.leaderNodeType = parsedBid[0]
if (config.nodeTypes or config.workers) and not (config.nodeTypes and config.workers):
raise RuntimeError("The --nodeTypes and --workers options must be specified together,")
if config.nodeTypes:
nodeTypesList = config.nodeTypes.split(",")
numWorkersList = config.workers.split(",")
if not len(nodeTypesList) == len(numWorkersList):
raise RuntimeError("List of node types must be same length as list of numbers of workers.")
for nodeTypeStr, num in zip(nodeTypesList, numWorkersList):
parsedBid = nodeTypeStr.split(':', 1)
if len(nodeTypeStr) != len(parsedBid[0]):
#Is a preemptable node
preemptableNodeTypes.append(parsedBid[0])
spotBids.append(float(parsedBid[1]))
numPreemptableNodes.append(int(num))
else:
nodeTypes.append(nodeTypeStr)
numNodes.append(int(num))
provisioner.launchCluster(leaderNodeType=config.leaderNodeType,
leaderSpotBid=leaderSpotBid,
nodeTypes=nodeTypes,
preemptableNodeTypes=preemptableNodeTypes,
numWorkers=numNodes,
numPreemptableWorkers = numPreemptableNodes,
keyName=config.keyPairName,
botoPath=config.botoPath,
clusterName=config.clusterName,
spotBids=spotBids,
userTags=tagsDict,
zone=config.zone,
leaderStorage=config.leaderStorage,
nodeStorage=config.nodeStorage,
vpcSubnet=config.vpcSubnet)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument(
"--leaderNodeType",
dest="leaderNodeType",
required=True,
help="Non-preemptable node type to use for the cluster leader.")
parser.add_argument(
"--keyPairName",
dest='keyPairName',
required=True,
help="On AWS, the name of the AWS key pair to include on the instance."
" On Google/GCE, this is the ssh key pair."
" On Azure, this will be used as the owner tag.")
parser.add_argument(
"--publicKeyFile",
dest='publicKeyFile',
default="~/.ssh/id_rsa.pub",
help="On Azure, the file"
" containing the key pairs (the first key pair will be used).")
parser.add_argument(
"--boto",
dest='botoPath',
help="The path to the boto credentials directory. This is transferred "
"to all nodes in order to access the AWS jobStore from non-AWS instances."
)
parser.add_argument(
"-t",
"--tag",
metavar='NAME=VALUE',
dest='tags',
default=[],
action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" -t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument(
"--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not "
"specified. This subnet needs to have auto assign IPs turned on.")
parser.add_argument(
"--nodeTypes",
dest='nodeTypes',
default=None,
type=str,
help="Comma-separated list of node types to create while launching the "
"leader. The syntax for each node type depends on the provisioner "
"used. For the aws provisioner this is the name of an EC2 instance "
"type followed by a colon and the price in dollar to bid for a spot "
"instance, for example 'c3.8xlarge:0.42'. Must also provide the "
"--workers argument to specify how many workers of each node type "
"to create.")
parser.add_argument(
"-w",
"--workers",
dest='workers',
default=None,
type=str,
help=
"Comma-separated list of the number of workers of each node type to "
"launch alongside the leader when the cluster is created. This can be "
"useful if running toil without auto-scaling but with need of more "
"hardware support")
parser.add_argument(
"--leaderStorage",
dest='leaderStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for the leader "
"instance. This is an EBS volume.")
parser.add_argument(
"--nodeStorage",
dest='nodeStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for any worker "
"instances created when using the -w flag. This is an EBS volume.")
parser.add_argument(
'--forceDockerAppliance',
dest='forceDockerAppliance',
action='store_true',
default=False,
help=
"Disables sanity checking the existence of the docker image specified "
"by TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for "
"autoscaling.")
parser.add_argument(
"--azureStorageCredentials",
dest='azureStorageCredentials',
type=str,
default=credential_file_path,
help=
"The location of the file containing the Azure storage credentials. If not specified,"
" the default file is used with Azure provisioning. Use 'None' to disable"
" the transfer of credentials.")
config = parseBasicOptions(parser)
tagsDict = None if config.tags is None else createTagsDict(config.tags)
# checks the validity of TOIL_APPLIANCE_SELF before proceeding
checkToilApplianceSelf = applianceSelf(
forceDockerAppliance=config.forceDockerAppliance)
spotBids = []
nodeTypes = []
preemptableNodeTypes = []
numNodes = []
numPreemptableNodes = []
leaderSpotBid = None
if config.provisioner == 'aws':
logger.info('Using aws provisioner.')
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
except ImportError:
logger.error(
'The aws extra must be installed to use this provisioner')
raise
provisioner = AWSProvisioner()
elif config.provisioner == 'azure':
try:
from toil.provisioners.azure.azureProvisioner import AzureProvisioner
except ImportError:
raise RuntimeError(
'The aws extra must be installed to use this provisioner')
provisioner = AzureProvisioner()
elif config.provisioner == 'gce':
logger.info('Using a gce provisioner.')
try:
from toil.provisioners.gceProvisioner import GCEProvisioner
except ImportError:
logger.error(
'The google extra must be installed to use this provisioner')
raise
provisioner = GCEProvisioner()
else:
assert False
#Parse leader node type and spot bid
parsedBid = config.leaderNodeType.split(':', 1)
if len(config.leaderNodeType) != len(parsedBid[0]):
leaderSpotBid = float(parsedBid[1])
config.leaderNodeType = parsedBid[0]
if (config.nodeTypes
or config.workers) and not (config.nodeTypes and config.workers):
raise RuntimeError(
"The --nodeTypes and --workers options must be specified together,"
)
if config.nodeTypes:
nodeTypesList = config.nodeTypes.split(",")
numWorkersList = config.workers.split(",")
if not len(nodeTypesList) == len(numWorkersList):
raise RuntimeError(
"List of node types must be the same length as the list of workers."
)
for nodeTypeStr, num in zip(nodeTypesList, numWorkersList):
parsedBid = nodeTypeStr.split(':', 1)
if len(nodeTypeStr) != len(parsedBid[0]):
#Is a preemptable node
preemptableNodeTypes.append(parsedBid[0])
spotBids.append(float(parsedBid[1]))
numPreemptableNodes.append(int(num))
else:
nodeTypes.append(nodeTypeStr)
numNodes.append(int(num))
provisioner.launchCluster(
leaderNodeType=config.leaderNodeType,
leaderSpotBid=leaderSpotBid,
nodeTypes=nodeTypes,
preemptableNodeTypes=preemptableNodeTypes,
numWorkers=numNodes,
numPreemptableWorkers=numPreemptableNodes,
keyName=config.keyPairName,
botoPath=config.botoPath,
clusterName=config.clusterName,
spotBids=spotBids,
userTags=tagsDict,
zone=config.zone,
leaderStorage=config.leaderStorage,
nodeStorage=config.nodeStorage,
vpcSubnet=config.vpcSubnet,
publicKeyFile=config.publicKeyFile,
azureStorageCredentials=config.azureStorageCredentials)
def main():
parser = getBasicOptionParser()
parser = addBasicProvisionerOptions(parser)
parser.add_argument("--nodeType", dest='nodeType', required=True,
help="Node type for {non-|}preemptable nodes. The syntax depends on the "
"provisioner used. For the aws provisioner this is the name of an "
"EC2 instance type followed by a colon and the price in dollar to "
"bid for a spot instance, for example 'c3.8xlarge:0.42'.")
parser.add_argument("--keyPairName", dest='keyPairName', required=True,
help="The name of the AWS key pair to include on the instance")
parser.add_argument("-t", "--tag", metavar='NAME=VALUE', dest='tags', default=[], action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" --t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument("--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not specified. "
"This subnet needs to have auto assign IPs turned on.")
parser.add_argument("-w", "--workers", dest='workers', default=0, type=int,
help="Specify a number of workers to launch alongside the leader when the "
"cluster is created. This can be useful if running toil without "
"auto-scaling but with need of more hardware support")
parser.add_argument("--leaderStorage", dest='leaderStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for the leader instance. "
"This is an EBS volume.")
parser.add_argument("--nodeStorage", dest='nodeStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for any worker instances "
"created when using the -w flag. This is an EBS volume.")
config = parseBasicOptions(parser)
tagsDict = None if config.tags is None else createTagsDict(config.tags)
spotBid = None
if config.provisioner == 'aws':
logger.info('Using aws provisioner.')
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
except ImportError:
raise RuntimeError('The aws extra must be installed to use this provisioner')
provisioner = AWSProvisioner()
parsedBid = config.nodeType.split(':', 1)
if len(config.nodeType) != len(parsedBid[0]):
# there is a bid
spotBid = float(parsedBid[1])
config.nodeType = parsedBid[0]
else:
assert False
provisioner.launchCluster(instanceType=config.nodeType,
keyName=config.keyPairName,
clusterName=config.clusterName,
workers=config.workers,
spotBid=spotBid,
userTags=tagsDict,
zone=config.zone,
leaderStorage=config.leaderStorage,
nodeStorage=config.nodeStorage,
vpcSubnet=config.vpcSubnet)
