def _getScript(self):
def userScript():
from toil.common import Toil
from toil.job import Job
# Because this is the only job in the pipeline and because it is preemptable,
# there will be no non-preemptable jobs. The non-preemptable scaler will therefore
# not request any nodes initially. And since we made it impossible for the
# preemptable scaler to allocate any nodes (using an abnormally low spot bid),
# we will observe a deficit of preemptable nodes that the non-preemptable scaler will
# compensate for by spinning up non-preemptable nodes instead.
#
def job(job, disk='10M', cores=1, memory='10M', preemptable=True):
pass
if __name__ == '__main__':
options = Job.Runner.getDefaultArgumentParser().parse_args()
with Toil(options) as toil:
if toil.config.restart:
toil.restart()
else:
toil.start(Job.wrapJobFn(job))
script = dedent('\n'.join(getsource(userScript).split('\n')[1:]))
# use appliance ssh method instead of sshutil so we can specify input param
cluster = cluster_factory(provisioner='aws',
clusterName=self.clusterName)
leader = cluster.getLeader()
leader.sshAppliance('tee', '/home/userScript.py', input=script)
def launchCluster(self):
from boto.ec2.blockdevicemapping import BlockDeviceType
from toil.lib.ec2 import wait_instances_running
self.createClusterUtil(args=[
'--leaderStorage',
str(self.requestedLeaderStorage), '--nodeTypes', ",".join(
self.instanceTypes), '-w', ",".join(self.numWorkers),
'--nodeStorage',
str(self.requestedLeaderStorage)
])
self.cluster = cluster_factory(provisioner='aws',
zone=self.zone,
clusterName=self.clusterName)
nodes = self.cluster._getNodesInCluster(both=True)
nodes.sort(key=lambda x: x.launch_time)
# assuming that leader is first
workers = nodes[1:]
# test that two worker nodes were created
self.assertEqual(2, len(workers))
# test that workers have expected storage size
# just use the first worker
worker = workers[0]
worker = next(wait_instances_running(self.cluster._boto2.ec2,
[worker]))
rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
rootVolume = self.cluster._boto2.ec2.get_all_volumes(
volume_ids=[rootBlockDevice.volume_id])[0]
self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
def _getScript(self):
def restartScript():
import argparse
import os
from toil.job import Job
def f0(job):
if 'FAIL' in os.environ:
raise RuntimeError('failed on purpose')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args()
rootJob = Job.wrapJobFn(f0,
cores=0.5,
memory='50 M',
disk='50 M')
Job.Runner.startToil(rootJob, options)
script = dedent('\n'.join(getsource(restartScript).split('\n')[1:]))
tempfile_path = '/tmp/temp-or-ary.txt'
with open(tempfile_path, 'w') as f:
# use appliance ssh method instead of sshutil so we can specify input param
f.write(script)
cluster = cluster_factory(provisioner='aws',
clusterName=self.clusterName)
leader = cluster.getLeader()
self.sshUtil([
'mkdir', '-p', self.scriptDir
]) # hot deploy doesn't seem permitted to work in normal /tmp or /home
leader.injectFile(tempfile_path, self.scriptName, 'toil_leader')
if os.path.exists(tempfile_path):
os.remove(tempfile_path)
def main():
parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
options = parser.parse_args()
set_logging_from_options(options)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
cluster.destroyCluster()
def main() -> None:
parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
options = parser.parse_args()
set_logging_from_options(options)
logger.info('Destroying cluster %s', options.clusterName)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
cluster.destroyCluster()
logger.info('Cluster %s is now gone.', options.clusterName)
def launchCluster(self):
from boto.ec2.blockdevicemapping import BlockDeviceType
from toil.lib.ec2 import wait_instances_running
self.createClusterUtil(args=[
'--leaderStorage',
str(self.requestedLeaderStorage), '--nodeTypes', ",".join(
self.instanceTypes), '--workers', ",".join(
[f'0-{c}' for c in self.numWorkers]), '--nodeStorage',
str(self.requestedLeaderStorage), '--clusterType', 'kubernetes'
])
self.cluster = cluster_factory(provisioner='aws',
zone=self.zone,
clusterName=self.clusterName)
def putScript(self, content: str):
"""
Helper method for _getScript to inject a script file at the configured script path, from text.
"""
cluster = cluster_factory(provisioner='aws',
zone=self.zone,
clusterName=self.clusterName)
leader = cluster.getLeader()
self.sshUtil(['mkdir', '-p', self.scriptDir])
with tempfile.NamedTemporaryFile(mode='w') as t:
# use appliance ssh method instead of sshutil so we can specify input param
t.write(content)
# This works to make writes visible on non-Windows
t.flush()
leader.injectFile(t.name, self.script(), 'toil_leader')
def main() -> None:
parser = parser_with_common_options(provisioner_options=True,
jobstore_option=False)
parser.add_argument("--insecure",
action='store_true',
help="Temporarily disable strict host key checking.")
parser.add_argument("--sshOption",
dest='sshOptions',
default=[],
action='append',
help="Pass an additional option to the SSH command.")
parser.add_argument(
"--grafana_port",
dest='grafana_port',
default=3000,
help="Assign a local port to be used for the Grafana dashboard.")
parser.add_argument('args', nargs=argparse.REMAINDER)
options = parser.parse_args()
set_logging_from_options(options)
# Since we collect all the remaining arguments at the end for a command to
# run, it's easy to lose options.
if len(options.args) > 0 and options.args[0].startswith('-'):
logger.warning(
'Argument \'%s\' interpreted as a command to run '
'despite looking like an option.', options.args[0])
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
command = options.args if options.args else ['bash']
sshOptions: List[str] = options.sshOptions
# Forward ports:
# 3000 for Grafana dashboard
# 9090 for Prometheus dashboard
sshOptions.extend([
'-L', f'{options.grafana_port}:localhost:3000', '-L',
'9090:localhost:9090'
])
cluster.getLeader().sshAppliance(*command,
strict=not options.insecure,
tty=sys.stdin.isatty(),
sshOptions=sshOptions)
def main():
parser = parser_with_common_options(provisioner_options=True,
jobstore_option=False)
parser.add_argument("--insecure",
action='store_true',
help="Temporarily disable strict host key checking.")
parser.add_argument("--sshOption",
dest='sshOptions',
default=[],
action='append',
help="Pass an additional option to the SSH command.")
parser.add_argument('args', nargs=argparse.REMAINDER)
options = parser.parse_args()
set_logging_from_options(options)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
command = options.args if options.args else ['bash']
cluster.getLeader().sshAppliance(*command,
strict=not options.insecure,
tty=sys.stdin.isatty(),
sshOptions=options.sshOptions)
def main():
parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
parser.add_argument("--insecure", action='store_true',
help="Temporarily disable strict host key checking.")
parser.add_argument("--sshOption", dest='sshOptions', default=[], action='append',
help="Pass an additional option to the SSH command.")
parser.add_argument('args', nargs=argparse.REMAINDER)
options = parser.parse_args()
set_logging_from_options(options)
# Since we collect all the remaining arguments at the end for a command to
# run, it's easy to lose options.
if len(options.args) > 0 and options.args[0].startswith('-'):
logger.warning('Argument \'%s\' interpreted as a command to run '
'despite looking like an option.', options.args[0])
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
command = options.args if options.args else ['bash']
cluster.getLeader().sshAppliance(*command, strict=not options.insecure, tty=sys.stdin.isatty(),
sshOptions=options.sshOptions)
def main() -> None:
parser = parser_with_common_options(provisioner_options=True,
jobstore_option=False)
parser.add_argument("--insecure",
dest='insecure',
action='store_true',
required=False,
help="Temporarily disable strict host key checking.")
parser.add_argument(
"args",
nargs=argparse.REMAINDER,
help="Arguments to pass to"
"`rsync`. Takes any arguments that rsync accepts. Specify the"
" remote with a colon. For example, to upload `example.py`,"
" specify `toil rsync-cluster -p aws test-cluster example.py :`."
"\nOr, to download a file from the remote:, `toil rsync-cluster"
" -p aws test-cluster :example.py .`")
options = parser.parse_args()
set_logging_from_options(options)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone)
cluster.getLeader().coreRsync(args=options.args,
strict=not options.insecure)
def main():
parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
parser.add_argument("-T", "--clusterType", dest="clusterType",
choices=['mesos', 'kubernetes'], default='mesos',
help="Cluster scheduler to use.")
parser.add_argument("--leaderNodeType", dest="leaderNodeType", required=True,
help="Non-preemptable node type to use for the cluster leader.")
parser.add_argument("--keyPairName", dest='keyPairName',
help="On AWS, the name of the AWS key pair to include on the instance."
" On Google/GCE, this is the ssh key pair.")
parser.add_argument("--owner", dest='owner',
help="The owner tag for all instances. If not given, the value in"
" --keyPairName will be used if given.")
parser.add_argument("--boto", dest='botoPath',
help="The path to the boto credentials directory. This is transferred "
"to all nodes in order to access the AWS jobStore from non-AWS instances.")
parser.add_argument("-t", "--tag", metavar='NAME=VALUE', dest='tags',
default=[], action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" -t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument("--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not "
"specified. This subnet needs to have auto assign IPs turned on.")
parser.add_argument("--nodeTypes", dest='nodeTypes', default=None, type=str,
help="Specifies a list of comma-separated node types, each of which is "
"composed of slash-separated instance types, and an optional spot "
"bid set off by a colon, making the node type preemptable. Instance "
"types may appear in multiple node types, and the same node type "
"may appear as both preemptable and non-preemptable.\n"
"Valid argument specifying two node types:\n"
"\tc5.4xlarge/c5a.4xlarge:0.42,t2.large\n"
"Node types:\n"
"\tc5.4xlarge/c5a.4xlarge:0.42 and t2.large\n"
"Instance types:\n"
"\tc5.4xlarge, c5a.4xlarge, and t2.large\n"
"Semantics:\n"
"\tBid $0.42/hour for either c5.4xlarge or c5a.4xlarge instances,\n"
"\ttreated interchangeably, while they are available at that price,\n"
"\tand buy t2.large instances at full price\n"
"Must also provide the --workers argument to specify how many "
"workers of each node type to create.")
parser.add_argument("-w", "--workers", dest='workers', default=None, type=str,
help="Comma-separated list of the ranges of numbers of workers of each "
"node type to launch, such as '0-2,5,1-3'. If a range is given, "
"workers will automatically be launched and terminated by the cluster "
"to auto-scale to the workload.")
parser.add_argument("--leaderStorage", dest='leaderStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for the leader "
"instance. This is an EBS volume.")
parser.add_argument("--nodeStorage", dest='nodeStorage', type=int, default=50,
help="Specify the size (in gigabytes) of the root volume for any worker "
"instances created when using the -w flag. This is an EBS volume.")
parser.add_argument('--forceDockerAppliance', dest='forceDockerAppliance', action='store_true',
default=False,
help="Disables sanity checking the existence of the docker image specified "
"by TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for "
"autoscaling.")
parser.add_argument('--awsEc2ProfileArn', dest='awsEc2ProfileArn', default=None, type=str,
help="If provided, the specified ARN is used as the instance profile for EC2 instances."
"Useful for setting custom IAM profiles. If not specified, a new IAM role is created "
"by default with sufficient access to perform basic cluster operations.")
parser.add_argument('--awsEc2ExtraSecurityGroupId', dest='awsEc2ExtraSecurityGroupIds', default=[], action='append',
help="Any additional security groups to attach to EC2 instances. Note that a security group "
"with its name equal to the cluster name will always be created, thus ensure that "
"the extra security groups do not have the same name as the cluster name.")
options = parser.parse_args()
set_logging_from_options(options)
tags = create_tags_dict(options.tags) if options.tags else dict()
# Get worker node types
worker_node_types = parse_node_types(options.nodeTypes)
check_valid_node_types(options.provisioner, worker_node_types + [({options.leaderNodeType}, None)])
# Holds string ranges, like "5", or "3-10"
worker_node_ranges = options.workers.split(',') if options.workers else []
# checks the validity of TOIL_APPLIANCE_SELF before proceeding
applianceSelf(forceDockerAppliance=options.forceDockerAppliance)
# This holds either ints to launch static nodes, or tuples of ints
# specifying ranges to launch managed auto-scaling nodes, for each type.
nodeCounts = []
if ((worker_node_types != [] or worker_node_ranges != []) and not
(worker_node_types != [] and worker_node_ranges != [])):
raise RuntimeError("The --nodeTypes option requires --workers, and visa versa.")
if worker_node_types and worker_node_ranges:
if not len(worker_node_types) == len(worker_node_ranges):
raise RuntimeError("List of worker count ranges must be the same length as the list of node types.")
for spec in worker_node_ranges:
if '-' in spec:
# Provision via autoscaling
parts = spec.split('-')
if len(parts) != 2:
raise RuntimeError("Unacceptable range: " + spec)
nodeCounts.append((int(parts[0]), int(parts[1])))
else:
# Provision fixed nodes
nodeCounts.append(int(spec))
owner = options.owner or options.keyPairName or 'toil'
# Check to see if the user specified a zone. If not, see if one is stored in an environment variable.
options.zone = options.zone or os.environ.get(f'TOIL_{options.provisioner.upper()}_ZONE')
if not options.zone:
raise RuntimeError(f'Please provide a value for --zone or set a default in the '
f'TOIL_{options.provisioner.upper()}_ZONE environment variable.')
logger.info('Creating cluster %s...', options.clusterName)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
clusterType=options.clusterType,
zone=options.zone,
nodeStorage=options.nodeStorage)
cluster.launchCluster(leaderNodeType=options.leaderNodeType,
leaderStorage=options.leaderStorage,
owner=owner,
keyName=options.keyPairName,
botoPath=options.botoPath,
userTags=tags,
vpcSubnet=options.vpcSubnet,
awsEc2ProfileArn=options.awsEc2ProfileArn,
awsEc2ExtraSecurityGroupIds=options.awsEc2ExtraSecurityGroupIds)
for typeNum, spec in enumerate(nodeCounts):
# For each batch of workers to make
wanted = worker_node_types[typeNum]
if isinstance(spec, int):
# Make static nodes
if spec == 0:
# Don't make anything
continue
if wanted[1] is None:
# Make non-spot instances
cluster.addNodes(nodeTypes=wanted[0], numNodes=spec, preemptable=False)
else:
# We have a spot bid
cluster.addNodes(nodeTypes=wanted[0], numNodes=spec, preemptable=True,
spotBid=wanted[1])
elif isinstance(spec, tuple):
# Make a range of auto-scaling nodes
max_count, min_count = spec
if max_count < min_count:
# Flip them around
min_count, max_count = max_count, min_count
if max_count == 0:
# Don't want any
continue
if wanted[1] is None:
# Make non-spot instances
cluster.addManagedNodes(nodeTypes=wanted[0], minNodes=min_count, maxNodes=max_count,
preemptable=False)
else:
# Bid at the given price.
cluster.addManagedNodes(nodeTypes=wanted[0], minNodes=min_count, maxNodes=max_count,
preemptable=True, spotBid=wanted[1])
logger.info('Cluster created successfully.')
def _test(self, preemptableJobs=False):
"""Does the work of the testing. Many features' tests are thrown in here in no particular order."""
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.cluster = cluster_factory(provisioner='aws',
zone=self.zone,
clusterName=self.clusterName)
self.leader = self.cluster.getLeader()
self.sshUtil(['mkdir', '-p', self.scriptDir])
self.sshUtil(['mkdir', '-p', self.dataDir])
assert len(self.getMatchingRoles()) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--python', exactPython,
'--never-download', self.venvDir
]
self.sshUtil(venv_command)
upgrade_command = [
self.pip(), 'install', 'setuptools==28.7.1', 'pyyaml==3.12'
]
self.sshUtil(upgrade_command)
log.info('Set up script...')
self._getScript()
toilOptions = [
self.jobStore, '--workDir=/var/lib/toil', '--clean=always',
'--retryCount=2', '--logDebug',
'--logFile=' + os.path.join(self.scriptDir, 'sort.log')
]
if preemptableJobs:
toilOptions.extend(['--defaultPreemptable'])
log.info('Run script...')
self._runScript(toilOptions)
assert len(self.getMatchingRoles()) == 1
from boto.exception import EC2ResponseError
volumeID = self.getRootVolID()
self.cluster.destroyCluster()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
self.cluster._boto2.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles()) == 0
def _test(self, preemptableJobs=False):
"""Does the work of the testing. Many features' tests are thrown in here in no particular order."""
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.cluster = cluster_factory(provisioner='aws',
clusterName=self.clusterName)
self.leader = self.cluster.getLeader()
self.sshUtil([
'mkdir', '-p', self.scriptDir
]) # hot deploy doesn't seem permitted to work in normal /tmp or /home
assert len(self.getMatchingRoles()) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--python', exactPython,
'--never-download', '/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1',
'pyyaml==3.12'
]
self.sshUtil(upgrade_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/tmp/t/',
'--logDebug', '--logFile=/tmp/t/sort.log', '--provisioner=aws'
]
toilOptions.extend([
'--nodeTypes=' + ",".join(self.instanceTypes),
'--maxNodes=' + ",".join(self.numWorkers)
])
if preemptableJobs:
toilOptions.extend(['--defaultPreemptable'])
self._runScript(toilOptions)
assert len(self.getMatchingRoles()) == 1
# check stats
self.sshUtil([
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/tmp/t/") if f.endswith(".json")].pop()))'
])
from boto.exception import EC2ResponseError
volumeID = self.getRootVolID()
self.cluster.destroyCluster()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
self.cluster._ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles()) == 0
def main():
parser = parser_with_common_options(provisioner_options=True,
jobstore_option=False)
parser.add_argument(
"--leaderNodeType",
dest="leaderNodeType",
required=True,
help="Non-preemptable node type to use for the cluster leader.")
parser.add_argument(
"--keyPairName",
dest='keyPairName',
help="On AWS, the name of the AWS key pair to include on the instance."
" On Google/GCE, this is the ssh key pair.")
parser.add_argument(
"--owner",
dest='owner',
help="The owner tag for all instances. If not given, the value in"
" --keyPairName will be used if given.")
parser.add_argument(
"--boto",
dest='botoPath',
help="The path to the boto credentials directory. This is transferred "
"to all nodes in order to access the AWS jobStore from non-AWS instances."
)
parser.add_argument(
"-t",
"--tag",
metavar='NAME=VALUE',
dest='tags',
default=[],
action='append',
help="Tags are added to the AWS cluster for this node and all of its "
"children. Tags are of the form:\n"
" -t key1=value1 --tag key2=value2\n"
"Multiple tags are allowed and each tag needs its own flag. By "
"default the cluster is tagged with "
" {\n"
" \"Name\": clusterName,\n"
" \"Owner\": IAM username\n"
" }. ")
parser.add_argument(
"--vpcSubnet",
help="VPC subnet ID to launch cluster in. Uses default subnet if not "
"specified. This subnet needs to have auto assign IPs turned on.")
parser.add_argument(
"--nodeTypes",
dest='nodeTypes',
default=None,
type=str,
help="Comma-separated list of node types to create while launching the "
"leader. The syntax for each node type depends on the provisioner "
"used. For the aws provisioner this is the name of an EC2 instance "
"type followed by a colon and the price in dollar to bid for a spot "
"instance, for example 'c3.8xlarge:0.42'. Must also provide the "
"--workers argument to specify how many workers of each node type "
"to create.")
parser.add_argument(
"-w",
"--workers",
dest='workers',
default=None,
type=str,
help=
"Comma-separated list of the number of workers of each node type to "
"launch alongside the leader when the cluster is created. This can be "
"useful if running toil without auto-scaling but with need of more "
"hardware support")
parser.add_argument(
"--leaderStorage",
dest='leaderStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for the leader "
"instance. This is an EBS volume.")
parser.add_argument(
"--nodeStorage",
dest='nodeStorage',
type=int,
default=50,
help="Specify the size (in gigabytes) of the root volume for any worker "
"instances created when using the -w flag. This is an EBS volume.")
parser.add_argument(
'--forceDockerAppliance',
dest='forceDockerAppliance',
action='store_true',
default=False,
help=
"Disables sanity checking the existence of the docker image specified "
"by TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for "
"autoscaling.")
parser.add_argument(
'--awsEc2ProfileArn',
dest='awsEc2ProfileArn',
default=None,
type=str,
help=
"If provided, the specified ARN is used as the instance profile for EC2 instances."
"Useful for setting custom IAM profiles. If not specified, a new IAM role is created "
"by default with sufficient access to perform basic cluster operations."
)
parser.add_argument(
'--awsEc2ExtraSecurityGroupId',
dest='awsEc2ExtraSecurityGroupIds',
default=[],
action='append',
help=
"Any additional security groups to attach to EC2 instances. Note that a security group "
"with its name equal to the cluster name will always be created, thus ensure that "
"the extra security groups do not have the same name as the cluster name."
)
options = parser.parse_args()
set_logging_from_options(options)
tags = create_tags_dict(options.tags) if options.tags else dict()
worker_node_types = options.nodeTypes.split(
',') if options.nodeTypes else []
worker_quantities = options.workers.split(',') if options.workers else []
check_valid_node_types(options.provisioner,
worker_node_types + [options.leaderNodeType])
# checks the validity of TOIL_APPLIANCE_SELF before proceeding
applianceSelf(forceDockerAppliance=options.forceDockerAppliance)
owner = options.owner or options.keyPairName or 'toil'
# Check to see if the user specified a zone. If not, see if one is stored in an environment variable.
options.zone = options.zone or os.environ.get(
f'TOIL_{options.provisioner.upper()}_ZONE')
if not options.zone:
raise RuntimeError(
f'Please provide a value for --zone or set a default in the '
f'TOIL_{options.provisioner.upper()}_ZONE environment variable.')
if (options.nodeTypes or
options.workers) and not (options.nodeTypes and options.workers):
raise RuntimeError(
"The --nodeTypes and --workers options must be specified together."
)
if not len(worker_node_types) == len(worker_quantities):
raise RuntimeError(
"List of node types must be the same length as the list of workers."
)
cluster = cluster_factory(provisioner=options.provisioner,
clusterName=options.clusterName,
zone=options.zone,
nodeStorage=options.nodeStorage)
cluster.launchCluster(
leaderNodeType=options.leaderNodeType,
leaderStorage=options.leaderStorage,
owner=owner,
keyName=options.keyPairName,
botoPath=options.botoPath,
userTags=tags,
vpcSubnet=options.vpcSubnet,
awsEc2ProfileArn=options.awsEc2ProfileArn,
awsEc2ExtraSecurityGroupIds=options.awsEc2ExtraSecurityGroupIds)
for worker_node_type, num_workers in zip(worker_node_types,
worker_quantities):
if ':' in worker_node_type:
worker_node_type, bid = worker_node_type.split(':', 1)
cluster.addNodes(nodeType=worker_node_type,
numNodes=int(num_workers),
preemptable=True,
spotBid=float(bid))
else:
cluster.addNodes(nodeType=worker_node_type,
numNodes=int(num_workers),
preemptable=False)
