def addBasicProvisionerOptions(parser):
parser.add_argument("--version", action='version', version=version)
parser.add_argument(
'-p',
"--provisioner",
dest='provisioner',
choices=['aws', 'azure', 'gce'],
required=False,
default="aws",
help="The provisioner for cluster auto-scaling. Only aws is currently "
"supported")
try:
from toil.provisioners.aws import getCurrentAWSZone
currentZone = getCurrentAWSZone()
except ImportError:
currentZone = None
zoneString = currentZone if currentZone else 'No zone could be determined'
parser.add_argument(
'-z',
'--zone',
dest='zone',
required=False,
default=currentZone,
help=
"The AWS availability zone of the master. This parameter can also be "
"set via the TOIL_AWS_ZONE environment variable, or by the ec2_region_name "
"parameter in your .boto file, or derived from the instance metadata if "
"using this utility on an existing EC2 instance. "
"Currently: %s" % zoneString)
parser.add_argument(
"clusterName",
help="The name that the cluster will be identifiable by. "
"Must be lowercase and may not contain the '_' "
"character.")
return parser
def _buildContext(self):
if self._zone is None:
self._zone = getCurrentAWSZone()
if self._zone is None:
raise RuntimeError(
'Could not determine availability zone. Ensure that one of the following '
'is true: the --zone flag is set, the TOIL_AWS_ZONE environment variable '
'is set, ec2_region_name is set in the .boto file, or that '
'you are running on EC2.')
logger.debug("Building AWS context in zone %s for cluster %s" % (self._zone, self.clusterName))
self._ctx = Context(availability_zone=self._zone, namespace=self._toNameSpace())
def checkValidNodeTypes(provisioner, nodeTypes):
"""
Raises if an invalid nodeType is specified for aws or gce.
:param str provisioner: 'aws' or 'gce' to specify which cloud provisioner used.
:param nodeTypes: A list of node types. Example: ['t2.micro', 't2.medium']
:return: Nothing. Raises if invalid nodeType.
"""
# TODO: Move out of "aws.__init__.py" >.>
if not nodeTypes:
return
if not isinstance(nodeTypes, list):
nodeTypes = [nodeTypes]
if not isinstance(nodeTypes[0], string_types):
return
# check if a valid node type for aws
from toil.lib.generatedEC2Lists import E2Instances, regionDict
if provisioner == 'aws':
from toil.provisioners.aws import getCurrentAWSZone
currentZone = getCurrentAWSZone()
if not currentZone:
currentZone = 'us-west-2'
else:
currentZone = currentZone[:
-1] # adds something like 'a' or 'b' to the end
# check if instance type exists in this region
for nodeType in nodeTypes:
if nodeType and ':' in nodeType:
nodeType = nodeType.split(':')[0]
if nodeType not in regionDict[currentZone]:
# They probably misspelled it and can't tell.
close = get_close_matches(nodeType, regionDict[currentZone], 1)
if len(close) > 0:
helpText = ' Did you mean ' + close[0] + '?'
else:
helpText = ''
raise RuntimeError(
'Invalid nodeType (%s) specified for AWS in region: %s.%s'
'' % (nodeType, currentZone, helpText))
elif provisioner == 'gce':
for nodeType in nodeTypes:
if nodeType and ':' in nodeType:
nodeType = nodeType.split(':')[0]
try:
E2Instances[nodeType]
raise RuntimeError(
"It looks like you've specified an AWS nodeType with the "
"{} provisioner. Please specify an {} nodeType."
"".format(provisioner, provisioner))
except KeyError:
pass
else:
raise RuntimeError("Invalid provisioner: {}".format(provisioner))
def __init__(self, clusterName, zone, nodeStorage, nodeStorageOverrides, sseKey):
super(AWSProvisioner, self).__init__(clusterName, zone, nodeStorage, nodeStorageOverrides)
self.cloud = 'aws'
self._sseKey = sseKey
self._zone = zone if zone else getCurrentAWSZone()
# establish boto3 clients
self.session = boto3.Session(region_name=zoneToRegion(zone))
self.ec2 = self.session.resource('ec2')
if clusterName:
self._buildContext() # create connection (self._ctx)
else:
self._readClusterSettings()
def addBasicProvisionerOptions(parser):
parser.add_argument("--version", action='version', version=version)
parser.add_argument('-p', "--provisioner", dest='provisioner', choices=['aws'], required=True,
help="The provisioner for cluster auto-scaling. Only aws is currently "
"supported")
currentZone = getCurrentAWSZone()
zoneString = currentZone if currentZone else 'No zone could be determined'
parser.add_argument('-z', '--zone', dest='zone', required=False, default=currentZone,
help="The AWS availability zone of the master. This parameter can also be "
"set via the TOIL_AWS_ZONE environment variable, or by the ec2_region_name "
"parameter in your .boto file, or derived from the instance metadata if "
"using this utility on an existing EC2 instance. "
"Currently: %s" % zoneString)
parser.add_argument("clusterName", help="The name that the cluster will be identifiable by. "
"Must be lowercase and may not contain the '_' "
"character.")
return parser
def checkValidNodeTypes(provisioner, nodeTypes):
"""
Raises if an invalid nodeType is specified for aws, azure, or gce.
:param str provisioner: 'aws', 'gce', or 'azure' to specify which cloud provisioner used.
:param nodeTypes: A list of node types. Example: ['t2.micro', 't2.medium']
:return: Nothing. Raises if invalid nodeType.
"""
if not nodeTypes:
return
if not isinstance(nodeTypes, list):
nodeTypes = [nodeTypes]
if not isinstance(nodeTypes[0], string_types):
return
# check if a valid node type for aws
from toil.lib.generatedEC2Lists import E2Instances, regionDict
if provisioner == 'aws':
from toil.provisioners.aws import getCurrentAWSZone
currentZone = getCurrentAWSZone()
if not currentZone:
currentZone = 'us-west-2'
else:
currentZone = currentZone[:
-1] # adds something like 'a' or 'b' to the end
# check if instance type exists in this region
for nodeType in nodeTypes:
if nodeType and ':' in nodeType:
nodeType = nodeType.split(':')[0]
if nodeType not in regionDict[currentZone]:
raise RuntimeError(
'Invalid nodeType (%s) specified for AWS in region: %s.'
'' % (nodeType, currentZone))
# Only checks if aws nodeType specified for gce/azure atm.
if provisioner == 'gce' or provisioner == 'azure':
for nodeType in nodeTypes:
if nodeType and ':' in nodeType:
nodeType = nodeType.split(':')[0]
try:
E2Instances[nodeType]
raise RuntimeError(
"It looks like you've specified an AWS nodeType with the "
"{} provisioner. Please specify an {} nodeType."
"".format(provisioner, provisioner))
except KeyError:
pass
