def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, zone=None):
ctx = cls._buildContext(clusterName=clusterName, zone=zone)
profileARN = cls._getProfileARN(ctx)
# the security group name is used as the cluster identifier
cls._createSecurityGroup(ctx, clusterName)
bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType])
leaderData = dict(role='leader',
image=applianceSelf(),
entrypoint='mesos-master',
args=leaderArgs.format(name=clusterName))
userData = awsUserData.format(**leaderData)
kwargs = {'key_name': keyName, 'security_groups': [clusterName],
'instance_type': instanceType,
'user_data': userData, 'block_device_map': bdm,
'instance_profile_arn': profileARN}
if not spotBid:
logger.info('Launching non-preemptable leader')
create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching preemptable leader')
# force generator to evaluate
list(create_spot_instances(ec2=ctx.ec2,
price=spotBid,
image_id=cls._discoverAMI(ctx),
tags={'clusterName': clusterName},
spec=kwargs,
num_instances=1))
return cls._getLeader(clusterName=clusterName, wait=True)
def _addNodes(self, instances, numNodes, preemptable=False):
bdm = self._getBlockDeviceMapping(self.instanceType)
arn = self._getProfileARN(self.ctx)
workerData = dict(role='worker',
image=applianceSelf(),
entrypoint='mesos-slave',
args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable))
userData = awsUserData.format(**workerData)
kwargs = {'key_name': self.keyName, 'security_groups': [self.clusterName],
'instance_type': self.instanceType.name,
'user_data': userData, 'block_device_map': bdm,
'instance_profile_arn': arn}
instancesLaunched = []
if not preemptable:
logger.info('Launching %s non-preemptable nodes', numNodes)
instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching %s preemptable nodes', numNodes)
# force generator to evaluate
instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2,
price=self.spotBid,
image_id=self._discoverAMI(self.ctx),
tags={'clusterName': self.clusterName},
spec=kwargs,
num_instances=numNodes))
wait_instances_running(self.ctx.ec2, instancesLaunched)
logger.info('Launched %s new instance(s)', numNodes)
return len(instancesLaunched)
def launchCluster(self, instanceType, keyName, clusterName, workers=0,
spotBid=None, userTags=None, zone=None, vpcSubnet=None):
if userTags is None:
userTags = {}
ctx = self._buildContext(clusterName=clusterName, zone=zone)
profileARN = self._getProfileARN(ctx)
# the security group name is used as the cluster identifier
sgs = self._createSecurityGroup(ctx, clusterName, vpcSubnet)
bdm = self._getBlockDeviceMapping(ec2_instance_types[instanceType])
self.masterPublicKey = 'AAAAB3NzaC1yc2Enoauthorizedkeyneeded'
leaderData = dict(role='leader',
image=applianceSelf(),
entrypoint='mesos-master',
sshKey=self.masterPublicKey,
args=leaderArgs.format(name=clusterName))
userData = awsUserData.format(**leaderData)
kwargs = {'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs],
'instance_type': instanceType,
'user_data': userData, 'block_device_map': bdm,
'instance_profile_arn': profileARN,
'placement': zone}
if vpcSubnet:
kwargs["subnet_id"] = vpcSubnet
if not spotBid:
logger.info('Launching non-preemptable leader')
create_ondemand_instances(ctx.ec2, image_id=self._discoverAMI(ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching preemptable leader')
# force generator to evaluate
list(create_spot_instances(ec2=ctx.ec2,
price=spotBid,
image_id=self._discoverAMI(ctx),
tags={'clusterName': clusterName},
spec=kwargs,
num_instances=1))
leader = self._getLeader(clusterName=clusterName, wait=True, zone=zone)
defaultTags = {'Name': clusterName, 'Owner': keyName}
defaultTags.update(userTags)
# if we running launch cluster we need to save this data as it won't be generated
# from the metadata. This data is needed to launch worker nodes.
self.leaderIP = leader.private_ip_address
self._addTags([leader], defaultTags)
self.ctx = ctx
self.spotBid = spotBid
preemptable = True if spotBid else False
self.instanceType[preemptable] = ec2_instance_types[instanceType]
self.clusterName = clusterName
self.keyName = keyName
self.tags = leader.tags
self.subnetID = leader.subnet_id
if workers:
# assuming that if the leader was launched without a spotbid then all workers
# will be non-preemptable
workersCreated = self.addNodes(workers, preemptable=bool(spotBid))
logger.info('Added %d workers with %d workers requested', workersCreated, workers)
return leader
def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, zone=None):
ctx = cls._buildContext(clusterName=clusterName, zone=zone)
profileARN = cls._getProfileARN(ctx)
# the security group name is used as the cluster identifier
cls._createSecurityGroup(ctx, clusterName)
bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType])
leaderData = dict(role='leader',
image=applianceSelf(),
entrypoint='mesos-master',
args=leaderArgs.format(name=clusterName))
userData = awsUserData.format(**leaderData)
kwargs = {'key_name': keyName, 'security_groups': [clusterName],
'instance_type': instanceType,
'user_data': userData, 'block_device_map': bdm,
'instance_profile_arn': profileARN}
if not spotBid:
logger.info('Launching non-preemptable leader')
create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching preemptable leader')
# force generator to evaluate
list(create_spot_instances(ec2=ctx.ec2,
price=spotBid,
image_id=cls._discoverAMI(ctx),
tags={'clusterName': clusterName},
spec=kwargs,
num_instances=1))
return cls._getLeader(clusterName=clusterName, wait=True)
def addNodes(self, nodeType, numNodes, preemptable):
instanceType = ec2_instance_types[nodeType]
bdm = self._getBlockDeviceMapping(instanceType, rootVolSize=self.nodeStorage)
arn = self._getProfileARN(self.ctx)
keyPath = '' if not self.config or not self.config.sseKey else self.config.sseKey
entryPoint = 'mesos-slave' if not self.config or not self.config.sseKey else "waitForKey.sh"
workerData = dict(role='worker',
image=applianceSelf(),
entrypoint=entryPoint,
sshKey=self.masterPublicKey,
args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable, keyPath=keyPath))
userData = awsUserData.format(**workerData)
sgs = [sg for sg in self.ctx.ec2.get_all_security_groups() if sg.name == self.clusterName]
kwargs = {'key_name': self.keyName,
'security_group_ids': [sg.id for sg in sgs],
'instance_type': instanceType.name,
'user_data': userData,
'block_device_map': bdm,
'instance_profile_arn': arn,
'placement': getCurrentAWSZone()}
kwargs["subnet_id"] = self.subnetID if self.subnetID else self._getClusterInstance(self.instanceMetaData).subnet_id
instancesLaunched = []
for attempt in retry(predicate=AWSProvisioner._throttlePredicate):
with attempt:
# after we start launching instances we want to insure the full setup is done
# the biggest obstacle is AWS request throttling, so we retry on these errors at
# every request in this method
if not preemptable:
logger.info('Launching %s non-preemptable nodes', numNodes)
instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx),
spec=kwargs, num_instances=numNodes)
else:
logger.info('Launching %s preemptable nodes', numNodes)
kwargs['placement'] = getSpotZone(self.spotBids[nodeType], instanceType.name, self.ctx)
# force generator to evaluate
instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2,
price=self.spotBids[nodeType],
image_id=self._discoverAMI(self.ctx),
tags={'clusterName': self.clusterName},
spec=kwargs,
num_instances=numNodes,
tentative=True)
)
# flatten the list
instancesLaunched = [item for sublist in instancesLaunched for item in sublist]
for attempt in retry(predicate=AWSProvisioner._throttlePredicate):
with attempt:
wait_instances_running(self.ctx.ec2, instancesLaunched)
# request throttling retry happens internally to these two methods to insure proper granularity
AWSProvisioner._addTags(instancesLaunched, self.tags)
self._propagateKey(instancesLaunched)
logger.info('Launched %s new instance(s)', numNodes)
return len(instancesLaunched)
def addNodes(self, numNodes, preemptable):
instanceType = self._getInstanceType(preemptable)
bdm = self._getBlockDeviceMapping(instanceType, rootVolSize=self.nodeStorage)
arn = self._getProfileARN(self.ctx)
keyPath = '' if not self.config or not self.config.sseKey else self.config.sseKey
entryPoint = 'mesos-slave' if not self.config or not self.config.sseKey else "waitForKey.sh"
workerData = dict(role='worker',
image=applianceSelf(),
entrypoint=entryPoint,
sshKey=self.masterPublicKey,
args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable, keyPath=keyPath))
userData = awsUserData.format(**workerData)
sgs = [sg for sg in self.ctx.ec2.get_all_security_groups() if sg.name == self.clusterName]
kwargs = {'key_name': self.keyName,
'security_group_ids': [sg.id for sg in sgs],
'instance_type': instanceType.name,
'user_data': userData,
'block_device_map': bdm,
'instance_profile_arn': arn,
'placement': getCurrentAWSZone()}
kwargs["subnet_id"] = self.subnetID if self.subnetID else self._getClusterInstance(self.instanceMetaData).subnet_id
instancesLaunched = []
for attempt in retry(predicate=AWSProvisioner._throttlePredicate):
with attempt:
# after we start launching instances we want to insure the full setup is done
# the biggest obstacle is AWS request throttling, so we retry on these errors at
# every request in this method
if not preemptable:
logger.info('Launching %s non-preemptable nodes', numNodes)
instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx),
spec=kwargs, num_instances=numNodes)
else:
logger.info('Launching %s preemptable nodes', numNodes)
kwargs['placement'] = getSpotZone(self.spotBid, instanceType.name, self.ctx)
# force generator to evaluate
instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2,
price=self.spotBid,
image_id=self._discoverAMI(self.ctx),
tags={'clusterName': self.clusterName},
spec=kwargs,
num_instances=numNodes,
tentative=True)
)
# flatten the list
instancesLaunched = [item for sublist in instancesLaunched for item in sublist]
for attempt in retry(predicate=AWSProvisioner._throttlePredicate):
with attempt:
wait_instances_running(self.ctx.ec2, instancesLaunched)
# request throttling retry happens internally to these two methods to insure proper granularity
AWSProvisioner._addTags(instancesLaunched, self.tags)
self._propagateKey(instancesLaunched)
logger.info('Launched %s new instance(s)', numNodes)
return len(instancesLaunched)
def launchCluster(cls,
instanceType,
keyName,
clusterName,
spotBid=None,
userTags=None,
zone=None,
vpcSubnet=None):
if userTags is None:
userTags = {}
ctx = cls._buildContext(clusterName=clusterName, zone=zone)
profileARN = cls._getProfileARN(ctx)
# the security group name is used as the cluster identifier
sgs = cls._createSecurityGroup(ctx, clusterName, vpcSubnet)
bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType])
leaderData = dict(role='leader',
image=applianceSelf(),
entrypoint='mesos-master',
sshKey='AAAAB3NzaC1yc2Enoauthorizedkeyneeded',
args=leaderArgs.format(name=clusterName))
userData = awsUserData.format(**leaderData)
kwargs = {
'key_name': keyName,
'security_group_ids': [sg.id for sg in sgs],
'instance_type': instanceType,
'user_data': userData,
'block_device_map': bdm,
'instance_profile_arn': profileARN,
'placement': zone
}
if vpcSubnet:
kwargs["subnet_id"] = vpcSubnet
if not spotBid:
logger.info('Launching non-preemptable leader')
create_ondemand_instances(ctx.ec2,
image_id=cls._discoverAMI(ctx),
spec=kwargs,
num_instances=1)
else:
logger.info('Launching preemptable leader')
# force generator to evaluate
list(
create_spot_instances(ec2=ctx.ec2,
price=spotBid,
image_id=cls._discoverAMI(ctx),
tags={'clusterName': clusterName},
spec=kwargs,
num_instances=1))
leader = cls._getLeader(clusterName=clusterName, wait=True, zone=zone)
defaultTags = {'Name': clusterName, 'Owner': keyName}
defaultTags.update(userTags)
cls._addTags([leader], defaultTags)
return leader
def createInstances():
"""
:rtype: Iterable[list[Instance]]
"""
if preemptable:
for batch in create_spot_instances(
self._ec2,
self.spotBid,
self.imageId,
spec,
# Don't insist on spot requests and don't raise
# if no requests were fulfilled:
tentative=True,
num_instances=numNodes,
timeout=deadline - time.time()):
yield batch
else:
yield create_ondemand_instances(self._ec2,
self.imageId,
spec,
num_instances=numNodes)
def _addNodes(self, instances, numNodes, preemptable=False):
bdm = self._getBlockDeviceMapping(self.instanceType)
arn = self._getProfileARN(self.ctx)
workerData = dict(role='worker',
image=applianceSelf(),
entrypoint='mesos-slave',
args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable))
userData = awsUserData.format(**workerData)
kwargs = {'key_name': self.keyName,
'security_groups': [self.clusterName],
'instance_type': self.instanceType.name,
'user_data': userData,
'block_device_map': bdm,
'instance_profile_arn': arn}
instancesLaunched = []
if not preemptable:
logger.info('Launching %s non-preemptable nodes', numNodes)
instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching %s preemptable nodes', numNodes)
kwargs['placement'] = getSpotZone(self.spotBid, self.instanceType.name, self.ctx)
# force generator to evaluate
instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2,
price=self.spotBid,
image_id=self._discoverAMI(self.ctx),
tags={'clusterName': self.clusterName},
spec=kwargs,
num_instances=numNodes,
tentative=True)
)
wait_instances_running(self.ctx.ec2, instancesLaunched)
logger.info('Launched %s new instance(s)', numNodes)
return len(instancesLaunched)
def launchCluster(self, instanceType, keyName, clusterName, workers=0,
spotBid=None, userTags=None, zone=None, vpcSubnet=None, leaderStorage=50, nodeStorage=50):
# only use this node storage value if launchCluster is called from cluster utility
if self.config is None:
self.nodeStorage = nodeStorage
if userTags is None:
userTags = {}
ctx = self._buildContext(clusterName=clusterName, zone=zone)
profileARN = self._getProfileARN(ctx)
# the security group name is used as the cluster identifier
sgs = self._createSecurityGroup(ctx, clusterName, vpcSubnet)
bdm = self._getBlockDeviceMapping(ec2_instance_types[instanceType], rootVolSize=leaderStorage)
self.masterPublicKey = 'AAAAB3NzaC1yc2Enoauthorizedkeyneeded'
leaderData = dict(role='leader',
image=applianceSelf(),
entrypoint='mesos-master',
sshKey=self.masterPublicKey,
args=leaderArgs.format(name=clusterName))
userData = awsUserData.format(**leaderData)
kwargs = {'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs],
'instance_type': instanceType,
'user_data': userData, 'block_device_map': bdm,
'instance_profile_arn': profileARN,
'placement': zone}
if vpcSubnet:
kwargs["subnet_id"] = vpcSubnet
if not spotBid:
logger.info('Launching non-preemptable leader')
create_ondemand_instances(ctx.ec2, image_id=self._discoverAMI(ctx),
spec=kwargs, num_instances=1)
else:
logger.info('Launching preemptable leader')
# force generator to evaluate
list(create_spot_instances(ec2=ctx.ec2,
price=spotBid,
image_id=self._discoverAMI(ctx),
tags={'clusterName': clusterName},
spec=kwargs,
num_instances=1))
leader = self._getLeader(clusterName=clusterName, wait=True, zone=zone)
defaultTags = {'Name': clusterName, 'Owner': keyName}
defaultTags.update(userTags)
# if we running launch cluster we need to save this data as it won't be generated
# from the metadata. This data is needed to launch worker nodes.
self.leaderIP = leader.private_ip_address
self._addTags([leader], defaultTags)
self.ctx = ctx
self.spotBid = spotBid
preemptable = True if spotBid else False
self.instanceType[preemptable] = ec2_instance_types[instanceType]
self.clusterName = clusterName
self.keyName = keyName
self.tags = leader.tags
self.subnetID = leader.subnet_id
if workers:
# assuming that if the leader was launched without a spotbid then all workers
# will be non-preemptable
workersCreated = self.addNodes(workers, preemptable=bool(spotBid))
logger.info('Added %d workers with %d workers requested', workersCreated, workers)
return leader