def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, zone=None): ctx = cls._buildContext(clusterName=clusterName, zone=zone) profileARN = cls._getProfileARN(ctx) # the security group name is used as the cluster identifier cls._createSecurityGroup(ctx, clusterName) bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType]) leaderData = dict(role='leader', image=applianceSelf(), entrypoint='mesos-master', args=leaderArgs.format(name=clusterName)) userData = awsUserData.format(**leaderData) kwargs = {'key_name': keyName, 'security_groups': [clusterName], 'instance_type': instanceType, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': profileARN} if not spotBid: logger.info('Launching non-preemptable leader') create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx), spec=kwargs, num_instances=1) else: logger.info('Launching preemptable leader') # force generator to evaluate list(create_spot_instances(ec2=ctx.ec2, price=spotBid, image_id=cls._discoverAMI(ctx), tags={'clusterName': clusterName}, spec=kwargs, num_instances=1)) return cls._getLeader(clusterName=clusterName, wait=True)
def _addNodes(self, instances, numNodes, preemptable=False): bdm = self._getBlockDeviceMapping(self.instanceType) arn = self._getProfileARN(self.ctx) workerData = dict(role='worker', image=applianceSelf(), entrypoint='mesos-slave', args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable)) userData = awsUserData.format(**workerData) kwargs = {'key_name': self.keyName, 'security_groups': [self.clusterName], 'instance_type': self.instanceType.name, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': arn} instancesLaunched = [] if not preemptable: logger.info('Launching %s non-preemptable nodes', numNodes) instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx), spec=kwargs, num_instances=1) else: logger.info('Launching %s preemptable nodes', numNodes) # force generator to evaluate instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2, price=self.spotBid, image_id=self._discoverAMI(self.ctx), tags={'clusterName': self.clusterName}, spec=kwargs, num_instances=numNodes)) wait_instances_running(self.ctx.ec2, instancesLaunched) logger.info('Launched %s new instance(s)', numNodes) return len(instancesLaunched)
def launchCluster(self, instanceType, keyName, clusterName, workers=0, spotBid=None, userTags=None, zone=None, vpcSubnet=None): if userTags is None: userTags = {} ctx = self._buildContext(clusterName=clusterName, zone=zone) profileARN = self._getProfileARN(ctx) # the security group name is used as the cluster identifier sgs = self._createSecurityGroup(ctx, clusterName, vpcSubnet) bdm = self._getBlockDeviceMapping(ec2_instance_types[instanceType]) self.masterPublicKey = 'AAAAB3NzaC1yc2Enoauthorizedkeyneeded' leaderData = dict(role='leader', image=applianceSelf(), entrypoint='mesos-master', sshKey=self.masterPublicKey, args=leaderArgs.format(name=clusterName)) userData = awsUserData.format(**leaderData) kwargs = {'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs], 'instance_type': instanceType, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': profileARN, 'placement': zone} if vpcSubnet: kwargs["subnet_id"] = vpcSubnet if not spotBid: logger.info('Launching non-preemptable leader') create_ondemand_instances(ctx.ec2, image_id=self._discoverAMI(ctx), spec=kwargs, num_instances=1) else: logger.info('Launching preemptable leader') # force generator to evaluate list(create_spot_instances(ec2=ctx.ec2, price=spotBid, image_id=self._discoverAMI(ctx), tags={'clusterName': clusterName}, spec=kwargs, num_instances=1)) leader = self._getLeader(clusterName=clusterName, wait=True, zone=zone) defaultTags = {'Name': clusterName, 'Owner': keyName} defaultTags.update(userTags) # if we running launch cluster we need to save this data as it won't be generated # from the metadata. This data is needed to launch worker nodes. self.leaderIP = leader.private_ip_address self._addTags([leader], defaultTags) self.ctx = ctx self.spotBid = spotBid preemptable = True if spotBid else False self.instanceType[preemptable] = ec2_instance_types[instanceType] self.clusterName = clusterName self.keyName = keyName self.tags = leader.tags self.subnetID = leader.subnet_id if workers: # assuming that if the leader was launched without a spotbid then all workers # will be non-preemptable workersCreated = self.addNodes(workers, preemptable=bool(spotBid)) logger.info('Added %d workers with %d workers requested', workersCreated, workers) return leader
def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, zone=None): ctx = cls._buildContext(clusterName=clusterName, zone=zone) profileARN = cls._getProfileARN(ctx) # the security group name is used as the cluster identifier cls._createSecurityGroup(ctx, clusterName) bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType]) leaderData = dict(role='leader', image=applianceSelf(), entrypoint='mesos-master', args=leaderArgs.format(name=clusterName)) userData = awsUserData.format(**leaderData) kwargs = {'key_name': keyName, 'security_groups': [clusterName], 'instance_type': instanceType, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': profileARN} if not spotBid: logger.info('Launching non-preemptable leader') create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx), spec=kwargs, num_instances=1) else: logger.info('Launching preemptable leader') # force generator to evaluate list(create_spot_instances(ec2=ctx.ec2, price=spotBid, image_id=cls._discoverAMI(ctx), tags={'clusterName': clusterName}, spec=kwargs, num_instances=1)) return cls._getLeader(clusterName=clusterName, wait=True)
def addNodes(self, nodeType, numNodes, preemptable): instanceType = ec2_instance_types[nodeType] bdm = self._getBlockDeviceMapping(instanceType, rootVolSize=self.nodeStorage) arn = self._getProfileARN(self.ctx) keyPath = '' if not self.config or not self.config.sseKey else self.config.sseKey entryPoint = 'mesos-slave' if not self.config or not self.config.sseKey else "waitForKey.sh" workerData = dict(role='worker', image=applianceSelf(), entrypoint=entryPoint, sshKey=self.masterPublicKey, args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable, keyPath=keyPath)) userData = awsUserData.format(**workerData) sgs = [sg for sg in self.ctx.ec2.get_all_security_groups() if sg.name == self.clusterName] kwargs = {'key_name': self.keyName, 'security_group_ids': [sg.id for sg in sgs], 'instance_type': instanceType.name, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': arn, 'placement': getCurrentAWSZone()} kwargs["subnet_id"] = self.subnetID if self.subnetID else self._getClusterInstance(self.instanceMetaData).subnet_id instancesLaunched = [] for attempt in retry(predicate=AWSProvisioner._throttlePredicate): with attempt: # after we start launching instances we want to insure the full setup is done # the biggest obstacle is AWS request throttling, so we retry on these errors at # every request in this method if not preemptable: logger.info('Launching %s non-preemptable nodes', numNodes) instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx), spec=kwargs, num_instances=numNodes) else: logger.info('Launching %s preemptable nodes', numNodes) kwargs['placement'] = getSpotZone(self.spotBids[nodeType], instanceType.name, self.ctx) # force generator to evaluate instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2, price=self.spotBids[nodeType], image_id=self._discoverAMI(self.ctx), tags={'clusterName': self.clusterName}, spec=kwargs, num_instances=numNodes, tentative=True) ) # flatten the list instancesLaunched = [item for sublist in instancesLaunched for item in sublist] for attempt in retry(predicate=AWSProvisioner._throttlePredicate): with attempt: wait_instances_running(self.ctx.ec2, instancesLaunched) # request throttling retry happens internally to these two methods to insure proper granularity AWSProvisioner._addTags(instancesLaunched, self.tags) self._propagateKey(instancesLaunched) logger.info('Launched %s new instance(s)', numNodes) return len(instancesLaunched)
def addNodes(self, numNodes, preemptable): instanceType = self._getInstanceType(preemptable) bdm = self._getBlockDeviceMapping(instanceType, rootVolSize=self.nodeStorage) arn = self._getProfileARN(self.ctx) keyPath = '' if not self.config or not self.config.sseKey else self.config.sseKey entryPoint = 'mesos-slave' if not self.config or not self.config.sseKey else "waitForKey.sh" workerData = dict(role='worker', image=applianceSelf(), entrypoint=entryPoint, sshKey=self.masterPublicKey, args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable, keyPath=keyPath)) userData = awsUserData.format(**workerData) sgs = [sg for sg in self.ctx.ec2.get_all_security_groups() if sg.name == self.clusterName] kwargs = {'key_name': self.keyName, 'security_group_ids': [sg.id for sg in sgs], 'instance_type': instanceType.name, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': arn, 'placement': getCurrentAWSZone()} kwargs["subnet_id"] = self.subnetID if self.subnetID else self._getClusterInstance(self.instanceMetaData).subnet_id instancesLaunched = [] for attempt in retry(predicate=AWSProvisioner._throttlePredicate): with attempt: # after we start launching instances we want to insure the full setup is done # the biggest obstacle is AWS request throttling, so we retry on these errors at # every request in this method if not preemptable: logger.info('Launching %s non-preemptable nodes', numNodes) instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx), spec=kwargs, num_instances=numNodes) else: logger.info('Launching %s preemptable nodes', numNodes) kwargs['placement'] = getSpotZone(self.spotBid, instanceType.name, self.ctx) # force generator to evaluate instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2, price=self.spotBid, image_id=self._discoverAMI(self.ctx), tags={'clusterName': self.clusterName}, spec=kwargs, num_instances=numNodes, tentative=True) ) # flatten the list instancesLaunched = [item for sublist in instancesLaunched for item in sublist] for attempt in retry(predicate=AWSProvisioner._throttlePredicate): with attempt: wait_instances_running(self.ctx.ec2, instancesLaunched) # request throttling retry happens internally to these two methods to insure proper granularity AWSProvisioner._addTags(instancesLaunched, self.tags) self._propagateKey(instancesLaunched) logger.info('Launched %s new instance(s)', numNodes) return len(instancesLaunched)
def launchCluster(cls, instanceType, keyName, clusterName, spotBid=None, userTags=None, zone=None, vpcSubnet=None): if userTags is None: userTags = {} ctx = cls._buildContext(clusterName=clusterName, zone=zone) profileARN = cls._getProfileARN(ctx) # the security group name is used as the cluster identifier sgs = cls._createSecurityGroup(ctx, clusterName, vpcSubnet) bdm = cls._getBlockDeviceMapping(ec2_instance_types[instanceType]) leaderData = dict(role='leader', image=applianceSelf(), entrypoint='mesos-master', sshKey='AAAAB3NzaC1yc2Enoauthorizedkeyneeded', args=leaderArgs.format(name=clusterName)) userData = awsUserData.format(**leaderData) kwargs = { 'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs], 'instance_type': instanceType, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': profileARN, 'placement': zone } if vpcSubnet: kwargs["subnet_id"] = vpcSubnet if not spotBid: logger.info('Launching non-preemptable leader') create_ondemand_instances(ctx.ec2, image_id=cls._discoverAMI(ctx), spec=kwargs, num_instances=1) else: logger.info('Launching preemptable leader') # force generator to evaluate list( create_spot_instances(ec2=ctx.ec2, price=spotBid, image_id=cls._discoverAMI(ctx), tags={'clusterName': clusterName}, spec=kwargs, num_instances=1)) leader = cls._getLeader(clusterName=clusterName, wait=True, zone=zone) defaultTags = {'Name': clusterName, 'Owner': keyName} defaultTags.update(userTags) cls._addTags([leader], defaultTags) return leader
def createInstances(): """ :rtype: Iterable[list[Instance]] """ if preemptable: for batch in create_spot_instances( self._ec2, self.spotBid, self.imageId, spec, # Don't insist on spot requests and don't raise # if no requests were fulfilled: tentative=True, num_instances=numNodes, timeout=deadline - time.time()): yield batch else: yield create_ondemand_instances(self._ec2, self.imageId, spec, num_instances=numNodes)
def _addNodes(self, instances, numNodes, preemptable=False): bdm = self._getBlockDeviceMapping(self.instanceType) arn = self._getProfileARN(self.ctx) workerData = dict(role='worker', image=applianceSelf(), entrypoint='mesos-slave', args=workerArgs.format(ip=self.leaderIP, preemptable=preemptable)) userData = awsUserData.format(**workerData) kwargs = {'key_name': self.keyName, 'security_groups': [self.clusterName], 'instance_type': self.instanceType.name, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': arn} instancesLaunched = [] if not preemptable: logger.info('Launching %s non-preemptable nodes', numNodes) instancesLaunched = create_ondemand_instances(self.ctx.ec2, image_id=self._discoverAMI(self.ctx), spec=kwargs, num_instances=1) else: logger.info('Launching %s preemptable nodes', numNodes) kwargs['placement'] = getSpotZone(self.spotBid, self.instanceType.name, self.ctx) # force generator to evaluate instancesLaunched = list(create_spot_instances(ec2=self.ctx.ec2, price=self.spotBid, image_id=self._discoverAMI(self.ctx), tags={'clusterName': self.clusterName}, spec=kwargs, num_instances=numNodes, tentative=True) ) wait_instances_running(self.ctx.ec2, instancesLaunched) logger.info('Launched %s new instance(s)', numNodes) return len(instancesLaunched)
def launchCluster(self, instanceType, keyName, clusterName, workers=0, spotBid=None, userTags=None, zone=None, vpcSubnet=None, leaderStorage=50, nodeStorage=50): # only use this node storage value if launchCluster is called from cluster utility if self.config is None: self.nodeStorage = nodeStorage if userTags is None: userTags = {} ctx = self._buildContext(clusterName=clusterName, zone=zone) profileARN = self._getProfileARN(ctx) # the security group name is used as the cluster identifier sgs = self._createSecurityGroup(ctx, clusterName, vpcSubnet) bdm = self._getBlockDeviceMapping(ec2_instance_types[instanceType], rootVolSize=leaderStorage) self.masterPublicKey = 'AAAAB3NzaC1yc2Enoauthorizedkeyneeded' leaderData = dict(role='leader', image=applianceSelf(), entrypoint='mesos-master', sshKey=self.masterPublicKey, args=leaderArgs.format(name=clusterName)) userData = awsUserData.format(**leaderData) kwargs = {'key_name': keyName, 'security_group_ids': [sg.id for sg in sgs], 'instance_type': instanceType, 'user_data': userData, 'block_device_map': bdm, 'instance_profile_arn': profileARN, 'placement': zone} if vpcSubnet: kwargs["subnet_id"] = vpcSubnet if not spotBid: logger.info('Launching non-preemptable leader') create_ondemand_instances(ctx.ec2, image_id=self._discoverAMI(ctx), spec=kwargs, num_instances=1) else: logger.info('Launching preemptable leader') # force generator to evaluate list(create_spot_instances(ec2=ctx.ec2, price=spotBid, image_id=self._discoverAMI(ctx), tags={'clusterName': clusterName}, spec=kwargs, num_instances=1)) leader = self._getLeader(clusterName=clusterName, wait=True, zone=zone) defaultTags = {'Name': clusterName, 'Owner': keyName} defaultTags.update(userTags) # if we running launch cluster we need to save this data as it won't be generated # from the metadata. This data is needed to launch worker nodes. self.leaderIP = leader.private_ip_address self._addTags([leader], defaultTags) self.ctx = ctx self.spotBid = spotBid preemptable = True if spotBid else False self.instanceType[preemptable] = ec2_instance_types[instanceType] self.clusterName = clusterName self.keyName = keyName self.tags = leader.tags self.subnetID = leader.subnet_id if workers: # assuming that if the leader was launched without a spotbid then all workers # will be non-preemptable workersCreated = self.addNodes(workers, preemptable=bool(spotBid)) logger.info('Added %d workers with %d workers requested', workersCreated, workers) return leader