def get_agent_resources(master, agent_ids):
"""Returns the total resources for each of the given agents
:param master: The address for the Mesos master
:type master: `util.host.HostAddress`
:param agent_ids: The set of agent IDs
:type agent_ids: set
:returns: The total resources for each agent stored by agent ID
:rtype: dict
"""
results = {}
resp = make_dcos_request(master, '/slaves')
for agent_dict in resp.json()['slaves']:
agent_id = agent_dict['id']
if agent_id in agent_ids:
resource_list = []
resource_dict = agent_dict['resources']
for name in resource_dict:
value = resource_dict[name]
if isinstance(value, float):
resource_list.append(ScalarResource(name, value))
resources = NodeResources(resource_list)
results[agent_id] = resources
return results
def get_agent_resources(hostname, port, agent_ids):
"""Returns the total resources for each of the given agents
:param hostname: The hostname of the master
:type hostname: str
:param port: The port of the master
:type port: int
:param agent_ids: The set of agent IDs
:type agent_ids: set
:returns: The total resources for each agent stored by agent ID
:rtype: dict
"""
results = {}
url = 'http://%s:%i/slaves' % (hostname, port)
response = urllib2.urlopen(url)
response_json = json.load(response)
for agent_dict in response_json['slaves']:
agent_id = agent_dict['id']
if agent_id in agent_ids:
resource_list = []
resource_dict = agent_dict['resources']
for name in resource_dict:
value = resource_dict[name]
if isinstance(value, float):
resource_list.append(ScalarResource(name, value))
resources = NodeResources(resource_list)
results[agent_id] = resources
return results
def get_node_resources(self):
"""Returns the node resources represented by this schema
:returns: The node resources
:rtype: :class:`node.resources.NodeResources`
"""
resource_list = []
for name, value in self._resources['resources'].items():
resource_list.append(ScalarResource(name, float(value)))
return NodeResources(resource_list)
def resourceOffers(self, driver, offers):
"""
Invoked when resources have been offered to this framework. A single
offer will only contain resources from a single slave. Resources
associated with an offer will not be re-offered to _this_ framework
until either (a) this framework has rejected those resources (see
SchedulerDriver.launchTasks) or (b) those resources have been
rescinded (see Scheduler.offerRescinded). Note that resources may be
concurrently offered to more than one framework at a time (depending
on the allocator being used). In that case, the first framework to
launch tasks using those resources will be able to use them while the
other frameworks will have those resources rescinded (or if a
framework has already launched tasks with those resources then those
tasks will fail with a TASK_LOST status and a message saying as much).
See documentation for :meth:`mesos_api.mesos.Scheduler.resourceOffers`.
"""
started = now()
agents = {}
resource_offers = []
total_resources = NodeResources()
for offer in offers:
offer_id = offer.id.value
agent_id = offer.slave_id.value
framework_id = offer.framework_id.value
hostname = offer.hostname
resource_list = []
for resource in offer.resources:
if resource.type == 0: # This is the SCALAR type
resource_list.append(
ScalarResource(resource.name, resource.scalar.value))
resources = NodeResources(resource_list)
total_resources.add(resources)
agents[agent_id] = Agent(agent_id, hostname)
resource_offers.append(
ResourceOffer(offer_id, agent_id, framework_id, resources,
started))
node_mgr.register_agents(agents.values())
resource_mgr.add_new_offers(resource_offers)
num_offers = len(resource_offers)
logger.info('Received %d offer(s) with %s from %d node(s)', num_offers,
total_resources, len(agents))
scheduler_mgr.add_new_offer_count(num_offers)
duration = now() - started
msg = 'Scheduler resourceOffers() took %.3f seconds'
if duration > ScaleScheduler.NORMAL_WARN_THRESHOLD:
logger.warning(msg, duration.total_seconds())
else:
logger.debug(msg, duration.total_seconds())
def get_resources(self, task_type):
"""Returns the resources for the given task type, None if the task type doesn't exist
:param task_type: The task type
:type task_type: string
:returns: The task resources, possibly None
:rtype: :class:`node.resources.node_resources.NodeResources`
"""
for task_dict in self._configuration['tasks']:
if task_dict['type'] == task_type and 'resources' in task_dict:
resources = []
for name, value in task_dict['resources'].items():
resources.append(ScalarResource(name, value))
return NodeResources(resources)
return None
def offers(self, offers):
"""
Invoked when resources have been offered to this framework. A single
offer will only contain resources from a single agent. Resources
associated with an offer will not be re-offered to _this_ framework
until either (a) this framework has rejected those resources
or (b) those resources have been rescinded. Note that resources may be
concurrently offered to more than one framework at a time (depending
on the allocator being used). In that case, the first framework to
launch tasks using those resources will be able to use them while the
other frameworks will have those resources rescinded (or if a
framework has already launched tasks with those resources then those
tasks will fail with a TASK_LOST status and a message saying as much).
"""
started = now()
agents = {}
offered_nodes = []
resource_offers = []
total_resources = NodeResources()
skipped_roles = set()
for offer in offers:
scale_offer = from_mesos_offer(offer)
offer_id = scale_offer.id.value
agent_id = scale_offer.agent_id.value
framework_id = scale_offer.framework_id.value
hostname = scale_offer.hostname
offered_nodes.append(hostname)
# ignore offers while we're paused
if scheduler_mgr.config.is_paused:
offer.decline()
continue
resource_list = []
for resource in scale_offer.resources:
# Only accept resource that are of SCALAR type and have a role matching our accept list
if resource.type == RESOURCE_TYPE_SCALAR:
if resource.role in settings.ACCEPTED_RESOURCE_ROLE:
logger.debug("Received scalar resource %s with value %i associated with role %s" %
(resource.name, resource.scalar.value, resource.role))
resource_list.append(ScalarResource(resource.name, resource.scalar.value))
else:
skipped_roles.add(resource.role)
offer.decline()
logger.debug("Number of resources: %i" % len(resource_list))
# Only register agent, if offers are being received
if len(resource_list) > 0:
resources = NodeResources(resource_list)
total_resources.add(resources)
agents[agent_id] = Agent(agent_id, hostname)
resource_offers.append(ResourceOffer(offer_id, agent_id, framework_id, resources, started, offer))
logger.debug("Offer analysis complete with %i resource offers." % len(resource_offers))
node_mgr.register_agents(agents.values())
logger.debug("Agents registered.")
resource_mgr.add_new_offers(resource_offers)
logger.debug("Resource offers added.")
Node.objects.update_node_offers(offered_nodes, now())
logger.debug("Node offer times updated.")
num_offers = len(resource_offers)
logger.info('Received %d offer(s) with %s from %d node(s)', num_offers, total_resources, len(agents))
if len(skipped_roles):
logger.warning('Skipped offers from roles that are not marked as accepted: %s', ','.join(skipped_roles))
scheduler_mgr.add_new_offer_count(num_offers)
duration = now() - started
msg = 'Scheduler resourceOffers() took %.3f seconds'
if duration > ScaleScheduler.NORMAL_WARN_THRESHOLD:
logger.warning(msg, duration.total_seconds())
else:
logger.debug(msg, duration.total_seconds())