def process_all(
client,
rack_id,
tag_name,
tag_definition,
system_ids,
xpath,
batch_size=None,
):
log.debug(
"Processing {nums} system_ids for tag {name}.",
nums=len(system_ids),
name=tag_name,
)
if batch_size is None:
batch_size = DEFAULT_BATCH_SIZE
batches = gen_batches(system_ids, batch_size)
node_details = gen_node_details(client, batches)
nodes_matched, nodes_unmatched = classify(
partial(try_match_xpath, xpath, logger=maaslog), node_details)
post_updated_nodes(
client,
rack_id,
tag_name,
tag_definition,
nodes_matched,
nodes_unmatched,
)
def test_subjects(self):
subjects = [("one", 1), ("two", 2), ("three", 3)]
def is_even(subject):
return subject % 2 == 0
self.assertSequenceEqual((["two"], ["one", "three"]),
classify(is_even, subjects))
def populate_tags_for_single_node(tags, node):
"""Reevaluate all tags for a single node.
Presumably this node's details have recently changed. Use
`populate_tags` when many nodes need reevaluating.
"""
probed_details = get_single_probed_details(node.system_id)
probed_details_doc = merge_details(probed_details)
# Same document, many queries: use XPathEvaluator.
evaluator = etree.XPathEvaluator(probed_details_doc, namespaces=tag_nsmap)
evaluator = partial(try_match_xpath, doc=evaluator, logger=logger)
tags_defined = ((tag, tag.definition) for tag in tags if tag.is_defined)
tags_matching, tags_nonmatching = classify(evaluator, tags_defined)
node.tags.remove(*tags_nonmatching)
node.tags.add(*tags_matching)
def populate_tags_for_single_node(tags, node):
"""Reevaluate all tags for a single node.
Presumably this node's details have recently changed. Use `populate_tags`
when many nodes need reevaluating AND there are rack controllers available
to which to farm-out work. Use `populate_tag_for_multiple_nodes` when many
nodes need reevaluating locally, i.e. when there are no rack controllers
connected.
"""
probed_details = get_single_probed_details(node)
probed_details_doc = merge_details(probed_details)
# Same document, many queries: use XPathEvaluator.
evaluator = etree.XPathEvaluator(probed_details_doc, namespaces=tag_nsmap)
evaluator = partial(try_match_xpath, doc=evaluator, logger=logger)
tags_defined = ((tag, tag.definition) for tag in tags if tag.is_defined)
tags_matching, tags_nonmatching = classify(evaluator, tags_defined)
node.tags.remove(*tags_nonmatching)
node.tags.add(*tags_matching)
def process_all(client, tag_name, tag_definition, nodegroup_uuid, system_ids,
xpath, batch_size=None):
logger.debug(
"processing %d system_ids for tag %s nodegroup %s",
len(system_ids), tag_name, nodegroup_uuid)
if batch_size is None:
batch_size = DEFAULT_BATCH_SIZE
batches = gen_batches(system_ids, batch_size)
node_details = gen_node_details(client, nodegroup_uuid, batches)
nodes_matched, nodes_unmatched = classify(
partial(try_match_xpath, xpath, logger=logger), node_details)
# Upload all updates for one nodegroup at one time. This should be no more
# than ~41*10,000 = 410kB. That should take <1s even on a 10Mbit network.
# This also allows us to track if a nodegroup has been processed in the DB,
# without having to add another API call.
post_updated_nodes(
client, tag_name, tag_definition, nodegroup_uuid,
nodes_matched, nodes_unmatched)
def populate_tag_for_multiple_nodes(tag, nodes, batch_size=DEFAULT_BATCH_SIZE):
"""Reevaluate a single tag for a multiple nodes.
Presumably this tag's expression has recently changed. Use `populate_tags`
when many nodes need reevaluating AND there are rack controllers available
to which to farm-out work. Use this only when many nodes need reevaluating
locally, i.e. when there are no rack controllers connected.
"""
# Same expression, multuple documents: compile expression with XPath.
xpath = etree.XPath(tag.definition, namespaces=tag_nsmap)
# The XML details documents can be large so work in batches.
for batch in gen_batches(nodes, batch_size):
probed_details = get_probed_details(batch)
probed_details_docs_by_node = {
node: merge_details(probed_details[node.system_id])
for node in batch
}
nodes_matching, nodes_nonmatching = classify(
partial(try_match_xpath, xpath, logger=maaslog),
probed_details_docs_by_node.items())
tag.node_set.remove(*nodes_nonmatching)
tag.node_set.add(*nodes_matching)
def process_all(client,
tag_name,
tag_definition,
nodegroup_uuid,
system_ids,
xpath,
batch_size=None):
logger.debug("processing %d system_ids for tag %s nodegroup %s",
len(system_ids), tag_name, nodegroup_uuid)
if batch_size is None:
batch_size = DEFAULT_BATCH_SIZE
batches = gen_batches(system_ids, batch_size)
node_details = gen_node_details(client, nodegroup_uuid, batches)
nodes_matched, nodes_unmatched = classify(
partial(try_match_xpath, xpath, logger=logger), node_details)
# Upload all updates for one nodegroup at one time. This should be no more
# than ~41*10,000 = 410kB. That should take <1s even on a 10Mbit network.
# This also allows us to track if a nodegroup has been processed in the DB,
# without having to add another API call.
post_updated_nodes(client, tag_name, tag_definition, nodegroup_uuid,
nodes_matched, nodes_unmatched)
def test_no_subjects(self):
self.assertSequenceEqual(([], []), classify(sentinel.func, []))
def test_subjects(self):
subjects = [("one", 1), ("two", 2), ("three", 3)]
is_even = lambda subject: subject % 2 == 0
self.assertSequenceEqual((["two"], ["one", "three"]),
classify(is_even, subjects))
