def process_all( client, rack_id, tag_name, tag_definition, system_ids, xpath, batch_size=None, ): log.debug( "Processing {nums} system_ids for tag {name}.", nums=len(system_ids), name=tag_name, ) if batch_size is None: batch_size = DEFAULT_BATCH_SIZE batches = gen_batches(system_ids, batch_size) node_details = gen_node_details(client, batches) nodes_matched, nodes_unmatched = classify( partial(try_match_xpath, xpath, logger=maaslog), node_details) post_updated_nodes( client, rack_id, tag_name, tag_definition, nodes_matched, nodes_unmatched, )
def test_subjects(self): subjects = [("one", 1), ("two", 2), ("three", 3)] def is_even(subject): return subject % 2 == 0 self.assertSequenceEqual((["two"], ["one", "three"]), classify(is_even, subjects))
def populate_tags_for_single_node(tags, node): """Reevaluate all tags for a single node. Presumably this node's details have recently changed. Use `populate_tags` when many nodes need reevaluating. """ probed_details = get_single_probed_details(node.system_id) probed_details_doc = merge_details(probed_details) # Same document, many queries: use XPathEvaluator. evaluator = etree.XPathEvaluator(probed_details_doc, namespaces=tag_nsmap) evaluator = partial(try_match_xpath, doc=evaluator, logger=logger) tags_defined = ((tag, tag.definition) for tag in tags if tag.is_defined) tags_matching, tags_nonmatching = classify(evaluator, tags_defined) node.tags.remove(*tags_nonmatching) node.tags.add(*tags_matching)
def populate_tags_for_single_node(tags, node): """Reevaluate all tags for a single node. Presumably this node's details have recently changed. Use `populate_tags` when many nodes need reevaluating AND there are rack controllers available to which to farm-out work. Use `populate_tag_for_multiple_nodes` when many nodes need reevaluating locally, i.e. when there are no rack controllers connected. """ probed_details = get_single_probed_details(node) probed_details_doc = merge_details(probed_details) # Same document, many queries: use XPathEvaluator. evaluator = etree.XPathEvaluator(probed_details_doc, namespaces=tag_nsmap) evaluator = partial(try_match_xpath, doc=evaluator, logger=logger) tags_defined = ((tag, tag.definition) for tag in tags if tag.is_defined) tags_matching, tags_nonmatching = classify(evaluator, tags_defined) node.tags.remove(*tags_nonmatching) node.tags.add(*tags_matching)
def process_all(client, tag_name, tag_definition, nodegroup_uuid, system_ids, xpath, batch_size=None): logger.debug( "processing %d system_ids for tag %s nodegroup %s", len(system_ids), tag_name, nodegroup_uuid) if batch_size is None: batch_size = DEFAULT_BATCH_SIZE batches = gen_batches(system_ids, batch_size) node_details = gen_node_details(client, nodegroup_uuid, batches) nodes_matched, nodes_unmatched = classify( partial(try_match_xpath, xpath, logger=logger), node_details) # Upload all updates for one nodegroup at one time. This should be no more # than ~41*10,000 = 410kB. That should take <1s even on a 10Mbit network. # This also allows us to track if a nodegroup has been processed in the DB, # without having to add another API call. post_updated_nodes( client, tag_name, tag_definition, nodegroup_uuid, nodes_matched, nodes_unmatched)
def populate_tag_for_multiple_nodes(tag, nodes, batch_size=DEFAULT_BATCH_SIZE): """Reevaluate a single tag for a multiple nodes. Presumably this tag's expression has recently changed. Use `populate_tags` when many nodes need reevaluating AND there are rack controllers available to which to farm-out work. Use this only when many nodes need reevaluating locally, i.e. when there are no rack controllers connected. """ # Same expression, multuple documents: compile expression with XPath. xpath = etree.XPath(tag.definition, namespaces=tag_nsmap) # The XML details documents can be large so work in batches. for batch in gen_batches(nodes, batch_size): probed_details = get_probed_details(batch) probed_details_docs_by_node = { node: merge_details(probed_details[node.system_id]) for node in batch } nodes_matching, nodes_nonmatching = classify( partial(try_match_xpath, xpath, logger=maaslog), probed_details_docs_by_node.items()) tag.node_set.remove(*nodes_nonmatching) tag.node_set.add(*nodes_matching)
def process_all(client, tag_name, tag_definition, nodegroup_uuid, system_ids, xpath, batch_size=None): logger.debug("processing %d system_ids for tag %s nodegroup %s", len(system_ids), tag_name, nodegroup_uuid) if batch_size is None: batch_size = DEFAULT_BATCH_SIZE batches = gen_batches(system_ids, batch_size) node_details = gen_node_details(client, nodegroup_uuid, batches) nodes_matched, nodes_unmatched = classify( partial(try_match_xpath, xpath, logger=logger), node_details) # Upload all updates for one nodegroup at one time. This should be no more # than ~41*10,000 = 410kB. That should take <1s even on a 10Mbit network. # This also allows us to track if a nodegroup has been processed in the DB, # without having to add another API call. post_updated_nodes(client, tag_name, tag_definition, nodegroup_uuid, nodes_matched, nodes_unmatched)
def test_no_subjects(self): self.assertSequenceEqual(([], []), classify(sentinel.func, []))
def test_subjects(self): subjects = [("one", 1), ("two", 2), ("three", 3)] is_even = lambda subject: subject % 2 == 0 self.assertSequenceEqual((["two"], ["one", "three"]), classify(is_even, subjects))