def has_condition(self, condition_id, condition_context) -> bool:
"""Return if the condition exist and avilible"""
if condition_id in self.conditions:
condition = self.get_condition(condition_id)
return condition.is_available(condition_context)
logger.warning("Condition id %s not found.", condition_id)
return False
def check_and_sort(collections, target_type, reply):
"""Check the collection and sort conditions"""
collection = collections.get(target_type)
if collection:
collection = sorted(collection, key=lambda x: x.get('id'))
reply.append({"id": target_type + "_condition_collection", "conditions": collection})
else:
logger.warning("Condition collection for %s is None.", target_type)
def recommend_watchpoints(condition_mgr: ConditionMgr, multi_card_graph_stream, condition_context):
"""
Recommend watchpoints.
Args:
condition_mgr (ConditionMgr): Condition manager instance.
multi_card_graph_stream (GraphHandler): Multi card graph handler instance.
condition_context (ConditionContext): Context for condition.
Returns:
list[WatchPointData], watch points to be created.
"""
watch_points = []
if not multi_card_graph_stream.has_graph:
logger.warning("Given graph is None.")
return watch_points
if not settings.ENABLE_RECOMMENDED_WATCHPOINTS:
return watch_points
# add weight watch points
merged_info = get_basic_node_info(TargetTypeEnum.WEIGHT.value, multi_card_graph_stream)
_recommend_weight_initialization(merged_info, condition_mgr, watch_points, condition_context)
_recommend_weight_change_too_large(merged_info, condition_mgr, watch_points, condition_context)
# Because we cannot identify trainable weights currently, weight_no_change and weight_change_too_small will not be
# recommended.
trainable_weight_nodes = []
_recommend_weight_not_changed(condition_mgr, trainable_weight_nodes, watch_points, condition_context)
_recommend_weight_change_too_small(condition_mgr, trainable_weight_nodes, watch_points, condition_context)
# add gradient watch points
merged_info = get_basic_node_info(TargetTypeEnum.GRADIENT.value, multi_card_graph_stream)
_recommend_gradient_vanishing(merged_info, condition_mgr, watch_points, condition_context)
# add tensor watch points
merged_info = get_basic_node_info(TargetTypeEnum.TENSOR.value, multi_card_graph_stream)
_recommend_operator_overflow(merged_info, condition_mgr, watch_points, condition_context)
_recommend_tensor_overflow(merged_info, condition_mgr, watch_points, condition_context)
_recommend_tensor_all_zero(merged_info, condition_mgr, watch_points, condition_context)
# add activation watch points
merged_info = get_basic_node_info(TargetTypeEnum.ACTIVATION.value, multi_card_graph_stream,
ActivationFuncEnum.TANH.value)
_recommend_activation_range(merged_info, condition_mgr, watch_points, condition_context,
ActivationFuncEnum.TANH.value)
merged_info = get_basic_node_info(TargetTypeEnum.ACTIVATION.value, multi_card_graph_stream,
ActivationFuncEnum.SIGMOID.value)
_recommend_activation_range(merged_info, condition_mgr, watch_points, condition_context,
ActivationFuncEnum.SIGMOID.value)
merged_info = get_basic_node_info(TargetTypeEnum.ACTIVATION.value, multi_card_graph_stream,
[ActivationFuncEnum.RELU.value, ActivationFuncEnum.RELUV2.value])
_recommend_activation_range(merged_info, condition_mgr, watch_points, condition_context,
ActivationFuncEnum.RELU.value)
return watch_points