def __init__(self, tagsets, model_dir, ratio_thres = 0.8, max_num = 2, \
slot_prob_thres = 0.6, value_prob_thres = 0.8, \
mode = 'hr', bs_mode = 'enhance', bs_alpha = 0.0, \
unified_thres = 0.5):
self.tagsets = tagsets
self.frame = {}
self.memory = {}
self.beliefstate = BeliefState(bs_mode, bs_alpha)
self.slot_prob_threshold = slot_prob_thres
self.value_prob_threshold = value_prob_thres
self.ratio_thres = ratio_thres
self.unified_thres = unified_thres
self.slot_prob_factor = math.log(self.unified_thres, self.slot_prob_threshold)
self.value_prob_factor = math.log(self.unified_thres, self.value_prob_threshold)
self.ratio_thres_factor = math.log(self.unified_thres, self.ratio_thres)
self.mode = mode
self.svc = slot_value_classifier()
self.svc.LoadModel(model_dir)
self.tuple_extractor = Tuple_Extractor()
self.rules = DSTC4_rules(tagsets)
self.appLogger = logging.getLogger(self.MY_ID)
if not self.svc.is_set:
self.appLogger.error('Error: Fail to load slot_value_classifier model!')
raise Exception('Error: Fail to load slot_value_classifier model!')
self.value_extractor = value_extractor(tagsets, ratio_thres, max_num)
def __init__(self, tagsets, ratio_thres = 0.85, \
bs_mode = 'enhance', bs_alpha = 0.0, \
unified_thres = 0.5):
self.tagsets = tagsets
self.frame = {}
self.memory = {}
self.beliefstate = BeliefState(bs_mode, bs_alpha)
self.ratio_thres = ratio_thres
self.unified_thres = unified_thres
self.ratio_thres_factor = math.log(self.unified_thres, self.ratio_thres)
self.rules = DSTC4_rules(tagsets)
self.appLogger = logging.getLogger(self.MY_ID)
def __init__(self, tagsets, association_rule_file, semtagger_model, prob_threshold = 0.8, mode = 'exact', bs_mode = 'max', bs_alpha = 0.0, unified_thres=0.5): self.appLogger = logging.getLogger(self.MY_ID) self.tagsets = tagsets self.unified_thres = unified_thres self.prob_threshold = prob_threshold self.prob_thres_factor = math.log(self.unified_thres, self.prob_threshold) input = codecs.open(association_rule_file, 'r', 'utf-8') self.association_rules = json.load(input) input.close() self.semtagger = SemanticTagger(semtagger_model) self.mode = mode self.beliefstate = BeliefState(bs_mode,bs_alpha) self.rules = DSTC4_rules(tagsets) self.reset()
class association_rule_tracker(object):
MY_ID = 'association_rule_tracker'
def __init__(self, tagsets, association_rule_file, semtagger_model, prob_threshold = 0.8, mode = 'exact', bs_mode = 'max', bs_alpha = 0.0, unified_thres=0.5):
self.appLogger = logging.getLogger(self.MY_ID)
self.tagsets = tagsets
self.unified_thres = unified_thres
self.prob_threshold = prob_threshold
self.prob_thres_factor = math.log(self.unified_thres, self.prob_threshold)
input = codecs.open(association_rule_file, 'r', 'utf-8')
self.association_rules = json.load(input)
input.close()
self.semtagger = SemanticTagger(semtagger_model)
self.mode = mode
self.beliefstate = BeliefState(bs_mode,bs_alpha)
self.rules = DSTC4_rules(tagsets)
self.reset()
def addUtter(self, utter):
self.appLogger.debug('utter_index: %d' % (utter['utter_index']))
output = {'utter_index': utter['utter_index']}
topic = utter['segment_info']['topic']
if topic in self.tagsets:
self._UpdateFrameProb(utter)
self._UpdateFrame()
self.frame = self.rules.prune_frame_label(topic, self.frame)
output['frame_prob'] = copy.deepcopy(self.beliefstate.state)
output['frame_label'] = copy.deepcopy(self.frame)
return output
def _get_prob(self, prob, occurrence_times, alpha = 2):
return prob * (1.0 - 1.0 / (1 + occurrence_times**alpha) )
def _get_frame(self, semtag, topic):
candidate_frames = []
semtag_key = 'SemTag_%s(%s)' %(semtag['tag'], '_'.join(semtag['content']))
target_rules = self.association_rules[topic]
if self.mode == 'exact':
if semtag_key in target_rules['v']['rules']:
for frame_key, frame_value in target_rules['v']['rules'][semtag_key].items():
if frame_key.startswith('Frame'):
slot = frame_key[frame_key.find('_') + 1: frame_key.find('(')]
value = ' '.join(frame_key[frame_key.find('(') + 1 : frame_key.find(')')].split('_'))
prob = self._get_prob(frame_value['prob'], frame_value['occurrence'])
candidate_frames.append((slot,value,prob))
elif self.mode == 'fuzzy':
for key, frame_candidates in target_rules['v']['rules'].items():
ratio = fuzz.ratio(semtag_key[7:], key[7:]) * 1.0 / 100
for frame_key, frame_value in frame_candidates.items():
if frame_key.startswith('Frame'):
slot = frame_key[frame_key.find('_') + 1: frame_key.find('(')]
value = ' '.join(frame_key[frame_key.find('(') + 1 : frame_key.find(')')].split('_'))
prob = self._get_prob(frame_value['prob'], frame_value['occurrence']) * ratio
candidate_frames.append((slot,value,prob))
else:
raise Exception('Error: unknown mode %s!' %(mode))
return candidate_frames
def _UpdateFrameProb(self, utter):
topic = utter['segment_info']['topic']
if utter['segment_info']['target_bio'] == 'B':
self.frame = {}
#self.frame_prob = {}
self.beliefstate.reset()
if topic in self.tagsets:
transcript = utter['transcript']
semtags = self.semtagger.ParseSent(transcript)
for semtag in semtags:
candidate_frames = self._get_frame(semtag, topic)
for (slot, value, prob) in candidate_frames:
#self._AddSLot2FrameProb(slot, -1)
#self._AddSlotValue2FrameProb(slot,value,prob)
self.beliefstate._AddSLot2State(slot, -1)
normalised_prob = math.pow(prob, self.prob_thres_factor)
self.beliefstate._AddSlotValue2State(slot,value,normalised_prob)
def _UpdateFrame(self):
self.frame = {}
for slot in self.beliefstate.state:
for value, prob in self.beliefstate.state[slot]['values'].items():
if prob >= self.unified_thres:
self._AddSlotValue2Frame(slot,value)
def _AddSlotValue2Frame(self,slot,value):
if slot not in self.frame:
self.frame[slot] = []
if value not in self.frame[slot]:
self.frame[slot].append(value)
def reset(self):
self.frame = {}
#self.frame_prob = {}
self.memory = {}
self.beliefstate.reset()
class msiip_nsvc_tracker(object):
MY_ID = 'msiip_baseline'
def __init__(self, tagsets, ratio_thres = 0.85, \
bs_mode = 'enhance', bs_alpha = 0.0, \
unified_thres = 0.5):
self.tagsets = tagsets
self.frame = {}
self.memory = {}
self.beliefstate = BeliefState(bs_mode, bs_alpha)
self.ratio_thres = ratio_thres
self.unified_thres = unified_thres
self.ratio_thres_factor = math.log(self.unified_thres, self.ratio_thres)
self.rules = DSTC4_rules(tagsets)
self.appLogger = logging.getLogger(self.MY_ID)
def addUtter(self, utter):
self.appLogger.debug('utter_index: %d' % (utter['utter_index']))
output = {'utter_index': utter['utter_index']}
topic = utter['segment_info']['topic']
if topic in self.tagsets:
self._UpdateFrameProb(utter)
self._UpdateFrame()
self.frame = self.rules.prune_frame_label(topic, self.frame)
output['frame_prob'] = copy.deepcopy(self.beliefstate.state)
output['frame_label'] = copy.deepcopy(self.frame)
return output
def _UpdateFrameProb(self, utter):
topic = utter['segment_info']['topic']
if utter['segment_info']['target_bio'] == 'B':
self.frame = {}
self.beliefstate.reset()
if topic in self.tagsets:
transcript = utter['transcript'].replace('Singapore', '')
prob_frame_labels = {}
for slot in self.tagsets[topic]:
for value in self.tagsets[topic][slot]:
ratio = fuzz.partial_ratio(value.lower(), transcript.lower()) * 1.0 / 100
if ratio > self.ratio_thres:
if slot not in prob_frame_labels:
prob_frame_labels[slot] = {'prob':-1, 'values':{}}
if value not in prob_frame_labels[slot]['values'] or ratio > prob_frame_labels[slot]['values'][value]:
prob_frame_labels[slot]['values'][value] = ratio
# normalise prob so as 0.5 is the threshold
self._NormalisProb(prob_frame_labels)
# update belief state
self.beliefstate.AddFrame(prob_frame_labels)
def _NormalisProb(self, prob_frame_labels):
'''
based on the threshold, normalise threshold to 0.5
'''
for slot in prob_frame_labels:
for value, ratio in prob_frame_labels[slot]['values'].items():
prob_frame_labels[slot]['values'][value] = math.pow(ratio, self.ratio_thres_factor)
def _UpdateFrame(self):
self.frame = {}
for slot in self.beliefstate.state:
for value, ratio in self.beliefstate.state[slot]['values'].items():
if ratio >= self.unified_thres:
self._AddSlotValue2Frame(slot,value)
def _AddSlotValue2Frame(self,slot,value):
if slot not in self.frame:
self.frame[slot] = []
if value not in self.frame[slot]:
self.frame[slot].append(value)
def reset(self):
self.frame = {}
self.memory = {}
self.beliefstate.reset()
class msiip_nsvc_tracker(object):
MY_ID = 'msiip_nsvc'
def __init__(self, tagsets, model_dir, ratio_thres = 0.8, max_num = 2, \
slot_prob_thres = 0.6, value_prob_thres = 0.8, \
mode = 'hr', bs_mode = 'enhance', bs_alpha = 0.0, \
unified_thres = 0.5):
self.tagsets = tagsets
self.frame = {}
self.memory = {}
self.beliefstate = BeliefState(bs_mode, bs_alpha)
self.slot_prob_threshold = slot_prob_thres
self.value_prob_threshold = value_prob_thres
self.ratio_thres = ratio_thres
self.unified_thres = unified_thres
self.slot_prob_factor = math.log(self.unified_thres, self.slot_prob_threshold)
self.value_prob_factor = math.log(self.unified_thres, self.value_prob_threshold)
self.ratio_thres_factor = math.log(self.unified_thres, self.ratio_thres)
self.mode = mode
self.svc = slot_value_classifier()
self.svc.LoadModel(model_dir)
self.tuple_extractor = Tuple_Extractor()
self.rules = DSTC4_rules(tagsets)
self.appLogger = logging.getLogger(self.MY_ID)
if not self.svc.is_set:
self.appLogger.error('Error: Fail to load slot_value_classifier model!')
raise Exception('Error: Fail to load slot_value_classifier model!')
self.value_extractor = value_extractor(tagsets, ratio_thres, max_num)
def addUtter(self, utter):
self.appLogger.debug('utter_index: %d' % (utter['utter_index']))
output = {'utter_index': utter['utter_index']}
topic = utter['segment_info']['topic']
if topic in self.tagsets:
self._UpdateFrameProb(utter)
self._UpdateFrame()
self.frame = self.rules.prune_frame_label(topic, self.frame)
output['frame_prob'] = copy.deepcopy(self.beliefstate.state)
output['frame_label'] = copy.deepcopy(self.frame)
return output
def _UpdateFrameProb(self, utter):
topic = utter['segment_info']['topic']
if utter['segment_info']['target_bio'] == 'B':
self.frame = {}
self.beliefstate.reset()
if topic in self.tagsets:
transcript = utter['transcript']
svc_result, result_prob = self.svc.PredictUtter(utter, self.svc.feature.feature_list)
tuples = []
probs = []
for key in svc_result:
label = svc_result[key]
prob = result_prob[key][1]
if label == 1:
tuples.append(key)
probs.append(prob)
self.appLogger.debug('transcript: %s' %(transcript))
self.appLogger.debug('tuples: %s, probs: %s' %(tuples.__str__(), probs.__str__()))
# get the prob_frame_labels for the single utterance
prob_frame_labels = self.tuple_extractor.generate_frame(tuples, probs, self.mode)
self._AddExtractValue(topic, transcript, prob_frame_labels)
# normalise prob so as 0.5 is the threshold
self._NormalisProb(prob_frame_labels)
# update belief state
self.beliefstate.AddFrame(prob_frame_labels)
def _NormalisProb(self, prob_frame_labels):
'''
based on the threshold, normalise threshold to 0.5
'''
for slot in prob_frame_labels:
if prob_frame_labels[slot]['prob'] != -1:
prob_frame_labels[slot]['prob'] = math.pow(prob_frame_labels[slot]['prob'], self.slot_prob_factor)
if self.tuple_extractor.enumerable(slot):
for value, prob in prob_frame_labels[slot]['values'].items():
prob_frame_labels[slot]['values'][value] = math.pow(prob, self.value_prob_factor)
else:
for value, ratio in prob_frame_labels[slot]['values'].items():
prob_frame_labels[slot]['values'][value] = math.pow(ratio, self.ratio_thres_factor)
def _AddExtractValue(self, topic, transcript, prob_frame_labels):
'''
extract values for non-enumerable slots
'''
for slot in prob_frame_labels:
if slot in self.tagsets[topic]:
if not self.tuple_extractor.enumerable(slot):
value_list = self.value_extractor.ExtractValue(topic, slot, transcript)
for value, ratio in value_list:
prob_frame_labels[slot]['values'][value] = ratio * 1.0 / 100
def _UpdateFrame(self):
self.frame = {}
for slot in self.beliefstate.state:
if self.tuple_extractor.enumerable(slot):
for value, prob in self.beliefstate.state[slot]['values'].items():
if prob >= self.unified_thres:
self._AddSlotValue2Frame(slot,value)
else:
if self.beliefstate.state[slot]['prob'] > self.unified_thres:
for value, ratio in self.beliefstate.state[slot]['values'].items():
if ratio >= self.unified_thres:
self._AddSlotValue2Frame(slot,value)
def _AddSlotValue2Frame(self,slot,value):
if slot not in self.frame:
self.frame[slot] = []
if value not in self.frame[slot]:
self.frame[slot].append(value)
def reset(self):
self.frame = {}
self.memory = {}
self.beliefstate.reset()
def __init__(self, playerNumber, initalState):
self.id = playerNumber
self.out = False
self.belief = BeliefState(initalState, numParticles)
print(f"Belief State for card 0: {self.belief.GetEstimateForCard(0)}")
return
def __init__(self, player, initalState):
self.id = player
self.out = False
self.belief = BeliefState(initalState, numParticles)
return
