class GoalGenerator(object): def __init__(self,data=None,init=False,prep=False): if init: if data == None: print 'Error: needs data' return corpus = lc.Corpus(data,prep=prep) self.goal_table = None self.goal_table = corpus.goal_table() tot = sum(self.goal_table.values()) for k, v in self.goal_table.items(): self.goal_table[k] = float(v)/tot ls.store_model(self.goal_table,'_goal_table.model') ls.store_model(corpus.val_list(),'_value_list.model') else: self.goal_table = ls.load_model('_goal_table.model') self.sampler = MultinomialSampler(self.goal_table) def goal(self): goal_str = self.sampler.sample() goal = ast.literal_eval(goal_str) return goal def show_goal_table(self): for key,value in sorted(self.goal_table.iteritems(),key=lambda(k,v):(v,k)): print "%s: %s" % (key,value)
def __init__(self): self.val_list = ls.load_model('_value_list.model') self.cm = {'bn':ls.load_model('_confusion_matrix_bn.model'),\ 'dp':ls.load_model('_confusion_matrix_p.model'),\ 'ap':ls.load_model('_confusion_matrix_p.model'),\ 'tt':ls.load_model('_confusion_matrix_tt.model')} self.cm_ua_template = [] self.co_cs = [] self.inco_cs = [] self.q_class_max = 4 for c in range(self.q_class_max): self.cm_ua_template.append(ls.load_model('_confusion_matrix_ua_class_%d.model'%c)) self.co_cs.append(ls.load_model('_correct_confidence_score_prob_dist_class_%d.model'%c)) self.inco_cs.append(ls.load_model('_incorrect_confidence_score_prob_dist_class_%d.model'%c)) self.q_class_sampler = MultinomialSampler(ls.load_model('_quality_class.model'))
def __init__(self,data=None,init=False,prep=False): if init: if data == None: print 'Error: needs data' return corpus = lc.Corpus(data,prep=prep) self.goal_table = None self.goal_table = corpus.goal_table() tot = sum(self.goal_table.values()) for k, v in self.goal_table.items(): self.goal_table[k] = float(v)/tot ls.store_model(self.goal_table,'_goal_table.model') ls.store_model(corpus.val_list(),'_value_list.model') else: self.goal_table = ls.load_model('_goal_table.model') self.sampler = MultinomialSampler(self.goal_table)
class ErrorSimulator(object): def __init__(self): self.val_list = ls.load_model('_value_list.model') self.cm = {'bn':ls.load_model('_confusion_matrix_bn.model'),\ 'dp':ls.load_model('_confusion_matrix_p.model'),\ 'ap':ls.load_model('_confusion_matrix_p.model'),\ 'tt':ls.load_model('_confusion_matrix_tt.model')} self.cm_ua_template = [] self.co_cs = [] self.inco_cs = [] self.q_class_max = 4 for c in range(self.q_class_max): self.cm_ua_template.append(ls.load_model('_confusion_matrix_ua_class_%d.model'%c)) self.co_cs.append(ls.load_model('_correct_confidence_score_prob_dist_class_%d.model'%c)) self.inco_cs.append(ls.load_model('_incorrect_confidence_score_prob_dist_class_%d.model'%c)) self.q_class_sampler = MultinomialSampler(ls.load_model('_quality_class.model')) def dialog_init(self,q_class=-1): if q_class == -1: q_class = self.q_class_sampler.sample() print 'Error quality class %d'%q_class self.c_cm_ua_template = self.cm_ua_template[q_class] self.c_co_cs = self.co_cs[q_class] self.c_inco_cs = self.inco_cs[q_class] def get_act(self,ua_ins,goal): import random # err_ua = [] # try: # err_ua_table = self.c_cm_ua_template[','.join(sorted(ua_ins.keys()))] # err_ua_templates = MultinomialSampler(err_ua_table).sample().split(',') # except KeyError: # print traceback.format_exc() # print 'Error template backup' # err_ua_templates = ['non-understanding'] # for k, key in enumerate(ua_ins.keys()): # if not ua_ins[key] == '': # err_ua_table = self.c_cm_ua_template[key] # err_ua_templates = MultinomialSampler(err_ua_table).sample().split(',') # break # cs = 0.0;inc_inco = False # for err_ua_template in err_ua_templates: # try: # act,field,val = err_ua_template.split(':') # print act, field, val # print goal['G_'+field] # if val == 'o' and not goal['G_'+field] == '': # err_val = goal['G_'+field] # cs = MultinomialSampler(self.c_co_cs[field]).sample() # elif val == 'x' and not goal['G_'+field] == '': # try: # err_val = MultinomialSampler(self.cm[field][goal['G_'+field]]).sample() # except KeyError: # err_val = random.sample(self.val_list[field],1)[0] # print 'chose random error' # cs = MultinomialSampler(self.c_inco_cs[field]).sample() # inc_inco = True # else: # err_val = random.sample(self.val_list[field],1)[0] # print 'chose random error' # cs = MultinomialSampler(self.c_inco_cs[field]).sample() # inc_inco = True # except ValueError: # print traceback.format_exc() # err_val = err_ua_template # if err_val in ua_ins and err_val in ['yes','no']: # cs = MultinomialSampler(self.c_co_cs[err_val]).sample() # else: # try: # inc_inco = True # cs = MultinomialSampler(self.c_inco_cs[err_val]).sample() # except KeyError,UnboundLocalError: # print traceback.format_exc() # err_ua.append(':'.join(err_ua_template.split(':')[:-1]+[err_val])) # try: # if len(err_ua) > 1: # if inc_inco: # cs = MultinomialSampler(self.c_inco_cs['multi%d'%len(err_ua)]).sample() # else: # cs = MultinomialSampler(self.c_co_cs['multi%d'%len(err_ua)]).sample() # except: # print traceback.format_exc() # if inc_inco: # cs = MultinomialSampler(self.c_inco_cs['total']).sample() # else: # cs = MultinomialSampler(self.c_co_cs['total']).sample() err_ua = [] if ','.join(sorted(ua_ins.keys())) in self.c_cm_ua_template: err_ua_table = self.c_cm_ua_template[','.join(sorted(ua_ins.keys()))] err_ua_templates = MultinomialSampler(err_ua_table).sample().split(',') else: no_template_key = ','.join(sorted(ua_ins.keys())) print 'no c_cm_ua_template key: ' + ','.join(sorted(ua_ins.keys())) print 'Error template backup' candidates = ['I:bn','I:tt','I:ap','I:dp'] random.shuffle(candidates) for candidate in candidates: if candidate in ua_ins: del ua_ins[candidate] if ','.join(sorted(ua_ins.keys())) in self.c_cm_ua_template: err_ua_table = self.c_cm_ua_template[','.join(sorted(ua_ins.keys()))] err_ua_templates = MultinomialSampler(err_ua_table).sample().split(',') print 'backup from %s to %s'%(no_template_key,','.join(sorted(ua_ins.keys()))) break else: print 'backup fail' err_ua_templates = ['non-understanding'] if len(err_ua_templates) > 1 and 'non-understanding' in err_ua_templates: print 'another action with non-understanding in err_ua_templates' print ua_ins print err_ua_templates print traceback.format_exc() exit() if ua_ins == {'non-understanding':'non-understanding'} and err_ua_templates != ['non-understanding']: print 'non-understanding maps to another action' print err_ua_templates err_ua_templates = ['non-understanding'] # realize errors by sampling from a entity-level confusion matrix # generate confidence score cs = 0.0;inc_inco = False print 'err_ua_templates: ' + str(err_ua_templates) for err_ua_template in err_ua_templates: print 'err_ua_template: ' + str(err_ua_template) if not isinstance(err_ua_template,str): print 'Invalid type: ' + str(err_ua_template) print traceback.format_exc() exit() if len(err_ua_template.split(':')) == 3: act,field,val = err_ua_template.split(':') if val == 'o' and goal['G_'+field] != '': err_val = goal['G_'+field] cs = MultinomialSampler(self.c_co_cs[field]).sample() # actCounts[field]['correct'] += 1 elif val == 'x' and goal['G_'+field] != '': if goal['G_'+field] in self.cm[field]: err_val = MultinomialSampler(self.cm[field][goal['G_'+field]]).sample() # except KeyError: # smoothing confusion matrix for values else: # smoothing confusion matrix for values print 'keyerror field: ' + field print 'keyerror goal: ' + goal['G_'+field] err_val = random.sample(self.val_list[field],1)[0] print 'chose random error #1' cs = MultinomialSampler(self.c_inco_cs[field]).sample() inc_inco = True # actCounts[field]['incorrect'] += 1 else: # no valid goal value for this field, generate a random value err_val = random.sample(self.val_list[field],1)[0] print 'chose random error #2' cs = MultinomialSampler(self.c_inco_cs[field]).sample() inc_inco = True if val == 'o': # actCounts[field]['nogoal_correct'] += 1 err_val = 'CORRECT_' + err_val elif val == 'x': pass # actCounts[field]['nogoal_incorrect'] += 1 else: # for yes,no,non-understanding err_val = err_ua_template if err_val == 'non-understanding': cs = 1.0 elif err_val in ['yes','no']: if err_val in ua_ins: if err_val in self.c_co_cs: cs = MultinomialSampler(self.c_co_cs[err_val]).sample() if cs == 'OOI': print 'not available value key %s'%err_val cs = MultinomialSampler(self.c_co_cs['single']).sample() else: print 'not available correct confidence key %s'%err_val cs = MultinomialSampler(self.c_co_cs['single']).sample() else: if err_val in self.c_inco_cs: cs = MultinomialSampler(self.c_inco_cs[err_val]).sample() if cs == 'OOI': print 'not available value key %s'%err_val cs = MultinomialSampler(self.c_inco_cs['single']).sample() else: print 'not available incorrect confidence key %s'%err_val cs = MultinomialSampler(self.c_inco_cs['single']).sample() inc_inco = True else: print 'unknown action: ' + err_val print traceback.format_exc() exit() err_ua.append(':'.join(err_ua_template.split(':')[:-1]+[err_val])) if len(err_ua) > 1: if inc_inco: if 'multi%d'%len(err_ua) in self.c_inco_cs: cs = MultinomialSampler(self.c_inco_cs['multi%d'%len(err_ua)]).sample() else: cs = MultinomialSampler(self.c_inco_cs['total']).sample() else: if 'multi%d'%len(err_ua) in self.c_co_cs: cs = MultinomialSampler(self.c_co_cs['multi%d'%len(err_ua)]).sample() else: cs = MultinomialSampler(self.c_co_cs['total']).sample() return err_ua,cs