def __init__(self, opt): self.second_resp = opt.get('second_response') self.examples_idx_stack_path = os.path.join( os.getcwd(), './{}_examples_stack{}.pkl'.format( 'second_response' if self.second_resp else 'first_response', '_sandbox' if opt['is_sandbox'] else '', ), ) self.OLD = OffensiveStringMatcher() self.opt = opt build_pc(opt) build_ic(opt) df = 'personality_captions' if not self.second_resp else 'image_chat' data_path = os.path.join(self.opt['datapath'], '{}/{}.json') self.data = [] for dt in ['train', 'val', 'test']: if self.second_resp and dt == 'val': dt = 'valid' with open(data_path.format(df, dt)) as f: self.data += json.load(f) if self.second_resp: self.data = [d for d in self.data if len(d['dialog']) > 1] if os.path.exists(self.examples_idx_stack_path): with open(self.examples_idx_stack_path, 'rb') as handle: self.idx_stack = pickle.load(handle) else: self.idx_stack = [] self.add_idx_stack() self.save_idx_stack()
def __init__(self, opt): self.opt = opt handle = './examples_stack{}{}{}.pkl'.format( '_sandbox' if opt['is_sandbox'] else '', opt['compare_key_1'], opt['compare_key_2']) self.examples_idx_stack_path = os.path.join(os.getcwd(), handle) build_pc(opt) data_path = opt.get('eval_data_path') with open(data_path) as f: self.data = json.load(f) if os.path.exists(self.examples_idx_stack_path): with open(self.examples_idx_stack_path, 'rb') as handle: self.idx_stack = pickle.load(handle) else: self.idx_stack = [] self.add_idx_stack() self.save_idx_stack()