def parse_to_layout(parse): #if parse not in [("color", "cat"), ("color", "shirt")]: # return None #if not isinstance(parse, tuple): # return None #if parse[0] not in LEGAL_QUERIES: # return None #if isinstance(parse[1], tuple): # return None layout_modules = [None, None] layout_indices = [None, None] if parse[0] in ("is", "is_there", "count"): layout_modules[0] = DenseAnswerModule else: layout_modules[0] = AttAnswerModule #else: # print parse # exit() #elif parse[0] == "count": # layout_modules[0] = DenseAnswerModule layout_indices[0] = LAYOUT_INDEX.index(parse[0]) layout_modules[1] = DetectModule layout_indices[1] = LAYOUT_INDEX.index(parse[1]) layout = Layout(tuple(layout_modules), tuple(layout_indices)) return layout
def parse_to_layout_helper(parse, internal): if isinstance(parse, str): #return (DetectModule, LAYOUT_INDEX.index(parse)) return (DetectModule, LAYOUT_INDEX.get_or_else(parse, LAYOUT_INDEX[UNK])) else: head = parse[0] #head_idx = LAYOUT_INDEX.index(parse) head_idx = LAYOUT_INDEX.get_or_else(head, LAYOUT_INDEX[UNK]) if internal: if head == "and": mod_head = ConjModule else: mod_head = RedetectModule else: if head == "count": mod_head = DenseAnswerModule else: mod_head = AttAnswerModule below = [ parse_to_layout_helper(child, internal=True) for child in parse[1:] ] mods_below, indices_below = zip(*below) return (mod_head, ) + tuple(mods_below), ( head_idx, ) + tuple(indices_below)
def parse_to_layout_helper(parse, internal): if isinstance(parse, str): return (DetectModule, LAYOUT_INDEX.index(parse)) else: head = parse[0] head_idx = LAYOUT_INDEX.index(parse) if internal: if head == "and": mod_head = ConjModule else: mod_head = RedetectModule else: if head == "how many": mod_head = DenseAnswerModule else: mod_head = AttAnswerModule below = [parse_to_layout_helper(child, internal=True) for child in parse[1:]] mods_below, indices_below = zip(*below) return (mod_head,) + tuple(mods_below), (head_idx,) + tuple(indices_below)
def __init__(self, config, set_name): self.config = config data = set() data_by_layout_type = defaultdict(list) data_by_string_length = defaultdict(list) data_by_layout_and_length = defaultdict(list) if set_name == "val": self.data = data self.by_layout_type = data_by_layout_type self.by_string_length = data_by_string_length self.by_layout_and_length = data_by_layout_and_length return if set_name == "train": # TODO better index pred_counter = defaultdict(lambda: 0) with open(PARSE_FILE % set_name) as parse_f: for parse_str in parse_f: parse_preds = parse_str.strip() \ .replace("'", "") \ .replace("(", "") \ .replace(")", "") \ .split() for pred in parse_preds: pred_counter[pred] += 1 for pred, count in pred_counter.items(): if count <= 1: continue LAYOUT_INDEX.index(pred) with open(STRING_FILE % set_name) as question_f, \ open(PARSE_FILE % set_name) as parse_f, \ open(ANN_FILE % set_name) as ann_f, \ open(IMAGE_ID_FILE % set_name) as image_id_f: unked = 0 i = 0 for question, parse_str, answer, image_id in \ zip(question_f, parse_f, ann_f, image_id_f): question = question.strip() parse_str = parse_str.strip().replace("'", "") answer = answer.strip() image_id = int(image_id.strip()) words = question.split() words = ["<s>"] + words + ["</s>"] parse = parse_tree(parse_str) answer = ANSWER_INDEX.index(answer) words = [STRING_INDEX.index(w) for w in words] if len(parse) == 1: parse = parse + ("object",) layout = parse_to_layout(parse) #if i == 300: # continue i += 1 coco_set_name = "train" if set_name == "train" else "val" try: datum = CocoQADatum(words, layout, image_id, answer, coco_set_name) datum.raw_query = parse_str data.add(datum) data_by_layout_type[datum.layout.modules].append(datum) data_by_string_length[len(datum.string)].append(datum) data_by_layout_and_length[(datum.layout.modules, len(datum.string))].append(datum) except IOError as e: pass self.data = data self.by_layout_type = data_by_layout_type self.by_string_length = data_by_string_length self.by_layout_and_length = data_by_layout_and_length logging.info("%s:", set_name.upper()) logging.info("%s items", len(self.data)) logging.info("%s words", len(STRING_INDEX)) logging.info("%s functions", len(LAYOUT_INDEX)) logging.info("%s answers", len(ANSWER_INDEX)) logging.info("%s layouts", len(self.by_layout_type.keys())) logging.info("")
def __init__(self, config, set_name): self.config = config data = set() data_by_layout_type = defaultdict(list) data_by_string_length = defaultdict(list) data_by_layout_and_length = defaultdict(list) if set_name == "val": self.data = data self.by_layout_type = data_by_layout_type self.by_string_length = data_by_string_length self.by_layout_and_length = data_by_layout_and_length return if set_name == "train": # TODO better index pred_counter = defaultdict(lambda: 0) with open(PARSE_FILE % set_name) as parse_f: for parse_str in parse_f: parse_preds = parse_str.strip() \ .replace("'", "") \ .replace("(", "") \ .replace(")", "") \ .split() for pred in parse_preds: pred_counter[pred] += 1 for pred, count in pred_counter.items(): if count <= 1: continue LAYOUT_INDEX.index(pred) with open(STRING_FILE % set_name) as question_f, \ open(PARSE_FILE % set_name) as parse_f, \ open(ANN_FILE % set_name) as ann_f, \ open(IMAGE_ID_FILE % set_name) as image_id_f: unked = 0 i = 0 for question, parse_str, answer, image_id in \ zip(question_f, parse_f, ann_f, image_id_f): question = question.strip() parse_str = parse_str.strip().replace("'", "") answer = answer.strip() image_id = int(image_id.strip()) words = question.split() words = ["<s>"] + words + ["</s>"] parse = parse_tree(parse_str) answer = ANSWER_INDEX.index(answer) words = [STRING_INDEX.index(w) for w in words] if len(parse) == 1: parse = parse + ("object", ) layout = parse_to_layout(parse) #if i == 300: # continue i += 1 coco_set_name = "train" if set_name == "train" else "val" try: datum = CocoQADatum(words, layout, image_id, answer, coco_set_name) datum.raw_query = parse_str data.add(datum) data_by_layout_type[datum.layout.modules].append(datum) data_by_string_length[len(datum.string)].append(datum) data_by_layout_and_length[( datum.layout.modules, len(datum.string))].append(datum) except IOError as e: pass self.data = data self.by_layout_type = data_by_layout_type self.by_string_length = data_by_string_length self.by_layout_and_length = data_by_layout_and_length logging.info("%s:", set_name.upper()) logging.info("%s items", len(self.data)) logging.info("%s words", len(STRING_INDEX)) logging.info("%s functions", len(LAYOUT_INDEX)) logging.info("%s answers", len(ANSWER_INDEX)) logging.info("%s layouts", len(self.by_layout_type.keys())) logging.info("")