Ejemplo n.º 1
0
def vis(model, batch_data, batch_output, first_input):
    fields = []
    fields.append(batch_data[0].raw_query)
    fields.append(" ".join([STRING_INDEX.get(w) for w in batch_data[0].string[1:-1]]))
    fields.append("<img src='../../%s' />" % batch_data[0].image_path)

    if hasattr(model, "apollo_net") and hasattr(model, "attention_layer"):
        att_data = model.apollo_net \
                        .blobs[model.attention_layer] \
                        .data[0] \
                        .reshape((20, 20))[:first_input.shape[1], 
                                           :first_input.shape[2]]
        fields.append(att_data)

    if hasattr(model, "apollo_net") and hasattr(model, "answer_layer"):
        preds = np.argsort(-model.apollo_net.blobs[model.answer_layer].data[0,...])
        predstrs = ", ".join([ANSWER_INDEX.get(p) for p in preds[:KBEST]])
        fields.append(predstrs)

    fields.append(ANSWER_INDEX.get(batch_output[0]))
    visualizer.show(fields)
Ejemplo n.º 2
0
    def __init__(self, config, set_name, size=None):
        self.config = config
        self.size = size

        data_by_id = dict()
        data_by_layout_type = defaultdict(list)

        if set_name == "test":
            self.by_id = data_by_id
            self.by_layout_type = data_by_layout_type
            self.layout_types = set()
            return

        with open(STRING_FILE % set_name) as question_f, \
             open(PARSE_FILE % set_name) as parse_f:
            questions = json.load(question_f)["questions"]
            parses = [l.strip() for l in parse_f]
            assert len(questions) == len(parses)
            pairs = zip(questions, parses)
            if size is not None:
                pairs = pairs[:size]
            for question, parse_str in pairs:
                id = question["question_id"]
                indexed_question = \
                    [STRING_INDEX.index(w) for w in question["question"].split()]
                parse = parse_tree(parse_str)
                layout = parse_to_layout(parse)
                if layout is None:
                    continue
                image_id = question["image_id"]
                #if os.path.exists(IMAGE_FILE % (set_name, set_name, image_id)):
                try:
                    datum = ImageDatum( id, indexed_question, layout, set_name, image_id, [])
                    data_by_id[id] = datum
                except IOError as e:
                    pass
                    

        with open(ANN_FILE % set_name) as ann_f:
            annotations = json.load(ann_f)["annotations"]
            for ann in annotations:
                question_id = ann["question_id"]
                if question_id not in data_by_id:
                    continue

                answer_counter = defaultdict(lambda: 0)
                for ans in ann["answers"]:
                    ans_words = ans["answer"]
                    if " " in ans_words or "/" in ans_words or "," in ans_words:
                        continue
                    answer_counter[ans_words] += 1
                    #ans_indexed = ANSWER_INDEX.index(ans_words)
                    #data_by_id[question_id].outputs.append(ans_indexed)

                counted_answers = [(count, word) for word, count in
                        answer_counter.items()]
                sorted_answers = sorted(counted_answers)
                if len(sorted_answers) == 0:
                    del data_by_id[question_id]
                    continue
                best_count = sorted_answers[-1][0]
                if best_count == 1:
                    del data_by_id[question_id]
                    continue
                best_answer = sorted_answers[-1][1]
                best_answer_indexed = ANSWER_INDEX.index(best_answer)
                data_by_id[question_id].outputs.append(best_answer_indexed)

        for datum in data_by_id.values():
            data_by_layout_type[datum.layout.modules].append(datum)

        self.by_id = data_by_id
        self.by_layout_type = data_by_layout_type
        self.layout_types = data_by_layout_type.keys()

        logging.info("%s:", set_name.upper())
        logging.info("%s items", len(self.by_id))
        logging.info("%d answers", len(ANSWER_INDEX))
        logging.info("%d layouts", len(self.layout_types))
        logging.info("")
Ejemplo n.º 3
0
    def __init__(self, config, set_name, size=None):
        self.config = config
        self.size = size

        data_by_id = dict()
        data_by_layout_type = defaultdict(list)

        if set_name == "test":
            self.by_id = data_by_id
            self.by_layout_type = data_by_layout_type
            self.layout_types = set()
            return

        with open(STRING_FILE % set_name) as question_f, \
             open(PARSE_FILE % set_name) as parse_f:
            questions = json.load(question_f)["questions"]
            parses = [l.strip() for l in parse_f]
            assert len(questions) == len(parses)
            pairs = zip(questions, parses)
            if size is not None:
                pairs = pairs[:size]
            for question, parse_str in pairs:
                id = question["question_id"]
                indexed_question = \
                    [STRING_INDEX.index(w) for w in question["question"].split()]
                parse = parse_tree(parse_str)
                layout = parse_to_layout(parse)
                if layout is None:
                    continue
                image_id = question["image_id"]
                #if os.path.exists(IMAGE_FILE % (set_name, set_name, image_id)):
                try:
                    datum = ImageDatum(id, indexed_question, layout, set_name,
                                       image_id, [])
                    data_by_id[id] = datum
                except IOError as e:
                    pass

        with open(ANN_FILE % set_name) as ann_f:
            annotations = json.load(ann_f)["annotations"]
            for ann in annotations:
                question_id = ann["question_id"]
                if question_id not in data_by_id:
                    continue

                answer_counter = defaultdict(lambda: 0)
                for ans in ann["answers"]:
                    ans_words = ans["answer"]
                    if " " in ans_words or "/" in ans_words or "," in ans_words:
                        continue
                    answer_counter[ans_words] += 1
                    #ans_indexed = ANSWER_INDEX.index(ans_words)
                    #data_by_id[question_id].outputs.append(ans_indexed)

                counted_answers = [(count, word)
                                   for word, count in answer_counter.items()]
                sorted_answers = sorted(counted_answers)
                if len(sorted_answers) == 0:
                    del data_by_id[question_id]
                    continue
                best_count = sorted_answers[-1][0]
                if best_count == 1:
                    del data_by_id[question_id]
                    continue
                best_answer = sorted_answers[-1][1]
                best_answer_indexed = ANSWER_INDEX.index(best_answer)
                data_by_id[question_id].outputs.append(best_answer_indexed)

        for datum in data_by_id.values():
            data_by_layout_type[datum.layout.modules].append(datum)

        self.by_id = data_by_id
        self.by_layout_type = data_by_layout_type
        self.layout_types = data_by_layout_type.keys()

        logging.info("%s:", set_name.upper())
        logging.info("%s items", len(self.by_id))
        logging.info("%d answers", len(ANSWER_INDEX))
        logging.info("%d layouts", len(self.layout_types))
        logging.info("")
Ejemplo n.º 4
0
    def __init__(self, config, set_name, filter_file=None):
        self.config = config
        size = config.train_size

        data = set()
        data_by_layout_type = defaultdict(list)
        data_by_string_length = defaultdict(list)
        data_by_layout_and_length = defaultdict(list)

        with open(STRING_FILE % (size, set_name)) as question_f, \
             open(PARSE_FILE % (size, set_name)) as parse_f, \
             open(ANN_FILE % (size, set_name)) as ann_f:

            img_filter = None
            if filter_file is not None:
                img_filter = set()
                with open(filter_file) as filt_h:
                    for line in filt_h:
                        img_filter.add(line.strip())

                
            for question, parse_str, answer in zip(question_f, parse_f, ann_f):
                question = question.strip()
                parse_str = parse_str.strip()
                #parse_str = "(what object)"
                answer = answer.strip()
                words = question.split()
                image_id = words[-2]
                words = ["<s>"] + words[:-4] + ["</s>"]

                # TODO multi answer
                if "," in answer:
                    continue
                if img_filter is not None and image_id not in img_filter:
                    continue

                answer = ANSWER_INDEX.index(answer)

                indexed_words = [STRING_INDEX.index(w) for w in words]

                parse = parse_tree(parse_str)
                #if parse[0] != "color":
                #    continue
                layout = parse_to_layout(parse)
                datum = DaquarDatum(indexed_words, layout, image_id, answer)

                data.add(datum)
                data_by_layout_type[datum.layout.modules].append(datum)
                data_by_string_length[len(datum.string)].append(datum)
                data_by_layout_and_length[(datum.layout.modules, len(datum.string))].append(datum)

        self.data = data
        self.by_layout_type = data_by_layout_type
        self.by_string_length = data_by_string_length
        self.by_layout_and_length = data_by_layout_and_length

        logging.info("%s:", set_name.upper())
        logging.info("%s items", len(self.data))
        logging.info("%s words", len(STRING_INDEX))
        logging.info("%s functions", len(LAYOUT_INDEX))
        logging.info("%s answers", len(ANSWER_INDEX))
        logging.info("%s layouts", len(self.by_layout_type.keys()))
        logging.info("")
Ejemplo n.º 5
0
    def __init__(self, config, set_name):
        self.config = config

        data = set()
        data_by_layout_type = defaultdict(list)
        data_by_string_length = defaultdict(list)
        data_by_layout_and_length = defaultdict(list)

        if set_name == "val":
            self.data = data
            self.by_layout_type = data_by_layout_type
            self.by_string_length = data_by_string_length
            self.by_layout_and_length = data_by_layout_and_length
            return

        if set_name == "train":
            # TODO better index
            pred_counter = defaultdict(lambda: 0)
            with open(PARSE_FILE % set_name) as parse_f:
                for parse_str in parse_f:
                    parse_preds = parse_str.strip() \
                                           .replace("'", "") \
                                           .replace("(", "") \
                                           .replace(")", "") \
                                           .split()
                    for pred in parse_preds:
                        pred_counter[pred] += 1
            for pred, count in pred_counter.items():
                if count <= 1:
                    continue
                LAYOUT_INDEX.index(pred)

        with open(STRING_FILE % set_name) as question_f, \
             open(PARSE_FILE % set_name) as parse_f, \
             open(ANN_FILE % set_name) as ann_f, \
             open(IMAGE_ID_FILE % set_name) as image_id_f:

            unked = 0
            i = 0
            for question, parse_str, answer, image_id in \
                    zip(question_f, parse_f, ann_f, image_id_f):
            
                question = question.strip()
                parse_str = parse_str.strip().replace("'", "")
                answer = answer.strip()
                image_id = int(image_id.strip())
                words = question.split()
                words = ["<s>"] + words + ["</s>"]

                parse = parse_tree(parse_str)

                answer = ANSWER_INDEX.index(answer)
                words = [STRING_INDEX.index(w) for w in words]
                if len(parse) == 1:
                    parse = parse + ("object",)
                layout = parse_to_layout(parse)

                #if i == 300:
                #    continue
                i += 1

                coco_set_name = "train" if set_name == "train" else "val"
                try:
                    datum = CocoQADatum(words, layout, image_id, answer, coco_set_name)
                    datum.raw_query = parse_str
                    data.add(datum)
                    data_by_layout_type[datum.layout.modules].append(datum)
                    data_by_string_length[len(datum.string)].append(datum)
                    data_by_layout_and_length[(datum.layout.modules, len(datum.string))].append(datum)
                except IOError as e:
                    pass

        self.data = data
        self.by_layout_type = data_by_layout_type
        self.by_string_length = data_by_string_length
        self.by_layout_and_length = data_by_layout_and_length

        logging.info("%s:", set_name.upper())
        logging.info("%s items", len(self.data))
        logging.info("%s words", len(STRING_INDEX))
        logging.info("%s functions", len(LAYOUT_INDEX))
        logging.info("%s answers", len(ANSWER_INDEX))
        logging.info("%s layouts", len(self.by_layout_type.keys()))
        logging.info("")
Ejemplo n.º 6
0
    def __init__(self, config, set_name):
        self.config = config

        data = set()
        data_by_layout_type = defaultdict(list)
        data_by_string_length = defaultdict(list)
        data_by_layout_and_length = defaultdict(list)

        if set_name == "val":
            self.data = data
            self.by_layout_type = data_by_layout_type
            self.by_string_length = data_by_string_length
            self.by_layout_and_length = data_by_layout_and_length
            return

        if set_name == "train":
            # TODO better index
            pred_counter = defaultdict(lambda: 0)
            with open(PARSE_FILE % set_name) as parse_f:
                for parse_str in parse_f:
                    parse_preds = parse_str.strip() \
                                           .replace("'", "") \
                                           .replace("(", "") \
                                           .replace(")", "") \
                                           .split()
                    for pred in parse_preds:
                        pred_counter[pred] += 1
            for pred, count in pred_counter.items():
                if count <= 1:
                    continue
                LAYOUT_INDEX.index(pred)

        with open(STRING_FILE % set_name) as question_f, \
             open(PARSE_FILE % set_name) as parse_f, \
             open(ANN_FILE % set_name) as ann_f, \
             open(IMAGE_ID_FILE % set_name) as image_id_f:

            unked = 0
            i = 0
            for question, parse_str, answer, image_id in \
                    zip(question_f, parse_f, ann_f, image_id_f):

                question = question.strip()
                parse_str = parse_str.strip().replace("'", "")
                answer = answer.strip()
                image_id = int(image_id.strip())
                words = question.split()
                words = ["<s>"] + words + ["</s>"]

                parse = parse_tree(parse_str)

                answer = ANSWER_INDEX.index(answer)
                words = [STRING_INDEX.index(w) for w in words]
                if len(parse) == 1:
                    parse = parse + ("object", )
                layout = parse_to_layout(parse)

                #if i == 300:
                #    continue
                i += 1

                coco_set_name = "train" if set_name == "train" else "val"
                try:
                    datum = CocoQADatum(words, layout, image_id, answer,
                                        coco_set_name)
                    datum.raw_query = parse_str
                    data.add(datum)
                    data_by_layout_type[datum.layout.modules].append(datum)
                    data_by_string_length[len(datum.string)].append(datum)
                    data_by_layout_and_length[(
                        datum.layout.modules, len(datum.string))].append(datum)
                except IOError as e:
                    pass

        self.data = data
        self.by_layout_type = data_by_layout_type
        self.by_string_length = data_by_string_length
        self.by_layout_and_length = data_by_layout_and_length

        logging.info("%s:", set_name.upper())
        logging.info("%s items", len(self.data))
        logging.info("%s words", len(STRING_INDEX))
        logging.info("%s functions", len(LAYOUT_INDEX))
        logging.info("%s answers", len(ANSWER_INDEX))
        logging.info("%s layouts", len(self.by_layout_type.keys()))
        logging.info("")