def ask():
global my_demo
global my_demo2
global my_demo3
global my_demo4
global mod
global order
units = request.form['units'].split(",")
question = request.form['question']
image = request.form['image']
disp = request.form['disp']
if disp == "lxmert_full_12heads_768hdims":
head_mask = empty_mask(12)
mod = [("lang", 9, 12), ("vis", 5, 12), ("vl", 5, 12), ("lv", 5, 12), ("vv", 5, 12), ("ll", 5, 12)]
order = makeOrder(mod)
else:
mod = [("lang", 9, 4), ("vis", 5, 4), ("vl", 5, 4), ("lv", 5, 4), ("vv", 5, 4), ("ll", 5, 4)]
order = makeOrder(mod)
head_mask = empty_mask(4)
if units is not None and not units == ['']:
for elem in units:
temp = elem.split("_")
# print(temp)
head_mask[temp[0]][int(temp[1])][int(temp[2])] = 1
if disp == "lxmert_tiny":
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size = my_demo2.ask(question, image,
head_mask)
elif disp == "lxmert_tiny_init_oracle_pretrain":
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size = my_demo3.ask(question, image,
head_mask)
elif disp == "lxmert_full_12heads_768hdims":
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size = my_demo4.ask(question, image,
head_mask)
else:
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size = my_demo.ask(question, image,
head_mask)
k_vals = toSliptDict(k_dist)
five = {}
for u in range(len(five_predictions)):
five[five_predictions[u][0]] = five_predictions[u][1].item()
heats = purgeHeats(AtttoSliptDict(attention_heads), input_size)
resp = Response(response=ujson.dumps({
"k_dist": k_vals,
"five": five,
"labels": input_labels,
"heatmaps": heats
}),
status=200,
mimetype="application/json")
return resp
def RCNNStats():
res = {"total": [[], []]}
ok = []
fail = []
spoon = []
with open("static/assets/data/info.json", 'r') as fjson:
imgs = ujson.load(fjson)
head_mask = empty_mask()
item = "knife"
for k, v in imgs.items():
if hasItem(v["scene"]["objects"], item):
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size \
= my_demo2.ask(v["questions"]["0"]["question"], k, head_mask)
if item in input_labels["visual"]:
ok.append(k)
else:
if "spoon" in input_labels["visual"] or "fork" in input_labels["visual"]:
spoon.append(k)
else:
fail.append(k)
# TODO CHECK if input labels has item
# print(ok)
# print(spoon)
# print(fail)
pass
def stater(mod, name, disp):
global order
global my_demo
if mod["head"] == 4:
my_demo.cfg["tiny_lxmert"] = 1
else:
my_demo.cfg["tiny_lxmert"] = 0
if name == "oracle":
my_demo.cfg["oracle"] = 1
my_demo.cfg['data_split'] = 'val'
else:
my_demo.cfg["oracle"] = 0
my_demo.cfg['data_split'] = 'testdev'
order = makeOrder([("lang", mod["lang"], mod["head"]),
("vis", mod["vis"], mod["head"]),
("vl", mod["cross"], mod["head"]),
("lv", mod["cross"], mod["head"]),
("vv", mod["cross"], mod["head"]),
("ll", mod["cross"], mod["head"])])
# print(args)
my_demo.initConf("model/src/pretrain/" + disp)
my_demo.load_data()
my_demo.load_model()
res = {}
for elem in order:
res[elem] = {"functs": {}, "groups": {}}
with open("static/assets/data/images.json", 'r') as fjson:
imgs = ujson.load(fjson)
for im in range(len(imgs["default"])):
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size \
= my_demo.ask(dataset[im].question, imgs["default"][im], empty_mask(12))
temp = toSliptD(k_dist)
for elem in order:
for func in dataset[im]["operations"]: # here change
if res[elem]["functs"][func] is not None:
res[elem]["functs"][func]["median"].append(temp[0])
res[elem]["functs"][func]["min"].append(temp[1])
res[elem]["functs"][func]["max"].append(temp[2])
else:
res[elem]["functs"][func]["median"] = [temp[0]]
res[elem]["functs"][func]["min"] = [temp[1]]
res[elem]["functs"][func]["max"] = [temp[2]]
with open('%s.json' % "info2", 'w') as wjson:
ujson.dump(res, wjson, ensure_ascii=False, sort_keys=True, indent=4)
def ask():
global my_demo
units = request.form['units'].split(",")
question = request.form['question']
image = request.form['image']
head_mask = empty_mask(12)
if units is not None and not units == ['']:
for elem in units:
temp = elem.split("_")
# print(temp)
head_mask[temp[0]][int(temp[1])][int(temp[2])] = 1
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size \
= my_demo.ask(question, image, head_mask)
# print(input_size["textual"])
k_vals = toSliptDict(k_dist)
five = {}
for u in range(len(five_predictions)):
five[five_predictions[u][0]] = five_predictions[u][1].item()
# print(five)
for k, v in alignment.items():
alignment[k]["xywh"] = alignment[k]["xywh"].tolist()
# print(input_labels)
# print(image)
return ujson.dumps({
# "pred": top_prediction[0],
# "confidence": top_prediction[1].item(),
"alignment":
alignment,
# "coords": umaper.transform([formatK_dist(k_dist)]).tolist(),
"k_dist":
k_vals,
"five":
five,
"labels":
input_labels,
"heatmaps":
purgeHeats(AtttoSliptDict(attention_heads), input_size)
})
def stackDat():
with open("static/assets/data/info.json", "r") as datFile:
data = ujson.load(datFile)
res = {}
head_mask = empty_mask(12)
for elem in order:
res[elem] = {"functions": {}, "groups": {}, "kmeds": [[], [], []]}
# skip = [51,54,101,138,151,204,206]
img = 0
for k, v in data.items():
print("doing img ..", img, " with id:", k)
# if img in skip:
# print("SKIPPED")
# img += 1
# continue
for k2, v2 in v["questions"].items():
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size \
= my_demo.ask(v2["question"], k, head_mask)
k_vals = toSliptDict(k_dist)
for k3, v3 in k_vals.items():
for op in v2["operations"]:
if not op in res[k3]["functions"]:
res[k3]["functions"][op] = [
k_split([0, 0, 0, 0], v3[0]),
k_split([0, 0, 0, 0], v3[1]),
k_split([0, 0, 0, 0], v3[2])
]
else:
res[k3]["functions"][op][0] = k_split(
res[k3]["functions"][op][0], v3[0])
res[k3]["functions"][op][1] = k_split(
res[k3]["functions"][op][1], v3[1])
res[k3]["functions"][op][2] = k_split(
res[k3]["functions"][op][2], v3[2])
if not v2["groups"]["global"] is None:
if not v2["groups"]["global"] in res[k3]["groups"]:
res[k3]["groups"][v2["groups"]["global"]] = [
k_split([0, 0, 0, 0], v3[0]),
k_split([0, 0, 0, 0], v3[1]),
k_split([0, 0, 0, 0], v3[2])
]
else:
tab = res[k3]["groups"][v2["groups"]["global"]]
tab[0] = k_split(tab[0], v3[0])
tab[0] = k_split(tab[1], v3[1])
tab[0] = k_split(tab[2], v3[1])
res[k3]["kmeds"][0].append(int(v3[0]))
res[k3]["kmeds"][1].append(int(v3[1]))
res[k3]["kmeds"][2].append(int(v3[2]))
# if not v2["functions"] in res[k3]["functions"]:
# res[k3]["functions"][v2["functions"]] = k_split([0, 0, 0, 0], v3)
# else:
# res[k3]["functions"][v2["functions"]] = k_split(res[k3]["functions"][v2["functions"]], v3)
#
# if not v2["groups"]["global"] is None:
# if not v2["groups"]["global"] in res[k3]["groups"]:
# res[k3]["groups"][v2["groups"]["global"]] = k_split([0, 0, 0, 0], v3)
# else:
# res[k3]["groups"][v2["groups"]["global"]] = k_split(
# res[k3]["groups"][v2["groups"]["global"]], v3)
# res[k3]["kmeds"].append(int(v3))
img += 1
with open('%s.json' % "tiny_oracle_full", 'w') as wjson:
ujson.dump(res,
wjson,
ensure_ascii=False,
sort_keys=True,
indent=4)
def pruneStats():
head_mask = empty_mask()
# units = ["lang_4_0", "lang_6_1", "lang_6_0", "lang_5_0", "lang_3_3"]
# units = ["lang_8_1", "lang_7_1", "lang_7_2", "lang_6_0", "lang_7_3", "lang_6_3", "lang_8_0"]
# units = ["lv_0_0", "lv_0_1", "lv_0_2", "lv_0_3","lang_8_2","lang_8_3","lang_8_1","lang_8_0"]
units = ["lang_8_0", "lang_6_2", "lang_6_3", "lang_6_1", "lang_6_0"]
# units = ["vl_2_3", "lang_1_2", "lang_2_1", "lang_3_2", "lang_5_2", "lang_6_3", "lang_6_0", "lang_7_0", "lang_8_3",
# "lang_8_2", "lang_7_2", "lang_6_2", "lang_6_1", "lang_5_3", "lang_4_0", "lang_3_3", "lang_2_0", "lang_1_1",
# "ll_0_0"]
head_mask2 = empty_mask()
stats1 = [0, 0, 0]
stats2 = [0, 0, 0]
jf = [0, 0]
jf2 = [0, 0]
ref = ["H", "M", "T"]
if units is not None and not units == ['']:
for elem in units:
temp = elem.split("_")
# print(temp)
head_mask[temp[0]][int(temp[1])][int(temp[2])] = 1
with open("static/assets/data/info.json", 'r') as fjson:
imgs = ujson.load(fjson)
for k, v in imgs.items():
for k2, v2 in v["questions"].items():
# print(v2)
if "and" in v2["operations"]:
# if "tail" == v2["ood"]: # "relate" in v2["operations"]:
# temp = v2["question"].split(" ")
#
# rof = temp.index("and")
# # print(temp)
# bob = temp[rof - 1]
#
# if "?" in temp[rof + 1]:
# temp[rof - 1] = temp[rof + 1].replace("?","")
# temp[rof + 1] = bob+"?"
# else:
# temp[rof - 1] = temp[rof + 1]
# temp[rof + 1] = bob
# v2["question"] =" ".join(temp)
five_predictions, attention_heads, alignment, k_dist, input_labels, input_size = my_demo3.ask(
v2["question"], k, head_mask2)
five_predictions2, _, _, _, _, _ = my_demo3.ask(
v2["question"], k, head_mask)
ood1 = getOod(v2, five_predictions[0][0])
ood2 = getOod(v2, five_predictions2[0][0])
if v2["answer"] == five_predictions[0][0]:
jf[0] += 1
else:
jf[1] += 1
if v2["answer"] == five_predictions2[0][0]:
jf2[0] += 1
else:
jf2[1] += 1
stats1[ref.index(ood1)] += 1
# if ood2 == "T":
# print(five_predictions2)
# print(k, " --- ", v2["question"], ' --- ', five_predictions2[0][0], " --conf-- ",
# five_predictions2[0][1].item())
stats2[ref.index(ood2)] += 1