def printTierResults(tierName, res, color):
if res is None:
return
print("{tierName} Loss: {loss}, {tierName} accuracy: {acc}".format(tierName = tierName,
loss = bcolored(res["loss"], color),
acc = bcolored(res["acc"], color)))
printAnalysis(res)
def loadWeights(sess, saver, init):
if config.restoreEpoch > 0 or config.restore:
# restore last epoch only if restoreEpoch isn't set
if config.restoreEpoch == 0:
# restore last logged epoch
config.restoreEpoch, config.lr = lastLoggedEpoch()
print(bcolored("Restoring epoch {} and lr {}".format(config.restoreEpoch, config.lr),"cyan"))
print(bcolored("Restoring weights", "blue"))
saver.restore(sess, config.weightsFile(config.restoreEpoch))
epoch = config.restoreEpoch
else:
print(bcolored("Initializing weights", "blue"))
sess.run(init)
logInit()
epoch = 0
return epoch
def statsToStr(stats, res, epoch, batchNum, dataLen, startTime):
formatStr = "\reb {epoch},{batchNum} ({dataProcessed} / {dataLen:5d}), " + \
"t = {time} ({loadTime:2.2f}+{trainTime:2.2f}), " + \
"lr {lr}, l = {loss}, a = {acc}, avL = {avgLoss}, " + \
"avA = {avgAcc}, g = {gradNorm:2.4f}, " + \
"emL = {emaLoss:2.4f}, emA = {emaAcc:2.4f}; " + \
"{expname}" # {machine}/{gpu}"
s_epoch = bcolored("{:2d}".format(epoch),"green")
s_batchNum = "{:3d}".format(batchNum)
s_dataProcessed = bcolored("{:5d}".format(stats["totalData"]),"green")
s_dataLen = dataLen
s_time = bcolored("{:2.2f}".format(time.time() - startTime),"green")
s_loadTime = res["readTime"]
s_trainTime = res["trainTime"]
s_lr = bold(config.lr)
s_loss = bcolored("{:2.4f}".format(res["loss"]), "blue")
s_acc = bcolored("{:2.4f}".format(res["acc"]),"blue")
s_avgLoss = bcolored("{:2.4f}".format(stats["loss"]), "blue")
s_avgAcc = bcolored("{:2.4f}".format(stats["acc"]),"red")
s_gradNorm = res["gradNorm"]
s_emaLoss = stats["emaLoss"]
s_emaAcc = stats["emaAcc"]
s_expname = config.expName
# s_machine = bcolored(config.dataPath[9:11],"green")
# s_gpu = bcolored(config.gpus,"green")
return formatStr.format(epoch = s_epoch, batchNum = s_batchNum, dataProcessed = s_dataProcessed,
dataLen = s_dataLen, time = s_time, loadTime = s_loadTime,
trainTime = s_trainTime, lr = s_lr, loss = s_loss, acc = s_acc,
avgLoss = s_avgLoss, avgAcc = s_avgAcc, gradNorm = s_gradNorm,
emaLoss = s_emaLoss, emaAcc = s_emaAcc, expname = s_expname)
def main():
with open(config.configFile(), "a+") as outFile:
json.dump(vars(config), outFile)
# set gpus
if config.gpus != "":
config.gpusNum = len(config.gpus.split(","))
os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
tf.logging.set_verbosity(tf.logging.ERROR)
# process data
print(bold("Preprocess data..."))
start = time.time()
preprocessor = Preprocesser()
data, embeddings, answerDict = preprocessor.preprocessData()
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# build model
print(bold("Building model..."))
start = time.time()
model = MACnet(embeddings, answerDict, raw=config.raw_image)
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# initializer
init = tf.global_variables_initializer()
# savers
savers = setSavers(model)
saver, emaSaver, resnet_saver = \
savers["saver"], savers["emaSaver"], savers['resnet_saver']
# sessionConfig
sessionConfig = setSession()
with tf.Session(config=sessionConfig) as sess:
# ensure no more ops are added after model is built
sess.graph.finalize()
# restore / initialize weights, initialize epoch variable
epoch = loadWeights(sess, saver, init, resnet_saver)
if config.train:
start0 = time.time()
bestEpoch = epoch
bestRes = None
prevRes = None
# epoch in [restored + 1, epochs]
for epoch in range(config.restoreEpoch + 1, config.epochs + 1):
print(bcolored("Training epoch {}...".format(epoch), "green"))
start = time.time()
# train
# calle = lambda: model.runEpoch(), collectRuntimeStats, writer
trainingData, alterData = chooseTrainingData(data)
trainRes = runEpoch(sess,
model,
trainingData,
train=True,
epoch=epoch,
saver=saver,
alterData=alterData,
raw=config.raw_image)
# save weights
saver.save(sess, config.weightsFile(epoch))
if config.saveSubset:
subsetSaver.save(sess, config.subsetWeightsFile(epoch))
# load EMA weights
if config.useEMA:
print(bold("Restoring EMA weights"))
emaSaver.restore(sess, config.weightsFile(epoch))
# evaluation
evalRes = runEvaluation(sess,
model,
data["main"],
epoch,
raw=config.raw_image)
extraEvalRes = runEvaluation(sess,
model,
data["extra"],
epoch,
evalTrain=not config.extraVal,
raw=config.raw_image)
# restore standard weights
if config.useEMA:
print(bold("Restoring standard weights"))
saver.restore(sess, config.weightsFile(epoch))
print("")
epochTime = time.time() - start
print("took {:.2f} seconds".format(epochTime))
# print results
printDatasetResults(trainRes, evalRes, extraEvalRes)
# stores predictions and optionally attention maps
if config.getPreds:
print(bcolored("Writing predictions...", "white"))
writePreds(preprocessor, evalRes, extraEvalRes)
logRecord(epoch, epochTime, config.lr, trainRes, evalRes,
extraEvalRes)
# update best result
# compute curr and prior
currRes = {
"train": trainRes,
"val": evalRes["val"],
'test': evalRes['test']
}
curr = {"res": currRes, "epoch": epoch}
if bestRes is None or better(currRes, bestRes):
bestRes = currRes
bestEpoch = epoch
prior = {
"best": {
"res": bestRes,
"epoch": bestEpoch
},
"prev": {
"res": prevRes,
"epoch": epoch - 1
}
}
# lr reducing
if config.lrReduce:
if not improveEnough(curr, prior, config.lr):
config.lr *= config.lrDecayRate
print(
colored("Reducing LR to {}".format(config.lr),
"red"))
# early stopping
if config.earlyStopping > 0:
if epoch - bestEpoch > config.earlyStopping:
break
# update previous result
prevRes = currRes
# reduce epoch back to the last one we trained on
epoch -= 1
print("Training took {:.2f} seconds ({:} epochs)".format(
time.time() - start0, epoch - config.restoreEpoch))
if config.finalTest:
print("Testing on epoch {}...".format(epoch))
start = time.time()
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
evalRes = runEvaluation(sess,
model,
data["main"],
epoch,
evalTest=True,
raw=config.raw_image)
extraEvalRes = runEvaluation(sess,
model,
data["extra"],
epoch,
evalTrain=not config.extraVal,
evalTest=True,
raw=config.raw_image)
print("took {:.2f} seconds".format(time.time() - start))
printDatasetResults(None, evalRes, extraEvalRes)
print("Writing predictions...")
writePreds(preprocessor, evalRes, extraEvalRes)
print(bcolored("Done!", "white"))
def main():
# Access the CLODUAMQP_URL environment variable and parse it (fallback to localhost)
url = os.environ.get('CLOUDAMQP_URL')
params = pika.URLParameters(url)
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue='rpc_queue')
################################ input ################################
global cap
if config.input_video == "camera":
# camera input
cap = cv2.VideoCapture(0)
H, W = 480, 640
cap.set(cv2.CAP_PROP_FRAME_WIDTH, W)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, H)
else:
# static image
cap = config.input_video
################################ feature extraction in PyTorch ################################
global fextract_model
fextract_model = build_model()
################################ MAC model in TF ################################
global preprocessor, sess, mac_model
with open(config.configFile(), "a+") as outFile:
json.dump(vars(config), outFile)
# set gpus
if config.gpus != "":
config.gpusNum = len(config.gpus.split(","))
os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
tf.logging.set_verbosity(tf.logging.ERROR)
assert config.batchSize == 1, 'Error: always batch size of one'
# process data
print(bold("Preprocess data..."))
start = time.time()
preprocessor = Preprocesser()
data, embeddings, answerDict = preprocessor.preprocessData(hasTrain=False,
hasDebug=True)
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# build model
print(bold("Building model..."))
start = time.time()
mac_model = MACnet(embeddings, answerDict)
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# initializer
init = tf.global_variables_initializer()
# savers
savers = setSavers(mac_model)
saver, emaSaver = savers["saver"], savers["emaSaver"]
# sessionConfig
sessionConfig = setSession()
sess = tf.Session(config=sessionConfig)
# ensure no more ops are added after model is built
sess.graph.finalize()
# restore / initialize weights, initialize epoch variable
epoch = loadWeights(sess, saver, init)
print("Testing on epoch {}...".format(epoch))
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
################################ RPC Queue ################################
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue='rpc_queue', on_message_callback=on_request)
print(" [x] Awaiting RPC requests")
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
sess.close()
if config.input_video == "camera":
cap.release()
def main():
with open(config.configFile(), "a+") as outFile:
json.dump(vars(config), outFile)
tf.set_random_seed(config.tfseed)
# set gpus
if config.gpus != "":
config.gpusNum = len(config.gpus.split(","))
os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
tf.logging.set_verbosity(tf.logging.ERROR)
# process data
print(bold("Preprocess data..."))
start = time.time()
preprocessor = Preprocesser()
data, embeddings, answerDict, questionDict = preprocessor.preprocessData()
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
nextElement = None
dataOps = None
# build model
print(bold("Building model..."))
start = time.time()
model = MACnet(embeddings, answerDict, questionDict, nextElement)
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# initializer
init = tf.global_variables_initializer()
# savers
savers = setSavers(model)
saver, emaSaver = savers["saver"], savers["emaSaver"]
# sessionConfig
sessionConfig = setSession()
with tf.Session(config=sessionConfig) as sess:
# ensure no more ops are added after model is built
sess.graph.finalize()
# restore / initialize weights, initialize epoch variable
epoch = loadWeights(sess, saver, init)
trainRes, evalRes = None, None
if config.train:
start0 = time.time()
bestEpoch = epoch
bestRes = None
prevRes = None
# epoch in [restored + 1, epochs]
for epoch in range(config.restoreEpoch + 1, config.epochs + 1):
print(bcolored("Training epoch {}...".format(epoch), "green"))
start = time.time()
# train
# calle = lambda: model.runEpoch(), collectRuntimeStats, writer
trainingData, alterData = chooseTrainingData(data)
trainRes = runEpoch(
sess,
model,
trainingData,
dataOps,
train=True,
epoch=epoch,
saver=saver,
alterData=alterData,
maxAcc=trainRes["maxAcc"] if trainRes else 0.0,
minLoss=trainRes["minLoss"] if trainRes else float("inf"),
)
# save weights
saver.save(sess, config.weightsFile(epoch))
if config.saveSubset:
subsetSaver.save(sess, config.subsetWeightsFile(epoch))
# load EMA weights
if config.useEMA:
print(bold("Restoring EMA weights"))
emaSaver.restore(sess, config.weightsFile(epoch))
# evaluation
getPreds = config.getPreds or (config.analysisType != "")
evalRes = runEvaluation(sess,
model,
data["main"],
dataOps,
epoch,
getPreds=getPreds,
prevRes=evalRes)
extraEvalRes = runEvaluation(sess,
model,
data["extra"],
dataOps,
epoch,
evalTrain=not config.extraVal,
getPreds=getPreds)
# restore standard weights
if config.useEMA:
print(bold("Restoring standard weights"))
saver.restore(sess, config.weightsFile(epoch))
print("")
epochTime = time.time() - start
print("took {:.2f} seconds".format(epochTime))
# print results
printDatasetResults(trainRes, evalRes, extraEvalRes)
# stores predictions and optionally attention maps
if config.getPreds:
print(bcolored("Writing predictions...", "white"))
writePreds(preprocessor, evalRes, extraEvalRes)
logRecord(epoch, epochTime, config.lr, trainRes, evalRes,
extraEvalRes)
# update best result
# compute curr and prior
currRes = {"train": trainRes, "val": evalRes["val"]}
curr = {"res": currRes, "epoch": epoch}
if bestRes is None or better(currRes, bestRes):
bestRes = currRes
bestEpoch = epoch
prior = {
"best": {
"res": bestRes,
"epoch": bestEpoch
},
"prev": {
"res": prevRes,
"epoch": epoch - 1
}
}
# lr reducing
if config.lrReduce:
if not improveEnough(curr, prior, config.lr):
config.lr *= config.lrDecayRate
print(
colored("Reducing LR to {}".format(config.lr),
"red"))
# early stopping
if config.earlyStopping > 0:
if epoch - bestEpoch > config.earlyStopping:
break
# update previous result
prevRes = currRes
# reduce epoch back to the last one we trained on
epoch -= 1
print("Training took {:.2f} seconds ({:} epochs)".format(
time.time() - start0, epoch - config.restoreEpoch))
if config.finalTest:
print("Testing on epoch {}...".format(epoch))
start = time.time()
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
evalRes = runEvaluation(sess,
model,
data["main"],
dataOps,
epoch,
evalTest=False,
getPreds=True)
extraEvalRes = runEvaluation(sess,
model,
data["extra"],
dataOps,
epoch,
evalTrain=not config.extraVal,
evalTest=False,
getPreds=True)
print("took {:.2f} seconds".format(time.time() - start))
printDatasetResults(trainRes, evalRes, extraEvalRes)
print("Writing predictions...")
writePreds(preprocessor, evalRes, extraEvalRes)
if config.interactive:
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
tier = config.interactiveTier
images = data["main"][tier]["images"]
imgsInfoFilename = config.imgsInfoFile(tier)
with open(imgsInfoFilename, "r") as file:
imageIndex = json.load(file)
openImageFiles(images)
resInter = {"preds": []}
while True:
text = inp("Enter <imageId>_<question>\n")
if len(text) == 0:
break
imageId, questionStr = text.split("_")
imageInfo = imageIndex[imageId]
imageId = {
"group": tier,
"id": imageId,
"idx": imageInfo["idx"]
} # int(imageId)
question = preprocessor.encodeQuestionStr(questionStr)
instance = {
"questionStr": questionStr,
"question": question,
"answer": "yes", # Dummy answer
"answerFreq": ["yes"], # Dummy answer
"imageId": imageId,
"tier": tier,
"index": 0
}
if config.imageObjects:
instance["objectsNum"] = imageInfo["objectsNum"]
print(instance)
datum = preprocessor.vectorizeData([instance])
image = loadImageBatch(images, {"imageIds": [imageId]})
res = model.runBatch(sess,
datum,
image,
train=False,
getPreds=True,
getAtt=True)
resInter["preds"].append(instance)
print(instance["prediction"])
if config.getPreds:
print(bcolored("Writing predictions...", "white"))
preprocessor.writePreds(resInter, "interactive".format())
closeImageFiles(images)
print(bcolored("Done!", "white"))
def question_answer(n_clicks, imageId, questionStr):
instance = None
print(questionStr)
if questionStr is not None:
questionStr = str(questionStr)
print(n_clicks, imageId, questionStr)
with tf.Session(config=sessionConfig) as sess:
# ensure no more ops are added after model is built
sess.graph.finalize()
# restore / initialize weights, initialize epoch variable
epoch = loadWeights(sess, saver, init)
trainRes, evalRes = None, None
if config.interactive:
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
tier = config.interactiveTier
images = data["main"][tier]["images"]
imgsInfoFilename = config.imgsInfoFile(tier)
with open(imgsInfoFilename, "r") as file:
imageIndex = json.load(file)
openImageFiles(images)
resInter = {"preds": []}
imageInfo = imageIndex[imageId]
print(imageInfo)
imageix = imageId
imageId = {
"group": tier,
"id": imageId,
"idx": imageInfo["index"]
} # int(imageId)
question = preprocessor.encodeQuestionStr(questionStr)
instance = {
"questionStr": questionStr,
"question": question,
"answer": "yes", # Dummy answer
"answerFreq": ["yes"], # Dummy answer
"imageId": imageId,
"tier": tier,
"index": 0
}
if config.imageObjects:
instance["objectsNum"] = imageInfo["objectsNum"]
datum = preprocessor.vectorizeData([instance])
image = loadImageBatch(images, {"imageIds": [imageId]})
res = model.runBatch(sess,
datum,
image,
train=False,
getPreds=True,
getAtt=True)
resInter["preds"].append(instance)
if config.getPreds:
print(bcolored("Writing predictions...", "white"))
preprocessor.writePreds(resInter, "interactive".format())
closeImageFiles(images)
# get bboxes
bboxes = data_obs['bboxes'][data_obs_mi[str(imageix)]['index']]
#images
fig, axes = plt.subplots(
4,
2,
figsize=(20, 20),
gridspec_kw={'height_ratios': [4, 1, 4, 1]})
for row in [0, 2]:
for col in [0, 1]:
image_ax = axes[row, col]
word_ax = axes[row + 1, col]
step = row + col + 1
attentions_objects = instance['attentions']['kb'][step
- 1]
generate_image_map(imageix, bboxes, attentions_objects,
image_ax, step)
q_vector = instance['question']
attentions_question = instance['attentions'][
'question'][step - 1][0:len(q_vector)]
generate_question_map(q_vector, attentions_question,
word_ax)
src = fig_to_uri(fig)
else:
src = 'None'
return 'Prediction {}: {}'.format(n_clicks, instance["prediction"]), src
json.dump(vars(config), outFile)
# set gpus
# if config.gpus != "":
# config.gpusNum = len(config.gpus.split(","))
# os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
tf.logging.set_verbosity(tf.logging.ERROR)
# process data
print(bold("Preprocess data..."))
start = time.time()
preprocessor = Preprocesser()
data, embeddings, answerDict, questionDict = preprocessor.preprocessData()
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
nextElement = None
dataOps = None
# build model
print(bold("Building model..."))
start = time.time()
model = MACnet(embeddings, answerDict, questionDict, nextElement)
print("took {} seconds".format(
bcolored("{:.2f}".format(time.time() - start), "blue")))
# initializer
init = tf.global_variables_initializer()
# savers
def main():
with open(config.configFile(), "a+") as outFile:
json.dump(vars(config), outFile)
# set gpus
if config.gpus != "":
config.gpusNum = len(config.gpus.split(","))
os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
tf.logging.set_verbosity(tf.logging.ERROR)
assert config.batchSize == 1, 'Error: always batch size of one'
# process data
print(bold("Preprocess data..."))
start = time.time()
preprocessor = Preprocesser()
data, embeddings, answerDict = preprocessor.preprocessData(hasTrain = False, hasDebug = True)
print("took {} seconds".format(bcolored("{:.2f}".format(time.time() - start), "blue")))
# build model
print(bold("Building model..."))
start = time.time()
model = MACnet(embeddings, answerDict)
print("took {} seconds".format(bcolored("{:.2f}".format(time.time() - start), "blue")))
# initializer
init = tf.global_variables_initializer()
# savers
savers = setSavers(model)
saver, emaSaver = savers["saver"], savers["emaSaver"]
# sessionConfig
sessionConfig = setSession()
with tf.Session(config = sessionConfig) as sess:
# ensure no more ops are added after model is built
sess.graph.finalize()
# restore / initialize weights, initialize epoch variable
epoch = loadWeights(sess, saver, init)
print("Testing on epoch {}...".format(epoch))
start = time.time()
if epoch > 0:
if config.useEMA:
emaSaver.restore(sess, config.weightsFile(epoch))
else:
saver.restore(sess, config.weightsFile(epoch))
#evalRes = runEvaluation(sess, model, data["main"], epoch, evalTest = False, evalDebug = True)
#####################################################
imagesFilename = './FRIDGR_v0.1/data/debug.h5'
datasetFilename = './FRIDGR_v0.1/data/FRIDGR_debug_questions.json'
images = {"imagesFilename": imagesFilename}
q = ["Do I have meat?",
"How many meats?",
"Do I have egg?",
"Do I have donut?",
"Do I have beer?"]
a = [True, 2, True, True, False]
with open(datasetFilename, "r") as datasetFile:
data = json.load(datasetFile)["questions"]
for i in range(5):
#evalRes = runDemo(sess, model, data["main"], epoch)
# data to instance:
instance = data[20]
question = q[i] #instance["question"]
questionSeq = preprocessor.tokenize(question)
answer = a[i] #instance.get("answer", "yes") # DUMMY_ANSWER
dummyProgram = [{"function": "FUNC", "value_inputs": [], "inputs": []}]
program = instance.get("program", dummyProgram)
postfixProgram = program #preprocessor.programTranslator.programToPostfixProgram(program)
programSeq = preprocessor.programTranslator.programToSeq(postfixProgram)
programInputs = preprocessor.programTranslator.programToInputs(postfixProgram, offset = 2)
print('DEBUG1:', question, questionSeq, answer, instance["image_index"])
#print('DEBUG2:', dummyProgram)
#print('DEBUG3:', program)
# pass other fields to instance?
instances = []
instances.append({
"question": question,
"questionSeq": questionSeq,
"answer": answer,
"imageId": instance["image_index"],
"program": program,
"programSeq": programSeq,
"programInputs": programInputs,
"index": 0
})
dataset = {"debug": None}
buckets = preprocessor.vectorizeData(instances)
#print('buckets =', buckets)
dataset["debug"] = {"data": buckets, "images": images, "train": False}
evalRes = runDemo(sess, model, dataset, epoch)
pred = evalRes["debug"]["preds"][0]["prediction"]
print('Answer/Predict is', answer, pred)
#print('ID/Text decoded is', i, preprocessor.answerDict.decodeId(pred))
#####################################################
#extraEvalRes = None
#print("took {:.2f} seconds".format(time.time() - start))
#printDatasetResults(None, evalRes, extraEvalRes)
#print("Writing predictions...")
#writePreds(preprocessor, evalRes, extraEvalRes)
print(bcolored("Done!","white"))
