def getAdditionalIntermediate(stateList, step, viewPoint, detail=False):
# 指定したステップ番号のデータを取得する
bState = stateList[step]
# 取得したデータに対して次の状態を予測する
aState = manager.predictwithViewPoint(bState, viewPoint)
# 予測した状態に最も近い状態を stateList から推定する
output = None
tempdiff = 1000000000
for state in stateList:
if state["step"] <= bState["step"]:
continue
curError = manager.calcDifference(state, aState)
if detail == True:
debugLine = "[DynamicalProgramming]getAdditionalIntermediate:"
debugLine += "step:" + str(state["step"])
debugLine += "\t\terror:" + str(curError)
print(debugLine)
# もしtempdiff より小さくなったなら更新する
if curError < tempdiff:
tempdiff = curError
output = state
# もし tempdiff + RANGE 以上になったなら，
# もうその先で更新の見込みはないとして終了する
# elif curError >= tempdiff + 1500:
elif curError >= 5 * tempdiff:
print("TOKEN")
break
return output
def getWorstData(stateDict, detail=False):
if detail == True:
print("[predictMatching]getWorstData:start")
# TODO 暫定的に「ステップ数の遠いデータにはペナルティ」としよう
# 本質的ではないので他の方法を考える
bMean = int(
sum([b["step"]
for b in stateDict["before"]]) / len(stateDict["before"]))
aMean = int(
sum([a["step"]
for a in stateDict["after"]]) / len(stateDict["before"]))
output = {}
output["score"] = 0
output["worstIndex"] = -1
output["worstScore"] = 0
for i in range(len(stateDict["before"])):
if detail == True:
log = "step " + str(i) + "/" + str(len(stateDict["before"]))
print("[predictMatching]getWorstData:" + log)
# 学習データをコピーし，テストデータ分をポップ
tempDict = copy.deepcopy(stateDict)
tempTest = {}
tempTest["before"] = tempDict["before"].pop(i)
tempTest["after"] = tempDict["after"].pop(i)
tempTest["fname"] = tempDict["fname"].pop(i)
tempTest["isadd"] = tempDict["isadd"].pop(i)
# 学習データで学習を行う
vp = manager.getViewPoint(tempDict)
# 学習した観点でテストデータの推定を行う
predicted = manager.predictwithViewPoint(tempTest["before"], vp)
# 推定結果と実際の状態とのずれを計算する
error = manager.calcDifference(predicted, tempTest["after"])
# TODO ステップ数のずれを考慮する
# TODO 他の方法を考える
tempsteprate = 2
error += tempsteprate * (tempTest["before"]["step"] -
bMean) * (tempTest["before"]["step"] - bMean)
error += tempsteprate * (tempTest["after"]["step"] -
aMean) * (tempTest["after"]["step"] - aMean)
# ずれが最大を更新したら記録する
output["score"] += error
if error > output["worstScore"]:
if detail == True:
print("[predictMatching]getWorstData: updateWorst")
output["worstIndex"] = i
output["worstScore"] = error
# score は平均化
output["score"] /= len(stateDict["before"])
if detail == True:
print("[predictMatching]getWorstData:score:" + str(output["score"]))
return output
def test_pick0and100(self):
output = True
filepaths = glob.glob("tmp/log_MakerMain/*")
# データ取得
datas = manager.getStateswithViewPoint(filepaths, [], [])
stateDict = {}
stateDict["before"] = []
stateDict["after"] = []
for count, d in enumerate(datas):
stateDict["before"].append(datas[d][0])
stateDict["after"].append(datas[d][100])
if count >= 49:
break
for _ in range(3):
tempDict = {}
tempDict["before"] = []
tempDict["after"] = []
for count, d in enumerate(datas):
tempDict["before"].append(datas[d][0])
tempDict["after"].append(datas[d][100])
if count >= 49:
break
if len(stateDict["before"]) != len(tempDict["before"]):
output = False
break
for i in range(len(stateDict["before"])):
output = output and (stateDict["before"][i]["step"]
== tempDict["before"][i]["step"])
if output == False:
break
self.assertTrue(output)
def test_ViewPointManager_getStatewithViewPoint(self):
output = True
filepaths = glob.glob("tmp/log_MakerMain/*")
# データ取得
datas = manager.getStateswithViewPoint(filepaths, [], [])
for _ in range(3):
temp = manager.getStateswithViewPoint(filepaths, [], [])
for fname in datas:
if len(datas[fname]) != len(temp[fname]):
output = False
break
for i in range(len(datas[fname])):
for o in datas[fname][i]:
a = datas[fname][i][o]
b = temp[fname][i][o]
if type(a) == type(np.array([])):
output = output and (np.allclose(a, b))
else:
output = output and (a == b)
if output == False:
break
self.assertTrue(output)
def pruningInterDict_2(datas, interDict):
output = {}
print(interDict)
for filename in interDict.keys():
output[filename] = []
before = None
after = None
for step in interDict[filename]:
after = datas[filename][step]
# 目測で大体 4500 前後
if len(output[filename]) == 0 or manager.calcDifference(before, after) > 4500:
output[filename].append(step)
before = datas[filename][step]
return output
def run():
if os.path.exists(DIR_PATH) == False:
os.mkdir(DIR_PATH)
print("start MAINSYU")
filepaths = glob.glob("tmp/log_MakerMain/*")
dirname = "3-2000-10-0.11-200-5-5-False-100"
# dirname = "CHEATNANODA_results"
# データ取得
datas = manager.getStateswithViewPoint(filepaths, [], [])
pm = ParamManager("MAINSYU")
pm.printREADME("MAINSYU")
p = pm.firstParams()
paramNum = len(p)
count = 0
while True:
print("[MAINSYU]iteration:" + str(count))
count += 1
if p == NOPARAMS:
break
print("---------- : make DP_main_results.dill")
with open("tmp/log_MakerMain/dills/DP_main_results.dill", "wb") as f:
# 指定したディレクトリの境界情報を取得
interdict = matching.getInterDict(dirname)
interdict = matching.pruningInterDict(interdict)
dill.dump(matching.DP_main(datas, interdict, \
sampleSize=pm.pick(p, "sampleSize"), \
n_iter=pm.pick(p, "n_iter"),\
distError=pm.pick(p, "distError")), f)
print("+++++----- : save Histogram as png")
summarizer.run([DIR_PATH] +
[p[name] for name in pm.getParamNameList()])
print("++++++++++ : Finished")
p = pm.nextParams()
# kirei で result.csv を p, r, f に集計
for y in range(paramNum):
for x in range(y):
kirei.execute(x, y)
# coding = utf-8
# log データを可視化してチェックするだけ
import glob
import os
import HDP_HMM.MakerMain as makerMain
import DPM.ViewPointEncoder as encoder
import DPM.ViewPointManager as manager
fpaths = glob.glob("tmp/log_MakerMain/*")
for fpath in fpaths:
if os.path.basename(fpath)[:3] != "log" \
or fpath[-4:] != ".csv":
continue
if int(os.path.basename(fpath)[-6:-4]) > 2:
continue
with open(fpath, "r", encoding="utf-8") as f:
count = 0
while True:
line = f.readline()
if line == "":
break
if count%100 == 0:
manager.show(encoder.encodeState(line), title=fpath)
count += 1
def DP_main(datas, interDict):
output = {"matching": [], "pending": []}
rests = copy.deepcopy(interDict)
# 境界が 0 番, 500番以外にないデータはここで除外する
temprests = {}
for r in rests:
if len(rests[r]) > 2:
temprests[r] = rests[r]
rests = temprests
# 最初の stateDict を作る
stateDict = {"before": [], "after": [], "fname": [], "isadd": []}
for fname in rests:
fdata = datas[fname]
stateDict["before"].append(fdata[rests[fname].pop(0)])
stateDict["after"].append(fdata[rests[fname].pop(0)])
stateDict["fname"].append(fname)
stateDict["isadd"].append(False)
while True:
# rests が空なら終了
flg = True
for fname in rests:
if len(rests[fname]) != 0:
flg = False
break
if flg == True:
break
# サンプリングを用いて vp を取得
vp = manager.getViewPointwothSampling(stateDict)
# これをもとに predicts を作成
predicts = {}
moved = manager.getMovedObject(stateDict)
for i in range(len(stateDict["before"])):
predicts[stateDict["fname"][i]] = getAdditionalIntermediate(
datas[pending["fname"][i]], pending["before"][i]["step"], vp)
# predicts と after の関係から，動作を決定する
matching = {"before": [], "after": [], "fname": [], "isadd": []}
# ignored = {"before":[], "after":[], "fname":[], "isadd":[]}
# pending = {"before":[], "after":[], "fname":[], "isadd":[]}
pending = {}
for i in range(len(stateDict["before"])):
# predicts と after の差が score の一定倍程度なら
# matching として保存
if manager.calcDifference(predicts[stateDict["fname"][i]],
stateDict["after"],
objs=moved) < vp["score"] * 0.8:
matching["before"].append(stateDict["before"][i])
matching["after"].append(predicts[stateDict["fname"][i]])
matching["fname"].append(stateDict["fname"])
matching["isadd"].append(stateDict["isadd"])
# 新手法は matching すべてに対して predicts を適用していくという
# ものなので，ignored に対して特別な処理はしない
# pending であるもののみ，rests に返却するために保存する
elif predicts[stateDict["fname"]
[i]]["step"] < stateDict["after"][i]["step"]:
pending[stateDict["fname"][i]] = stateDict["after"][i]["step"]
# rests に pending を返却
for fname in pending:
rests[fname] = [pending[fname]] + rests[fname]
# output に matching と pending を追加
output["matching"].append(matching)
output["pending"].append(pending)
# 次の stateDict を作る
stateDict = {"before": [], "after": [], "fname": [], "isadd": []}
for fname in rests:
fdata = datas[fname]
stateDict["before"].append(predicts[fname])
stateDict["after"].append(fdata[rests[fname].pop(0)])
stateDict["fname"].append(fname)
stateDict["isadd"].append(False)
for fname in stateDict["fname"]:
# すでに境界状態を使い果たした（本来そうならないでほしいが）fnameを表示して無視
if len(rests[fname]) == 0:
print("[predictMatching]DP_main:Empty - " + fname)
for i in range(len(stateDict["fname"])):
if stateDict["fname"][i] == fname:
stateDict["before"].pop(i)
stateDict["fname"].pop(i)
break
continue
return output
if len(rests[fname]) == 0:
print("[predictMatching]DP_main:Empty - " + fname)
for i in range(len(stateDict["fname"])):
if stateDict["fname"][i] == fname:
stateDict["before"].pop(i)
stateDict["fname"].pop(i)
break
continue
return output
if __name__ == "__main__":
filepaths = glob.glob("tmp/forTest_predictMatching/*")
# filepaths = glob.glob("tmp/log_MakerMain/*")
# データ取得
datas = manager.getStateswithViewPoint(filepaths, [], [])
stateDict = {}
stateDict["before"] = []
stateDict["after"] = []
stateDict["fname"] = []
stateDict["isadd"] = []
for count, d in enumerate(sorted(list(datas.keys()))):
stateDict["before"].append(datas[d][0])
stateDict["after"].append(datas[d][200])
stateDict["fname"].append(d)
stateDict["isadd"].append(False)
if count >= 249:
break
if count >= 239:
stateDict["before"].append(datas[d][0])
def DP_main_2(datas, interDict, sampleSize=0.5, n_iter=50, distError=0.005):
output = {"matching": [], "pending": []}
rests = copy.deepcopy(interDict)
# 境界が 0 番, 500番以外にないデータはここで除外する
temprests = {}
for r in rests:
if len(rests[r]) > 2:
temprests[r] = rests[r]
rests = temprests
for r in rests:
print(rests[r])
# 最初の stateDict を作る
stateDict = {"before": [], "after": [], "fname": [], "isadd": []}
for fname in rests:
fdata = datas[fname]
stateDict["before"].append(fdata[rests[fname].pop(0)])
stateDict["after"].append(fdata[rests[fname].pop(0)])
stateDict["fname"].append(fname)
stateDict["isadd"].append(False)
while True:
# rests が空なら終了
flg = True
for fname in rests:
if len(rests[fname]) != 0:
flg = False
break
if flg == True:
break
# サンプリングを用いて vp を取得
vp = manager.getViewPointwithSampling(stateDict, sampleSize, n_iter)
print("get vp:" + str(vp[0]["base"]) + "," + str(vp[0]["ref"]))
# これをもとに predicts を作成
predicts = {}
moved = manager.getMovedObject(stateDict)
for i in range(len(stateDict["before"])):
# step を使う場合は -1
predicts[stateDict["fname"][i]] = getAdditionalIntermediate(
datas[stateDict["fname"][i]],
stateDict["before"][i]["step"] - 1, vp)
# predicts と after の関係から，動作を決定する
matching = {"before": [], "after": [], "fname": [], "isadd": []}
pending = {}
for i in range(len(stateDict["before"])):
# predicts と after の差が score の一定倍程度なら
# matching として保存
matching["before"].append(stateDict["before"][i])
matching["after"].append(predicts[stateDict["fname"][i]])
matching["fname"].append(stateDict["fname"][i])
matching["isadd"].append(stateDict["isadd"][i])
# 新手法は matching すべてに対して predicts を適用していくという
# ものなので，ignored に対して特別な処理はしない
# pending であるもののみ，rests に返却するために保存する
print("aaa" + str(predicts[stateDict["fname"][i]]))
print("bbb" + str(stateDict["after"][i]))
if manager.calcDifference(
predicts[stateDict["fname"][i]],
stateDict["after"][i],
objs=moved) >= vp[0]["score"] * distError and predicts[
stateDict["fname"]
[i]]["step"] < stateDict["after"][i]["step"]:
pending[stateDict["fname"][i]] = stateDict["after"][i]["step"]
# rests に pending を返却
for fname in pending:
# インデックスのずれを修正するために -1
rests[fname] = [pending[fname] - 1] + rests[fname]
# output に matching と pending を追加
output["matching"].append(matching)
output["pending"].append(pending)
with open("tmp/log_MakerMain/dills/DP_main_2_temp.dill", "wb") as f:
dill.dump(output, f)
print("new Matching")
print([[matching["before"][i]["step"], matching["after"][i]["step"]]
for i in range(len(matching["after"]))])
print("new Pending")
print([pending[fname] for fname in pending])
# 次の stateDict を作る
stateDict = {"before": [], "after": [], "fname": [], "isadd": []}
for fname in rests:
fdata = datas[fname]
stateDict["before"].append(predicts[fname])
if len(rests[fname]) > 0:
stateDict["after"].append(fdata[rests[fname].pop(0)])
else:
print("EMPTY - " + fname)
stateDict["after"].append(fdata[-1])
stateDict["fname"].append(fname)
stateDict["isadd"].append(False)
return output
def DP_sub(datas, stateDict):
output = {}
matching = {"before": [], "after": [], "fname": [], "isadd": []}
ignored = {"before": [], "after": [], "fname": [], "isadd": []}
pending = {"before": [], "after": [], "fname": [], "isadd": []}
# 閾値以下になるまでworstState を抽出
matching = copy.deepcopy(stateDict)
while True:
result = getWorstData(matching)
# if result["score"] < THRESHOLD:
rate = result["worstScore"] / result["score"]
print("[predictMatching]DP_sub:rate:" + str(rate))
if rate < THRESHOLD:
break
elif len(matching) < 3:
print("失敗よ！ばか！")
exit()
pending["before"].append(matching["before"].pop(result["worstIndex"]))
pending["after"].append(matching["after"].pop(result["worstIndex"]))
pending["fname"].append(matching["fname"].pop(result["worstIndex"]))
pending["isadd"].append(matching["isadd"].pop(result["worstIndex"]))
print("[predictMatching]DP_sub:pended:" +
str(pending["after"][-1]["step"]))
# 除去した状態のbefore でgetAdditional
vp = manager.getViewPoint(matching)
additionals = []
for i in range(len(pending["before"])):
additionals.append(
getAdditionalIntermediate(datas[pending["fname"][i]],
pending["before"][i]["step"], vp))
temptext = "[predictMatching]DP_sub:additionals-(before, after, additional):("
temptext += str(pending["before"][i]["step"]) + ", "
temptext += str(pending["after"][i]["step"]) + ", "
temptext += str(additionals[-1]["step"]) + ")"
print(temptext)
# additional のステップが after のステップよりも手前なら保留データ
for i in range(len(additionals)):
# 無視だろうが保留だろうが，additionals したデータは適切なので
# additionals を用いて matching に復帰
matching["before"].append(pending["before"][i])
matching["after"].append(additionals[i])
matching["fname"].append(pending["fname"][i])
matching["isadd"].append(True)
# additional のステップが after のステップよりも奥なら無視データ
if additionals[i]["step"] > pending["after"][i]["step"]:
ignored["before"].append(pending["before"][i])
ignored["after"].append(pending["after"][i])
ignored["fname"].append(pending["fname"][i])
ignored["isadd"].append(pending["isadd"][i])
# pending[after] を消しておいて，あとで
# 「消されてないのは保留データ」というように処理する
pending["after"][i] = None
temppend = {"before": [], "after": [], "fname": [], "isadd": []}
for i in range(len(pending["before"])):
if pending["after"][i] is not None:
# temppend["before"].append(pending["before"][i])
# pending の前状態は additionals で更新
temppend["before"].append(additionals[i])
temppend["after"].append(pending["after"][i])
temppend["fname"].append(pending["fname"][i])
temppend["isadd"].append(True)
pending = temppend
# 返す
output["matching"] = matching
output["ignored"] = ignored
output["pending"] = pending
return output
if len(rests[fname]) > 0:
stateDict["after"].append(fdata[rests[fname].pop(0)])
else:
print("EMPTY - " + fname)
stateDict["after"].append(fdata[-1])
stateDict["fname"].append(fname)
stateDict["isadd"].append(False)
return output
if __name__ == "__main__":
# filepaths = glob.glob("tmp/forTest_predictMatching/*")
filepaths = glob.glob("tmp/log_MakerMain/*")
# データ取得
datas = manager.getStateswithViewPoint(filepaths, [], [])
"""
# 0 番と 100 番のデータのみを取り出してみる
stateDict = {}
stateDict["before"] = []
stateDict["after"] = []
flag = False
for d in datas:
stateDict["before"].append(datas[d][0])
stateDict["after"].append(datas[d][100])
# 一個だけ, 200番と 300番のデータを入れてみる
if flag == False:
stateDict["before"].append(datas[d][200])
stateDict["after"].append(datas[d][300])
flag = True
worstData = getWorstData(stateDict)
def reproduction(initState, matchingDict):
manager.show(initState)
state = copy.deepcopy(initState)
for vp in matchingDict["viewpoint"]:
state = manager.predictwithViewPoint(state, vp)
manager.show(state)
def DP_main(datas, interDict_, sampleSize=0.7, n_iter=50, distError=50):
interDict = pruningInterDict_2(datas, interDict_)
output = {"matching":[], "viewpoint":[]}
rests = copy.deepcopy(interDict)
# 最初の stateDict を作る
stateDict = {}
for fname in rests.keys():
fdata = datas[fname]
stateDict[fname] = fdata[rests[fname].pop(0)]
# matching に登録する
output["matching" ].append(stateDict)
output["viewpoint"].append(None)
while True:
# rests が空なら終了
flg = True
for fname in rests.keys():
if len(rests[fname]) != 0:
flg = False
break
if flg == True:
break
# 次の stateDict を作る
stateDict = {}
for fname in rests.keys():
fdata = datas[fname]
while True:
if len(rests[fname])>0 and rests[fname][0] < output["matching"][-1][fname]["step"]+10:
rests[fname].pop(0)
else:
break
if len(rests[fname]) > 0:
stateDict[fname] = fdata[rests[fname].pop(0)]
else:
# print("EMPTY - " + fname)
stateDict[fname] = fdata[-1]
# サンプリングを用いて vp を取得
fnameList = sorted(list(stateDict.keys()))
beforeDict = output["matching"][-1]
beforeList = [beforeDict[fn] for fn in fnameList]
afterList = [stateDict[fn] for fn in fnameList]
tempDict = {"before":beforeList, "after":afterList, "fname":fnameList}
vp = manager.getViewPointwithSampling(tempDict,sampleSize,n_iter)
print("get vp:" + str(vp["base"]) + "," + str(vp["ref"]))
matching = {}
moved = manager.getMovedObject(tempDict)
for fname in stateDict.keys():
# vp を用いて，境界の推定をやり直す
bStep = output["matching"][-1][fname]["step"]
addInter = getAdditionalIntermediate(datas[fname], bStep, vp)
# matching の更新
matching[fname] = addInter
# output に matching と pending と vp を追加
output["matching" ].append(matching)
output["viewpoint"].append(vp)
with open("tmp/log_MakerMain/dills/DP_main_temp.dill", "wb") as f:
dill.dump(output, f)
return output
# coding = utf-8
import DPM.ViewPointEncoder as encoder
import DPM.ViewPointManager as manager
interList = []
path="tmp/log_MakerMain/GettingIntermediated/3-2500-2500-9/inter000000000.csv"
with open(path) as f:
while True:
line = f.readline()
if line == "":
break
interList.append(int(line.split(",")[0]))
print("inters:"+str(interList))
with open("tmp/log_MakerMain/log000000000.csv", "r", encoding="utf-8") as f:
counter = 0
while True:
counter+=1
line = f.readline()
if line == "":
break
elif counter%6 != 0 and counter not in interList:
continue
state = encoder.encodeState(line)
name = "log000000000_"+str(counter)+".png"
inter = counter in interList
manager.savefig(state, name=name, log=True, inter=inter)