def getAdditionalIntermediate(stateList, step, viewPoint, detail=False): # 指定したステップ番号のデータを取得する bState = stateList[step] # 取得したデータに対して次の状態を予測する aState = manager.predictwithViewPoint(bState, viewPoint) # 予測した状態に最も近い状態を stateList から推定する output = None tempdiff = 1000000000 for state in stateList: if state["step"] <= bState["step"]: continue curError = manager.calcDifference(state, aState) if detail == True: debugLine = "[DynamicalProgramming]getAdditionalIntermediate:" debugLine += "step:" + str(state["step"]) debugLine += "\t\terror:" + str(curError) print(debugLine) # もしtempdiff より小さくなったなら更新する if curError < tempdiff: tempdiff = curError output = state # もし tempdiff + RANGE 以上になったなら, # もうその先で更新の見込みはないとして終了する # elif curError >= tempdiff + 1500: elif curError >= 5 * tempdiff: print("TOKEN") break return output
def getWorstData(stateDict, detail=False): if detail == True: print("[predictMatching]getWorstData:start") # TODO 暫定的に「ステップ数の遠いデータにはペナルティ」としよう # 本質的ではないので他の方法を考える bMean = int( sum([b["step"] for b in stateDict["before"]]) / len(stateDict["before"])) aMean = int( sum([a["step"] for a in stateDict["after"]]) / len(stateDict["before"])) output = {} output["score"] = 0 output["worstIndex"] = -1 output["worstScore"] = 0 for i in range(len(stateDict["before"])): if detail == True: log = "step " + str(i) + "/" + str(len(stateDict["before"])) print("[predictMatching]getWorstData:" + log) # 学習データをコピーし,テストデータ分をポップ tempDict = copy.deepcopy(stateDict) tempTest = {} tempTest["before"] = tempDict["before"].pop(i) tempTest["after"] = tempDict["after"].pop(i) tempTest["fname"] = tempDict["fname"].pop(i) tempTest["isadd"] = tempDict["isadd"].pop(i) # 学習データで学習を行う vp = manager.getViewPoint(tempDict) # 学習した観点でテストデータの推定を行う predicted = manager.predictwithViewPoint(tempTest["before"], vp) # 推定結果と実際の状態とのずれを計算する error = manager.calcDifference(predicted, tempTest["after"]) # TODO ステップ数のずれを考慮する # TODO 他の方法を考える tempsteprate = 2 error += tempsteprate * (tempTest["before"]["step"] - bMean) * (tempTest["before"]["step"] - bMean) error += tempsteprate * (tempTest["after"]["step"] - aMean) * (tempTest["after"]["step"] - aMean) # ずれが最大を更新したら記録する output["score"] += error if error > output["worstScore"]: if detail == True: print("[predictMatching]getWorstData: updateWorst") output["worstIndex"] = i output["worstScore"] = error # score は平均化 output["score"] /= len(stateDict["before"]) if detail == True: print("[predictMatching]getWorstData:score:" + str(output["score"])) return output
def test_pick0and100(self): output = True filepaths = glob.glob("tmp/log_MakerMain/*") # データ取得 datas = manager.getStateswithViewPoint(filepaths, [], []) stateDict = {} stateDict["before"] = [] stateDict["after"] = [] for count, d in enumerate(datas): stateDict["before"].append(datas[d][0]) stateDict["after"].append(datas[d][100]) if count >= 49: break for _ in range(3): tempDict = {} tempDict["before"] = [] tempDict["after"] = [] for count, d in enumerate(datas): tempDict["before"].append(datas[d][0]) tempDict["after"].append(datas[d][100]) if count >= 49: break if len(stateDict["before"]) != len(tempDict["before"]): output = False break for i in range(len(stateDict["before"])): output = output and (stateDict["before"][i]["step"] == tempDict["before"][i]["step"]) if output == False: break self.assertTrue(output)
def test_ViewPointManager_getStatewithViewPoint(self): output = True filepaths = glob.glob("tmp/log_MakerMain/*") # データ取得 datas = manager.getStateswithViewPoint(filepaths, [], []) for _ in range(3): temp = manager.getStateswithViewPoint(filepaths, [], []) for fname in datas: if len(datas[fname]) != len(temp[fname]): output = False break for i in range(len(datas[fname])): for o in datas[fname][i]: a = datas[fname][i][o] b = temp[fname][i][o] if type(a) == type(np.array([])): output = output and (np.allclose(a, b)) else: output = output and (a == b) if output == False: break self.assertTrue(output)
def pruningInterDict_2(datas, interDict): output = {} print(interDict) for filename in interDict.keys(): output[filename] = [] before = None after = None for step in interDict[filename]: after = datas[filename][step] # 目測で大体 4500 前後 if len(output[filename]) == 0 or manager.calcDifference(before, after) > 4500: output[filename].append(step) before = datas[filename][step] return output
def run(): if os.path.exists(DIR_PATH) == False: os.mkdir(DIR_PATH) print("start MAINSYU") filepaths = glob.glob("tmp/log_MakerMain/*") dirname = "3-2000-10-0.11-200-5-5-False-100" # dirname = "CHEATNANODA_results" # データ取得 datas = manager.getStateswithViewPoint(filepaths, [], []) pm = ParamManager("MAINSYU") pm.printREADME("MAINSYU") p = pm.firstParams() paramNum = len(p) count = 0 while True: print("[MAINSYU]iteration:" + str(count)) count += 1 if p == NOPARAMS: break print("---------- : make DP_main_results.dill") with open("tmp/log_MakerMain/dills/DP_main_results.dill", "wb") as f: # 指定したディレクトリの境界情報を取得 interdict = matching.getInterDict(dirname) interdict = matching.pruningInterDict(interdict) dill.dump(matching.DP_main(datas, interdict, \ sampleSize=pm.pick(p, "sampleSize"), \ n_iter=pm.pick(p, "n_iter"),\ distError=pm.pick(p, "distError")), f) print("+++++----- : save Histogram as png") summarizer.run([DIR_PATH] + [p[name] for name in pm.getParamNameList()]) print("++++++++++ : Finished") p = pm.nextParams() # kirei で result.csv を p, r, f に集計 for y in range(paramNum): for x in range(y): kirei.execute(x, y)
# coding = utf-8 # log データを可視化してチェックするだけ import glob import os import HDP_HMM.MakerMain as makerMain import DPM.ViewPointEncoder as encoder import DPM.ViewPointManager as manager fpaths = glob.glob("tmp/log_MakerMain/*") for fpath in fpaths: if os.path.basename(fpath)[:3] != "log" \ or fpath[-4:] != ".csv": continue if int(os.path.basename(fpath)[-6:-4]) > 2: continue with open(fpath, "r", encoding="utf-8") as f: count = 0 while True: line = f.readline() if line == "": break if count%100 == 0: manager.show(encoder.encodeState(line), title=fpath) count += 1
def DP_main(datas, interDict): output = {"matching": [], "pending": []} rests = copy.deepcopy(interDict) # 境界が 0 番, 500番以外にないデータはここで除外する temprests = {} for r in rests: if len(rests[r]) > 2: temprests[r] = rests[r] rests = temprests # 最初の stateDict を作る stateDict = {"before": [], "after": [], "fname": [], "isadd": []} for fname in rests: fdata = datas[fname] stateDict["before"].append(fdata[rests[fname].pop(0)]) stateDict["after"].append(fdata[rests[fname].pop(0)]) stateDict["fname"].append(fname) stateDict["isadd"].append(False) while True: # rests が空なら終了 flg = True for fname in rests: if len(rests[fname]) != 0: flg = False break if flg == True: break # サンプリングを用いて vp を取得 vp = manager.getViewPointwothSampling(stateDict) # これをもとに predicts を作成 predicts = {} moved = manager.getMovedObject(stateDict) for i in range(len(stateDict["before"])): predicts[stateDict["fname"][i]] = getAdditionalIntermediate( datas[pending["fname"][i]], pending["before"][i]["step"], vp) # predicts と after の関係から,動作を決定する matching = {"before": [], "after": [], "fname": [], "isadd": []} # ignored = {"before":[], "after":[], "fname":[], "isadd":[]} # pending = {"before":[], "after":[], "fname":[], "isadd":[]} pending = {} for i in range(len(stateDict["before"])): # predicts と after の差が score の一定倍程度なら # matching として保存 if manager.calcDifference(predicts[stateDict["fname"][i]], stateDict["after"], objs=moved) < vp["score"] * 0.8: matching["before"].append(stateDict["before"][i]) matching["after"].append(predicts[stateDict["fname"][i]]) matching["fname"].append(stateDict["fname"]) matching["isadd"].append(stateDict["isadd"]) # 新手法は matching すべてに対して predicts を適用していくという # ものなので,ignored に対して特別な処理はしない # pending であるもののみ,rests に返却するために保存する elif predicts[stateDict["fname"] [i]]["step"] < stateDict["after"][i]["step"]: pending[stateDict["fname"][i]] = stateDict["after"][i]["step"] # rests に pending を返却 for fname in pending: rests[fname] = [pending[fname]] + rests[fname] # output に matching と pending を追加 output["matching"].append(matching) output["pending"].append(pending) # 次の stateDict を作る stateDict = {"before": [], "after": [], "fname": [], "isadd": []} for fname in rests: fdata = datas[fname] stateDict["before"].append(predicts[fname]) stateDict["after"].append(fdata[rests[fname].pop(0)]) stateDict["fname"].append(fname) stateDict["isadd"].append(False) for fname in stateDict["fname"]: # すでに境界状態を使い果たした(本来そうならないでほしいが)fnameを表示して無視 if len(rests[fname]) == 0: print("[predictMatching]DP_main:Empty - " + fname) for i in range(len(stateDict["fname"])): if stateDict["fname"][i] == fname: stateDict["before"].pop(i) stateDict["fname"].pop(i) break continue return output
if len(rests[fname]) == 0: print("[predictMatching]DP_main:Empty - " + fname) for i in range(len(stateDict["fname"])): if stateDict["fname"][i] == fname: stateDict["before"].pop(i) stateDict["fname"].pop(i) break continue return output if __name__ == "__main__": filepaths = glob.glob("tmp/forTest_predictMatching/*") # filepaths = glob.glob("tmp/log_MakerMain/*") # データ取得 datas = manager.getStateswithViewPoint(filepaths, [], []) stateDict = {} stateDict["before"] = [] stateDict["after"] = [] stateDict["fname"] = [] stateDict["isadd"] = [] for count, d in enumerate(sorted(list(datas.keys()))): stateDict["before"].append(datas[d][0]) stateDict["after"].append(datas[d][200]) stateDict["fname"].append(d) stateDict["isadd"].append(False) if count >= 249: break if count >= 239: stateDict["before"].append(datas[d][0])
def DP_main_2(datas, interDict, sampleSize=0.5, n_iter=50, distError=0.005): output = {"matching": [], "pending": []} rests = copy.deepcopy(interDict) # 境界が 0 番, 500番以外にないデータはここで除外する temprests = {} for r in rests: if len(rests[r]) > 2: temprests[r] = rests[r] rests = temprests for r in rests: print(rests[r]) # 最初の stateDict を作る stateDict = {"before": [], "after": [], "fname": [], "isadd": []} for fname in rests: fdata = datas[fname] stateDict["before"].append(fdata[rests[fname].pop(0)]) stateDict["after"].append(fdata[rests[fname].pop(0)]) stateDict["fname"].append(fname) stateDict["isadd"].append(False) while True: # rests が空なら終了 flg = True for fname in rests: if len(rests[fname]) != 0: flg = False break if flg == True: break # サンプリングを用いて vp を取得 vp = manager.getViewPointwithSampling(stateDict, sampleSize, n_iter) print("get vp:" + str(vp[0]["base"]) + "," + str(vp[0]["ref"])) # これをもとに predicts を作成 predicts = {} moved = manager.getMovedObject(stateDict) for i in range(len(stateDict["before"])): # step を使う場合は -1 predicts[stateDict["fname"][i]] = getAdditionalIntermediate( datas[stateDict["fname"][i]], stateDict["before"][i]["step"] - 1, vp) # predicts と after の関係から,動作を決定する matching = {"before": [], "after": [], "fname": [], "isadd": []} pending = {} for i in range(len(stateDict["before"])): # predicts と after の差が score の一定倍程度なら # matching として保存 matching["before"].append(stateDict["before"][i]) matching["after"].append(predicts[stateDict["fname"][i]]) matching["fname"].append(stateDict["fname"][i]) matching["isadd"].append(stateDict["isadd"][i]) # 新手法は matching すべてに対して predicts を適用していくという # ものなので,ignored に対して特別な処理はしない # pending であるもののみ,rests に返却するために保存する print("aaa" + str(predicts[stateDict["fname"][i]])) print("bbb" + str(stateDict["after"][i])) if manager.calcDifference( predicts[stateDict["fname"][i]], stateDict["after"][i], objs=moved) >= vp[0]["score"] * distError and predicts[ stateDict["fname"] [i]]["step"] < stateDict["after"][i]["step"]: pending[stateDict["fname"][i]] = stateDict["after"][i]["step"] # rests に pending を返却 for fname in pending: # インデックスのずれを修正するために -1 rests[fname] = [pending[fname] - 1] + rests[fname] # output に matching と pending を追加 output["matching"].append(matching) output["pending"].append(pending) with open("tmp/log_MakerMain/dills/DP_main_2_temp.dill", "wb") as f: dill.dump(output, f) print("new Matching") print([[matching["before"][i]["step"], matching["after"][i]["step"]] for i in range(len(matching["after"]))]) print("new Pending") print([pending[fname] for fname in pending]) # 次の stateDict を作る stateDict = {"before": [], "after": [], "fname": [], "isadd": []} for fname in rests: fdata = datas[fname] stateDict["before"].append(predicts[fname]) if len(rests[fname]) > 0: stateDict["after"].append(fdata[rests[fname].pop(0)]) else: print("EMPTY - " + fname) stateDict["after"].append(fdata[-1]) stateDict["fname"].append(fname) stateDict["isadd"].append(False) return output
def DP_sub(datas, stateDict): output = {} matching = {"before": [], "after": [], "fname": [], "isadd": []} ignored = {"before": [], "after": [], "fname": [], "isadd": []} pending = {"before": [], "after": [], "fname": [], "isadd": []} # 閾値以下になるまでworstState を抽出 matching = copy.deepcopy(stateDict) while True: result = getWorstData(matching) # if result["score"] < THRESHOLD: rate = result["worstScore"] / result["score"] print("[predictMatching]DP_sub:rate:" + str(rate)) if rate < THRESHOLD: break elif len(matching) < 3: print("失敗よ!ばか!") exit() pending["before"].append(matching["before"].pop(result["worstIndex"])) pending["after"].append(matching["after"].pop(result["worstIndex"])) pending["fname"].append(matching["fname"].pop(result["worstIndex"])) pending["isadd"].append(matching["isadd"].pop(result["worstIndex"])) print("[predictMatching]DP_sub:pended:" + str(pending["after"][-1]["step"])) # 除去した状態のbefore でgetAdditional vp = manager.getViewPoint(matching) additionals = [] for i in range(len(pending["before"])): additionals.append( getAdditionalIntermediate(datas[pending["fname"][i]], pending["before"][i]["step"], vp)) temptext = "[predictMatching]DP_sub:additionals-(before, after, additional):(" temptext += str(pending["before"][i]["step"]) + ", " temptext += str(pending["after"][i]["step"]) + ", " temptext += str(additionals[-1]["step"]) + ")" print(temptext) # additional のステップが after のステップよりも手前なら保留データ for i in range(len(additionals)): # 無視だろうが保留だろうが,additionals したデータは適切なので # additionals を用いて matching に復帰 matching["before"].append(pending["before"][i]) matching["after"].append(additionals[i]) matching["fname"].append(pending["fname"][i]) matching["isadd"].append(True) # additional のステップが after のステップよりも奥なら無視データ if additionals[i]["step"] > pending["after"][i]["step"]: ignored["before"].append(pending["before"][i]) ignored["after"].append(pending["after"][i]) ignored["fname"].append(pending["fname"][i]) ignored["isadd"].append(pending["isadd"][i]) # pending[after] を消しておいて,あとで # 「消されてないのは保留データ」というように処理する pending["after"][i] = None temppend = {"before": [], "after": [], "fname": [], "isadd": []} for i in range(len(pending["before"])): if pending["after"][i] is not None: # temppend["before"].append(pending["before"][i]) # pending の前状態は additionals で更新 temppend["before"].append(additionals[i]) temppend["after"].append(pending["after"][i]) temppend["fname"].append(pending["fname"][i]) temppend["isadd"].append(True) pending = temppend # 返す output["matching"] = matching output["ignored"] = ignored output["pending"] = pending return output
if len(rests[fname]) > 0: stateDict["after"].append(fdata[rests[fname].pop(0)]) else: print("EMPTY - " + fname) stateDict["after"].append(fdata[-1]) stateDict["fname"].append(fname) stateDict["isadd"].append(False) return output if __name__ == "__main__": # filepaths = glob.glob("tmp/forTest_predictMatching/*") filepaths = glob.glob("tmp/log_MakerMain/*") # データ取得 datas = manager.getStateswithViewPoint(filepaths, [], []) """ # 0 番と 100 番のデータのみを取り出してみる stateDict = {} stateDict["before"] = [] stateDict["after"] = [] flag = False for d in datas: stateDict["before"].append(datas[d][0]) stateDict["after"].append(datas[d][100]) # 一個だけ, 200番と 300番のデータを入れてみる if flag == False: stateDict["before"].append(datas[d][200]) stateDict["after"].append(datas[d][300]) flag = True worstData = getWorstData(stateDict)
def reproduction(initState, matchingDict): manager.show(initState) state = copy.deepcopy(initState) for vp in matchingDict["viewpoint"]: state = manager.predictwithViewPoint(state, vp) manager.show(state)
def DP_main(datas, interDict_, sampleSize=0.7, n_iter=50, distError=50): interDict = pruningInterDict_2(datas, interDict_) output = {"matching":[], "viewpoint":[]} rests = copy.deepcopy(interDict) # 最初の stateDict を作る stateDict = {} for fname in rests.keys(): fdata = datas[fname] stateDict[fname] = fdata[rests[fname].pop(0)] # matching に登録する output["matching" ].append(stateDict) output["viewpoint"].append(None) while True: # rests が空なら終了 flg = True for fname in rests.keys(): if len(rests[fname]) != 0: flg = False break if flg == True: break # 次の stateDict を作る stateDict = {} for fname in rests.keys(): fdata = datas[fname] while True: if len(rests[fname])>0 and rests[fname][0] < output["matching"][-1][fname]["step"]+10: rests[fname].pop(0) else: break if len(rests[fname]) > 0: stateDict[fname] = fdata[rests[fname].pop(0)] else: # print("EMPTY - " + fname) stateDict[fname] = fdata[-1] # サンプリングを用いて vp を取得 fnameList = sorted(list(stateDict.keys())) beforeDict = output["matching"][-1] beforeList = [beforeDict[fn] for fn in fnameList] afterList = [stateDict[fn] for fn in fnameList] tempDict = {"before":beforeList, "after":afterList, "fname":fnameList} vp = manager.getViewPointwithSampling(tempDict,sampleSize,n_iter) print("get vp:" + str(vp["base"]) + "," + str(vp["ref"])) matching = {} moved = manager.getMovedObject(tempDict) for fname in stateDict.keys(): # vp を用いて,境界の推定をやり直す bStep = output["matching"][-1][fname]["step"] addInter = getAdditionalIntermediate(datas[fname], bStep, vp) # matching の更新 matching[fname] = addInter # output に matching と pending と vp を追加 output["matching" ].append(matching) output["viewpoint"].append(vp) with open("tmp/log_MakerMain/dills/DP_main_temp.dill", "wb") as f: dill.dump(output, f) return output
# coding = utf-8 import DPM.ViewPointEncoder as encoder import DPM.ViewPointManager as manager interList = [] path="tmp/log_MakerMain/GettingIntermediated/3-2500-2500-9/inter000000000.csv" with open(path) as f: while True: line = f.readline() if line == "": break interList.append(int(line.split(",")[0])) print("inters:"+str(interList)) with open("tmp/log_MakerMain/log000000000.csv", "r", encoding="utf-8") as f: counter = 0 while True: counter+=1 line = f.readline() if line == "": break elif counter%6 != 0 and counter not in interList: continue state = encoder.encodeState(line) name = "log000000000_"+str(counter)+".png" inter = counter in interList manager.savefig(state, name=name, log=True, inter=inter)