def valueiteration(v, reward, a): # a = [[None, None, None], [None, None, None], [None, None, None]] # initializing a dummy last value matrix to enter Error loop # vlast = [[1, 0, 0], [0, 0, 0], [0, 0, 0]] # ipdb.set_trace() vlast = generateDummy(v) print(error(v, vlast)) while error(v, vlast) >= 10**(-5): vlast = deepcopy(v) # copying current V in Vlast m = np.shape(v) # size of value matrix for i in range(0, m[0]): # Nos. of Raws for j in range(0, m[1]): # Nos. of Columns for k in range(0, 4): # Nos. of actions if k == 0 and i > 0: # Upper movement for all Raws -1st Raw temp0 = reward[i][j][k] + gama * v[i - 1][j] elif k == 0 and i == 0: # Upper movement for 1st Raw (All Cols) temp0 = None if k == 1 and i < m[ 0] - 1: # Down movement for all raws - last Raw temp1 = reward[i][j][k] + gama * v[i + 1][j] elif k == 1 and i == m[ 0] - 1: # Down movement for last Raw(All Cols) temp1 = None if k == 2 and j > 0: # Left movement for all Cols - 1st Cols temp2 = reward[i][j][k] + gama * v[i][j - 1] elif k == 2 and j == 0: # Left movement for 1st Cols(all raws) temp2 = None if k == 3 and j < m[ 1] - 1: # Right movement for all Cols - last Cols temp3 = reward[i][j][k] + gama * v[i][j + 1] elif k == 3 and j == m[ 1] - 1: # Right movement for last Cols(all raws) temp3 = None v[i][j] = max(temp0, temp1, temp2, temp3) # taking max of all actions # a stores which action is taken to get the value if v[i][j] == temp0: a[i][j] = 0 if v[i][j] == temp1: a[i][j] = 1 if v[i][j] == temp2: a[i][j] = 2 if v[i][j] == temp3: a[i][j] = 3 if v[i][j] == None: v[i][j] = 0 # ipdb.set_trace() return v, a
def qLearning(n, p, p1, encoder, ENClast): import qinitial v = initvalact.initvalact(n) Q = qinitial.qinitial(n) #Q = [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]] a = v[1] # a = [[None, None, None], [None, None, None], [None, None, None]] # initializing action matrix # size = reading size of the given Q matrix [Nos of raws, Nos. of col, Nos. of actions possible per state] size = np.shape(Q) # storing size of Q-matrix n = size[0] Qlast = generateDummy(Q) # generating dummy of same sizq as Q to enter the while loop iteration = 0 # initializing the iteration reward = generate_rewardmatrix.generate_rewardmatrix(n) while qError(Q, Qlast) > 1.5 or Q == Qlast: # check for the error value to be 10**-3 or Q = Qlast # we want here Q!=Qlast becauses in starting phase if reward is zero in next step we will read error = 0 # and this will cause us to fall out of the loop iteration += 1 # incresing iteration value Qlast = deepcopy(Q) # copying Q to Qlast # state = selecting state randomly 1every time depending on the Q size state = random.randint(1, size[0] * size[1]) # temp = to retrive raw and column from Nos of state generated by random selector # state / Nos.of column will give us information about the raw number... # for 3x4 (raw x column) state 1 to 4 are raw in 1 and state 5 to 8 are raw in 2 # for raw1(state 1 to 4)/4 (total columns) will be 0 < temp <= 1 # for raw1(state 5 to 8)/4 (total columns) will be 1 < temp <= 2 temp = state / (size[1] * 1.0) # defining a temporary variable if ((temp).is_integer()): raw = int(temp) - 1 else: raw = int(temp) # temp = modulo of state and Total column # for column1(state 1,5,9) % 4 (total columns) will be 1 [i.e colum = 1-1 = 0] # for column1(state 2,6,10) % 4 (total columns) will be 2 [i.e colum = 2-1 = 1] temp = state % size[1] col = temp - 1 if col < 0: col = size[1] - 1 else: pass gotopos.gotopos(raw, col, p, p1, n) # to go to state that is selected randomly time.sleep(0.3) # ipdb.set_trace() for i in range(0, 20): # action selection according to selction of state if raw == 0 and col == 0: action = random.choice([1, 3]) elif raw == 0 and (col == -1 or col == size[1]-1): action = random.choice([1, 2]) elif raw == 0: action = random.choice([1, 2, 3]) elif raw == size[0]-1 and col == 0: action = random.choice([0, 3]) elif raw == size[0]-1 and (col == -1 or col == size[1]-1): action = random.choice([0, 2]) elif raw == size[0]-1: action = random.choice([0, 2, 3]) elif col == 0: action = random.choice([0, 1, 3]) elif (col == -1 or col == size[1]-1): action = random.choice([0, 1, 2]) else: action = random.randint(0, 3) # cells where all four actions are possible # defining nextstate according to choosen action if action == 0: # Up movement nextstate = Q[raw-1][col] rawtemp = raw - 1 # raw of nextstep coltemp = col # col of nextstep elif action == 1: # Down movememt nextstate = Q[raw+1][col] rawtemp = raw + 1 # raw of nextstep coltemp = col # col of nextstep elif action == 2: # Left movement nextstate = Q[raw][col-1] rawtemp = raw # raw of nextstep coltemp = col - 1 # col of nextstep else: # Right movement # ipdb.set_trace() nextstate = Q[raw][col+1] rawtemp = raw # raw of nextstep coltemp = col + 1 # col of nextstep # ipdb.set_trace() # try executing the Q-iteration formula with no errors.. ''' _____ADD HERE____ ACTION_PERFORMANCE FUNCTION UPDATE_REWARD FUNCTION ''' ENClast = encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) if action == 0 or action == 1: ENClast = encoder.getData() ENC = encoder.getData() reward[raw][col][action] = ENC - ENClast # update_reward.update_reward(reward, raw, col, action, diff) try: Q[raw][col][action] = reward[raw][col][action] + gama * (max(nextstate)) #print "Q", Q # tracking if there is a type error (i.e. datatype missmatch) or not in above equation except TypeError as e: print("TypeError") raw = rawtemp col = coltemp print "qerror is", qError(Q,Qlast) print "reward is", reward # getting the appropriate action back from the given calculated values of Q matrix for r in range(0, size[0]): for c in range(0, size[1]): # ipdb.set_trace() a[r][c] = Q[r][c].index(max(Q[r][c])) # ipdb.set_trace() # function returns Q matrix, action matrix and nos of iteration return Q, a, iteration
def qLearning(n, reward1): v = initvalact.initvalact(n) Q = qinitial.qinitial(n) kMatrix = qinitial.qinitial(n) bita = 1 / 2 # Q = [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]] a = v[1] # a = [[None, None, None], [None, None, None], [None, None, None]] # initializing action matrix # size = reading size of the given Q matrix [Nos of raws, Nos. of col, Nos. of actions possible per state] size = np.shape(Q) # storing size of Q-matrix n = size[0] Qlast = generateDummy( Q) # generating dummy of same sizq as Q to enter the while loop iteration = 0 # initializing the iteration reward = generate_rewardmatrix.generate_rewardmatrix(n) # check for the error value to be 10**-3 or Q = Qlast ipdb.set_trace() while qError(Q, Qlast) > 1.5 or Q == Qlast or iteration <= 12: # we want here Q!=Qlast becauses in starting phase if reward is zero in next step we will read error = 0 # and this will cause us to fall out of the loop iteration += 1 # incresing iteration value Qlast = deepcopy(Q) # copying Q to Qlast # state = selecting state randomly every time depending on the Q size state = random.randint(1, size[0] * size[1]) # temp = to retrive raw and column from Nos of state generated by random selector # state / Nos.of column will give us information about the raw number... # for 3x4 (raw x column) state 1 to 4 are raw in 1 and state 5 to 8 are raw in 2 # for raw1(state 1 to 4)/4 (total columns) will be 0 < temp <= 1 # for raw1(state 5 to 8)/4 (total columns) will be 1 < temp <= 2 temp = state / (size[1] * 1.0) # defining a temporary variable if ((temp).is_integer()): raw = int(temp) - 1 else: raw = int(temp) # temp = modulo of state and Total column # for column1(state 1,5,9) % 4 (total columns) will be 1 [i.e colum = 1-1 = 0] # for column1(state 2,6,10) % 4 (total columns) will be 2 [i.e colum = 2-1 = 1] temp = state % size[1] col = temp - 1 if col < 0: col = size[1] - 1 else: pass # ipdb.set_trace() NumOfSelAct = 100 * (1 - 1 / (np.exp(0.05 * (iteration - 2)))) NumOfSelAct = round(NumOfSelAct * 25 / 100) print NumOfSelAct for i in range(0, 20): # action selection according to selction of state if i < NumOfSelAct: possibleActions = action_select(raw, col, n) tempList = [] for i in possibleActions: tempList.append(Q[raw][col][i]) action = possibleActions[tempList.index(max(tempList))] else: possibleActions = action_select(raw, col, n) action = random.choice(possibleActions) # defining nextstate according to choosen action if action == 0: # Up movement nextstate = Q[raw - 1][col] rawtemp = raw - 1 # raw of nextstep coltemp = col # col of nextstep elif action == 1: # Down movememt nextstate = Q[raw + 1][col] rawtemp = raw + 1 # raw of nextstep coltemp = col # col of nextstep elif action == 2: # Left movement nextstate = Q[raw][col - 1] rawtemp = raw # raw of nextstep coltemp = col - 1 # col of nextstep else: # Right movement # ipdb.set_trace() nextstate = Q[raw][col + 1] rawtemp = raw # raw of nextstep coltemp = col + 1 # col of nextstep # ipdb.set_trace() # try executing the Q-iteration formula with no errors.. ''' _____ADD HERE____ ACTION_PERFORMANCE FUNCTION UPDATE_REWARD FUNCTION ''' reward[raw][col][action] = reward1[raw][col][action] kMatrix[raw][col][action] = kMatrix[raw][col][action] + 1 try: alpha = 1 / ((kMatrix[raw][col][action])**bita) Q[raw][col][action] = (1-alpha) * Q[raw][col][action] + \ alpha * (reward[raw][col][action] + gama * (max(nextstate))) # tracking if there is a type error (i.e. datatype missmatch) or not in above equation except TypeError as e: print("TypeError") ipdb.set_trace() raw = rawtemp col = coltemp # getting the appropriate action back from the given calculated values of Q matrix for r in range(0, size[0]): for c in range(0, size[1]): # ipdb.set_trace() possibleActions = action_select(r, c, n) tempList = [] for i in possibleActions: tempList.append(Q[r][c][i]) a[r][c] = possibleActions[tempList.index(max(tempList))] # ipdb.set_trace() # function returns Q matrix, action matrix and nos of iteration print kMatrix print NumOfSelAct return Q, a, iteration
def qLearning(n): simulatedRewards = [ [ { 0: None, 1: 0, 2: None, 3: 0 }, # state = 1 { 0: None, 1: 0, 2: 0, 3: 0 }, # State = 2 { 0: None, 1: 0, 2: 0, 3: None } ], # State = 3 [ { 0: 0, 1: 0, 2: None, 3: 0 }, # State = 4 { 0: 0, 1: 0, 2: 0, 3: 0 }, # State = 5 { 0: 0, 1: 0, 2: 0, 3: None } ], # State = 6 [ { 0: 0, 1: None, 2: None, 3: -1 }, # State = 7 { 0: 0, 1: None, 2: 1, 3: -1 }, # State = 8 { 0: 0, 1: None, 2: 1, 3: None } ] ] # State = 9 v = initvalact.initvalact(n) Q = qinitial.qinitial(n) '''Q = [[[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0]], # State1,State2, Stete3 [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0]], # State4, State5, State6 [[1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 1, 0]]] # State7, State8, State9''' kMatrix = qinitial.qinitial(n) Tr = qinitial.qinitial(n) a = v[1] size = np.shape(Q) # storing size of Q-matrix n = size[0] Qlast = generateDummy( Q) # generating dummy of same sizq as Q to enter the while loop iteration = 0 # initializing the iteration reward = generate_rewardmatrix.generate_rewardmatrix(n) state = random.randint(1, size[0] * size[1]) while qError(Q, Qlast) > 1.5 or Q == Qlast or iteration <= 100: iteration += 1 # incresing iteration value Qlast = deepcopy(Q) # copying Q to Qlast # state = selecting state randomly 1every time depending on the Q size # temp = to retrive raw and column from Nos of state generated by random selector # state / Nos.of column will give us information about the raw number... # for 3x4 (raw x column) state 1 to 4 are raw in 1 and state 5 to 8 are raw in 2 # for raw1(state 1 to 4)/4 (total columns) will be 0 < temp <= 1 # for raw1(state 5 to 8)/4 (total columns) will be 1 < temp <= 2 if iteration == 1: temp = state / (size[1] * 1.0) # defining a temporary variable if ((temp).is_integer()): raw = int(temp) - 1 else: raw = int(temp) # temp = modulo of state and Total column # for column1(state 1,5,9) % 4 (total columns) will be 1 [i.e colum = 1-1 = 0] # for column1(state 2,6,10) % 4 (total columns) will be 2 [i.e colum = 2-1 = 1] temp = state % size[1] col = temp - 1 if col < 0: col = size[1] - 1 else: pass # *** gotopos.gotopos(raw, col, p, p1, n) # to go to state that is selected randomly # *** time.sleep(0.3) # ipdb.set_trace() possibleActions = action_select(raw, col, n) # ipdb.set_trace() probablity = epsilon_greedy_policy(Q[raw][col], possibleActions, epsilon) # ipdb.set_trace() actionIndex = np.random.choice(len(probablity), p=probablity) # ipdb.set_trace() action = possibleActions[actionIndex] # ipdb.set_trace() # for i in range(0, 20): # action selection according to selction of state ''' ------------------------------------------------------------- ***************REPLACED BY EPSILON GREDDY POLICY************* ------------------------------------------------------------- if i < NumOfSelAct: possibleActions = action_select(raw, col, n) tempList = [] for i in possibleActions: tempList.append(Q[raw][col][i]) action = possibleActions[tempList.index(max(tempList))] else: possibleActions = action_select(raw, col, n) action = random.choice(possibleActions) ------------------------------------------------------------- ***************REPLACED BY EPSILON GREDDY POLICY************* ------------------------------------------------------------- ''' # defining nextstate according to choosen action if action == 0: # Up movement nextstate = Q[raw - 1][col] rawtemp = raw - 1 # raw of nextstep coltemp = col # col of nextstep elif action == 1: # Down movememt nextstate = Q[raw + 1][col] rawtemp = raw + 1 # raw of nextstep coltemp = col # col of nextstep elif action == 2: # Left movement nextstate = Q[raw][col - 1] rawtemp = raw # raw of nextstep coltemp = col - 1 # col of nextstep else: # Right movement # ipdb.set_trace() nextstate = Q[raw][col + 1] rawtemp = raw # raw of nextstep coltemp = col + 1 # col of nextstep # ipdb.set_trace() # try executing the Q-iteration formula with no errors.. ''' _____ADD HERE____ ACTION_PERFORMANCE FUNCTION UPDATE_REWARD FUNCTION ENClast= encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) # if action == 0 or action == 1: # ENClast= encoder.getData() ENC= encoder.getData() reward[raw][col][action]= ENC - ENClast # update_reward.update_reward(reward, raw, col, action, diff) ''' # ipdb.set_trace() reward[raw][col][action] = simulatedRewards[raw][col][action] # ipdb.set_trace() kMatrix[raw][col][action] = kMatrix[raw][col][action] + 1 # ipdb.set_trace() try: alpha = 1 / ((kMatrix[raw][col][action])**bita) # ipdb.set_trace() eComplement = reward[raw][col][action] + gama * max( nextstate) - Q[raw][col][action] # ipdb.set_trace() e = reward[raw][col][action] + gama * max(nextstate) - max( Q[raw][col]) # ipdb.set_trace() for r in range(size[0]): for c in range(size[1]): for actn in action_select(raw, col, n): Tr[r][c][actn] = gama * lamda * Tr[r][c][actn] Q[r][c][ actn] = Q[r][c][actn] + alpha * Tr[r][c][actn] * e # ipdb.set_trace() Q[raw][col][action] = Q[raw][col][action] + alpha * eComplement # ipdb.set_trace() Tr[raw][col][action] += 1 # ipdb.set_trace() # print "Q", Q # tracking if there is a type error (i.e. datatype missmatch) or not in above equation except TypeError as e: print("TypeError") # ipdb.set_trace() print possibleActions print probablity print "raw= ", raw, "col = ", col, "action = ", action raw = rawtemp col = coltemp print "qerror is", qError(Q, Qlast) print "reward is", reward print "iteration = ", iteration # time.sleep(0.1) # ipdb.set_trace() # getting the appropriate action back from the given calculated values of Q matrix print Tr print Q for r in range(0, size[0]): for c in range(0, size[1]): possibleActions = action_select(r, c, n) tempList = [] for i in possibleActions: tempList.append(Q[r][c][i]) a[r][c] = possibleActions[tempList.index(max(tempList))] # ipdb.set_trace() # function returns Q matrix, action matrix and nos of iteration return Q, a, iteration
def qLearning(n, p, p1, encoder, ENClast): v = initvalact.initvalact(n) Q = qinitial.qinitial(n) kMatrix = qinitial.qinitial(n) restriCount = qinitial.qinitial(n) bita = 1/2 # Q = [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]] a = v[1] # a = [[None, None, None], [None, None, None], [None, None, None]] # initializing action matrix # size = reading size of the given Q matrix [Nos of raws, Nos. of col, Nos. of actions possible per state] size = np.shape(Q) # storing size of Q-matrix n = size[0] Qlast = generateDummy(Q) # generating dummy of same sizq as Q to enter the while loop iteration = 0 # initializing the iteration reward = generate_rewardmatrix.generate_rewardmatrix(n) # check for the error value to be 10**-3 or Q = Qlast global val1 val1 = pinSetup.valueRead_ON() while (qError(Q, Qlast) > 1.5 or Q == Qlast or iteration <= 4*n) and (val1 == 0): # we want here Q!=Qlast becauses in starting phase if reward is zero in next step we will read error = 0 # and this will cause us to fall out of the loop iteration += 1 # incresing iteration value Qlast = deepcopy(Q) # copying Q to Qlast # state = selecting state randomly 1every time depending on the Q size state = random.randint(1, size[0] * size[1]) # temp = to retrive raw and column from Nos of state generated by random selector # state / Nos.of column will give us information about the raw number... # for 3x4 (raw x column) state 1 to 4 are raw in 1 and state 5 to 8 are raw in 2 # for raw1(state 1 to 4)/4 (total columns) will be 0 < temp <= 1 # for raw1(state 5 to 8)/4 (total columns) will be 1 < temp <= 2 temp = state / (size[1] * 1.0) # defining a temporary variable if ((temp).is_integer()): raw = int(temp) - 1 else: raw = int(temp) # temp = modulo of state and Total column # for column1(state 1,5,9) % 4 (total columns) will be 1 [i.e colum = 1-1 = 0] # for column1(state 2,6,10) % 4 (total columns) will be 2 [i.e colum = 2-1 = 1] temp = state % size[1] col = temp - 1 if col < 0: col = size[1] - 1 else: pass gotopos.gotopos(raw, col, p, p1, n) # to go to state that is selected randomly time.sleep(0.3) # ipdb.set_trace() NumOfSelAct = 100*(1-1/(np.exp(0.05*(iteration-2)))) NumOfSelAct = round(NumOfSelAct*25/100) for i in range(0, 20): # action selection according to selction of state if i < NumOfSelAct: possibleActions = action_select(raw, col, n) tempList = [] for j in possibleActions: tempList.append(Q[raw][col][j]) action = possibleActions[tempList.index(max(tempList))] print ("for i") , i, ("selected action is"), action else: possibleActions = action_select(raw, col, n) action = random.choice(possibleActions) # defining nextstate according to choosen action if action == 0: # Up movement nextstate = Q[raw-1][col] rawtemp = raw - 1 # raw of nextstep coltemp = col # col of nextstep elif action == 1: # Down movememt nextstate = Q[raw+1][col] rawtemp = raw + 1 # raw of nextstep coltemp = col # col of nextstep elif action == 2: # Left movement nextstate = Q[raw][col-1] rawtemp = raw # raw of nextstep coltemp = col - 1 # col of nextstep else: # Right movement # ipdb.set_trace() nextstate = Q[raw][col+1] rawtemp = raw # raw of nextstep coltemp = col + 1 # col of nextstep # ipdb.set_trace() # try executing the Q-iteration formula with no errors.. ''' _____ADD HERE____ ACTION_PERFORMANCE FUNCTION UPDATE_REWARD FUNCTION ''' ENClast = encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) if action == 0 or action == 1: ENClast = encoder.getData() ENC = encoder.getData() if ENC - ENClast > 0: diff = 1 if ENC - ENClast > 1: diff = 2 elif ENC - ENClast < 0: diff = -1 if ENC - ENClast < -1: diff = -2 else: diff = 0 oldreward = reward[raw][col][action] if (oldreward != 0 and diff == 0) or (np.sign(oldreward)!=np.sign(diff)): # restriCount[raw][col][action] += 1 # if restriCount[raw][col][action] < 3: print ("!! restriction applied !!") restriCount[raw][col][action] = 0 gotopos.gotopos(raw, col, p, p1, n) time.sleep(0.3) ENClast = encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) if action == 0 or action == 1: ENClast = encoder.getData() ENC = encoder.getData() diff = ENC - ENClast direction = pinSetup.valueRead_dir() reward[raw][col][action] = ((-1)**direction)*diff # update_reward.update_reward(reward, raw, col, action, diff) kMatrix[raw][col][action] = kMatrix[raw][col][action] + 1 try: alpha = 1/((kMatrix[raw][col][action])**bita) Q[raw][col][action] = (1-alpha) * Q[raw][col][action] + \ alpha * (reward[raw][col][action] + gama * (max(nextstate))) # print "Q", Q # tracking if there is a type error (i.e. datatype missmatch) or not in above equation except TypeError as e: print("TypeError") raw = rawtemp col = coltemp print "iteration is", iteration print "qerror is", qError(Q, Qlast) print "reward is", reward val1 = pinSetup.valueRead_ON() # getting the appropriate action back from the given calculated values of Q matrix if val1 == 1: #import os print "Stop" #os.system("shutdown now") for r in range(0, size[0]): for c in range(0, size[1]): # ipdb.set_trace() possibleActions = action_select(r, c, n) tempList = [] for i in possibleActions: tempList.append(Q[r][c][i]) a[r][c] = possibleActions[tempList.index(max(tempList))] # ipdb.set_trace() # function returns Q matrix, action matrix and nos of iteration print kMatrix # print NumOfSelAct return Q, a, iteration
def qLearning(n, p, p1, encoder, ENClast): v = initvalact.initvalact(n) Q = qinitial.qinitial(n) kMatrix = qinitial.qinitial(n) Tr = qinitial.qinitial(n) a = v[1] size = np.shape(Q) # storing size of Q-matrix n = size[0] Qlast = generateDummy( Q) # generating dummy of same sizq as Q to enter the while loop iteration = 0 # initializing the iteration reward = generate_rewardmatrix.generate_rewardmatrix(n) state = random.randint(1, size[0] * size[1]) global val1 val1 = pinSetup.valueRead_ON() while True and val1 == 0: iteration += 1 # incresing iteration value epsilon = (0.9 / (np.exp(0.05 * (iteration - 1)))) + 0.1 Qlast = deepcopy(Q) # copying Q to Qlast # state = selecting state randomly 1every time depending on the Q size # temp = to retrive raw and column from Nos of state generated by random selector # state / Nos.of column will give us information about the raw number... # for 3x4 (raw x column) state 1 to 4 are raw in 1 and state 5 to 8 are raw in 2 # for raw1(state 1 to 4)/4 (total columns) will be 0 < temp <= 1 # for raw1(state 5 to 8)/4 (total columns) will be 1 < temp <= 2 if iteration == 1: temp = state / (size[1] * 1.0) # defining a temporary variable if ((temp).is_integer()): raw = int(temp) - 1 else: raw = int(temp) # temp = modulo of state and Total column # for column1(state 1,5,9) % 4 (total columns) will be 1 [i.e colum = 1-1 = 0] # for column1(state 2,6,10) % 4 (total columns) will be 2 [i.e colum = 2-1 = 1] temp = state % size[1] col = temp - 1 if col < 0: col = size[1] - 1 else: pass gotopos.gotopos(raw, col, p, p1, n) # to go to state that is selected randomly time.sleep(0.3) possibleActions = action_select(raw, col, n) probablity = epsilon_greedy_policy(Q[raw][col], possibleActions, epsilon) actionIndex = np.random.choice(len(probablity), p=probablity) action = possibleActions[actionIndex] # defining nextstate according to choosen action if action == 0: # Up movement nextstate = Q[raw - 1][col] rawtemp = raw - 1 # raw of nextstep coltemp = col # col of nextstep elif action == 1: # Down movememt nextstate = Q[raw + 1][col] rawtemp = raw + 1 # raw of nextstep coltemp = col # col of nextstep elif action == 2: # Left movement nextstate = Q[raw][col - 1] rawtemp = raw # raw of nextstep coltemp = col - 1 # col of nextstep else: # Right movement nextstate = Q[raw][col + 1] rawtemp = raw # raw of nextstep coltemp = col + 1 # col of nextstep # try executing the Q-iteration formula with no errors.. ''' _____ADD HERE____ ACTION_PERFORMANCE FUNCTION UPDATE_REWARD FUNCTION ''' ENClast = encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) if action == 0 or action == 1: ENClast = encoder.getData() ENC = encoder.getData() diff = ENC - ENClast oldreward = reward[raw][col][action] if (oldreward != 0 and diff == 0) or (np.sign(oldreward) != np.sign(diff) and oldreward != 0): print("!! restriction applied !!") gotopos.gotopos(raw, col, p, p1, n) time.sleep(0.3) ENClast = encoder.getData() act.playAction(action, raw, col, size[0], p, p1) time.sleep(0.1) if action == 0 or action == 1: ENClast = encoder.getData() ENC = encoder.getData() diff = ENC - ENClast direction = pinSetup.valueRead_dir() reward[raw][col][action] = ((-1)**direction) * diff kMatrix[raw][col][action] = kMatrix[raw][col][action] + 1 try: alpha = 1 / ((kMatrix[raw][col][action])**bita) eComplement = reward[raw][col][action] + gama * max( nextstate) - Q[raw][col][action] e = reward[raw][col][action] + gama * max(nextstate) - max( Q[raw][col]) for r in range(size[0]): for c in range(size[1]): for actn in action_select(raw, col, n): Tr[r][c][actn] = gama * lamda * Tr[r][c][actn] Q[r][c][ actn] = Q[r][c][actn] + alpha * Tr[r][c][actn] * e Q[raw][col][action] = Q[raw][col][action] + alpha * eComplement Tr[raw][col][action] += 1 # tracking if there is a type error (i.e. datatype missmatch) or not in above equation except TypeError as e: print("TypeError") print possibleActions print probablity print "raw= ", raw, "col = ", col, "action = ", action raw = rawtemp col = coltemp print "qerror is", qError(Q, Qlast) print "reward is", reward print "iteration = ", iteration val1 = pinSetup.valueRead_ON() # getting the appropriate action back from the given calculated values of Q matrix print Tr print Q for r in range(0, size[0]): for c in range(0, size[1]): possibleActions = action_select(r, c, n) tempList = [] for i in possibleActions: tempList.append(Q[r][c][i]) a[r][c] = possibleActions[tempList.index(max(tempList))] # function returns Q matrix, action matrix and nos of iteration return Q, a, iteration