def filter_open_cases(log):
log_selected = attributes_filter.apply(
log, ["Payment Handled"],
parameters={
xes_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY:
constants.concept_key,
"positive": True
})
util.print_filtered_cases_count(len(log), len(log_selected))
return log_selected
def train_prioritized(Q_Network, train_batch, w_batch, Exp, s_scale,
input_size, num_actions):
state_t_batch = [item[0] for item in train_batch]
state_t_batch = np.array(state_t_batch)
state_t_1_batch = [item[1] for item in train_batch]
state_t_1_batch = np.array(state_t_1_batch)
action_batch = [item[2] for item in train_batch]
reward_batch = [item[3] for item in train_batch]
reward_batch = np.array(reward_batch)
done_batch = [item[4] for item in train_batch]
done_batch = np.array(done_batch)
action_t_batch = Q_Network.evaluate_critic(state_t_batch)
w_batch = np.transpose(np.tile(w_batch, (num_actions, 1)))
if Exp == 'epsilon':
action_t_1_batch = Q_Network.evaluate_critic(state_t_1_batch)
q_t_1 = Q_Network.evaluate_target_critic(state_t_1_batch)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][action_batch[i]] = reward_batch[
i] + GAMMA * q_t_1[i][np.argmax(action_t_1_batch[i])]
elif Exp == 'softmax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.softV(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][
action_batch[i]] = reward_batch[i] + GAMMA * q_t_1[i]
elif Exp == 'sparsemax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.sparsemax(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][
action_batch[i]] = reward_batch[i] + GAMMA * q_t_1[i]
#Q_Network.train_critic_prioritized(state_t_batch, action_t_batch, w_batch)
errors, cost, _ = Q_Network.train_critic_prioritized(
state_t_batch, action_t_batch, w_batch)
errors = np.sum(errors, axis=1)
return errors, cost
def train(Q_Network, train_batch, Exp, s_scale, input_size, num_actions):
state_t_batch = [item[0] for item in train_batch]
state_t_batch = np.array(state_t_batch)
state_t_1_batch = [item[1] for item in train_batch]
state_t_1_batch = np.array(state_t_1_batch)
action_batch = [item[2] for item in train_batch]
reward_batch = [item[3] for item in train_batch]
reward_batch = np.array(reward_batch)
done_batch = [item[4] for item in train_batch]
done_batch = np.array(done_batch)
action_t_batch = Q_Network.evaluate_critic(state_t_batch)
if Exp == 'epsilon':
action_t_1_batch = Q_Network.evaluate_critic(state_t_1_batch)
q_t_1 = Q_Network.evaluate_target_critic(state_t_1_batch)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][action_batch[i]] = reward_batch[
i] + GAMMA * q_t_1[i][np.argmax(action_t_1_batch[i])]
elif Exp == 'softmax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.softV(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][
action_batch[i]] = reward_batch[i] + GAMMA * q_t_1[i]
elif Exp == 'sparsemax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.sparsemax(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][
action_batch[i]] = reward_batch[i] + GAMMA * q_t_1[i]
# Update critic by minimizing the loss
Q_Network.train_critic(state_t_batch, action_t_batch)
def main():
algo = input('Algorithm? 1 - Random direction, 2 - RSS gradient\n')
max_it = input('Max iterations?\n')
nb_samples = input('Nb of RSS samples for averaging?\n')
mode = input('Mode? 1 - Thread, 2 - Procedural\n')
write = input('Output file? 1/0 (y/n)\n')
sim = input('Simulation? 1/0 (y/n)\n')
roomba_sim = None
if sim:
roomba_sim = Roomba_sim.Roomba_sim()
roomba_sim.daemon = True # Thread is killed when main thread killed
roomba_sim.start() # Start thread
explore = Exploration.Exploration(roomba_sim)
explore.daemon = True # Thread is killed when main thread killed
explore.start(
) # Start thread (Needed for procedural too, it activate the control of the robot)
rssi_obj = RSSI_Measure.RSSI_Measure()
localize = Localization.Localization(explore, rssi_obj, algo, max_it,
nb_samples, mode, write)
localize.daemon = True # Thread is killed when main thread killed
localize.start() # Start thread
print 'Algorithm Start'
localize.join()
explore.stop()
explore.join()
def train_prioritized(Q_Network, train_batch, w_batch, Exp, s_scale, input_size, num_actions, size_action_batch):
state_t_batch, state_t_1_batch, action_batch, reward_batch, done_batch = zip(*train_batch)
state_t_batch = np.array(state_t_batch)
state_t_1_batch = np.array(state_t_1_batch)
action_batch = np.array(action_batch)
reward_batch = np.array(reward_batch)
done_batch = np.array(done_batch)
batch_size = len(done_batch)
q_t_1_batch = []
if Exp == 'epsilon':
# 'target-Q네트워크'로 계산한 action별 q값 계산
q_t_1_batch = Q_Network.get_target_q_batch(state_t_1_batch)
q_t_1_batch = np.reshape(q_t_1_batch, [batch_size, -1])
# Exploration 기법 별로 V(next_state)를 계산
q_t_1_batch = np.max(q_t_1_batch, axis = 1)
# 측정 Q(state, action)을 q_t_1_batch로 저장
q_t_1_batch = reward_batch + GAMMA*q_t_1_batch#*(1-done_batch)
elif Exp == 'softmax':
q_t_1_batch = Q_Network.get_target_q_batch(state_t_1_batch)
q_t_1_batch = np.reshape(q_t_1_batch, [batch_size, -1])
q_t_1_batch = Exploration.softV(q_t_1_batch, s_scale)
q_t_1_batch = reward_batch + GAMMA*q_t_1_batch#*(1-done_batch)
elif Exp == 'sparsemax':
q_t_1_batch = Q_Network.get_target_q_batch(state_t_1_batch)
q_t_1_batch = np.reshape(q_t_1_batch, [batch_size, -1])
q_t_1_batch = Exploration.sparsemax(q_t_1_batch, s_scale)
q_t_1_batch = reward_batch + GAMMA*q_t_1_batch#*(1-done_batch)
q_t_1_batch = np.reshape(q_t_1_batch,[-1,1])
w_batch = np.reshape(w_batch,[-1,1])
# q_t_1_batch와 오차가 가장 작아지는 방향으로 Q네트워크 training, weight 적용
errors, cost, _ = Q_Network.train_critic_prioritized(state_t_batch, action_batch, q_t_1_batch, w_batch)
errors = np.sum(errors, axis=1)
return errors, cost, state_t_batch
def getExplorer(self):
#
# Replay.
replay_buffer = ReplayBuffer(self.replaySize, self.frameHist)
explorer = Exploration.EpsilonGreedy(self.explorationSched,
TensorConfig.TensorConfig(),
replay_buffer, self.env,
self.q_func, self.maxSteps)
return explorer
def train(Q_Network, train_batch, Exp, s_scale, input_size, num_actions, size_action_batch):
state_t_batch, state_t_1_batch, action_batch, reward_batch, done_batch = zip(*train_batch)
state_t_batch = np.array(state_t_batch)
state_t_1_batch = np.array(state_t_1_batch)
action_batch = np.array(action_batch)
reward_batch = np.array(reward_batch)
done_batch = np.array(done_batch)
q_t_1_batch = []
if Exp == 'epsilon':
for i in range(0, len(train_batch)):
q_t_1_batch.append(np.reshape(Q_Network.get_target_q_batch(np.reshape(state_t_1_batch[i],[1,-1])),[1,-1])[0])
q_t_1_batch = reward_batch + GAMMA*np.max(q_t_1_batch, axis=1)#*(1-done_batch)
elif Exp == 'softmax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.softV(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][action_batch[i]] = reward_batch[i] + GAMMA*q_t_1[i]
elif Exp == 'sparsemax':
action_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
q_t_1 = Exploration.sparsemax(action_t_1_batch, s_scale)
for i in range(0, len(train_batch)):
if done_batch[i]:
action_t_batch[i][action_batch[i]] = reward_batch[i]
else:
action_t_batch[i][action_batch[i]] = reward_batch[i] + GAMMA*q_t_1[i]
q_t_1_batch = np.reshape(q_t_1_batch,[-1,1])
cost, _ = Q_Network.train_critic(state_t_batch, action_batch, q_t_1_batch)
return cost, state_t_batch
def CreateAndTrainModel(modelToSavePath):
discountFactor = 0.9
learningRate = 1
trainingEstimator = Model.GetModel(modelToSavePath + "_trainning")
targetEstimator = Model.GetModel(modelToSavePath)
Model.CreateModelIfDoesntExist(modelToSavePath + "_trainning",
trainingEstimator)
Model.CreateModelIfDoesntExist(modelToSavePath, targetEstimator)
experienceContainer = ER.ExperienceContainer(10000)
maxMovesCount = 10
epochCount = 1000
totalMoves = 0
for epoch in range(epochCount):
print('epoch:' + str(epoch))
state = Game.CreateState(None, Game.Pos(1, 2), None)
print(state)
nbMoves = 0
tooManyMoves = False
while (not (state.IsFinished() or tooManyMoves)):
nbMoves += 1
totalMoves += 1
action = Exploration.GetNextActionBoltzmann(
Model.GetQValue(state, targetEstimator), epoch, epochCount)
print(str(action) + '\n')
nextState = Game.Move(state, action)
experienceContainer.Add(state, action, nextState.GetReward(),
nextState)
state = nextState
tooManyMoves = nbMoves == maxMovesCount
print(state
if not tooManyMoves else 'lost because did too many moves')
if totalMoves % 5 == 0:
Model.TrainModelWithExperienceReplay(experienceContainer,
trainingEstimator,
targetEstimator,
discountFactor,
learningRate)
if totalMoves % 25 == 0:
targetEstimator = trainingEstimator
OverrideDir(modelToSavePath, modelToSavePath + "_trainning")
def Game():
os.system("cls")
print("Welcome to A Text Based RPG")
print("Developed by Malachi Wadas \n\n\n\n")
itp = input("(n)ew / (l)oad game? > ")
if itp in ["n", "N"]:
Player = NewGame()
Easyregions = [
gen.GenerateRegion(rnd.choice([1, 2, 3]),
themes=rnd.choice([
"fire", "ice", "earth", "lightning",
"random", "special", ""
]))
] * 200
Medregions = [
gen.GenerateRegion(rnd.choice([4, 5]),
themes=rnd.choices([
"fire", "ice", "earth", "lightning",
"random", "special", ""
],
k=2))
] * 100
Hardregions = [
gen.GenerateRegion(rnd.choice([6, 7]),
themes=rnd.choices([
"fire", "ice", "earth", "lightning",
"random", "special", ""
],
k=2))
] * 25
Reg = rnd.sample(Easyregions + Medregions + Hardregions, 5)
for i in range(len(Reg)):
print(
str(i) + ": " + Reg[i]["Name"] + ", Regions: " +
str(Reg[i]["Total Regions"]))
itp = int(
input("Which one do you wnat to explore? 0 - " +
str(len(Reg) - 1) + " > "))
while not (0 <= itp <= len(Reg) - 1):
for i in range(len(Reg)):
print(
str(i) + ": " + Reg[i]["Name"] + ", Regions: " +
str(Reg[i]["Total Regions"]))
itp = int(
input("Which one do you wnat to explore? 0 - " +
str(len(Reg) - 1) + " > "))
reg = Reg[itp]
Player["Current Location"] = reg
expr.ExploreRegion(Player, reg)
if itp in ["l", "L"]:
quit()
def __init__(self, seed, envName, expName):
super(Config, self).__init__(seed, envName, expName=expName)
self.parallelCfg = Exploration.ExploreParallelCfg()
self.parallelCfg.model = self.q_func
self.parallelCfg.exploreSched = self.explorationSched
self.parallelCfg.numFramesPerBuffer = self.frameHist + 1
self.parallelCfg.sampleLatest = True
self.parallelCfg.numEnv = 32
self.batch_size = self.parallelCfg.numEnv
self.logPeriod = int(self.parallelCfg.numEnv * 100)
#
# Dont need to wait since we are going sequentially, but allow for some randomness.
self.learning_starts = 50
self.learning_freq = 1
self.epsilonStepSize = 1 # To match single step learning frequency.
def loadSprites(self):
self.tank = Tank(self.screen)
self.exploration = Exploration(self.screen, (100, 100))
self.map = Map(self.height, self.height, 32, 32)
enemy_tank0 = EnemyTank(self.screen)
enemy_tank1 = EnemyTank(self.screen)
enemy_tank2 = EnemyTank(self.screen)
enemy_tank3 = EnemyTank(self.screen)
enemy_tank4 = EnemyTank(self.screen)
self.enemy_group = pygame.sprite.Group()
self.enemy_group.add(enemy_tank0, enemy_tank1, enemy_tank2,
enemy_tank3, enemy_tank4)
self.empty_map = EmptyMap(self.screen, self.width, self.height)
"""currently it is tank_sprites,later, you should decouple these"""
def train_error(Q_Network, train_batch, Exp, s_scale, input_size, num_actions):
state_t_batch = [item[0] for item in train_batch]
state_t_batch = np.array(state_t_batch)
state_t_1_batch = [item[1] for item in train_batch]
state_t_1_batch = np.array(state_t_1_batch)
action_batch = [item[2] for item in train_batch]
reward_batch = [item[3] for item in train_batch]
reward_batch = np.array(reward_batch)
done_batch = [1 if item[4] == False else 0 for item in train_batch]
done_batch = np.array(done_batch)
q_t_batch = Q_Network.evaluate_critic(state_t_batch)
q_t = [q_t_batch[i][action_batch[i]] for i in range(len(train_batch))]
q_t = np.array(q_t)
action_t_1_batch = Q_Network.evaluate_critic(state_t_1_batch)
q_t_1_batch = Q_Network.evaluate_target_critic(state_t_1_batch)
if Exp == 'epsilon':
q_t_1 = [
q_t_1_batch[i][np.argmax(action_t_1_batch[i])]
for i in range(len(train_batch))
]
q_t_1 = np.array(q_t_1)
error_batch = GAMMA * q_t_1 * done_batch + reward_batch - q_t
elif Exp == 'softmax':
q_t_1 = Exploration.softV(q_t_1_batch, s_scale)
error_batch = GAMMA * q_t_1 * done_batch + reward_batch - q_t
elif Exp == 'sparsemax':
#q_t_1 = Exploration.sparsemax(q_t_1_batch, s_scale)
#error_batch = GAMMA*q_t_1*done_batch + reward_batch - q_t
q_t_1 = [
q_t_1_batch[i][np.argmax(action_t_1_batch[i])]
for i in range(len(train_batch))
]
q_t_1 = np.array(q_t_1)
error_batch = GAMMA * q_t_1 * done_batch + reward_batch - q_t
return error_batch
def PlayWithModel(modelPath):
if not os.path.isdir(modelPath):
print("No model: " + modelPath)
return
estimator = Model.GetModel(modelPath)
state = Game.CreateState(None, Game.Pos(1, 2), None)
print(state)
maxMovesCount = 5
nbMoves = 0
while (not state.IsFinished()):
if nbMoves == maxMovesCount:
print('lost because did too many moves')
return
action = Exploration.GetBestAction(Model.GetQValue(state, estimator))
print(str(action) + '\n')
state.Move(action)
print(state)
nbMoves += 1
def explore_loop():
global init_probing
reached_goal = False
finished = False
if init_probing:
ardoutQ.append("a")
AhBot.turn_left()
init_probed = False
while not init_probed:
if len(ardinQ) > 0:
sensors = ardinQ.popleft()
print("init ignored" + str(sensors))
init_probed = True
detect = exp_move("d")
# main loop
while not finished:
arrow = 0 # detection result
Explored.set_cleared(
AhBot.x,
AhBot.y) # add the grids occuppied by the robot to the set CLEARED
steps = Exploration.next_step(
) # compute the next step base on current explored map
prev_bot_state = str(AhBot.x - 1) + "_" + str(20 -
AhBot.y) + "_" + str(
AhBot.face % 4)
for action in steps:
arrow = exp_move(action)
if AhBot.x == END_PT[0] and AhBot.y == END_PT[1]:
reached_goal = True
if reached_goal and AhBot.x == START_PT[0] and AhBot.y == START_PT[
1]: # back to the starting point
finished = True
break
# send exp_map and ahbot state to bluetooth
update_bt(arrow, prev_bot_state)
def run_DQN(self, case_n, seed_n, Exp, Double, Dueling, Prioritized):
sess = self.sess
dis = self.dis
REPLAY_MEMORY = self.REPLAY_MEMORY
batch_size = self.batch_size
Game = self.Game
save_epi = self.save_epi
max_episodes = self.max_episodes
env = self.env
input_size = self.input_size
output_size = self.output_size
alpha = self.alpha
beta_init = self.beta_init
eps = self.eps
eps_div = self.eps_div
s_scale = self.s_scale
training_step = self.training_step
copy_step = self.copy_step
repu_num = self.repu_num
ending_cond_epis = self.ending_cond_epis
ending_cond_reward = self.ending_cond_reward
conti_action_flag = self.conti_action_flag
action_map = self.action_map
env.seed(seed_n)
np.random.seed(seed_n)
tf.set_random_seed(seed_n)
random.seed(seed_n)
Q_Network = self.Q_Network
end_episode = 0
step_count_total = 0
global_step = 0
loss = 0
replay_buffer = deque()
Q_list = []
TD_buffer = deque()
steps_list = []
step_avg_list = []
global_step_list = []
print("")
print("CASE {}".format(case_n))
print(" STATE DIM : {}, ACTION DIM : {}".format(input_size, self.action_dim))
print(" Exp : {}".format(Exp))
print(" Strategy : Double : {}, Dueling : {}, Prioritized : {}".format(Double, Dueling, Prioritized))
for episode in range(1, max_episodes+1):
done = False
step_count = 0
current_step = 0
TD_error = 0
state = env.reset()
while not done:
e = 1. / ((float(episode - 1) / eps_div) + 1)
action = Exploration.choice_action(Exp, e, s_scale, Q_Network.evaluate_critic(np.reshape(state, [1, input_size]))[0])
if conti_action_flag:
#action = np.array(action_map[action])
action0 = [action_map[action]]
else:
action0 = action
next_state, reward, done, _ = env.step(action0)
step_count += reward
global_step += 1
current_step += 1
if Prioritized:
q_t = np.max(Q_Network.evaluate_critic(np.reshape(state, [1, input_size])))
q_t_1 = np.max(Q_Network.evaluate_critic(np.reshape(next_state, [1, input_size])))
if done:
q_t_1 = reward
else:
q_t_1 = reward + dis*q_t_1
TD_buffer.append(pow(abs(q_t_1-q_t)+eps,alpha))
if len(TD_buffer) > REPLAY_MEMORY:
TD_buffer.popleft()
replay_buffer.append((state, next_state, action, reward, done))
if len(replay_buffer) > REPLAY_MEMORY:
replay_buffer.popleft()
state = next_state
minibatch = []
TD_choice = []
if global_step > batch_size and global_step % training_step == 0:
for re in range(repu_num):
minibatch = []
TD_choice = []
if Prioritized:
#TD_batch = Train.if_prioritized(Q_Network, replay_buffer, input_size, self.action_dim, eps, alpha)
TD_batch = np.array(TD_buffer)/sum(TD_buffer)
TD_choice = np.random.choice(len(TD_batch), size = batch_size, replace = False, p = TD_batch)
for i in range(batch_size):
minibatch.append(replay_buffer[TD_choice[i]])
else:
minibatch = random.sample(replay_buffer, batch_size)
Train.train(Q_Network, minibatch, Exp, s_scale, input_size, self.action_dim)
if Prioritized:
for i in range(batch_size):
state_m, next_state_m, action_m, reward_m, done_m = minibatch[i]
q_t = np.max(Q_Network.evaluate_critic(np.reshape(state_m, [1, input_size])))
q_t_1 = np.max(Q_Network.evaluate_critic(np.reshape(next_state_m, [1, input_size])))
if done:
q_t_1 = reward_m
else:
q_t_1 = reward_m + dis*q_t_1
TD_buffer[TD_choice[i]] = pow(abs(q_t_1-q_t)+eps,alpha)
if global_step > batch_size and global_step % copy_step == 0:
Train.copy(Q_Network)
steps_list.append(step_count)
global_step_list.append(global_step)
# Print the average of result
if episode < ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / episode)
if episode == ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / ending_cond_epis)
if episode > ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 - ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(episode, round(step_avg_list[episode - 1], 3)))
if Exp == 'epsilon':
print (" ( Result : {}, Loss : {}, Epsilon : {}, Steps : {}, Global Steps : {} )"
.format(round(step_count, 5), round(loss, 8), round(e, 5), current_step, global_step))
else:
print (" ( Result : {}, Loss : {}, Steps : {}, Global Steps : {} )"
.format(round(step_count, 5), round(loss, 8), current_step, global_step))
# Save the networks
if episode % save_epi == 0:
# #Q_Network.save_network(episode = episode, save_epi = save_epi)
# Action_Network.save_network(episode = episode, save_epi = save_epi)
file_case = str(case_n)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_seed'+file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_global_'+'_seed'+file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode+1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
end_episode += 1
if step_avg_list[episode - 1] > ending_cond_reward:
break
print("--------------------------------------------------")
print("--------------------------------------------------")
for episode in range(end_episode + 1, max_episodes+1):
s = env.reset()
reward_sum = 0
done = False
while not done :
#env.render()
action = np.argmax(Q_Network.evaluate_critic(np.reshape(state, [1, input_size])))
if conti_action_flag:
action = [action_map[action]]
else:
action = action
state, reward, done, _ = env.step(action)
reward_sum += reward
global_step += 1
#if episode % save_epi == 0:
# Q_Network.save_network(episode = episode, save_epi = save_epi)
# Action_Network.save_network(episode = episode, save_epi = save_epi)
if done :
steps_list.append(reward_sum)
global_step_list.append(global_step)
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 - ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(episode, round(step_avg_list[episode - 1], 3)))
print (" ( Result : {} )".format(reward_sum))
if episode % save_epi == 0:
file_case = str(case_n)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_seed'+file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_global_'+'_seed'+file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode+1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
file_case = str(case_n)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_seed'+file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open('/home/jolp/Desktop/Data/'+self.file_name+'_global_'+'_seed'+file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, max_episodes+1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
def MiniBatchTrain(config, load_model=False):
# Set learning parameters
gamma = config["discount-factor"]
num_episodes = config["epochs"]
learning_rate = config["learning-rate"]
[eps_start, eps_stop, eps_steps] = config["epsilon-params"]
model_id = config["model-id"]
memory_size = config["replay-size"]
exploration = Exploration.getExplorationFromArgs(config["exploration"])
batch_size = config["batch-size"]
update_mode = config["update-mode"]
tensorboard_port = config["tensorboard"]
# Initialize TF Network and variables
Qout, inputs = Network.getNetworkFromArgs(config["architecture"])
Qmean = tf.reduce_mean(Qout)
Qmax = tf.reduce_max(Qout)
predict = tf.argmax(Qout, 1)
# Initialize TF output and optimizer
nextQ = tf.placeholder(shape=[None, 4], dtype=tf.float32)
loss = Losses.getLossFromArgs(config["loss"])(nextQ, Qout)
trainer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
updateModel = trainer.minimize(loss)
init = tf.global_variables_initializer()
# Initialize tensorboard summary
summary_op = Summary.init_summary_writer(model_name=model_id,
var_list=[("loss", loss),
("Qmean", Qmean),
("Qmax", Qmax)],
tb_port=tensorboard_port)
# Random action parameter
_epsilon = Gradients.Exponential(start=eps_start, stop=eps_stop)
def epsilon(step):
# Exponentially decreasing epsilon to 0.1 for first 25% epochs, constant value of 0.1 from there on
if step < num_episodes / (100.0 / eps_steps):
return _epsilon(step / (num_episodes / (100.0 / eps_steps)))
else:
return eps_stop
memory = ReplayMemory(memory_size)
def update_model():
if memory.full:
replay = memory.sample(batch_size)
state_list = []
target_list = []
for sample in replay:
input = sample[0]
action = sample[1]
reward_list = sample[2]
possible_states = sample[3]
targetQ = []
_, allQ = sess.run([predict, Qout],
feed_dict={inputs: [input]})
if update_mode == "single":
next_state = possible_states[action]
next_input = normalize(next_state.grid_to_input())
Q1 = sess.run(Qout, feed_dict={inputs: [next_input]})
maxQ1 = np.max(Q1)
targetQ = allQ
targetQ[0, action] = reward_list[action] + \
(0 if next_state.halt else gamma * maxQ1)
elif update_mode == "all":
next_inputs = [
normalize(s.grid_to_input()) for s in possible_states
]
Q1 = sess.run(Qout, feed_dict={inputs: next_inputs})
maxQs = [np.max(Q) for Q in Q1]
targetQ = allQ
for k in range(4):
if possible_states[k].valid:
targetQ[0, k] = reward_list[k] + \
(0 if possible_states[k].halt else gamma * maxQs[k])
state_list.insert(0, input)
target_list.insert(0, targetQ[0])
_, summary = sess.run([updateModel, summary_op],
feed_dict={
inputs: state_list,
nextQ: target_list
})
Summary.write_summary_operation(summary, total_steps + steps)
with tf.Session() as sess:
sess.run(init)
total_steps = 0
for i in range(num_episodes):
# Reset environment and get first new observation
state = Game.new_game(4)
reward_sum = 0
steps = 0
rand_steps = 0
invalid_steps = 0
# The Q-Network
while not state.halt:
s = normalize(state.grid_to_input())
steps += 1
if i == 0:
state.printstate()
print ""
# Choose an action by greedily (with e chance of random action) from the Q-network
a, allQ = sess.run([predict, Qout], feed_dict={inputs: [s]})
possible_states, action, ra, invalid_prediction = exploration(
a[0], allQ, i, epsilon, state)
if ra:
rand_steps += 1
if invalid_prediction:
invalid_steps += 1
reward_list = []
for k, nextstate in enumerate(possible_states):
r = reward(state, nextstate)
if r is not 0:
r = np.log2(nextstate.score - state.score) / 2.0
reward_list.insert(k, r)
reward_sum += reward_list[action]
# [normailzed input, action_taken, rewards for all action, all possible states]
memory.push([s, action, reward_list, possible_states])
# update step
update_model()
state = possible_states[action]
maxtile = max([max(state.grid[k]) for k in range(len(state.grid))])
stat = {
'max-tile': maxtile,
'score': state.score,
'steps': steps,
'r': reward_sum,
'rand-steps': "{0:.3f}".format(float(rand_steps) / steps)
}
total_steps += steps
Summary.write_scalar_summaries(
[
("steps", steps),
("epsilon", epsilon(i)),
("score", state.score),
("rand-steps", float(rand_steps) / steps),
("maxtile", maxtile),
# ("invalid-steps", steps)
],
i)
print i, "\t", stat
sess.close()
def run_DQN(self, case_n, seed_n, Exp, Double, Dueling, Prioritized):
sess = self.sess
dis = self.dis
REPLAY_MEMORY = self.REPLAY_MEMORY
replay_memory = self.replay_memory
batch_size = self.batch_size
Game = self.Game
save_epi = self.save_epi
max_episodes = self.max_episodes
env = self.env
input_size = self.input_size
output_size = self.output_size
alpha = self.alpha
beta_init = self.beta_init
eps = self.eps
eps_div = self.eps_div
s_scale = self.s_scale
training_step = self.training_step
copy_step = self.copy_step
repu_num = self.repu_num
ending_cond_epis = self.ending_cond_epis
ending_cond_reward = self.ending_cond_reward
conti_action_flag = self.conti_action_flag
action_map = self.action_map
env.seed(seed_n)
np.random.seed(seed_n)
tf.set_random_seed(seed_n)
random.seed(seed_n)
Q_Network = self.Q_Network
end_episode = 0
step_count_total = 0
global_step = 0
loss = 0
replay_buffer = deque()
Q_list = []
TD_buffer = deque()
steps_list = []
step_avg_list = []
global_step_list = []
print("")
print("CASE {}".format(case_n))
print(" STATE DIM : {}, ACTION DIM : {}".format(
input_size, self.action_dim))
print(" Exp : {}".format(Exp))
print(
" Strategy : Double : {}, Dueling : {}, Prioritized : {}".format(
Double, Dueling, Prioritized))
t = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = 0
for episode in range(1, max_episodes + 1):
t1 = time.time()
#print( "TIME {} --- EPISODE {}".format(t1-t,episode))
t = t1
done = False
step_count = 0
current_step = 0
cost = 0
state = env.reset()
while not done:
t7 = time.time()
e = 1. / ((float(episode - 1) / eps_div) + 1)
action = Exploration.choice_action(
Exp, e, s_scale,
Q_Network.evaluate_critic(
np.reshape(state, [1, input_size]))[0])
if conti_action_flag:
action0 = [action_map[action]]
else:
action0 = action
next_state, reward, done, _ = env.step(action0)
step_count += reward
global_step += 1
current_step += 1
#t2 = time.time()
#print( "TIME {} --- 1 {} {}".format(t2-t7,episode,current_step))
if Prioritized:
replay_memory.save_experience(state, action, reward,
next_state, done)
else:
replay_buffer.append(
(state, next_state, action, reward, done))
if len(replay_buffer) > REPLAY_MEMORY:
replay_buffer.popleft()
#t2 = time.time()
#print( "TIME {} --- 2 {} {}".format(t2-t7,episode,current_step))
state = next_state
replay_memory.anneal_per_importance_sampling(
global_step, max_episodes * 1000)
if global_step > batch_size and global_step % training_step == 0:
for re in range(repu_num):
minibatch = []
if Prioritized:
idx, priorities, w_batch, experience = replay_memory.retrieve_experience(
batch_size)
minibatch = self.format_experience(
experience, minibatch)
errors, cost = Train.train_prioritized(
Q_Network, minibatch, w_batch, Exp, s_scale,
input_size, output_size)
#print(errors)
#t2 = time.time()
#print( "TIME {} --- 3 {} {}".format(t2-t7,episode,current_step))
"""
errors = []
for i in range(batch_size):
state_m, next_state_m, action_m, reward_m, done_m = minibatch[i]
q_t = np.max(Q_Network.evaluate_critic(np.reshape(state_m, [1, input_size])))
q_t_1 = np.max(Q_Network.evaluate_critic(np.reshape(next_state_m, [1, input_size])))
if done_m:
q_t_1 = reward_m
else:
q_t_1 = reward_m + dis*q_t_1
errors.append(q_t_1-q_t)
"""
#errors = Train.train_error(Q_Network, minibatch, Exp, s_scale, input_size, output_size)
#t2 = time.time()
#print( "TIME {} --- 4 {} {}".format(t2-t7,episode,current_step))
replay_memory.update_experience_weight(idx, errors)
else:
minibatch = random.sample(replay_buffer,
batch_size)
Train.train(Q_Network, minibatch, Exp, s_scale,
input_size, self.action_dim)
if global_step > batch_size and global_step % copy_step == 0:
Train.copy(Q_Network)
#t8 = time.time()
#print( "TIME {} --- CYCLE {} {}".format(t8-t7,episode,current_step))
steps_list.append(step_count)
global_step_list.append(global_step)
# Print the average of result
if episode < ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / episode)
if episode == ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / ending_cond_epis)
if episode > ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 - ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(
episode, round(step_avg_list[episode - 1], 3)))
if Exp == 'epsilon' or Exp == 'sparsemax':
print(
" ( Result : {}, Loss : {}, Epsilon : {}, Steps : {}, Global Steps : {} )"
.format(round(step_count, 5), round(cost, 5), round(e, 5),
current_step, global_step))
else:
print(
" ( Result : {}, Loss : {}, Steps : {}, Global Steps : {} )"
.format(round(step_count, 5), round(cost, 5), current_step,
global_step))
# Save the networks
if episode % save_epi == 0:
file_case = str(case_n)
Q_Network.save_network(game_name=self.file_name + '_seed' +
file_case,
episode=episode,
save_epi=save_epi)
with open(
'/home/jolp/Desktop/Data/' + self.file_name + '_seed' +
file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/jolp/Desktop/Data/' + self.file_name +
'_global_' + '_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
end_episode += 1
if step_avg_list[episode - 1] > ending_cond_reward:
break
print("--------------------------------------------------")
print("--------------------------------------------------")
for episode in range(end_episode + 1, max_episodes + 1):
s = env.reset()
reward_sum = 0
done = False
while not done:
#env.render()
action = np.argmax(
Q_Network.evaluate_critic(
np.reshape(state, [1, input_size])))
if conti_action_flag:
action = [action_map[action]]
else:
action = action
state, reward, done, _ = env.step(action)
reward_sum += reward
global_step += 1
#if episode % save_epi == 0:
# Q_Network.save_network(episode = episode, save_epi = save_epi)
# Action_Network.save_network(episode = episode, save_epi = save_epi)
if done:
steps_list.append(reward_sum)
global_step_list.append(global_step)
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 -
ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(
episode, round(step_avg_list[episode - 1], 3)))
print(" ( Result : {} )".format(
reward_sum))
if episode % save_epi == 0:
file_case = str(case_n)
Q_Network.save_network(game_name=self.file_name + '_seed' +
file_case,
episode=episode,
save_epi=save_epi)
with open(
'/home/jolp/Desktop/Data/' + self.file_name + '_seed' +
file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/jolp/Desktop/Data/' + self.file_name +
'_global_' + '_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
file_case = str(case_n)
with open(
'/home/jolp/Desktop/Data/' + self.file_name + '_seed' +
file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/jolp/Desktop/Data/' + self.file_name + '_global_' +
'_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, max_episodes + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
def getExplorer(self):
explorer = Exploration.ParallelExplorer(self.parallelCfg)
return explorer
def SmoothMFD (Db, a, Wkt, Window=GaussWin, Par=50.,
Delta=0.1, SphereGrid=False,
Box=[], Buffer=[], Grid=[],
Threshold=-100, Unwrap=False,
ZeroRates=False):
if Par <= 0:
Par = np.inf
# Catalogue selection
DbS = Sel.AreaSelect(Db, Wkt, Owrite=0, Buffer=Buffer, Unwrap=Unwrap)
x,y,z = Exp.GetHypocenter(DbS)
# Creating the mesh grid
P = CU.Polygon()
P.Load(Wkt)
# Unwrapping coordinates
if Unwrap:
x = [i if i > 0. else i+360. for i in x]
P.Unwrap()
if Grid:
XY = [G for G in Grid if P.IsInside(G[0], G[1])]
else:
if SphereGrid:
XY = P.SphereGrid(Delta=Delta, Unwrap=Unwrap)
else:
XY = P.CartGrid(Dx=Delta, Dy=Delta, Bounds=Box)
Win = []
for xyP in XY:
Win.append(0)
for xyE in zip(x,y):
Dis = CU.WgsDistance(xyP[1], xyP[0], xyE[1], xyE[0])
Win[-1] += Window(Dis, Par)
# Scaling and normalising the rates
Norm = np.sum(Win)
A = []; X = []; Y = []
for I,W in enumerate(Win):
aT = -np.inf
if Norm > 0. and W > 0.:
aT = a + np.log10(W/Norm)
if aT < Threshold:
# Filter below threshold
aT = -np.inf
if ZeroRates:
A.append(aT)
X.append(XY[I][0])
Y.append(XY[I][1])
else:
if aT > -np.inf:
A.append(aT)
X.append(XY[I][0])
Y.append(XY[I][1])
if Unwrap:
# Wrap back longitudes
X = [x if x < 180. else x-360. for x in X]
return X, Y, A
def run_DQN(self, seed_n, Exp, Double, Prioritized):
############## parameter 복사 ##############
sess = self.sess
dis = self.dis
REPLAY_MEMORY = self.REPLAY_MEMORY
replay_memory = self.replay_memory
batch_size = self.batch_size
size_action_batch = self.size_action_batch
Game = self.Game
save_epi = self.save_epi
save_network = self.save_network
max_episodes = self.max_episodes
max_steps = self.max_steps
env = self.env
random_action = self.random_action
input_size = self.input_size
output_size = self.output_size
alpha = self.alpha
beta_init = self.beta_init
beta_max_step = self.beta_max_step
eps = self.eps
eps_div = self.eps_div
s_scale = self.s_scale
training_step = self.training_step
copy_step = self.copy_step
action_copy_step = self.action_copy_step
action_train = self.action_train
weighted_train = self.weighted_train
repu_num = self.repu_num
DDPG = self.DDPG
ending_cond_epis = self.ending_cond_epis
ending_cond_reward = self.ending_cond_reward
env.seed(seed_n)
np.random.seed(seed_n)
tf.set_random_seed(seed_n)
random.seed(seed_n)
#############################################
Q_Network = self.Q_Network
A_batch = Q_Network.get_action_batch()
if DDPG:
Action_Network = self.Action_Network
# DDPG Action Network 학습 시 사용되는 grad_inv 설정
action_max = np.array(env.action_space.high).tolist()
action_min = np.array(env.action_space.low).tolist()
action_bounds = [action_max, action_min]
grad_inv = grad_inverter(sess, action_bounds)
case_n = seed_n + 1
end_episode = 0
step_count_total = 0
global_step = 0
loss = 0
e = 1.
replay_buffer = deque()
Q_list = []
TD_buffer = deque()
steps_list = []
step_avg_list = []
global_step_list = []
average_distance = []
rate_of_adjacent = []
print("")
print("CASE {}".format(case_n))
print(" STATE DIM : {}, ACTION DIM : {}".format(
input_size, self.action_dim))
print(" Exp : {}".format(Exp))
if DDPG:
print(" Strategy : Double : {}, Prioritized : {}, DDPG : {}".
format(Double, Prioritized, DDPG))
elif random_action:
if action_train:
print(
" Strategy : Double : {}, Prioritized : {}, ACTION : RANDOM, ACTION TRAIN 'ON'"
.format(Double, Prioritized))
else:
print(
" Strategy : Double : {}, Prioritized : {}, ACTION : RANDOM"
.format(Double, Prioritized))
else:
if action_train:
print(
" Strategy : Double : {}, Prioritized : {}, ACTION : DISCRETIZATION, ACTION TRAIN 'ON'"
.format(Double, Prioritized))
else:
print(
" Strategy : Double : {}, Prioritized : {}, ACTION : DISCRETIZATION"
.format(Double, Prioritized))
print("")
for episode in range(1, max_episodes + 1):
done = False
step_count = 0
current_step = 0
cost = 0
state = env.reset()
while not done:
# 입실론 값 조정, 0.001미만이 될 시 더 이상 작아지지 않는다.
if e > 0.001:
#e = 1. / ((float(episode - 1) / eps_div) + 1)
e = 1. / ((float(global_step) / eps_div) + 1)
t4 = time.time()
if DDPG: # DDPG true 시, 액션네트워크로부터 행동을 결정받음
action = Action_Network.evaluate_actor(
np.reshape(state, [1, input_size]))[0]
else: # DDPG false 시, state에 따른 각 행동 별 q 값을 get_q_batch로 받은 후 Exploration 방식에 따라 행동 결정
action0 = Exploration.choice_action(Exp, e, s_scale,\
np.reshape(Q_Network.get_q_batch(np.reshape(state,[1,-1])),[1,-1])[0])
action = A_batch[action0]
next_state, reward, done, _ = env.step(action)
step_count += reward
global_step += 1
current_step += 1
# Prioritized 시 tree(replay_memory)에 저장, 아닐 시 랜덤으로 추출할 replay_beffer에 저장
if Prioritized:
replay_memory.save_experience(state, action, reward,
next_state, done)
else:
replay_buffer.append(
(state, next_state, action, reward, done))
if len(replay_buffer) > REPLAY_MEMORY:
replay_buffer.popleft()
state = next_state
if global_step <= beta_max_step:
replay_memory.anneal_per_importance_sampling(
global_step, beta_max_step)
# training step마다 traing 실행
if global_step > batch_size and global_step % training_step == 0:
for re in range(
repu_num): # repu_num만큼 반복 training. 거의 1로 사용.
if Prioritized:
# replay_memory로부터 batch를 추출
idx, priorities, w_batch, experience = replay_memory.retrieve_experience(
batch_size)
minibatch = self.format_experience(experience)
if DDPG:
# DDPG true시 Q네트워크와 Action네트워크 모두 training
errors, cost = Train.train_prioritized_DDPG(
Q_Network, Action_Network, minibatch,
w_batch, output_size, grad_inv)
replay_memory.update_experience_weight(
idx, errors)
else:
# DDPG false시 Q네트워크 training
errors, cost, state_t_batch = Train.train_prioritized(
Q_Network, minibatch, w_batch, Exp,
s_scale, input_size, output_size,
size_action_batch)
replay_memory.update_experience_weight(
idx, errors)
# action_copy_step 마다 action set을 training, action_train이 false 시 RAS 알고리즘
if action_train and global_step % action_copy_step == 0:
action_weight = []
if weighted_train: # WARAS 알고리즘
# weight 계산
for k in range(batch_size):
state_t = np.reshape(
state_t_batch[k], [1, -1])
q_batch = Q_Network.get_q_batch(
state_t)
q_batch = np.reshape(
q_batch, [1, -1])[0]
q_batch = q_batch * 10.
max_q = np.max(q_batch)
q_batch = np.exp(q_batch - max_q)
action_weight.append(q_batch)
else: # ARAS 알고리즘
# 모든 weight를 1로 설정
action_weight = np.ones(
[batch_size, size_action_batch])
# weight 값을 이용한 Q네트워크 training
Q_Network.train_weighted_actor(
state_t_batch, action_weight)
# target-action set을 update
Q_Network.update_action_target_critic()
A_batch = Q_Network.get_action_batch()
t_A_batch = Q_Network.get_target_action_batch(
)
"""
# 거리가 가까운 action 쌍을 찾아 resampling
A_batch, t_A_batch = self.realign_action_batch(A_batch, t_A_batch)
Q_Network.realign_action_batch(A_batch, t_A_batch)
A_batch = Q_Network.get_action_batch()
t_A_batch = Q_Network.get_target_action_batch()
"""
else: # Prioritized가 아닐 시 랜덤하게 minibatch를 생성해 training
minibatch = random.sample(replay_buffer,
batch_size)
if DDPG:
cost = Train.train_DDPG(
Q_Network, Action_Network, minibatch,
output_size, grad_inv)
else:
cost, state_t_batch = Train.train(
Q_Network, minibatch, Exp, s_scale,
input_size, output_size, size_action_batch)
# copy_step 마다 Q네트워크 업데이트
if global_step % copy_step == 0:
if DDPG:
# Update target Critic and actor network
Q_Network.update_target_critic()
Q_Network.update_action_target_critic()
Action_Network.update_target_actor()
else:
Q_Network.update_target_critic()
Q_Network.update_action_target_critic()
steps_list.append(step_count)
global_step_list.append(global_step)
# Print the average of result
if episode < ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / episode)
if episode == ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_avg_list.append(step_count_total / ending_cond_epis)
if episode > ending_cond_epis:
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 - ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(
episode, round(step_avg_list[episode - 1], 3)))
if DDPG:
print (" ( Result : {}, Loss : {}, Steps : {}, Global Steps : {} )"
#.format(round(step_count, 3), round(cost, 5), current_step, global_step))
.format(round(step_count, 3), 0, current_step, global_step))
elif Exp == 'epsilon' or Exp == 'sparsemax':
print (" ( Result : {}, Loss : {}, Epsilon : {}, Steps : {}, Global Steps : {} )"
#.format(round(step_count, 3), round(cost, 5), round(e, 4), current_step, global_step))
.format(round(step_count, 3), 0, round(e, 5), current_step, global_step))
else:
print (" ( Result : {}, Loss : {}, Steps : {}, Global Steps : {} )"
#.format(round(step_count, 3), round(cost, 5), current_step, global_step))
.format(round(step_count, 3), 0, current_step, global_step))
distance, per_of_sim, per_of_sim2 = self.get_action_variance(
A_batch)
print(
" ( Action Batch :::: Distance : {}, Percent : {}%({}%) )"
.format(distance, per_of_sim, per_of_sim2))
average_distance.append(distance)
rate_of_adjacent.append(per_of_sim)
# Save the networks
if episode % save_epi == 0:
file_case = str(case_n)
if save_network:
Q_Network.save_network(game_name=self.file_name + '_seed' +
file_case,
episode=episode,
save_epi=save_epi)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name +
'_seed' + file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name +
'_global_' + '_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
with open(
'/home/minjae/Desktop/JOLP/' +
'Average_of_Distance_(' + self.file_name + '_seed' +
file_case + ')', 'wb') as fout:
pickle.dump(average_distance, fout)
with open(
'/home/minjae/Desktop/JOLP/' + 'Rate_of_Adjacent_(' +
self.file_name + '_global_' + '_seed' + file_case +
')', 'wb') as fout2:
pickle.dump(rate_of_adjacent, fout2)
p_values = list(range(1, episode + 1))
q_values = average_distance[:]
r_values = rate_of_adjacent[:]
plt.plot(p_values, q_values, c='r')
plt.title('Average of Distance between Actions')
plt.grid(True)
plt.show()
plt.plot(p_values, r_values, c='b')
plt.title('Rate of Adjacent Actions')
plt.grid(True)
plt.show()
end_episode += 1
# 결과가 목표치를 달성하면 학습 중단
if step_avg_list[episode - 1] > ending_cond_reward:
break
# max_steps 만큼 학습되었으면 학습 중단
if global_step > max_steps:
break
print("--------------------------------------------------")
print("--------------------------------------------------")
# 목표치를 달성하여 학습 중단 시, 남은 episode 만큼 실행
for episode in range(end_episode + 1, max_episodes + 1):
if global_step > max_steps:
break
s = env.reset()
reward_sum = 0
done = False
while not done:
# 최대 Q 값을 나타내는 행동 선택
action = np.argmax(
Q_Network.evaluate_critic(
np.reshape(state, [1, input_size])))
if conti_action_flag:
action = [action_map[action]]
else:
action = action
state, reward, done, _ = env.step(action)
reward_sum += reward
global_step += 1
if done:
steps_list.append(reward_sum)
global_step_list.append(global_step)
step_count_total += steps_list[episode - 1]
step_count_total -= steps_list[episode - 1 -
ending_cond_epis]
step_avg_list.append(step_count_total / ending_cond_epis)
print("{} {}".format(
episode, round(step_avg_list[episode - 1], 3)))
print(" ( Result : {} )".format(
reward_sum))
if episode % save_epi == 0:
file_case = str(case_n)
if save_network:
Q_Network.save_network(game_name=self.file_name + '_seed' +
file_case,
episode=episode,
save_epi=save_epi)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name +
'_seed' + file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name +
'_global_' + '_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
x_values = list(range(1, episode + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
# parameter 저장
file_case = str(case_n)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name + '_seed' +
file_case, 'wb') as fout:
pickle.dump(step_avg_list, fout)
with open(
'/home/minjae/Desktop/JOLP/' + self.file_name + '_global_' +
'_seed' + file_case, 'wb') as fout2:
pickle.dump(global_step_list, fout2)
# 그래프 출력
x_values = list(range(1, len(step_avg_list) + 1))
y_values = step_avg_list[:]
plt.plot(x_values, y_values, c='green')
plt.title(self.file_name)
plt.grid(True)
plt.show()
with open(
'/home/minjae/Desktop/JOLP/' + 'Average_of_Distance_(' +
self.file_name + '_seed' + file_case + ')', 'wb') as fout:
pickle.dump(average_distance, fout)
with open(
'/home/minjae/Desktop/JOLP/' + 'Rate_of_Adjacent_(' +
self.file_name + '_global_' + '_seed' + file_case + ')',
'wb') as fout2:
pickle.dump(rate_of_adjacent, fout2)
p_values = list(range(1, episode + 1))
q_values = average_distance[:]
r_values = rate_of_adjacent[:]
plt.plot(p_values, q_values, c='r')
plt.title('Average of Distance between Actions')
plt.grid(True)
plt.show()
plt.plot(p_values, r_values, c='b')
plt.title('Rate of Adjacent Actions')
plt.grid(True)
plt.show()
def filter_cases_before_2018(log):
log_filtered = EventLog(
[case for case in log if case[0]['time:timestamp'].year >= 2018])
util.print_filtered_cases_count(len(log), len(log_filtered))
return log_filtered
for key, values in criteria.items():
for value in values:
tofilter_log = attributes_filter.apply(
tofilter_log, [value],
parameters={
xes_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: key,
"positive": True
})
tofilter_cases = [
case.attributes[constants.concept_key] for case in tofilter_log
]
log_filtered = EventLog([
case for case in log
if case.attributes[constants.concept_key] not in tofilter_cases
])
util.print_filtered_cases_count(len(log), len(log) - len(tofilter_log))
return log_filtered
def determine_stage(words):
stages = ['declaration', 'permit', 'request', 'trip']
for word in words:
if word.lower() in stages:
return stages[stages.index(word.lower())]
else:
return ' '.join(words)
def determine_action(words):
for word in words:
if word.isupper():