def train(self, env, args, summary_writer):
results = []
# First, gather experience
# tqdm_e = tqdm(range(args.nb_episodes), desc='Score', leave=True, unit=" episodes")
# for e in tqdm_e:
for e in range(args.nb_episodes):
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
actions, states, rewards = [], [], []
noise = OrnsteinUhlenbeckProcess(size=self.act_dim)
while not done:
if args.render: env.render()
# env.render()
# Actor picks an action (following the deterministic policy)
a = self.policy_action(old_state)
# Clip continuous values to be valid w.r.t. environment
a = np.clip(a + noise.generate(time), -self.act_range,
self.act_range)
# Retrieve new state, reward, and whether the state is terminal
new_state, r, done, _ = env.step(a)
# Add outputs to memory buffer
self.memorize(old_state, a, r, done, new_state)
# Sample experience from buffer
states, actions, rewards, dones, new_states, _ = self.sample_batch(
args.batch_size)
# Predict target q-values using target networks
q_values = self.critic.target_predict(
[new_states,
self.actor.target_predict(new_states)])
# Compute critic target
critic_target = self.bellman(rewards, q_values, dones)
# Train both networks on sampled batch, update target networks
self.update_models(states, actions, critic_target)
# Update current state
old_state = new_state
cumul_reward += r
time += 1
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
print("score", score)
# Display score
# tqdm_e.set_description("Score: " + str(cumul_reward))
# tqdm_e.refresh()
return results
def train(self, env, args, summary_writer):
""" Main DDQN Training Algorithm
"""
results = []
tqdm_e = tqdm(range(args.nb_episodes),
desc='Score',
leave=True,
unit=" episodes")
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
while not done:
if args.render: env.render()
# Actor picks an action (following the policy)
a = self.policy_action(old_state)
# print('action ', a)
# Retrieve new state, reward, and whether the state is terminal
new_state, r, done, _ = env.step(a)
# print('reward', r)
# Memorize for experience replay
self.memorize(old_state, a, r, done, new_state)
# Update current state
old_state = new_state
cumul_reward += r
time += 1
# Train DDQN and transfer weights to target network
if (self.buffer.size() > args.batch_size):
# print('train agent')
self.train_agent(args.batch_size)
self.agent.transfer_weights()
# print('memory buffer:', self.buffer.size())
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
# print('e', e)
if (e % self.save_interval == 0) & (e != 0):
# print('save')
self.save_weights(self.export_path, e)
return results
def train(self, env, args, summary_writer):
""" Main A2C Training Algorithm
"""
results = []
# Main Loop
tqdm_e = tqdm(range(args.nb_episodes),
desc='Score',
leave=True,
unit=" episodes")
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
actions, states, rewards = [], [], []
while not done:
if args.render: env.render()
# Actor picks an action (following the policy)
a = self.policy_action(old_state)
# Retrieve new state, reward, and whether the state is terminal
new_state, r, done, _ = env.step(a)
# Memorize (s, a, r) for training
actions.append(to_categorical(a, self.act_dim))
rewards.append(r)
states.append(old_state)
# Update current state
old_state = new_state
cumul_reward += r
time += 1
# Train using discounted rewards ie. compute updates
self.train_models(states, actions, rewards, done)
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
return results
def train(self, env, args, summary_writer):
""" Main DDQN Training Algorithm DDQN主要训练算法
"""
results = []
tqdm_e = tqdm(range(args.nb_episodes),
desc='Score',
leave=True,
unit=" episodes")
for e in tqdm_e:
# Reset episode 重设episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
while not done:
if args.render: env.render()
# Actor picks an action (following the policy) 演员选择动作（遵循政策）
a = self.policy_action(old_state)
# Retrieve new state, reward, and whether the state is terminal 检索新状态，奖励以及该状态是否为终端
new_state, r, done, _ = env.step(a)
# Memorize for experience replay 保存经验重播
self.memorize(old_state, a, r, done, new_state)
# Update current state 更新当前状态
old_state = new_state
cumul_reward += r
time += 1
# Train DDQN and transfer weights to target network 训练DDQN并将权重转移到目标网络
if (self.buffer.size() > args.batch_size):
self.train_agent(args.batch_size)
self.agent.transfer_weights()
# Gather stats every episode for plotting 收集每个情节的统计数据以进行绘图
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard 为Tensorboard导出结果
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score 显示分数
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
return results
def train(self, env, args, summary_writer, envtest=None):
""" Main DDQN Training Algorithm
"""
results = []
tqdm_e = tqdm(range(args.nb_episodes),
desc='Score',
leave=True,
unit=" episodes")
epoch = 0
gross_profit = 0
WritetoCsvFile("logFile_1.csv", [
"stage", "file", "history_win", "stop", "usevol", "dueling",
"traineval", "allprices", "allprices2", "allprices3", "ma1", "ma2",
"madifference", "hidema", "candlenum", "hidden_dim", "maxProfit",
"maxLOSS", "avgProfit", "avgLOSS", "countprofit", "countloss",
"maxdrop", "Total profit", "total_reward", "TRADES", "epoch"
])
WritetoCsvFile("logFileDetail.csv", [
"stage", "file", "history_win", "stop", "usevol", "dueling",
"traineval", "allprices", "allprices2", "allprices3", "ma1", "ma2",
"madifference", "hidema", "candlenum", "hidden_dim", 'maxProfit',
'maxLOSS', 'avgProfit', 'avgLOSS', 'maxdrop', 'Total profit',
'gross profit', "total_reward", 'TRADES', 'epoch'
])
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
##########################################
total_reward = 0
total_profit = 0
total_loss = 0
total_profitMax = 0
total_profitMin = 0
max_drop = 0
profitLst = []
lossLst = []
trades = 0
step = 0
#####################################3####
while not done:
#if args.render: env.render()
# Actor picks an action (following the policy)
a = self.policy_action(old_state)
# Retrieve new state, reward, and whether the state is terminal
#new_state, r, done, _ = env.step(a)
#######################################################
new_state, r, done, buy, sell, profit = env.step(a)
total_reward += r
if profit != 0:
trades += 1
total_profit += profit
if total_profit > total_profitMax:
total_profitMax = total_profit
total_profitMin = total_profit
if total_profit < total_profitMin:
total_profitMin = total_profit
try:
if total_profitMax != 0 and max_drop < (
total_profitMax -
total_profitMin) / total_profitMax:
max_drop = (total_profitMax -
total_profitMin) / total_profitMax
except:
max_drop = 0
if profit > 0:
profitLst.append(profit)
elif profit < 0:
lossLst.append(profit)
step += 1
if step % 1500 == 0:
print(
'maxProfit: {} maxLOSS: {} avgProfit: {:01.2f} avgLOSS: {:01.2f} maxdrop: {:.2%} Total profit: {}/{} TRADES: {} '
.format(np.max(profitLst + [0]),
-np.min(lossLst + [0]),
np.mean(profitLst + [0]),
-np.mean(lossLst + [0]), max_drop,
total_profit, gross_profit, trades))
WritetoCsvFile("logFileDetail.csv", [
"train", args.trainf, args.history_win, args.stop,
args.usevol, args.dueling, args.traineval,
args.allprices, args.allprices2, args.allprices3,
args.ma1, args.ma2, args.madifference, args.hidema,
args.candlenum, args.hidden_dim,
np.max(profitLst + [0]), -np.min(lossLst + [0]),
np.mean(profitLst + [0]), -np.mean(lossLst + [0]),
max_drop, total_profit, gross_profit, total_reward,
trades, epoch
])
#done = True if step == len(env.data) - 3 else False
######################################################
# Memorize for experience replay
self.memorize(old_state, a, r, done, new_state)
# Update current state
old_state = new_state
cumul_reward += r
time += 1
# Train DDQN and transfer weights to target network
if (self.buffer.size() > args.batch_size):
self.train_agent(args.batch_size)
self.agent.transfer_weights()
gross_profit += total_profit
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
l_profit = tfSummary('profit', total_profit)
l_aprofit = tfSummary('average profit', np.mean(profitLst))
l_aloss = tfSummary('l_aloss', -np.mean(lossLst))
l_trades = tfSummary('l_trades', trades)
np.mean(profitLst), -np.mean(lossLst)
summary_writer.add_summary(score, global_step=e)
summary_writer.add_summary(l_profit, global_step=e)
summary_writer.add_summary(l_aprofit, global_step=e)
summary_writer.add_summary(l_aloss, global_step=e)
summary_writer.add_summary(l_trades, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
self.agent.saveModel("./models/model_ep", "")
results = [
np.max(profitLst + [0]), -np.min(lossLst + [0]),
np.mean(profitLst + [0]), -np.mean(lossLst + [0]),
len(profitLst),
len(lossLst), max_drop, total_profit, total_reward, trades
]
WritetoCsvFile("logFile_1.csv", [
"train", args.trainf, args.history_win, args.stop, args.usevol,
args.dueling, args.traineval, args.allprices, args.allprices2,
args.allprices3, args.ma1, args.ma2, args.madifference,
args.hidema, args.candlenum, args.hidden_dim
] + results + [epoch])
if envtest: # Если задано окружение для тестирования то тестируем каждую эпоху
newargs = args
newargs.traineval = False
self.evaluate(envtest,
newargs,
summary_writer,
model=None,
epoch=epoch)
epoch += 1
return results
def train(self, summary_writer):
env = CarEnv()
results = []
i = 0
# First, gather experience
tqdm_e = tqdm(range(2000), desc='Score', leave=True, unit=" episodes")
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
old_state = np.array(old_state).reshape(40, )
actions, states, rewards = [], [], []
noise = OrnsteinUhlenbeckProcess(size=self.act_dim)
while not done:
# if args.render: env.render()
# Actor picks an action (following the deterministic policy)
a = self.policy_action(old_state)
# Clip continuous values to be valid w.r.t. environment
a = np.clip(a + noise.generate(time), -self.act_range,
self.act_range)
a = float(a[0])
# Retrieve new state, reward, and whether the state is terminal
new_state, r, done, _ = env.step(a, time)
print("Now r is {}".format(r))
# Add outputs to memory buffer
temp_next = old_state.copy()
temp_next[:4] = temp_next[4:8]
temp_next[4:8] = temp_next[8:12]
temp_next[8:12] = temp_next[12:16]
temp_next[12:16] = temp_next[16:20]
temp_next[16:20] = temp_next[20:24]
temp_next[20:24] = temp_next[24:28]
temp_next[24:28] = temp_next[28:32]
temp_next[28:32] = temp_next[32:36]
temp_next[32:36] = temp_next[36:40]
temp_next[36:40] = new_state
temp_next = np.array(temp_next).reshape(40, )
self.memorize(old_state, a, r, done, temp_next)
old_state = temp_next.copy()
cumul_reward += r
time += 1
# since episode is over destroying actors in the scenario
for actor in env.actor_list:
actor.destroy()
# Sample experience from buffer
for i in range(50):
states, actions, rewards, dones, new_states, _ = self.sample_batch(
64)
# Predict target q-values using target networks
q_values = self.critic.target_predict(
[new_states,
self.actor.target_predict(new_states)])
# Compute critic target
critic_target = self.bellman(rewards, q_values, dones)
# Train both networks on sampled batch, update target networks
self.update_models(states, actions, critic_target)
print("learning happened")
# mean, stdev, velocity_data, action_data, vehicle_x_data, vehicle_y_data, obj_x_data, obj_y_data = gather_stats(self, env, velocity_data, action_data, vehicle_x_data, vehicle_y_data, obj_x_data, obj_y_data)
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
print(cumul_reward)
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
i += 1
if i % 10 == 0:
df = pd.DataFrame(np.array(results))
df.to_csv("DDPG" + "/logs.csv",
header=['Episode', 'Mean', 'Stddev'],
float_format='%10.5f')
return results
def train(self, env, args, summary_writer):
""" Main A2C Training Algorithm
"""
# self.pretrain_random(env, args, summary_writer)
results = []
possible_states = [np.asarray(0), np.asarray(1)]
# Main Loop
tqdm_e = tqdm(range(args.nb_episodes),
desc='Score',
leave=True,
unit=" episodes")
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
actions, states, rewards = [], [], []
while not done:
if args.render: env.render()
# if (e%64==1)&(e>30):
# if args.render: env.render()
# Actor picks an action (following the policy)
if e < 30:
a = [
random.choice(possible_states),
random.choice(possible_states),
random.choice(possible_states),
random.choice(possible_states)
]
elif np.random.rand() < 0.5:
a = [
random.choice(possible_states),
random.choice(possible_states),
random.choice(possible_states),
random.choice(possible_states)
]
else:
a = self.policy_action(old_state, e)
#feedforward
# Retrieve new state, reward, and whether the state is terminal
new_state, r, done, _ = env.step(a)
#self.c_opt([states, discounted_rewards])
# print(novelty)
# Memorize (s, a, r) for training
# actions.append(to_categorical(a, self.act_dim))
actions.append(a)
states.append(old_state)
# compute the novelty
# print(self.env_dim[1])
last_state = states[-1].reshape((1, 4, self.env_dim[1]))
novelty = 0
# novelty=self.rnd_opt([last_state,last_state])[0]
rewards.append(r + 0.0001 * novelty)
# Update current state
old_state = new_state
cumul_reward += r + 0.0001 * novelty
time += 1
# Train using discounted rewards ie. compute updates
self.her.add(states, np.asarray(actions), rewards)
# print(np.asarray(states).shape, np.asarray(actions).shape, np.asarray(rewards).shape)
# only update every 10 episodes?
if e > 24:
for item in self.her.sample():
states, actions, rewards, completed = item
# print(np.asarray(states).shape, np.asarray(actions).shape, np.asarray(rewards).shape)
states = np.asarray(states)[-min(1000, len(rewards)):]
actions = np.asarray(actions)[-min(1000, len(rewards)):]
rewards = np.asarray(rewards)[-min(1000, len(rewards)):]
self.train_models(states, actions, rewards, completed)
# try:
# for item in self.her.sample():
# states, actions, rewards, completed=item
# print(np.asarray(states).shape, np.asarray(actions).shape, np.asarray(rewards).shape)
# states=np.asarray(states)[-max(1000, len(rewards)):]
# actions=np.asarray(actions)[-max(1000, len(rewards)):]
# rewards=np.asarray(rewards)[-max(1000, len(rewards)):]
# self.train_models(states, actions, rewards, completed)
# except:
# print('error training critic')
# for item in self.her.sample():
# states, actions, rewards, completed=item
# self.train_models(states, np.asarray(actions), rewards, done)
# # self.train_models(states, np.asarray(actions), rewards, done)
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score: " + str(cumul_reward))
tqdm_e.refresh()
return results
def pretrain_random(self,
env,
args,
summary_writer,
train_steps=200,
env_steps=100):
"""
Generate a somewhat random output so that the agent explores.
"""
results = []
# Main Loop
tqdm_e = tqdm(range(train_steps),
desc='pretrain',
leave=True,
unit=" episodes")
for e in tqdm_e:
# Reset episode
time, cumul_reward, done = 0, 0, False
old_state = env.reset()
actions, states, rewards = [], [], []
old_a = np.asarray(np.zeros_like(self.policy_action(old_state, e)))
while not done:
# if args.render: env.render()
# Actor picks an action (following the policy)
a = self.policy_action(old_state, e)
# if np.random.rand()<0.1:
# print(a)
# print(a)
#feedforward
# Retrieve new state, reward, and whether the state is terminal
new_state, _, done, _ = env.step(a)
r = np.random.choice(
((np.asarray(a).reshape(-1) - old_a.reshape(-1))**2)[:2])
old_a = np.asarray(a)
#self.c_opt([states, discounted_rewards])
# Memorize (s, a, r) for training
actions.append(to_categorical(a, self.act_dim))
rewards.append(r)
states.append(old_state)
# compute the novelty
last_state = states[-1].reshape((1, 4, 4))
novelty = self.rnd_opt([last_state, last_state])
# Update current state
old_state = new_state
cumul_reward += r
time += 1
# Train using discounted rewards ie. compute updates
try:
self.train_models(states, np.asarray(actions), rewards, done)
except:
print('error training critic')
self.train_models(states, np.asarray(actions), rewards, done)
# Gather stats every episode for plotting
if (args.gather_stats):
mean, stdev = gather_stats(self, env)
results.append([e, mean, stdev])
# Export results for Tensorboard
score = tfSummary('score', cumul_reward)
summary_writer.add_summary(score, global_step=e)
summary_writer.flush()
# Display score
tqdm_e.set_description("Score:{}, Nov.: {}".format(
str(cumul_reward), novelty))
tqdm_e.refresh()