-
Notifications
You must be signed in to change notification settings - Fork 0
/
base_optimizer.py
125 lines (105 loc) · 3.72 KB
/
base_optimizer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import threading
import zmq
from msgpack_numpy import dumps as dump
from msgpack_numpy import loads as load
from tornado.concurrent import Future
from six.moves import queue
import time
import random
import os
from tqdm import tqdm
from queue import Queue
from intelActNet import Network
from config import config
from util import *
class MasterProcess(threading.Thread):
def __init__(self, pipe_c2s=config.ip_addr+':8100', pipe_s2c=config.ip_addr+':8101'):
super(MasterProcess, self).__init__()
self.context = zmq.Context()
self.c2s_socket = self.context.socket(zmq.PULL)
self.c2s_socket.bind(pipe_c2s)
self.s2c_socket = self.context.socket(zmq.ROUTER)
self.s2c_socket.bind(pipe_s2c)
self.start_time = time.time()
self.network = Network()
if config.continue_training:
self.network.restore()
self.predict_queue = queue.Queue(maxsize = config.agent_num)
self.training_queue = queue.Queue(maxsize=config.agent_num*100)
self.predictor = PredictThread(self.predict_queue, self.network)
self.trainer = TrainingThread(self.training_queue, self.network)
self.client_memory = [Queue(maxsize=35) for _ in range(config.agent_num)]
self.predictor.start()
self.trainer.start()
def _put_predict_task(self, observation, callback):
f = Future()
f.add_done_callback(callback)
if observation is not None:
self.predict_queue.put([observation, f])
else:
f.set_result(None)
return f
def parse_memory(self, ident, observation, predicting_result):
raise(NotImplementedError())
def _on_state(self, index, obs):
def cb(output):
predicting_result = output.result()
if predicting_result is not None:
self.s2c_socket.send_multipart([nameClient(index), dump(predicting_result['action'])])
training_data = self.parse_memory(index, obs, predicting_result)
if training_data is not None:
self.training_queue.put(training_data)
self._put_predict_task(obs, cb) # add one dimension as time axis
def run(self):
while True:
for _ in tqdm(range(config.save_freq)):
client_id, observations = load(self.c2s_socket.recv(copy = False).bytes)
self._on_state(client_id, observations)
self.network.save()
config.update()
class PredictThread(threading.Thread):
def __init__(self, predicting_queue, network):
super(PredictThread, self).__init__()
self.recv_queue = predicting_queue
self.network = network
def run(self):
while True:
predicting_batch = []
future_batch = []
observation, future = self.recv_queue.get()
predicting_batch.append(observation)
future_batch.append(future)
while len(predicting_batch) < config.batch_size:
try:
observation, future = self.recv_queue.get_nowait()
predicting_batch.append(observation)
future_batch.append(future)
except queue.Empty:
break
predicting_results = self.network.predict(predicting_batch)
for c in range(predicting_results.shape[0]):
predicting_result = predicting_results[c]
if random.random() > config.p_explore:
rtn_action = np.where(predicting_result == max(predicting_result))[0][0]
else:
rtn_action = np.random.choice(range(config.num_actions))
final_result = dict()
final_result['action'] = rtn_action
future_batch[c].set_result(final_result)
class TrainingThread(threading.Thread):
def __init__(self, training_queue, network):
super(TrainingThread, self).__init__()
self.recv_queue = training_queue
self.network = network
def run(self):
while True:
training_batch = []
training_data = self.recv_queue.get()
training_batch.append(training_data)
while len(training_batch) < config.batch_size:
try:
training_data = self.recv_queue.get_nowait()
training_batch.append(training_data)
except queue.Empty:
break
self.network.train(training_batch)