def run(self):
"""
Run relative threads and wait until they finish (due to error)
"""
if self._supress_output:
sys.stdout = open('/tmp/' + 'latest' + ".out", "w")
sys.stderr = open('/tmp/' + 'latest' + ".err", "w")
self._param_reciever = ZmqServer(
host='*',
port=self.publish_port,
serializer=U.serialize,
deserializer=U.deserialize,
)
self._server = ZmqServer(
host='*',
port=self.serving_port,
# handler=self._handle_agent_request,
serializer=U.serialize,
deserializer=U.deserialize,
)
self._subscriber_thread = self._param_reciever.start_loop(
handler=self._set_storage, blocking=False)
self._server_thread = self._server.start_loop(
handler=self._handle_agent_request, blocking=False)
logging.info('Parameter server started')
self._subscriber_thread.join()
self._server_thread.join()
def run(self):
self._server = ZmqServer(host=self.serving_host,
port=self.serving_port,
serializer=U.serialize,
deserializer=U.deserialize,
bind=True)
self._server.start_loop(handler=self._handle_request, blocking=True)
def __init__(self,
seed,
evict_interval,
compress_before_send,
load_balanced=True,
index=0,
**kwargs):
self.config = ConfigDict(kwargs)
self.index = index
if load_balanced:
collector_port = os.environ['SYMPH_COLLECTOR_BACKEND_PORT']
sampler_port = os.environ['SYMPH_SAMPLER_BACKEND_PORT']
else:
collector_port = os.environ['SYMPH_COLLECTOR_FRONTEND_PORT']
sampler_port = os.environ['SYMPH_SAMPLER_FRONTEND_PORT']
self._collector_server = ExperienceCollectorServer(
host='localhost' if load_balanced else '*',
port=collector_port,
exp_handler=self._insert_wrapper,
load_balanced=load_balanced,
compress_before_send=compress_before_send)
self._sampler_server = ZmqServer(
host='localhost' if load_balanced else '*',
port=sampler_port,
bind=not load_balanced,
serializer=get_serializer(compress_before_send),
deserializer=get_deserializer(compress_before_send))
self._sampler_server_thread = None
self._evict_interval = evict_interval
self._evict_thread = None
self._setup_logging()
def __init__(self, learner_config, env_config, session_config, index=0):
"""
"""
# Note that there're 2 replay configs:
# one in learner_config that controls algorithmic
# part of the replay logic
# one in session_config that controls system settings
self.learner_config = learner_config
self.env_config = env_config
self.session_config = session_config
self.index = index
collector_port = os.environ['SYMPH_COLLECTOR_BACKEND_PORT']
sampler_port = os.environ['SYMPH_SAMPLER_BACKEND_PORT']
self._collector_server = ExperienceCollectorServer(
host='localhost',
port=collector_port,
exp_handler=self._insert_wrapper,
load_balanced=True,
)
self._sampler_server = ZmqServer(host='localhost',
port=sampler_port,
bind=False)
self._sampler_server_thread = None
self._evict_interval = self.session_config.replay.evict_interval
self._evict_thread = None
self._setup_logging()
def run(self):
"""
Run relative threads and wait until they finish (due to error)
"""
self._subscriber = ZmqSub(
host=self.publisher_host,
port=self.publisher_port,
# handler=self._set_storage,
topic='ps',
deserializer=U.deserialize,
)
self._server = ZmqServer(
host=self.serving_host,
port=self.serving_port,
# handler=self._handle_agent_request,
serializer=U.serialize,
deserializer=U.deserialize,
bind=not self.load_balanced,
)
self._subscriber_thread = self._subscriber.start_loop(
handler=self._set_storage, blocking=False)
self._server_thread = self._server.start_loop(
handler=self._handle_agent_request, blocking=False)
print('Parameter server started')
self._subscriber_thread.join()
self._server_thread.join()
def test_client_server_eventloop():
host = '127.0.0.1'
port = 7001
N = 20
data = [None, None]
server = ZmqServer(host=host,
port=port,
serializer='pickle',
deserializer='pickle')
all_requests = []
def handler(x):
if x == N - 1:
server.stop()
all_requests.append(x)
return x + 1
server_thread = server.start_loop(handler, blocking=False)
client = ZmqClient(host=host,
port=port,
serializer='pickle',
deserializer='pickle')
all_responses = []
for i in range(N):
all_responses.append(client.request(i))
data[1] = all_responses
server_thread.join()
assert all_requests == [n for n in range(N)]
assert all_responses == [n + 1 for n in range(N)]
def server_fn():
server = ZmqServer(host='*',
port=args.port,
serializer='pyarrow',
deserializer='pyarrow')
def f(msg):
assert msg == MSG
print('message received succesfully')
server.start_loop(f)
def server(N):
server = ZmqServer(host=host,
port=port,
serializer='pickle',
deserializer='pickle')
all_requests = []
for i in range(N):
n = server.recv()
all_requests.append(n)
server.send(n + 1)
data[0] = all_requests
def run(self):
self._server = ZmqServer(
host='*',
port=self.port,
serializer=U.pickle_serialize,
deserializer=U.pickle_deserialize,
bind=True,
)
self._server_thread = self._server.start_loop(handler=self._handle_request,
blocking=False)
logging.info('Spec server started')
self._server_thread.join()
def server():
server = ZmqServer(host=host,
port=port,
serializer='pickle',
deserializer='pickle')
while True:
req = server.recv()
if req == 'request-1':
server.send('received-request-1')
elif req == 'request-2':
time.sleep(0.5)
server.send('received-request-2')
elif req == 'request-3':
server.send('received-request-3')
break
def _start_remote_server(self):
server = ZmqServer(host='*',
port=REMOTE_PORT,
serializer=get_serializer(),
deserializer=get_deserializer(),
auth=False)
graph = self._get_graph_features()
def handler(_):
return [graph]
thr = Thread(target=lambda: server.start_loop(handler, blocking=True))
thr.daemon = True
thr.start()
return thr
def main(_):
server = ZmqServer(host='localhost',
port=PORT,
serializer=U.serialize,
deserializer=U.deserialize,
bind=True)
server_thread = server.start_loop(handler=server_f, blocking=False)
# client = ZmqClient(host='localhost',
# port=PORT,
# timeout=2,
# serializer=U.serialize,
# deserializer=U.deserialize)
client = get_ps_client()
for _ in range(10):
# client.request(['info', ['x']])
client.fetch_parameter_with_info(['x'])
print('Done!')
def __init__(self):
self.param_info = {
'time': time.time(),
'iteration': 0,
'variable_list': [],
'hash': U.pyobj_hash({}),
}
self._server = ZmqServer(
host=_LOCALHOST,
port=PS_FRONTEND_PORT,
serializer=U.serialize,
deserializer=U.deserialize,
bind=True,
)
self._server_thread = self._server.start_loop(
handler=self._handle_agent_request, blocking=False)
def main(_):
proxy = ZmqProxyThread("tcp://*:%d" % FRONTEND_PORT,
"tcp://*:%d" % BACKEND_PORT)
proxy.start()
server = ZmqServer(host='localhost',
port=BACKEND_PORT,
serializer=U.serialize,
deserializer=U.deserialize,
bind=False)
server_thread = server.start_loop(handler=server_f, blocking=False)
# client = ZmqClient(host='localhost',
# port=PORT,
# timeout=2,
# serializer=U.serialize,
# deserializer=U.deserialize)
client = get_ps_client()
for _ in range(10):
# client.request(['info', ['x']])
client.fetch_parameter_with_info(['x'])
print('Done!')