def onMessage(self, payload, isBinary):
if not self.factory.agent_conn.check_message(self):
return
assert not isBinary, "Binary websocket not supported"
payload = ujson.loads(payload)
context = {
'start': time.time(),
'conn': self,
}
latency = context['start'] - payload['headers']['sent_at']
pyprofile.incr('rewarder_protocol.messages')
pyprofile.incr('rewarder_protocol.messages.{}'.format(
payload['method']))
pyprofile.timing('rewarder_protocol.latency.rtt.skew_unadjusted',
2 * latency)
if latency < 0:
pyprofile.incr(
'rewarder_protocol.latency.rtt.skew_unadjusted.negative')
if payload['method'] == 'v0.env.reset':
logger.info('Received reset message: %s', payload)
self.factory.agent_conn.control_buffer.recv_rpc(context, payload)
elif payload['method'] == 'v0.control.ping':
logger.debug('Received ping message: %s', payload)
parent_message_id = payload['headers']['message_id']
headers = {'parent_message_id': parent_message_id}
self.send_message('v0.reply.control.ping', {}, headers)
else:
logger.warn('Received unsupported message: %s', payload)
def __call__(self, img, available_at):
# Choose the return value
if len(self.deque) > 0 and self.deque[0].ready():
last = self.deque.popleft()
res = last.get()
if res is not None:
pyprofile.timing('vnc_env.diagnostics.async_decode.latency',
time.time() - res['available_at'])
else:
res = False
pyprofile.gauge('vnc_env.diagnostics.async_decode.queue_depth',
len(self.deque))
# Just grayscale it by keeping only one component. Should be
# good enough as this region is black and white anyway.
grayscale = img[self.y:self.y + self.height,
self.x:self.x + self.width, 0]
# Apply processing if needed
match = np.array_equal(self._last_img, grayscale)
if not match:
pyprofile.incr('vnc_env.diagnostics.async_decode.schedule')
# sneakily copy if numpy hasn't, so it can be cached
self._last_img = np.ascontiguousarray(grayscale)
async = self.qr_pool.apply_async(
self.method, (self._last_img, time.time(), available_at))
self.deque.append(async)
else:
pyprofile.incr('vnc_env.diagnostics.async_decode.cache_hit')
return res
def _step(self, action_n):
start = time.time()
with pyprofile.push('vnc_env.Timer.step'):
observation_n, reward_n, done_n, info = self.env.step(action_n)
# Calculate how much time was spent actually doing work
sleep = info.get('stats.throttle.sleep')
if sleep is None or sleep < 0:
sleep = 0
pyprofile.timing('vnc_env.Timer.step.excluding_sleep',
time.time() - start - sleep)
return observation_n, reward_n, done_n, info
def onMessage(self, payload, isBinary):
extra_logger.debug('[%s] Received payload: %s', self.factory.label,
payload)
assert not isBinary
payload = ujson.loads(payload)
context = self._make_context()
latency = context['start'] - payload['headers']['sent_at']
pyprofile.incr('rewarder_protocol.messages')
pyprofile.incr('rewarder_protocol.messages.{}'.format(
payload['method']))
# Double latency to model RTT
pyprofile.timing('rewarder_protocol.latency.rtt.skew_unadjusted',
2 * latency)
if latency < 0:
pyprofile.incr(
'rewarder_protocol.latency.rtt.skew_unadjusted.negative')
self.recv(context, payload)
def _step(self, action_n):
if self._steps is None:
self._start_timer()
self._steps += 1
accum_observation_n, accum_reward_n, accum_done_n, accum_info = self._substep(
action_n)
accum_info['throttle.action.available_at'] = time.time()
# Record which indexes we were just peeking at, so when we
# make the follow-up we'll be sure to peek there too.
peek_n = [
any(spaces.PeekReward for peek in action) for action in action_n
]
if self.fps is None:
return accum_observation_n, accum_reward_n, accum_done_n, accum_info
accum_info['stats.throttle.sleep'] = 0
while True:
# See how much time we have to idle
delta = self._start + 1. / self.fps * self._steps - time.time()
# The following assumes that our control loop
if delta < 0:
# We're out of time. Just get out of here.
delta = abs(delta)
if delta >= 1:
logger.info(
'Throttle fell behind by %.2fs; lost %.2f frames',
delta, self.fps * delta)
pyprofile.timing('vnc_env.Throttle.lost_sleep', delta)
self._start_timer()
break
# elif delta < 0.008:
# # Only have 8ms. Let's spend it sleeping, and
# # return an image which may have up to an
# # additional 8ms lag.
# #
# # 8ms is reasonably arbitrary; we just want something
# # that's small where it's not actually going to help
# # if we make another step call. Step with 32 parallel
# # envs takes about 6ms (about half of which is
# # diagnostics, which could be totally async!), so 8 is
# # a reasonable choice for now..
# pyprofile.timing('vnc_env.Throttle.sleep', delta)
# accum_info['stats.throttle.sleep'] += delta
# time.sleep(delta)
# break
else:
# We've got plenty of time. Sleep for up to 16ms, and
# then refresh our current frame. We need to
# constantly be calling step so that our lags are
# reported correctly, within 16ms. (The layering is
# such that the vncdriver doesn't know which pixels
# correspond to metadata, and the diagnostics don't
# know when pixels first got painted. So we do our
# best to present frames as they're ready to the
# diagnostics.)
delta = min(delta, 0.016)
pyprofile.timing('vnc_env.Throttle.sleep', delta)
accum_info['stats.throttle.sleep'] += delta
time.sleep(delta)
# We want to merge in the latest reward/done/info so that our
# agent has the most up-to-date info post-sleep, but also want
# to avoid popping any rewards where done=True (since we'd
# have to merge across episode boundaries).
action_n = []
for done, peek in zip(accum_done_n, peek_n):
if done or peek:
# No popping of reward/done
action_n.append([spaces.PeekReward])
else:
action_n.append([])
observation_n, reward_n, done_n, info = self._substep(action_n)
# Merge observation, rewards and metadata.
# Text observation has order in which the messages are sent.
rewarder.merge_n(
accum_observation_n,
accum_reward_n,
accum_done_n,
accum_info,
observation_n,
reward_n,
done_n,
info,
)
return accum_observation_n, accum_reward_n, accum_done_n, accum_info