def sendMessage(self, data):
pyprofile.incr('vnc_client.data.sent.messages')
pyprofile.incr('vnc_client.data.sent.bytes',
len(data),
unit=pyprofile.BYTES)
if self.transport:
self.transport.write(data)
def parse_rectangle(cls, client, x, y, width, height, data):
decompressed = client.zlib_decompressor.decompress(data)
logger.debug('[zlib] Decompressed from %s bytes -> %s bytes', len(data), len(decompressed))
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding.decompressed_bytes', len(decompressed), unit=pyprofile.BYTES)
data = np.frombuffer(decompressed, np.uint8).reshape((height, width, 4))[:, :, [0, 1, 2]]
encoding = cls(data)
return Rectangle(x, y, width, height, encoding)
def dataReceived(self, data):
pyprofile.incr('vnc_proxy_server.data.sent.messages')
pyprofile.incr('vnc_proxy_server.data.sent.bytes', len(data))
self.buf.append(data)
self.buf_len += len(data)
self.flush()
def __call__(self, img, available_at):
# Choose the return value
if len(self.deque) > 0 and self.deque[0].ready():
last = self.deque.popleft()
res = last.get()
if res is not None:
pyprofile.timing('vnc_env.diagnostics.async_decode.latency', time.time() - res['available_at'])
else:
res = False
pyprofile.gauge('vnc_env.diagnostics.async_decode.queue_depth', len(self.deque))
# Just grayscale it by keeping only one component. Should be
# good enough as this region is black and white anyway.
grayscale = img[self.y:self.y+self.height, self.x:self.x+self.width, 0]
# Apply processing if needed
match = np.array_equal(self._last_img, grayscale)
if not match:
pyprofile.incr('vnc_env.diagnostics.async_decode.schedule')
# sneakily copy if numpy hasn't, so it can be cached
self._last_img = np.ascontiguousarray(grayscale)
async = self.qr_pool.apply_async(self.method, (self._last_img, time.time(), available_at))
self.deque.append(async)
else:
pyprofile.incr('vnc_env.diagnostics.async_decode.cache_hit')
return res
def send_env_reward(self, reward, done, info, episode_id):
pyprofile.incr('agent_conn.reward', reward)
if done:
pyprofile.incr('agent_conn.done')
reactor.callFromThread(self._send_env_reward, reward, done, info,
episode_id)
def __call__(self, img, available_at):
# Choose the return value
if len(self.deque) > 0 and self.deque[0].ready():
last = self.deque.popleft()
res = last.get()
if res is not None:
pyprofile.timing('vnc_env.diagnostics.async_decode.latency',
time.time() - res['available_at'])
else:
res = False
pyprofile.gauge('vnc_env.diagnostics.async_decode.queue_depth',
len(self.deque))
# Just grayscale it by keeping only one component. Should be
# good enough as this region is black and white anyway.
grayscale = img[self.y:self.y + self.height,
self.x:self.x + self.width, 0]
# Apply processing if needed
match = np.array_equal(self._last_img, grayscale)
if not match:
pyprofile.incr('vnc_env.diagnostics.async_decode.schedule')
# sneakily copy if numpy hasn't, so it can be cached
self._last_img = np.ascontiguousarray(grayscale)
async = self.qr_pool.apply_async(
self.method, (self._last_img, time.time(), available_at))
self.deque.append(async)
else:
pyprofile.incr('vnc_env.diagnostics.async_decode.cache_hit')
return res
def update_rectangle(self, x, y, width, height, data):
bytes = data.tobytes()
pyprofile.incr('vncdriver.pyglet_screen.blit')
pyprofile.incr('vncdriver.pyglet_screen.blit.bytes', len(bytes), unit=pyprofile.BYTES)
import pyglet
image = pyglet.image.ImageData(width, height, 'RGB', bytes, pitch=width * -3)
self.texture.blit_into(image, x, self._height-height-y, 0)
self._is_updated = True
def recv_DecodeZlib_value(self, block, x, y, width, height):
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding.bytes',
len(block),
unit=pyprofile.BYTES)
rectangle = server_messages.ZlibEncoding.parse_rectangle(
self, x, y, width, height, block)
self._rectangles.append(rectangle)
self._process_rectangles()
def recv_DecodeZlib(self, block, x, y, width, height):
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding')
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding.bytes',
len(block),
unit=pyprofile.BYTES)
(length, ) = struct.unpack('!I', block)
self.expect(self.recv_DecodeZlib_value, length, x, y, width, height)
def dataReceived(self, data):
pyprofile.incr('vnc_client.data.received.messages')
pyprofile.incr('vnc_client.data.received.bytes', len(data), unit=pyprofile.BYTES)
self.buf.append(data)
self.buf_len += len(data)
logger.debug('Received data: %s bytes (brings us to %s total)', len(data), self.buf_len)
self.flush()
def recv_FramebufferUpdate(self, block):
(number_of_rectangles, ) = struct.unpack('!xH', block)
logger.debug('Receiving %d rectangles', number_of_rectangles)
pyprofile.incr('vncdriver.framebuffer_update')
pyprofile.incr('vncdriver.framebuffer_update.number_of_rectangles', number_of_rectangles)
self._remaining_rectangles = number_of_rectangles
self._rectangles = []
self._process_rectangles()
def get(self, cropped):
if self._last_img is None:
return
match = np.array_equal(self._last_img, cropped)
if match:
pyprofile.incr('score.crop_cache.hit.{}'.format(self.name))
return self._last_value
else:
self._last_img = None
def reward(self):
info = {}
if self.env is None:
return 0, False, info
screen, _, done, observation_info = self.env.step([])
# Copy over the staleness of the observation and the number of
# VNC updates in the last frame. This gets sent to the client.
lag = observation_info.get('diagnostics.lag.observation')
if lag is not None:
info['rewarder.lag.observation'] = lag[0]
info['rewarder.lag.observation.timestamp'] = time.time() - lag[0]
info['rewarder.vnc.updates.n'] = updates_n = observation_info.get(
'vnc.updates.n')
info['rewarder.vnc.updates.pixels'] = observation_info.get(
'vnc.updates.pixels')
info['rewarder.vnc.updates.bytes'] = observation_info.get(
'vnc.updates.bytes')
if updates_n is not None:
pyprofile.incr('reward.vnc.updates.n', updates_n)
now = time.time()
if self._has_initial_reward and info[
'rewarder.vnc.updates.n'] == 0: # Nothing new!
# Timeout after 100 seconds without VNC updates.
# This means nothing on the screen has changed, which is probably very bad.
# We log the error and end the episode, hopefully it will recover nicely.
if now > self._vnc_last_update + 100:
logger.error('No vnc updates since {}'.format(
self._vnc_last_update))
done = True
return 0, done, info
elif self.reward_parser is None:
return 0, done, info
self._vnc_last_update = now
self._has_initial_reward = True
reward, done, reward_info = self.reward_parser.reward(screen)
if (self.env_id == 'flashgames.NeonRace2-v0' and reward > 20000) or \
(self.env_id == 'flashgames.CoasterRacer-v0' and reward > 20000) or \
(self.env_id == 'internet.SlitherIO-v0' and reward > 1000):
import tempfile
f = tempfile.NamedTemporaryFile()
path = f.name + '.png'
f.close()
logger.info(
'[%s] Abnormally large reward of %s received! This may indicate a bug in the OCR. Saving a screenshot to %s for investigation.',
utils.thread_name(), reward, path)
Image.fromarray(screen).save(path)
return reward, done, info
def dataReceived(self, data):
if self.expected is None:
# We're in direct proxy mode
self.proxyData(data)
return
pyprofile.incr('vnc_proxy_server.data.sent.messages')
pyprofile.incr('vnc_proxy_server.data.sent.bytes', len(data))
self.buf.append(data)
self.buf_len += len(data)
self.flush()
def parse_rectangle(cls, client, x, y, width, height, data):
decompressed = client.zlib_decompressor.decompress(data)
logger.debug('[zlib] Decompressed from %s bytes -> %s bytes',
len(data), len(decompressed))
pyprofile.incr(
'vncdriver.recv_rectangle.zlib_encoding.decompressed_bytes',
len(decompressed),
unit=pyprofile.BYTES)
data = np.frombuffer(decompressed, np.uint8).reshape(
(height, width, 4))[:, :, [0, 1, 2]]
encoding = cls(data)
return Rectangle(x, y, width, height, encoding)
def update_rectangle(self, x, y, width, height, data):
bytes = data.tobytes()
pyprofile.incr('vncdriver.pyglet_screen.blit')
pyprofile.incr('vncdriver.pyglet_screen.blit.bytes',
len(bytes),
unit=pyprofile.BYTES)
import pyglet
image = pyglet.image.ImageData(width,
height,
'RGB',
bytes,
pitch=width * -3)
self.texture.blit_into(image, x, self._height - height - y, 0)
self._is_updated = True
def parse_rectangle(cls, client, x, y, width, height, data):
decompressed = client.zlib_decompressor.decompress(data)
logger.debug('[zrle] Decompressed from %s bytes -> %s bytes', len(data), len(decompressed))
pyprofile.incr('vncdriver.recv_rectangle.zrle_encoding.decompressed_bytes', len(decompressed), unit=pyprofile.BYTES)
if client.framebuffer.bpp > 24:
bytes_per_pixel = 3
else:
bytes_per_pixel = client.framebuffer.bypp
buf = BytesIO(decompressed)
data = cls._read(x, y, width, height, buf, bytes_per_pixel)
encoding = cls(data)
return Rectangle(x, y, width, height, encoding)
def parse_rectangle(cls, client, x, y, width, height, data):
decompressed = client.zlib_decompressor.decompress(data)
logger.debug('[zrle] Decompressed from %s bytes -> %s bytes',
len(data), len(decompressed))
pyprofile.incr(
'vncdriver.recv_rectangle.zrle_encoding.decompressed_bytes',
len(decompressed),
unit=pyprofile.BYTES)
if client.framebuffer.bpp > 24:
bytes_per_pixel = 3
else:
bytes_per_pixel = client.framebuffer.bypp
buf = BytesIO(decompressed)
data = cls._read(x, y, width, height, buf, bytes_per_pixel)
encoding = cls(data)
return Rectangle(x, y, width, height, encoding)
def reward(self, img):
score, done = self.score(img)
now = time.time()
# See how long we're running without score, and error out if too long
if done:
logger.info('RESET CAUSE: gameover state reached')
elif score is None:
# If the scorer can't find anything and the episode isn't over,
# we're probably stuck in some kind of weird or stale state.
# It's been more than 100 seconds, we're done here.
#
# If no vexpect is provided, then we assume the user
# is testing vexpect and will handle resets on their
# own.
if now > self._last_score_time + 70 and self.vexpect != None and self.scorer != None:
pyprofile.incr('rewarder.reset.unable_to_parse_score_parse')
logger.error(
'RESET CAUSE: Rewarder has been unable to parse a score since %.2f (%.2fs ago)',
self._last_score_time, now - self._last_score_time)
done = True
else:
# We got a good score, reset our timeout
self._last_score_time = time.time()
if score is None:
# TODO: could return negative score here...
return 0, done, self.UNKNOWN_SCORE
reward_to_return, info = self.reward_from_score.reward(
score, time.time())
if reward_to_return == 0:
if self.nonzero_reward_timeout is not None and \
now - self._last_nonzero_reward_time > self.nonzero_reward_timeout:
logger.error(
'RESET CAUSE: No non-zero rewards were generated since %.2f (%.2fs ago), which exceeds the configured timeout of %.2fs',
self._last_nonzero_reward_time,
now - self._last_nonzero_reward_time,
self.nonzero_reward_timeout)
done = True
else:
self._last_nonzero_reward_time = time.time()
return reward_to_return, done, info
def onMessage(self, payload, isBinary):
if not self.factory.agent_conn.check_message(self):
return
assert not isBinary, "Binary websocket not supported"
payload = ujson.loads(payload)
context = {
'start': time.time(),
'conn': self,
}
latency = context['start'] - payload['headers']['sent_at']
pyprofile.incr('rewarder_protocol.messages')
pyprofile.incr('rewarder_protocol.messages.{}'.format(payload['method']))
pyprofile.timing('rewarder_protocol.latency.rtt.skew_unadjusted', 2*latency)
if latency < 0:
pyprofile.incr('rewarder_protocol.latency.rtt.skew_unadjusted.negative')
if payload['method'] == 'v0.env.reset':
logger.info('Received reset message: %s', payload)
self.factory.agent_conn.control_buffer.recv_rpc(context, payload)
elif payload['method'] == 'v0.control.ping':
logger.debug('Received ping message: %s', payload)
parent_message_id = payload['headers']['message_id']
headers = {'parent_message_id': parent_message_id}
self.send_message('v0.reply.control.ping', {}, headers)
else:
logger.warn('Received unsupported message: %s', payload)
def onMessage(self, payload, isBinary):
extra_logger.debug('[%s] Received payload: %s', self.factory.label,
payload)
assert not isBinary
payload = ujson.loads(payload)
context = self._make_context()
latency = context['start'] - payload['headers']['sent_at']
pyprofile.incr('rewarder_protocol.messages')
pyprofile.incr('rewarder_protocol.messages.{}'.format(
payload['method']))
# Double latency to model RTT
pyprofile.timing('rewarder_protocol.latency.rtt.skew_unadjusted',
2 * latency)
if latency < 0:
pyprofile.incr(
'rewarder_protocol.latency.rtt.skew_unadjusted.negative')
self.recv(context, payload)
def recv_DecodeZlib(self, block, x, y, width, height):
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding')
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding.bytes', len(block), unit=pyprofile.BYTES)
(length,) = struct.unpack('!I', block)
self.expect(self.recv_DecodeZlib_value, length, x, y, width, height)
def recv_DecodeZlib_value(self, block, x, y, width, height):
pyprofile.incr('vncdriver.recv_rectangle.zlib_encoding.bytes', len(block), unit=pyprofile.BYTES)
rectangle = server_messages.ZlibEncoding.parse_rectangle(self, x, y, width, height, block)
self._rectangles.append(rectangle)
self._process_rectangles()
def recv(self, context, response):
method = response['method']
body = response['body']
headers = response['headers']
remote_time = headers['sent_at']
local_time = context['start']
episode_id = headers.get('episode_id')
if episode_id is not None:
self.reward_buffer.push_time(episode_id, remote_time, local_time)
# Gets called by RewarderClient
if method == 'v0.env.reward':
episode_id = headers['episode_id']
reward = body['reward']
done = body['done']
info = body['info']
extra_logger.debug(
'[%s] Received %s: reward=%s done=%s info=%s episode_id=%s',
self.factory.label, method, reward, done, info, episode_id)
pyprofile.incr('rewarder_client.reward', reward)
if done:
pyprofile.incr('rewarder_client.done')
self.reward_buffer.push(episode_id, reward, done, info)
elif method == 'v0.env.text':
episode_id = headers['episode_id']
text = body['text']
extra_logger.debug('[%s] Received %s: text=%s episode_id=%s',
self.factory.label, method, text, episode_id)
self.reward_buffer.push_text(episode_id, text)
elif method == 'v0.env.observation':
episode_id = headers['episode_id']
jsonable = body['observation']
extra_logger.debug(
'[%s] Received %s: observation=%s episode_id=%s',
self.factory.label, method, jsonable, episode_id)
self.reward_buffer.set_observation(episode_id=episode_id,
observation=jsonable)
elif method == 'v0.env.describe':
episode_id = headers['episode_id']
env_id = body['env_id']
env_state = body['env_state']
fps = body['fps']
extra_logger.info(
'[%s] Received %s: env_id=%s env_state=%s episode_id=%s',
self.factory.label, method, env_id, env_state, episode_id)
self.reward_buffer.set_env_info(env_state,
env_id=env_id,
episode_id=episode_id,
fps=fps)
elif method == 'v0.reply.env.reset':
episode_id = headers['episode_id']
self._finish_reset(episode_id)
elif method in ['v0.reply.error', 'v0.reply.control.ping']:
assert headers.get('parent_message_id') is not None
elif method == 'v0.connection.close':
assert headers.get('parent_message_id') is None
logger.debug('Server hanging up: %s', body['message'])
self._close_message = body['message']
e = error.Error(body['message'])
self.factory.record_error(e)
else:
logger.error(
'Unrecognized websocket method: method=%s body=%s headers=%s (consider adding to rewarder_state.py)',
method, body, headers)
return
parent_id = headers.get('parent_message_id')
if parent_id is not None:
try:
spec = self._requests.pop(parent_id)
except KeyError:
logger.error(
'[%s] Received extra reply to %d; ignoring: method=%s body=%s headers=%s ',
self.factory.label, parent_id, method, body, headers)
else:
request, d = spec
if method != 'v0.reply.error':
d.callback((context, request, response))
else:
e = RemoteError('[{}] Remote error: {}'.format(
self.factory.label, body['message']))
d.errback(e)
def sendMessage(self, data):
pyprofile.incr('vnc_client.data.sent.messages')
pyprofile.incr('vnc_client.data.sent.bytes', len(data), unit=pyprofile.BYTES)
if self.transport:
self.transport.write(data)
def process_rpc(self, context, message):
if message['method'] == 'v0.env.reset':
env_info = self.env_status.env_info()
env_id = message['body']['env_id']
fps = message['body']['fps']
if fps < 0 or fps > 60:
logger.warn('[%s] Ignoring request for bad fps=%s', fps)
fps = None
changing_id = env_id != env_info['env_id']
changing_fps = fps != env_info['fps']
if changing_id:
# Validate the env ID before altering any state
try:
gym_controlplane.spec(env_id)
except error.UserError as e:
self.agent_conn.send_reply_error(
e.user_message,
parent_message_id=message['headers']['message_id'],
parent_context=context)
pyprofile.incr('control.env_id_change.user_error')
return
episode_id = message['headers']['episode_id']
if not changing_id and not changing_fps and \
rewarder.compare_ids(episode_id, env_info['episode_id']) < 0 and \
self._sweep_id == self._last_reset_sweep_id:
# The user hasn't requested any changes, and our
# current episode_id is ahead of the requested one,
# and we're in the middle of resetting -- they may as
# well just piggyback on our current reset.
logger.info(
'[%s] SKIPPING RESET: user requested reset, but already reset in this sweep and no changes requested: request: env_id=%s fps=%s episode_id=%s current: episode_id=%s. Short circuiting by just using the current reset.',
utils.thread_name(), env_id, fps, episode_id,
env_info['episode_id'])
short_circuit = True
else:
logger.debug(
'Change requested: changing_id=%s changing_fps=%s compare_ids=%s state=%s sweep_id=%s last_reset_sweep_id=%s',
changing_id, changing_fps,
rewarder.compare_ids(episode_id, env_info['episode_id']),
env_info['env_state'], self._sweep_id,
self._last_reset_sweep_id)
# Make sure we're resetting before changing the env_id
env_info = self.env_status.set_env_info('resetting',
env_id=env_id,
bump_past=episode_id)
short_circuit = False
if not short_circuit:
if changing_id:
self.load_env(env_id)
pyprofile.incr('control.env_id_change')
pyprofile.incr('control.env_id_change.{}'.format(env_id))
logger.info(
'[%s] RESET CAUSE: changing out environments due to v0.env.reset (with episode_id=%s): %s -> %s (new episode_id=%s fps=%s)',
utils.thread_name(), episode_id, env_info['env_id'],
env_id, env_info['episode_id'], fps)
else:
logger.info(
'[%s] RESET CAUSE: Resetting environment due to v0.env.reset (with episode_id=%s), keeping same env_id=%s: new episode_id=%s fps=%s',
utils.thread_name(), episode_id, env_id,
env_info['episode_id'], fps)
self.reset()
# We let the agent know the new episode_id to care
# about. (In theory, we could do this before the reset
# completes, but the aggregation case would then behave
# differently.)
self.agent_conn.send_reply_env_reset(
episode_id=env_info['episode_id'],
parent_message_id=message['headers']['message_id'],
parent_context=context,
)
else:
logger.warn('Ignoring unsupported message: %s', message)
