def allocate_ports(self, num):
if self.reuse and self._next_port in self._ports:
vnc_id = self._ports[self._next_port]
rewarder_id = self._ports.get(self._next_port + 10000)
# Reuse an existing docker container if it exists
if (self._next_port + 10000) not in self._ports:
raise error.Error(
"Port {} was allocated but {} was not. This indicates unexpected state with spun-up VNC docker instances."
.format(self._next_port, self._next_port + 1))
elif vnc_id != rewarder_id:
raise error.Error(
"Port {} is exposed from {} while {} is exposed from {}. Both should come from a single Docker instance running your environment."
.format(vnc_id, self._next_port, rewarder_id,
self._next_port + 10000))
base = self._next_port
self._next_port += 1
return base, base + 10000, vnc_id
elif not self.reuse:
# Otherwise, allocate find the lowest free pair of
# ports. This doesn't work for the reuse case since on
# restart we won't remember where we spun up our
# containers.
while self._next_port in self._ports or (self._next_port +
10000) in self._ports:
self._next_port += 1
base = self._next_port
self._next_port += 1
# And get started!
return base, base + 10000, None
def _decode(self, observation, start, available_at):
# This method gets wrapped by AsyncDecode.__call__
with pyprofile.push(
'vnc_env.diagnostics.QRCodeMetadataDecoder.qr_code_scanner'):
encoded = fastzbarlight.qr_code_scanner(observation.tobytes(),
self.width, self.height)
if encoded is None:
# Failed to parse!
return
if encoded.startswith(b'v1:'):
encoded = encoded.decode('utf-8')
if len(encoded) != len('v1:') + 12 + 12:
raise error.Error(
'Bad length for metadata from enviroment: {}'.format(
encoded))
encoded = encoded[len('v1:'):]
last_update = int(encoded[:12], 16) / 1000.0
last_action = int(encoded[12:24], 16) / 1000.
return {
# Timestamp on the image
'now': last_update,
# When the last probe was received
'probe_received_at': last_action,
'processing_start': start,
'processing_end': time.time(),
'available_at': available_at,
}
else:
raise error.Error(
'Bad version string for metadata from environment: {}'.format(
encoded))
def processExited(self, reason):
if isinstance(reason.value, twisted.internet.error.ProcessDone):
out = b''.join(self.out).decode('utf-8')
match = re.search('offset ([\d.-]+) sec', out)
if match is not None:
offset = float(match.group(1))
self.deferred.callback(offset)
else:
self.deferred.errback(
error.Error('Could not parse offset: %s', out))
else:
err = b''.join(self.err)
self.deferred.errback(
error.Error('{} failed with status {}: stderr={!r}'.format(
self._cmd, reason.value.exitCode, err)))
def gym_core_action_space(gym_core_id):
spec = gym.spec(gym_core_id)
if spec.id == 'CartPole-v0':
return spaces.Hardcoded([
[spaces.KeyEvent.by_name('left', down=True)],
[spaces.KeyEvent.by_name('left', down=False)],
])
elif spec._entry_point.startswith('gym.envs.atari:'):
actions = []
env = spec.make()
for action in env.unwrapped.get_action_meanings():
z = 'FIRE' in action
left = 'LEFT' in action
right = 'RIGHT' in action
up = 'UP' in action
down = 'DOWN' in action
translated = atari_vnc(up=up,
down=down,
left=left,
right=right,
z=z)
actions.append(translated)
return spaces.Hardcoded(actions)
else:
raise error.Error('Unsupported env type: {}'.format(spec.id))
def from_remotes(cls, client_id, remotes, runtime_id, start_timeout, tag,
api_key):
parsed = urlparse.urlparse(remotes)
if not (parsed.scheme == 'http' or parsed.scheme == 'https'):
raise error.Error(
'AllocatorManager must start with http:// or https://: {}'.
format(remotes))
base_url = parsed.scheme + '://' + parsed.netloc
if parsed.path:
base_url += '/' + parsed.path
query = urlparse.parse_qs(parsed.query)
n = query.get('n', [1])[0]
cpu = query.get('cpu', [None])[0]
if cpu is not None:
cpu = float(cpu)
placement = query.get('address', ['public'])[0]
params = {}
if tag is not None: params['tag'] = tag
if cpu is not None: params['cpu'] = cpu
return cls(client_id=client_id,
runtime_id=runtime_id,
base_url=base_url,
start_timeout=start_timeout,
params=params,
placement=placement,
api_key=api_key), int(n)
def fail(reason):
reason = error.Error('[{}] Connection failed: {}'.format(
factory.label, reason.value))
try:
d.errback(utils.format_error(reason))
except defer.AlreadyCalledError:
pass
def _register_vnc(self, address, start_time=None):
if start_time is None:
start_time = time.time()
host, port = host_port(address, default_port=5900)
while True:
# In VNC, the server sends bytes upon connection
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((host, port))
except (socket.error, socket.gaierror) as e:
# ECONNREFUSED: VNC env hasn't come up yet
# ETIMEDOUT: the packets can't be delivered yet, such as can happen on kubernetes
# gaierror: can't resolve the address yet, which can also happen on kubernetes
expected = socket.errno.ECONNREFUSED == e.errno or socket.errno.ETIMEDOUT == e.errno or isinstance(e, socket.gaierror)
if self.start_timeout is None or not expected:
reraise(suffix='while connecting to VNC server {}'.format(address))
logger.info('VNC server %s did not come up yet (error: %s). Sleeping for 1s.', address, e)
time.sleep(1)
else:
break
if time.time() - start_time > self.start_timeout:
raise error.Error('VNC server {} did not come up within {}s'.format(address, self.start_timeout))
self.sockets[sock] = ('vnc', address)
def _register_rewarder(self, address, start_time=None):
if start_time is None:
start_time = time.time()
host, port = host_port(address, default_port=15900)
while True:
# In WebSockets, the server sends bytes once we've upgraded the protocol
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((host, port))
except (socket.error, socket.gaierror) as e:
# ECONNREFUSED: VNC env hasn't come up yet
# ETIMEDOUT: the packets can't be delivered yet, such as can happen on kubernetes
# gaierror: can't resolve the address yet, which can also happen on kubernetes
expected = socket.errno.ECONNREFUSED == e.errno or socket.errno.ETIMEDOUT == e.errno or isinstance(e, socket.gaierror)
if self.start_timeout is None or not expected:
reraise(suffix='while connecting to Rewarder server {}'.format(address))
logger.info('Rewarder server %s did not come up yet (error: %s). Sleeping for 1s.', address, e)
time.sleep(1)
else:
break
if time.time() - start_time > self.start_timeout:
raise error.Error('Rewarder server {} did not come up within {}s'.format(address, self.start_timeout))
# Send a websocket handshake.
# https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers
#
# The port 10003 is an arbitrary port that we don't actually connect to, but needs to be a valid part
# e.g Host: 127.0.0.1:GARBAGE results in the following error: (invalid port 'GARBAGE' in HTTP Host header '127.0.0.1:GARBAGE')
sock.send(b'GET / HTTP/1.1\r\nHost: 127.0.0.1:10003\r\nUpgrade: WebSocket\r\nConnection:Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\nauthorization: ' + utils.basic_auth_encode('openai').encode('utf-8') + b'\r\nopenai-observer: true\r\n\r\n')
self.sockets[sock] = ('rewarder', address)
def keycode(key):
if key in constants.KEYMAP:
return constants.KEYMAP.get(key)
elif len(key) == 1:
return ord(key)
else:
raise error.Error('Not sure how to translate to keycode: {!r}'.format(key))
def monitor(self):
if not self.metadata['runtime.vectorized']:
# Just delegate if we're not actually vectorized (like
# Unvectorize)
return super(Env, self).monitor
if not hasattr(self, '_monitor'):
# Not much we can do if we don't know how wide we'll
# be. This can happen when closing.
if self.n is None:
raise error.Error(
'You must call "configure()" before accesssing the monitor for {}'
.format(self))
# Circular dependencies :(
from universe import wrappers
from universe.vectorized import monitoring
# We need to maintain pointers to these to them being
# GC'd. They have a weak reference to us to avoid cycles.
self._unvectorized = [
wrappers.WeakUnvectorize(self, i) for i in range(self.n)
]
# Store reference to avoid GC
# self._render_cached = monitoring.RenderCache(self)
self._monitor = monitoring.Monitor(self._unvectorized)
return self._monitor
def allocate(self, handles, initial=False, params={}):
if len(handles) > self.available_n:
raise error.Error(
'Requested {} handles, but only have {} envs'.format(
len(handles), self.available_n))
self.n = len(handles)
self._handles = handles
def _handle_crashed_n(self, info_n):
# for i in self.crashed:
# info_n[i]['env_status.crashed'] = True
if self.allow_reconnect:
return
if len(self.crashed) > 0:
errors = {}
for i, info in enumerate(info_n):
if 'error' in info:
errors[self.crashed[i]] = info['error']
if len(errors) == 0:
raise error.Error('{}/{} environments have crashed. No error key in info_n: {}'.format(len(self.crashed), self.n, info_n))
else:
raise error.Error('{}/{} environments have crashed! Most recent error: {}'.format(len(self.crashed), self.n, errors))
def rewarder_session(which):
if which is None:
which = rewarder.RewarderSession
if isinstance(which, type):
return which
else:
raise error.Error('Invalid RewarderSession driver: {!r}'.format(which))
def __init__(self, client_id, base_url=allocator_base,
address_type=None, start_timeout=None, api_key=None,
runtime_id=None, tag=None, params=None, placement=None,
use_recorder_ports=False,
):
super(AllocatorManager, self).__init__()
self.label = 'AllocatorManager'
self.supports_reconnect = True
self.connect_vnc = True
self.connect_rewarder = True
if address_type is None: address_type = 'public'
if address_type not in ['public', 'pod', 'private']:
raise error.Error('Bad address type specified: {}. Must be public, pod, or private.'.format(address_type))
self.tag = tag
self.client_id = client_id
self.address_type = address_type
if start_timeout is None:
start_timeout = 20 * 60
self.start_timeout = start_timeout
self.params = params
self.placement = placement
self.use_recorder_ports = use_recorder_ports
# if base_url is None:
# base_url = scoreboard.api_base
# if base_url is None:
# base_url = gym_base_url
# if api_key is None:
# api_key = scoreboard.api_key
# if api_key is None:
# raise gym.error.AuthenticationError("""You must provide an OpenAI Gym API key.
# (HINT: Set your API key using "gym.scoreboard.api_key = .." or "export OPENAI_GYM_API_KEY=..."). You can find your API key in the OpenAI Gym web interface: https://gym.openai.com/settings/profile.""")
if api_key is None:
api_key = _api_key
self._requestor = AllocatorClient(self.label, api_key, base_url=base_url)
self.base_url = base_url
# These could be overridden on a per-allocation basis, if you
# want heterogeoneous envs. We don't support those currently
# in the higher layers, but this layer could support it
# easily.
self.runtime_id = runtime_id
self.tag = tag
self.pending = {}
self.error_buffer = utils.ErrorBuffer()
self.requests = queue.Queue()
self.ready = queue.Queue()
self._reconnect_history = {}
self._sleep = 1
def build(cls, metadata_encoding, pool, qr_pool, label):
metadata_encoding = metadata_encoding.copy()
type = metadata_encoding.pop('type')
if type == 'qrcode':
return QRCodeMetadataDecoder(label=label, pool=pool, qr_pool=qr_pool, **metadata_encoding)
elif type == 'pixels':
return PixelsMetadataDecoder(label=label)
else:
raise error.Error('Invalid encoding: {}'.format(type))
def __init__(self, env=None):
super(Wrapper, self).__init__(env)
if env is not None and not env.metadata.get('runtime.vectorized'):
if self.autovectorize:
# Circular dependency :(
from universe import wrappers
env = wrappers.Vectorize(env)
else:
raise error.Error(
'This wrapper can only wrap vectorized envs (i.e. where env.metadata["runtime.vectorized"] = True), not {}. Set "self.autovectorize = True" to automatically add a Vectorize wrapper.'
.format(env))
if env is None and not self.standalone:
raise error.Error(
'This env requires a non-None env to be passed. Set "self.standalone = True" to allow env to be omitted or None.'
)
self.env = env
def __init__(self, env):
super(Wrapper, self).__init__(env)
if not env.metadata.get('runtime.vectorized'):
if self.autovectorize:
# Circular dependency :(
from universe import wrappers
env = wrappers.Vectorize(env)
else:
raise error.Error('This wrapper can only wrap vectorized envs (i.e. where env.metadata["runtime.vectorized"] = True), not {}. Set "self.autovectorize = True" to automatically add a Vectorize wrapper.'.format(env))
self.env = env
def env(self):
# Called upon instantiation
if not hasattr(self, '_env_ref'):
return
env = self._env_ref()
if env is None:
raise error.Error(
"env has been garbage collected. To keep using WeakUnvectorize, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)"
)
return env
def onClose(self, wasClean, code, reason):
if self._close_message:
return
if not self._connected:
assert not self._connection_result.called
self._connection_result.errback(error.ConnectionError(reason))
return
if not self._closed:
error_message = 'Lost connection: {} (clean={} code={})'.format(
reason, wasClean, code)
reason = error.Error(error_message)
# TODO: it's not an error if we requested it
self.factory.record_error(reason)
else:
reason = error.Error("We closed the connection: {}".format(reason))
for request, d in self._requests.values():
d.errback(reason)
def onConnect(self, request):
if not os.path.exists('/usr/local/openai/privileged_state/password'):
raise error.Error('No such file: /usr/local/openai/privileged_state/password. (HINT: did the init script run /app/universe-envs/base/openai-setpassword?)')
with open('/usr/local/openai/privileged_state/password') as f:
password = f.read().strip()
self._message_id = 0
self._request = request
self._observer = request.headers.get('openai-observer') == 'true'
self.password = password
logger.info('Client connecting: peer=%s observer=%s', request.peer, self._observer)
def __init__(self,
remotes,
error_buffer,
encoding=None,
compress_level=None,
fine_quality_level=None,
subsample_level=None):
"""compress_level: 0-9 [9 is highest compression]
fine_quality_level: 0-100 [100 is best quality]
subsample_level: 0-3 [0 is best quality]
Lots of references for this, but
https://github.com/TurboVNC/turbovnc/blob/master/doc/performance.txt
is decent.
"""
load_pygame()
import libvncdriver
if encoding is None:
encoding = os.environ.get('LIBVNC_ENCODING', 'tight')
if compress_level is None:
compress_level = int(os.environ.get('LIBVNC_COMPRESS_LEVEL', '0'))
if fine_quality_level is None:
fine_quality_level = int(
os.environ.get('LIBVNC_FINE_QUALITY_LEVEL', '100'))
if subsample_level is None:
subsample_level = int(os.environ.get('LIBVNC_SUBSAMPLE_LEVEL',
'0'))
if not hasattr(libvncdriver, 'VNCSession'):
raise error.Error('''
*=================================================*
|| libvncdriver is not installed ||
|| Try installing with "pip install libvncdriver" ||
|| or use the go or python driver by setting ||
|| UNIVERSE_VNCDRIVER=go ||
|| UNIVERSE_VNCDRIVER=py ||
*=================================================*''')
logger.info("Using libvncdriver's %s encoding" % encoding)
self.driver = libvncdriver.VNCSession(
remotes=remotes,
error_buffer=error_buffer,
encoding=encoding,
compress_level=compress_level,
fine_quality_level=fine_quality_level,
subsample_level=subsample_level,
)
self.screen = None
self.render_called_once = False
if PYGAME_INSTALLED:
pygame.init()
def build(client_id, remotes, runtime=None, start_timeout=None, **kwargs):
if isinstance(remotes, int):
remotes = str(remotes)
elif not isinstance(remotes, str):
raise error.Error(
'remotes argument must be a string, got {} which is of type {}'.
format(remotes, type(remotes)))
if re.search('^\d+$', remotes): # an integer, like -r 20
n = int(remotes)
return DockerManager(
runtime=runtime,
start_timeout=start_timeout,
reuse=kwargs.get('reuse', False),
n=n,
), n
elif remotes.startswith('vnc://'):
return HardcodedAddresses.build(remotes, start_timeout=start_timeout)
elif remotes.startswith('http://') or remotes.startswith('https://'):
if runtime is None:
raise error.Error(
'Must provide a runtime. HINT: try creating your env instance via gym.make("flashgames.DuskDrive-v0")'
)
manager, n = AllocatorManager.from_remotes(
client_id,
remotes,
runtime_id=runtime.id,
runtime_tag=runtime.image.split(':')[-1],
start_timeout=start_timeout,
api_key=kwargs.get('api_key'),
use_recorder_ports=kwargs.get('use_recorder_ports', False),
)
manager.start()
return manager, n
else:
raise error.Error(
'Invalid remotes: {!r}. Must be an integer or must start with vnc:// or https://'
.format(remotes))
def _apply(self, framebuffer_update):
if self.paint_cursor:
self._unpaint_cursor()
for rect in framebuffer_update.rectangles:
if isinstance(rect.encoding,
(server_messages.RAWEncoding, server_messages.ZRLEEncoding, server_messages.ZlibEncoding)):
self._update_rectangle(rect.x, rect.y, rect.width, rect.height, rect.encoding.data)
elif isinstance(rect.encoding, server_messages.PseudoCursorEncoding):
self._update_cursor_shape(rect.x, rect.y, rect.width, rect.height, rect.encoding.image, rect.encoding.mask)
else:
raise error.Error('Unrecognized encoding: {}'.format(rect.encoding))
if self.paint_cursor:
self._paint_cursor()
def __init__(self, env_m):
self.env_m = env_m
for env in self.env_m:
if not env._configured:
raise error.Error(
'Joint env should have been initialized: {}'.format(env))
# TODO: generalize this. Doing so requires adding a vectorized
# space mode.
self.action_space = env_m[0].action_space
self.observation_space = env_m[0].observation_space
self.pool = pool.ThreadPool(min(len(env_m), 5))
def apply(self, framebuffer_update):
pyprofile.push('vncdriver.pyglet_screen.apply')
for rect in framebuffer_update.rectangles:
if isinstance(
rect.encoding,
(server_messages.RAWEncoding, server_messages.ZRLEEncoding,
server_messages.ZlibEncoding)):
self.update_rectangle(rect.x, rect.y, rect.width, rect.height,
rect.encoding.data)
else:
raise error.Error('Unrecognized encoding: {}'.format(
rect.encoding))
pyprofile.pop()
def _initialize(self):
if not os.environ.get('DISPLAY') and sys.platform.startswith('linux'):
raise error.Error(
"Cannot render with mode='human' with no DISPLAY variable set."
)
import pyglet
self._window = pyglet.window.Window(width=self._width,
height=self._height,
visible=True)
self._window.dispatch_events()
self.texture = pyglet.image.Texture.create(width=self._width,
height=self._height)
def build(cls, remotes, **kwargs):
parsed = urlparse.urlparse(remotes)
if parsed.scheme != 'vnc':
raise error.Error('HardcodedAddresses must be initialized with a string starting with vnc://: {}'.format(remotes))
addresses = parsed.netloc.split(',')
query = urlparse.parse_qs(parsed.query)
# We could support per-backend passwords, but no need for it
# right now.
password = query.get('password', [utils.default_password()])[0]
vnc_addresses, rewarder_addresses = parse_remotes(addresses)
res = cls(vnc_addresses, rewarder_addresses, vnc_password=password, rewarder_password=password, **kwargs)
return res, res.available_n
def _spawn(self):
if self.runtime.image is None:
raise error.Error('No image specified')
assert self._container_id is None
self.vnc_port, self.rewarder_port, self._container_id = self.assigner.allocate_ports(
2)
if self._container_id is not None:
logger.info('[%s] Reusing container %s on ports %s and %s',
self.label, self._container_id[:12], self.vnc_port,
self.rewarder_port)
self.reusing = True
self.started = True
return
self.reusing = False
logger.info(
'[%s] Creating container: image=%s. Run the same thing by hand as: %s',
self.label, self.runtime.image,
pretty_command(
self.runtime.cli_command(self.vnc_port, self.rewarder_port)))
try:
container = self._spawn_container()
except docker.errors.NotFound as e:
# Looks like we need to pull the image
assert 'No such image' in e.explanation.decode(
'utf-8'
), 'Expected NotFound error message message to include "No such image", but it was: {}. This is probably just a bug in this assertion and the assumption was incorrect'.format(
e.explanation)
logger.info('Image %s not present locally; pulling',
self.runtime.image)
self._pull_image()
# If we called pull_image from multiple processes (as we do with universe-starter-agent A3C)
# these will all return at the same time. We probably all got the same port numbers before the pull started,
# so wait a short random time and refresh our port numbers
time.sleep(random.uniform(0.5, 2.5))
self.assigner._refresh_ports()
self.vnc_port, self.rewarder_port, self._container_id = self.assigner.allocate_ports(
2)
if self._container_id is not None:
logger.info('[%s] Reusing container %s on ports %s and %s',
self.label, self._container_id[:12], self.vnc_port,
self.rewarder_port)
self.reusing = True
self.started = True
return
# Try spawning again.
container = self._spawn_container()
self._container_id = container['Id']
def wait_for_step(self, error_buffer=None, timeout=None):
# TODO: this might be cleaner using channels
with self.cv:
start = time.time()
while True:
if self.count != 0:
return
elif timeout is not None and time.time() - start > timeout:
raise error.Error('No rewards received in {}s'.format(timeout))
if error_buffer:
error_buffer.check()
self.cv.wait(timeout=0.5)
def _step(self, action):
try:
for a, client in zip(action, self._clients):
for event in a:
if event[0] == 'KeyEvent':
key, down = event[1:]
client.send_KeyEvent(key, down)
elif event[0] == 'PointerEvent':
x, y, buttomask = event[1:]
client.send_PointerEvent(x, y, buttomask)
else:
raise error.Error('Bad event type: {}'.format(type))
except Exception as e:
self.error_buffer.record(e)
