def connect(self, reconnect=False):
# type: (bool) -> None
if self.is_connected():
if not reconnect:
return
self._disconnect()
# Set up socket
self._socket = self._zmq_context.socket(zmq.DEALER)
self._socket.setsockopt(zmq.LINGER, self._linger)
if self._ssh_thru_server is None:
self._socket.connect(self.broker_url)
else:
ssh.tunnel_connection(self._socket,
self.broker_url,
self._ssh_thru_server,
password=self._ssh_password)
self._log.info(
"Connected to broker at {} via ssh-tunnel {}".format(
self.broker_url, self._ssh_thru_server))
self._log.debug("Connected to broker on ZMQ DEALER socket at %s",
self.broker_url)
self._expect_reply = False
def __init__(self,
frontend=DEFAULT_FRONTEND,
timeout=DEFAULT_TIMEOUT,
timeout_max_overflow=DEFAULT_TIMEOUT_MOVF,
timeout_overflows=DEFAULT_TIMEOUT_OVF,
debug=False,
ctx=None,
ssh=None):
self.ssh = ssh
self.kill_ctx = ctx is None
self.ctx = ctx or zmq.Context()
self.frontend = frontend
self.master = self.ctx.socket(zmq.REQ)
if ssh:
from zmq import ssh
ssh.tunnel_connection(self.master, frontend, self.ssh)
else:
self.master.connect(frontend)
self.poller = zmq.Poller()
self.poller.register(self.master, zmq.POLLIN)
self.timeout = timeout * 1000
self.lock = threading.Lock()
self.timeout_max_overflow = timeout_max_overflow * 1000
self.timeout_overflows = timeout_overflows
self.debug = debug
def connect(self, reconnect=False):
# type: (bool) -> None
if self.is_connected():
if not reconnect:
return
self._disconnect()
# Set up socket
self._socket = self._context.socket(zmq.DEALER)
# make the type of connection requested
if self._ssh_thru_server is None:
# regular connection, a simple request/reply socket
self._socket.connect(self._broker_address)
self._log.info("Connected to broker at %s", self._broker_address)
else:
self._log.info(
"Connecting to broker at {} via ssh-tunnel {}".format(
self._broker_address, self._ssh_thru_server))
ssh.tunnel_connection(self._socket,
self._broker_address,
self._ssh_thru_server,
password=self._ssh_password)
self._log.info("Connected to broker.")
self._poller = zmq.Poller()
self._poller.register(self._socket, zmq.POLLIN)
self._send_ready()
self._last_broker_hb = time.time()
def obtainTaskSpecFromServer(ServerIP,
ServerPort,
TeamName,
use_ssh=False,
ssh_server="youbot-hbrs2-pc2"):
context = zmq.Context()
connection_address = "tcp://" + str(ServerIP) + ":" + str(ServerPort)
print "Start connection to " + connection_address
# Socket to talk to server
print "Connecting to server..."
socket = context.socket(zmq.REQ)
if not use_ssh:
print "connecting directly, not through an ssh tunnel"
socket.connect(connection_address)
else:
print "connecting via ssh tunnel through: " + ssh_server
ssh.tunnel_connection(socket, connection_address, ssh_server)
print "Sending request for task specification"
socket.send(TeamName)
# Get the reply.
message = socket.recv()
socket.send("ACK")
socket.close()
print "Received task specification: " + message
return message
def client(container,command,output,tunnel):
context = zmq.Context()
if tunnel:
ssh.tunnel.openssh_tunnel(6000,6000,'lxplus','lheinric-dockerinteractive')
webhost = 'localhost'
else:
webhost = 'lheinric-dockerinteractive'
url = 'http://{0}:6000/start?'.format(webhost)
parameters = []
parameters.append('container={0}'.format(container))
parameters.append('command={0}'.format(command))
if output:
parameters.append('afsdirmount={0}'.format(output))
url = url + '&'.join(parameters)
r = requests.get(url)
if not r.ok:
e = click.ClickException(message = click.style('sorry, there is no spot available on the server', fg = 'red'))
e.exit_code = 1
raise e
readfrom = r.json()['readfrom']
# readfrom = 5556
click.secho('starting remote docker session', fg = 'green')
#incoming messages
socket = context.socket(zmq.PAIR)
if tunnel:
ssh.tunnel_connection(socket,'tcp://lheinric-dockerinteractive:{0}'.format(readfrom),'lxplus')
else:
socket.connect("tcp://lheinric-dockerinteractive:{0}".format(readfrom))
poller = zmq.Poller()
poller.register(socket,zmq.POLLIN)
sockets = [socket]
socket.send_json({'ctrl':'start'})
ack = socket.recv()
istty = os.isatty(sys.stdin.fileno())
socket.send_json({'ctrl':{'tty':istty}})
signal.signal(signal.SIGWINCH, get_sigwinch_handler(socket))
signal.signal(signal.SIGINT, get_sigint_handler(socket))
signal.signal(signal.SIGHUP, get_sighup_handler(socket))
signal.signal(signal.SIGTERM, get_sigterm_handler(socket))
if istty:
handle_tty(socket)
else:
handle_nontty(socket)
click.secho('Bye.', fg = 'green')
sys.exit(0)
return
def connect(self, passw = None): ''' Connect to a remote host ''' try: #if not port: ip = self.ip port = self.port if not self.context: self.context = zmq.Context() if not self.socket: socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) #self.socket = socket self._set_socket(socket) else: socket = self.socket #for port in ports: server = "tcp://%s:%s" %(ip, port) if ip == 'localhost' or ip == '127.0.0.1': socket.connect(server) else: ssh_server = ip if not passw: tunnel = ssh.tunnel_connection(socket, server, ssh_server, paramiko=True, timeout=5) else: tunnel = ssh.tunnel_connection(socket, server, ssh_server, password = passw, paramiko=True, timeout=5) except: raise return 0
def connect_socket(self, addr, tunnel=None):
"""Connect socket to endpoint, possible using an ssh tunnel
The tunnel argument specifies the hostname of the ssh server to
tunnel through.
"""
if(tunnel):
from zmq import ssh
ssh.tunnel_connection(self._socket, addr, tunnel)
else:
self._socket.connect(addr)
def connect_socket(self, addr, tunnel=None):
"""Connect socket to endpoint, possible using an ssh tunnel
The tunnel argument specifies the hostname of the ssh server to
tunnel through.
"""
if (tunnel):
from zmq import ssh
ssh.tunnel_connection(self._socket, addr, tunnel)
else:
self._socket.connect(addr)
def connect(host, sub_port, ssh_host):
ctx = zmq.Context()
sub_sock = ctx.socket(zmq.SUB)
sub_sock.setsockopt(zmq.SUBSCRIBE, b'')
remote = f'tcp://{host}:{sub_port}'
if ssh_host:
from zmq import ssh
ssh.tunnel_connection(sub_sock, remote, ssh_host)
else:
sub_sock.connect(remote)
return sub_sock
def _connect_socket(
sock,
address,
ssh_server: Optional[str] = None,
ssh_keyfile: Optional[str] = None,
ssh_password: Optional[str] = None,
):
if ssh_server is not None:
tunnel_connection(sock, address, ssh_server, ssh_keyfile, ssh_password)
else:
sock.connect(address)
def get_connection(socket, endpoint, ssh_server=None, ssh_keyfile=None):
if ssh_server is None:
socket.connect(endpoint)
else:
try:
try:
ssh.tunnel_connection(socket, endpoint, ssh_server, keyfile=ssh_keyfile)
except ImportError:
ssh.tunnel_connection(socket, endpoint, ssh_server, keyfile=ssh_keyfile, paramiko=True)
except ImportError:
raise ImportError("pexpect was not found, and failed to use " "Paramiko. You need to install Paramiko")
def get_connection(socket, endpoint, ssh_server=None, ssh_keyfile=None):
if ssh_server is None:
socket.connect(endpoint)
else:
try:
try:
ssh.tunnel_connection(socket, endpoint, ssh_server,
keyfile=ssh_keyfile)
except ImportError:
ssh.tunnel_connection(socket, endpoint, ssh_server,
keyfile=ssh_keyfile, paramiko=True)
except ImportError:
raise ImportError("pexpect was not found, and failed to use "
"Paramiko. You need to install Paramiko")
def connect_socket(self, addr, tunnel=None):
"""Connect socket to endpoint, possible using an ssh tunnel
The tunnel argument specifies the hostname of the ssh server to
tunnel through.
Note that this still succeeds even if there's no ZMQ server on
the other end as the connection is asynchronous. For more details
check the zmq_connect(3) documentation.
"""
if(tunnel):
from zmq import ssh
# If there's no ssh server listening we're gonna
# get stuck here for a long time as there's no timeout
ssh.tunnel_connection(self._socket, addr, tunnel)
else:
self._socket.connect(addr)
def connect_socket(self, addr, tunnel=None):
"""Connect socket to endpoint, possible using an ssh tunnel
The tunnel argument specifies the hostname of the ssh server to
tunnel through.
Note that this still succeeds even if there's no ZMQ server on
the other end as the connection is asynchronous. For more details
check the zmq_connect(3) documentation.
"""
if tunnel:
from zmq import ssh
# If there's no ssh server listening we're gonna
# get stuck here for a long time as there's no timeout
ssh.tunnel_connection(self._socket, addr, tunnel)
else:
self._socket.connect(addr)
def setup(socket, addr, ssh):
"""Tunnel zmq connection with SSH.
Args:
socket (obj): the zmq socket to use
addr (string): connect to the address via ssh tunnel
ssh (obj): the namedtuple containing the ssh options
"""
from zmq.ssh import tunnel_connection
tunnel_connection(
socket,
addr,
ssh.server,
keyfile=ssh.key_file,
password=ssh.password,
timeout=ssh.timeout,
paramiko=ssh.paramiko,
)
def connect_to_pullsocket(self):
""" create new socket. connect to server. send READY message """
self._socket = self._context.socket(zmq.PUSH)
self._socket.setsockopt(zmq.IDENTITY, self._identity)
if self._ssh_thru_server is None:
# regular connection
#self._socket.connect("tcp://%s:%d"%(self._broker_ipaddress,self._broker_port))
self._socket.connect("%s/%d" %
(self._pullsocket_address, self._port))
if self._worker_verbosity >= 0:
print "BaseWorker[{}] socket connected".format(self._identity)
else:
ssh.tunnel_connection(
self._socket,
"tcp://%s:%d" % (self._broker_ipaddress, self._broker_port),
self._ssh_thru_server)
if self._worker_verbosity >= 0:
print "BaseWorker[{}] socket connected via ssh-tunnel".format(
self._identity)
def _set_up(self, parent, popen_args=None):
# http://learning-0mq-with-pyzmq.readthedocs.io/en/latest/pyzmq/patterns/pushpull.html
self.info("Starting node: {}".format(self.unique_node))
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PUSH)
# self.socket.setsockopt(zmq.SNDBUF, 2048)
if self.msg_queue_size is None:
self.socket.setsockopt(zmq.CONFLATE, 1)
else:
self.socket.set_hwm(self.msg_queue_size)
if self.tunnel is not None:
from zmq import ssh
self.info("Subscribe to {}:{} over server {}".format(
self.host, self.port, self.tunnel))
ssh.tunnel_connection(self.socket,
"tcp://{}:{}".format(self.host,
self.port), self.tunnel)
else:
self.info("Subscribe to {}:{}".format(self.host, self.port))
self.socket.connect("tcp://{}:{}".format(self.host, self.port))
def __init__(self, frontend=DEFAULT_FRONTEND, timeout=DEFAULT_TIMEOUT,
timeout_max_overflow=DEFAULT_TIMEOUT_MOVF,
timeout_overflows=DEFAULT_TIMEOUT_OVF,
debug=False, ctx=None, ssh=None):
self.ssh = ssh
self.kill_ctx = ctx is None
self.ctx = ctx or zmq.Context()
self.frontend = frontend
self.master = self.ctx.socket(zmq.REQ)
if ssh:
from zmq import ssh
ssh.tunnel_connection(self.master, frontend, self.ssh)
else:
self.master.connect(frontend)
self.poller = zmq.Poller()
self.poller.register(self.master, zmq.POLLIN)
self.timeout = timeout * 1000
self.lock = threading.Lock()
self.timeout_max_overflow = timeout_max_overflow * 1000
self.timeout_overflows = timeout_overflows
self.debug = debug
def connect_to_broker(self):
""" create new socket. connect to server. send READY message """
self._socket = self._context.socket(zmq.DEALER)
self._socket.setsockopt(zmq.IDENTITY, self._identity)
self._poller.register(self._socket, zmq.POLLIN)
if self._ssh_thru_server is None:
# regular connection
self._socket.connect("tcp://%s:%d" %
(self._broker_ipaddress, self._broker_port))
print "SSNetWorker[{}] socket connected".format(self._identity)
else:
ssh.tunnel_connection(
self._socket,
"tcp://%s:%d" % (self._broker_ipaddress, self._broker_port),
self._ssh_thru_server)
print "SSNetWorker[{}] socket connected via ssh-tunnel".format(
self._identity)
self._socket.send(PPP_READY)
print "SSNetWorker[{}] sent PPP_READY".format(self._identity)
def _connect(self):
# Get Context
if self._ASYNC:
self._CLIENT = self._CONTEXT.socket(self.zmq.PUSH)
else:
self._CLIENT = self._CONTEXT.socket(self.zmq.REQ)
# Check if SSH is requested
if self._USE_SSH:
from zmq import ssh
# Ensure there exists an SSH Host
if self._SSH_HOST:
try:
ssh.tunnel_connection(self._CLIENT, self._BROKER_HOST, self._SSH_HOST)
except RuntimeError as e:
raise e
else:
raise AttributeError("No SSH Host.")
else:
self._CLIENT.connect(self._BROKER_HOST)
# Register Poll
self._POLL.register(self._CLIENT, self.zmq.POLLIN)
def _attach_publisher(self):
zmq_publisher = self.args.get('zmq_publisher')
if zmq_publisher in (None, DEFAULT_PUBLISHER):
# if this option is not provided by the command line,
# we ask the broker about it
res = self.client.ping()
endpoint = res['endpoints']['publisher']
if endpoint.startswith('ipc'):
# IPC - lets hope we're on the same box
zmq_publisher = endpoint
elif endpoint.startswith('tcp'):
# TCP, let's see what IP & port we have
splitted = split_endpoint(endpoint)
if splitted['ip'] == '0.0.0.0':
# let's use the broker ip
broker = self.args['broker']
broker_ip = split_endpoint(broker)['ip']
zmq_publisher = 'tcp://%s:%d' % (broker_ip,
splitted['port'])
else:
# let's use the original ip
zmq_publisher = endpoint
else:
zmq_publisher = DEFAULT_PUBLISHER
if not self.ssh:
self.sub.connect(zmq_publisher)
else:
if zmq_publisher == DEFAULT_PUBLISHER:
zmq_publisher = DEFAULT_SSH_PUBLISHER
from zmq import ssh
ssh.tunnel_connection(self.sub, zmq_publisher, self.ssh)
self.zstream = zmqstream.ZMQStream(self.sub, self.loop)
self.zstream.on_recv(self._recv_result)
self.zmq_publisher = zmq_publisher
def main():
# put the imports in here, because ssh opens a file descriptor (/dev/urandom I think)
# which gets broken if you do it before daemonizing
import zmq
from zmq import ssh
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.XREP)
frontend.bind("tcp://*:7620")
# Socket facing services
backend = context.socket(zmq.XREQ)
ssh.tunnel_connection(backend,
'tcp://127.0.0.1:7621',
'certainty-b',
timeout=24 * 60 * 60)
zmq.device(zmq.QUEUE, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def _connect(self):
# Get Context
if self._ASYNC:
self._CLIENT = self._CONTEXT.socket(self.zmq.PUSH)
else:
self._CLIENT = self._CONTEXT.socket(self.zmq.REQ)
# Check if SSH is requested
if self._USE_SSH:
from zmq import ssh
# Ensure there exists an SSH Host
if self._SSH_HOST:
try:
ssh.tunnel_connection(self._CLIENT, self._BROKER_HOST,
self._SSH_HOST)
except RuntimeError as e:
raise e
else:
raise AttributeError("No SSH Host.")
else:
self._CLIENT.connect(self._BROKER_HOST)
# Register Poll
self._POLL.register(self._CLIENT, self.zmq.POLLIN)
def client_ssh_connection():
context = zmq.Context()
socket = context.socket(zmq.PAIR)
password = SSH_PWD
username = USER_NAME
server_port = SSH_PORT
server_name = SSH_NAME
server = "{}@{}:{}".format(username, server_name, server_port)
addr = "tcp://{}:{}".format(HOSTNAME, SERVER_PORT)
tunnuel = ssh.tunnel_connection(socket, addr=addr, server=server, password=password)
print("Waiting for server")
msg = socket.recv_json()
# resend message to ensure the integrity of the msg
socket.send_json(msg)
print("send: {}".format(msg))
print("Server Connected")
return socket
import sys, zmq, os, datetime
import zmq.ssh as ssh
msg = {}
msg['name'] = sys.argv[1]
msg['media'] = sys.argv[2]
msg['text'] = sys.argv[3]
msg['posted'] = datetime.datetime.now().strftime('%m/%d/%Y @ %H:%M')
# define sockets
ctx = zmq.Context()
channel = ctx.socket(zmq.REQ)
# tunnel connections using ssh
ssh.tunnel_connection(channel,
"tcp://127.0.0.1:3003",
"root@d")
channel.send_json(msg)
print(channel.recv())
from zmq import ssh
# initialize Flask web server
app = Flask(__name__)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.INFO)
app.config.from_object('config')
# initialize zmq message queue
context = zmq.Context()
port = "6666"
ssh_password = environ.get("SSH_PASSWORD")
app.logger.info("Connecting to titan.elka.pw.edu.pl...")
socket = context.socket(zmq.REQ)
ssh.tunnel_connection(socket,
"tcp://*****:*****@mion.elka.pw.edu.pl",
password=ssh_password)
app.logger.info("Connected!")
@app.route('/')
def homepage():
return render_template('index.html')
@app.route('/rate-photo', methods=["PUT"])
def rate_photo():
app.logger.info('Received image: {} bytes, {}.'.format(
request.content_length, request.content_type))
app.logger.info('Reading data...')
stream = io.BytesIO(request.data)
def client(container, command, output, tunnel):
context = zmq.Context()
if tunnel:
ssh.tunnel.openssh_tunnel(6000, 6000, 'lxplus',
'lheinric-dockerinteractive')
webhost = 'localhost'
else:
webhost = 'lheinric-dockerinteractive'
url = 'http://{0}:6000/start?'.format(webhost)
parameters = []
parameters.append('container={0}'.format(container))
parameters.append('command={0}'.format(command))
if output:
parameters.append('afsdirmount={0}'.format(output))
url = url + '&'.join(parameters)
r = requests.get(url)
if not r.ok:
e = click.ClickException(message=click.style(
'sorry, there is no spot available on the server', fg='red'))
e.exit_code = 1
raise e
readfrom = r.json()['readfrom']
# readfrom = 5556
click.secho('starting remote docker session', fg='green')
#incoming messages
socket = context.socket(zmq.PAIR)
if tunnel:
ssh.tunnel_connection(
socket, 'tcp://lheinric-dockerinteractive:{0}'.format(readfrom),
'lxplus')
else:
socket.connect("tcp://lheinric-dockerinteractive:{0}".format(readfrom))
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
sockets = [socket]
socket.send_json({'ctrl': 'start'})
ack = socket.recv()
istty = os.isatty(sys.stdin.fileno())
socket.send_json({'ctrl': {'tty': istty}})
signal.signal(signal.SIGWINCH, get_sigwinch_handler(socket))
signal.signal(signal.SIGINT, get_sigint_handler(socket))
signal.signal(signal.SIGHUP, get_sighup_handler(socket))
signal.signal(signal.SIGTERM, get_sigterm_handler(socket))
if istty:
handle_tty(socket)
else:
handle_nontty(socket)
click.secho('Bye.', fg='green')
sys.exit(0)
return
def __init__(
self,
args=None, # popen style args
bufsize=0,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=False,
shell=False,
cwd=None,
env=None,
universal_newlines=False,
startupinfo=None,
creationflags=0,
nrv_endpoint=None, # endpoint of center
nrv_identity=None, # id of 0mq socket
nrv_recv_in=False, # recv stdin from cetner?
nrv_send_out=False, # send stdout to center?
nrv_send_err=False, # send stderr to center?
nrv_send_all=True, # send/recv stdin/out/err?
nrv_restart_retries=3, # # of process restarts
nrv_autorestart=False, # restart failed process?
nrv_startsecs=1, # how long to try restarting
nrv_exitcodes=(0, 2), # "good" exit codes, no restart
nrv_poll_interval=0.1, # interval to poll subprocess for life
nrv_ping_interval=1, # interval to ping the center with stats
nrv_wait_to_die=3, # time to wait for the subproc to die
nrv_linger=0, # 0mq socket linger
nrv_ssh_server=None, # ssh server to tunnel to center endpoint
):
"""
Spawn a subprocess.Popen with 'args', watch the process. See
'nrvopen --help' for description of options.
"""
self.endpoint = nrv_endpoint
self.args = args
# pass most args to Popen, but force PIPEs for stdio
self.kwargs = dict(
bufsize=bufsize,
executable=executable,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=preexec_fn,
close_fds=close_fds,
shell=shell,
cwd=cwd,
env=env,
universal_newlines=universal_newlines,
startupinfo=startupinfo,
creationflags=creationflags,
)
self.active = Event()
self.active.set()
self.context = zmq.Context()
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.recv_in = nrv_recv_in
self.send_out = nrv_send_out
self.send_err = nrv_send_err
if nrv_send_all:
self.recv_in = self.send_out = self.send_err = True
self.restart_retries = nrv_restart_retries
self.restart_attempts = 0
self.autorestart = nrv_autorestart
self.startsecs = nrv_startsecs
self.exitcodes = set(nrv_exitcodes)
self.ping_interval = nrv_ping_interval
self.poll_interval = nrv_poll_interval
self.wait_to_die = nrv_wait_to_die
self.uptime = None
# create a connection to the zerovisor router
self.io = self.context.socket(zmq.DEALER)
if nrv_identity is not None:
self.io.setsockopt(zmq.IDENTITY, nrv_identity)
if nrv_ssh_server is not None:
from zmq import ssh
self.tunnel = ssh.tunnel_connection(self.io, self.endpoint, nrv_ssh_server)
else:
self.io.connect(self.endpoint)
self.io.setsockopt(zmq.LINGER, nrv_linger)
def __init__(self,
ID,
env_id='Pong-ram-v0',
collector_ip=None,
psw="",
traj_length=20,
batch_size=16,
max_train=11,
early_stop=100,
round_length=300,
max_eval=40000,
min_games=1,
subprocess=True,
mutation_chance=0.5,
mutation_rate=1.0,
crossover_chance=0.8):
if collector_ip is None:
self.ip = socket.gethostbyname(socket.gethostname())
else:
self.ip = collector_ip
self.ID = ID
self.gpu = -int(int(os.environ['CUDA_VISIBLE_DEVICES']) < 0)
physical_devices = tf.config.list_physical_devices('GPU')
print(physical_devices, self.gpu)
if len(physical_devices) > 0:
print('setting memory limit')
tf.config.experimental.set_memory_growth(physical_devices[0],
enable=True)
# tf.config.experimental.set_virtual_device_configuration(physical_devices[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4096)])
self.envs = [gym.make(env_id) for _ in range(5)]
self.obs = [None] * 5
self.util = name2class[env_id]
self.action_dim = self.util.action_space_dim
self.state_shape = (self.util.full_state_dim, )
#self.env = make_env_mario(self.util.name, 2, 4)
#self.env = JoypadSpace(self.env, SIMPLE_MOVEMENT)
#self.state_shape = self.util.state_dim
#self.action_dim = self.util.action_space_dim
self.mutation_rate = mutation_rate
self.mutation_chance = mutation_chance
self.crossover_chance = crossover_chance
self.player = Individual(self.state_shape,
self.action_dim,
self.util.goal_dim,
traj_length=traj_length,
batch_size=batch_size)
self.frame_skip = 4
if subprocess:
context = zmq.Context()
self.mating_pipe = context.socket(zmq.PULL)
self.evolved_pipe = context.socket(zmq.PUSH)
self.tunneling = (psw != "")
self.psw = psw
self.mating_pipe.setsockopt(zmq.RCVTIMEO, 60 * 1000)
self.mating_pipe.setsockopt(zmq.LINGER, 0)
if self.tunneling:
print('tunnel')
ssh.tunnel_connection(self.mating_pipe,
"tcp://%s:5655" % self.ip,
"villinvic@%s" % self.ip,
password=psw)
ssh.tunnel_connection(self.evolved_pipe,
"tcp://%s:5656" % self.ip,
"villinvic@%s" % self.ip,
password=psw)
else:
self.mating_pipe.connect("tcp://%s:5655" % self.ip)
self.evolved_pipe.connect("tcp://%s:5656" % self.ip)
self.traj_length = traj_length
self.max_train = max_train
self.early_stop = early_stop
self.round_length = round_length
self.batch_size = batch_size
self.max_eval = max_eval
self.min_games = min_games
self.trajectory = {
'state':
np.zeros((batch_size, traj_length) + self.state_shape,
dtype=np.float32),
'action':
np.zeros((batch_size, self.traj_length), dtype=np.int32),
'rew':
np.zeros((batch_size, self.traj_length), dtype=np.float32),
'base_rew':
np.zeros((batch_size, self.traj_length), dtype=np.float32),
}
if subprocess:
signal.signal(signal.SIGINT, self.exit)
def connectSocketViaSSH(self, socket: zmq.Socket, host: str, port: int,
server_ssh: str):
#self.addClientAuth( socket ) # TODO: this is commented to avoid key checking
ssh.tunnel_connection(socket, "tcp://{0}:{1}".format(host, port),
server_ssh)
return port
def run_once(self, timeout=None):
"""run the subscriber, if a timeout occurs return from run loop
:param timeout: timeout in seconds, None means don't timeout
"""
# Prepare our context and subscriber
snapshot = SolidSocket(self._ctx, zmq.DEALER, default_timeout=timeout)
snapshot.linger = 0
addr = "tcp://localhost:%d" % self._ports[1]
if self._ssh_args:
tunnel = ssh.tunnel_connection(snapshot, addr, **self._ssh_args)
self.tunnels += [tunnel]
else:
snapshot.connect(addr)
subscriber = SolidSocket(self._ctx, zmq.SUB, default_timeout=timeout)
subscriber.linger = 0
subscriber.setsockopt(zmq.SUBSCRIBE, '')
addr = "tcp://localhost:%d" % self._ports[0]
if self._ssh_args:
tunnel = ssh.tunnel_connection(subscriber, addr, **self._ssh_args)
self.tunnels += [tunnel]
else:
subscriber.connect(addr)
# Get state snapshot
if self._sequence >= 0:
logger.info("Requesting shapshot")
if self._runid is None:
runid = ""
else:
runid = self._runid
snapshot.send_multipart(["sync", runid, str(self._sequence)])
while self.go:
#try:
msg = Message.recv(snapshot)
#except:
# break # Interrupted
if msg.topic == "command":
if msg.data == "finished":
logger.info("Received snapshot=%d" % self._sequence)
break # Done
elif self.data == "exit":
break
else:
self._sequence = msg.sequence
self._runid = msg.runid
self.handle(msg)
if self._seqfile is not None:
self._seqfile.write(self._sequence, self._runid)
# Now apply pending updates, discard out-of-sequence messages
while self.go:
#try:
msg = Message.recv(subscriber)
#except:
# self.stop()
# break # Interrupted
if msg.runid != self._runid and self._runid is not None:
logger.info("Detecting new runid runid %s != %s" %
(uuid.UUID(bytes=self._runid),
uuid.UUID(bytes=msg.runid)))
self._runid = msg.runid
self._sequence = 0
if msg.topic == "command" and msg.data == "exit":
logger.info("Explicit shutdown received lastseq=%d" %
self._sequence)
break # Done
#TODO > or >=
if msg.sequence > self._sequence:
self._sequence = msg.sequence
self.handle(msg)
if self._seqfile is not None:
self._seqfile.write(self._sequence, self._runid)
subscriber.close()
snapshot.close()
def run_once(self, timeout=None):
"""run the subscriber, if a timeout occurs return from run loop
:param timeout: timeout in seconds, None means don't timeout
"""
# Prepare our context and subscriber
snapshot = SolidSocket(self._ctx, zmq.DEALER, default_timeout=timeout)
snapshot.linger = 0
addr = "tcp://localhost:%d" % self._ports[1]
if self._ssh_args:
tunnel = ssh.tunnel_connection(snapshot, addr, **self._ssh_args)
self.tunnels += [tunnel]
else:
snapshot.connect(addr)
subscriber = SolidSocket(self._ctx, zmq.SUB, default_timeout=timeout)
subscriber.linger = 0
subscriber.setsockopt(zmq.SUBSCRIBE, '')
addr = "tcp://localhost:%d" % self._ports[0]
if self._ssh_args:
tunnel = ssh.tunnel_connection(subscriber, addr, **self._ssh_args)
self.tunnels += [tunnel]
else:
subscriber.connect(addr)
# Get state snapshot
if self._sequence >= 0:
logger.info("Requesting shapshot")
if self._runid is None:
runid = ""
else:
runid = self._runid
snapshot.send_multipart(["sync", runid, str(self._sequence)])
while self.go:
#try:
msg = Message.recv(snapshot)
#except:
# break # Interrupted
if msg.topic == "command":
if msg.data == "finished":
logger.info("Received snapshot=%d" % self._sequence)
break # Done
elif self.data == "exit":
break
else:
self._sequence = msg.sequence
self._runid = msg.runid
self.handle(msg)
if self._seqfile is not None:
self._seqfile.write(self._sequence, self._runid)
# Now apply pending updates, discard out-of-sequence messages
while self.go:
#try:
msg = Message.recv(subscriber)
#except:
# self.stop()
# break # Interrupted
if msg.runid != self._runid and self._runid is not None:
logger.info(
"Detecting new runid runid %s != %s" %
(uuid.UUID(bytes=self._runid), uuid.UUID(bytes=msg.runid)))
self._runid = msg.runid
self._sequence = 0
if msg.topic == "command" and msg.data == "exit":
logger.info("Explicit shutdown received lastseq=%d" %
self._sequence)
break # Done
#TODO > or >=
if msg.sequence > self._sequence:
self._sequence = msg.sequence
self.handle(msg)
if self._seqfile is not None:
self._seqfile.write(self._sequence, self._runid)
subscriber.close()
snapshot.close()
def _init_socket(
ctx: 'zmq.Context',
host: str,
port: Optional[int],
socket_type: 'SocketType',
identity: Optional[str] = None,
use_ipc: bool = False,
ssh_server: Optional[str] = None,
ssh_keyfile: Optional[str] = None,
ssh_password: Optional[str] = None,
) -> Tuple['zmq.Socket', str]:
sock = {
SocketType.PULL_BIND: lambda: ctx.socket(zmq.PULL),
SocketType.PULL_CONNECT: lambda: ctx.socket(zmq.PULL),
SocketType.SUB_BIND: lambda: ctx.socket(zmq.SUB),
SocketType.SUB_CONNECT: lambda: ctx.socket(zmq.SUB),
SocketType.PUB_BIND: lambda: ctx.socket(zmq.PUB),
SocketType.PUB_CONNECT: lambda: ctx.socket(zmq.PUB),
SocketType.PUSH_BIND: lambda: ctx.socket(zmq.PUSH),
SocketType.PUSH_CONNECT: lambda: ctx.socket(zmq.PUSH),
SocketType.PAIR_BIND: lambda: ctx.socket(zmq.PAIR),
SocketType.PAIR_CONNECT: lambda: ctx.socket(zmq.PAIR),
SocketType.ROUTER_BIND: lambda: ctx.socket(zmq.ROUTER),
SocketType.DEALER_CONNECT: lambda: ctx.socket(zmq.DEALER),
}[socket_type]()
sock.setsockopt(zmq.LINGER, 0)
if socket_type == SocketType.DEALER_CONNECT:
sock.set_string(zmq.IDENTITY, identity)
# if not socket_type.is_pubsub:
# sock.hwm = int(os.environ.get('JINA_SOCKET_HWM', 1))
if socket_type.is_bind:
if use_ipc:
sock.bind(host)
else:
# JEP2, if it is bind, then always bind to local
if host != __default_host__:
default_logger.warning(
f'host is set from {host} to {__default_host__} as the socket is in BIND type'
)
host = __default_host__
if port is None:
sock.bind_to_random_port(f'tcp://{host}')
else:
try:
sock.bind(f'tcp://{host}:{port}')
except zmq.error.ZMQError:
default_logger.error(
f'error when binding port {port} to {host}, this port is occupied. '
f'If you are using Linux, try `lsof -i :{port}` to see which process '
f'occupies the port.'
)
raise
else:
if port is None:
address = host
else:
address = f'tcp://{host}:{port}'
# note that ssh only takes effect on CONNECT, not BIND
# that means control socket setup does not need ssh
if ssh_server:
tunnel_connection(sock, address, ssh_server, ssh_keyfile, ssh_password)
else:
sock.connect(address)
if socket_type in {SocketType.SUB_CONNECT, SocketType.SUB_BIND}:
# sock.setsockopt(zmq.SUBSCRIBE, identity.encode('ascii') if identity else b'')
sock.subscribe('') # An empty shall subscribe to all incoming messages
return sock, sock.getsockopt_string(zmq.LAST_ENDPOINT)
import zmq from zmq import ssh import time # import json import struct context = zmq.Context() socket = context.socket(zmq.REQ) ssh.tunnel_connection(socket, "tcp://127.0.0.1:5563", "[email protected]") #socket.bind("tcp://127.0.0.1:5563") requesId = 0 while True: request = "Message %d" % requesId print("Sending '%s'.." % request) socket.send_string(request) response = socket.recv() # decode bytes to string # data_string = response.decode('utf8') data_received = struct.unpack('dddd', response) # data_json = json.loads(data_string) print("Response data:", data_received) requesId += 1 time.sleep(1)
