def test_rpyc(self):
p1, p2 = PipeStream.create_pair()
client = rpyc.connect_stream(p1)
server = rpyc.connect_stream(p2)
server_thread = rpyc.spawn(server.serve_all)
assert client.root.get_service_name() == "VOID"
t = rpyc.BgServingThread(client)
assert server.root.get_service_name() == "VOID"
t.stop()
client.close()
server.close()
server_thread.join()
def test_rpyc(self):
p1, p2 = PipeStream.create_pair()
client = rpyc.connect_stream(p1)
server = rpyc.connect_stream(p2)
server_thread = rpyc.spawn(server.serve_all)
assert client.root.get_service_name() == "VOID"
t = rpyc.BgServingThread(client)
assert server.root.get_service_name() == "VOID"
t.stop()
client.close()
server.close()
server_thread.join()
def connect_device_service(addr,
port=18812,
timeout=3,
attempts=2,
error_on_fail=True):
"""
Connect to the :class:`DeviceService` running at the given address and port
`timeout` and `attempts` define respectively timeout of a single connection attempt, and the number of attempts
(RPyC default is 3 seconds timeout and 6 attempts).
If ``error_on_fail==True``, raise error if the connection failed; otherwise, return ``None``
"""
addr, port = net.as_addr_port(addr, port)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
try:
s = rpyc.SocketStream.connect(addr,
port,
timeout=timeout,
attempts=attempts)
return rpyc.connect_stream(s, SocketTunnelService).root
except net.socket.timeout:
if error_on_fail:
raise
return None
def connect(self, service=VoidService, config=None):
"""Same as :func:`connect <rpyc.utils.factory.connect>`, but with the ``host`` and ``port``
parameters fixed"""
config = config or {}
stream = SocketStream(
self.remote_machine.connect_sock(self.remote_port))
return rpyc.connect_stream(stream, service=service, config=config)
def connect(self, service = VoidService, config = {}):
"""Same as :func:`connect <rpyc.utils.factory.connect>`, but with the ``host`` and ``port``
parameters fixed"""
if self.local_port is None:
# ParamikoMachine
stream = SocketStream(self.remote_machine.connect_sock(self.remote_port))
return rpyc.connect_stream(stream, service = service, config = config)
else:
return rpyc.connect("localhost", self.local_port, service = service, config = config)
def connect(self, service = VoidService, config = {}):
"""Same as :func:`connect <rpyc.utils.factory.connect>`, but with the ``host`` and ``port``
parameters fixed"""
if self.local_port is None:
# ParamikoMachine
stream = SocketStream(self.remote_machine.connect_sock(self.remote_port))
return rpyc.connect_stream(stream, service = service, config = config)
else:
return rpyc.connect("localhost", self.local_port, service = service, config = config)
def call_request_vote(self, otherNodeDesc):
'''sends a vote request message to some other node by calling that RPC on that node
:param otherNodeDesc NodeRef: a description of the other node
:return tuple(int, boolean): what term the other node thinks is most recent and
whether the other node will vote for this one
'''
assert self.isCandidate
requestVoteRetVal = None
voteRequestRpcStartTime = time.time()
try:
nodeConnStream = rpyc.SocketStream.connect(
otherNodeDesc.host,
otherNodeDesc.port,
timeout=RaftNode.CONNECTION_TIMEOUT,
attempts=1)
nodeConn = rpyc.connect_stream(nodeConnStream)
otherNodeRoot = nodeConn.root
timedRequestVoteProxy = helpers.timed(otherNodeRoot.request_vote,
RaftNode.CONNECTION_TIMEOUT)
voteRequestPromise = timedRequestVoteProxy(self.currTerm,
self.identityIndex)
requestVoteRetVal = voteRequestPromise.value
except AsyncResultTimeout:
self.nodeLogger.info(
"connection timed out while candidate node %d in term %d tried to send request_vote "
"to node %s", self.identityIndex, self.currTerm,
otherNodeDesc.name)
except (socket.timeout, ConnectionRefusedError):
self.nodeLogger.info(
"candidate node %d in term %d was unable to connect to another node %s",
self.identityIndex, self.currTerm, otherNodeDesc.name)
except EOFError:
self.nodeLogger.info(
"candidate node %d in term %d lost connection to another node %s",
self.identityIndex, self.currTerm, otherNodeDesc.name)
except Exception as e:
self.nodeLogger.error(
"Exception for candidate node %d in term %d: %s\n%s\n%s",
self.identityIndex, self.currTerm, e.__doc__, str(e),
traceback.format_exc())
voteRequestRpcDuration = time.time() - voteRequestRpcStartTime
self.nodeLogger.debug(
"sending vote request to node %s took %f seconds",
otherNodeDesc.name, voteRequestRpcDuration)
return requestVoteRetVal
def call_append_entries(self, otherNodeDesc):
'''send an append_entries/heartbeat 'message' to another node by calling that RPC on that node
:param otherNodeDesc NodeRef: a description of the other node
:return tuple(int, boolean): what term the other node thinks is most recent and whether the other node accepts
this node as the leader
'''
# assert self.exposed_is_leader() this might not always be true because of concurrency
appendEntriesRetVal = None
heartbeatRpcStartTime = time.time()
try:
nodeConnStream = rpyc.SocketStream.connect(
otherNodeDesc.host,
otherNodeDesc.port,
timeout=RaftNode.CONNECTION_TIMEOUT,
attempts=1)
nodeConn = rpyc.connect_stream(nodeConnStream)
otherNodeRoot = nodeConn.root
timedAppendEntriesProxy = helpers.timed(
otherNodeRoot.append_entries, RaftNode.CONNECTION_TIMEOUT)
appendEntriesPromise = timedAppendEntriesProxy(
self.currTerm, self.identityIndex)
appendEntriesRetVal = appendEntriesPromise.value
except AsyncResultTimeout:
self.nodeLogger.info(
"connection timed out while leader node %d in term %d tried to send append_entries "
"to node %s", self.identityIndex, self.currTerm,
otherNodeDesc.name)
except (socket.timeout, ConnectionRefusedError):
self.nodeLogger.info(
"leader node %d in term %d was unable to connect to another node %s",
self.identityIndex, self.currTerm, otherNodeDesc.name)
except EOFError:
self.nodeLogger.info(
"leader node %d in term %d lost connection to another node %s",
self.identityIndex, self.currTerm, otherNodeDesc.name)
except Exception as e:
self.nodeLogger.error(
"Exception for leader node %d in term %d: %s\n%s\n%s",
self.identityIndex, self.currTerm, e.__doc__, str(e),
traceback.format_exc())
heartbeatRpcDuration = time.time() - heartbeatRpcStartTime
self.nodeLogger.debug(
"sending append_entries to other node %s took %f seconds",
otherNodeDesc.name, heartbeatRpcDuration)
return appendEntriesRetVal
def __try_connect(self):
import rpyc
try:
stream = rpyc.SocketStream.connect(host="127.0.0.1",
port=self.rpyc_port,
nodelay=True,
keepalive=True)
self.__connection = rpyc.connect_stream(
stream,
self._get_service(),
config={"sync_request_timeout": RPYC_REQUEST_TIMEOUT},
)
except (ConnectionRefusedError, EOFError):
if self.proc.poll() is not None:
raise ClusterError(
f"SSH tunnel died, return code: {self.proc.returncode}")
def newRPyCConnectionOverSsh(command, host, username, password):
"""get a RPyC connection to a given host via ssh
Open a ssh connection to the given host and
run command. Command must start a RPyC server, that
is connected to its stdio. Usually command will look
similar to::
stackless_python2.7 .../rpyc/scripts/rpyc_classic.py -m stdio -q
(You Run the command on the given host via host and return a RpycStream object
The RpycStream can be used to communicate with the command.
"""
if not isinstance(command, basestring) and isinstance(command, collections.Sequence):
# convert the argv string vector to a single string
command = argv2command(command)
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, username=username, password=password) # there are more authentication options
transport = client.get_transport()
transport.setName("paramiko.Transport for rpyc stream")
channel = transport.open_session()
channel.exec_command(command)
# append stderr to sys.stderr or self.stderr
# the thread must hold a copy of the paramiko client object in order to keep the connection alive
closeEvent = threading.Event()
closeEvent.clear()
stderrThread = threading.Thread(target=__copy2stderr,
args=(None, channel, client, closeEvent),
name="Stderr copy for %s" % (channel.get_name(),))
stderrThread.daemon = False # keep the interpreter alive until this thread is done
stderrThread.start()
prs = SshRpycStream(channel)
# keep the paramiko-client alive. This is very important, otherwise the connection will shutdown
prs._sshClient = client
c = rpyc.connect_stream(prs, rpyc.SlaveService)
return c
def setup(self):
self.pipe_server_thread = Thread(target=self.pipe_server)
self.pipe_server_thread.start()
time.sleep(1) # make sure server is accepting already
self.np_client = NamedPipeStream.create_client("floop")
self.client = rpyc.connect_stream(self.np_client)
def pipe_server(self):
self.np_server = NamedPipeStream.create_server("floop")
self.server = rpyc.connect_stream(self.np_server)
self.server.serve_all()
def _create_conn(self, ce_ip, ce_port, ep_names, debug=False):
"""
Helper for creating a Central Engine connection, the most basic func.
"""
proxy = None
config = {
'allow_pickle': True,
'allow_getattr': True,
'allow_setattr': True,
'allow_delattr': True,
'allow_all_attrs': True,
}
def close_conn():
try:
proxy.close()
except Exception:
pass
proxy = None
# Connect to RPyc server
try:
r_stream = rpyc.SocketStream.connect(ce_ip, ce_port, timeout=3.0)
proxy = rpyc.connect_stream(r_stream,
service=TwisterClientService,
config=config)
logPrint('Client Debug: Connected to CE at `{}:{}`...'.format(
ce_ip, ce_port))
except Exception as e:
if debug:
logPrint('*ERROR* Cannot connect to CE path `{}:{}`! '\
'Exception: `{}`!'.format(ce_ip, ce_port, e))
close_conn()
return None
# Authenticate on RPyc server
try:
check = proxy.root.login(self.user_name, 'EP')
if check:
logPrint('Client Debug: Authentication successful!')
except Exception as e:
check = False
if not check:
if debug:
logPrint(
'*ERROR* Cannot authenticate on CE path `{}:{}`! Invalid login!'
.format(ce_ip, ce_port))
close_conn()
return None
# Say Hello and Register all EPs on the current Central Engine
if ep_names:
try:
proxy.ping(data='Hello', timeout=3)
# Call the user status to create the User Project
s = proxy.root.get_user_variable('user_roles')
if not s:
logPrint(
'*ERROR* Cannot register! Cannot get roles for user `{}`!'
.format(self.user_name))
close_conn()
return None
# Fire up the User Service
proxy.root.read_file('~/twister/config/fwmconfig.xml')
except Exception as e:
logPrint('Exception: `{}`'.format(e))
check = False
try:
proxy.root.hello('client', {'eps': ep_names})
logPrint('Client Debug: Register EPs successful!')
except Exception as e:
logPrint('Exception: `{}`'.format(e))
check = False
if not check:
if debug:
logPrint(
'*ERROR* Cannot register! Cannot send hello on CE path `{}:{}`!'
.format(ce_ip, ce_port))
close_conn()
return None
BgServingThread(proxy)
return proxy
def _usr_service(self, user, oper='read'):
"""
Launch a user service.
"""
if oper not in ['read', 'write']:
logWarning(
'Invalid FS operation `{}`, for user `{}`! Will reset to "read".'
.format(oper, user))
oper = 'read'
# Must block here, so more users cannot launch Logs at the same time and lose the PID
with self._srv_lock:
# Try to re-use the logger server, if available
conn = self._services.get(user, {}).get('conn_' + oper, None)
if conn:
try:
conn.ping(data='Hello', timeout=30.0)
# logDebug('Reuse old {} User Service connection for `{}` OK.'.format(op, user))
return conn
except Exception as exp_err:
logWarning(
'Cannot reuse {} User Service for `{}`: `{}`.'.format(
oper, user, exp_err))
self._kill(user)
else:
logInfo('Launching a User Service for `{}`, the first time...'.
format(user))
port = None
# If the server is not available, search for a free port in the safe range...
while 1:
port = random.randrange(63000, 65000)
try:
socket.create_connection((None, port), 1)
except Exception:
break
p_cmd = 'su {} -c "{} -u {}/server/UserService.py {} {}"'.\
format(user, sys.executable, TWISTER_PATH, port, self.name)
proc = subprocess.Popen(p_cmd, cwd='{}/twister'.\
format(userHome(user)), shell=True, close_fds=True,\
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.poll()
time.sleep(2.0)
config = {
'allow_pickle': True,
'allow_getattr': True,
'allow_setattr': True,
'allow_delattr': True
}
retry = 10
delay = 0.5
success = False
while retry > 0:
if success:
break
try:
stream_r = rpyc.SocketStream.connect('127.0.0.1',
port,
timeout=5.0)
conn_read = rpyc.connect_stream(stream_r, config=config)
conn_read.root.hello()
logDebug(
'Connected to User Service for `{}`, operation `read`.'
.format(user))
success = True
except Exception as exp_err:
logWarning('Cannot connect to User Service for `{}` - \
Exception: `{}`! Wait {}s...'.format(user, exp_err, delay))
if success:
try:
stream_w = rpyc.SocketStream.connect('127.0.0.1',
port,
timeout=5.0)
conn_write = rpyc.connect_stream(stream_w,
config=config)
conn_write.root.hello()
logDebug(
'Connected to User Service for `{}`, operation `write`.'
.format(user))
break
except Exception as exp_err:
logWarning('Cannot connect to User Service for `{}` \
- Exception: `{}`! Wait {}s...' .\
format(user, exp_err, delay))
success = False
time.sleep(delay)
retry -= 1
delay += 0.75
if not success:
logError(
'Error on starting User Service for `{}`!'.format(user))
return None
# Save the process inside the block. 99% of the time, this block is executed instantly!
self._services[user] = {
'proc': proc,
'conn_read': conn_read,
'conn_write': conn_write,
'port': port
}
logDebug(
'User Service for `{}` launched on `127.0.0.1:{}` - PID `{}`.'.
format(user, port, proc.pid))
return self._services[user].get('conn_' + oper, None)
def _usr_service(self, user_view_actv):
"""
Launch a user service.
Open a ClearCase view first.
"""
if user_view_actv[-1] != ':':
user_view_actv += ':'
try:
user, view, actv = user_view_actv.split(':')
except Exception:
# We don't really know the user in here !
msg = 'Invalid ClearCase user-view-activity parameter: `{}`!'.format(
user_view_actv)
logWarning(msg)
return '*ERROR* ' + msg
view = view.strip()
actv = actv.strip()
user_view = user + ':' + view
if not view:
# We don't know the view in here !
msg = 'Empty view in `{}`!'.format(user_view_actv)
logWarning(msg)
return '*ERROR* ' + msg
# Must block here, so more users cannot launch Logs at the same time and lose the PID
with self._srv_lock:
def pread():
""" Read proc stdout """
while 1:
try:
line = proc.readline().strip()
if not line:
continue
plog.append(line)
except:
break
# Try to re-use the FS, if available
conn = self._services.get(user_view, {}).get('conn', None)
if conn:
try:
conn.ping(data='Hello', timeout=30.0)
# logDebug('Reuse old ClearCase Service connection for `{}` OK.'.format(user))
proc = self._services.get(user_view, {}).get('proc', None)
old_actv = self._services.get(user_view,
{}).get('actv', None)
if actv != old_actv:
logInfo('Changing activity to `{}`, for `{}`.'.format(
actv, user_view))
# Set cc activity again !
proc.sendline('cleartool setactivity {}'.format(actv))
time.sleep(1.0)
pread()
self._services.get(user_view, {})['actv'] = actv
return conn
except Exception as e:
logWarning(
'Cannot connect to ClearCase Service for `{}`: `{}`.'.
format(user_view, e))
self._kill(user)
proc = self._services.get(user_view, {}).get('proc', None)
PID = proc.pid
proc.terminate()
logInfo('Terminated CC User Service `{}` for user `{}`.'.
format(PID, user))
else:
logInfo(
'Launching a ClearCase Service for `{}`, the first time...'
.format(user_view))
proc = pexpect.spawn(['bash'], timeout=2.5, maxread=2048)
time.sleep(2.0)
plog = []
proc.sendline('su {}'.format(user))
time.sleep(2.0)
pread()
# User's home folder
proc.sendline('cd ~/twister')
pread()
# Set cc view only the first time !
proc.sendline('cleartool setview {}'.format(view))
time.sleep(2.0)
pread()
# Empty line after set view
proc.sendline('')
pread()
if actv:
# Set cc activity for the first time !
proc.sendline('cleartool setactivity {}'.format(actv))
time.sleep(1.0)
pread()
port = None
# If the server is not available, search for a free port in the safe range...
while 1:
port = random.randrange(63000, 65000)
try:
socket.create_connection((None, port), 1)
except:
break
# Launching 1 UserService inside the SSH terminal, with ClearCase View open
p_cmd = '{} -u {}/server/UserService.py {} ClearCase & '.format(
sys.executable, TWISTER_PATH, port)
proc.sendline(p_cmd)
time.sleep(2.0)
pread()
# Empty line after proc start
proc.sendline('')
pread()
logDebug(
'ClearCase startup log \n:: -------\n{}\n:: -------'.format(
'\n'.join(plog)))
config = {
'allow_pickle': True,
'allow_getattr': True,
'allow_setattr': True,
'allow_delattr': True
}
retry = 10
delay = 0.5
success = False
while retry > 0:
try:
stream = rpyc.SocketStream.connect('127.0.0.1',
port,
timeout=5.0)
conn = rpyc.connect_stream(stream, config=config)
conn.root.hello()
logDebug('Connected to ClearCase Service for `{}`.'.format(
user_view))
success = True
break
except Exception as e:
logWarning('Cannot connect to ClearCase Service for `{}` \
- Exception: `{}`! Retry...'.format(user_view, e))
time.sleep(delay)
retry -= 1
delay += 0.75
if not success:
logError(
'Error on starting ClearCase Service for `{}`!'.format(
user_view))
return None
# Save the process inside the block.
self._services[user_view] = {
'proc': proc,
'conn': conn,
'port': port,
'actv': actv
}
logDebug(
'ClearCase Service for `{}` launched on `127.0.0.1:{}`.'.format(
user_view, port))
return conn
def setUp(self):
self.pipe_server_thread = rpyc.spawn(self.pipe_server)
time.sleep(1) # make sure server is accepting already
self.np_client = NamedPipeStream.create_client("floop")
self.client = rpyc.connect_stream(self.np_client)
def pipe_server(self):
self.np_server = NamedPipeStream.create_server("floop")
self.server = rpyc.connect_stream(self.np_server)
self.server.serve_all()
def setUp(self):
self.pipe_server_thread = Thread(target=self.pipe_server)
self.pipe_server_thread.start()
time.sleep(1) # make sure server is accepting already
self.np_client = NamedPipeStream.create_client("floop")
self.client = rpyc.connect_stream(self.np_client)
