def start(self, mode):
"""
In CLIENT_MODE:
Establish a connection with the server before sending a new message.
In SERVER_MODE:
Start listening to the incoming messages from the clients.
:param mode: (int) Either CLIENT_MODE or SERVER_MODE.
:return: None
"""
address = 'tcp://{}:{}'.format(self.ip, self.port)
if mode == CLIENT_MODE:
self.mode = mode
self.logger.info(
"Starting Scoreboard Client listening on {}:{} ...".format(
self.ip, self.port))
self.instance = Proxy(address)
elif mode == SERVER_MODE:
self.mode = mode
self.scoreboard = Scoreboard()
server = Server(self, address)
self.logger.info(
"Starting Scoreboard Server listening on {}:{} ...".format(
self.ip, self.port))
server.serve_forever()
def test_signal_two_proxies(self):
address = 'tcp://127.0.0.1:6009'
s = Server(Example(), rep_endpoint=address)
proxy1 = Proxy(address)
proxy2 = Proxy(address)
class MemMethod(object):
def __init__(self_):
self_.called = 0
def __call__(self_, value, old_value, others):
self_.called += 1
fun = MemMethod()
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy2.rw_prop_changed.connect(fun)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy1.p = 28
proxy1._proxy_stop_server()
proxy1._proxy_stop_me()
proxy2._proxy_stop_me()
def test_signal(self):
address = 'tcp://127.0.0.1:6008'
s = Server(Example(), rep_endpoint=address)
proxy = Proxy(address)
class MemMethod(object):
def __init__(self_):
self_.called = 0
def __call__(self_, value, old_value, others):
self_.called += 1
fun1 = MemMethod()
self.assertEqual(fun1.called, 0)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy.rw_prop_changed.connect(fun1)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 28
time.sleep(SLEEP_SECS)
self.assertEqual(proxy.rw_prop, 28)
self.assertEqual(fun1.called, 1)
fun2 = MemMethod()
self.assertEqual(fun2.called, 0)
proxy.rw_prop_changed.connect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 29
time.sleep(SLEEP_SECS)
self.assertEqual(proxy.rw_prop, 29)
self.assertEqual(fun1.called, 2)
self.assertEqual(fun2.called, 1)
proxy.rw_prop_changed.disconnect(fun1)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 30
self.assertEqual(fun1.called, 2)
proxy.rw_prop_changed.disconnect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy.rw_prop_changed.connect(fun1)
proxy.rw_prop_changed.connect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop_changed.disconnect(None)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy._proxy_stop_server()
proxy._proxy_stop_me()
def test_signal(self):
address = 'tcp://127.0.0.1:6008'
s = Server(Example(), rep_endpoint=address)
proxy = Proxy(address)
class MemMethod(object):
def __init__(self_):
self_.called = 0
def __call__(self_, value, old_value, others):
self_.called += 1
fun1 = MemMethod()
self.assertEqual(fun1.called, 0)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy.rw_prop_changed.connect(fun1)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 28
time.sleep(SLEEP_SECS)
self.assertEqual(proxy.rw_prop, 28)
self.assertEqual(fun1.called, 1)
fun2 = MemMethod()
self.assertEqual(fun2.called, 0)
proxy.rw_prop_changed.connect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 29
time.sleep(SLEEP_SECS)
self.assertEqual(proxy.rw_prop, 29)
self.assertEqual(fun1.called, 2)
self.assertEqual(fun2.called, 1)
proxy.rw_prop_changed.disconnect(fun1)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop = 30
self.assertEqual(fun1.called, 2)
proxy.rw_prop_changed.disconnect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy.rw_prop_changed.connect(fun1)
proxy.rw_prop_changed.connect(fun2)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy.rw_prop_changed.disconnect(None)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy._proxy_stop_server()
proxy._proxy_stop_me()
def get_mmvt_object(subject):
mmvt = None
pizco_log_fname = op.join(get_mmvt_dir(), subject, 'logs', 'pizco.log')
waits_for_file(pizco_log_fname)
with open(pizco_log_fname, 'r') as log:
pizco_address = log.read()
try:
from pizco import Proxy
devnull = open(os.devnull, 'w')
with RedirectStdStreams(stdout=devnull, stderr=devnull):
mmvt = Proxy(pizco_address)
except:
pass
return mmvt
def test_server_return_dict(self):
s = ReturnDictsServer(Example())
proxy = Proxy(s.rep_endpoint)
self.assertEqual(s.served_object.dict_attribute[1], 2)
self.assertEqual(s.served_object.dict_attribute, {1: 2})
self.assertEqual(proxy.dict_attribute[1], 2)
self.assertEqual(proxy.dict_attribute, {1: 2})
# This should not work as the dictionary is not remotely linked
proxy.dict_attribute[2] = 4
self.assertEqual(s.served_object.dict_attribute, {1: 2})
proxy._proxy_stop_server()
proxy._proxy_stop_me()
def test_future(self):
s = Server(Example())
proxy = Proxy(s.rep_endpoint)
fut = proxy.fut()
self.assertEqual(fut.result(), 10)
fut = proxy.fut_raise()
self.assertIsInstance(fut.exception(), ValueError)
fut = proxy.fut_raise()
self.assertRaises(ValueError, fut.result)
proxy._proxy_stop_me()
s.stop()
def test_future(self):
s = Server(Example())
proxy = Proxy(s.rep_endpoint)
fut = proxy.fut()
self.assertEqual(fut.result(), 10)
fut = proxy.fut_raise()
self.assertIsInstance(fut.exception(), ValueError)
fut = proxy.fut_raise()
self.assertRaises(ValueError, fut.result)
proxy._proxy_stop_me()
s.stop()
def test_server_return_dict(self):
s = ReturnDictsServer(Example())
proxy = Proxy(s.rep_endpoint)
self.assertEqual(s.served_object.dict_attribute[1], 2)
self.assertEqual(s.served_object.dict_attribute, {1: 2})
self.assertEqual(proxy.dict_attribute[1], 2)
self.assertEqual(proxy.dict_attribute, {1: 2})
# This should not work as the dictionary is not remotely linked
proxy.dict_attribute[2] = 4
self.assertEqual(s.served_object.dict_attribute, {1: 2})
proxy._proxy_stop_server()
proxy._proxy_stop_me()
# -*- coding: utf-8 -*-
import sys
import time
import random
if sys.version_info < (3,0):
input = raw_input
from pizco import Proxy
proxy = Proxy('tcp://127.0.0.1:8000')
colors = ('green', 'blue', 'white', 'yellow')
while True:
input('Press enter to run ...\n')
proxy.door_open = True
proxy.lights_on = True
time.sleep(.1)
proxy.paint(random.choice(colors))
proxy.lights_on = False
proxy.door_open = False
self.layout3 = QHBoxLayout()
self.label3 = QLabel()
self.label3.setMinimumSize(QSize(100, 0))
self.label3.setText('Color:')
self.color = QLineEdit()
self.color.setReadOnly(True)
self.color.textChanged.connect(self.on_color_text_changed)
self.color_box = QLabel()
self.color_box.setText(' ')
self.layout3.addWidget(self.label3)
self.layout3.addWidget(self.color)
self.layout3.addWidget(self.color_box)
self.lights.stateChanged.connect(self.on_lights_on_changed)
self.door_open.stateChanged.connect(self.on_door_open_changed)
self.layout.addLayout(self.layout1)
self.layout.addLayout(self.layout2)
self.layout.addLayout(self.layout3)
if __name__ == "__main__":
proxy = Proxy('tcp://127.0.0.1:8000')
app = QtGui.QApplication(sys.argv)
main = ControlForm(proxy)
main.show()
if sys.platform.startswith('darwin'):
main.raise_()
sys.exit(app.exec_())
# -*- coding: utf-8 -*-
import sys
import time
if sys.version_info < (3, 0):
input = raw_input
from pizco import Proxy
proxy = Proxy('tcp://127.0.0.1:8000')
while True:
input('Press enter to run ...\n')
fut = proxy.change_roof()
print('I am doing something while changing the roof')
print('The door is open?: {}'.format(proxy.door_open))
print('The lights are on?: {}'.format(proxy.lights_on))
print('I have finished doing this and now I will wait for the result')
print(fut.result())
# -*- coding: utf-8 -*-
from __future__ import print_function
import sys
import time
if sys.version_info < (3,0):
input = raw_input
from pizco import Proxy
proxy = Proxy('tcp://127.0.0.1:8000')
while True:
try:
proxy.paint(input('New color for the house: '))
except ValueError as ex:
print('Oops!')
print(ex)
# -*- coding: utf-8 -*-
import sys
import time
if sys.version_info < (3,0):
input = raw_input
from pizco import Proxy
proxy = Proxy('tcp://127.0.0.1:8000')
while True:
input('Press enter to run ...\n')
fut = proxy.change_roof()
print('I am doing something while changing the roof')
print('The door is open?: {}'.format(proxy.door_open))
print('The lights are on?: {}'.format(proxy.lights_on))
print('I have finished doing this and now I will wait for the result')
print(fut.result())
def test_server_no_inspect(self):
s = NoInspectServer(Example())
proxy = Proxy(s.rep_endpoint)
self.assertEqual(s.served_object.simple_attribute, 12)
self.assertEqual(proxy.simple_attribute, 12)
proxy.simple_attribute = 24
self.assertEqual(s.served_object.simple_attribute, 24)
self.assertEqual(proxy.simple_attribute, 24)
self.assertRaises(AttributeError, getattr, proxy, 'not_an_attribute')
self.assertEqual(s.served_object.dict_attribute[1], 2)
self.assertEqual(proxy.dict_attribute[1], 2)
self.assertRaises(KeyError, operator.getitem, proxy.dict_attribute, 2)
print(proxy.dict_attribute)
proxy.dict_attribute[2] = 4
self.assertEqual(s.served_object.dict_attribute[2], 4)
self.assertEqual(proxy.dict_attribute[2], 4)
self.assertEqual(s.served_object.rw_prop, 42)
self.assertEqual(proxy.rw_prop, 42)
proxy.rw_prop = 21
self.assertEqual(s.served_object.rw_prop, 21)
self.assertEqual(proxy.rw_prop, 21)
self.assertEqual(proxy.fun_simple(), 46)
self.assertEqual(proxy.fun_arg1(2), 4)
self.assertEqual(proxy.fun_arg2(2, 3), 8)
self.assertEqual(proxy.fun_arg2(y=2), 4)
self.assertRaises(ValueError, proxy.fun_raise)
proxy._proxy_stop_server()
proxy._proxy_stop_me()
class ScoreboardWrapper():
"""
Wraps the Scoreboard into two types of remotely accessible components:
* Server: wraps a real in-memory instance of the Scoreboard and exposes it via a ZMQ socket.
* Client: connects to a server, redirects attribute request to it, and collect the response.
Using this kind of wrapper we can easily implement a multiprocess scoreboardwrapper in which multiple clients,
each one in a different process, listen to a the HTTP RESTful API that the clients will use to access the
scoreboard. Besides the multiple clients (as many as allowed by the underlying hardware) there will be a single
server instance, that receives clients communications and keeps the unique and shared Scoreboard instance.
This is the typical type of situation in which we have to make a commitment in terms of latency. The scalability
of the final architecture is hindered by the need of keeping the Scoreboard Data Consistency (SDC). The more
scalable the architecture the harder to keep the SDC. The proposed architecture is the straighforward initial
approach, that gets rid of the SDC problem by having a single and centralized server that manages the shared
scoreboard data. Obviously, this makes the server the critical piece of the architecture in terms of latency,
since all requests must be attended by the same process.
If the language allows simultaneous execution of multiple threads (Python does not due to the GIL, or Global
Interpreter Lock), the next straightforward step towards increasing the scalability would be to make the server
thread-safe and allow multiple read threads plus a single write thread, to simultaneously run on the same
server. This is implemented using mutual exclusion (mutex) tools (e.g. semaphores) to protect the critical
section (i.e., the shared data). Read operations, such as Top100, Top200, Top500, and At100/3, can be done
simultaneously (one per thread). There will be as many simultaneous threads as allowed by the underlying
hardware. However, whenever a user reported a new score, regardless of it was either absolute or relative, it
must be assured that no other thread is within the critical section (unless we follow a policy that does not
need to report 100% accurate scores on each read).
The next step towards increasing the scalability of our architecture, that is to reduce the latency, would need
some kind of partition of the single server to allow accessing it in less time. That kind of vertical
scalability solution would require adding some kind of scheduling mechanism to properly orchestrate all the
server partitions to keep SDC yielding better performance, in terms of lower latencies.
In order not to end up with a overdesign situation, the steps towards the scalability of our architecture should
be guided by a performance analysis, measuring the latency of one step before taking the decision to go for the
next one. Take into account that the more steps taken the more complexity in our final design, and therefore
the less maintainable.
"""
def __init__(self, port=DEFAULT_PORT, ip=DEFAULT_IP, elogger=logger):
self.ip = ip
self.port = port
self.logger = elogger
self.mode = None
self.instance = None # Server or Proxy, according to mode
self.scoreboard = None # The in-memory scoreboard (only in the server)
def is_valid_info(self, client_info):
"""
True if the format of the specified client info is as expected.
:param client_info: (dict) The client info to be checked. The Valid format is:
{"user": <client_id>, ("total": <total_score>|"score": <relative_score>)}
where:
<client_id> : (int) Id of the client.
<total_score>: (int) Absolute score of the client.
<relative_score>: (int) Relative score of the client.
Examples:
{"user": 123, "total": 250}
{"user": 456, "score": "+10"}
{"user": 789, "score": "-20"}
:return:
"""
result = False
try:
if isinstance(client_info,
dict) and len(client_info) == 2 and isinstance(
client_info["user"], int):
try:
if isinstance(client_info["total"], int):
result = True
except KeyError:
# There is no "total" (absolute). Try with "score" (relative)
if isinstance(client_info["score"],
str) and (client_info["score"][0] == '+'
or client_info["score"][0] == '-'):
value = int(client_info["score"][1:])
if value >= 0:
result = True
except (TypeError, KeyError, ValueError):
pass
return result
def start(self, mode):
"""
In CLIENT_MODE:
Establish a connection with the server before sending a new message.
In SERVER_MODE:
Start listening to the incoming messages from the clients.
:param mode: (int) Either CLIENT_MODE or SERVER_MODE.
:return: None
"""
address = 'tcp://{}:{}'.format(self.ip, self.port)
if mode == CLIENT_MODE:
self.mode = mode
self.logger.info(
"Starting Scoreboard Client listening on {}:{} ...".format(
self.ip, self.port))
self.instance = Proxy(address)
elif mode == SERVER_MODE:
self.mode = mode
self.scoreboard = Scoreboard()
server = Server(self, address)
self.logger.info(
"Starting Scoreboard Server listening on {}:{} ...".format(
self.ip, self.port))
server.serve_forever()
def reset(self):
"""
Resets the info in the server.
:return: None
"""
if self.mode == SERVER_MODE:
self.scoreboard.reset()
self.logger.debug("Server Scoreboard reset")
elif self.mode == CLIENT_MODE:
self.instance.reset()
self.logger.debug("Client Scoreboard reset (sent to server)")
def update(self, client_info):
"""
In CLIENT_MODE:
Sends updated client score to the shared Scoreboard.
In SERVER_MODE:
Updates the client score and sends back the updated client score to the client.
:param client_info: (dict) A JSON submitted by the client, as specified in the Code Challenge:
Examples:
{"user": 123, "total": 250}
{"user": 456, "score": "+10"}
{"user": 789, "score": "-20"}
:return: (dict) The updated client score.
"""
if self.mode == SERVER_MODE:
if isinstance(client_info, str):
client_info = loads(client_info)
self.scoreboard.update(client_info)
client = self.scoreboard.get(client_info["user"])
result = {"user": client.id, "total": client.score}
self.logger.debug("Server Scoreboard updated : {}".format(result))
result = dumps(result)
elif self.mode == CLIENT_MODE and self.is_valid_info(client_info):
result = self.instance.update(dumps(client_info))
self.logger.debug(
"Client Scoreboard obtained update response from server : {}".
format(result))
result = loads(result)
else:
result = {"error": "Invalid client info"}
return result
def top(self, top_size):
"""
Asks the shared Scoreboard for the clients that occupy the specified number of top ranking positions
(i.e., those with the higher score values), according to the absolute ranking.
:param top_size: (int) Number of higher ranking positions to retrieve.
:return: (list of dict) The clients that occupies the specified ranking positions.
"""
if self.mode == SERVER_MODE:
result = self.scoreboard.top(int(top_size))
# Serialize to stringified JSON to be sent to the client
# From list of tuples to list of dicts
serial_result = []
for client in result:
serial_result.append(client.to_json())
self.logger.debug("Server Scoreboard top ({}) : {}".format(
top_size, result))
result = dumps(serial_result)
elif self.mode == CLIENT_MODE and isinstance(top_size, int):
result = self.instance.top(str(top_size))
self.logger.debug("Client Scoreboard top ({}) : {}".format(
top_size, result))
result = loads(result)
else:
result = {"error": "Invalid top size"}
return result
def relative_top(self, ranking_position, scope_size):
"""
Asks the shared Scoreboard for the relative top (see Scoreboard.relative_top)
:param ranking_position: (int) Ranking position to retrieve scope around. Must be a positive value, from 1 to N.
:param scope_size: (int) Scope size (see explanation above). Must be a positive value.
:return: (list of dict) The clients that occupies the specified ranking positions.
"""
if self.mode == SERVER_MODE:
result = self.scoreboard.relative_top(int(ranking_position),
int(scope_size))
# Serialize to stringified JSON to be sent to the client
# From list of tuples to list of dicts
serial_result = []
for client in result:
serial_result.append(client.to_json())
self.logger.debug(
"Server Scoreboard relative top ({}, {}) : {}".format(
ranking_position, scope_size, result))
result = dumps(serial_result)
elif self.mode == CLIENT_MODE and isinstance(
ranking_position, int) and isinstance(scope_size, int):
result = self.instance.relative_top(str(ranking_position),
str(scope_size))
self.logger.debug(
"Client Scoreboard relative top ({}, {}) : {}".format(
ranking_position, scope_size, result))
result = loads(result)
else:
result = {"error": "Invalid ranking_position, scope_size values"}
return result
def test_signal_two_proxies(self):
address = 'tcp://127.0.0.1:6009'
s = Server(Example(), rep_endpoint=address)
proxy1 = Proxy(address)
proxy2 = Proxy(address)
class MemMethod(object):
def __init__(self_):
self_.called = 0
def __call__(self_, value, old_value, others):
self_.called += 1
fun = MemMethod()
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 0)
proxy2.rw_prop_changed.connect(fun)
time.sleep(SLEEP_SECS)
self.assertEqual(len(s.served_object.rw_prop_changed.slots), 1)
proxy1.p = 28
proxy1._proxy_stop_server()
proxy1._proxy_stop_me()
proxy2._proxy_stop_me()
import os
import os.path as op
from pizco import Proxy
os.chdir(os.environ['MMVT_CODE'])
from src.mmvt_addon.scripts import run_mmvt
from src.utils import utils
subject, atlas = 'DC', 'laus250'
run_mmvt.run(subject, atlas, run_in_background=False, debug=False)
mmvt = Proxy('tcp://127.0.0.1:8001')
meg_dir = utils.get_link_dir(utils.get_links_dir(), 'meg')
stc_fname = op.join(meg_dir, subject, 'left-MNE-1-15-lh.stc')
mmvt.plot_stc(stc_fname, 1270, threshold=0.1)
def test_server_no_inspect(self):
s = NoInspectServer(Example())
proxy = Proxy(s.rep_endpoint)
self.assertEqual(s.served_object.simple_attribute, 12)
self.assertEqual(proxy.simple_attribute, 12)
proxy.simple_attribute = 24
self.assertEqual(s.served_object.simple_attribute, 24)
self.assertEqual(proxy.simple_attribute, 24)
self.assertRaises(AttributeError, getattr, proxy, 'not_an_attribute')
self.assertEqual(s.served_object.dict_attribute[1], 2)
self.assertEqual(proxy.dict_attribute[1], 2)
self.assertRaises(KeyError, operator.getitem, proxy.dict_attribute, 2)
print(proxy.dict_attribute)
proxy.dict_attribute[2] = 4
self.assertEqual(s.served_object.dict_attribute[2], 4)
self.assertEqual(proxy.dict_attribute[2], 4)
self.assertEqual(s.served_object.rw_prop, 42)
self.assertEqual(proxy.rw_prop, 42)
proxy.rw_prop = 21
self.assertEqual(s.served_object.rw_prop, 21)
self.assertEqual(proxy.rw_prop, 21)
self.assertEqual(proxy.fun_simple(), 46)
self.assertEqual(proxy.fun_arg1(2), 4)
self.assertEqual(proxy.fun_arg2(2, 3), 8)
self.assertEqual(proxy.fun_arg2(y=2), 4)
self.assertRaises(ValueError, proxy.fun_raise)
proxy._proxy_stop_server()
proxy._proxy_stop_me()
# -*- coding: utf-8 -*-
from __future__ import print_function
import sys
import time
if sys.version_info < (3, 0):
input = raw_input
from pizco import Proxy
proxy = Proxy('tcp://127.0.0.1:8000')
while True:
try:
proxy.paint(input('New color for the house: '))
except ValueError as ex:
print('Oops!')
print(ex)
