def send_to_butler(self, command_line, widget_proxy, live):
you = u"You > {0}".format(command_line)
self.parent.value.set_values(self.parent.value.get() + [you])
self.parent.wMain.values = self.parent.value.get()
self.parent.wMain.display()
# handle special commands
if command_line.lower() == "quit":
IOLoop.instance().stop()
sys.exit(0) # why is this needed ?
elif command_line.lower() == "reload":
try:
cli = MQSyncReq(zmq.Context())
msg = MQMessage()
msg.set_action('butler.reload.do')
result = cli.request('butler', msg.get(), timeout=10).get()
if result:
msg = "*** action reload : {0}".format(result)
self.parent.value.set_values(self.parent.value.get() + [msg])
except:
msg = u"*** action reload : error (is butler component ok ?)"
self.parent.value.set_values(self.parent.value.get() + [msg])
# handle butler
else:
self.parent.butler_cb(command_line, identity = "cli user", media = "chat", location = None, mood = None)
self.parent.wMain.values = self.parent.value.get()
self.parent.wMain.display()
def ready(self, ioloopstart=1):
""" to call at the end of the __init__ of classes that inherits of XplPlugin
"""
if self.dont_run_ready == True:
return
### activate xpl hbeat
if self.enable_hbeat_called == True:
self.log.error(u"in ready() : enable_hbeat() function already called : the plugin may not be fully converted to the 0.4+ Domogik format")
else:
self.enable_hbeat()
# send the status for the xpl hbeat
self.myxpl.update_status(2)
### send plugin status : STATUS_ALIVE
# TODO : why the dbmgr has no self._name defined ???????
# temporary set as unknown to avoir blocking bugs
if not hasattr(self, '_name'):
self._name = "unknown"
self._set_status(STATUS_ALIVE)
### Instantiate the MQ
# nothing can be launched after this line (blocking call!!!!)
self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!")
if ioloopstart == 1:
IOLoop.instance().start()
def _connect(self):
context = zmq.Context()
url = "{0}:{1}".format(BinderDSettings.BROKER_HOST,
BinderDSettings.BROKER_PORT)
worker = BinderDModule.Worker(self, context, url, bytes(self.TAG))
IOLoop.instance().start()
worker.shutdown()
def _setup_pipe_in(self):
"""setup listening pipe for subprocesses"""
ctx = self.pub_socket.context
# use UUID to authenticate pipe messages
self._pipe_uuid = uuid.uuid4().bytes
self._pipe_in = ctx.socket(zmq.PULL)
self._pipe_in.linger = 0
try:
self._pipe_port = self._pipe_in.bind_to_random_port(
"tcp://127.0.0.1")
except zmq.ZMQError as e:
warn("Couldn't bind IOStream to 127.0.0.1: %s" % e +
"\nsubprocess output will be unavailable.")
self._pipe_flag = False
self._pipe_in.close()
del self._pipe_in
return
self._pipe_poller = zmq.Poller()
self._pipe_poller.register(self._pipe_in, zmq.POLLIN)
if IOLoop.initialized():
# subprocess flush should trigger flush
# if kernel is idle
IOLoop.instance().add_handler(
self._pipe_in,
lambda s, event: self.flush(),
IOLoop.READ,
)
def main():
"""main method"""
url_worker = "ipc://backend.ipc"
url_client = "ipc://frontend.ipc"
# Prepare our context and sockets
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind(url_client)
backend = context.socket(zmq.ROUTER)
backend.bind(url_worker)
# create workers and clients threads
for i in range(NBR_WORKERS):
thread = threading.Thread(target=worker_thread, args=(url_worker, i, ))
thread.daemon = True
thread.start()
for i in range(NBR_CLIENTS):
thread_c = threading.Thread(target=client_thread, args=(url_client, i, ))
thread_c.daemon = True
thread_c.start()
# create queue with the sockets
queue = LRUQueue(backend, frontend)
# start reactor
IOLoop.instance().start()
def main():
http_fetcher = fetcher.HTTPFetcher()
rb = robot.Robot(http_fetcher, [
('/.*', AHandler),
])
worker.Worker(rb).start()
IOLoop.instance().start()
def main(basedir, n_workers, daemon_flag,
client_socket, worker_socket,
worker_bin, deployment, notify_socket,
journal_socket):
"""main method"""
# create queue with the sockets
signal.signal(signal.SIGCHLD, signal_handler)
LRUQueue(worker_socket, client_socket,
basedir, daemon_flag, worker_bin,
deployment)
worker_env = {
'WORKER_SOCKET': worker_socket,
'LAF_DEPLOYMENT': deployment
}
if notify_socket:
os.environ['NOTIFICATION_SOCK'] = notify_socket
worker_env['NOTIFICATION_SOCK'] = notify_socket
if journal_socket:
os.environ['JOURNAL_SOCK'] = journal_socket
worker_env['JOURNAL_SOCK'] = journal_socket
for _ in range(int(n_workers)):
subprocess.Popen([worker_bin, basedir],
env=worker_env,
shell=False)
# start reactor
IOLoop.instance().start()
def device_manager(context):
global manager
global publisher
worker_address = Config.get('services', 'worker_address')
worker_port = Config.get('services', 'worker_port')
worker_address = "tcp://" + worker_address + ":" + worker_port
publisher_address = Config.get('device_service', 'publisher_address')
publisher_port = Config.get('device_service', 'publisher_port')
publisher_address = "tcp://" + publisher_address + ":" + publisher_port
service = Config.get('device_service', 'service_name')
publisher = DeviceServicePublisher(context, publisher_address)
manager = DeviceServiceManager(context, worker_address, service)
try:
IOLoop.instance().start()
except KeyboardInterrupt:
IOLoop.instance().stop()
publisher.shutdown()
manager.shutdown()
return
def on_message(self, msgid, content):
""" Handle MQ messages
@param msgid : message id
@content : message content
"""
if msgid == "plugin.status":
# we may miss starting and stop-request events but we only want to do some checks on alive and stopped...
# and sometimes it happens that we still receive a last 'alive' status before the 'stop' one
if self.count == 0:
print(u"Message skipped (we skip the first one) : msgid={0}, content={1}".format(msgid, content))
self.count = 1
return
print(u"Message received : msgid={0}, content={1}".format(msgid, content))
if content['name'] == self.name and \
content['type'] == self.type and \
content['host'] == self.host:
self.plugin_status = content['event']
# plugin started
if content['event'] == STATUS_ALIVE:
print(u"Plugin is started")
print(u"Stop listening to MQ as we get our result")
IOLoop.instance().stop()
# plugin stopped
elif content['event'] == STATUS_STOPPED:
print(u"Plugin is stopped")
print(u"Stop listening to MQ as we get our result")
IOLoop.instance().stop()
def main():
"""main method"""
url_worker = "ipc://backend.ipc"
url_client = "ipc://frontend.ipc"
# Prepare our context and sockets
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind(url_client)
backend = context.socket(zmq.ROUTER)
backend.bind(url_worker)
# create workers and clients threads
for i in range(NBR_WORKERS):
thread = threading.Thread(target=worker_thread, args=(url_worker, i, ))
thread.daemon = True
thread.start()
for i in range(NBR_CLIENTS):
thread_c = threading.Thread(target=client_thread,
args=(url_client, i, ))
thread_c.daemon = True
thread_c.start()
# create queue with the sockets
queue = LRUQueue(backend, frontend)
# start reactor
IOLoop.instance().start()
def _setup_pipe_in(self):
"""setup listening pipe for subprocesses"""
ctx = self.pub_socket.context
# use UUID to authenticate pipe messages
self._pipe_uuid = uuid.uuid4().bytes
self._pipe_in = ctx.socket(zmq.PULL)
self._pipe_in.linger = 0
try:
self._pipe_port = self._pipe_in.bind_to_random_port("tcp://127.0.0.1")
except zmq.ZMQError as e:
warn("Couldn't bind IOStream to 127.0.0.1: %s" % e +
"\nsubprocess output will be unavailable."
)
self._pipe_flag = False
self._pipe_in.close()
del self._pipe_in
return
self._pipe_poller = zmq.Poller()
self._pipe_poller.register(self._pipe_in, zmq.POLLIN)
if IOLoop.initialized():
# subprocess flush should trigger flush
# if kernel is idle
IOLoop.instance().add_handler(self._pipe_in,
lambda s, event: self.flush(),
IOLoop.READ,
)
def on_message(self, msgid, content):
""" Handle MQ messages
@param msgid : message id
@content : message content
"""
if msgid == "plugin.status":
# we may miss starting and stop-request events but we only want to do some checks on alive and stopped...
# and sometimes it happens that we still receive a last 'alive' status before the 'stop' one
if self.count == 0:
print(
u"Message skipped (we skip the first one) : msgid={0}, content={1}"
.format(msgid, content))
self.count = 1
return
print(u"Message received : msgid={0}, content={1}".format(
msgid, content))
if content['name'] == self.name and \
content['type'] == self.type and \
content['host'] == self.host:
self.plugin_status = content['event']
# plugin started
if content['event'] == STATUS_ALIVE:
print(u"Plugin is started")
print(u"Stop listening to MQ as we get our result")
IOLoop.instance().stop()
# plugin stopped
elif content['event'] == STATUS_STOPPED:
print(u"Plugin is stopped")
print(u"Stop listening to MQ as we get our result")
IOLoop.instance().stop()
def on_message(self, msg):
if _do_print:
print 'client received:',
pprint(msg)
self.last_msg = msg
IOLoop.instance().stop()
return
def _publish_msg(self,
msg_type,
data=None,
metadata=None,
buffers=None,
**keys):
"""Helper for sending a comm message on IOPub"""
if threading.current_thread(
).name != 'MainThread' and IOLoop.initialized():
# make sure we never send on a zmq socket outside the main IOLoop thread
IOLoop.instance().add_callback(lambda: self._publish_msg(
msg_type, data, metadata, buffers, **keys))
return
data = {} if data is None else data
metadata = {} if metadata is None else metadata
content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
self.session.send(
self.iopub_socket,
msg_type,
content,
metadata=json_clean(metadata),
parent=self.kernel._parent_header,
ident=self.topic,
buffers=buffers,
)
def ready(self, ioloopstart=1):
""" to call at the end of the __init__ of classes that inherits of XplPlugin
"""
if self.dont_run_ready == True:
return
### activate xpl hbeat
if self.enable_hbeat_called == True:
self.log.error(u"in ready() : enable_hbeat() function already called : the plugin may not be fully converted to the 0.4+ Domogik format")
else:
self.enable_hbeat()
# send the status for the xpl hbeat
self.myxpl.update_status(2)
### send plugin status : STATUS_ALIVE
# TODO : why the dbmgr has no self._name defined ???????
# temporary set as unknown to avoir blocking bugs
if not hasattr(self, '_name'):
self._name = "unknown"
self._set_status(STATUS_ALIVE)
### Instantiate the MQ
# nothing can be launched after this line (blocking call!!!!)
self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!")
if ioloopstart == 1:
IOLoop.instance().start()
def main():
http_fetcher = fetcher.HTTPFetcher()
rb = robot.Robot(http_fetcher, [
('/.*', AHandler),
])
worker.Worker(rb).start()
IOLoop.instance().start()
def main(pat):
fname = find_connection_file(pat)
with open(fname) as f:
cfg = json.load(f)
url = "%s://%s:%s" % (cfg.get('transport', 'tcp'), cfg['ip'], cfg['iopub_port'])
session = Session(key=cfg['key'])
ctx = zmq.Context.instance()
sub = ctx.socket(zmq.SUB)
sub.subscribe = b''
sub.connect(url)
# import IPython
# IPython.embed()
# return
stream = ZMQStream(sub)
stream.on_recv(lambda msg_list: log_msg(session, msg_list))
pc = PeriodicCallback(print_time, 5 * 60 * 1000)
pc.start()
IOLoop.instance().start()
def on_message(self, msg):
if _do_print:
print 'client received:',
pprint(msg)
self.last_msg = msg
IOLoop.instance().stop()
return
def __init__(self, socket, pipe=False):
# type: (object, object) -> object
"""Create IOPub thread
Parameters
----------
socket: zmq.PUB Socket
the socket on which messages will be sent.
pipe: bool
Whether this process should listen for IOPub messages
piped from subprocesses.
"""
self.socket = socket
self.background_socket = BackgroundSocket(self)
self._master_pid = os.getpid()
self._pipe_flag = pipe
self.io_loop = IOLoop()
if pipe:
self._setup_pipe_in()
self._local = threading.local()
self._events = {}
self._setup_event_pipe()
self.thread = threading.Thread(target=self._thread_main)
self.thread.daemon = True
def subscriber_manager(context):
global subscriber
global publisher
global subscriber_topics
session = Session()
# Set up forwarder device
logger.debug("Configuring forwarder device")
forwarder_subscriber_address = session.query(RoutingDefiniton.route) \
.filter_by(name="internal_pub") \
.scalar()
forwarder_publisher_address = session.query(RoutingDefiniton.route) \
.filter_by(name="internal_sub") \
.scalar()
forwarder = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
forwarder.connect_in(forwarder_subscriber_address)
forwarder.setsockopt_in(zmq.SUBSCRIBE, "")
logger.debug("forwarder connect in: {0}".format(forwarder_subscriber_address))
forwarder.bind_out(forwarder_publisher_address)
logger.debug("forwarder bind out: {0}".format(forwarder_publisher_address))
# Set up subscriber listening to Farm-Monitor server
subscriber_address = session.query(RoutingDefiniton.route)\
.filter_by(name="subscriber_address").scalar()
subscriber_port = session.query(RoutingDefiniton.route)\
.filter_by(name="subscriber_port").scalar()
subscriber_address = "tcp://" + subscriber_address + ":" + subscriber_port
device_id = session.query(HardwareDefinition.serial_number).scalar()
subscriber_topics.append(device_id)
subscriber = Subscriber(context, subscriber_address, subscriber_topics)
subscriber.subscriber.on_recv(callback=server_subscriber_recv)
logger.debug("Subscriber listening to: {0} topics: {1}".format(subscriber_address,
subscriber_topics))
# Set up publisher to forward messages from Farm-Monitor to internal
publisher_address = session.query(RoutingDefiniton.route)\
.filter_by(name="internal_pub").scalar()
publisher = Publisher(context, publisher_address)
logger.debug("Publisher configured to: {0}".format(publisher_address))
session.close()
try:
logger.info("starting forwarder, Subscriber, and Publisher")
# Start the forwarder
forwarder.start()
IOLoop.instance().start()
except KeyboardInterrupt:
logger.info("stopping forwarder, Subscriber, and Publisher")
IOLoop.instance().stop()
subscriber.shutdown()
publisher.shutdown()
return
def test_03_timeout_01(self):
"""Test MDPclient request w/ timeout.
"""
client = MyClient(self.context, self.endpoint, self.service)
client.request(b'XXX', 20) # 20 millisecs timeout
IOLoop.instance().start()
client.shutdown()
self.assertEquals(client.timed_out, True)
return
def test_03_timeout_01(self):
"""Test MNClient request w/ timeout.
"""
client = MyClient(self.context, self.endpoint, self.service)
client.request(b'XXX', 20) # 20 millisecs timeout
IOLoop.instance().start()
client.shutdown()
self.assertEqual(client.timed_out, True)
return
class AsyncServerAdapter(object):
producer_class = Producer
def __init__(self, backend_rep_uri, frontend_rep_uri, frontend_pub_uri,
control_pipe=None):
self.uris = OrderedDict([
('backend_rep', backend_rep_uri),
('consumer_push_be', unique_ipc_uri()),
('consumer_pull_be', unique_ipc_uri()),
('frontend_rep_uri', frontend_rep_uri),
('frontend_pub_uri', frontend_pub_uri)
])
self.control_pipe = control_pipe
self.done = False
logging.getLogger(log_label(self)).info("uris: %s", self.uris)
def watchdog(self):
if self.control_pipe is None:
return
elif not self.done and self.control_pipe.poll():
self.done = True
self.finish()
def run(self):
consumer = Process(target=Consumer(self.uris['backend_rep'],
self.uris['consumer_push_be'],
self.uris['consumer_pull_be']).run
)
producer = Process(target=self.producer_class(
self.uris['frontend_rep_uri'],
self.uris['frontend_pub_uri'],
self.uris['consumer_pull_be'],
self.uris['consumer_push_be']).run
)
self.io_loop = IOLoop()
periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop)
periodic_callback.start()
try:
consumer.start()
producer.start()
self.io_loop.start()
except KeyboardInterrupt:
pass
producer.terminate()
consumer.terminate()
logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have '
'been terminated')
def __del__(self):
uris = [self.uris[label] for label in ('consumer_push_be',
'consumer_pull_be', )]
cleanup_ipc_uris(uris)
def finish(self):
logging.getLogger(log_label(self)).debug('"finish" request received')
self.io_loop.stop()
def _schedule_flush(self):
"""schedule a flush in the main thread
only works with a tornado/pyzmq eventloop running
"""
if IOLoop.initialized():
IOLoop.instance().add_callback(self.flush)
else:
# no async loop, at least force the timer
self._start = 0
def main(start_script_file, running_class_def=None):
'''主函数，传入开始执行的主函数，然后以指定的类作为起始工作实例，为单实例机器的程序入口'''
current_environment_pack = get_current_environment_pack(
start_script_absolute_filename=start_script_file)
if running_class_def:
current_working_instance = running_class_def(current_environment_pack)
#print id(current_working_engine),'这里是主进程'
IOLoop.instance().start()
pass
def __init__(self, socket, pipe=False):
self.socket = socket
self.background_socket = BackgroundSocket(self)
self._master_pid = os.getpid()
self._pipe_flag = pipe
self.io_loop = IOLoop()
if pipe:
self._setup_pipe_in()
self.thread = threading.Thread(target=self._thread_main)
self.thread.daemon = True
def _schedule_flush(self):
"""schedule a flush in the main thread
only works with a tornado/pyzmq eventloop running
"""
if IOLoop.initialized():
IOLoop.instance().add_callback(self.flush)
else:
# no async loop, at least force the timer
self._start = 0
def _on_msg(self, msg):
self._msgs.append(msg)
if _do_print:
print('broker received:', msg)
if self.broker.do_reply:
new_msg = msg[:4]
new_msg.append(b'REPLY')
self.broker.send_multipart(new_msg)
else:
IOLoop.instance().stop()
return
def main():
from frontier import Frontier
from options import parse_command_line
parse_command_line()
ft = Frontier([
('http://localhost/', 1),
])
Master(ft).start()
IOLoop.instance().start()
def run(address, optional_address=None):
context = zmq.Context()
broker = DedalusRunner(context, address, optional_address)
try:
IOLoop.instance().start()
except KeyboardInterrupt:
_LOG.info("Interrupt received, stopping.")
finally:
# clean up
broker.shutdown()
context.term()
def main():
from frontier import Frontier
from options import parse_command_line
parse_command_line()
ft = Frontier([
('http://localhost/', 1),
])
Master(ft).start()
IOLoop.instance().start()
def test_01_simple_01(self):
"""Test MDPWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = MyWorker(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 1000)
sender.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def _on_msg(self, msg):
self._msgs.append(msg)
if _do_print:
print 'broker received:',
pprint(msg)
if self.broker.do_reply:
new_msg = msg[:4]
new_msg.append(b'REPLY')
self.broker.send_multipart(new_msg)
else:
IOLoop.instance().stop()
return
def run(address):
context = zmq.Context()
worker = WorkerRunner(context, address, WK_SERVICE)
try:
IOLoop.instance().start()
worker.shutdown()
except KeyboardInterrupt:
_LOG.info("Interrupt received, stopping!")
finally:
# clean up
worker.shutdown()
context.term()
def _send_fireball(self, mtype, data, f_size):
if len(data) < 3:
logger.error('no data to send')
return []
self.loop = IOLoop().instance()
self.socket.close()
self.socket = self.context.socket(zmq.DEALER)
self.socket.connect(self.remote)
self.stream = ZMQStream(self.socket)
self.stream.on_recv(self._handle_message_fireball)
self.stream.io_loop.call_later(SNDTIMEO, self._fireball_timeout)
self.response = []
if PYVERSION == 3:
if isinstance(data, bytes):
data = data.decode('utf-8')
data = json.loads(data)
if not isinstance(data, list):
data = [data]
if (len(data) % f_size) == 0:
self.num_responses = int((len(data) / f_size))
else:
self.num_responses = int((len(data) / f_size)) + 1
logger.debug('responses expected: %i' % self.num_responses)
batch = []
for d in data:
batch.append(d)
if len(batch) == f_size:
Msg(mtype=Msg.INDICATORS_CREATE, token=self.token, data=batch).send(self.socket)
batch = []
if len(batch):
Msg(mtype=Msg.INDICATORS_CREATE, token=self.token, data=batch).send(self.socket)
logger.debug("starting loop to receive")
self.loop.start()
# clean up FDs
self.loop.close()
self.stream.close()
self.socket.close()
return self.response
def main():
context = zmq.Context(1)
#qq = q.TheQueue(context, CLIENT_URL, WORKER_URL)
qq = q.TheQueue(context, CLIENT_URL, WORKER_URL)
for i in range(2):
t = threading.Thread(target=w.TheWorker,args=(context,))
t.start()
#t.join()
try:
IOLoop.instance().start()
except KeyboardInterrupt:
times_str('ctrlc')
def main():
cfg = Loader('mq')
my_conf = cfg.load()
config = dict(my_conf[1])
ctx = DaemonContext()
ctx.open()
context = zmq.Context()
print(("tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])))
broker = MDPBroker(context, "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']))
IOLoop.instance().start()
broker.shutdown()
def run(address):
context = zmq.Context()
client = ClientRunner(context, address, SERVICE_ECHO)
try:
IOLoop.instance().start()
client.shutdown()
except KeyboardInterrupt:
_LOG.info("Interrupt received, stopping!")
finally:
# clean up
client.shutdown()
context.term()
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug("Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (
nb_test, DATABASE_CONNECTION_NUM_TRY)
print(msg)
self.log.error(msg)
msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
print(msg)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
print(msg)
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
print(msg)
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error("Error while starting database engine : %s" %
traceback.format_exc())
self.force_leave()
return
Listener(self._request_config_cb, self.myxpl, {
'schema': 'domogik.config',
'xpltype': 'xpl-cmnd'
})
self.enable_hbeat()
IOLoop.instance().start()
def main():
ft = frontier.Frontier([
('http://m.sohu.com/', 1),
])
http_fetcher = fetcher.HTTPFetcher()
rb = robot.Robot(http_fetcher, [
('/.*', AHandler),
])
id = fork_processes(0)
if id == master_id():
master.Master(ft).start()
else:
worker.Worker(rb).start()
IOLoop.instance().start()
def test_01_simple_01(self):
"""Test MDPWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = MyWorker(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 500)
stopper = DelayedCallback(self.stop_test, 2500)
sender.start()
stopper.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def test_04_receive_01(self):
"""Test MNClient message receive.
"""
self._start_broker(do_reply=True)
client = MyClient(self.context, self.endpoint, self.service)
client.request(b'XXX')
IOLoop.instance().start()
client.shutdown()
self._stop_broker()
self.assertEqual(True, hasattr(client, 'last_msg'))
self.assertEqual(3, len(client.last_msg))
self.assertEqual(b'REPLY', client.last_msg[-1])
self.assertEqual(self.service, client.last_msg[-2])
return
def test_04_receive_01(self):
"""Test MDPclient message receive.
"""
self._start_broker(do_reply=True)
client = MyClient(self.context, self.endpoint, self.service)
client.request(b'XXX')
IOLoop.instance().start()
client.shutdown()
self._stop_broker()
self.assertEquals(True, hasattr(client, b'last_msg'))
self.assertEquals(3, len(client.last_msg))
self.assertEquals(b'REPLY', client.last_msg[-1])
self.assertEquals(self.service, client.last_msg[-2])
return
def main():
ft = frontier.Frontier([
('http://m.sohu.com/', 1),
])
http_fetcher = fetcher.HTTPFetcher()
rb = robot.Robot(http_fetcher, [
('/.*', AHandler),
])
id = fork_processes(0)
if id == master_id():
master.Master(ft).start()
else:
worker.Worker(rb).start()
IOLoop.instance().start()
def main():
cfg = Loader('mq')
my_conf = cfg.load()
config = dict(my_conf[1])
ctx = DaemonContext()
ctx.open()
context = zmq.Context()
print(("tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])))
broker = MDPBroker(
context, "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']))
IOLoop.instance().start()
broker.shutdown()
def test_simple_worker(self):
"""Test MNWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = WorkerRunner(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 500)
stopper = DelayedCallback(self.stop_loop, 2500)
sender.start()
stopper.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def _send_fireball(self, mtype, data):
if len(data) < 3:
logger.error('no data to send')
return []
self.loop = IOLoop()
self.socket = self.context.socket(zmq.DEALER)
self.socket.connect(self.remote)
timeout = time.time() + SNDTIMEO
self.loop.add_timeout(timeout, self._fireball_timeout)
self.response = []
self.loop.add_handler(self.socket, self._handle_message_fireball,
zmq.POLLIN)
if PYVERSION == 3:
if isinstance(data, bytes):
data = data.decode('utf-8')
data = json.loads(data)
if not isinstance(data, list):
data = [data]
if (len(data) % FIREBALL_SIZE) == 0:
self.num_responses = int((len(data) / FIREBALL_SIZE))
else:
self.num_responses = int((len(data) / FIREBALL_SIZE)) + 1
logger.debug('responses expected: %i' % self.num_responses)
batch = []
for d in data:
batch.append(d)
if len(batch) == FIREBALL_SIZE:
Msg(mtype=Msg.INDICATORS_CREATE, token=self.token,
data=batch).send(self.socket)
batch = []
if len(batch):
Msg(mtype=Msg.INDICATORS_CREATE, token=self.token,
data=batch).send(self.socket)
logger.debug("starting loop to receive")
self.loop.start()
self.loop.close()
self.socket.close()
return self.response
def _on_msg(self, msg):
if _do_print:
print 'broker received:',
pprint(msg)
self.target = msg.pop(0)
if msg[1] == chr(1): # ready
print 'READY'
self.target = msg[0]
return
if msg[1] == chr(4): # ready
print 'HB'
return
if msg[1] == chr(3): # reply
IOLoop.instance().stop()
return
return
def __init__(self, socket, io_loop=None):
self.socket = socket
self.io_loop = io_loop or IOLoop.instance()
self.poller = zmq.Poller()
self._send_queue = Queue()
self._recv_callback = None
self._send_callback = None
self._close_callback = None
self._recv_copy = False
self._flushed = False
self._state = self.io_loop.ERROR
with stack_context.NullContext():
self.io_loop.add_handler(self.socket, self._handle_events,
self._state)
# shortcircuit some socket methods
self.bind = self.socket.bind
self.bind_to_random_port = self.socket.bind_to_random_port
self.connect = self.socket.connect
self.setsockopt = self.socket.setsockopt
self.getsockopt = self.socket.getsockopt
self.setsockopt_unicode = self.socket.setsockopt_unicode
self.getsockopt_unicode = self.socket.getsockopt_unicode
def stop(self):
self.terminated = True
loop = IOLoop.instance()
with loop._callback_lock:
for timeout in loop._timeouts:
timeout.callback = None
for dc_or_stream in event_mod.callbacks.values():
if hasattr(dc_or_stream, 'close'):
def close_stream_callback(stream):
stream.close()
loop.add_callback(close_stream_callback, dc_or_stream)
else:
dc_or_stream.stop()
event_mod.callbacks = {}
# if getattr(self, 'stream', None) is not None:
# self.stream.close()
with loop._callback_lock:
for timeout in loop._timeouts:
timeout.callback = None
loop.stop()
def __init__(self, socket, io_loop=None):
self.socket = socket
self.io_loop = io_loop or IOLoop.instance()
self.poller = zmq.Poller()
self._send_queue = Queue()
self._recv_callback = None
self._send_callback = None
self._close_callback = None
self._recv_copy = False
self._flushed = False
self._state = self.io_loop.ERROR
with stack_context.NullContext():
self.io_loop.add_handler(
self.socket, self._handle_events, self._state)
# shortcircuit some socket methods
self.bind = self.socket.bind
self.bind_to_random_port = self.socket.bind_to_random_port
self.connect = self.socket.connect
self.setsockopt = self.socket.setsockopt
self.getsockopt = self.socket.getsockopt
self.setsockopt_unicode = self.socket.setsockopt_unicode
self.getsockopt_unicode = self.socket.getsockopt_unicode
class UdpListener (CCDaemon):
""" UDP server to handle UDP stream. """
log = skytools.getLogger ('d:UdpListener')
def reload (self):
super(UdpListener, self).reload()
self.listen_host = self.cf.get ('listen-host')
self.listen_port = self.cf.getint ('listen-port')
self.stats_period = self.cf.getint ('stats-period', 30)
def startup (self):
super(UdpListener, self).startup()
# plugins should be ready before we start receiving udp stream
self.init_plugins()
self.listen_addr = (self.listen_host, self.listen_port)
self.sock = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setblocking (0)
try:
self.sock.bind (self.listen_addr)
except Exception, e:
self.log.exception ("failed to bind to %s - %s", self.listen_addr, e)
raise
self.ioloop = IOLoop.instance()
callback = functools.partial (self.handle_udp, self.sock)
self.ioloop.add_handler (self.sock.fileno(), callback, self.ioloop.READ)
self.timer_stats = PeriodicCallback (self.send_stats, self.stats_period * 1000, self.ioloop)
self.timer_stats.start()
def __init__(self):
#procControllerAddr = '165.227.24.226' # I am client to HostController
#procControllerPort = '5557'
hostControllerPort = '5556' # I server to device
print("Host Controller Starting\n")
self.context = zmq.Context() # get context
self.loop = IOLoop.instance()
# self.clientSetup = ClientSetup(context) # instantiate the ClientSetup object
self.serverSetup = ServerSetup(self.context) # instantiate the ServerSetup object
# set up separate server and client sockets
self.serverSocket = self.serverSetup.createServerSocket() # get a server socket
self.serverSetup.serverBind(hostControllerPort, self.serverSocket) # bind to an address
# self.clientSocket = self.clientSetup.createClientSocket() # get a client socket
# NOTE: setIdentity() MUST BE CALLED BEFORE clientConnect or the identity will
# not take effect
# self.clientSetup.setIdentity(MasterId().getDevId()) # get the device id
# self.clientSetup.clientConnect(hostControllerAddr, hostControllerPort, self.clientSocket) # connect to server using clientSocket
self.serverSocket = ZMQStream(self.serverSocket)
self.serverSocket.on_recv(self.onServerRecv)
self.messages = Messages() # instantiate a Messages object
self.inDict = {}
self.outDict = {}
def __init__(self, id, uuid, socket, addr, time_=None, beacon=None, **kwargs):
self.id = id
self.uuid = uuid
self.socket = socket
self.addr = addr
self.time = time_ or time.time()
self.transport, host = addr.split('://', 1)
self.host, self.beacon_router_port = host.rsplit(':', 1)
# Set callbacks
for k, v in kwargs.iteritems():
if k.startswith('on_') and k.endswith('_cb'):
setattr(self, k, v)
if not self.ctx:
self.ctx = zmq.Context.instance()
self.loop = IOLoop.instance()
# Set up our own dkv client interface to peer
self.subscriber = self.ctx.socket(zmq.SUB)
self.subscriber.setsockopt(zmq.SUBSCRIBE, b'')
self.subscriber.connect(self.publisher_endpoint)
# Wrap sockets in ZMQStreams for IOLoop handlers
self.subscriber = ZMQStream(self.subscriber)
self.subscriber.on_recv(self._on_subscriber_recv)
def __init__(self, frontier,
data_in_sock='ipc:///tmp/robot-data-w2m.sock',
data_out_sock='ipc:///tmp/robot-data-m2w.sock',
msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
io_loop=None):
self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())
context = zmq.Context()
self._io_loop = io_loop or IOLoop.instance()
self._in_socket = context.socket(zmq.SUB)
self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
self._in_socket.bind(data_in_sock)
self._in_stream = ZMQStream(self._in_socket, io_loop)
self._out_socket = context.socket(zmq.PUSH)
self._out_socket.bind(data_out_sock)
self._out_stream = ZMQStream(self._out_socket, io_loop)
self._online_workers = set()
self._running = False
self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)
self.frontier = frontier
self.messenger = ServerMessenger(msg_in_sock, msg_out_sock,
context, io_loop)
def __init__(self, port=5556):
self.port = port
self.ctx = zmq.Context()
self.kvmap = {}
self.loop = IOLoop.instance()
# Set up our clone server sockets
self.snapshot = self.ctx.socket(zmq.ROUTER)
self.publisher = self.ctx.socket(zmq.PUB)
self.collector = self.ctx.socket(zmq.PULL)
self.snapshot.bind("tcp://*:%d" % self.port)
self.publisher.bind("tcp://*:%d" % (self.port + 1))
self.collector.bind("tcp://*:%d" % (self.port + 2))
# Wrap sockets in ZMQStreams for IOLoop handlers
self.snapshot = ZMQStream(self.snapshot)
self.publisher = ZMQStream(self.publisher)
self.collector = ZMQStream(self.collector)
# Register our handlers with reactor
self.snapshot.on_recv(self.handle_snapshot)
self.collector.on_recv(self.handle_collect)
self.flush_callback = PeriodicCallback(self.flush_ttl, 1000)
# basic log formatting:
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
def _publish_msg(self, msg_type, data=None, metadata=None, buffers=None, **keys):
"""Helper for sending a comm message on IOPub"""
if threading.current_thread().name != 'MainThread' and IOLoop.initialized():
# make sure we never send on a zmq socket outside the main IOLoop thread
IOLoop.instance().add_callback(lambda : self._publish_msg(msg_type, data, metadata, buffers, **keys))
return
data = {} if data is None else data
metadata = {} if metadata is None else metadata
content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
self.session.send(self.iopub_socket, msg_type,
content,
metadata=json_clean(metadata),
parent=self.kernel._parent_header,
ident=self.topic,
buffers=buffers,
)
def __init__(self, primary, local, remote):
# initialize the Binary Star
self.ctx = zmq.Context() # Our private context
self.loop = IOLoop.instance() # Reactor loop
self.state = STATE_PRIMARY if primary else STATE_BACKUP
self.event = None # Current event
self.peer_expiry = 0 # When peer is considered 'dead'
self.voter_callback = None # Voting socket handler
self.master_callback = None # Call when become master
self.slave_callback = None # Call when become slave
# Create publisher for state going to peer
self.statepub = self.ctx.socket(zmq.PUB)
self.statepub.bind(local)
# Create subscriber for state coming from peer
self.statesub = self.ctx.socket(zmq.SUB)
self.statesub.setsockopt_string(zmq.SUBSCRIBE, u'')
self.statesub.connect(remote)
# wrap statesub in ZMQStream for event triggers
self.statesub = ZMQStream(self.statesub, self.loop)
# setup basic reactor events
self.heartbeat = PeriodicCallback(self.send_state,
HEARTBEAT, self.loop)
self.statesub.on_recv(self.recv_state)
def __init__(self, context=None, ioloop=None, **kwargs): #{
"""
Parameters
==========
ioloop : IOLoop
An existing IOLoop instance, if not passed, zmq.IOLoop.instance()
will be used.
context : Context
An existing Context instance, if not passed, zmq.Context.instance()
will be used.
serializer : Serializer
An instance of a Serializer subclass that will be used to serialize
and deserialize args, kwargs and the result.
"""
Context, _ = get_zmq_classes()
if context is None:
self.context = Context.instance()
else:
assert isinstance(context, Context)
self.context = context
self.ioloop = IOLoop.instance() if ioloop is None else ioloop
self._futures = {} # {<req_id> : <Future>}
super(TornadoRPCClient, self).__init__(**kwargs)
def wait_for_event(self, event, timeout = 30):
""" Wait for a plugin to be in a state
@param event : the event (STATUS_ALIVE, STATUS_STOPPED, ...)
This is done by subscribing on the MQ plugin.status publisher
If no status has been catched before the timeout, raise an error
"""
print(u"Start listening to MQ...")
IOLoop.instance().start()
# TODO : handle timeout
# the following line will be processed when a IOLoop.instance().stop() will be called
if self.plugin_status == event:
return True
else:
print(u"Plugin not in status '{0}' : status = {1}".format(event, self.plugin_status))
return False