def run(self):
target = self.target
context = zmq.Context()
channel_send = context.socket(zmq.PUSH)
channel_send.connect("tcp://127.0.0.1:" + str(self.push_port))
# Wait a second to wake up and connect
time.sleep(1)
for item in target(self.items(), **self.kwargs):
send_ujson(channel_send, item)
for x in range(self.consumers):
send_ujson(channel_send, 'STOP')
time.sleep(1)
def run(self):
target = self.target
context = zmq.Context()
channel_send = context.socket(zmq.PUSH)
channel_send.connect("tcp://127.0.0.1:" + str(self.push_port))
# Wait a second to wake up and connect
time.sleep(1)
for item in target(self.items(), **self.kwargs):
send_ujson(channel_send, item)
for x in range(self.consumers):
send_ujson(channel_send, 'STOP')
time.sleep(1)
def run(self):
target = self.target
context = zmq.Context()
# Set up sending channel for page elements
channel_pages_send = context.socket(zmq.PUSH)
channel_pages_send.bind("tcp://127.0.0.1:%s" % self.push_pages_port)
# Set up sending channel for revision elements
channel_revs_send = context.socket(zmq.PUSH)
channel_revs_send.bind("tcp://127.0.0.1:%s" % self.push_revs_port)
channel_control = context.socket(zmq.PUB)
channel_control.bind("tcp://127.0.0.1:%s" % self.control_port)
# Wait a second to wake up and connect
time.sleep(1)
for item in target(*self.args, **self.kwargs):
# Classify outcome elements in their corresponding queue
# for later processing
if isinstance(item, Page):
send_ujson(channel_pages_send, item)
elif isinstance(item, Revision):
send_ujson(channel_revs_send, item)
# elif isinstance(item, LogItem):
# if self.out_logitem_queue is not None:
# self.out_logitem_queue.put(item)
#
# elif isinstance(item, User):
# if self.out_user_queue is not None:
# self.output_user_queue.put(item)
# Wait few seconds to let workers empty data pipeline
time.sleep(20)
#channel_pages_send.close()
#channel_revs_send.close()
# Send STOP message to all workers and quit
if self.page_consumers > 0 and self.rev_consumers > 0:
channel_control.send('STOP')
time.sleep(5)
def run(self):
target = self.target
# Set up sending ZMQ data and control channels
context = zmq.Context()
if (self.push_pages_port):
channel_pages_send = context.socket(zmq.PUSH)
channel_pages_send.bind("tcp://127.0.0.1:%s" %
self.push_pages_port)
if (self.push_revs_port):
channel_revs_send = context.socket(zmq.PUSH)
channel_revs_send.bind("tcp://127.0.0.1:%s" %
self.push_revs_port)
if (self.push_logs_port):
channel_logs_send = context.socket(zmq.PUSH)
channel_logs_send.bind("tcp://127.0.0.1:%s" %
self.push_logs_port)
channel_control = context.socket(zmq.PUB)
channel_control.bind("tcp://127.0.0.1:%s" %
self.control_port)
# Wait a second to wake up and connect
time.sleep(1)
for item in target(*self.args, **self.kwargs):
# Classify outcome elements in their corresponding queue
# for later processing
if isinstance(item, Page):
send_ujson(channel_pages_send, item)
elif isinstance(item, Revision):
send_ujson(channel_revs_send, item)
elif isinstance(item, LogItem):
send_ujson(channel_logs_send, item)
# Wait few seconds to let workers empty data pipeline
time.sleep(20)
#channel_pages_send.close()
#channel_revs_send.close()
# Send STOP message to all workers and quit
if self.consumers > 0:
channel_control.send('STOP')
time.sleep(5)
