class ParsingProtocol(protocol.ProcessProtocol):
def __init__(self, xmpp):
self._xmpp = xmpp
self._proto = PipeProtocol()
self._callbacks = {}
self._id = 0
def start(self):
reactor.spawnProcess(self, "parsing_worker.py")
def stop(self):
try:
self.transport.signalProcess("KILL")
except error.ProcessExitedAlready:
pass
def outReceived(self, out):
packets = self._proto.decode(out)
for packet in packets:
parsed = cPickle.loads(packet)
self._callbacks[parsed["_id"]].callback(parsed)
del self._callbacks[parsed["_id"]]
def errReceived(self, err):
report = u"PARSING WORKER ERROR:\n\n%s" % err
log.msg(report)
self._xmpp.send_message(
to=config.error_report_jid, from_=config.main_full_jid,
body=report)
def parse(self, task, data):
self._id += 1
d = defer.Deferred()
self._callbacks[self._id] = d
task = task.copy()
task["_id"] = self._id
task["_data"] = data
encoded = self._proto.encode(cPickle.dumps(task, protocol=2))
self.transport.write(encoded)
return d
class ParsingProtocol(protocol.ProcessProtocol):
def __init__(self, xmpp):
self._xmpp = xmpp
self._proto = PipeProtocol()
self._callbacks = {}
self._id = 0
def start(self):
reactor.spawnProcess(self, "parsing_worker.py")
def stop(self):
try:
self.transport.signalProcess("KILL")
except error.ProcessExitedAlready:
pass
def outReceived(self, out):
packets = self._proto.decode(out)
for packet in packets:
parsed = cPickle.loads(packet)
self._callbacks[parsed["_id"]].callback(parsed)
del self._callbacks[parsed["_id"]]
def errReceived(self, err):
report = u"PARSING WORKER ERROR:\n\n%s" % err
log.msg(report)
self._xmpp.send_message(to=config.error_report_jid,
from_=config.main_full_jid,
body=report)
def parse(self, task, data):
self._id += 1
d = defer.Deferred()
self._callbacks[self._id] = d
task = task.copy()
task["_id"] = self._id
task["_data"] = data
encoded = self._proto.encode(cPickle.dumps(task, protocol=2))
self.transport.write(encoded)
return d
def __init__(self, xmpp):
self._xmpp = xmpp
self._proto = PipeProtocol()
self._callbacks = {}
self._id = 0
import sys
import time
import fcntl
import select
import cPickle
import traceback
from parsers import parsers
from pipe_protocol import PipeProtocol
# Set stdin in nonblocking-mode
fd = sys.stdin.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
proto = PipeProtocol()
while True:
select.select([sys.stdin], [], [])
data = sys.stdin.read()
packets = proto.decode(data)
for packet in packets:
task = cPickle.loads(packet)
try:
res = parsers[task["parser"]].do_task(task)
if not res:
res = {}
except Exception:
del task["_data"]
err = "TASK:\n%s\n\nTRACEBACK:\n%s" % (
repr(task), traceback.format_exc()[:-1])
sys.stderr.write(err)
def __init__(self, xmpp):
self._xmpp = xmpp
self._proto = PipeProtocol()
self._callbacks = {}
self._id = 0
import os
import sys
import time
import fcntl
import select
import cPickle
import traceback
from parsers import parsers
from pipe_protocol import PipeProtocol
# Set stdin in nonblocking-mode
fd = sys.stdin.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
proto = PipeProtocol()
while True:
select.select([sys.stdin], [], [])
data = sys.stdin.read()
packets = proto.decode(data)
for packet in packets:
task = cPickle.loads(packet)
try:
res = parsers[task["parser"]].do_task(task)
if not res:
res = {}
except Exception:
del task["_data"]
err = "TASK:\n%s\n\nTRACEBACK:\n%s" % (repr(task),
traceback.format_exc()[:-1])
sys.stderr.write(err)