Exemple #1
0
class ParsingProtocol(protocol.ProcessProtocol):

    def __init__(self, xmpp):
        self._xmpp = xmpp
        self._proto = PipeProtocol()
        self._callbacks = {}
        self._id = 0

    def start(self):
        reactor.spawnProcess(self, "parsing_worker.py")

    def stop(self):
        try:
            self.transport.signalProcess("KILL")
        except error.ProcessExitedAlready:
            pass

    def outReceived(self, out):
        packets = self._proto.decode(out)
        for packet in packets:
            parsed = cPickle.loads(packet)
            self._callbacks[parsed["_id"]].callback(parsed)
            del self._callbacks[parsed["_id"]]

    def errReceived(self, err):
        report = u"PARSING WORKER ERROR:\n\n%s" % err
        log.msg(report)
        self._xmpp.send_message(
            to=config.error_report_jid, from_=config.main_full_jid,
            body=report)

    def parse(self, task, data):
        self._id += 1
        d = defer.Deferred()
        self._callbacks[self._id] = d
        task = task.copy()
        task["_id"] = self._id
        task["_data"] = data
        encoded = self._proto.encode(cPickle.dumps(task, protocol=2))
        self.transport.write(encoded)
        return d
Exemple #2
0
class ParsingProtocol(protocol.ProcessProtocol):
    def __init__(self, xmpp):
        self._xmpp = xmpp
        self._proto = PipeProtocol()
        self._callbacks = {}
        self._id = 0

    def start(self):
        reactor.spawnProcess(self, "parsing_worker.py")

    def stop(self):
        try:
            self.transport.signalProcess("KILL")
        except error.ProcessExitedAlready:
            pass

    def outReceived(self, out):
        packets = self._proto.decode(out)
        for packet in packets:
            parsed = cPickle.loads(packet)
            self._callbacks[parsed["_id"]].callback(parsed)
            del self._callbacks[parsed["_id"]]

    def errReceived(self, err):
        report = u"PARSING WORKER ERROR:\n\n%s" % err
        log.msg(report)
        self._xmpp.send_message(to=config.error_report_jid,
                                from_=config.main_full_jid,
                                body=report)

    def parse(self, task, data):
        self._id += 1
        d = defer.Deferred()
        self._callbacks[self._id] = d
        task = task.copy()
        task["_id"] = self._id
        task["_data"] = data
        encoded = self._proto.encode(cPickle.dumps(task, protocol=2))
        self.transport.write(encoded)
        return d
Exemple #3
0
 def __init__(self, xmpp):
     self._xmpp = xmpp
     self._proto = PipeProtocol()
     self._callbacks = {}
     self._id = 0
Exemple #4
0
import sys
import time
import fcntl
import select
import cPickle
import traceback
from parsers import parsers
from pipe_protocol import PipeProtocol


# Set stdin in nonblocking-mode
fd = sys.stdin.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

proto = PipeProtocol()
while True:
    select.select([sys.stdin], [], [])
    data = sys.stdin.read()
    packets = proto.decode(data)
    for packet in packets:
        task = cPickle.loads(packet)
        try:
            res = parsers[task["parser"]].do_task(task)
            if not res:
                res = {}
        except Exception:
            del task["_data"]
            err = "TASK:\n%s\n\nTRACEBACK:\n%s" % (
                repr(task), traceback.format_exc()[:-1])
            sys.stderr.write(err)
Exemple #5
0
 def __init__(self, xmpp):
     self._xmpp = xmpp
     self._proto = PipeProtocol()
     self._callbacks = {}
     self._id = 0
Exemple #6
0
import os
import sys
import time
import fcntl
import select
import cPickle
import traceback
from parsers import parsers
from pipe_protocol import PipeProtocol

# Set stdin in nonblocking-mode
fd = sys.stdin.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

proto = PipeProtocol()
while True:
    select.select([sys.stdin], [], [])
    data = sys.stdin.read()
    packets = proto.decode(data)
    for packet in packets:
        task = cPickle.loads(packet)
        try:
            res = parsers[task["parser"]].do_task(task)
            if not res:
                res = {}
        except Exception:
            del task["_data"]
            err = "TASK:\n%s\n\nTRACEBACK:\n%s" % (repr(task),
                                                   traceback.format_exc()[:-1])
            sys.stderr.write(err)