def _process(self, rule, feed, limit=None): fetch = Fetcher(rule, feed) parser_name = rule.parser or PARSER_DEFAULT parser = load_plugin(cif.smrt.parser.__path__[0], parser_name) if parser is None: self.logger.info('trying z{}'.format(parser_name)) parser = load_plugin(cif.smrt.parser.__path__[0], 'z{}'.format(parser_name)) if parser is None: raise SystemError('Unable to load parser: {}'.format(parser_name)) self.logger.debug("loading parser: {}".format(parser)) parser = parser(self.client, fetch, rule, feed, limit=limit) rv = parser.process() return rv
def __init__(self, store=STORE_DEFAULT, storage_address=STORAGE_ADDR, *args, **kv): self.logger = logging.getLogger(__name__) self.context = zmq.Context() self.router = self.context.socket(zmq.ROUTER) self.storage_addr = storage_address self.connected = False self.ctrl = None self.loop = ioloop.IOLoop.instance() self.store = load_plugin(cif.store.__path__[0], store)(*args, **kv) self.router = self.context.socket(zmq.ROUTER)
def _process(self, rule, feed, limit=None): fetch = Fetcher(rule, feed) parser_name = rule.parser or PARSER_DEFAULT parser = load_plugin(cif.smrt.parser.__path__[0], parser_name) if parser is None: self.logger.info('trying z{}'.format(parser_name)) parser = load_plugin(cif.smrt.parser.__path__[0], 'z{}'.format(parser_name)) if parser is None: raise SystemError( 'Unable to load parser: {}'.format(parser_name)) self.logger.debug("loading parser: {}".format(parser)) parser = parser(self.client, fetch, rule, feed, limit=limit) rv = parser.process() return rv
def _process(self, rule, feed, limit=None): fetch = Fetcher(rule, feed) parser = rule.parser or PARSER_DEFAULT parser = load_plugin(cif.smrt.parser.__path__[0], parser) self.logger.debug("loading parser: {}".format(parser)) parser = parser(self.client, fetch, rule, feed, limit=limit) rv = parser.process() return rv
def __init__(self, remote=REMOTE_ADDR, token=TOKEN, client='http'): self.logger = logging.getLogger(__name__) self.client = load_plugin(cif.client.__path__[0], client)(remote, token)