def _process(self, rule, feed, limit=None):
fetch = Fetcher(rule, feed)
parser_name = rule.parser or PARSER_DEFAULT
parser = load_plugin(cif.smrt.parser.__path__[0], parser_name)
if parser is None:
self.logger.info('trying z{}'.format(parser_name))
parser = load_plugin(cif.smrt.parser.__path__[0], 'z{}'.format(parser_name))
if parser is None:
raise SystemError('Unable to load parser: {}'.format(parser_name))
self.logger.debug("loading parser: {}".format(parser))
parser = parser(self.client, fetch, rule, feed, limit=limit)
rv = parser.process()
return rv
def __init__(self, store=STORE_DEFAULT, storage_address=STORAGE_ADDR, *args, **kv):
self.logger = logging.getLogger(__name__)
self.context = zmq.Context()
self.router = self.context.socket(zmq.ROUTER)
self.storage_addr = storage_address
self.connected = False
self.ctrl = None
self.loop = ioloop.IOLoop.instance()
self.store = load_plugin(cif.store.__path__[0], store)(*args, **kv)
self.router = self.context.socket(zmq.ROUTER)
def _process(self, rule, feed, limit=None):
fetch = Fetcher(rule, feed)
parser_name = rule.parser or PARSER_DEFAULT
parser = load_plugin(cif.smrt.parser.__path__[0], parser_name)
if parser is None:
self.logger.info('trying z{}'.format(parser_name))
parser = load_plugin(cif.smrt.parser.__path__[0],
'z{}'.format(parser_name))
if parser is None:
raise SystemError(
'Unable to load parser: {}'.format(parser_name))
self.logger.debug("loading parser: {}".format(parser))
parser = parser(self.client, fetch, rule, feed, limit=limit)
rv = parser.process()
return rv
def _process(self, rule, feed, limit=None):
fetch = Fetcher(rule, feed)
parser = rule.parser or PARSER_DEFAULT
parser = load_plugin(cif.smrt.parser.__path__[0], parser)
self.logger.debug("loading parser: {}".format(parser))
parser = parser(self.client, fetch, rule, feed, limit=limit)
rv = parser.process()
return rv
def __init__(self, remote=REMOTE_ADDR, token=TOKEN, client='http'):
self.logger = logging.getLogger(__name__)
self.client = load_plugin(cif.client.__path__[0], client)(remote,
token)
def __init__(self, remote=REMOTE_ADDR, token=TOKEN, client='http'):
self.logger = logging.getLogger(__name__)
self.client = load_plugin(cif.client.__path__[0], client)(remote, token)
