def process(self, rule, feed, limit=None, data=None, filters=None): parser = self.load_parser(rule, feed, limit=limit, data=data, filters=filters) feed_indicators = parser.process() if not limit: limit = rule.feeds[feed].get('limit') if limit: feed_indicators = itertools.islice(feed_indicators, int(limit)) feed_indicators = (i for i in feed_indicators if self.is_valid(i, rule)) feed_indicators = (self.clean_indicator(i, rule) for i in feed_indicators) # check to see if the indicator is too old if self.goback: feed_indicators = (i for i in feed_indicators if not self.is_old(i)) feed_indicators = (i for i in feed_indicators if not self.is_archived_with_log(i)) feed_indicators_batches = chunk(feed_indicators, int(FIREBALL_SIZE)) for indicator_batch in feed_indicators_batches: self.archiver.begin() self.send_indicators(indicator_batch) for i in indicator_batch: if self.is_archived_with_log(i): continue # TODO- this affects a lot of tests # converted i.format_keys to generator in indicator-0.0.0b0 yield list(i.format_keys())[0] self.archive(i) self.archiver.commit() if limit: self.logger.debug("limit reached...")
def _process(self, rule, feed, limit=None): fetch = Fetcher(rule, feed) parser_name = rule.parser or PARSER_DEFAULT parser = load_plugin(csirtg_smrt.parser.__path__[0], parser_name) if parser is None: self.logger.info('trying z{}'.format(parser_name)) parser = load_plugin(csirtg_smrt.parser.__path__[0], 'z{}'.format(parser_name)) if parser is None: raise SystemError('Unable to load parser: {}'.format(parser_name)) self.logger.debug("loading parser: {}".format(parser)) parser = parser(self.client, fetch, rule, feed, limit=limit) rv = parser.process() return rv
def process(self, rule, feed, limit=None, data=None, filters=None): parser = self.load_parser(rule, feed, limit=limit, data=data, filters=filters) feed_indicators = parser.process() if not limit: limit = rule.feeds[feed].get('limit') if limit: feed_indicators = itertools.islice(feed_indicators, int(limit)) feed_indicators = (i for i in feed_indicators if self.is_valid(i, rule)) feed_indicators = (self.clean_indicator(i, rule) for i in feed_indicators) # check to see if the indicator is too old if self.goback: feed_indicators = (i for i in feed_indicators if not self.is_old(i)) feed_indicators = (i for i in feed_indicators if not self.is_archived_with_log(i)) feed_indicators_batches = chunk(feed_indicators, int(FIREBALL_SIZE)) for indicator_batch in feed_indicators_batches: self.archiver.begin() self.send_indicators(indicator_batch) for i in indicator_batch: if self.is_archived_with_log(i): continue yield i.format_keys() self.archive(i) self.archiver.commit() if limit: self.logger.debug("limit reached...")