def generate_reports(self, minimum_severity='WARNING', status_filter=None, minimum_confidence=50): cur = self.dbconn.cursor() q, params = build_sql_query(minimum_severity, status_filter, minimum_confidence) cur.execute(q, params) for res in ResultIter(cur): new_reports = processing_engines.process_ioc(res, minimum_severity, status_filter, minimum_confidence) for report in new_reports: yield report
def perform_feed_retrieval(self): new_feed = self.create_feed() tx_limit = self.bridge_options.get('tx_request_limit', 500) or 500 tx_retries = self.bridge_options.get('tx_request_retries', 5) or 5 now = datetime.utcnow() since_date = now - timedelta(days=self.bridge_options["historical_days"]) since_date_str = since_date.strftime("%Y-%m-%d") until_date = since_date while until_date < now + timedelta(1): until_date += timedelta(days=1) until_date_str = until_date.strftime("%Y-%m-%d") for ioc_type in self.bridge_options["ioc_types"]: self.logger.info("Pulling %s IOCs (%s to %s)" % (ioc_type, since_date_str, until_date_str)) try: count = 0 for result in ThreatDescriptor.objects(since=since_date_str, until=until_date_str, type_=ioc_type, dict_generator=True, limit=tx_limit, retries=tx_retries, fields="raw_indicator,owner,indicator{id,indicator},type,last_updated,share_level,severity,description,report_urls,status,confidence"): new_reports = processing_engines.process_ioc(ioc_type, result, minimum_severity=self.bridge_options["minimum_severity"], status_filter=self.bridge_options["status_filter"], minimum_confidence=self.bridge_options["minimum_confidence"]) for report in new_reports: new_feed.add_report(report) count += 1 if count % 1000 == 0: self.logger.info("%s added %d reports for this iteration" % (ioc_type, count)) self.logger.info("%s added %d reports TOTAL" % (ioc_type, count)) except pytxFetchError: self.logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type) except Exception: self.logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type) # update the start date since_date_str = until_date_str with self.feed_lock: self.feed = new_feed