def _process_item(self, item): tc_date_added = item[0].get('dateAdded', None) tc_last_modified = item[0].get('lastModified', None) f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec( datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace( tzinfo=pytz.utc)) l_seen = utc_millisec( ) if tc_last_modified is None else dt_to_millisec( datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace( tzinfo=pytz.utc)) return [ self.tc.group_indicator_processing(item[0], item[1], item[2], f_seen, l_seen) ]
def _eval_send_exabgp(self, message, source=None, indicator=None, value=None): indicators = [indicator] if '-' in indicator: a1, a2 = indicator.split('-', 1) indicators = map(str, netaddr.iprange_to_cidrs(a1, a2)) # Already in our format? Just convert it to IPNetwork object elif '/' in indicator: indicators = map(str, netaddr.iprange_to_cidrs(indicator, indicator)) # Single host one per line elif re.match(r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$", indicator): indicators = map(str, netaddr.iprange_to_cidrs(indicator, indicator)) try: for i in indicators: value['__indicator'] = i now = utc_millisec() age_out = now+self.age_out*1000 try: feed_community = value['feed_community'] except: feed_community = self.exabgp_defcomm value['_age_out'] = age_out values = { 'command': str(message) + ' route ' + i + ' next-hop self community ' + feed_community } data = urllib.urlencode(values) req = urllib2.Request('http://' + self.exabgp_host + ':' + str(self.exabgp_port)) req.add_header('Content-Type', 'application/x-www-form-urlencoded') # req.add_header('Content-Type', 'application/json') response = urllib2.urlopen(req, data) #LOG.info("%s: %s", str(message).upper(), i) self.statistics['message.sent'] += 1 except: LOG.error("%s: %s", str(message).upper(), i) pass
def _age_out_run(self): """ Checks for indicators that are too old and triggers their removal. :return: None """ while True: now = utc_millisec() low_watermark = now - self.age_out_interval * 1000 otimestamp, oindicator = self._read_oldest_indicator() LOG.debug('{} - low watermark: {} otimestamp: {}'.format( self.name, low_watermark, otimestamp)) while otimestamp is not None and otimestamp < low_watermark: self._delete_indicator(oindicator) otimestamp, oindicator = self._read_oldest_indicator() wait_time = 30 if otimestamp is not None: next_expiration = ( (otimestamp + self.age_out_interval * 1000) - now) wait_time = max(wait_time, next_expiration / 1000 + 1) LOG.debug('%s - sleeping for %d secs', self.name, wait_time) gevent.sleep(wait_time)
def _process_item(self, item): tc_date_added = item[1].get('dateAdded', None) tc_last_modified = item[1].get('lastModified', None) f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec( datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace( tzinfo=pytz.utc)) l_seen = utc_millisec( ) if tc_last_modified is None else dt_to_millisec( datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace( tzinfo=pytz.utc)) if l_seen > self.last_tc_run: self.last_tc_run = l_seen if item[0] == "IP": return self.tc.ip_processing(item[1], item[2], f_seen, l_seen) if item[0] == "GENERAL": return self.tc.general_processing(item[1], item[2], f_seen, l_seen) return []
def _build_iterator(self, now): if self.tc is None: raise RuntimeError('{} - API Key or API Secret not set, ' 'poll not performed'.format(self.name)) if self.last_successful_run is None: self.last_successful_run = utc_millisec( ) - self.initial_interval * 86400000.0 if self.last_tc_run is None: self.last_tc_run = self.last_successful_run return self.tc.indicator_iterator(self.last_tc_run)
def filtered_update(self, source=None, indicator=None, value=None): """ Processes updates, i.e. triggers addition of new indicators. :param source: ignored :param indicator: str, actual indicator :param value: dict, indicator object holding additional information :return: None """ now = utc_millisec() self._add_indicator(now, indicator, value)