def _process_item(self, item):
tc_date_added = item[0].get('dateAdded', None)
tc_last_modified = item[0].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
l_seen = utc_millisec(
) if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
return [
self.tc.group_indicator_processing(item[0], item[1], item[2],
f_seen, l_seen)
]
def _eval_send_exabgp(self, message, source=None, indicator=None, value=None):
indicators = [indicator]
if '-' in indicator:
a1, a2 = indicator.split('-', 1)
indicators = map(str, netaddr.iprange_to_cidrs(a1, a2))
# Already in our format? Just convert it to IPNetwork object
elif '/' in indicator:
indicators = map(str, netaddr.iprange_to_cidrs(indicator, indicator))
# Single host one per line
elif re.match(r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$", indicator):
indicators = map(str, netaddr.iprange_to_cidrs(indicator, indicator))
try:
for i in indicators:
value['__indicator'] = i
now = utc_millisec()
age_out = now+self.age_out*1000
try:
feed_community = value['feed_community']
except:
feed_community = self.exabgp_defcomm
value['_age_out'] = age_out
values = { 'command': str(message) + ' route ' + i + ' next-hop self community ' + feed_community }
data = urllib.urlencode(values)
req = urllib2.Request('http://' + self.exabgp_host + ':' + str(self.exabgp_port))
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
# req.add_header('Content-Type', 'application/json')
response = urllib2.urlopen(req, data)
#LOG.info("%s: %s", str(message).upper(), i)
self.statistics['message.sent'] += 1
except:
LOG.error("%s: %s", str(message).upper(), i)
pass
def _age_out_run(self):
"""
Checks for indicators that are too old and triggers their removal.
:return: None
"""
while True:
now = utc_millisec()
low_watermark = now - self.age_out_interval * 1000
otimestamp, oindicator = self._read_oldest_indicator()
LOG.debug('{} - low watermark: {} otimestamp: {}'.format(
self.name, low_watermark, otimestamp))
while otimestamp is not None and otimestamp < low_watermark:
self._delete_indicator(oindicator)
otimestamp, oindicator = self._read_oldest_indicator()
wait_time = 30
if otimestamp is not None:
next_expiration = (
(otimestamp + self.age_out_interval * 1000) - now)
wait_time = max(wait_time, next_expiration / 1000 + 1)
LOG.debug('%s - sleeping for %d secs', self.name, wait_time)
gevent.sleep(wait_time)
def _process_item(self, item):
tc_date_added = item[1].get('dateAdded', None)
tc_last_modified = item[1].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
l_seen = utc_millisec(
) if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
if l_seen > self.last_tc_run:
self.last_tc_run = l_seen
if item[0] == "IP":
return self.tc.ip_processing(item[1], item[2], f_seen, l_seen)
if item[0] == "GENERAL":
return self.tc.general_processing(item[1], item[2], f_seen, l_seen)
return []
def _build_iterator(self, now):
if self.tc is None:
raise RuntimeError('{} - API Key or API Secret not set, '
'poll not performed'.format(self.name))
if self.last_successful_run is None:
self.last_successful_run = utc_millisec(
) - self.initial_interval * 86400000.0
if self.last_tc_run is None:
self.last_tc_run = self.last_successful_run
return self.tc.indicator_iterator(self.last_tc_run)
def filtered_update(self, source=None, indicator=None, value=None):
"""
Processes updates, i.e. triggers addition of new indicators.
:param source: ignored
:param indicator: str, actual indicator
:param value: dict, indicator object holding additional information
:return: None
"""
now = utc_millisec()
self._add_indicator(now, indicator, value)