def _process_item(self, item):
tc_date_added = item[0].get('dateAdded', None)
tc_last_modified = item[0].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))
l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))
return [self.tc.group_indicator_processing(item[0], item[1], item[2], f_seen, l_seen)]
def _build_iterator(self, now):
rkwargs = dict(
stream=True,
verify=self.verify_cert,
timeout=self.polling_timeout
)
if self.headers is not None:
rkwargs['headers'] = self.headers
if self.username is not None and self.password is not None:
rkwargs['auth'] = (self.username, self.password)
else:
raise RuntimeError('%s - credentials not set' % self.name)
if self.client_cert_required and self.key_file is not None and self.cert_file is not None:
rkwargs['cert'] = (self.cert_file, self.key_file)
else:
raise RuntimeError('%s - client certificate/key not set' % self.name)
if self.last_successful_run is None:
self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0
if self.last_vti_run is None:
self.last_vti_run = self.last_successful_run
start_date = datetime.fromtimestamp(self.last_vti_run / 1000)
end_date = datetime.fromtimestamp(utc_millisec() / 1000)
payload = {'startDate': start_date.strftime('%Y-%m-%d'),
'endDate': end_date.strftime('%Y-%m-%d')}
if self.indicator_type is not None:
payload['indicatorType'] = self.indicator_type
if self.victim_type is not None:
payload['victimType'] = self.victim_type
r = requests.get(
self.url,
params=payload,
**rkwargs
)
try:
r.raise_for_status()
except:
LOG.debug('%s - exception in request: %s %s',
self.name, r.status_code, r.content)
raise
result = self.extractor.search(r.json())
if result is None:
result = []
return result
def _process_item(self, item):
if self.indicator not in item:
LOG.debug('%s not in %s', self.indicator, item)
return [[None, None]]
indicator = item[self.indicator]
if not (isinstance(indicator, str) or
isinstance(indicator, unicode)):
LOG.error(
'Wrong indicator type: %s - %s',
indicator, type(indicator)
)
return [[None, None]]
indicator_type = item.get('indicatorType', None)
if indicator_type is not None:
indicator_type = VTI_INDICATOR_TYPES.get(indicator_type, None)
if indicator_type == 'HASH':
indicator_type = self._detect_sha_version(indicator)
if indicator_type == 'IP':
indicator_type = self._detect_ip_version(indicator)
upload_date = item.get('uploadDate', None)
if upload_date is None:
upload_date = utc_millisec()
else:
try:
dt = datetime.strptime(upload_date, '%Y-%m-%d')
upload_date = dt_to_millisec(dt)
except ValueError:
upload_date = utc_millisec()
if upload_date > self.last_vti_run:
self.last_vti_run = upload_date
fields = self.fields
if fields is None:
fields = item.keys()
fields.remove(self.indicator)
if 'indicatorType' in fields:
fields.remove('indicatorType')
if 'uploadDate' in fields:
fields.remove('uploadDate')
attributes = {'type': indicator_type, 'first_seen': upload_date, 'last_seen': upload_date}
for field in fields:
if field not in item:
continue
attributes['%s_%s' % (self.prefix, field)] = item[field]
return [[indicator, attributes]]
def _process_item(self, item):
tc_date_added = item[1].get('dateAdded', None)
tc_last_modified = item[1].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))
l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))
if l_seen > self.last_tc_run:
self.last_tc_run = l_seen
if item[0] == "IP":
return self.tc.ip_processing(item[1], item[2], f_seen, l_seen)
if item[0] == "GENERAL":
return self.tc.general_processing(item[1], item[2], f_seen, l_seen)
return []
def _process_item(self, item):
tc_date_added = item[0].get('dateAdded', None)
tc_last_modified = item[0].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
l_seen = utc_millisec(
) if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
return [
self.tc.group_indicator_processing(item[0], item[1], item[2],
f_seen, l_seen)
]
def _process_item(self, item):
tc_date_added = item[1].get('dateAdded', None)
tc_last_modified = item[1].get('lastModified', None)
f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(
datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
l_seen = utc_millisec(
) if tc_last_modified is None else dt_to_millisec(
datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=pytz.utc))
if l_seen > self.last_tc_run:
self.last_tc_run = l_seen
if item[0] == "IP":
return self.tc.ip_processing(item[1], item[2], f_seen, l_seen)
if item[0] == "GENERAL":
return self.tc.general_processing(item[1], item[2], f_seen, l_seen)
return []
def _build_iterator(self, now):
if self.tc is None:
raise RuntimeError('{} - API Key or API Secret not set, '
'poll not performed'.format(self.name))
if self.last_successful_run is None:
self.last_successful_run = utc_millisec(
) - self.initial_interval * 86400000.0
if self.last_tc_run is None:
self.last_tc_run = self.last_successful_run
return self.tc.indicator_iterator(self.last_tc_run)
def _build_iterator(self, now):
if self.tc is None:
raise RuntimeError(
'{} - API Key or API Secret not set, '
'poll not performed'.format(self.name)
)
if self.last_successful_run is None:
self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0
if self.last_tc_run is None:
self.last_tc_run = self.last_successful_run
return self.tc.indicator_iterator(self.last_tc_run)