def process_all(self, config_to_process=None, force_run=False):
results = {}
fields = ['frequency', 'modules', 'ttl', 'direction']
date_hour = datetime.datetime.now().hour
for name, config in self.configs.iteritems():
if not _valid_json(fields, config):
app.logger.warn('Bad config from feed.json')
continue
if config_to_process:
if not name == config_to_process:
continue
app.logger.info('Processing Feed: %s' % name)
modules = config.get('modules')
if force_run:
freq = '*'
else:
freq = config.get('frequency').split(',')
if not ('*' in freq or str(date_hour) in freq):
continue
if 'parse' in modules.keys() and 'collect' in modules.keys():
try:
coll_cls = _dynamic_load(modules['collect'].get('name'))
parse_cls = _dynamic_load(modules['parse'].get('name'))
except Exception, e:
app.logger.warn('error loading classes: %s' % e)
continue
collect_config = modules['collect'].get('config')
parse_config = modules['parse'].get('config')
if not collect_config and not parse_config:
app.logger.warn('error loading module configs')
continue
collector = coll_cls(collect_config)
data = collector.get()
if not data:
app.logger.warn('error loading data from collector')
continue
parser = parse_cls(parse_config, data)
logs = parser.run()
#results[config.get('name', 'n/a')] = logs
results[name] = _add_indicators(logs, name, config.get('ttl'),
config.get('direction'))
elif 'custom' in modules.keys():
pass
def process_all(self, config_to_process=None):
results = {}
fields = ['name', 'frequency', 'event_id', 'modules']
date_hour = datetime.datetime.now().hour
for config in self.configs:
if not _valid_json(fields, config):
app.logger.warn('Bad config from feed.json')
continue
if config_to_process:
if not config.get('name') == config_to_process:
continue
app.logger.info('Processing Feed: %s' % config['name'])
modules = config.get('modules')
event_id = config.get('event_id')
freq = config.get('frequency').split(',')
if not ('*' in freq or date_hour in freq):
continue
if 'parse' in modules.keys() and 'collect' in modules.keys():
try:
coll_cls = _dynamic_load(modules['collect'].get('name'))
parse_cls = _dynamic_load(modules['parse'].get('name'))
except Exception, e:
app.logger.warn('error loading classes: %s' % e)
continue
collect_config = modules['collect'].get('config')
parse_config = modules['parse'].get('config')
if not collect_config and not parse_config:
app.logger.warn('error loading module configs')
continue
collector = coll_cls(collect_config)
data = collector.get()
if not data:
app.logger.warn('error loading data from collector')
continue
parser = parse_cls(parse_config, event_id, data)
logs = parser.run()
#results[config.get('name', 'n/a')] = logs
results[config.get('name', 'n/a')] = _add_indicators(logs)
elif 'custom' in modules.keys():
pass
class ParseScrapedData:
def __init__(self, data, source):
self.data = data
self.source = source
if not Event.query.filter(
Event.name == "{} scraping".format(self.source)).first():
# add source
obj = Source()
obj.name = self.source
db.session.add(obj)
db.session.commit()
# create event
self.add_event({
"name":
"{} scraping".format(self.source),
"details":
"Observables mined from {}".format(self.source),
"confidence":
10,
"source":
source,
"tlp":
"Green",
"impact":
"Low",
"likelihood":
"Low"
})
def get_source_id(self):
return Event.query.filter(
Event.name == '{} scraping'.format(self.source)).first().id
def add_event(self, payload):
req_keys = ('name', 'details', 'confidence', 'source', 'tlp', 'impact',
'likelihood')
try:
pld = payload
except Exception, e:
app.logger.info('data error: {}'.format(e))
return
if utils._valid_json(req_keys, pld):
impact = Level.query.filter(Level.name == pld['impact']).first()
likelihood = Likelihood.query.filter(
Likelihood.name == pld['likelihood']).first()
source = Source.query.filter(Source.name == pld['source']).first()
tlp = Tlp.query.filter(Tlp.name == pld['tlp']).first()
if not (impact and likelihood and source and tlp):
app.logger.warning(
'impact, likelihood, source, or tlp not found')
try:
confidence = int(pld['confidence'])
if confidence < 0 or confidence > 100:
raise Exception
except Exception, e:
app.logger.warning(
'confidence was not a number between 0 and 100')
return
ev = Event(pld['name'], pld['details'], source, tlp, impact,
likelihood, confidence)
db.session.add(ev)
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
app.logger.warning('Integrity error - rolled back')
app.logger.info('success - event_id {}'.format(ev.id))
db.session.rollback()
app.logger.warning('Integrity error - rolled back')
app.logger.info('success - event_id {}'.format(ev.id))
else:
app.logger.warning('error: bad json')
def add_data(self, data):
req_keys = ('control', 'data_type', 'event_id', 'pending', 'data')
try:
pld = json.loads(data)
except Exception, e:
app.logger.warning('error when uploading indicator: {}'.format(e))
return
if utils._valid_json(req_keys, pld):
# load related stuff
res_dict = ResultsDict(pld['event_id'], pld['control'])
for val, desc in pld['data']:
res_dict.new_ind(data_type=pld['data_type'],
indicator=val,
date=None,
description=desc)
utils._add_indicators(res_dict, pld['pending'])
app.logger.info('success - added indicator from {}'.format(
self.source))
else:
app.logger.warning('error: bad json')
def run(self):
app.logger.info("Processing ParseScrapedData")