def __init__(self, cfgPath): """ Source class' constructor. :param config: Config object that contains informations about the feed. :type config: Config object instance. """ self.cfgPath = cfgPath #conf from hippocampe/core/conf/feeds conf = getConf.getConf(cfgPath) #conf from hippocampe/core/conf/hippo cfg = getConf.getHippoConf() super(Source, self).__init__() self.source = conf.get('source', 'url') self.firstQuery = str() self.lastQuery = str() self.description = conf.get('source', 'description') self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = cfg.get('elasticsearch', 'typeNameESSource') self.score = conf.getint('source', 'score') self.coreIntelligence = conf.get('source', 'coreIntelligence') self.typeNameESIntel = conf.get('elasticsearch', 'typeIntel') self.validityDate = conf.get('source', 'validityDate') self.useByDate = conf.get('source', 'useBydate')
def indexNew(coreIntelligence, listData): logger.info('bulkOp.indexNew launched') hippoCfg = getHippoConf() indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') typeNameES = hippoCfg.get('elasticsearch', 'typeNameESNew') indexNew = IndexNew() indexNew.createIndexNew() es = getES() k = ({ '_op_type': 'index', '_index': indexNameES, '_type': typeNameES, '_source': { 'type': coreIntelligence, 'toSearch': data[coreIntelligence] } } for data in listData) #k.next() gives: #{'_op_type': 'index', '_index':'hippocampe', '_type':'new', '_source': {'typeIntel': 'ip', 'intelligence': '1.1.1.1'} res = helpers.bulk(es, k) logger.info('bulkOp.index res: %s', res) logger.info('bulkOp.indexNew end') return res[0]
def index(cfgPath, listData): logger.info('bulkOp.index launched') hippoCfg = getHippoConf() indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') cfg = getConf(cfgPath) typeNameES = cfg.get('elasticsearch', 'typeIntel') #creating the index, only if does not exist index = IndexIntel(cfgPath) index.createIndexIntel() es = getES() k = ({ '_op_type': 'index', '_index': indexNameES, '_type': typeNameES, '_source': data } for data in listData) res = helpers.bulk(es, k, raise_on_error=False) #res = helpers.bulk(es,k, raise_on_exception=False) #res = helpers.bulk(es,k) logger.info('bulkOp.index res: %s', res) logger.info('bulkOp.index end') return res
def bigMsearch(coreIntelligence, listParsedData): logger.info('searchIntel.bigMsearch launched') es = getES() cfg = getHippoConf() indexNameES = cfg.get('elasticsearch', 'indexNameES') req = list() req_head = {'index': indexNameES} coreIntelligence = coreIntelligence for element in listParsedData: req_body = { 'query': { 'bool': { 'must': [{ 'match': { coreIntelligence: element[coreIntelligence] } }] } } } req.extend([req_head, req_body]) res = es.msearch(body=req) logger.info('searchIntel.bigMsearch end') return res
def __init__(self, typeIntel): cfg = getHippoConf() self.typeIntel = typeIntel self.es = getES() self.docSearch = dict() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') #contains every distinct value from a field self.size = int()
def __init__(self): cfg = getHippoConf() self.docSearch = str() self.matchResponse = str() self.matchDict = dict() self.es = getES() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew') self.nbDoc = int()
def __init__(self): super(Job, self).__init__() cfg = getHippoConf() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs') self.status = str() self.startTime = str() self.endTime = str() self.duration = str() self.report = dict()
def __init__(self, typeNameES): """ ObjToIndex class' constructor. """ cfg = getHippoConf() self.es = getES() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = typeNameES self.docSearch = dict() self.size = int()
def __init__(self, field): cfg = getHippoConf() self.field = field self.es = getES() self.docSearch = dict() self.matchResponse = dict() self.indexName = cfg.get('elasticsearch', 'indexNameES') #contains every distinct value from a field self.distinctList = list() #number of distinct values self.size = int()
def __init__(self, typeIntel, ioc): cfg = getHippoConf() self.typeIntel = typeIntel self.value = ioc self.docMatch = str() self.matchResponse = str() self.matchList = list() self.es = getES() #data stored in index hippocampe, so search is only in this index self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
def __init__(self): super(IndexJob, self).__init__() cfg = getHippoConf() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs') #the mapping has to be indicated at the creation of the index #the confMapping, is the mapping indicated in conf file for each field #it has to be parsed bellow self.confMapping = dict()
def __init__(self, cfgPath): """ IndexSource class' constructor. :param source: shadowBook process's configuration :type config: Config object instance """ conf = getHippoConf() super(IndexSource, self).__init__() self.indexNameES = conf.get('elasticsearch', 'indexNameES') self.typeNameES = conf.get('elasticsearch', 'typeNameESSource')
def __init__(self, typeIntel, intelligence): """ Ioc class' constructor. :param config: Config object that contains informations about the ioc. :type config: Config object instance. :param source: Ioc's Source object. :type source: Source object instance. """ super(NewIntel, self).__init__() cfg = getHippoConf() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = 'new' self.typeIntel = typeIntel self.intelligence = intelligence
def __init__(self, cfgPath): """ IndexIOC class' constructor. :param config: shadowBook process's configuration :type config: Config object instance """ super(IndexIntel, self).__init__() self.conf = getConf.getConf(cfgPath) hippoConf = getConf.getHippoConf() self.indexNameES = hippoConf.get('elasticsearch', 'indexNameES') self.typeNameES = self.conf.get('elasticsearch', 'typeIntel') #the mapping has to be indicated at the creation of the index #the confMapping, is the mapping indicated in conf file for each field #it has to be parsed bellow self.confMapping = dict()
def checkData(checkList): #checkList is the list of types to check #check if the hippocampe's index exists in ES #and check if ES type exists according to checkList logger.info('ES.checkData launched') logger.info(checkList) ES = getES() index = IndicesClient(ES) cfg = getHippoConf() indexName = cfg.get('elasticsearch', 'indexNameES') #references contains the name of types used in Hippocampe references = dict() references['sourceType'] = cfg.get('elasticsearch', 'typeNameESSource') references['newType'] = cfg.get('elasticsearch', 'typeNameESNew') references['jobsType'] = cfg.get('elasticsearch', 'typeNameESJobs') #listType = list() #listType.append(sourceType) #listType.append(newType) #listType.append(jobsType) #check index if index.exists(index=indexName): #check types for check in checkList: if index.exists_type(index=indexName, doc_type=references[check]): logger.info('index %s and type %s exist', indexName, references[check]) else: logger.info('index %s exists but type %s does not', indexName, references[check]) return False return True else: logger.info('index %s does not exist', indexName) return False
def update(typeNameES, listId): logger.info('bulkOp.update launched') hippoCfg = getHippoConf() es = getES() now = strftime("%Y%m%dT%H%M%S%z") indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') # k is a generator expression that produces # dict to update every doc wich id is in listId k = ({ '_op_type': 'update', '_index': indexNameES, '_type': typeNameES, 'doc': { 'lastQuery': now }, '_id': id } for id in listId) res = helpers.bulk(es, k) logger.info('bulkOp.update res: %s', res) #res looks like #(2650, []) logger.info('bulkOp.update end') return res[0]
def littleMsearch(coreIntelligence, typeNameES, listParsedData): logger.info('searchIntel.littleMsearch launched') cfg = getHippoConf() indexNameES = cfg.get('elasticsearch', 'indexNameES') es = getES() #forging littleMsearch request #request to be sent to ES for littleMsearch req = list() #request header req_head = {'index': indexNameES, 'type': typeNameES} coreIntelligence = coreIntelligence #in the previous example, coreIntelligence is 'domain' for element in listParsedData: req_body = { 'query': { 'bool': { 'must': [{ 'match': { coreIntelligence: element[coreIntelligence] } }] } } } req.extend([req_head, req_body]) #req will look like #[{ # 'index': 'hippocampe', # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' #}, { # 'query': { # 'bool': { # 'must': [{ # 'match': { # u 'domain': 'skandastech.com' # } # }] # } # } #}, { # 'index': 'hippocampe', # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' #}, { # 'query': { # 'bool': { # 'must': [{ # 'match': { # u 'domain': 'stie.pbsoedirman.com' # } # }] # } # } #}] res = es.msearch(body=req) # res will look like #{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0', # u'_index': u'hippocampe', # u'_score': 7.470799, # u'_source': {u'firstAppearance': u'20160325T145146+0100', # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', # u'lastAppearance': u'20160325T145146+0100', # u'source': u'https://openphish.com/feed.txt', # u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'}, # u'_type': u'openphishFree_feedURL'}], # u'max_score': 7.470799, # u'total': 1}, # u'timed_out': False, # u'took': 124}, # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [], u'max_score': None, u'total': 0}, # u'timed_out': False, # u'took': 107}, # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0', # u'_index': u'hippocampe', # u'_score': 7.4480977, # u'_source': {u'firstAppearance': u'20160325T145117+0100', # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', # u'lastAppearance': u'20160325T145117+0100', # u'source': u'https://openphish.com/feed.txt', # u'url': u'http://www.rutzcellars.com/dd-dd/art/'}, # u'_type': u'openphishFree_feedURL'}], # u'max_score': 7.4480977, # u'total': 1}, # u'timed_out': False, # u'took': 117}]} logger.info('searchIntel.littleMsearch end') return res
def __init__(self): super(IndexNew, self).__init__() cfg = getHippoConf() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = 'new'