def index(cfgPath, listData): logger.info('bulkOp.index launched') hippoCfg = getHippoConf() indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') cfg = getConf(cfgPath) typeNameES = cfg.get('elasticsearch', 'typeIntel') #creating the index, only if does not exist index = IndexIntel(cfgPath) index.createIndexIntel() es = getES() k = ({ '_op_type': 'index', '_index': indexNameES, '_type': typeNameES, '_source': data } for data in listData) res = helpers.bulk(es, k, raise_on_error=False) #res = helpers.bulk(es,k, raise_on_exception=False) #res = helpers.bulk(es,k) logger.info('bulkOp.index res: %s', res) logger.info('bulkOp.index end') return res
def indexNew(coreIntelligence, listData): logger.info('bulkOp.indexNew launched') hippoCfg = getHippoConf() indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') typeNameES = hippoCfg.get('elasticsearch', 'typeNameESNew') indexNew = IndexNew() indexNew.createIndexNew() es = getES() k = ({ '_op_type': 'index', '_index': indexNameES, '_type': typeNameES, '_source': { 'type': coreIntelligence, 'toSearch': data[coreIntelligence] } } for data in listData) #k.next() gives: #{'_op_type': 'index', '_index':'hippocampe', '_type':'new', '_source': {'typeIntel': 'ip', 'intelligence': '1.1.1.1'} res = helpers.bulk(es, k) logger.info('bulkOp.index res: %s', res) logger.info('bulkOp.indexNew end') return res[0]
def bigMsearch(coreIntelligence, listParsedData): logger.info('searchIntel.bigMsearch launched') es = getES() cfg = getHippoConf() indexNameES = cfg.get('elasticsearch', 'indexNameES') req = list() req_head = {'index': indexNameES} coreIntelligence = coreIntelligence for element in listParsedData: req_body = { 'query': { 'bool': { 'must': [{ 'match': { coreIntelligence: element[coreIntelligence] } }] } } } req.extend([req_head, req_body]) res = es.msearch(body=req) logger.info('searchIntel.bigMsearch end') return res
def __init__(self, typeIntel): cfg = getHippoConf() self.typeIntel = typeIntel self.es = getES() self.docSearch = dict() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') #contains every distinct value from a field self.size = int()
def __init__(self): """ Index class' constructor. """ self.indexNameES = str() self.typeNameES = str() self.docMapping = dict() self.es = getES()
def __init__(self): cfg = getHippoConf() self.docSearch = str() self.matchResponse = str() self.matchDict = dict() self.es = getES() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew') self.nbDoc = int()
def __init__(self, typeNameES): """ ObjToIndex class' constructor. """ cfg = getHippoConf() self.es = getES() self.indexNameES = cfg.get('elasticsearch', 'indexNameES') self.typeNameES = typeNameES self.docSearch = dict() self.size = int()
def __init__(self, field): cfg = getHippoConf() self.field = field self.es = getES() self.docSearch = dict() self.matchResponse = dict() self.indexName = cfg.get('elasticsearch', 'indexNameES') #contains every distinct value from a field self.distinctList = list() #number of distinct values self.size = int()
def __init__(self, typeIntel, ioc): cfg = getHippoConf() self.typeIntel = typeIntel self.value = ioc self.docMatch = str() self.matchResponse = str() self.matchList = list() self.es = getES() #data stored in index hippocampe, so search is only in this index self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
def __init__(self): """ ObjToIndex class' constructor. """ self.es = getES() self.idInES = str() self.indexNameES = str() self.typeNameES = str() self.docIndex = dict() self.docSearch = dict() self.docUpdate = dict() self.resSearch = dict()
def fixThisBrokenBullshit(source): es = getES() data = {'query': {'bool': {'must': [{'match': {'source': source}}]}}} res = es.search(body=data) for i in res['hits']['hits']: if i["_source"]["idSource"] != "": return i["_source"]["idSource"] break else: continue
def update(typeNameES, listId): logger.info('bulkOp.update launched') hippoCfg = getHippoConf() es = getES() now = strftime("%Y%m%dT%H%M%S%z") indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') # k is a generator expression that produces # dict to update every doc wich id is in listId k = ({ '_op_type': 'update', '_index': indexNameES, '_type': typeNameES, 'doc': { 'lastQuery': now }, '_id': id } for id in listId) res = helpers.bulk(es, k) logger.info('bulkOp.update res: %s', res) #res looks like #(2650, []) logger.info('bulkOp.update end') return res[0]
def __init__(self, idSource): self.idSource = idSource self.docMatch = str() self.matchResponse = str() self.scoreSource = float() self.es = getES()
def littleMsearch(coreIntelligence, typeNameES, listParsedData): logger.info('searchIntel.littleMsearch launched') cfg = getHippoConf() indexNameES = cfg.get('elasticsearch', 'indexNameES') es = getES() #forging littleMsearch request #request to be sent to ES for littleMsearch req = list() #request header req_head = {'index': indexNameES, 'type': typeNameES} coreIntelligence = coreIntelligence #in the previous example, coreIntelligence is 'domain' for element in listParsedData: req_body = { 'query': { 'bool': { 'must': [{ 'match': { coreIntelligence: element[coreIntelligence] } }] } } } req.extend([req_head, req_body]) #req will look like #[{ # 'index': 'hippocampe', # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' #}, { # 'query': { # 'bool': { # 'must': [{ # 'match': { # u 'domain': 'skandastech.com' # } # }] # } # } #}, { # 'index': 'hippocampe', # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' #}, { # 'query': { # 'bool': { # 'must': [{ # 'match': { # u 'domain': 'stie.pbsoedirman.com' # } # }] # } # } #}] res = es.msearch(body=req) # res will look like #{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0', # u'_index': u'hippocampe', # u'_score': 7.470799, # u'_source': {u'firstAppearance': u'20160325T145146+0100', # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', # u'lastAppearance': u'20160325T145146+0100', # u'source': u'https://openphish.com/feed.txt', # u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'}, # u'_type': u'openphishFree_feedURL'}], # u'max_score': 7.470799, # u'total': 1}, # u'timed_out': False, # u'took': 124}, # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [], u'max_score': None, u'total': 0}, # u'timed_out': False, # u'took': 107}, # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0', # u'_index': u'hippocampe', # u'_score': 7.4480977, # u'_source': {u'firstAppearance': u'20160325T145117+0100', # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', # u'lastAppearance': u'20160325T145117+0100', # u'source': u'https://openphish.com/feed.txt', # u'url': u'http://www.rutzcellars.com/dd-dd/art/'}, # u'_type': u'openphishFree_feedURL'}], # u'max_score': 7.4480977, # u'total': 1}, # u'timed_out': False, # u'took': 117}]} logger.info('searchIntel.littleMsearch end') return res