def index(cfgPath, listData):
logger.info('bulkOp.index launched')
hippoCfg = getHippoConf()
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
cfg = getConf(cfgPath)
typeNameES = cfg.get('elasticsearch', 'typeIntel')
#creating the index, only if does not exist
index = IndexIntel(cfgPath)
index.createIndexIntel()
es = getES()
k = ({
'_op_type': 'index',
'_index': indexNameES,
'_type': typeNameES,
'_source': data
} for data in listData)
res = helpers.bulk(es, k, raise_on_error=False)
#res = helpers.bulk(es,k, raise_on_exception=False)
#res = helpers.bulk(es,k)
logger.info('bulkOp.index res: %s', res)
logger.info('bulkOp.index end')
return res
def indexNew(coreIntelligence, listData):
logger.info('bulkOp.indexNew launched')
hippoCfg = getHippoConf()
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
typeNameES = hippoCfg.get('elasticsearch', 'typeNameESNew')
indexNew = IndexNew()
indexNew.createIndexNew()
es = getES()
k = ({
'_op_type': 'index',
'_index': indexNameES,
'_type': typeNameES,
'_source': {
'type': coreIntelligence,
'toSearch': data[coreIntelligence]
}
} for data in listData)
#k.next() gives:
#{'_op_type': 'index', '_index':'hippocampe', '_type':'new', '_source': {'typeIntel': 'ip', 'intelligence': '1.1.1.1'}
res = helpers.bulk(es, k)
logger.info('bulkOp.index res: %s', res)
logger.info('bulkOp.indexNew end')
return res[0]
def bigMsearch(coreIntelligence, listParsedData):
logger.info('searchIntel.bigMsearch launched')
es = getES()
cfg = getHippoConf()
indexNameES = cfg.get('elasticsearch', 'indexNameES')
req = list()
req_head = {'index': indexNameES}
coreIntelligence = coreIntelligence
for element in listParsedData:
req_body = {
'query': {
'bool': {
'must': [{
'match': {
coreIntelligence: element[coreIntelligence]
}
}]
}
}
}
req.extend([req_head, req_body])
res = es.msearch(body=req)
logger.info('searchIntel.bigMsearch end')
return res
def __init__(self, typeIntel):
cfg = getHippoConf()
self.typeIntel = typeIntel
self.es = getES()
self.docSearch = dict()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
#contains every distinct value from a field
self.size = int()
def __init__(self): """ Index class' constructor. """ self.indexNameES = str() self.typeNameES = str() self.docMapping = dict() self.es = getES()
def __init__(self):
cfg = getHippoConf()
self.docSearch = str()
self.matchResponse = str()
self.matchDict = dict()
self.es = getES()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew')
self.nbDoc = int()
def __init__(self, typeNameES):
"""
ObjToIndex class' constructor.
"""
cfg = getHippoConf()
self.es = getES()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = typeNameES
self.docSearch = dict()
self.size = int()
def __init__(self, field):
cfg = getHippoConf()
self.field = field
self.es = getES()
self.docSearch = dict()
self.matchResponse = dict()
self.indexName = cfg.get('elasticsearch', 'indexNameES')
#contains every distinct value from a field
self.distinctList = list()
#number of distinct values
self.size = int()
def __init__(self, typeIntel, ioc):
cfg = getHippoConf()
self.typeIntel = typeIntel
self.value = ioc
self.docMatch = str()
self.matchResponse = str()
self.matchList = list()
self.es = getES()
#data stored in index hippocampe, so search is only in this index
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
def __init__(self): """ ObjToIndex class' constructor. """ self.es = getES() self.idInES = str() self.indexNameES = str() self.typeNameES = str() self.docIndex = dict() self.docSearch = dict() self.docUpdate = dict() self.resSearch = dict()
def fixThisBrokenBullshit(source):
es = getES()
data = {'query': {'bool': {'must': [{'match': {'source': source}}]}}}
res = es.search(body=data)
for i in res['hits']['hits']:
if i["_source"]["idSource"] != "":
return i["_source"]["idSource"]
break
else:
continue
def update(typeNameES, listId):
logger.info('bulkOp.update launched')
hippoCfg = getHippoConf()
es = getES()
now = strftime("%Y%m%dT%H%M%S%z")
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
# k is a generator expression that produces
# dict to update every doc wich id is in listId
k = ({
'_op_type': 'update',
'_index': indexNameES,
'_type': typeNameES,
'doc': {
'lastQuery': now
},
'_id': id
} for id in listId)
res = helpers.bulk(es, k)
logger.info('bulkOp.update res: %s', res)
#res looks like
#(2650, [])
logger.info('bulkOp.update end')
return res[0]
def __init__(self, idSource): self.idSource = idSource self.docMatch = str() self.matchResponse = str() self.scoreSource = float() self.es = getES()
def littleMsearch(coreIntelligence, typeNameES, listParsedData):
logger.info('searchIntel.littleMsearch launched')
cfg = getHippoConf()
indexNameES = cfg.get('elasticsearch', 'indexNameES')
es = getES()
#forging littleMsearch request
#request to be sent to ES for littleMsearch
req = list()
#request header
req_head = {'index': indexNameES, 'type': typeNameES}
coreIntelligence = coreIntelligence
#in the previous example, coreIntelligence is 'domain'
for element in listParsedData:
req_body = {
'query': {
'bool': {
'must': [{
'match': {
coreIntelligence: element[coreIntelligence]
}
}]
}
}
}
req.extend([req_head, req_body])
#req will look like
#[{
# 'index': 'hippocampe',
# 'type': u 'malwaredomainsFree_dnsbhDOMAIN'
#}, {
# 'query': {
# 'bool': {
# 'must': [{
# 'match': {
# u 'domain': 'skandastech.com'
# }
# }]
# }
# }
#}, {
# 'index': 'hippocampe',
# 'type': u 'malwaredomainsFree_dnsbhDOMAIN'
#}, {
# 'query': {
# 'bool': {
# 'must': [{
# 'match': {
# u 'domain': 'stie.pbsoedirman.com'
# }
# }]
# }
# }
#}]
res = es.msearch(body=req)
# res will look like
#{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0',
# u'_index': u'hippocampe',
# u'_score': 7.470799,
# u'_source': {u'firstAppearance': u'20160325T145146+0100',
# u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
# u'lastAppearance': u'20160325T145146+0100',
# u'source': u'https://openphish.com/feed.txt',
# u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'},
# u'_type': u'openphishFree_feedURL'}],
# u'max_score': 7.470799,
# u'total': 1},
# u'timed_out': False,
# u'took': 124},
# {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [], u'max_score': None, u'total': 0},
# u'timed_out': False,
# u'took': 107},
# {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0',
# u'_index': u'hippocampe',
# u'_score': 7.4480977,
# u'_source': {u'firstAppearance': u'20160325T145117+0100',
# u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
# u'lastAppearance': u'20160325T145117+0100',
# u'source': u'https://openphish.com/feed.txt',
# u'url': u'http://www.rutzcellars.com/dd-dd/art/'},
# u'_type': u'openphishFree_feedURL'}],
# u'max_score': 7.4480977,
# u'total': 1},
# u'timed_out': False,
# u'took': 117}]}
logger.info('searchIntel.littleMsearch end')
return res