def __init__(self, cfgPath):
"""
Source class' constructor.
:param config: Config object that contains informations about the feed.
:type config: Config object instance.
"""
self.cfgPath = cfgPath
#conf from hippocampe/core/conf/feeds
conf = getConf.getConf(cfgPath)
#conf from hippocampe/core/conf/hippo
cfg = getConf.getHippoConf()
super(Source, self).__init__()
self.source = conf.get('source', 'url')
self.firstQuery = str()
self.lastQuery = str()
self.description = conf.get('source', 'description')
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = cfg.get('elasticsearch', 'typeNameESSource')
self.score = conf.getint('source', 'score')
self.coreIntelligence = conf.get('source', 'coreIntelligence')
self.typeNameESIntel = conf.get('elasticsearch', 'typeIntel')
self.validityDate = conf.get('source', 'validityDate')
self.useByDate = conf.get('source', 'useBydate')
def indexNew(coreIntelligence, listData):
logger.info('bulkOp.indexNew launched')
hippoCfg = getHippoConf()
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
typeNameES = hippoCfg.get('elasticsearch', 'typeNameESNew')
indexNew = IndexNew()
indexNew.createIndexNew()
es = getES()
k = ({
'_op_type': 'index',
'_index': indexNameES,
'_type': typeNameES,
'_source': {
'type': coreIntelligence,
'toSearch': data[coreIntelligence]
}
} for data in listData)
#k.next() gives:
#{'_op_type': 'index', '_index':'hippocampe', '_type':'new', '_source': {'typeIntel': 'ip', 'intelligence': '1.1.1.1'}
res = helpers.bulk(es, k)
logger.info('bulkOp.index res: %s', res)
logger.info('bulkOp.indexNew end')
return res[0]
def index(cfgPath, listData):
logger.info('bulkOp.index launched')
hippoCfg = getHippoConf()
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
cfg = getConf(cfgPath)
typeNameES = cfg.get('elasticsearch', 'typeIntel')
#creating the index, only if does not exist
index = IndexIntel(cfgPath)
index.createIndexIntel()
es = getES()
k = ({
'_op_type': 'index',
'_index': indexNameES,
'_type': typeNameES,
'_source': data
} for data in listData)
res = helpers.bulk(es, k, raise_on_error=False)
#res = helpers.bulk(es,k, raise_on_exception=False)
#res = helpers.bulk(es,k)
logger.info('bulkOp.index res: %s', res)
logger.info('bulkOp.index end')
return res
def bigMsearch(coreIntelligence, listParsedData):
logger.info('searchIntel.bigMsearch launched')
es = getES()
cfg = getHippoConf()
indexNameES = cfg.get('elasticsearch', 'indexNameES')
req = list()
req_head = {'index': indexNameES}
coreIntelligence = coreIntelligence
for element in listParsedData:
req_body = {
'query': {
'bool': {
'must': [{
'match': {
coreIntelligence: element[coreIntelligence]
}
}]
}
}
}
req.extend([req_head, req_body])
res = es.msearch(body=req)
logger.info('searchIntel.bigMsearch end')
return res
def __init__(self, typeIntel):
cfg = getHippoConf()
self.typeIntel = typeIntel
self.es = getES()
self.docSearch = dict()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
#contains every distinct value from a field
self.size = int()
def __init__(self):
cfg = getHippoConf()
self.docSearch = str()
self.matchResponse = str()
self.matchDict = dict()
self.es = getES()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew')
self.nbDoc = int()
def __init__(self):
super(Job, self).__init__()
cfg = getHippoConf()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs')
self.status = str()
self.startTime = str()
self.endTime = str()
self.duration = str()
self.report = dict()
def __init__(self, typeNameES):
"""
ObjToIndex class' constructor.
"""
cfg = getHippoConf()
self.es = getES()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = typeNameES
self.docSearch = dict()
self.size = int()
def __init__(self, field):
cfg = getHippoConf()
self.field = field
self.es = getES()
self.docSearch = dict()
self.matchResponse = dict()
self.indexName = cfg.get('elasticsearch', 'indexNameES')
#contains every distinct value from a field
self.distinctList = list()
#number of distinct values
self.size = int()
def __init__(self, typeIntel, ioc):
cfg = getHippoConf()
self.typeIntel = typeIntel
self.value = ioc
self.docMatch = str()
self.matchResponse = str()
self.matchList = list()
self.es = getES()
#data stored in index hippocampe, so search is only in this index
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
def __init__(self):
super(IndexJob, self).__init__()
cfg = getHippoConf()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs')
#the mapping has to be indicated at the creation of the index
#the confMapping, is the mapping indicated in conf file for each field
#it has to be parsed bellow
self.confMapping = dict()
def __init__(self, cfgPath):
"""
IndexSource class' constructor.
:param source: shadowBook process's configuration
:type config: Config object instance
"""
conf = getHippoConf()
super(IndexSource, self).__init__()
self.indexNameES = conf.get('elasticsearch', 'indexNameES')
self.typeNameES = conf.get('elasticsearch', 'typeNameESSource')
def __init__(self, typeIntel, intelligence):
"""
Ioc class' constructor.
:param config: Config object that contains informations about the ioc.
:type config: Config object instance.
:param source: Ioc's Source object.
:type source: Source object instance.
"""
super(NewIntel, self).__init__()
cfg = getHippoConf()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = 'new'
self.typeIntel = typeIntel
self.intelligence = intelligence
def __init__(self, cfgPath):
"""
IndexIOC class' constructor.
:param config: shadowBook process's configuration
:type config: Config object instance
"""
super(IndexIntel, self).__init__()
self.conf = getConf.getConf(cfgPath)
hippoConf = getConf.getHippoConf()
self.indexNameES = hippoConf.get('elasticsearch', 'indexNameES')
self.typeNameES = self.conf.get('elasticsearch', 'typeIntel')
#the mapping has to be indicated at the creation of the index
#the confMapping, is the mapping indicated in conf file for each field
#it has to be parsed bellow
self.confMapping = dict()
def checkData(checkList):
#checkList is the list of types to check
#check if the hippocampe's index exists in ES
#and check if ES type exists according to checkList
logger.info('ES.checkData launched')
logger.info(checkList)
ES = getES()
index = IndicesClient(ES)
cfg = getHippoConf()
indexName = cfg.get('elasticsearch', 'indexNameES')
#references contains the name of types used in Hippocampe
references = dict()
references['sourceType'] = cfg.get('elasticsearch', 'typeNameESSource')
references['newType'] = cfg.get('elasticsearch', 'typeNameESNew')
references['jobsType'] = cfg.get('elasticsearch', 'typeNameESJobs')
#listType = list()
#listType.append(sourceType)
#listType.append(newType)
#listType.append(jobsType)
#check index
if index.exists(index=indexName):
#check types
for check in checkList:
if index.exists_type(index=indexName, doc_type=references[check]):
logger.info('index %s and type %s exist', indexName,
references[check])
else:
logger.info('index %s exists but type %s does not', indexName,
references[check])
return False
return True
else:
logger.info('index %s does not exist', indexName)
return False
def update(typeNameES, listId):
logger.info('bulkOp.update launched')
hippoCfg = getHippoConf()
es = getES()
now = strftime("%Y%m%dT%H%M%S%z")
indexNameES = hippoCfg.get('elasticsearch', 'indexNameES')
# k is a generator expression that produces
# dict to update every doc wich id is in listId
k = ({
'_op_type': 'update',
'_index': indexNameES,
'_type': typeNameES,
'doc': {
'lastQuery': now
},
'_id': id
} for id in listId)
res = helpers.bulk(es, k)
logger.info('bulkOp.update res: %s', res)
#res looks like
#(2650, [])
logger.info('bulkOp.update end')
return res[0]
def littleMsearch(coreIntelligence, typeNameES, listParsedData):
logger.info('searchIntel.littleMsearch launched')
cfg = getHippoConf()
indexNameES = cfg.get('elasticsearch', 'indexNameES')
es = getES()
#forging littleMsearch request
#request to be sent to ES for littleMsearch
req = list()
#request header
req_head = {'index': indexNameES, 'type': typeNameES}
coreIntelligence = coreIntelligence
#in the previous example, coreIntelligence is 'domain'
for element in listParsedData:
req_body = {
'query': {
'bool': {
'must': [{
'match': {
coreIntelligence: element[coreIntelligence]
}
}]
}
}
}
req.extend([req_head, req_body])
#req will look like
#[{
# 'index': 'hippocampe',
# 'type': u 'malwaredomainsFree_dnsbhDOMAIN'
#}, {
# 'query': {
# 'bool': {
# 'must': [{
# 'match': {
# u 'domain': 'skandastech.com'
# }
# }]
# }
# }
#}, {
# 'index': 'hippocampe',
# 'type': u 'malwaredomainsFree_dnsbhDOMAIN'
#}, {
# 'query': {
# 'bool': {
# 'must': [{
# 'match': {
# u 'domain': 'stie.pbsoedirman.com'
# }
# }]
# }
# }
#}]
res = es.msearch(body=req)
# res will look like
#{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0',
# u'_index': u'hippocampe',
# u'_score': 7.470799,
# u'_source': {u'firstAppearance': u'20160325T145146+0100',
# u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
# u'lastAppearance': u'20160325T145146+0100',
# u'source': u'https://openphish.com/feed.txt',
# u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'},
# u'_type': u'openphishFree_feedURL'}],
# u'max_score': 7.470799,
# u'total': 1},
# u'timed_out': False,
# u'took': 124},
# {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [], u'max_score': None, u'total': 0},
# u'timed_out': False,
# u'took': 107},
# {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
# u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0',
# u'_index': u'hippocampe',
# u'_score': 7.4480977,
# u'_source': {u'firstAppearance': u'20160325T145117+0100',
# u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
# u'lastAppearance': u'20160325T145117+0100',
# u'source': u'https://openphish.com/feed.txt',
# u'url': u'http://www.rutzcellars.com/dd-dd/art/'},
# u'_type': u'openphishFree_feedURL'}],
# u'max_score': 7.4480977,
# u'total': 1},
# u'timed_out': False,
# u'took': 117}]}
logger.info('searchIntel.littleMsearch end')
return res
def __init__(self):
super(IndexNew, self).__init__()
cfg = getHippoConf()
self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
self.typeNameES = 'new'
