def test_ingest_cortex_registration(self):
data1 = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]}
data2 = {'foo': [{'fqdn': 'weallfloat.com', 'haha': ['fooboat', 'sewer']}]}
data3 = {'foo': [{'fqdn': 'woot.com', 'haha': ['fooboat', 'sewer']}]}
ingest_def = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'tags': [
{'iter': 'haha/*',
'vars': [
['tag', {'regex': '^foo'}],
],
'template': 'zoom.{{tag}}'}
],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
ingest_def2 = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
gest = s_ingest.Ingest(ingest_def)
gest2 = s_ingest.Ingest(ingest_def2)
with s_cortex.openurl('ram:///') as core:
ret1 = s_ingest.register_ingest(core=core, gest=gest, evtname='ingest:test')
ret2 = s_ingest.register_ingest(core=core, gest=gest2, evtname='ingest:test2', ret_func=True)
self.none(ret1)
self.true(callable(ret2))
# Dump data into the core an event at a time.
core.fire('ingest:test', data=data1)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(isinstance(node, tuple))
self.true(s_tufo.tagged(node, 'zoom.foofoo'))
self.false(s_tufo.tagged(node, 'zoom.barbar'))
core.fire('ingest:test', data=data2)
node = core.getTufoByProp('inet:fqdn', 'weallfloat.com')
self.true(isinstance(node, tuple))
self.true(s_tufo.tagged(node, 'zoom.fooboat'))
self.false(s_tufo.tagged(node, 'zoom.sewer'))
# Try another ingest attached to the core. This won't have any tags applied.
core.fire('ingest:test2', data=data3)
node = core.getTufoByProp('inet:fqdn', 'woot.com')
self.true(isinstance(node, tuple))
self.false(s_tufo.tagged(node, 'zoom.fooboat'))
self.false(s_tufo.tagged(node, 'zoom.sewer'))
def test_ingest_files(self):
# s_encoding.encode('utf8,base64,-utf8','
data = {'foo': ['dmlzaQ==']}
info = {
'ingest': {
'iters': [[
"foo/*", {
'tags': ['woo.woo'],
'files': [{
'mime': 'hehe/haha',
'decode': '+utf8,base64'
}],
}
]]
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
tufo = core.getTufoByProp('file:bytes',
'442f602ecf8230b2a59a44b4f845be27')
self.true(s_tufo.tagged(tufo, 'woo.woo'))
self.eq(tufo[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes:mime'), 'hehe/haha')
# do it again with an outer iter and non-iter path
data = {'foo': ['dmlzaQ==']}
info = {
'ingest': {
'tags': ['woo.woo'],
'iters': [
('foo/*', {
'files': [{
'mime': 'hehe/haha',
'decode': '+utf8,base64'
}],
}),
]
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
tufo = core.getTufoByProp('file:bytes',
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes:mime'), 'hehe/haha')
self.true(s_tufo.tagged(tufo, 'woo.woo'))
def test_ingest_pivot(self):
data = {'foo': ['dmlzaQ=='], 'bar': ['1b2e93225959e3722efed95e1731b764']}
info = {'ingest': {
'tags': ['woo.woo'],
'iters': [
['foo/*', {
'files': [{'mime': 'hehe/haha', 'decode': '+utf8,base64'}],
}],
['bar/*', {
'forms': [('hehe:haha', {'pivot': ('file:bytes:md5', 'file:bytes')})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
core.addTufoForm('hehe:haha', ptype='file:bytes')
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('hehe:haha', '442f602ecf8230b2a59a44b4f845be27'))
def test_ingest_xref(self):
data = {
"fhash": "e844031e309ce19520f563c38239190f59e7e1a67d4302eaea563c3ad36a8d81",
"ip": "8.8.8.8"
}
ingdef = {
'ingest': {
'vars': [
[
"fhash",
{
"path": "fhash"
}
],
[
"ip",
{
"path": "ip"
}
]
],
'forms': [
[
"file:bytes:sha256",
{
"var": "fhash",
"savevar": "file_guid"
}
],
[
"inet:ipv4",
{
"var": "ip",
"savevar": "ip_guid"
}
],
[
"file:txtref",
{
"template": "{{file_guid}}|inet:ipv4|{{ip_guid}}"
}
]
]
}
}
with s_cortex.openurl('ram://') as core:
ingest = s_ingest.Ingest(info=ingdef)
ingest.ingest(core=core, data=data)
nodes1 = core.eval('file:bytes')
self.eq(len(nodes1), 1)
nodes2 = core.eval('inet:ipv4')
self.eq(len(nodes2), 1)
nodes3 = core.eval('file:txtref')
self.eq(len(nodes3), 1)
xrefnode = nodes3[0]
self.eq(xrefnode[1].get('file:txtref:file'), nodes1[0][1].get('file:bytes'))
self.eq(xrefnode[1].get('file:txtref:xref:inet:ipv4'), nodes2[0][1].get('inet:ipv4'))
def test_ingest_embed_props(self):
with s_cortex.openurl('ram://') as core:
info = {
"embed": [
{
"props": {"sfx": 1},
"nodes": [
["inet:fqdn", [
"com",
"net",
"org"
]],
],
}
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:fqdn', 'com'))
self.nn(core.getTufoByProp('inet:fqdn', 'net'))
self.nn(core.getTufoByProp('inet:fqdn', 'org'))
self.eq(3, len(core.eval('inet:fqdn:sfx=1')))
def test_ingest_embed_nodes(self):
with s_cortex.openurl('ram://') as core:
info = {
"embed": [
{
"nodes": [
["inet:fqdn", [
"woot.com",
"vertex.link"
]],
["inet:ipv4", [
"1.2.3.4",
0x05060708,
]],
]
}
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:ipv4', 0x01020304))
self.nn(core.getTufoByProp('inet:ipv4', 0x05060708))
self.nn(core.getTufoByProp('inet:fqdn', 'woot.com'))
self.nn(core.getTufoByProp('inet:fqdn', 'vertex.link'))
def test_ingest_varprop(self):
data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['zoom', {
'path': 'fqdn'
}]],
'forms': [('inet:fqdn', {
'var': 'zoom'
})],
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('inet:fqdn', 'vertex.link'))
def test_ingest_condform_with_missing_var(self):
data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['hehe', {
'path': 'heho'
}]],
'forms': [('inet:fqdn', {
'path': 'fqdn',
'cond': 'hehe != 3'
})],
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.none(core.getTufoByProp('inet:fqdn', 'vertex.link'))
def test_ingest_embed_pernode_tagsprops(self):
with s_cortex.openurl('ram://') as core:
info = {
"embed": [
{
"nodes": [
["inet:fqdn", [
["link", {"props": {"tld": 1}}],
]],
["inet:netuser", [
["rootkit.com/metr0", {"props": {"email": "*****@*****.**"}}],
["twitter.com/invisig0th", {"props": {"email": "*****@*****.**"}}]
]],
["inet:email", [
["*****@*****.**", {"tags": ["foo.bar", "baz.faz"]}]
]]
]
}
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:netuser', 'rootkit.com/metr0'))
self.nn(core.getTufoByProp('inet:netuser', 'twitter.com/invisig0th'))
self.eq(1, len(core.eval('inet:netuser:email="*****@*****.**"')))
self.eq(1, len(core.eval('inet:netuser:email="*****@*****.**"')))
node = core.eval('inet:email*tag=foo.bar')[0]
self.eq(node[1].get('inet:email'), '*****@*****.**')
def test_ingest_taginfo(self):
with self.getRamCore() as core:
info = {
'ingest': {
'iters': [
('foo/*', {
'vars': [['baz', {
'path': '1'
}]],
'tags': [{
'template': 'foo.bar.{{baz}}'
}],
'forms': [('inet:fqdn', {
'path': '0'
})]
}),
]
}
}
data = {'foo': [('vertex.link', 'LULZ')]}
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.len(1, core.eval('inet:fqdn*tag="foo.bar.lulz"'))
def test_ingest_xml_search(self):
with self.getRamCore() as core:
with self.getTestDir() as path:
xpth = os.path.join(path, 'woot.xml')
with genfile(xpth) as fd:
fd.write(testxml)
info = {
'sources': [(xpth, {
'open': {
'format': 'xml'
},
'ingest': {
'iters': [
['~badurl', {
'forms': [
('inet:url', {}),
],
}],
]
}
})]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:url', 'http://evil.com/'))
self.nn(core.getTufoByProp('inet:url', 'http://badguy.com/'))
def test_ingest_template(self):
data = {'foo': [('1.2.3.4', 'vertex.link')]}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['ipv4', {
'path': '0'
}], ['fqdn', {
'path': '1'
}]],
'forms': [('inet:dns:a', {
'template': '{{fqdn}}/{{ipv4}}'
})]
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('inet:ipv4', 0x01020304))
self.nn(core.getTufoByProp('inet:fqdn', 'vertex.link'))
self.nn(core.getTufoByProp('inet:dns:a', 'vertex.link/1.2.3.4'))
def test_ingest_iter_object(self):
data = {
'foo': {
'boosh': {
'fqdn': 'vertex.link'
},
'woot': {
'fqdn': 'foo.bario'
}
}
}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [
['bar', {'path': '0'}],
['fqdn', {'path': '1/fqdn'}]
],
'forms': [('inet:fqdn', {'template': '{{bar}}.{{fqdn}}'})]
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'boosh.vertex.link')
self.nn(node)
node = core.getTufoByProp('inet:fqdn', 'woot.foo.bario')
self.nn(node)
def test_ingest_embed_tags(self):
with s_cortex.openurl('ram://') as core:
info = {
"embed": [
{
"tags": [
"hehe.haha.hoho"
],
"nodes":[
["inet:fqdn", [
"rofl.com",
"laughitup.edu"
]],
["inet:email", [
"*****@*****.**"
]]
]
}
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:fqdn', 'rofl.com'))
self.nn(core.getTufoByProp('inet:fqdn', 'laughitup.edu'))
self.nn(core.getTufoByProp('inet:email', '*****@*****.**'))
self.eq(2, len(core.eval('inet:fqdn*tag=hehe.haha.hoho')))
self.eq(1, len(core.eval('inet:email*tag=hehe.haha.hoho')))
def test_ingest_tagiter(self):
data = {'foo': [{'fqdn': 'vertex.link', 'haha': ['foo', 'bar']}]}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['zoom', {
'path': 'fqdn'
}]],
'tags': [{
'iter': 'haha/*',
'vars': [['zoomtag', {}]],
'template': 'zoom.{{zoomtag}}'
}],
'forms': [('inet:fqdn', {
'path': 'fqdn'
})],
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'zoom.foo'))
self.true(s_tufo.tagged(node, 'zoom.bar'))
def test_ingest_addFormat(self):
def _fmt_woot_old(fd, info):
yield 'old.bad'
def _fmt_woot(fd, info):
yield 'woot'
opts = {'mode': 'r', 'encoding': 'utf8'}
s_ingest.addFormat('woot', _fmt_woot_old, opts)
self.nn(s_ingest.fmtyielders.get('woot'))
s_ingest.addFormat('woot', _fmt_woot, opts) # last write wins
with s_cortex.openurl('ram://') as core:
with self.getTestDir() as path:
wootpath = os.path.join(path, 'woot.woot')
with genfile(wootpath) as fd:
fd.write(b'this is irrelevant, we always yield woot :-)')
info = {
'sources': (
(wootpath, {'open': {'format': 'woot'}, 'ingest': {
'tags': ['hehe.haha'],
'forms': [
('inet:fqdn', {}),
]
}}),
)
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:fqdn', 'woot'))
def test_ingest_jsonl(self):
testjsonl = b'''{"fqdn": "spooky.com", "ipv4": "192.168.1.1"}
{"fqdn":"spookier.com", "ipv4":"192.168.1.2"}'''
with s_cortex.openurl('ram://') as core:
with self.getTestDir() as path:
xpth = os.path.join(path, 'woot.jsonl')
with genfile(xpth) as fd:
fd.write(testjsonl)
info = {
'sources': [(xpth,
{'open': {'format': 'jsonl'},
'ingest': {
'tags': ['leljsonl'],
'iters':[
['fqdn', {
'forms': [('inet:fqdn', {})]
}],
['ipv4', {
'forms': [('inet:ipv4', {})]
}]
]}})]}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:fqdn', 'spooky.com'))
self.nn(core.getTufoByProp('inet:ipv4', '192.168.1.1'))
self.nn(core.getTufoByProp('inet:fqdn', 'spookier.com'))
self.nn(core.getTufoByProp('inet:ipv4', '192.168.1.2'))
def test_ingest_tag_template_whif(self):
data = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'tags': [
{'iter': 'haha/*',
'vars': [
['tag', {'regex': '^foo'}],
],
'template': 'zoom.{{tag}}'}
],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'zoom.foofoo'))
self.false(s_tufo.tagged(node, 'zoom.barbar'))
def test_ingest_condtag(self):
data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [['hehe', {'path': 'hehe'}]],
'tags': [{'value': 'hehe.haha', 'cond': 'hehe != 3'}],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.false(s_tufo.tagged(node, 'hehe.haha'))
data['foo'][0]['hehe'] = 9
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'hehe.haha'))
def test_ingest_condform(self):
data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [['hehe', {'path': 'hehe'}]],
'forms': [('inet:fqdn', {'path': 'fqdn', 'cond': 'hehe != 3'})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.none(core.getTufoByProp('inet:fqdn', 'vertex.link'))
data['foo'][0]['hehe'] = 9
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('inet:fqdn', 'vertex.link'))
def test_ingest_lines(self):
with s_cortex.openurl('ram://') as core:
with self.getTestDir() as path:
path = os.path.join(path, 'woot.txt')
with genfile(path) as fd:
fd.write(testlines)
info = {
'sources': [
(path, {
'open': {'format': 'lines'},
'ingest': {
'forms': [['inet:fqdn', {}]]
}
})
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:fqdn', 'foo.com'))
self.nn(core.getTufoByProp('inet:fqdn', 'bar.com'))
def test_ingest_xml(self):
with s_cortex.openurl('ram://') as core:
with self.getTestDir() as path:
xpth = os.path.join(path, 'woot.xml')
with genfile(xpth) as fd:
fd.write(testxml)
info = {
'sources': [
(xpth, {
'open': {'format': 'xml'},
'ingest': {
'tags': ['lolxml'],
'iters':[
['data/dnsa', {
#explicitly opt fqdn into the optional attrib syntax
'vars': [
['fqdn', {'path': '$fqdn'}],
['ipv4', {'path': 'ipv4'}],
],
'forms': [
('inet:dns:a', {'template': '{{fqdn}}/{{ipv4}}'}),
]
}],
['data/urls/*', {
'forms': [
('inet:url', {}),
],
}],
]
}
})
]
}
gest = s_ingest.Ingest(info)
gest.ingest(core)
self.nn(core.getTufoByProp('inet:dns:a', 'foo.com/1.2.3.4'))
self.nn(core.getTufoByProp('inet:dns:a', 'bar.com/5.6.7.8'))
self.nn(core.getTufoByProp('inet:url', 'http://evil.com/'))
self.nn(core.getTufoByProp('inet:url', 'http://badguy.com/'))
self.eq(len(core.eval('inet:dns:a*tag=lolxml')), 2)
self.eq(len(core.eval('inet:url*tag=lolxml')), 2)
def test_ingest_reqprops(self):
tick = now()
ingdef = {
"ingest": {
"forms": [
[
"inet:dns:look",
{
"props": {
"time": {
"var": "time"
},
"a": {
"template": "{{fqdn}}/{{ipv4}}"
}
},
"value": "*"
}
]
],
"vars": [
[
"time",
{
"path": "time"
}
],
[
"fqdn",
{
"path": "fqdn"
}
],
[
"ipv4",
{
"path": "ipv4"
}
]
]
}
}
data = {"time": tick, "ipv4": "1.2.3.4", "fqdn": "vertex.link"}
with self.getRamCore() as core:
ingest = s_ingest.Ingest(ingdef)
ingest.ingest(core, data=data)
nodes = core.eval('inet:dns:look')
self.len(1, nodes)
node = nodes[0]
self.eq(node[1].get('inet:dns:look:time'), tick)
self.eq(node[1].get('inet:dns:look:a'), 'vertex.link/1.2.3.4')
def test_ingest_basic(self):
with self.getRamCore() as core:
info = {
'ingest': {
'iters': (('foo/*/fqdn', {
'forms': [
('inet:fqdn', {
'props': {
'sfx': {
'path': '../tld'
},
}
}),
]
}), ),
},
}
data = {
'foo': [
{
'fqdn': 'com',
'tld': True
},
{
'fqdn': 'woot.com'
},
],
'bar': [
{
'fqdn': 'vertex.link',
'tld': 0
},
],
'newp': [
{
'fqdn': 'newp.com',
'tld': 0
},
],
}
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.eq(
core.getTufoByProp('inet:fqdn', 'com')[1].get('inet:fqdn:sfx'),
1)
self.eq(
core.getTufoByProp('inet:fqdn',
'woot.com')[1].get('inet:fqdn:zone'), 1)
self.none(core.getTufoByProp('inet:fqdn', 'newp.com'))
def test_ingest_basic_bufio(self):
with s_cortex.openurl('ram://') as core:
info = {
'ingest': {
'iters': (
('foo/*/fqdn', {
'forms': [
('inet:fqdn', {
'props': {
'sfx': {'path': '../tld'},
}
}),
]
}),
),
},
'open': {
'format': 'json'
}
}
data = {
'foo': [
{'fqdn': 'com', 'tld': True},
{'fqdn': 'woot.com'},
],
'bar': [
{'fqdn': 'vertex.link', 'tld': 0},
],
'newp': [
{'fqdn': 'newp.com', 'tld': 0},
],
}
buf = s_compat.BytesIO(json.dumps(data).encode())
ingdata = s_ingest.iterdata(fd=buf, **info.get('open'))
gest = s_ingest.Ingest(info)
for _data in ingdata:
gest.ingest(core, data=_data)
self.eq(core.getTufoByProp('inet:fqdn', 'com')[1].get('inet:fqdn:sfx'), 1)
self.eq(core.getTufoByProp('inet:fqdn', 'woot.com')[1].get('inet:fqdn:zone'), 1)
self.none(core.getTufoByProp('inet:fqdn', 'newp.com'))
def test_ingest_xref(self):
data = {
"fhash":
"e844031e309ce19520f563c38239190f59e7e1a67d4302eaea563c3ad36a8d81",
"ip": "8.8.8.8"
}
ingdef = {
'ingest': {
'vars': [["fhash", {
"path": "fhash"
}], ["ip", {
"path": "ip"
}]],
'forms': [[
"file:bytes:sha256", {
"var": "fhash",
"savevar": "file_guid"
}
], ["inet:ipv4", {
"var": "ip",
"savevar": "ip_guid"
}],
[
"file:txtref", {
"template":
"({{file_guid}},inet:ipv4={{ip_guid}})"
}
]]
}
}
with self.getRamCore() as core:
ingest = s_ingest.Ingest(info=ingdef)
ingest.ingest(core=core, data=data)
nodes1 = core.eval('file:bytes')
self.len(1, nodes1)
nodes2 = core.eval('inet:ipv4')
self.len(1, nodes2)
nodes3 = core.eval('file:txtref')
self.len(1, nodes3)
xrefnode = nodes3[0]
self.eq(xrefnode[1].get('file:txtref:file'),
nodes1[0][1].get('file:bytes'))
self.eq(xrefnode[1].get('file:txtref:xref'), 'inet:ipv4=8.8.8.8')
self.eq(xrefnode[1].get('file:txtref:xref:prop'), 'inet:ipv4')
self.eq(xrefnode[1].get('file:txtref:xref:intval'),
nodes2[0][1].get('inet:ipv4'))
def _parseGestConfig(self, gest_data=None): # type: (dict) -> None
if gest_data is None:
return
self.gest_name = gest_data.get('name')
gestdef = gest_data.get('definition')
self.gest_open = gestdef.get('open')
self.gest = s_ingest.Ingest(gestdef)
# Blow up on missing data early
if not self.gest_name:
raise s_common.NoSuchName(
name='name', mesg='API Ingest definition is missing its name.')
if not self.gest_open:
raise s_common.NoSuchName(
name='open',
mesg='Ingest definition is missing a open directive.')
def test_ingest_savevar(self):
data = {'foo': [{'md5': '9e107d9d372bb6826bd81d3542a419d6',
'sha1': '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12',
'sha256': 'd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592',
'signame': 'Meme32.LazyDog',
'vendor': 'memeSec'},
{'md5': 'e4d909c290d0fb1ca068ffaddf22cbd0',
'sha1': '408d94384216f890ff7a0c3528e8bed1e0b01621',
'sha256': 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c',
'signame': 'Meme32.LazyDog.Puntuation',
'vendor': 'memeSec'}
]
}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [
['md5', {'path': 'md5'}],
['sha1', {'path': 'sha1'}],
['sha256', {'path': 'sha256'}],
['sig_name', {'path': 'signame'}],
['vendor', {'path': 'vendor'}]
],
'forms': [
['file:bytes:sha256',
{'props': {'md5': {'var': 'md5'}, 'sha1': {'var': 'sha1'}, 'sha256': {'var': 'sha256'}},
'var': 'sha256',
'savevar': 'file_guid'}],
['it:av:filehit', {'template': '{{file_guid}}/{{vendor}}/{{sig_name}}'}]
]
}],
],
}}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('file:bytes:sha256',
'd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592'))
self.nn(core.getTufoByProp('it:av:filehit:sig', 'memesec/meme32.lazydog'))
self.nn(core.getTufoByProp('it:av:sig:sig', 'meme32.lazydog'))
self.nn(core.getTufoByProp('file:bytes:sha256',
'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c'))
self.nn(core.getTufoByProp('it:av:filehit:sig', 'memesec/meme32.lazydog.puntuation'))
self.nn(core.getTufoByProp('it:av:sig:sig', 'meme32.lazydog.puntuation'))
self.len(2, core.getTufosByProp('it:av:sig:org', 'memesec'))
def test_ingest_cast(self):
with self.getRamCore() as core:
info = {
'ingest': {
'iters': [
('foo/*', {
'forms': [('strform', {'path': '1', 'cast': 'str:lwr'})]
}),
]
}
}
data = {'foo': [('vertex.link', 'LULZ')]}
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
self.nn(core.getTufoByProp('strform', 'lulz'))
def test_ingest_iter_objectish_array(self):
data = {
'foo': [{
0: 'boosh',
1: {
'fqdn': 'vertex.link'
},
}, {
0: 'woot',
1: {
'fqdn': 'foo.bario'
}
}]
}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['bar', {
'path': '0'
}], ['fqdn', {
'path': '1/fqdn'
}]],
'forms': [('inet:fqdn', {
'template': '{{bar}}.{{fqdn}}'
})]
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'boosh.vertex.link')
self.nn(node)
node = core.getTufoByProp('inet:fqdn', 'woot.foo.bario')
self.nn(node)
