def test_ingest_tag_template_whif(self):
data = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'tags': [
{'iter': 'haha/*',
'vars': [
['tag', {'regex': '^foo'}],
],
'template': 'zoom.{{tag}}'}
],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'zoom.foofoo'))
self.false(s_tufo.tagged(node, 'zoom.barbar'))
def test_ingest_tagiter(self):
data = {'foo': [{'fqdn': 'vertex.link', 'haha': ['foo', 'bar']}]}
info = {
'ingest': {
'iters': [
[
"foo/*", {
'vars': [['zoom', {
'path': 'fqdn'
}]],
'tags': [{
'iter': 'haha/*',
'vars': [['zoomtag', {}]],
'template': 'zoom.{{zoomtag}}'
}],
'forms': [('inet:fqdn', {
'path': 'fqdn'
})],
}
],
],
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'zoom.foo'))
self.true(s_tufo.tagged(node, 'zoom.bar'))
def test_ingest_condtag(self):
data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]}
info = {'ingest': {
'iters': [
["foo/*", {
'vars': [['hehe', {'path': 'hehe'}]],
'tags': [{'value': 'hehe.haha', 'cond': 'hehe != 3'}],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
with s_cortex.openurl('ram://') as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.false(s_tufo.tagged(node, 'hehe.haha'))
data['foo'][0]['hehe'] = 9
gest.ingest(core, data=data)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(s_tufo.tagged(node, 'hehe.haha'))
def test_ingest_files(self):
# s_encoding.encode('utf8,base64,-utf8','
data = {'foo': ['dmlzaQ==']}
info = {
'ingest': {
'iters': [[
"foo/*", {
'tags': ['woo.woo'],
'files': [{
'mime': 'hehe/haha',
'decode': '+utf8,base64'
}],
}
]]
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
tufo = core.getTufoByProp('file:bytes',
'442f602ecf8230b2a59a44b4f845be27')
self.true(s_tufo.tagged(tufo, 'woo.woo'))
self.eq(tufo[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes:mime'), 'hehe/haha')
# do it again with an outer iter and non-iter path
data = {'foo': ['dmlzaQ==']}
info = {
'ingest': {
'tags': ['woo.woo'],
'iters': [
('foo/*', {
'files': [{
'mime': 'hehe/haha',
'decode': '+utf8,base64'
}],
}),
]
}
}
with self.getRamCore() as core:
gest = s_ingest.Ingest(info)
gest.ingest(core, data=data)
tufo = core.getTufoByProp('file:bytes',
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(tufo[1].get('file:bytes:mime'), 'hehe/haha')
self.true(s_tufo.tagged(tufo, 'woo.woo'))
def test_ingest_cortex_registration(self):
data1 = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]}
data2 = {'foo': [{'fqdn': 'weallfloat.com', 'haha': ['fooboat', 'sewer']}]}
data3 = {'foo': [{'fqdn': 'woot.com', 'haha': ['fooboat', 'sewer']}]}
ingest_def = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'tags': [
{'iter': 'haha/*',
'vars': [
['tag', {'regex': '^foo'}],
],
'template': 'zoom.{{tag}}'}
],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
ingest_def2 = {'ingest': {
'iters': [
["foo/*", {
'vars': [['zoom', {'path': 'fqdn'}]],
'forms': [('inet:fqdn', {'path': 'fqdn'})],
}],
],
}}
gest = s_ingest.Ingest(ingest_def)
gest2 = s_ingest.Ingest(ingest_def2)
with s_cortex.openurl('ram:///') as core:
ret1 = s_ingest.register_ingest(core=core, gest=gest, evtname='ingest:test')
ret2 = s_ingest.register_ingest(core=core, gest=gest2, evtname='ingest:test2', ret_func=True)
self.none(ret1)
self.true(callable(ret2))
# Dump data into the core an event at a time.
core.fire('ingest:test', data=data1)
node = core.getTufoByProp('inet:fqdn', 'vertex.link')
self.true(isinstance(node, tuple))
self.true(s_tufo.tagged(node, 'zoom.foofoo'))
self.false(s_tufo.tagged(node, 'zoom.barbar'))
core.fire('ingest:test', data=data2)
node = core.getTufoByProp('inet:fqdn', 'weallfloat.com')
self.true(isinstance(node, tuple))
self.true(s_tufo.tagged(node, 'zoom.fooboat'))
self.false(s_tufo.tagged(node, 'zoom.sewer'))
# Try another ingest attached to the core. This won't have any tags applied.
core.fire('ingest:test2', data=data3)
node = core.getTufoByProp('inet:fqdn', 'woot.com')
self.true(isinstance(node, tuple))
self.false(s_tufo.tagged(node, 'zoom.fooboat'))
self.false(s_tufo.tagged(node, 'zoom.sewer'))