def test_ingest_basic_bufio(self):
with s_cortex.openurl('ram://') as core:
info = {
'ingest': {
'iters': (
('foo/*/fqdn', {
'forms': [
('inet:fqdn', {
'props': {
'sfx': {'path': '../tld'},
}
}),
]
}),
),
},
'open': {
'format': 'json'
}
}
data = {
'foo': [
{'fqdn': 'com', 'tld': True},
{'fqdn': 'woot.com'},
],
'bar': [
{'fqdn': 'vertex.link', 'tld': 0},
],
'newp': [
{'fqdn': 'newp.com', 'tld': 0},
],
}
buf = s_compat.BytesIO(json.dumps(data).encode())
ingdata = s_ingest.iterdata(fd=buf, **info.get('open'))
gest = s_ingest.Ingest(info)
for _data in ingdata:
gest.ingest(core, data=_data)
self.eq(core.getTufoByProp('inet:fqdn', 'com')[1].get('inet:fqdn:sfx'), 1)
self.eq(core.getTufoByProp('inet:fqdn', 'woot.com')[1].get('inet:fqdn:zone'), 1)
self.none(core.getTufoByProp('inet:fqdn', 'newp.com'))
def test_ingest_iterdata(self):
data = {
'foo': [
{'fqdn': 'com', 'tld': True},
{'fqdn': 'woot.com'},
],
'bar': [
{'fqdn': 'vertex.link', 'tld': 0},
],
'newp': [
{'fqdn': 'newp.com', 'tld': 0},
],
}
buf = s_compat.BytesIO(json.dumps(data).encode())
ingdata = s_ingest.iterdata(fd=buf, **{'format': 'json'})
for _data in ingdata:
self.nn(_data)
self.true(buf.closed)
buf2 = s_compat.BytesIO(json.dumps(data).encode())
# Leave the file descriptor open.
ingdata = s_ingest.iterdata(buf2,
close_fd=False,
**{'format': 'json'})
for _data in ingdata:
self.nn(_data)
self.false(buf2.closed)
buf2.close()
def __init__(self, fd=None, **opts):
s_eventbus.EventBus.__init__(self)
if fd is None:
fd = s_compat.BytesIO()
fd.seek(0, os.SEEK_END)
self.fd = fd
# track these to prevent context switches
self.size = fd.tell()
self.fdoff = self.size
self.opts = opts
self.fdlock = threading.Lock()
self.onfini(self._onFileFini)
def test_cortex_savefd(self):
fd = s_compat.BytesIO()
core0 = s_cortex.openurl('ram://', savefd=fd)
t0 = core0.formTufoByProp('foo','one', baz='faz')
t1 = core0.formTufoByProp('foo','two', baz='faz')
core0.setTufoProps(t0,baz='gronk')
core0.delTufoByProp('foo','two')
core0.fini()
fd.seek(0)
core1 = s_cortex.openurl('ram://', savefd=fd)
self.assertIsNone( core1.getTufoByProp('foo','two') )
t0 = core1.getTufoByProp('foo','one')
self.assertIsNotNone( t0 )
self.assertEqual( t0[1].get('foo:baz'), 'gronk' )
def __init__(self):
OutPutFd.__init__(self, s_compat.BytesIO())