def test_fmt_bufio(self):
data = {
'foo': [
{
'fqdn': 'com',
'tld': True
},
{
'fqdn': 'woot.com'
},
],
'bar': [
{
'fqdn': 'vertex.link',
'tld': 0
},
],
'newp': [
{
'fqdn': 'newp.com',
'tld': 0
},
],
}
buf = io.BytesIO(json.dumps(data).encode())
lines = list(s_encoding.iterdata(buf, format='json'))
self.len(1, lines)
self.eq(lines[0], data)
def test_fmt_lines(self):
with self.getTestDir() as dirn:
linep = s_common.genpath(dirn, 'woot.txt')
with s_common.genfile(linep) as fd:
fd.write(testlines)
with s_common.genfile(linep) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='lines'))
self.len(2, lines)
e = ['foo.com', 'bar.com']
self.eq(lines, e)
def test_fmt_mpk(self):
with self.getTestDir() as dirn:
fp = s_common.genpath(dirn, 'woot.mpk')
with s_common.genfile(fp) as fd:
fd.write(s_msgpack.en('foo.com'))
fd.write(s_msgpack.en('bar.com'))
with s_common.genfile(fp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='mpk'))
self.len(2, lines)
e = ['foo.com', 'bar.com']
self.eq(lines, e)
def test_fmt_xml(self):
with self.getTestDir() as dirn:
xmlp = s_common.genpath(dirn, 'woot.xml')
with s_common.genfile(xmlp) as fd:
fd.write(testxml)
with s_common.genfile(xmlp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='xml'))
self.len(1, lines)
line = lines[0]
elem = line.get('data')
self.len(3, list(elem))
def test_fmt_csv(self):
with self.getTestDir() as dirn:
csvp = s_common.genpath(dirn, 'woot.csv')
with s_common.genfile(csvp) as fd:
fd.write(b'#THIS IS A COMMENT\n')
fd.write(b'foo.com,1.2.3.4\n')
fd.write(b'vertex.link,5.6.7.8\n')
fd = s_common.genfile(csvp)
genr = s_encoding.iterdata(fd, format='csv')
lines = list(genr)
self.len(3, lines)
self.eq(lines[0], ['#THIS IS A COMMENT'])
self.eq(lines[1], ['foo.com', '1.2.3.4'])
self.eq(lines[2], ['vertex.link', '5.6.7.8'])
self.true(fd.closed)
# keep fd open if we want it left open
with s_common.genfile(csvp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='csv'))
self.false(fd.closed)
def test_fmt_json(self):
testjson = b'''{
"fqdn": "spooky.com",
"ipv4": "192.168.1.1",
"aliases": ["foo", "bar", "baz"]
}'''
with self.getTestDir() as dirn:
jsonp = s_common.genpath(dirn, 'woot.json')
with s_common.genfile(jsonp) as fd:
fd.write(testjson)
with s_common.genfile(jsonp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='json'))
e = [{
'fqdn': 'spooky.com',
'ipv4': '192.168.1.1',
'aliases': ['foo', 'bar', 'baz']
}]
self.eq(lines, e)
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.jsonl'):
with s_common.genfile(path) as fd:
item = list(s_encoding.iterdata(fd, False, format='jsonl'))
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
def test_fmt_jsonl(self):
testjsonl = b'''{"fqdn": "spooky.com", "ipv4": "192.168.1.1"}
{"fqdn":"spookier.com", "ipv4":"192.168.1.2"}'''
with self.getTestDir() as dirn:
jsonlp = s_common.genpath(dirn, 'woot.jsonl')
with s_common.genfile(jsonlp) as fd:
fd.write(testjsonl)
with s_common.genfile(jsonlp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='jsonl'))
self.len(2, lines)
e = [
{
'ipv4': '192.168.1.1',
'fqdn': 'spooky.com'
},
{
'ipv4': '192.168.1.2',
'fqdn': 'spookier.com'
},
]
self.eq(lines, e)
def test_fmt_addFormat(self):
def _fmt_woot_old(fd, info):
yield 'old.bad'
def _fmt_woot(fd, info):
yield 'woot'
opts = {'mode': 'r', 'encoding': 'utf8'}
s_encoding.addFormat('woot', _fmt_woot_old, opts)
self.nn(s_encoding.fmtyielders.get('woot'))
s_encoding.addFormat('woot', _fmt_woot, opts) # last write wins
with self.getTestDir() as dirn:
wootp = s_common.genpath(dirn, 'woot.woot')
with s_common.genfile(wootp) as fd:
fd.write(b'this is irrelevant, we always yield woot :-)')
with s_common.genfile(wootp) as fd:
lines = list(
s_encoding.iterdata(fd, close_fd=False, format='woot'))
self.len(1, lines)
self.eq(lines, ['woot'])
async def test_log(self):
def check_locs_cleanup(cobj):
keys = list(cobj.locs.keys())
for key in keys:
if key.startswith('log:'):
self.fail(f'Key with "log:" prefix found. [{key}]')
async with self.getTestCoreAndProxy() as (realcore, core):
with self.getTestSynDir() as dirn:
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --format jsonl')
fp = cmdr.locs.get('log:fp')
await cmdr.runCmdLine(
'storm --editformat splices [test:str=hi :tick=2018 +#haha.hehe]'
)
await cmdr.runCmdLine(
'storm --editformat nodeedits [test:str=hi2 :tick=2018 +#haha.hehe]'
)
await cmdr.runCmdLine('storm [test:comp=(42, bar)]')
# Try calling on a second time - this has no effect on the
# state of cmdr, but prints a warning
await cmdr.runCmdLine('log --on --format jsonl')
await cmdr.runCmdLine('log --off')
await cmdr.fini()
check_locs_cleanup(cmdr)
self.true(outp.expect('Starting logfile'))
e = 'Must call --off to disable current file before starting a new file.'
self.true(outp.expect(e))
self.true(outp.expect('Closing logfile'))
self.true(os.path.isfile(fp))
# Ensure that jsonl is how the data was saved
with s_common.genfile(fp) as fd:
genr = s_encoding.iterdata(fd,
close_fd=False,
format='jsonl')
objs = list(genr)
self.eq(objs[0][0], 'init')
nodeedits = [m for m in objs if m[0] == 'node:edits']
self.ge(len(nodeedits), 2)
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
# Our default format is mpk
fp = os.path.join(dirn, 'loggyMcLogFace.mpk')
await cmdr.runCmdLine(f'log --on --edits-only --path {fp}')
fp = cmdr.locs.get('log:fp')
await cmdr.runCmdLine(
'storm [test:str="I am a message!" :tick=1999 +#oh.my] ')
await cmdr.runCmdLine('log --off')
await cmdr.fini()
check_locs_cleanup(cmdr)
self.true(os.path.isfile(fp))
with s_common.genfile(fp) as fd:
genr = s_encoding.iterdata(fd,
close_fd=False,
format='mpk')
objs = list(genr)
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
# Our default format is mpk
fp = os.path.join(dirn, 'loggyMcNodeFace.mpk')
await cmdr.runCmdLine(f'log --on --nodes-only --path {fp}')
fp = cmdr.locs.get('log:fp')
await cmdr.runCmdLine(
'storm [test:str="I am a message!" :tick=1999 +#oh.my] ')
await cmdr.runCmdLine('log --off')
await cmdr.fini()
check_locs_cleanup(cmdr)
self.true(os.path.isfile(fp))
with s_common.genfile(fp) as fd:
genr = s_encoding.iterdata(fd,
close_fd=False,
format='mpk')
objs = list(genr)
self.eq(objs[0][0], ('test:str', 'I am a message!'))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --off')
await cmdr.fini()
self.true(
outp.expect(
'log: error: argument --off: not allowed with argument --on'
))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log')
await cmdr.fini()
self.true(
outp.expect(
'log: error: one of the arguments --on --off is required'
))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --edits-only --nodes-only')
await cmdr.fini()
e = 'log: error: argument --nodes-only: not allowed with argument --edits-only'
self.true(outp.expect(e))
# Bad internal state
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --nodes-only')
cmdr.locs['log:fmt'] = 'newp'
with self.getAsyncLoggerStream(
'synapse.cmds.cortex',
'Unknown encoding format: newp') as stream:
await cmdr.runCmdLine('storm test:str')
self.true(await stream.wait(2))
await cmdr.fini()
async def test_log(self):
async with self.getTestDmon('dmoncore') as dmon:
with self.getTestSynDir() as dirn:
async with await self.agetTestProxy(dmon, 'core') as core:
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --format jsonl')
fp = cmdr.locs.get('log:fp')
await cmdr.runCmdLine(
'storm [teststr=hi :tick=2018 +#haha.hehe]')
await cmdr.runCmdLine('log --off')
cmdr.fini()
self.true(outp.expect('Starting logfile'))
self.true(outp.expect('Closing logfile'))
self.true(os.path.isfile(fp))
# Ensure that jsonl is how the data was saved
with s_common.genfile(fp) as fd:
genr = s_encoding.iterdata(fd,
close_fd=False,
format='jsonl')
objs = list(genr)
self.eq(objs[0][0], 'init')
async with await self.agetTestProxy(dmon, 'core') as core:
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
# Our default format is mpk
fp = os.path.join(dirn, 'loggyMcLogFace.mpk')
await cmdr.runCmdLine(
f'log --on --splices-only --path {fp}')
fp = cmdr.locs.get('log:fp')
await cmdr.runCmdLine(
'storm [teststr="I am a message!" :tick=1999 +#oh.my] '
)
await cmdr.runCmdLine('log --off')
cmdr.fini()
self.true(os.path.isfile(fp))
with s_common.genfile(fp) as fd:
genr = s_encoding.iterdata(fd,
close_fd=False,
format='mpk')
objs = list(genr)
self.eq(objs[0][0], 'node:add')
async with await self.agetTestProxy(dmon, 'core') as core:
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log --on --off')
cmdr.fini()
self.true(outp.expect('Pick one'))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('log')
cmdr.fini()
self.true(outp.expect('Pick one'))