def test_msgpack_unpk(self):
byts = b'\x92\xa4hehe\n' * 3
unpk = s_msgpack.Unpk()
rets = unpk.feed(byts)
self.eq(rets, [(7, ('hehe', 10))] * 3)
def test_msgpack_byte(self):
unpk = s_msgpack.Unpk()
self.len(0, unpk.feed(b'\xa4'))
self.len(0, unpk.feed(b'v'))
self.len(0, unpk.feed(b'i'))
self.len(0, unpk.feed(b's'))
self.eq(unpk.feed(b'i')[0], (5, 'visi'))
def test_msgpack_surrogates(self):
bads = '\u01cb\ufffd\ud842\ufffd\u0012'
obyts = s_msgpack.en(bads)
self.isinstance(obyts, bytes)
outs = s_msgpack.un(obyts)
self.eq(outs, bads)
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'test.mpk')
fd.write(obyts)
fd.close()
fd = s_common.genfile(fdir, 'test.mpk')
gen = s_msgpack.iterfd(fd)
items = [obj for obj in gen]
self.len(1, items)
self.eq(outs, bads)
fd.close()
unpk = s_msgpack.Unpk()
ret = unpk.feed(obyts)
self.len(1, ret)
self.eq([(13, bads)], ret)
def test_msgpack_types(self):
# This is a future-proofing test for msgpack to ensure that
buf = b'\x92\xa4hehe\x85\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12'
struct = (
'hehe',
{
'str': '1234',
'int': 1234,
'float': 12.34,
'bin': b'1234',
'realworld': '\u01cb\ufffd\ud842\ufffd\u0012'
}
)
unode = s_msgpack.un(buf)
self.eq(unode, struct)
# Ensure our use of msgpack.Unpacker can also handle this data
with self.getTestDir() as dirn:
with s_common.genfile(dirn, 'test.mpk') as fd:
fd.write(buf)
with s_common.genfile(dirn, 'test.mpk') as fd:
genr = s_msgpack.iterfd(fd)
objs = list(genr)
self.len(1, objs)
self.eq(objs[0], struct)
# Ensure that our streaming Unpk object can also handle this data
unpk = s_msgpack.Unpk()
objs = unpk.feed(buf)
self.len(1, objs)
self.eq(objs[0], (71, struct))
def test_msgpack_large_data(self):
big_string = s_const.mebibyte * 129 * 'V'
struct = ('test', {'key': big_string})
buf = s_msgpack.en(struct)
unpacked_struct = s_msgpack.un(buf)
self.eq(struct, unpacked_struct)
# Ensure our use of msgpack.Unpacker can also handle this data
with self.getTestDir() as dirn:
with s_common.genfile(dirn, 'test.mpk') as fd:
fd.write(buf)
with s_common.genfile(dirn, 'test.mpk') as fd:
genr = s_msgpack.iterfd(fd)
objs = list(genr)
self.len(1, objs)
self.eq(objs[0], struct)
# Ensure that our streaming Unpk object can also handle this data
unpk = s_msgpack.Unpk()
objs = unpk.feed(buf)
self.len(1, objs)
self.eq(objs[0], (135266320, struct))
async def iterMpkFile(self, sha256):
'''
Yield items from a .mpk message pack stream file.
'''
unpk = s_msgpack.Unpk()
async for byts in self.get(s_common.uhex(sha256)):
for _, item in unpk.feed(byts):
yield item
async def __anit__(self, reader, writer, info=None):
await s_base.Base.__anit__(self)
self.iden = s_common.guid()
self.reader = reader
self.writer = writer
self.rxqu = collections.deque()
self.sock = self.writer.get_extra_info('socket')
self.peercert = self.writer.get_extra_info('peercert')
self._drain_lock = asyncio.Lock()
if info is None:
info = {}
if not info.get('unix'):
# disable nagle ( to minimize latency for small xmit )
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# enable TCP keep alives...
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if hasattr(socket, 'TCP_KEEPIDLE'):
# start sending a keep alives after 3 sec of inactivity
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
3)
# send keep alives every 3 seconds once started
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
3)
# close the socket after 5 failed keep alives (15 sec)
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
self.info = info
self.hostname = self.info.get('hostname')
self.unpk = s_msgpack.Unpk()
async def fini():
self.writer.close()
try:
await self.writer.wait_closed()
except (BrokenPipeError, ConnectionResetError) as e:
logger.debug('Link error waiting on close: %s', str(e))
self.onfini(fini)
if self.hostname is not None:
if self.hostname != self.getTlsPeerCn():
mesg = f'Expected: {self.hostname} Got: {self.getTlsPeerCn()}'
await self.fini()
raise s_exc.BadCertHost(mesg=mesg)
def checkTypes(self, enfunc):
# This is a future-proofing test for msgpack to ensure that
buf = b'\x92\xa4hehe\x85\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12'
struct = (
'hehe',
{
'str': '1234',
'int': 1234,
'float': 12.34,
'bin': b'1234',
'realworld': '\u01cb\ufffd\ud842\ufffd\u0012'
}
)
unode = s_msgpack.un(buf)
self.eq(unode, struct)
# Ensure our use of msgpack.Unpacker can also handle this data
with self.getTestDir() as dirn:
with s_common.genfile(dirn, 'test.mpk') as fd:
fd.write(buf)
with s_common.genfile(dirn, 'test.mpk') as fd:
genr = s_msgpack.iterfd(fd)
objs = list(genr)
self.len(1, objs)
self.eq(objs[0], struct)
# Ensure that our streaming Unpk object can also handle this data
unpk = s_msgpack.Unpk()
objs = unpk.feed(buf)
self.len(1, objs)
self.eq(objs[0], (71, struct))
# Generic isok helper
self.true(s_msgpack.isok(1))
self.true(s_msgpack.isok('1'))
self.true(s_msgpack.isok(1.1))
self.true(s_msgpack.isok(b'1'))
self.true(s_msgpack.isok(None))
self.true(s_msgpack.isok(True))
self.true(s_msgpack.isok(False))
self.true(s_msgpack.isok([1]))
self.true(s_msgpack.isok((1,)))
self.true(s_msgpack.isok({1: 1}))
# unpackage types
self.false(s_msgpack.isok({1, 2})) # set
self.false(s_msgpack.isok(print)) # function
buf2 = b'\x81\xc0\xcd\x04\xd2'
struct2 = {
None: 1234
}
ustruct2 = s_msgpack.un(buf2)
self.eq(ustruct2, struct2)
pbuf2 = enfunc(ustruct2)
self.eq(buf2, pbuf2)
async def iterMpkFile(self, sha256):
'''
Yield items from a MsgPack (.mpk) file in the Axon.
Args:
sha256 (str): The sha256 hash of the file as a string.
Yields:
Unpacked items from the bytes.
'''
unpk = s_msgpack.Unpk()
async for byts in self.get(s_common.uhex(sha256)):
for _, item in unpk.feed(byts):
yield item
def __init__(self, plex, sock):
Link.__init__(self, None)
self.plex = plex
self.sock = sock
self.txbuf = b''
self.txque = collections.deque() # (byts, info)
self.txlock = threading.Lock()
self.unpk = s_msgpack.Unpk()
self.flags = selectors.EVENT_READ
def fini():
self.plex._finiPlexSock(self.sock)
self.onfini(fini)
async def __anit__(self, reader, writer, info=None):
await s_base.Base.__anit__(self)
self.iden = s_common.guid()
self.reader = reader
self.writer = writer
self.rxqu = collections.deque()
self.sock = self.writer.get_extra_info('socket')
self._drain_lock = asyncio.Lock()
if info is None:
info = {}
if not info.get('unix'):
# disable nagle ( to minimize latency for small xmit )
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# enable TCP keep alives...
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if hasattr(socket, 'TCP_KEEPIDLE'):
# start sending a keep alives after 3 sec of inactivity
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
3)
# send keep alives every 3 seconds once started
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
3)
# close the socket after 5 failed keep alives (15 sec)
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
self.info = info
self.unpk = s_msgpack.Unpk()
async def fini():
self.writer.close()
self.onfini(fini)
def __init__(self, plex, sock):
Link.__init__(self, None)
self.plex = plex
self.sock = sock
self.fino = sock.fileno()
self.txbuf = b''
self.txque = collections.deque() #(byts, info)
self.txlock = threading.Lock()
self.unpk = s_msgpack.Unpk()
self.flags = select.EPOLLIN | select.EPOLLERR | select.EPOLLET
def fini():
self.plex._finiPlexSock(self.sock)
self.onfini(fini)
def __init__(self, fdir, path, xmit=None):
s_eventbus.EventBus.__init__(self)
self.wmin = fdir.getConfOpt('window:min')
self.wmax = fdir.getConfOpt('window:max')
self.wfill = fdir.getConfOpt('window:fill')
if not os.path.isfile(path):
with open(path, 'wb') as fd:
fd.write(b'\x00' * 8)
self.lock = threading.RLock()
self.dirty = False
self.caught = False
self.filling = s_atomic.CmpSet(False)
self.fdir = fdir
self._xmit = xmit
self.unpk = s_msgpack.Unpk()
self.dequ = collections.deque()
# open our state machine header atom
self.head = s_atomfile.openAtomFile(path, memok=True)
self.onfini(self.head.fini)
# the next expected ack
self.nack = struct.unpack('<Q', self.head.readoff(0, 8))[0]
# if the FifiDir has moved past us, catch up... :(
if self.nack < fdir.seqs[0]:
self.nack = fdir.seqs[0]
self.nseq = self.nack
self._initFifoAtom(self.nseq)
# what's the last sequence in the window...
self._fill()
async def _httpRespMsgpack(self):
byts = self.valu.get('body')
unpk = s_msgpack.Unpk()
for _, item in unpk.feed(byts):
yield item
async def test_spawncore(self):
# This test makes a real Cortex in a remote process, and then
# gets the spawninfo from that real Cortex in order to make a
# local SpawnCore. This avoids the problem of being unable to
# open lmdb environments multiple times by the same process
# and allows direct testing of the SpawnCore object.
mpctx = multiprocessing.get_context('spawn')
queue = mpctx.Queue()
event = mpctx.Event()
conf = {
'storm:log': True,
'storm:log:level': logging.INFO,
'modules': [('synapse.tests.utils.TestModule', {})],
}
queries = [
'[test:str="Cortex from the aether!"]',
]
with self.getTestDir() as dirn:
args = (dirn, conf, queries, queue, event)
proc = mpctx.Process(target=make_core, args=args)
try:
proc.start()
spawninfo = queue.get(timeout=30)
async with await s_spawn.SpawnCore.anit(spawninfo) as core:
root = await core.auth.getUserByName('root')
q = '''test:str
$lib.print($lib.str.format("{n}", n=$node.repr()))
| limit 1'''
item = {
'user': root.iden,
'view': list(core.views.keys())[0],
'storm': {
'query': q,
'opts': None,
}
}
# Test the storm implementation used by spawncore
msgs = await s_test.alist(s_spawn.storm(core, item))
podes = [m[1] for m in msgs if m[0] == 'node']
e = 'Cortex from the aether!'
self.len(1, podes)
self.eq(podes[0][0], ('test:str', e))
self.stormIsInPrint(e, msgs)
# Direct test of the _innerloop code.
todo = mpctx.Queue()
done = mpctx.Queue()
# Test poison - this would cause the corework to exit
todo.put(None)
self.none(await s_spawn._innerloop(core, todo, done))
# Test a real item with a link associated with it. This ends
# up getting a bunch of telepath message directly.
todo_item = item.copy()
link0, sock0 = await s_link.linksock()
todo_item['link'] = link0.getSpawnInfo()
todo.put(todo_item)
self.true(await s_spawn._innerloop(core, todo, done))
resp = done.get(timeout=12)
self.false(resp)
buf0 = sock0.recv(1024 * 16)
unpk = s_msgpack.Unpk()
msgs = [msg for (offset, msg) in unpk.feed(buf0)]
self.eq({'t2:genr', 't2:yield'}, {m[0] for m in msgs})
await link0.fini() # We're done with the link now
todo.close()
done.close()
# Test the workloop directly - this again just gets telepath
# messages back. This does use poison to kill the workloop.
todo = mpctx.Queue()
done = mpctx.Queue()
task = asyncio.create_task(
s_spawn._workloop(spawninfo, todo, done))
await asyncio.sleep(0.01)
link1, sock1 = await s_link.linksock()
todo_item = item.copy()
todo_item['link'] = link1.getSpawnInfo()
todo.put(todo_item)
# Don't block the IO loop!
resp = await s_coro.executor(done.get, timeout=12)
self.false(resp)
buf0 = sock1.recv(1024 * 16)
unpk = s_msgpack.Unpk()
msgs = [msg for (offset, msg) in unpk.feed(buf0)]
self.eq({'t2:genr', 't2:yield'}, {m[0] for m in msgs})
await link1.fini() # We're done with the link now
# Poison the queue - this should close the task
todo.put(None)
self.none(await asyncio.wait_for(task, timeout=12))
todo.close()
done.close()
finally:
queue.close()
event.set()
proc.join(12)
async def test_tools_storm(self):
async with self.getTestCore() as core:
await core.addTagProp('foo', ('int', {}), {})
pars = s_t_storm.getArgParser()
opts = pars.parse_args(('woot', ))
self.eq('woot', opts.cortex)
q = '$lib.model.ext.addFormProp(inet:ipv4, "_test:score", (int, $lib.dict()), $lib.dict())'
await core.callStorm(q)
async with core.getLocalProxy() as proxy:
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine(
'[inet:ipv4=1.2.3.4 +#foo=2012 +#bar +#baz:foo=10 :_test:score=7]'
)
text = str(outp)
self.isin('.....', text)
self.isin('inet:ipv4=1.2.3.4', text)
self.isin(':type = unicast', text)
self.isin(':_test:score = 7', text)
self.isin('.created = ', text)
self.isin('#bar', text)
self.isin('#baz:foo = 10', text)
self.isin(
'#foo = (2012/01/01 00:00:00.000, 2012/01/01 00:00:00.001)',
text)
self.isin('complete. 1 nodes in', text)
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('!quit')
self.isin('o/', str(outp))
self.true(scli.isfini)
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('!help')
self.isin('!quit', str(outp))
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('$lib.print(woot)')
self.isin('woot', str(outp))
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('$lib.warn(woot)')
self.isin('WARNING: woot', str(outp))
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('---')
self.isin(
"---\n ^\nSyntax Error: No terminal defined for '-' at line 1 col 2",
str(outp))
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('spin |' + ' ' * 80 + '---')
self.isin(
"... ---\n ^",
str(outp))
outp = s_output.OutPutStr()
async with await s_t_storm.StormCli.anit(proxy,
outp=outp) as scli:
await scli.runCmdLine('---' + ' ' * 80 + 'spin')
self.isin("--- ...\n ^",
str(outp))
lurl = core.getLocalUrl()
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, '$lib.print(woot)'), outp=outp)
self.isin('woot', str(outp))
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, f'!runfile --help'), outp=outp)
self.isin('Run a local storm file', str(outp))
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'foo.storm')
with open(path, 'wb') as fd:
fd.write(b'$lib.print(woot)')
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, f'!runfile {path}'), outp=outp)
self.isin(f'running storm file: {path}', str(outp))
self.isin('woot', str(outp))
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, f'!runfile /newp.storm'),
outp=outp)
self.isin(f'no such file: /newp.storm', str(outp))
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, f'!pushfile /newp'), outp=outp)
self.isin(f'no such file: /newp', str(outp))
outp = s_output.OutPutStr()
await s_t_storm.main((lurl, f'!pushfile {path}'), outp=outp)
text = str(outp)
self.isin(f'uploading file: {path}', text)
self.isin(':name = foo.storm', text)
self.isin(
':sha256 = c00adfcc316f8b00772cdbce2505b9ea539d74f42861801eceb1017a44344ed3',
text)
outp = s_output.OutPutStr()
path = os.path.join(dirn, 'bar.storm')
await s_t_storm.main((
lurl,
f'!pullfile c00adfcc316f8b00772cdbce2505b9ea539d74f42861801eceb1017a44344ed3 {path}'
),
outp=outp)
text = str(outp)
self.isin(
'downloading sha256: c00adfcc316f8b00772cdbce2505b9ea539d74f42861801eceb1017a44344ed3',
text)
self.isin(f'saved to: {path}', text)
with s_common.genfile(path) as fd:
self.isin('woot', fd.read().decode())
outp = s_output.OutPutStr()
await s_t_storm.main((
lurl,
f'!pullfile c11adfcc316f8b00772cdbce2505b9ea539d74f42861801eceb1017a44344ed3 {path}'
),
outp=outp)
text = str(outp)
self.isin('Axon does not contain the requested file.', text)
await scli.runCmdLine('[test:str=foo +#foo +#bar +#baz]')
await scli.runCmdLine('[test:str=bar +#foo +#bar +#baz]')
path = os.path.join(dirn, 'export1.nodes')
await s_t_storm.main((lurl, f'!export {path} {{ test:str }}'),
outp=outp)
text = str(outp)
self.isin(f'saved 2 nodes to: {path}', text)
with open(path, 'rb') as fd:
byts = fd.read()
podes = [i[1] for i in s_msgpack.Unpk().feed(byts)]
self.sorteq(('bar', 'foo'), [p[0][1] for p in podes])
for pode in podes:
self.sorteq(('bar', 'baz', 'foo'), pode[1]['tags'])
path = os.path.join(dirn, 'export2.nodes')
q = f'!export {path} {{ test:str }} --include-tags foo bar'
await s_t_storm.main((lurl, q), outp=outp)
text = str(outp)
self.isin(f'saved 2 nodes to: {path}', text)
with open(path, 'rb') as fd:
byts = fd.read()
podes = [i[1] for i in s_msgpack.Unpk().feed(byts)]
self.sorteq(('bar', 'foo'), [p[0][1] for p in podes])
for pode in podes:
self.sorteq(('bar', 'foo'), pode[1]['tags'])
path = os.path.join(dirn, 'export3.nodes')
q = f'!export {path} {{ test:str }} --no-tags'
await s_t_storm.main((lurl, q), outp=outp)
text = str(outp)
self.isin(f'saved 2 nodes to: {path}', text)
with open(path, 'rb') as fd:
byts = fd.read()
podes = [i[1] for i in s_msgpack.Unpk().feed(byts)]
self.sorteq(('bar', 'foo'), [p[0][1] for p in podes])
for pode in podes:
self.eq({}, pode[1]['tags'])
await s_t_storm.main((lurl, f'!export {path} {{ test:newp }}'),
outp=outp)
text = str(outp)
self.isin('No property named test:newp.', text)
