async def test_it_app_snort(self):
async with self.getTestCore() as core:
hit = s_common.guid()
rule = s_common.guid()
flow = s_common.guid()
host = s_common.guid()
opts = {'vars': {'rule': rule, 'flow': flow, 'host': host, 'hit': hit}}
nodes = await core.eval('[ it:app:snort:rule=$rule :text=gronk :name=foo :version=1.2.3 ]', opts=opts).list()
self.len(1, nodes)
self.eq('foo', nodes[0].get('name'))
self.eq('gronk', nodes[0].get('text'))
self.eq(0x10000200003, nodes[0].get('version'))
nodes = await core.eval('[ it:app:snort:hit=$hit :rule=$rule :flow=$flow :src="tcp://[::ffff:0102:0304]:0" :dst="tcp://[::ffff:0505:0505]:80" :time=2015 :sensor=$host :version=1.2.3 ]', opts=opts).list()
self.len(1, nodes)
self.eq(rule, nodes[0].get('rule'))
self.eq(flow, nodes[0].get('flow'))
self.eq(host, nodes[0].get('sensor'))
self.eq(1420070400000, nodes[0].get('time'))
self.eq('tcp://[::ffff:1.2.3.4]:0', nodes[0].get('src'))
self.eq(0, nodes[0].get('src:port'))
self.eq(0x01020304, nodes[0].get('src:ipv4'))
self.eq('::ffff:1.2.3.4', nodes[0].get('src:ipv6'))
self.eq('tcp://[::ffff:5.5.5.5]:80', nodes[0].get('dst'))
self.eq(80, nodes[0].get('dst:port'))
self.eq(0x05050505, nodes[0].get('dst:ipv4'))
self.eq('::ffff:5.5.5.5', nodes[0].get('dst:ipv6'))
self.eq(0x10000200003, nodes[0].get('version'))
async def test_it_forms_simple(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode('it:hostname', 'Bobs Computer')
self.eq(node.ndef[1], 'bobs computer')
host0 = s_common.guid()
sver0 = s_common.guid()
hprops = {
'name': 'Bobs laptop',
'desc': 'Bobs paperweight',
'ipv4': '1.2.3.4',
'latlong': '0.0, 0.0',
'os': sver0,
}
node = await snap.addNode('it:host', host0, hprops)
self.eq(node.ndef[1], host0)
self.eq(node.get('name'), 'bobs laptop')
self.eq(node.get('desc'), 'Bobs paperweight')
self.eq(node.get('ipv4'), 0x01020304)
self.eq(node.get('latlong'), (0.0, 0.0))
self.eq(node.get('os'), sver0)
node = await snap.addNode('it:hosturl', (host0, 'http://vertex.ninja/cool.php'))
self.eq(node.ndef[1], (host0, 'http://vertex.ninja/cool.php'))
self.eq(node.get('host'), host0)
self.eq(node.get('url'), 'http://vertex.ninja/cool.php')
node = await snap.addNode('it:dev:int', 0x61C88648)
self.eq(node.ndef[1], 1640531528)
cprops = {
'desc': 'Some words.',
}
node = await snap.addNode('it:sec:cve', 'CVE-2013-9999', cprops)
self.eq(node.ndef[1], 'cve-2013-9999')
self.eq(node.get('desc'), 'Some words.')
hash0 = s_common.guid()
hprops = {
'salt': 'B33F',
'hash:md5': s_m_crypto.ex_md5,
'hash:sha1': s_m_crypto.ex_sha1,
'hash:sha256': s_m_crypto.ex_sha256,
'hash:sha512': s_m_crypto.ex_sha512,
'hash:lm': s_m_crypto.ex_md5,
'hash:ntlm': s_m_crypto.ex_md5,
'passwd': "I've got the same combination on my luggage!",
}
node = await snap.addNode('it:auth:passwdhash', hash0, hprops)
self.eq(node.ndef[1], hash0)
self.eq(node.get('salt'), 'b33f')
self.eq(node.get('hash:md5'), s_m_crypto.ex_md5)
self.eq(node.get('hash:sha1'), s_m_crypto.ex_sha1)
self.eq(node.get('hash:sha256'), s_m_crypto.ex_sha256)
self.eq(node.get('hash:sha512'), s_m_crypto.ex_sha512)
self.eq(node.get('hash:lm'), s_m_crypto.ex_md5)
self.eq(node.get('hash:ntlm'), s_m_crypto.ex_md5)
self.eq(node.get('passwd'), "I've got the same combination on my luggage!")
async def test_cortex_readonly_toplayer(self):
'''
Test the various ways to incorrectly put a remote layer as the write layer
'''
async with t_cortex.CortexTest.getTestCore(self) as core0:
async with t_cortex.CortexTest.getTestCore(self) as core1:
conf = {'url': core0.getLocalUrl('*/layer')}
layr = await core1.addLayer(type='remote', config=conf)
await self.asyncraises(s_exc.ReadOnlyLayer, core1.view.addLayer(layr, indx=0))
await self.asyncraises(s_exc.ReadOnlyLayer, core1.view.setLayers([layr.iden]))
await self.asyncraises(s_exc.ReadOnlyLayer, core1.addView(s_common.guid(), 'root', [layr.iden]))
view = await core1.addView(s_common.guid(), 'root', [])
await self.asyncraises(s_exc.ReadOnlyLayer, view.addLayer(layr))
async def test_cortex_remote_layer(self):
async with self.getRemoteCores() as (directcore, core):
# We write to directcore and make sure we can read from core
await s_common.aspin(directcore.eval('[ test:str=woot :tick=2015 ]'))
layr = core.view.layers[1]
self.true(isinstance(layr, s_remotelayer.RemoteLayer))
self.len(1, [x async for x in layr.iterFormRows('test:str')])
self.len(1, [x async for x in layr.iterPropRows('test:str', 'tick')])
iden = s_common.guid()
buid = s_common.buid(('test:str', 'woot'))
props = await layr.getBuidProps(buid)
self.eq('woot', props.get('*test:str'))
await layr.setOffset(iden, 200)
self.eq(200, await layr.getOffset(iden))
self.ne((), tuple([x async for x in layr.splices(0, 200)]))
self.eq(s_modelrev.maxvers, await layr.getModelVers())
await self.asyncraises(s_exc.SynErr, layr.setModelVers((9, 9, 9)))
async def __anit__(self, boss, task, name, user, info=None, root=None):
await s_base.Base.__anit__(self)
if info is None:
info = {}
self.boss = boss
task._syn_task = self
self.task = task # the real task...
self.iden = s_common.guid()
self.tick = s_common.now()
self.boss.tasks[self.iden] = self
if root is not None:
root.kids[self.iden] = self
self.task.add_done_callback(self._onTaskDone)
self.name = name
self.user = user
self.root = root
self.info = info
self.kids = {}
self.onfini(self._onTaskFini)
async def test_infotech_android(self):
softver = s_common.guid()
async with self.getTestCore() as core:
async with await core.snap() as snap:
perm = await snap.addNode('it:os:android:perm', 'Foo Perm')
self.eq(perm.ndef[1], 'Foo Perm')
intent = await snap.addNode('it:os:android:intent', 'Foo Intent')
self.eq(intent.ndef[1], 'Foo Intent')
ilisn = await snap.addNode('it:os:android:ilisten', (softver, 'Listen Test'))
self.eq(ilisn.get('app'), softver)
self.eq(ilisn.get('intent'), 'Listen Test')
ibcast = await snap.addNode('it:os:android:ibroadcast', (softver, 'Broadcast Test'))
self.eq(ibcast.get('app'), softver)
self.eq(ibcast.get('intent'), 'Broadcast Test')
reqperm = await snap.addNode('it:os:android:reqperm', (softver, 'Test Perm'))
self.eq(reqperm.get('app'), softver)
self.eq(reqperm.get('perm'), 'Test Perm')
valu = 'someIdentifier'
aaid = await snap.addNode('it:os:android:aaid', valu)
self.eq(aaid.ndef[1], 'someidentifier')
async def init(self, name, conf=None):
'''
Generate a new CryoTank with a given name or get an reference to an existing CryoTank.
Args:
name (str): Name of the CryoTank.
Returns:
CryoTank: A CryoTank instance.
'''
tank = self.tanks.get(name)
if tank is not None:
return tank
iden = s_common.guid()
logger.info('Creating new tank: %s', name)
path = s_common.genpath(self.dirn, 'tanks', iden)
tank = await CryoTank.anit(path, conf)
node = await self.names.open((name,))
await node.set((iden, conf))
self.tanks.put(name, tank)
return tank
async def test_it_reveng(self):
async with self.getTestCore() as core:
baseFile = s_common.ehex(s_common.buid())
fva = 0x404438
fopt = {'vars': {'file': baseFile,
'func': s_common.guid(),
'fva': fva}}
vstr = 'VertexBrandArtisanalBinaries'
sopt = {'vars': {'func': fopt['vars']['func'],
'string': vstr}}
fnode = await core.eval('[it:reveng:filefunc=($file, $func) :va=$fva]', opts=fopt).list()
snode = await core.eval('[it:reveng:funcstr=($func, $string)]', opts=sopt).list()
self.len(1, fnode)
self.eq(f'sha256:{baseFile}', fnode[0].get('file'))
self.eq(fva, fnode[0].get('va'))
self.len(1, snode)
self.eq(fnode[0].get('function'), snode[0].get('function'))
self.eq(vstr, snode[0].get('string'))
funcnode = await core.eval('it:reveng:function [ :name="FunkyFunction" :description="Test Function" ]').list()
self.len(1, funcnode)
self.eq("FunkyFunction", funcnode[0].get('name'))
self.eq("Test Function", funcnode[0].get('description'))
nodes = await core.eval(f'file:bytes={baseFile} -> it:reveng:filefunc :function -> it:reveng:funcstr:function').list()
self.len(1, nodes)
self.eq(vstr, nodes[0].get('string'))
async def __anit__(self):
await s_base.Base.__anit__(self)
self.items = {}
self.iden = s_common.guid()
self.user = None
def _getTankIden(self):
path = s_common.genpath(self.dirn, 'guid')
if os.path.isfile(path):
with open(path, 'r') as fd:
return fd.read().strip()
# legacy cell code...
cellpath = s_common.genpath(self.dirn, 'cell.guid')
if os.path.isfile(cellpath):
with open(cellpath, 'r') as fd:
iden = fd.read().strip()
with open(path, 'w') as fd:
fd.write(iden)
os.unlink(cellpath)
return iden
iden = s_common.guid()
with open(path, 'w') as fd:
fd.write(iden)
return iden
async def test_server(self):
with self.getTestDir() as dirn:
outp = self.getTestOutp()
guid = s_common.guid()
argv = [dirn,
'--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--name', 'telecore']
async with await s_s_cortex.main(argv, outp=outp) as core:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
# Make a node with the cortex
podes = await s_t_utils.alist(proxy.eval(f'[ou:org={guid}]'))
self.len(1, podes)
self.true(core.dmon.shared.get('telecore') is core)
# And data persists...
async with await s_s_cortex.main(argv, outp=outp) as core:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
podes = await s_t_utils.alist(proxy.eval(f'ou:org={guid}'))
self.len(1, podes)
async def test_cmdrcore(self):
async with self.getTestCoreAndProxy() as (realcore, core):
outp = self.getTestOutp()
async with await s_jupyter.CmdrCore.anit(core, outp=outp) as cmdrcore:
podes = await cmdrcore.eval('[test:str=beep]',
num=1, cmdr=False)
self.len(1, podes)
self.false(outp.expect('[test:str=beep]', throw=False))
mesgs = await cmdrcore.storm('[test:str=boop]',
num=1, cmdr=True)
self.true(outp.expect('[test:str=boop]', throw=False))
podes = [m[1] for m in mesgs if m[0] == 'node']
self.gt(len(mesgs), len(podes))
self.len(1, podes)
self.eq(podes[0][0], ('test:str', 'boop'))
# Opts works for cmdr=False
podes = await cmdrcore.eval('[test:str=$foo]',
{'vars': {'foo': 'duck'}},
num=1, cmdr=False)
self.len(1, podes)
self.eq(podes[0][0], ('test:str', 'duck'))
# Opts does not work with cmdr=True - we have no way to plumb it through.
with self.getAsyncLoggerStream('synapse.cortex',
'Error during storm execution') as stream:
ret = await cmdrcore.eval('[test:str=$foo]',
{'vars': {'foo': 'fowl'}},
cmdr=True)
await stream.wait(1)
self.eq(ret, [])
# Assertion based tests
podes = await cmdrcore.eval('test:int', num=0)
self.len(0, podes)
podes = await cmdrcore.eval('test:str', num=3)
self.len(3, podes)
await self.asyncraises(AssertionError, cmdrcore.eval('test:str', num=1))
# Feed function for data loading
data = [
(('test:int', 137), {}),
]
guid = s_common.guid()
ret = await cmdrcore.addFeedData('syn.nodes', data, (guid, 1))
self.eq(ret, 2)
podes = await cmdrcore.eval('test:int=137',
num=1, cmdr=False)
self.len(1, podes)
# Raw cmdline test
async with self.getTestCoreAndProxy() as (realcore, core):
outp = self.getTestOutp()
async with await s_jupyter.CmdrCore.anit(core, outp=outp) as cmdrcore:
await cmdrcore.runCmdLine('help')
self.true(outp.expect('cli> help'))
self.true(outp.expect('List commands and display help output.'))
async def add(self, useriden, query: str, reqs, incunit=None, incvals=None):
'''
Persistently adds an appointment
Args:
query (str):
storm query to run
reqs (Union[None, Dict[TimeUnit, Union[int, Tuple[int]], List[...]):
one or more dicts of the fixed aspects of the appointment. dict value may be a single or multiple.
May be an empty dict or None.
incunit (Union[None, TimeUnit]):
the unit that changes for recurring, or None for non-recurring. It is an error for this value to match
a key in reqdict.
incvals (Union[None, int, Iterable[int]): count of units of incunit or explicit day of week or day of month.
Not allowed for incunit == None, required for others (1 would be a typical
value)
Notes:
For values in reqs that are lists and incvals if a list, all combinations of all values (the product) are
used
Returns:
iden of new appointment
'''
iden = s_common.guid()
recur = incunit is not None
indx = self._next_indx
self._next_indx += 1
if reqs is None:
reqs = {}
if not query:
raise ValueError('empty query')
if not reqs and incunit is None:
raise ValueError('at least one of reqs and incunit must be non-empty')
if incunit is not None and incvals is None:
raise ValueError('incvals must be non-None if incunit is non-None')
if isinstance(reqs, Mapping):
reqs = [reqs]
# Find all combinations of values in reqdict values and incvals values
recs = []
for req in reqs:
reqdicts = self._dictproduct(req)
if not isinstance(incvals, Iterable):
incvals = (incvals, )
recs.extend(ApptRec(rd, incunit, v) for (rd, v) in itertools.product(reqdicts, incvals))
appt = _Appt(iden, recur, indx, query, useriden, recs)
self._addappt(iden, appt)
await self._storeAppt(appt)
return iden
async def test_server(self):
with self.getTestDir() as dirn:
outp = self.getTestOutp()
guid = s_common.guid()
argv = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--name', 'univtest',
]
argu = list(argv)
argu.extend(['synapse.cortex.Cortex', dirn])
# Start a cortex with the universal loader
async with await s_s_univ.main(argu, outp=outp) as core:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
podes = await s_t_utils.alist(proxy.eval(f'[ou:org={guid}]'))
self.len(1, podes)
self.eq('cortex', await proxy.getCellType())
self.true(core.dmon.shared.get('univtest') is core)
# And data persists... and can be seen with the regular synapse cortex server
argu = list(argv)
argu.append(dirn)
async with await s_s_cortex.main(argu, outp=outp) as core:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
podes = await s_t_utils.alist(proxy.eval(f'ou:org={guid}'))
self.len(1, podes)
argu = list(argv)
argu.extend(['synapse.lib.cell.Cell', dirn])
# Start a cortex as a regular Cell
async with await s_s_univ.main(argu, outp=outp) as cell:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
self.eq('cell', await proxy.getCellType())
argu = list(argv)
argu.extend(['synapse.tests.test_lib_cell.EchoAuth', dirn])
# Or start the Cortex off a a EchoAuth (don't do this in practice...)
async with await s_s_univ.main(argu, outp=outp) as cell:
async with await s_telepath.openurl(f'cell://{dirn}') as proxy:
self.eq('echoauth', await proxy.getCellType())
argu = list(argv)
argu.extend(['synapse.lib.newp.Newp', dirn])
with self.raises(s_exc.NoSuchCtor):
async with await s_s_univ.main(argu, outp=outp) as core:
pass
argu = ['synapse.lib.cell.Cell', dirn,
'--telepath', 'tcp://127.0.0.1:9999999/',
'--https', '0',
'--name', 'telecore']
# Coverage test, for a bad configuration
with self.raises(OverflowError):
obj = await s_s_univ.main(argu, outp=outp)
async def lotsofwrites(path):
os.remove(pathlib.Path(path).with_suffix('.opts.yaml'))
async with await s_lmdbslab.Slab.anit(path, map_size=100000) as slab:
foo = slab.initdb('foo', dupsort=True)
mapsize = slab.mapsize
count = 0
while mapsize == slab.mapsize:
count += 1
slab.put(b'abcd', s_common.guid(count).encode('utf8') + byts, dupdata=True, db=foo)
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock
self.unpk = msgpack.Unpacker(use_list=0,encoding='utf8')
self.ident = s_common.guid()
self.xforms = [] # list of SockXform instances
self.crypto = None
self.sockinfo = info
self.onfini(self._finiSocket)
async def test_slab_infinite_loop(self):
'''
Trigger a map full when replaying the log from a prior map full.
'''
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
byts = b'\x00' * 256
count = 0
async with await s_lmdbslab.Slab.anit(path, map_size=32000, growsize=5000) as slab:
foo = slab.initdb('foo')
slab.put(b'abcd', s_common.guid(count).encode('utf8') + byts, db=foo)
await asyncio.sleep(1.1)
count += 1
slab.put(b'abcd', s_common.guid(count).encode('utf8') + byts, db=foo)
# If we got here we're good
self.true(True)
async def test_cell_setuser(self):
with self.getTestDir() as dirn:
async with await s_cell.Cell.anit(dirn) as cell:
async with cell.getLocalProxy() as prox:
self.eq('root', (await prox.getCellUser())['name'])
with self.raises(s_exc.NoSuchUser):
await prox.setCellUser(s_common.guid())
user = await prox.addAuthUser('visi')
self.true(await prox.setCellUser(user['iden']))
self.eq('visi', (await prox.getCellUser())['name'])
with self.raises(s_exc.AuthDeny):
await prox.setCellUser(s_common.guid())
def __init__(self, boss):
s_eventbus.EventBus.__init__(self)
self.jid = s_common.guid()
self.boss = boss
self.info = {}
self.task = None # (meth,args,kwargs)
self.retval = None
self.retexc = None
def add(self, useriden, condition, query, info):
iden = s_common.guid()
if not query:
raise ValueError('empty query')
self.core.getStormQuery(query)
rule = self._load_rule(iden, 1, condition, useriden, query, info=info)
self.core.slab.put(iden.encode(), rule.en(), db=self.trigdb)
return iden
async def addRole(self, name):
if self.rolesbyname.get(name) is not None:
raise s_exc.DupRoleName(name=name)
iden = s_common.guid()
path = self.node.full + ('roles', iden)
# directly set the nodes value and let events prop
await self.node.hive.set(path, name)
node = await self.node.hive.open(path)
return await self._addRoleNode(node)
async def test_lmdbslab_iternext_repeat_regression(self):
'''
Test for a scan being bumped in an iternext where the cursor is in the middle of a list of values with the same
key
'''
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
my_maxsize = 500000
async with await s_lmdbslab.Slab.anit(path, map_size=100000, growsize=50000, maxsize=my_maxsize) as slab:
foo = slab.initdb('foo', dupsort=True)
key = b'foo'
for i in range(100):
slab.put(key, s_common.guid(i).encode('utf8'), db=foo)
count = 0
for _, _ in slab.scanByRange(b'', db=foo):
count += 1
self.eq(count, 100)
# Partially read through scan
iter = slab.scanByRange(lmin=key, lmax=key, db=foo)
for i in range(60):
next(iter)
# Trigger a bump by writing a bunch; make sure we're not writing into the middle of the scan
multikey = b'\xff\xff\xff\xff' + s_common.guid(200).encode('utf8')
mapsize = slab.mapsize
count = 0
while mapsize == slab.mapsize:
count += 1
slab.put(multikey, s_common.guid(count).encode('utf8') + b'0' * 256, dupdata=True, db=foo)
# we wrote 100, read 60. We should read only another 40
self.len(40, list(iter))
def init(self, sock):
from Crypto.Cipher import ARC4
txnonce = s_common.guid()
sock.sendall(txnonce)
rxnonce = sock.recvall(16)
if rxnonce == None:
return
txkey = hashlib.sha256( txnonce + self.rc4key ).digest()
rxkey = hashlib.sha256( rxnonce + self.rc4key ).digest()
self.txcrypt = ARC4.new( txkey )
self.rxcrypt = ARC4.new( rxkey )
async def test_models_cngov_mucd(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
org0 = s_common.guid()
props = {
'org': org0
}
node = await snap.addNode('gov:cn:icp', 12345678, props)
self.eq(node.ndef[1], 12345678)
self.eq(node.get('org'), org0)
node = await snap.addNode('gov:cn:mucd', 61786)
self.eq(node.ndef[1], 61786)
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock
self.plex = None
self.unpk = msgpack.Unpacker(use_list=0,encoding='utf8')
self.iden = s_common.guid()
self.xforms = [] # list of SockXform instances
self.info = info
# used by Plex() tx
self.txbuf = None
self.txque = collections.deque()
self.onfini(self._finiSocket)
async def test_ou_code_prefixes(self):
guid0 = s_common.guid()
guid1 = s_common.guid()
guid2 = s_common.guid()
guid3 = s_common.guid()
omap = {
guid0: {'naics': '221121',
'sic': '0111'},
guid1: {'naics': '221122',
'sic': '0112'},
guid2: {'naics': '221113',
'sic': '2833'},
guid3: {'naics': '221320',
'sic': '0134'}
}
async with self.getTestCore() as core:
async with await core.snap() as snap:
for g, props in omap.items():
await snap.addNode('ou:org', g, props)
nodes = await alist(snap.getNodesBy('ou:org:sic', '01', cmpr='^='))
self.len(3, nodes)
nodes = await alist(snap.getNodesBy('ou:org:sic', '011', cmpr='^='))
self.len(2, nodes)
nodes = await alist(snap.getNodesBy('ou:org:naics', '22', cmpr='^='))
self.len(4, nodes)
nodes = await alist(snap.getNodesBy('ou:org:naics', '221', cmpr='^='))
self.len(4, nodes)
nodes = await alist(snap.getNodesBy('ou:org:naics', '2211', cmpr='^='))
self.len(3, nodes)
nodes = await alist(snap.getNodesBy('ou:org:naics', '22112', cmpr='^='))
self.len(2, nodes)
async def test_it_form_callbacks(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
# it:dev:str kicks out the :norm property on him when he is made
node = await snap.addNode('it:dev:str', 'evil RAT')
self.eq(node.ndef[1], 'evil RAT')
self.eq(node.get('norm'), 'evil rat')
pipe = 'MyPipe'
node = await snap.addNode('it:dev:pipe', pipe)
self.eq(node.ndef[1], pipe)
nodes = await alist(snap.getNodesBy('it:dev:str', pipe))
self.len(1, nodes)
# The callback created node also has norm set on it
self.eq(nodes[0].get('norm'), pipe.lower())
mutex = 'MyMutxex'
node = await snap.addNode('it:dev:mutex', mutex)
self.eq(node.ndef[1], mutex)
nodes = await alist(snap.getNodesBy('it:dev:str', mutex))
self.len(1, nodes)
key = 'HKEY_LOCAL_MACHINE\\Foo\\Bar'
node = await snap.addNode('it:dev:regkey', key)
self.eq(node.ndef[1], key)
nodes = await alist(snap.getNodesBy('it:dev:str', key))
self.len(1, nodes)
fbyts = 'sha256:' + 64 * 'f'
key = 'HKEY_LOCAL_MACHINE\\DUCK\\QUACK'
valus = [
('str', 'knight'),
('int', 20),
('bytes', fbyts),
]
for prop, valu in valus:
guid = s_common.guid((key, valu))
props = {
'key': key,
prop: valu,
}
node = await snap.addNode('it:dev:regval', guid, props)
self.eq(node.ndef[1], guid)
self.eq(node.get('key'), key)
self.eq(node.get(prop), valu)
nodes = await alist(snap.getNodesBy('it:dev:str', key))
self.len(1, nodes)
async def test_model_inet_dns_answer(self):
ip0 = 0x01010101
ip1 = '::2'
fqdn0 = 'woot.com'
fqdn1 = 'haha.com'
email0 = '*****@*****.**'
async with self.getTestCore() as core:
async with await core.snap() as snap:
# a record
props = {'a': (fqdn0, ip0)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('a'), (fqdn0, ip0))
# ns record
props = {'ns': (fqdn0, fqdn1)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('ns'), (fqdn0, fqdn1))
# rev record
props = {'rev': (ip0, fqdn0)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('rev'), (ip0, fqdn0))
# aaaa record
props = {'aaaa': (fqdn0, ip1)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('aaaa'), (fqdn0, ip1))
# rev6 record
props = {'rev6': (ip1, fqdn0)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('rev6'), (ip1, fqdn0))
# cname record
props = {'cname': (fqdn0, fqdn1)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('cname'), (fqdn0, fqdn1))
# mx record
props = {'mx': (fqdn0, fqdn1)}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('mx'), (fqdn0, fqdn1))
# soa record
guid = s_common.guid((fqdn0, fqdn1, email0))
props = {'soa': guid}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('soa'), guid)
# txt record
props = {'txt': (fqdn0, 'Oh my!')}
node = await snap.addNode('inet:dns:answer', '*', props)
self.eq(node.get('txt'), (fqdn0, 'Oh my!'))
async def test_model_base_node(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
iden = s_common.guid()
props = {
'type': 'hehe haha',
'data': ('some', 'data', 'here'),
}
node = await snap.addNode('graph:node', iden, props=props)
self.eq(node.ndef, ('graph:node', iden))
self.eq(node.get('type'), 'hehe haha')
self.eq(node.get('data'), ('some', 'data', 'here'))
async def sess(self, gen=True):
if self._web_sess is None:
iden = self.get_secure_cookie('sess')
if iden is None and not gen:
return None
if iden is None:
iden = s_common.guid()
opts = {'expires_days': 14, 'secure': True, 'httponly': True}
self.set_secure_cookie('sess', iden, **opts)
self._web_sess = await self.cell.genHttpSess(iden)
return self._web_sess
async def test_it_reveng(self):
async with self.getTestCore() as core:
baseFile = s_common.ehex(s_common.buid())
func = s_common.guid()
fva = 0x404438
rank = 33
complexity = 60
funccalls = ((baseFile, func), )
fopt = {
'vars': {
'file': baseFile,
'func': func,
'fva': fva,
'rank': rank,
'cmplx': complexity,
'funccalls': funccalls
}
}
vstr = 'VertexBrandArtisanalBinaries'
sopt = {'vars': {'func': func, 'string': vstr}}
name = "FunkyFunction"
descrp = "Test Function"
impcalls = ("libr.foo", "libr.foo2", "libr.foo3")
funcopt = {
'vars': {
'name': name,
'descrp': descrp,
'impcalls': impcalls
}
}
fnode = await core.nodes(
'[it:reveng:filefunc=($file, $func) :va=$fva :rank=$rank :complexity=$cmplx :funccalls=$funccalls]',
opts=fopt)
snode = await core.nodes('[it:reveng:funcstr=($func, $string)]',
opts=sopt)
self.len(1, fnode)
self.eq(f'sha256:{baseFile}', fnode[0].get('file'))
self.eq(fva, fnode[0].get('va'))
self.eq(rank, fnode[0].get('rank'))
self.eq(complexity, fnode[0].get('complexity'))
self.eq((f'sha256:{baseFile}', func), fnode[0].get('funccalls')[0])
self.len(1, snode)
self.eq(fnode[0].get('function'), snode[0].get('function'))
self.eq(vstr, snode[0].get('string'))
funcnode = await core.nodes('''
it:reveng:function [
:name=$name
:description=$descrp
:impcalls=$impcalls
:strings=(bar,foo,foo)
]''',
opts=funcopt)
self.len(1, funcnode)
self.eq(name, funcnode[0].get('name'))
self.eq(descrp, funcnode[0].get('description'))
self.len(len(impcalls), funcnode[0].get('impcalls'))
self.eq(impcalls[0], funcnode[0].get('impcalls')[0])
self.sorteq(('bar', 'foo'), funcnode[0].get('strings'))
nodes = await core.nodes('it:reveng:function -> it:dev:str')
self.len(2, nodes)
nodes = await core.nodes(
f'file:bytes={baseFile} -> it:reveng:filefunc :function -> it:reveng:funcstr:function'
)
self.len(1, nodes)
self.eq(vstr, nodes[0].get('string'))
nodes = await core.nodes(
f'file:bytes={baseFile} -> it:reveng:filefunc -> it:reveng:function -> it:reveng:impfunc'
)
self.len(len(impcalls), nodes)
async def test_model_econ(self):
async with self.getTestCore() as core:
# test card number 4024007150779444
card = (await core.nodes(
'[ econ:pay:card="*" :expr=201802 :name="Bob Smith" :cvv=123 :pin=1234 :pan=4024007150779444 ]'
))[0]
self.eq('bob smith', card.get('name'))
self.eq(1517443200000, card.get('expr'))
self.eq('4024007150779444', card.get('pan'))
self.eq(4, card.get('pan:mii'))
self.eq(402400, card.get('pan:iin'))
place = s_common.guid()
bycont = s_common.guid()
fromcont = s_common.guid()
text = f'''[
econ:purchase="*"
:price=13.37
:currency=USD
:by:contact={bycont}
:from:contact={fromcont}
:time=20180202
:place={place}
:paid=true
:paid:time=20180202
]'''
perc = (await core.nodes(text))[0]
self.eq('13.37', perc.get('price'))
self.eq('usd', perc.get('currency'))
self.len(1, await core.nodes('econ:purchase:price=13.37'))
self.len(1, await core.nodes('econ:purchase:price=13.370'))
self.len(0, await core.nodes('econ:purchase:price=13.372'))
with self.raises(s_exc.BadTypeValu):
await core.nodes(
'econ:purchase [ :price=170141183460469231731688 ]')
with self.raises(s_exc.BadTypeValu):
await core.nodes(
'econ:purchase [ :price=-170141183460469231731688 ]')
self.len(1, await core.nodes('econ:purchase:price*range=(13,14)'))
self.len(1, await core.nodes('econ:purchase:price>10.00'))
self.len(1, await core.nodes('econ:purchase:price<20.00'))
self.len(1, await core.nodes('econ:purchase:price>=10.00'))
self.len(1, await core.nodes('econ:purchase:price>=13.37'))
self.len(1, await core.nodes('econ:purchase:price<=20.00'))
self.len(1, await core.nodes('econ:purchase:price<=13.37'))
self.len(0, await core.nodes('econ:purchase:price<10.00'))
self.len(0, await core.nodes('econ:purchase:price>20.00'))
self.len(0, await core.nodes('econ:purchase:price>=20.00'))
self.len(0, await core.nodes('econ:purchase:price<=10.00'))
# runtime filter/cmpr test for econ:price
self.len(1, await core.nodes('econ:purchase:price +:price=13.37'))
self.len(1, await core.nodes('econ:purchase:price +:price=13.370'))
self.len(0, await core.nodes('econ:purchase:price +:price=13.372'))
self.len(
1, await
core.nodes('econ:purchase:price +:price*range=(13,14)'))
self.len(1, await core.nodes('econ:purchase:price +:price>10.00'))
self.len(1, await core.nodes('econ:purchase:price +:price<20.00'))
self.len(1, await core.nodes('econ:purchase:price +:price>=10.00'))
self.len(1, await core.nodes('econ:purchase:price +:price>=13.37'))
self.len(1, await core.nodes('econ:purchase:price +:price<=20.00'))
self.len(1, await core.nodes('econ:purchase:price +:price<=13.37'))
self.len(0, await core.nodes('econ:purchase:price +:price<10.00'))
self.len(0, await core.nodes('econ:purchase:price +:price>20.00'))
self.len(0, await core.nodes('econ:purchase:price +:price>=20.00'))
self.len(0, await core.nodes('econ:purchase:price +:price<=10.00'))
self.eq(bycont, perc.get('by:contact'))
self.eq(fromcont, perc.get('from:contact'))
self.eq(True, perc.get('paid'))
self.eq(1517529600000, perc.get('paid:time'))
self.eq(1517529600000, perc.get('time'))
self.eq(place, perc.get('place'))
self.len(1, await core.nodes('econ:purchase -> geo:place'))
self.len(2, await core.nodes('econ:purchase -> ps:contact | uniq'))
acqu = (await core.nodes(
f'[ econ:acquired=({perc.ndef[1]}, (inet:fqdn,vertex.link)) ]')
)[0]
self.eq(perc.ndef[1], acqu.get('purchase'))
self.len(1, await core.nodes('econ:acquired:item:form=inet:fqdn'))
self.eq(('inet:fqdn', 'vertex.link'), acqu.get('item'))
text = f'''[
econ:acct:payment="*"
:to:contact={bycont}
:to:coinaddr=(btc, 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2)
:from:contact={fromcont}
:from:coinaddr=(btc, 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2)
:from:pay:card={card.ndef[1]}
:amount = 20.30
:currency = usd
:time=20180202
:purchase={perc.ndef[1]}
]'''
await core.nodes(text)
self.len(
1, await core.nodes(
'econ:acct:payment +:time@=(2017,2019) +{-> econ:pay:card +:name="bob smith"}'
))
self.len(1, await core.nodes('econ:acct:payment -> econ:purchase'))
self.len(1, await core.nodes('econ:acct:payment -> econ:pay:card'))
self.len(
2, await core.nodes('econ:acct:payment -> ps:contact | uniq'))
nodes = await core.nodes('''
[ econ:fin:exchange=(us,nasdaq) :name=nasdaq :currency=usd :org=* ]
''')
self.len(1, nodes)
self.nn(nodes[0].ndef[1])
self.nn(nodes[0].get('org'))
self.eq('usd', nodes[0].get('currency'))
self.eq('nasdaq', nodes[0].get('name'))
nodes = await core.nodes('''
[
econ:fin:security=(us, nasdaq, tsla)
:exchange=(us, nasdaq)
:ticker=nasdaq/tsla
:type=STOCK
:price=9999.00
:time=202002
]
''')
self.len(1, nodes)
self.eq('947183947f2e2c7bdc55264c20670f19', nodes[0].ndef[1])
self.eq('stock', nodes[0].get('type'))
self.eq('nasdaq/tsla', nodes[0].get('ticker'))
self.eq('9999.00', nodes[0].get('price'))
self.eq(1580515200000, nodes[0].get('time'))
self.len(
1, await core.nodes(
'econ:fin:security -> econ:fin:exchange +:name=nasdaq'))
nodes = await core.nodes('''
[
econ:fin:tick=*
:time=20200202
:security=(us, nasdaq, tsla)
:price=9999.00
]
''')
self.len(1, nodes)
self.eq(1580601600000, nodes[0].get('time'))
self.eq('947183947f2e2c7bdc55264c20670f19',
nodes[0].get('security'))
self.eq('9999.00', nodes[0].get('price'))
nodes = await core.nodes('''
[
econ:fin:bar=*
:ival=(20200202, 20200203)
:security=(us, nasdaq, tsla)
:price:open=9999.00
:price:close=9999.01
:price:high=999999999999.00
:price:low=0.00001
]
''')
self.len(1, nodes)
self.eq((1580601600000, 1580688000000), nodes[0].get('ival'))
self.eq('947183947f2e2c7bdc55264c20670f19',
nodes[0].get('security'))
self.eq('9999.00', nodes[0].get('price:open'))
self.eq('9999.01', nodes[0].get('price:close'))
self.eq('999999999999.00', nodes[0].get('price:high'))
self.eq('0.00001', nodes[0].get('price:low'))
async def test_ps_simple(self):
person0 = s_common.guid()
persona0 = s_common.guid()
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode('ps:tokn', ' BOB ')
self.eq(node.ndef[1], 'bob')
node = await snap.addNode('ps:name',
' robert GREY the\t3rd ')
self.eq(node.ndef[1], 'robert grey the 3rd')
file0 = 'sha256:' + 64 * '0'
person_props = {
'dob': '1971',
'img': file0,
'nick': 'pennywise',
# 'guidname': '', # fixme guid aliases
'name': 'robert clown grey',
'name:sur': 'grey',
'name:middle': 'clown',
'name:given': 'robert',
}
node = await snap.addNode('ps:person', person0, person_props)
self.eq(node.ndef[1], person0)
self.eq(node.get('img'), file0)
self.eq(node.get('dob'), 31536000000)
self.eq(node.get('nick'), 'pennywise')
self.eq(node.get('name'), 'robert clown grey')
self.eq(node.get('name:sur'), 'grey')
self.eq(node.get('name:middle'), 'clown')
self.eq(node.get('name:given'), 'robert')
# self.eq(node.get('img'), '') # fixme file:bytes
# self.eq(node.get('guidname'), '') # fixme guid aliases
persona_props = {
'dob': '2000',
'img': file0,
'nick': 'acid burn',
# 'guidname': '', # fixme guid aliases
'name': 'Эммануэль брат Гольдштейн',
'name:sur': 'Гольдштейн',
'name:middle': 'брат',
'name:given': 'эммануэль',
}
node = await snap.addNode('ps:persona', persona0,
persona_props)
self.eq(node.ndef[1], persona0)
self.eq(node.get('img'), file0)
self.eq(node.get('dob'), 946684800000)
self.eq(node.get('nick'), 'acid burn')
self.eq(node.get('name'), 'эммануэль брат гольдштейн')
self.eq(node.get('name:sur'), 'гольдштейн')
self.eq(node.get('name:middle'), 'брат')
self.eq(node.get('name:given'), 'эммануэль')
# self.eq(node.get('img'), '') # fixme file:bytes
# self.eq(node.get('guidname'), '') # fixme guid aliases
node = await snap.addNode('ps:person:has',
(person0, ('test:str', 'sewer map')))
self.eq(node.ndef[1], (person0, ('test:str', 'sewer map')))
self.eq(node.get('person'), person0)
self.eq(node.get('node'), ('test:str', 'sewer map'))
self.eq(node.get('node:form'), 'test:str')
node = await snap.addNode('ps:persona:has',
(persona0,
('test:str', 'the gibson')))
self.eq(node.ndef[1], (persona0, ('test:str', 'the gibson')))
self.eq(node.get('persona'), persona0)
self.eq(node.get('node'), ('test:str', 'the gibson'))
self.eq(node.get('node:form'), 'test:str')
org0 = s_common.guid()
con0 = s_common.guid()
cprops = {
'org': org0,
'asof': '20080414',
'person': person0,
'name': 'Tony Stark',
'title': 'CEO',
'orgname': 'Stark Industries, INC',
# 'img': '', # fixme file:bytes
'user': '******',
'web:acct': ('twitter.com', 'ironman'),
'dob': '1976-12-17',
'url': 'https://starkindustries.com/',
'email': '*****@*****.**',
'email:work': '*****@*****.**',
'phone': '12345678910',
'phone:fax': '12345678910',
'phone:work': '12345678910',
'address':
'1 Iron Suit Drive, San Francisco, CA, 22222, USA',
}
node = await snap.addNode('ps:contact', con0, cprops)
self.eq(node.ndef[1], con0)
self.eq(node.get('org'), org0)
self.eq(node.get('asof'), 1208131200000)
self.eq(node.get('person'), person0)
self.eq(node.get('name'), 'tony stark')
self.eq(node.get('title'), 'ceo')
self.eq(node.get('orgname'), 'stark industries, inc')
self.eq(node.get('user'), 'ironman')
self.eq(node.get('web:acct'), ('twitter.com', 'ironman'))
self.eq(node.get('dob'), 219628800000)
self.eq(node.get('url'), 'https://starkindustries.com/')
self.eq(node.get('email'), '*****@*****.**')
self.eq(node.get('email:work'), '*****@*****.**')
self.eq(node.get('phone'), '12345678910')
self.eq(node.get('phone:fax'), '12345678910')
self.eq(node.get('phone:work'), '12345678910')
self.eq(node.get('address'),
'1 iron suit drive, san francisco, ca, 22222, usa')
async def test_lmdbslab_grow(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
async with await s_lmdbslab.Slab.anit(path,
map_size=100000,
growsize=10000) as slab:
foo = slab.initdb('foo', dupsort=True)
foo2 = slab.initdb('foo2', dupsort=False)
byts = b'\x00' * 256
for i in range(100):
slab.put(s_common.guid(i).encode('utf8'), byts, db=foo)
slab.put(s_common.guid(1000 + i).encode('utf8'),
byts,
db=foo2)
count = 0
for _, _ in slab.scanByRange(b'', db=foo):
count += 1
self.eq(count, 100)
count = 0
for _, _ in slab.scanByRangeBack(
b'ffffffffffffffffffffffffffffffff', db=foo):
count += 1
self.eq(count, 100)
# Trigger a grow/bump in the middle of a scan; make sure new nodes come after current scan position
iter = slab.scanByRange(b'', db=foo)
for _ in range(50):
next(iter)
iterback = slab.scanByRangeBack(
b'ffffffffffffffffffffffffffffffff', db=foo)
for _ in range(50):
next(iterback)
multikey = b'\xff\xff\xff\xfe' + s_common.guid(2000).encode(
'utf8')
mapsize = slab.mapsize
count = 0
# Write until we grow
while mapsize == slab.mapsize:
count += 1
rv = slab.put(
multikey,
s_common.guid(count + 100000).encode('utf8') + byts,
dupdata=True,
db=foo)
self.true(rv)
self.eq(50 + count, sum(1 for _ in iter))
self.eq(50, sum(1 for _ in iterback))
self.true(os.path.isfile(slab.optspath))
# Trigger a grow/bump in the middle of a dup scan
iter = slab.scanByDups(multikey, db=foo)
next(iter)
iter2 = slab.scanByFull(db=foo2)
next(iter2)
iterback = slab.scanByDupsBack(multikey, db=foo)
next(iterback)
iterback2 = slab.scanByFullBack(db=foo2)
next(iterback2)
multikey = b'\xff\xff\xff\xff' + s_common.guid(
i + 150000).encode('utf8')
for i in range(200):
slab.put(multikey,
s_common.guid(i + 200000).encode('utf8') + byts,
dupdata=True,
db=foo)
self.eq(count - 1, sum(1 for _ in iter))
self.eq(99, sum(1 for _ in iter2))
self.eq(count - 1, sum(1 for _ in iterback))
self.eq(99, sum(1 for _ in iterback2))
# lets ensure our mapsize / growsize persisted, and make sure readonly works
async with await s_lmdbslab.Slab.anit(path,
map_size=100000,
readonly=True) as newdb:
self.eq(10000, newdb.growsize)
foo = newdb.initdb('foo', dupsort=True)
for _, _ in newdb.scanByRange(b'', db=foo):
count += 1
self.gt(count, 200)
# Make sure readonly is really readonly
self.raises(s_exc.IsReadOnly, newdb.dropdb, 'foo')
self.raises(s_exc.IsReadOnly, newdb.put, b'1234', b'3456')
self.raises(s_exc.IsReadOnly, newdb.replace, b'1234', b'3456')
self.raises(s_exc.IsReadOnly, newdb.pop, b'1234')
self.raises(s_exc.IsReadOnly, newdb.delete, b'1234')
self.raises(s_exc.IsReadOnly, newdb.putmulti,
((b'1234', b'3456'), ))
# While we have the DB open in readonly, have another process write a bunch of data to cause the
# map size to be increased
def anotherproc(path):
async def lotsofwrites(path):
os.remove(pathlib.Path(path).with_suffix('.opts.yaml'))
async with await s_lmdbslab.Slab.anit(
path, map_size=100000) as slab:
foo = slab.initdb('foo', dupsort=True)
mapsize = slab.mapsize
count = 0
while mapsize == slab.mapsize:
count += 1
slab.put(b'abcd',
s_common.guid(count).encode('utf8') +
byts,
dupdata=True,
db=foo)
asyncio.run(lotsofwrites(path))
proc = multiprocessing.Process(target=anotherproc,
args=(path, ))
proc.start()
proc.join()
# Now trigger a remap for me
newdb.get(multikey, db=foo)
async def test_it_forms_simple(self):
async with self.getTestCore() as core:
place = s_common.guid()
async with await core.snap() as snap:
node = await snap.addNode('it:hostname', 'Bobs Computer')
self.eq(node.ndef[1], 'bobs computer')
org0 = s_common.guid()
host0 = s_common.guid()
sver0 = s_common.guid()
cont0 = s_common.guid()
hprops = {
'name': 'Bobs laptop',
'desc': 'Bobs paperweight',
'ipv4': '1.2.3.4',
'latlong': '0.0, 0.0',
'place': place,
'os': sver0,
'manu': 'Dull',
'model': 'Lutitude 8249',
'serial': '111-222',
'loc': 'us.hehe.haha',
'operator': cont0,
'org': org0,
}
node = await snap.addNode('it:host', host0, hprops)
self.eq(node.ndef[1], host0)
self.eq(node.get('name'), 'bobs laptop')
self.eq(node.get('desc'), 'Bobs paperweight')
self.eq(node.get('ipv4'), 0x01020304)
self.eq(node.get('latlong'), (0.0, 0.0))
self.eq(node.get('place'), place)
self.eq(node.get('os'), sver0)
self.eq(node.get('loc'), 'us.hehe.haha')
self.eq(node.get('org'), org0)
self.eq(node.get('operator'), cont0)
node = await snap.addNode(
'it:hosturl', (host0, 'http://vertex.ninja/cool.php'))
self.eq(node.ndef[1], (host0, 'http://vertex.ninja/cool.php'))
self.eq(node.get('host'), host0)
self.eq(node.get('url'), 'http://vertex.ninja/cool.php')
node = await snap.addNode('it:dev:int', 0x61C88648)
self.eq(node.ndef[1], 1640531528)
cprops = {
'desc': 'Some words.',
}
node = await snap.addNode('it:sec:cve', 'CVE-2013-9999',
cprops)
self.eq(node.ndef[1], 'cve-2013-9999')
self.eq(node.get('desc'), 'Some words.')
hash0 = s_common.guid()
hprops = {
'salt': 'B33F',
'hash:md5': s_m_crypto.ex_md5,
'hash:sha1': s_m_crypto.ex_sha1,
'hash:sha256': s_m_crypto.ex_sha256,
'hash:sha512': s_m_crypto.ex_sha512,
'hash:lm': s_m_crypto.ex_md5,
'hash:ntlm': s_m_crypto.ex_md5,
'passwd': "I've got the same combination on my luggage!",
}
node = await snap.addNode('it:auth:passwdhash', hash0, hprops)
self.eq(node.ndef[1], hash0)
self.eq(node.get('salt'), 'b33f')
self.eq(node.get('hash:md5'), s_m_crypto.ex_md5)
self.eq(node.get('hash:sha1'), s_m_crypto.ex_sha1)
self.eq(node.get('hash:sha256'), s_m_crypto.ex_sha256)
self.eq(node.get('hash:sha512'), s_m_crypto.ex_sha512)
self.eq(node.get('hash:lm'), s_m_crypto.ex_md5)
self.eq(node.get('hash:ntlm'), s_m_crypto.ex_md5)
self.eq(node.get('passwd'),
"I've got the same combination on my luggage!")
nodes = await core.nodes('[ it:adid=visi ]')
self.eq(('it:adid', 'visi'), nodes[0].ndef)
async def setPasswd(self, passwd):
salt = s_common.guid()
hashed = s_common.guid((salt, passwd))
await self.info.set('passwd', (salt, hashed))
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
outp.printf(s_common.guid())
async def __anit__(self, dirn, conf=None, readonly=False):
await s_base.Base.__anit__(self)
s_telepath.Aware.__init__(self)
self.dirn = s_common.gendir(dirn)
self.auth = None
# each cell has a guid
path = s_common.genpath(dirn, 'cell.guid')
# generate a guid file if needed
if not os.path.isfile(path):
with open(path, 'w') as fd:
fd.write(s_common.guid())
# read our guid file
with open(path, 'r') as fd:
self.iden = fd.read().strip()
boot = self._loadCellYaml('boot.yaml')
self.boot = s_common.config(boot, bootdefs)
await self._initCellDmon()
if conf is None:
conf = {}
[
conf.setdefault(k, v)
for (k, v) in self._loadCellYaml('cell.yaml').items()
]
self.conf = s_common.config(conf, self.confdefs + self.confbase)
self.cmds = {}
self.insecure = self.boot.get('insecure', False)
self.sessions = {}
self.httpsonly = self.conf.get('https:only', False)
self.boss = await s_boss.Boss.anit()
self.onfini(self.boss)
await self._initCellSlab(readonly=readonly)
self.hive = await self._initCellHive()
self.auth = await self._initCellAuth()
# check and migrate old cell auth
oldauth = s_common.genpath(self.dirn, 'auth')
if os.path.isdir(oldauth):
await s_compat.cellAuthToHive(oldauth, self.auth)
os.rename(oldauth, oldauth + '.old')
admin = self.boot.get('auth:admin')
if admin is not None:
name, passwd = admin.split(':', 1)
user = self.auth.getUserByName(name)
if user is None:
user = await self.auth.addUser(name)
await user.setAdmin(True)
await user.setPasswd(passwd)
self.insecure = False
await self._initCellHttp()
async def fini():
[await s.fini() for s in self.sessions.values()]
self.onfini(fini)
async def test_it_forms_prodsoft(self):
# Test all prodsoft and prodsoft associated linked forms
async with self.getTestCore() as core:
async with await core.snap() as snap:
# it:prod:soft
prod0 = s_common.guid()
org0 = s_common.guid()
person0 = s_common.guid()
file0 = 'a' * 64
acct0 = ('vertex.link', 'pennywise')
url0 = 'https://vertex.link/products/balloonmaker'
sprops = {
'name': 'Balloon Maker',
'desc': "Pennywise's patented balloon blower upper",
'desc:short': 'Balloon blower',
'author:org': org0,
'author:email': '*****@*****.**',
'author:acct': acct0,
'author:person': person0,
'url': url0,
}
node = await snap.addNode('it:prod:soft', prod0, sprops)
self.eq(node.ndef[1], prod0)
self.eq(node.get('name'), 'balloon maker')
self.eq(node.get('desc'),
"Pennywise's patented balloon blower upper")
self.eq(node.get('desc:short'), 'balloon blower')
self.eq(node.get('author:org'), org0)
self.eq(node.get('author:acct'), acct0)
self.eq(node.get('author:email'), '*****@*****.**')
self.eq(node.get('author:person'), person0)
self.false(node.get('isos'))
self.false(node.get('islib'))
await node.set('isos', True)
await node.set('islib', True)
self.true(node.get('isos'))
self.true(node.get('islib'))
self.eq(node.get('url'), url0)
# it:prod:softver - this does test a bunch of property related callbacks
url1 = 'https://vertex.link/products/balloonmaker/release_101-beta.exe'
vprops = {
'vers': 'V1.0.1-beta+exp.sha.5114f85',
'url': url1,
'software': prod0,
'arch': 'amd64'
}
ver0 = s_common.guid()
node = await snap.addNode('it:prod:softver', ver0, vprops)
self.eq(node.ndef[1], ver0)
self.eq(node.get('arch'), 'amd64')
self.eq(node.get('software'), prod0)
self.eq(node.get('software:name'), 'balloon maker')
self.eq(node.get('vers'), 'V1.0.1-beta+exp.sha.5114f85')
self.eq(node.get('vers:norm'), 'v1.0.1-beta+exp.sha.5114f85')
self.eq(node.get('semver'), 0x000010000000001)
self.eq(node.get('semver:major'), 1)
self.eq(node.get('semver:minor'), 0)
self.eq(node.get('semver:patch'), 1)
self.eq(node.get('semver:pre'), 'beta')
self.eq(node.get('semver:build'), 'exp.sha.5114f85')
self.eq(node.get('url'), url1)
# callback node creation checks
nodes = await snap.nodes(
'it:dev:str=V1.0.1-beta+exp.sha.5114f85')
self.len(1, nodes)
nodes = await snap.nodes('it:dev:str=amd64')
self.len(1, nodes)
host0 = s_common.guid()
node = await snap.addNode('it:hostsoft', (host0, ver0))
self.eq(node.ndef[1], (host0, ver0))
self.eq(node.get('host'), host0)
self.eq(node.get('softver'), ver0)
softfile = await snap.addNode('it:prod:softfile',
(ver0, file0))
self.eq(softfile.get('soft'), ver0)
self.eq(softfile.get('file'), f'sha256:{file0}')
ver1 = s_common.guid()
softlib = await snap.addNode('it:prod:softlib', (ver0, ver1))
self.eq(softlib.get('soft'), ver0)
self.eq(softlib.get('lib'), ver1)
os0 = s_common.guid()
softos = await snap.addNode('it:prod:softos', (ver0, os0))
self.eq(softos.get('soft'), ver0)
self.eq(softos.get('os'), os0)
prod1 = s_common.guid()
sigprops = {
'desc': 'The evil balloon virus!',
'url': url1,
}
sig0 = (prod1, 'Bar.BAZ.faZ')
node = await snap.addNode('it:av:sig', sig0, sigprops)
self.eq(node.ndef[1], (prod1, 'Bar.BAZ.faZ'.lower()))
self.eq(node.get('soft'), prod1)
self.eq(node.get('name'), 'bar.baz.faz')
self.eq(node.get('desc'), 'The evil balloon virus!')
self.eq(node.get('url'), url1)
node = await snap.addNode('it:av:filehit', (file0, sig0))
self.eq(node.ndef[1],
(f'sha256:{file0}', (prod1, 'Bar.BAZ.faZ'.lower())))
self.eq(node.get('file'), f'sha256:{file0}')
self.eq(node.get('sig'), (prod1, 'Bar.BAZ.faZ'.lower()))
self.eq(node.get('sig:name'), 'bar.baz.faz')
self.eq(node.get('sig:soft'), prod1)
# Test 'vers' semver brute forcing
testvectors = [
('1', 0x000010000000000, {
'major': 1,
'minor': 0,
'patch': 0
}),
('2.0A1', 0x000020000000000, {
'major': 2,
'minor': 0,
'patch': 0
}),
('2016-03-01', 0x007e00000300001, {
'major': 2016,
'minor': 3,
'patch': 1
}),
('1.2.windows-RC1', 0x000010000200000, {
'major': 1,
'minor': 2,
'patch': 0
}),
('3.4', 0x000030000400000, {
'major': 3,
'minor': 4,
'patch': 0
}),
('1.3a2.dev12', 0x000010000000000, {
'major': 1,
'minor': 0,
'patch': 0
}),
('v2.4.0.0-1', 0x000020000400000, {
'major': 2,
'minor': 4,
'patch': 0
}),
('v2.4.1.0-0.3.rc1', 0x000020000400001, {
'major': 2,
'minor': 4,
'patch': 1
}),
('0.18rc2', 0, {
'major': 0,
'minor': 0,
'patch': 0
}),
('OpenSSL_1_0_2l', 0x000010000000000, {
'major': 1,
'minor': 0,
'patch': 0
}),
]
for tv, te, subs in testvectors:
props = {'vers': tv}
node = await snap.addNode('it:prod:softver', '*', props)
self.eq(node.get('semver'), te)
self.eq(node.get('semver:major'), subs.get('major'))
self.eq(node.get('semver:minor'), subs.get('minor'))
self.eq(node.get('semver:patch'), subs.get('patch'))
node = await snap.addNode('it:prod:softver', '*', {'vers': ''})
self.eq(node.get('vers'), '')
self.none(node.get('vers:norm'))
self.none(node.get('semver'))
with self.getLoggerStream(
'synapse.models.infotech',
'Unable to brute force version parts out of the string'
) as stream:
node = await snap.addNode('it:prod:softver', '*',
{'vers': 'Alpha'})
self.none(node.get('semver'))
self.true(stream.is_set())
async def _guid(self, *args):
if args:
return s_common.guid(args)
return s_common.guid()
async def test_types_forms(self):
formlat = 'geo:latitude'
formlon = 'geo:longitude'
formlatlon = 'geo:latlong'
async with self.getTestCore() as core:
# Latitude Type Tests =====================================================================================
t = core.model.type(formlat)
self.raises(s_exc.BadTypeValu, t.norm, '-90.1')
self.eq(t.norm('-90')[0], -90.0)
self.eq(t.norm('-12.345678901234567890')[0], -12.3456789)
self.eq(t.norm('-0')[0], 0.0)
self.eq(t.norm('0')[0], 0.0)
self.eq(t.norm('12.345678901234567890')[0], 12.3456789)
self.eq(t.norm('90')[0], 90.0)
self.raises(s_exc.BadTypeValu, t.norm, '90.1')
self.raises(s_exc.BadTypeValu, t.norm, 'newp')
self.eq(t.indx(-90), b'\x00\x00\x00\x00\x00'
) # index starts at 0 and goes to 9000000000
self.eq(t.indx(-12.34567890123456789), b'\x01\xce\xdb\x17-')
self.eq(t.indx(0), b'\x02\x18q\x1a\x00')
self.eq(t.indx(12.34567890123456789), b'\x02b\x07\x1c\xd2')
self.eq(t.indx(90), b'\x040\xe24\x00')
# Longitude Type Tests =====================================================================================
t = core.model.type(formlon)
self.raises(s_exc.BadTypeValu, t.norm, '-180.1')
self.eq(t.norm('-180')[0], -180.0)
self.eq(t.norm('-12.345678901234567890')[0], -12.3456789)
self.eq(t.norm('-0')[0], 0.0)
self.eq(t.norm('0')[0], 0.0)
self.eq(t.norm('12.345678901234567890')[0], 12.3456789)
self.eq(t.norm('180')[0], 180.0)
self.raises(s_exc.BadTypeValu, t.norm, '180.1')
self.raises(s_exc.BadTypeValu, t.norm, 'newp')
self.eq(t.indx(-180), b'\x00\x00\x00\x00\x00'
) # index starts at 0 and goes to 18000000000
self.eq(t.indx(-12.34567890123456789), b'\x03\xe7L1-')
self.eq(t.indx(0), b'\x040\xe24\x00')
self.eq(t.indx(12.34567890123456789), b'\x04zx6\xd2')
self.eq(t.indx(180), b'\x08a\xc4h\x00')
# Latlong Type Tests =====================================================================================
t = core.model.type(formlatlon)
self.eq(t.norm('0,-0'), ((0.0, 0.0), {
'subs': {
'lat': 0.0,
'lon': 0.0
}
}))
self.eq(t.norm('89.999,179.999'), ((89.999, 179.999), {
'subs': {
'lat': 89.999,
'lon': 179.999
}
}))
self.eq(t.norm('-89.999,-179.999'), ((-89.999, -179.999), {
'subs': {
'lat': -89.999,
'lon': -179.999
}
}))
self.eq(t.norm([89.999, 179.999]), ((89.999, 179.999), {
'subs': {
'lat': 89.999,
'lon': 179.999
}
}))
self.eq(t.norm((89.999, 179.999)), ((89.999, 179.999), {
'subs': {
'lat': 89.999,
'lon': 179.999
}
}))
self.raises(s_exc.BadTypeValu, t.norm, '-91,0')
self.raises(s_exc.BadTypeValu, t.norm, '91,0')
self.raises(s_exc.BadTypeValu, t.norm, '0,-181')
self.raises(s_exc.BadTypeValu, t.norm, '0,181')
self.raises(s_exc.BadTypeValu, t.norm,
('newp', 'newp', 'still newp'))
# Demonstrate precision
self.eq(t.norm('12.345678,-12.345678'), ((12.345678, -12.345678), {
'subs': {
'lat': 12.345678,
'lon': -12.345678
}
}))
self.eq(t.norm('12.3456789,-12.3456789'),
((12.3456789, -12.3456789), {
'subs': {
'lat': 12.3456789,
'lon': -12.3456789
}
}))
self.eq(t.norm('12.34567890,-12.34567890'),
((12.3456789, -12.3456789), {
'subs': {
'lat': 12.3456789,
'lon': -12.3456789
}
}))
self.eq(t.indx((-90, -180)),
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
self.eq(t.indx((90, 180)), b'\x040\xe24\x00\x08a\xc4h\x00')
self.eq(t.indx((0, 0)), b'\x02\x18q\x1a\x00\x040\xe24\x00')
self.eq(t.indx((0, -0)), b'\x02\x18q\x1a\x00\x040\xe24\x00')
self.eq(t.indx((0, 1)), b'\x02\x18q\x1a\x00\x046\xd8\x15\x00')
self.eq(t.indx((0, -1)), b'\x02\x18q\x1a\x00\x04*\xecS\x00')
self.eq(t.indx((-90, 180)), b'\x00\x00\x00\x00\x00\x08a\xc4h\x00')
self.eq(t.indx((90, -180)), b'\x040\xe24\x00\x00\x00\x00\x00\x00')
self.eq(t.indx((12.3456789, -12.3456789)),
b'\x02b\x07\x1c\xd2\x03\xe7L1.')
self.eq(t.indx((12.34567890, -12.34567890)),
b'\x02b\x07\x1c\xd2\x03\xe7L1.')
self.eq(t.repr((0, 0)), '0,0')
self.eq(t.repr((0, -0)), '0,0')
self.eq(t.repr((12.345678, -12.345678)), '12.345678,-12.345678')
# Geo-dist tests
formname = 'geo:dist'
t = core.model.type(formname)
self.eq(t.norm('100km')[0], 100000000)
self.eq(t.norm('100 km')[0], 100000000)
self.eq(t.norm('837.33 m')[0], 837330)
self.eq(t.norm('11.2 km'), (11200000, {}))
self.eq(t.norm(11200000), (11200000, {}))
self.eq(t.repr(5), '5 mm')
self.eq(t.repr(500), '50.0 cm')
self.eq(t.repr(1000), '1.0 m')
self.eq(t.repr(10000), '10.0 m')
self.eq(t.repr(1000000), '1.0 km')
self.raises(s_exc.BadTypeValu, t.norm, '1.3 pc')
# geo:nloc
formname = 'geo:nloc'
t = core.model.type(formname)
ndef = ('inet:ipv4', '0.0.0.0')
latlong = ('0.000000000', '0')
stamp = -0
async with await core.snap() as snap:
node = await snap.addNode('geo:nloc', (ndef, latlong, stamp))
self.eq(node.ndef[1], (('inet:ipv4', 0), (0.0, 0.0), stamp))
self.eq(node.get('ndef'), ('inet:ipv4', 0))
self.eq(node.get('ndef:form'), 'inet:ipv4')
self.eq(node.get('latlong'), (0.0, 0.0))
self.eq(node.get('time'), 0)
self.nn(await snap.getNodeByNdef(('inet:ipv4', 0)))
# geo:place
# test inline tuple/float with negative syntax...
node = (await
alist(core.eval('[ geo:place="*" :latlong=(-30.0,20.22) ]')
))[0]
self.eq(node.get('latlong'), (-30.0, 20.22))
async with await core.snap() as snap:
guid = s_common.guid()
props = {
'name': 'Vertex HQ',
'desc': 'The place where Vertex Project hangs out at!',
'address':
'208 Datong Road, Pudong District, Shanghai, China',
'loc': 'us.hehe.haha',
'latlong': '34.1341, -118.3215',
'radius': '1.337km'
}
node = await snap.addNode('geo:place', guid, props)
self.eq(node.ndef[1], guid)
self.eq(node.get('name'), 'vertex hq')
self.eq(node.get('loc'), 'us.hehe.haha')
self.eq(node.get('latlong'), (34.13409999, -118.3215))
self.eq(node.get('radius'), 1337000)
self.eq(node.get('desc'),
'The place where Vertex Project hangs out at!')
self.eq(node.get('address'),
'208 datong road, pudong district, shanghai, china')
async def test_near(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
# These two nodes are 2,605m apart
guid0 = s_common.guid()
props = {
'name': 'Vertex HQ',
'latlong': '34.1341, -118.3215', # hollywood sign
'radius': '1.337km'
}
node = await snap.addNode('geo:place', guid0, props)
self.nn(node)
guid1 = s_common.guid()
props = {
'name': 'Griffith Observatory',
'latlong': '34.118560, -118.300370',
'radius': '75m'
}
node = await snap.addNode('geo:place', guid1, props)
self.nn(node)
guid2 = s_common.guid()
props = {'name': 'unknown location'}
node = await snap.addNode('geo:place', guid2, props)
self.nn(node)
# A telemetry node for example by the observatory
guid3 = s_common.guid()
props = {'latlong': '34.118660, -118.300470'}
node = await snap.addNode('tel:mob:telem', guid3, props)
self.nn(node)
# A telemetry node for example by the HQ
guid4 = s_common.guid()
props = {'latlong': '34.13412, -118.32153'}
node = await snap.addNode('tel:mob:telem', guid4, props)
self.nn(node)
# Node filtering behavior
nodes = await alist(
core.eval('geo:place +:latlong*near=((34.1, -118.3), 10km)'))
self.len(2, nodes)
nodes = await alist(
core.eval(
'geo:place +geo:place:latlong*near=((34.1, -118.3), 10km)')
)
self.len(2, nodes)
nodes = await alist(
core.eval('geo:place +:latlong*near=((34.1, -118.3), 50m)'))
self.len(0, nodes)
# +1's come from the unknown loc without a latlong prop
nodes = await alist(
core.eval('geo:place -:latlong*near=((34.1, -118.3), 10km)'))
self.len(0 + 1, nodes)
nodes = await alist(
core.eval('geo:place -:latlong*near=((34.1, -118.3), 50m)'))
self.len(2 + 1, nodes)
# Storm variable use to filter nodes based on a given location.
q = f'geo:place={guid0} $latlong=:latlong $radius=:radius | spin | geo:place +:latlong*near=($latlong, ' \
f'$radius)'
nodes = await alist(core.eval(q))
self.len(1, nodes)
q = f'geo:place={guid0} $latlong=:latlong $radius=:radius | spin | geo:place +:latlong*near=($latlong, 5km)'
nodes = await alist(core.eval(q))
self.len(2, nodes)
# Lifting nodes by *near=((latlong), radius)
nodes = await alist(
core.eval('geo:place:latlong*near=((34.1, -118.3), 10km)'))
self.len(2, nodes)
nodes = await alist(
core.eval(
'geo:place:latlong*near=(("34.118560", "-118.300370"), 50m)'
))
self.len(1, nodes)
nodes = await alist(
core.eval('geo:place:latlong*near=((0, 0), 50m)'))
self.len(0, nodes)
# Use a radius to lift nodes which will be inside the bounding box,
# but outside the cmpr implemented using haversine filtering.
nodes = await alist(
core.eval(
'geo:place:latlong*near=(("34.118560", "-118.300370"), 2600m)'
))
self.len(1, nodes)
# Storm variable use to lift nodes based on a given location.
q = f'geo:place={guid1} $latlong=:latlong $radius=:radius ' \
f'tel:mob:telem:latlong*near=($latlong, 3km) +tel:mob:telem'
nodes = await alist(core.eval(q))
self.len(2, nodes)
q = f'geo:place={guid1} $latlong=:latlong $radius=:radius ' \
f'tel:mob:telem:latlong*near=($latlong, $radius) +tel:mob:telem'
nodes = await alist(core.eval(q))
self.len(1, nodes)
async with self.getTestCore() as core:
await core.loadCoreModule(
'synapse.tests.test_model_geospace.GeoTstModule')
# Lift behavior for a node whose has a latlong as their primary property
nodes = await core.eval('[(test:latlong=(10, 10) :dist=10m) '
'(test:latlong=(10.1, 10.1) :dist=20m) '
'(test:latlong=(3, 3) :dist=5m)]').list()
self.len(3, nodes)
nodes = await core.eval('test:latlong*near=((10, 10), 5km)').list()
self.len(1, nodes)
nodes = await core.eval('test:latlong*near=((10, 10), 30km)'
).list()
self.len(2, nodes)
# Ensure geo:dist inherits from IntBase correctly
nodes = await core.nodes('test:latlong +:dist>5m')
self.len(2, nodes)
nodes = await core.nodes('test:latlong +:dist>=5m')
self.len(3, nodes)
nodes = await core.nodes('test:latlong +:dist<5m')
self.len(0, nodes)
nodes = await core.nodes('test:latlong +:dist<=5m')
self.len(1, nodes)
nodes = await core.nodes('test:latlong:dist>5m')
self.len(2, nodes)
nodes = await core.nodes('test:latlong:dist>=5m')
self.len(3, nodes)
nodes = await core.nodes('test:latlong:dist<5m')
self.len(0, nodes)
nodes = await core.nodes('test:latlong:dist<=5m')
self.len(1, nodes)
nodes = await core.nodes('test:latlong +:dist*range=(8m, 10m)')
self.len(1, nodes)
nodes = await core.nodes('test:latlong:dist*range=(8m, 10m)')
self.len(1, nodes)
async def test_model_file_mime_msoffice(self):
async with self.getTestCore() as core:
fileguid = s_common.guid()
opts = {'vars': {'fileguid': f'guid:{fileguid}'}}
def testmsoffice(n):
self.eq('lolz', n.get('title'))
self.eq('deep_value', n.get('author'))
self.eq('GME stonks', n.get('subject'))
self.eq('stonktrader3000', n.get('application'))
self.eq(1611100800000, n.get('created'))
self.eq(1611187200000, n.get('lastsaved'))
self.eq(f'guid:{fileguid}', n.get('file'))
self.eq(0, n.get('file:offs'))
self.eq(('foo', 'bar'), n.get('file:data'))
nodes = await core.nodes('''[
file:mime:msdoc=*
:file=$fileguid
:file:offs=0
:file:data=(foo, bar)
:title=lolz
:author=deep_value
:subject="GME stonks"
:application=stonktrader3000
:created=20210120
:lastsaved=20210121
]''',
opts=opts)
self.len(1, nodes)
testmsoffice(nodes[0])
nodes = await core.nodes('''[
file:mime:msxls=*
:file=$fileguid
:file:offs=0
:file:data=(foo, bar)
:title=lolz
:author=deep_value
:subject="GME stonks"
:application=stonktrader3000
:created=20210120
:lastsaved=20210121
]''',
opts=opts)
self.len(1, nodes)
testmsoffice(nodes[0])
nodes = await core.nodes('''[
file:mime:msppt=*
:file=$fileguid
:file:offs=0
:file:data=(foo, bar)
:title=lolz
:author=deep_value
:subject="GME stonks"
:application=stonktrader3000
:created=20210120
:lastsaved=20210121
]''',
opts=opts)
self.len(1, nodes)
testmsoffice(nodes[0])
async def test_layer_stortype_float(self):
async with self.getTestCore() as core:
layr = core.view.layers[0]
tmpdb = layr.layrslab.initdb('temp', dupsort=True)
stor = s_layer.StorTypeFloat(s_layer.STOR_TYPE_FLOAT64, 8)
vals = [
math.nan, -math.inf, -99999.9, -0.0000000001, -42.1, -0.0, 0.0,
0.000001, 42.1, 99999.9, math.inf
]
indxby = s_layer.IndxBy(layr, b'', tmpdb)
self.raises(s_exc.NoSuchImpl, indxby.getNodeValu, s_common.guid())
for key, val in ((stor.indx(v), s_msgpack.en(v)) for v in vals):
layr.layrslab.put(key[0], val, db=tmpdb)
# = -99999.9
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '=', -99999.9)
]
self.eq(retn, [-99999.9])
# <= -99999.9
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '<=', -99999.9)
]
self.eq(retn, [-math.inf, -99999.9])
# < -99999.9
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '<', -99999.9)
]
self.eq(retn, [-math.inf])
# > 99999.9
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '>', 99999.9)
]
self.eq(retn, [math.inf])
# >= 99999.9
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '>=', 99999.9)
]
self.eq(retn, [99999.9, math.inf])
# <= 0.0
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '<=', 0.0)
]
self.eq(retn,
[-math.inf, -99999.9, -42.1, -0.0000000001, -0.0, 0.0])
# >= -0.0
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '>=', -0.0)
]
self.eq(retn, [-0.0, 0.0, 0.000001, 42.1, 99999.9, math.inf])
# >= -42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '>=', -42.1)
]
self.eq(retn, [
-42.1, -0.0000000001, -0.0, 0.0, 0.000001, 42.1, 99999.9,
math.inf
])
# > -42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '>', -42.1)
]
self.eq(
retn,
[-0.0000000001, -0.0, 0.0, 0.000001, 42.1, 99999.9, math.inf])
# < 42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '<', 42.1)
]
self.eq(retn, [
-math.inf, -99999.9, -42.1, -0.0000000001, -0.0, 0.0, 0.000001
])
# <= 42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, '<=', 42.1)
]
self.eq(retn, [
-math.inf, -99999.9, -42.1, -0.0000000001, -0.0, 0.0, 0.000001,
42.1
])
# -42.1 to 42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, 'range=', (-42.1, 42.1))
]
self.eq(retn, [-42.1, -0.0000000001, -0.0, 0.0, 0.000001, 42.1])
# 1 to 42.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, 'range=', (1.0, 42.1))
]
self.eq(retn, [42.1])
# -99999.9 to -0.1
retn = [
s_msgpack.un(valu)
async for valu in stor.indxBy(indxby, 'range=', (-99999.9,
-0.1))
]
self.eq(retn, [-99999.9, -42.1])
# <= NaN
await self.agenraises(s_exc.NotANumberCompared,
stor.indxBy(indxby, '<=', math.nan))
# >= NaN
await self.agenraises(s_exc.NotANumberCompared,
stor.indxBy(indxby, '>=', math.nan))
# 1.0 to NaN
await self.agenraises(
s_exc.NotANumberCompared,
stor.indxBy(indxby, 'range=', (1.0, math.nan)))
async def test_forms_dns_simple(self):
async with self.getTestCore() as core:
async with await core.snap() as snap:
# inet:dns:a
node = await snap.addNode('inet:dns:a', ('hehe.com', '1.2.3.4'))
self.eq(node.ndef[1], ('hehe.com', 0x01020304))
self.eq(node.get('fqdn'), 'hehe.com')
self.eq(node.get('ipv4'), 0x01020304)
node = await snap.addNode('inet:dns:a', ('www.\u0915\u0949\u092e.com', '1.2.3.4'))
self.eq(node.ndef[1], ('www.xn--11b4c3d.com', 0x01020304))
self.eq(node.get('fqdn'), 'www.xn--11b4c3d.com')
self.eq(node.get('ipv4'), 0x01020304)
# inet:dns:aaaa
node = await snap.addNode('inet:dns:aaaa', ('localhost', '::1'))
self.eq(node.ndef[1], ('localhost', '::1'))
self.eq(node.get('fqdn'), 'localhost')
self.eq(node.get('ipv6'), '::1')
node = await snap.addNode('inet:dns:aaaa', ('hehe.com', '2001:0db8:85a3:0000:0000:8a2e:0370:7334'))
self.eq(node.ndef[1], ('hehe.com', '2001:db8:85a3::8a2e:370:7334'))
self.eq(node.get('fqdn'), 'hehe.com')
self.eq(node.get('ipv6'), '2001:db8:85a3::8a2e:370:7334')
# inet:dns:rev
node = await snap.addNode('inet:dns:rev', ('1.2.3.4', 'bebe.com'))
self.eq(node.ndef[1], (0x01020304, 'bebe.com'))
self.eq(node.get('ipv4'), 0x01020304)
self.eq(node.get('fqdn'), 'bebe.com')
# inet:dns:rev6
node = await snap.addNode('inet:dns:rev6', ('FF::56', 'bebe.com'))
self.eq(node.ndef[1], ('ff::56', 'bebe.com'))
self.eq(node.get('ipv6'), 'ff::56')
self.eq(node.get('fqdn'), 'bebe.com')
# inet:dns:ns
node = await snap.addNode('inet:dns:ns', ('haha.com', 'ns1.haha.com'))
self.eq(node.ndef[1], ('haha.com', 'ns1.haha.com'))
self.eq(node.get('zone'), 'haha.com')
self.eq(node.get('ns'), 'ns1.haha.com')
# inet:dns:cname
node = await snap.addNode('inet:dns:cname', ('HAHA.vertex.link', 'vertex.link'))
self.eq(node.ndef[1], ('haha.vertex.link', 'vertex.link'))
self.eq(node.get('fqdn'), 'haha.vertex.link')
self.eq(node.get('cname'), 'vertex.link')
# inet:dns:mx
node = await snap.addNode('inet:dns:mx', ('vertex.link', 'mail.vertex.link'))
self.eq(node.ndef[1], ('vertex.link', 'mail.vertex.link'))
self.eq(node.get('fqdn'), 'vertex.link')
self.eq(node.get('mx'), 'mail.vertex.link')
# inet:dns:soa
guid = s_common.guid()
props = {'fqdn': 'haha.vertex.link', 'ns': 'ns1.vertex.link', 'email': '*****@*****.**'}
node = await snap.addNode('inet:dns:soa', guid, props)
self.eq(node.get('fqdn'), 'haha.vertex.link')
self.eq(node.get('email'), '*****@*****.**')
self.eq(node.get('ns'), 'ns1.vertex.link')
# inet:dns:txt
node = await snap.addNode('inet:dns:txt', ('clowns.vertex.link', 'we all float down here'))
self.eq(node.ndef[1], ('clowns.vertex.link', 'we all float down here'))
self.eq(node.get('fqdn'), 'clowns.vertex.link')
self.eq(node.get('txt'), 'we all float down here')
async def test_model_risk(self):
async with self.getTestCore() as core:
attk = s_common.guid()
camp = s_common.guid()
org0 = s_common.guid()
pers = s_common.guid()
host = s_common.guid()
vuln = s_common.guid()
soft = s_common.guid()
hasv = s_common.guid()
plac = s_common.guid()
spec = s_common.guid()
item = s_common.guid()
async def addNode(text):
nodes = await core.nodes(text)
return nodes[0]
node = await addNode(f'''[
risk:attack={attk}
:time=20200202
:success=true
:targeted=true
:goal=*
:type=foo.bar
:desc=wootwoot
:campaign={camp}
:prev={attk}
:actor:org={org0}
:actor:person={pers}
:target = *
:attacker = *
:target:org={org0}
:target:host={host}
:target:place={plac}
:target:person={pers}
:via:ipv4=1.2.3.4
:via:ipv6=ff::01
:via:[email protected]
:via:phone=1234567890
:used:vuln={vuln}
:used:url=https://attacker.com/
:used:host={host}
:used:[email protected]
:used:file="*"
:used:server=tcp://1.2.3.4/
:used:software={soft}
]''')
self.eq(node.ndef, ('risk:attack', attk))
self.eq(node.get('time'), 1580601600000)
self.eq(node.get('desc'), 'wootwoot')
self.eq(node.get('type'), 'foo.bar.')
self.eq(node.get('success'), True)
self.eq(node.get('targeted'), True)
self.eq(node.get('campaign'), camp)
self.eq(node.get('prev'), attk)
self.eq(node.get('actor:org'), org0)
self.eq(node.get('actor:person'), pers)
self.eq(node.get('target:org'), org0)
self.eq(node.get('target:host'), host)
self.eq(node.get('target:place'), plac)
self.eq(node.get('target:person'), pers)
self.eq(node.get('via:ipv4'), 0x01020304)
self.eq(node.get('via:ipv6'), 'ff::1')
self.eq(node.get('via:email'), '*****@*****.**')
self.eq(node.get('via:phone'), '1234567890')
self.eq(node.get('used:vuln'), vuln)
self.eq(node.get('used:url'), 'https://attacker.com/')
self.eq(node.get('used:host'), host)
self.eq(node.get('used:email'), '*****@*****.**')
self.eq(node.get('used:server'), 'tcp://1.2.3.4')
self.eq(node.get('used:software'), soft)
self.nn(node.get('used:file'))
self.nn(node.get('goal'))
self.nn(node.get('target'))
self.nn(node.get('attacker'))
self.len(1, await core.nodes('risk:attack -> risk:attacktype'))
node = await addNode(f'''[
risk:vuln={vuln}
:name=myvuln
:type=mytype
:desc=mydesc
:cve=cve-2013-0000
]''')
self.eq(node.ndef, ('risk:vuln', vuln))
self.eq(node.get('name'), 'myvuln')
self.eq(node.get('type'), 'mytype')
self.eq(node.get('desc'), 'mydesc')
self.eq(node.get('cve'), 'cve-2013-0000')
self.len(1, await core.nodes('risk:attack :target -> ps:contact'))
self.len(1, await
core.nodes('risk:attack :attacker -> ps:contact'))
node = await addNode(f'''[
risk:hasvuln={hasv}
:vuln={vuln}
:person={pers}
:org={org0}
:place={plac}
:software={soft}
:hardware=*
:spec={spec}
:item={item}
:host={host}
]''')
self.eq(node.ndef, ('risk:hasvuln', hasv))
self.eq(node.get('vuln'), vuln)
self.eq(node.get('person'), pers)
self.eq(node.get('org'), org0)
self.eq(node.get('place'), plac)
self.eq(node.get('software'), soft)
self.eq(node.get('spec'), spec)
self.eq(node.get('item'), item)
self.eq(node.get('host'), host)
self.nn(node.get('hardware'))
self.len(1, await core.nodes('risk:hasvuln -> it:prod:hardware'))
nodes = await core.nodes('''
[ risk:alert=*
:type=BazFaz
:name=FooBar
:desc=BlahBlah
:detected=20501217
:attack=*
:vuln=*
]
''')
self.len(1, nodes)
self.eq('bazfaz', nodes[0].get('type'))
self.eq('FooBar', nodes[0].get('name'))
self.eq('BlahBlah', nodes[0].get('desc'))
self.eq(2554848000000, nodes[0].get('detected'))
self.len(1, await core.nodes('risk:alert -> risk:vuln'))
self.len(1, await core.nodes('risk:alert -> risk:attack'))
nodes = await core.nodes('''[
risk:compromise=*
:name = "Visi Wants Pizza"
:desc = "Visi wants a pepperoni and mushroom pizza"
:type = when.noms.attack
:target = {[ ps:contact=* :name=ledo ]}
:attacker = {[ ps:contact=* :name=visi ]}
:campaign = *
:time = 20210202
:lasttime = 20210204
:duration = 2D
:loss:pii = 400
:loss:econ = 1337
:loss:life = 0
:loss:bytes = 1024
:ransom:paid = 1
:ransom:price = 99
:response:cost = 1010
:econ:currency = usd
]''')
self.eq('visi wants pizza', nodes[0].get('name'))
self.eq('Visi wants a pepperoni and mushroom pizza',
nodes[0].get('desc'))
self.eq('when.noms.attack.', nodes[0].get('type'))
self.nn(nodes[0].get('target'))
self.nn(nodes[0].get('attacker'))
self.nn(nodes[0].get('campaign'))
self.eq(1612224000000, nodes[0].get('time'))
self.eq(1612396800000, nodes[0].get('lasttime'))
self.eq(172800000, nodes[0].get('duration'))
self.eq(400, nodes[0].get('loss:pii'))
self.eq('1337', nodes[0].get('loss:econ'))
self.eq(0, nodes[0].get('loss:life'))
self.eq(1024, nodes[0].get('loss:bytes'))
self.eq('1', nodes[0].get('ransom:paid'))
self.eq('99', nodes[0].get('ransom:price'))
self.eq('1010', nodes[0].get('response:cost'))
self.eq('usd', nodes[0].get('econ:currency'))
self.len(1, await core.nodes('risk:compromise -> ou:campaign'))
self.len(
1, await core.nodes('risk:compromise -> risk:compromisetype'))
self.len(
1, await core.nodes(
'risk:compromise :target -> ps:contact +:name=ledo'))
self.len(
1, await core.nodes(
'risk:compromise :attacker -> ps:contact +:name=visi'))
def _getSyncIden(self):
iden = s_common.guid()
evnt = asyncio.Event()
self.syncevents[iden] = evnt
return iden, evnt
def _onTeleCallMesg(self, sock, mesg):
# tele:call - call a method on a shared object
jid = mesg[1].get('jid')
sid = mesg[1].get('sid')
# check if the socket knows about their auth
# ( most likely via SSL client cert )
user = sock.get('syn:user')
with s_scope.enter({'dmon': self, 'sock': sock, 'syn:user': user, 'syn:auth': self.auth}):
try:
name = mesg[1].get('name')
item = self.shared.get(name)
if item is None:
# is it a pushed object?
pushsock = self.pushed.get(name)
if pushsock is not None:
# pass along how to reply
mesg[1]['suid'] = sock.iden
return pushsock.tx(mesg)
raise s_common.NoSuchObj(name)
task = mesg[1].get('task')
meth, args, kwargs = task
self._reqUserAllowed(user, 'tele:call', name, meth)
func = getattr(item, meth, None)
if func is None:
raise s_common.NoSuchMeth(meth)
if getattr(func, '_tele_clientside', False):
name = s_reflect.getMethName(func)
raise s_common.TeleClientSide(name=name)
logger.debug('Executing %s/%r for [%r]', jid, func, user)
ret = func(*args, **kwargs)
logger.debug('Done executing %s', jid)
# handle generator returns specially
if isinstance(ret, types.GeneratorType):
iden = s_common.guid()
txwait = threading.Event()
# start off set...
txwait.set()
self._dmon_yields.add(iden)
sock.tx(s_common.tufo('tele:yield:init', jid=jid, iden=iden))
# FIXME opt
maxsize = 100000000
def ontxsize(m):
size = m[1].get('size')
if size >= maxsize:
txwait.clear()
else:
txwait.set()
try:
sock.onfini(txwait.set)
sock.on('sock:tx:size', ontxsize)
for item in ret:
txwait.wait()
# check if we woke due to fini
if sock.isfini:
break
sock.tx(s_common.tufo('tele:yield:item', iden=iden, item=item))
if iden not in self._dmon_yields:
break
finally:
sock.off('sock:tx:size', ontxsize)
self._dmon_yields.discard(iden)
sock.tx(s_common.tufo('tele:yield:fini', iden=iden))
return
sock.tx(s_common.tufo('job:done', jid=jid, ret=ret))
except Exception as e:
errinfo = s_common.excinfo(e)
sock.tx(s_common.tufo('job:done', jid=jid, err=errinfo.get('err'), errinfo=errinfo))
async def add(self,
useriden,
query: str,
reqs,
incunit=None,
incvals=None):
'''
Persistently adds an appointment
Args:
query (str):
storm query to run
reqs (Union[None, Dict[TimeUnit, Union[int, Tuple[int]], List[...]):
one or more dicts of the fixed aspects of the appointment. dict value may be a single or multiple.
May be an empty dict or None.
incunit (Union[None, TimeUnit]):
the unit that changes for recurring, or None for non-recurring. It is an error for this value to match
a key in reqdict.
incvals (Union[None, int, Iterable[int]): count of units of incunit or explicit day of week or day of month.
Not allowed for incunit == None, required for others (1 would be a typical
value)
Notes:
For values in reqs that are lists and incvals if a list, all combinations of all values (the product) are
used
Returns:
iden of new appointment
'''
iden = s_common.guid()
recur = incunit is not None
indx = self._next_indx
self._next_indx += 1
if reqs is None:
reqs = {}
if not query:
raise ValueError('empty query')
if not reqs and incunit is None:
raise ValueError(
'at least one of reqs and incunit must be non-empty')
if incunit is not None and incvals is None:
raise ValueError('incvals must be non-None if incunit is non-None')
if isinstance(reqs, Mapping):
reqs = [reqs]
# Find all combinations of values in reqdict values and incvals values
recs = []
for req in reqs:
reqdicts = self._dictproduct(req)
if not isinstance(incvals, Iterable):
incvals = (incvals, )
recs.extend(
ApptRec(rd, incunit, v)
for (rd, v) in itertools.product(reqdicts, incvals))
appt = _Appt(iden, recur, indx, query, useriden, recs)
self._addappt(iden, appt)
await self._storeAppt(appt)
return iden
def __init__(self):
self.retn = None
self.iden = s_common.guid()
self.done = asyncio.Event()
async def test_model_x509(self):
async with self.getTestCore() as core:
crl = s_common.guid()
cert = s_common.guid()
icert = s_common.guid()
fileguid = 'guid:' + s_common.guid()
nodes = await core.nodes('''
[ crypto:x509:cert=$icert
:subject="CN=issuer.link"
:issuer:cert=$icert
:selfsigned=$lib.true
]
''', opts={'vars': {'icert': icert}})
self.eq(nodes[0].ndef, ('crypto:x509:cert', icert))
self.eq(nodes[0].get('subject'), "CN=issuer.link")
self.eq(nodes[0].get('issuer:cert'), icert)
self.eq(nodes[0].get('selfsigned'), True)
nodes = await core.nodes('''
[ crypto:x509:cert=$cert
:subject="CN=vertex.link"
:issuer="DN FOO THING"
:issuer:cert=$icert
:serial=12345
:version=v3
:validity:notafter=2019
:validity:notbefore=2015
:md5=$md5
:sha1=$sha1
:sha256=$sha256
:algo=1.2.840.113549.1.1.11
:rsa:key=(ff00ff00, 100)
:signature=ff00ff00
:ext:sans=((dns, vertex.link), (dns, "*.vertex.link"))
:ext:crls = ((dns, http://vertex.link/crls),)
:crl:urls = ("http://vertex.link/crls",)
:identities:urls=(http://woot.com/1, http://woot.com/2)
:identities:fqdns=(vertex.link, woot.com)
:identities:ipv4s=(1.2.3.4, 5.5.5.5)
:identities:ipv6s=(ff::11, ff::aa)
:identities:emails=([email protected], [email protected])
]
''', opts={'vars': {'icert': icert, 'cert': cert, 'md5': TEST_MD5, 'sha1': TEST_SHA1, 'sha256': TEST_SHA256}})
self.eq(nodes[0].ndef, ('crypto:x509:cert', cert))
self.eq(nodes[0].get('subject'), "CN=vertex.link")
self.eq(nodes[0].get('issuer'), "DN FOO THING")
self.eq(nodes[0].get('issuer:cert'), icert)
self.eq(nodes[0].get('serial'), "12345")
self.eq(nodes[0].get('version'), 2)
self.eq(nodes[0].get('validity:notafter'), 1546300800000)
self.eq(nodes[0].get('validity:notbefore'), 1420070400000)
self.eq(nodes[0].get('md5'), TEST_MD5)
self.eq(nodes[0].get('sha1'), TEST_SHA1)
self.eq(nodes[0].get('sha256'), TEST_SHA256)
self.eq(nodes[0].get('algo'), '1.2.840.113549.1.1.11')
self.eq(nodes[0].get('rsa:key'), ('ff00ff00', 100))
self.eq(nodes[0].get('signature'), 'ff00ff00')
self.eq(nodes[0].get('ext:crls'), (('dns', 'http://vertex.link/crls'),))
self.eq(nodes[0].get('crl:urls'), ('http://vertex.link/crls',))
self.eq(nodes[0].get('ext:sans'), (('dns', 'vertex.link'), ('dns', '*.vertex.link')))
self.eq(nodes[0].get('identities:urls'), ('http://woot.com/1', 'http://woot.com/2'))
self.eq(nodes[0].get('identities:fqdns'), ('vertex.link', 'woot.com'))
self.eq(nodes[0].get('identities:ipv4s'), (0x01020304, 0x05050505))
self.eq(nodes[0].get('identities:ipv6s'), ('ff::11', 'ff::aa'))
nodes = await core.nodes('''
[
crypto:x509:crl=$crl
:url=http://vertex.link/crls
:file="*"
]
''', opts={'vars': {'crl': crl}})
self.eq(nodes[0].ndef, ('crypto:x509:crl', crl))
self.nn(nodes[0].get('file'))
self.eq(nodes[0].get('url'), 'http://vertex.link/crls')
opts = {'vars': {'cert': cert, 'file': fileguid}}
nodes = await core.nodes('[ crypto:x509:signedfile = ($cert, $file) ]', opts=opts)
self.eq(nodes[0].ndef, ('crypto:x509:signedfile', (cert, fileguid)))
self.eq(nodes[0].get('cert'), cert)
self.nn(nodes[0].get('file'), fileguid)
opts = {'vars': {'cert': cert, 'crl': crl}}
nodes = await core.nodes('[ crypto:x509:revoked = ($crl, $cert) ]', opts=opts)
self.eq(nodes[0].ndef, ('crypto:x509:revoked', (crl, cert)))
self.eq(nodes[0].get('crl'), crl)
self.nn(nodes[0].get('cert'), cert)
async def test_storm(self):
help_msg = 'Execute a storm query.'
async with self.getTestCoreAndProxy() as (realcore, core):
await self.agenlen(
1, core.eval("[ test:str=abcd :tick=2015 +#cool ]"))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('help storm')
outp.expect(help_msg)
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm help')
outp.expect('For detailed help on any command')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm')
outp.expect(help_msg)
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --debug test:str=abcd')
outp.expect("('init',")
outp.expect("('node',")
outp.expect("('fini',")
outp.expect("tick")
outp.expect("tock")
outp.expect("took")
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --debug test:str=zzz')
outp.expect("('init',")
self.false(outp.expect("('node',", throw=False))
outp.expect("('fini',")
outp.expect("tick")
outp.expect("tock")
outp.expect("took")
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm test:str=b')
outp.expect('complete. 0 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm test:str=abcd')
outp.expect(':tick = 2015/01/01 00:00:00.000')
outp.expect('#cool')
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --hide-tags test:str=abcd')
outp.expect(':tick = 2015/01/01 00:00:00.000')
self.false(outp.expect('#cool', throw=False))
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --hide-props test:str=abcd')
self.false(
outp.expect(':tick = 2015/01/01 00:00:00.000', throw=False))
outp.expect('#cool')
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --show print,foo:bar test:str=abcd')
self.false(
outp.expect(':tick = 2015/01/01 00:00:00.000', throw=False))
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine(
'storm --hide-tags --hide-props test:str=abcd')
self.false(
outp.expect(':tick = 2015/01/01 00:00:00.000', throw=False))
self.false(outp.expect('#cool', throw=False))
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --raw test:str=abcd')
outp.expect("'tick': 1420070400000")
outp.expect("'tags': {'cool': (None, None)")
outp.expect('complete. 1 nodes')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --bad')
outp.expect('Syntax Error')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm newpz')
outp.expect('NoSuchName')
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm --hide-unknown [test:str=1234]')
s = str(outp)
self.notin('node:add', s)
self.notin('prop:set', s)
await self.agenlen(1, core.eval('[test:comp=(1234, 5678)]'))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
q = 'storm --raw --path test:comp -> test:int'
await cmdr.runCmdLine(q)
self.true(outp.expect("('test:int', 1234)"))
self.true(outp.expect("'path'"))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm [ test:str=foo +#bar.baz=(2015,?) ]')
self.true(
outp.expect('#bar.baz = (2015/01/01 00:00:00.000, ?)',
throw=False))
self.false(outp.expect('#bar ', throw=False))
outp.expect('complete. 1 nodes')
# Warning test
guid = s_common.guid()
podes = await alist(core.eval(f'[test:guid={guid}]'))
podes = await alist(
core.eval(
f'[test:edge=(("test:guid", {guid}), ("test:str", abcd))]')
)
q = 'storm test:str=abcd <- test:edge :n1:form -> *'
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine(q)
e = 'WARNING: The source property "n1:form" type "str" is not a form. Cannot pivot.'
self.true(outp.expect(e))
# Err case
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine('storm test:str -> test:newp')
self.true(outp.expect('ERROR'))
self.true(outp.expect('NoSuchProp'))
self.true(outp.expect('test:newp'))
# Cancelled case
evnt = asyncio.Event()
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
def setEvt(event):
smsg = event[1].get('mesg')
if smsg[0] == 'node':
evnt.set()
async def runLongStorm():
with cmdr.onWith('storm:mesg', setEvt):
await cmdr.runCmdLine('storm .created | sleep 10')
task = realcore.schedCoro(runLongStorm())
self.true(await asyncio.wait_for(evnt.wait(), timeout=6))
ps = await core.ps()
self.len(1, ps)
iden = ps[0].get('iden')
await core.kill(iden)
await asyncio.sleep(0)
self.true(outp.expect('query canceled.'))
self.true(task.done())
# Color test
outp.clear()
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
await cmdr.runCmdLine(f'storm test:{"x"*50} -> * -> $')
outp.expect('-> *')
outp.expect('Syntax Error')
outp.clear()
with self.withCliPromptMock() as patch:
cmdr = await s_cmdr.getItemCmdr(core, outp=outp)
cmdr.colorsenabled = True
await cmdr.runCmdLine('storm [#foo]')
await cmdr.runCmdLine('storm test:str ->')
lines = self.getMagicPromptColors(patch)
clines = []
for (color, text) in lines:
if text.startswith('Syntax Error:'):
text = 'Syntax Error'
clines.append((color, text))
self.isin(('#6faef2', '[#foo]'), clines)
self.isin(('#6faef2', ' ^'), clines)
self.isin(('#ff0066', 'Syntax Error'), clines)
self.isin(('#6faef2', 'test:str ->'), clines)
self.isin(('#6faef2', ' ^'), clines)
# Test that trying to print an \r doesn't assert (prompt_toolkit bug)
# https://github.com/prompt-toolkit/python-prompt-toolkit/issues/915
await core.addNode('test:str',
'foo',
props={'hehe': 'windows\r\nwindows\r\n'})
await cmdr.runCmdLine('storm test:str=foo')
self.true(1)
def test_eip55(self):
# Test bad input on eip55
v = s_common.guid() + 'X'
self.none(s_coin.ether_eip55(v))
async def test_telco_simple(self):
async with self.getTestCore() as core:
typ = core.model.type('tel:mob:mcc')
self.eq(typ.norm('001')[0], '001')
self.raises(s_exc.BadTypeValu, typ.norm, '01')
self.raises(s_exc.BadTypeValu, typ.norm, '0001')
typ = core.model.type('tel:mob:mnc')
self.eq(typ.norm('01')[0], '01')
self.eq(typ.norm('001')[0], '001')
self.raises(s_exc.BadTypeValu, typ.norm, '0001')
self.raises(s_exc.BadTypeValu, typ.norm, '1')
async with await core.snap() as snap:
# tel:mob:tac
oguid = s_common.guid()
props = {
'manu': 'Acme Corp',
'model': 'eYephone 9000',
'internal': 'spYphone 9000',
'org': oguid,
}
node = await snap.addNode('tel:mob:tac', 1, props)
self.eq(node.ndef[1], 1)
self.eq(node.get('manu'), 'acme corp')
self.eq(node.get('model'), 'eyephone 9000')
self.eq(node.get('internal'), 'spyphone 9000')
self.eq(node.get('org'), oguid)
# defvals
node = await snap.addNode('tel:mob:tac', 2)
self.eq(node.get('manu'), '??')
self.eq(node.get('model'), '??')
self.eq(node.get('internal'), '??')
# tel:mob:imid
node = await snap.addNode('tel:mob:imid',
(490154203237518, 310150123456789))
self.eq(node.ndef[1], (490154203237518, 310150123456789))
self.eq(node.get('imei'), 490154203237518)
self.eq(node.get('imsi'), 310150123456789)
# tel:mob:imsiphone
node = await snap.addNode(
'tel:mob:imsiphone',
(310150123456789, '+7(495) 124-59-83'))
self.eq(node.ndef[1], (310150123456789, '74951245983'))
self.eq(node.get('imsi'), 310150123456789)
self.eq(node.get('phone'), '74951245983')
# tel:mob:mcc
node = await snap.addNode('tel:mob:mcc', '611', {'loc': 'gn'})
self.eq(node.ndef[1], '611')
self.eq(node.get('loc'), 'gn')
# tel:mob:carrier
node = await snap.addNode('tel:mob:carrier', ('001', '02'), {
'org': oguid,
'loc': 'us'
})
self.eq(node.ndef[1], ('001', '02'))
self.eq(node.get('mcc'), '001')
self.eq(node.get('mnc'), '02')
self.eq(node.get('org'), oguid)
self.eq(node.get('loc'), 'us')
# tel:mob:cell
node = await snap.addNode('tel:mob:cell',
(('001', '02'), 3, 4), {
'radio': 'Pirate ',
'latlong': (0, 0),
'loc': 'us.ca.la'
})
self.eq(node.get('carrier'), ('001', '02'))
self.eq(node.get('carrier:mcc'), '001')
self.eq(node.get('carrier:mnc'), '02')
self.eq(node.get('lac'), 3)
self.eq(node.get('cid'), 4)
self.eq(node.get('loc'), 'us.ca.la')
self.eq(node.get('radio'), 'pirate')
self.eq(node.get('latlong'), (0.0, 0.0))
# tel:mob:telem
guid = s_common.guid()
softguid = s_common.guid()
props = {
'time': '2001',
'latlong': (-1, 1),
'accuracy': '100mm',
'cell': (('001', '02'), 3, 4),
'imsi': '310150123456789',
'imei': '490154203237518',
'phone': '123 456 7890',
'mac': '00:00:00:00:00:00',
'ipv4': '1.2.3.4',
'ipv6': '::1',
'wifi:ssid': 'The Best SSID2',
'wifi:bssid': '00:11:22:33:44:55',
'aaid': 'somestr',
'idfa': 'someotherstr',
'name': 'Robert Grey',
'email': '*****@*****.**',
'acct': ('vertex.link', 'clown'),
'app': softguid,
'data': {
'some key': 'some valu',
'BEEP': 1
}
}
node = await snap.addNode('tel:mob:telem', guid, props)
self.eq(node.ndef[1], guid)
self.eq(node.get('time'), 978307200000)
self.eq(node.get('latlong'), (-1.0, 1.0))
self.eq(node.get('accuracy'), 100)
self.eq(node.get('cell'), (('001', '02'), 3, 4))
self.eq(node.get('cell:carrier'), ('001', '02'))
self.eq(node.get('imsi'), 310150123456789)
self.eq(node.get('imei'), 490154203237518)
self.eq(node.get('phone'), '1234567890')
self.eq(node.get('mac'), '00:00:00:00:00:00')
self.eq(node.get('ipv4'), 0x01020304)
self.eq(node.get('ipv6'), '::1')
self.eq(node.get('wifi:ssid'), 'The Best SSID2')
self.eq(node.get('wifi:bssid'), '00:11:22:33:44:55')
self.eq(node.get('aaid'), 'somestr')
self.eq(node.get('idfa'), 'someotherstr')
self.eq(node.get('name'), 'robert grey')
self.eq(node.get('email'), '*****@*****.**')
self.eq(node.get('acct'), ('vertex.link', 'clown'))
self.eq(node.get('app'), softguid)
self.eq(node.get('data'), {'some key': 'some valu', 'BEEP': 1})
async def test_ou_simple(self):
async with self.getTestCore() as core:
# type norming first
# ou:name
t = core.model.type('ou:name')
norm, subs = t.norm('Acme Corp ')
self.eq(norm, 'acme corp')
# ou:naics
t = core.model.type('ou:naics')
norm, subs = t.norm(541715)
self.eq(norm, '541715')
self.raises(s_exc.BadTypeValu, t.norm, 'newp')
self.raises(s_exc.BadTypeValu, t.norm, 1000000)
self.raises(s_exc.BadTypeValu, t.norm, 1000)
# ou:sic
t = core.model.type('ou:sic')
norm, subs = t.norm('7999')
self.eq(norm, '7999')
norm, subs = t.norm(9999)
self.eq(norm, '9999')
norm, subs = t.norm('0111')
self.eq(norm, '0111')
self.raises(s_exc.BadTypeValu, t.norm, -1)
self.raises(s_exc.BadTypeValu, t.norm, 0)
self.raises(s_exc.BadTypeValu, t.norm, 111)
self.raises(s_exc.BadTypeValu, t.norm, 10000)
# ou:alias
t = core.model.type('ou:alias')
self.raises(s_exc.BadTypeValu, t.norm, 'asdf.asdf.asfd')
self.eq(t.norm('HAHA1')[0], 'haha1')
self.eq(t.norm('GOV_MFA')[0], 'gov_mfa')
async with await core.snap() as snap:
guid0 = s_common.guid()
name = '\u21f1\u21f2 Inc.'
normname = '\u21f1\u21f2 inc.'
altnames = (
'altarrowname',
'otheraltarrow',
)
oprops = {
'loc': 'US.CA',
'name': name,
'names': altnames,
'alias': 'arrow',
'phone': '+15555555555',
'sic': '0119',
'naics': 541715,
'url': 'http://www.arrowinc.link',
'us:cage': '7qe71',
'founded': '2015',
'dissolved': '2019',
}
node = await snap.addNode('ou:org', guid0, oprops)
self.eq(node.ndef[1], guid0),
self.eq(node.get('loc'), 'us.ca')
self.eq(node.get('name'), normname)
self.eq(node.get('names'), altnames)
self.eq(node.get('alias'), 'arrow')
self.eq(node.get('phone'), '15555555555')
self.eq(node.get('sic'), '0119')
self.eq(node.get('naics'), '541715')
self.eq(node.get('url'), 'http://www.arrowinc.link')
self.eq(node.get('us:cage'), '7qe71')
self.eq(node.get('founded'), 1420070400000)
self.eq(node.get('dissolved'), 1546300800000)
nodes = await snap.nodes('ou:name')
self.sorteq([x.ndef[1] for x in nodes],
(normname, ) + altnames)
nodes = await snap.nodes('ou:org:names*[=otheraltarrow]')
self.len(1, nodes)
opts = {'var': {'name': name}}
nodes = await snap.nodes('ou:org:names*contains=$name',
opts=opts)
self.len(0,
nodes) # primary ou:org:name is not in ou:org:names
person0 = s_common.guid()
mprops = {
'title': 'Dancing Clown',
'start': '2001',
'end': '2010',
}
node = await snap.addNode('ou:member', (guid0, person0),
mprops)
self.eq(node.ndef[1], (guid0, person0))
self.eq(node.get('title'), 'dancing clown')
self.eq(node.get('start'), 978307200000)
self.eq(node.get('end'), 1262304000000)
# ou:suborg
guid1 = s_common.guid()
subprops = {
'perc': 50,
'current': True,
}
node = await snap.addNode('ou:suborg', (guid0, guid1),
subprops)
self.eq(node.ndef[1], (guid0, guid1))
self.eq(node.get('perc'), 50)
self.eq(node.get('current'), 1)
await self.asyncraises(s_exc.BadPropValu, node.set('perc', -1))
await self.asyncraises(s_exc.BadPropValu,
node.set('perc', 101))
# ou:user
node = await snap.addNode('ou:user', (guid0, 'arrowman'))
self.eq(node.ndef[1], (guid0, 'arrowman'))
self.eq(node.get('org'), guid0)
self.eq(node.get('user'), 'arrowman')
# ou:hasalias
node = await snap.addNode('ou:hasalias', (guid0, 'EVILCORP'))
self.eq(node.ndef[1], (guid0, 'evilcorp'))
self.eq(node.get('alias'), 'evilcorp')
self.eq(node.get('org'), guid0)
# ou:orgnet4
node = await snap.addNode('ou:orgnet4',
(guid0,
('192.168.1.1', '192.168.1.127')))
self.eq(node.ndef[1], (guid0, (3232235777, 3232235903)))
self.eq(node.get('net'), (3232235777, 3232235903))
self.eq(node.get('org'), guid0)
# ou:orgnet6
node = await snap.addNode('ou:orgnet6',
(guid0, ('fd00::1', 'fd00::127')))
self.eq(node.ndef[1], (guid0, ('fd00::1', 'fd00::127')))
self.eq(node.get('net'), ('fd00::1', 'fd00::127'))
self.eq(node.get('org'), guid0)
# ou:org:has
node = await snap.addNode(
'ou:org:has',
(guid0, ('test:str', 'pretty floral bonnet')))
self.eq(node.ndef[1],
(guid0, ('test:str', 'pretty floral bonnet')))
self.eq(node.get('org'), guid0)
self.eq(node.get('node'), ('test:str', 'pretty floral bonnet'))
self.eq(node.get('node:form'), 'test:str')
# ou:meet
place0 = s_common.guid()
m0 = s_common.guid()
mprops = {
'name': 'Working Lunch',
'start': '201604011200',
'end': '201604011300',
'place': place0,
}
node = await snap.addNode('ou:meet', m0, mprops)
self.eq(node.ndef[1], m0)
self.eq(node.get('name'), 'working lunch')
self.eq(node.get('start'), 1459512000000)
self.eq(node.get('end'), 1459515600000)
self.eq(node.get('place'), place0)
mprops = {
'arrived': '201604011201',
'departed': '201604011259',
}
node = await snap.addNode('ou:meet:attendee', (m0, person0),
mprops)
self.eq(node.ndef[1], (m0, person0))
self.eq(node.get('arrived'), 1459512060000)
self.eq(node.get('departed'), 1459515540000)
# ou:conference
c0 = s_common.guid()
cprops = {
'org': guid0,
'name': 'arrowcon 2018',
'base': 'arrowcon',
'start': '20180301',
'end': '20180303',
'place': place0,
'url': 'http://arrowcon.org/2018',
}
node = await snap.addNode('ou:conference', c0, cprops)
self.eq(node.ndef[1], c0)
self.eq(node.get('name'), 'arrowcon 2018')
self.eq(node.get('base'), 'arrowcon')
self.eq(node.get('org'), guid0)
self.eq(node.get('start'), 1519862400000)
self.eq(node.get('end'), 1520035200000)
self.eq(node.get('place'), place0)
self.eq(node.get('url'), 'http://arrowcon.org/2018')
cprops = {
'arrived': '201803010800',
'departed': '201803021500',
'role:staff': False,
'role:speaker': True,
'roles': ['usher', 'coatcheck'],
}
node = await snap.addNode('ou:conference:attendee',
(c0, person0), cprops)
self.eq(node.ndef[1], (c0, person0))
self.eq(node.get('arrived'), 1519891200000)
self.eq(node.get('departed'), 1520002800000)
self.eq(node.get('role:staff'), 0)
self.eq(node.get('role:speaker'), 1)
self.eq(node.get('roles'), ('usher', 'coatcheck'))
# ou:conference:event
confguid = c0
con0 = s_common.guid()
cprops = {
'org':
guid0,
'name':
'Steve Rogers',
'title':
'The First Avenger',
'orgname':
'Avengers',
'user':
'******',
'web:acct': ('twitter.com', 'captainamerica'),
'dob':
'1918-07-04',
'url':
'https://captainamerica.com/',
'email':
'*****@*****.**',
'email:work':
'*****@*****.**',
'phone':
'12345678910',
'phone:fax':
'12345678910',
'phone:work':
'12345678910',
'address':
'222 Avenger Row, Washington, DCSan Francisco, CA, 22222, USA',
}
pscon = await snap.addNode('ps:contact', con0, cprops)
c0 = s_common.guid()
cprops = {
'conference': confguid,
'name': 'arrowcon 2018 dinner',
'desc': 'arrowcon dinner',
'start': '201803011900',
'end': '201803012200',
'contact': con0,
'place': place0,
'url': 'http://arrowcon.org/2018/dinner',
}
node = await snap.addNode('ou:conference:event', c0, cprops)
self.eq(node.ndef[1], c0)
self.eq(node.get('name'), 'arrowcon 2018 dinner')
self.eq(node.get('desc'), 'arrowcon dinner')
self.eq(node.get('conference'), confguid)
self.eq(node.get('start'), 1519930800000)
self.eq(node.get('end'), 1519941600000)
self.eq(node.get('contact'), con0)
self.eq(node.get('place'), place0)
self.eq(node.get('url'), 'http://arrowcon.org/2018/dinner')
cprops = {
'arrived': '201803011923',
'departed': '201803012300',
'roles': ['staff', 'speaker'],
}
node = await snap.addNode('ou:conference:event:attendee',
(c0, person0), cprops)
self.eq(node.ndef[1], (c0, person0))
self.eq(node.get('arrived'), 1519932180000)
self.eq(node.get('departed'), 1519945200000)
self.eq(node.get('roles'), ('staff', 'speaker'))
async def test_cell_activecoro(self):
evt0 = asyncio.Event()
evt1 = asyncio.Event()
evt2 = asyncio.Event()
evt3 = asyncio.Event()
evt4 = asyncio.Event()
async def coro():
try:
evt0.set()
await evt1.wait()
evt2.set()
await evt3.wait()
except asyncio.CancelledError:
evt4.set()
raise
with self.getTestDir() as dirn:
async with await s_cell.Cell.anit(dirn) as cell:
# Note: cell starts active, so coro should immediate run
cell.addActiveCoro(coro)
async def step():
await asyncio.wait_for(evt0.wait(), timeout=2)
# step him through...
evt1.set()
await asyncio.wait_for(evt2.wait(), timeout=2)
evt0.clear()
evt1.clear()
evt3.set()
await asyncio.wait_for(evt0.wait(), timeout=2)
await step()
self.none(await cell.delActiveCoro('notacoro'))
# Make sure a fini'd base takes its activecoros with it
async with await s_base.Base.anit() as base:
cell.addActiveCoro(coro, base=base)
self.len(2, cell.activecoros)
self.len(1, cell.activecoros)
self.raises(s_exc.IsFini, cell.addActiveCoro, coro, base=base)
# now deactivate and it gets cancelled
await cell.setCellActive(False)
await asyncio.wait_for(evt4.wait(), timeout=2)
evt0.clear()
evt1.clear()
evt2.clear()
evt3.clear()
evt4.clear()
# make him active post-init and confirm
await cell.setCellActive(True)
await step()
self.none(await cell.delActiveCoro(s_common.guid()))
async def test_ou_simple(self):
async with self.getTestCore() as core:
goal = s_common.guid()
org0 = s_common.guid()
camp = s_common.guid()
acto = s_common.guid()
async with await core.snap() as snap:
props = {
'name': 'MyGoal',
'type': 'MyType',
'desc': 'MyDesc',
'prev': goal,
}
node = await snap.addNode('ou:goal', goal, props=props)
self.eq(node.get('name'), 'MyGoal')
self.eq(node.get('type'), 'MyType')
self.eq(node.get('desc'), 'MyDesc')
self.eq(node.get('prev'), goal)
props = {
'stated': True,
'window': '2019,2020',
}
node = await snap.addNode('ou:hasgoal', (org0, goal), props=props)
self.eq(node.get('org'), org0)
self.eq(node.get('goal'), goal)
self.eq(node.get('stated'), True)
self.eq(node.get('window'), (1546300800000, 1577836800000))
props = {
'org': org0,
'goal': goal,
'goals': (goal,),
'actors': (acto,),
'camptype': 'get.pizza',
'name': 'MyName',
'type': 'MyType',
'desc': 'MyDesc',
'success': 1,
}
node = await snap.addNode('ou:campaign', camp, props=props)
self.eq(node.get('org'), org0)
self.eq(node.get('goal'), goal)
self.eq(node.get('goals'), (goal,))
self.eq(node.get('actors'), (acto,))
self.eq(node.get('name'), 'MyName')
self.eq(node.get('type'), 'MyType')
self.eq(node.get('desc'), 'MyDesc')
self.eq(node.get('success'), 1)
self.eq(node.get('camptype'), 'get.pizza.')
# type norming first
# ou:name
t = core.model.type('ou:name')
norm, subs = t.norm('Acme Corp ')
self.eq(norm, 'acme corp')
# ou:naics
t = core.model.type('ou:naics')
norm, subs = t.norm(541715)
self.eq(norm, '541715')
self.raises(s_exc.BadTypeValu, t.norm, 'newp')
self.raises(s_exc.BadTypeValu, t.norm, 1000000)
self.raises(s_exc.BadTypeValu, t.norm, 1000)
# ou:sic
t = core.model.type('ou:sic')
norm, subs = t.norm('7999')
self.eq(norm, '7999')
norm, subs = t.norm(9999)
self.eq(norm, '9999')
norm, subs = t.norm('0111')
self.eq(norm, '0111')
self.raises(s_exc.BadTypeValu, t.norm, -1)
self.raises(s_exc.BadTypeValu, t.norm, 0)
self.raises(s_exc.BadTypeValu, t.norm, 111)
self.raises(s_exc.BadTypeValu, t.norm, 10000)
# ou:isic
t = core.model.type('ou:isic')
self.eq('C', t.norm('C')[0])
self.eq('C13', t.norm('C13')[0])
self.eq('C139', t.norm('C139')[0])
self.eq('C1393', t.norm('C1393')[0])
self.raises(s_exc.BadTypeValu, t.norm, 'C1')
self.raises(s_exc.BadTypeValu, t.norm, 'C12345')
self.raises(s_exc.BadTypeValu, t.norm, 'newp')
self.raises(s_exc.BadTypeValu, t.norm, 1000000)
# ou:alias
t = core.model.type('ou:alias')
self.raises(s_exc.BadTypeValu, t.norm, 'asdf.asdf.asfd')
self.eq(t.norm('HAHA1')[0], 'haha1')
self.eq(t.norm('GOV_MFA')[0], 'gov_mfa')
# ou:position / ou:org:subs
orgiden = s_common.guid()
contact = s_common.guid()
position = s_common.guid()
subpos = s_common.guid()
suborg = s_common.guid()
opts = {'vars': {
'orgiden': orgiden,
'contact': contact,
'position': position,
'subpos': subpos,
'suborg': suborg,
}}
nodes = await core.nodes('''
[ ou:org=$orgiden :orgchart=$position ]
-> ou:position
[ :contact=$contact :title=ceo :org=$orgiden ]
''', opts=opts)
self.eq('ceo', nodes[0].get('title'))
self.eq(orgiden, nodes[0].get('org'))
self.eq(contact, nodes[0].get('contact'))
nodes = await core.nodes('''
ou:org=$orgiden
-> ou:position
[ :reports+=$subpos ]
-> ou:position
''', opts=opts)
self.eq(('ou:position', subpos), nodes[0].ndef)
nodes = await core.nodes('''
ou:org=$orgiden
[ :subs+=$suborg ]
-> ou:org
''', opts=opts)
self.eq(('ou:org', suborg), nodes[0].ndef)
async with await core.snap() as snap:
guid0 = s_common.guid()
name = '\u21f1\u21f2 Inc.'
normname = '\u21f1\u21f2 inc.'
altnames = ('altarrowname', 'otheraltarrow', )
oprops = {
'loc': 'US.CA',
'name': name,
'type': 'corp',
'orgtype': 'Corp.Lolz',
'names': altnames,
'logo': '*',
'alias': 'arrow',
'phone': '+15555555555',
'sic': '0119',
'naics': 541715,
'url': 'http://www.arrowinc.link',
'us:cage': '7qe71',
'founded': '2015',
'dissolved': '2019',
}
node = await snap.addNode('ou:org', guid0, oprops)
self.eq(node.ndef[1], guid0),
self.eq(node.get('loc'), 'us.ca')
self.eq(node.get('type'), 'corp')
self.eq(node.get('orgtype'), 'corp.lolz.')
self.eq(node.get('name'), normname)
self.eq(node.get('names'), altnames)
self.eq(node.get('alias'), 'arrow')
self.eq(node.get('phone'), '15555555555')
self.eq(node.get('sic'), '0119')
self.eq(node.get('naics'), '541715')
self.eq(node.get('url'), 'http://www.arrowinc.link')
self.eq(node.get('us:cage'), '7qe71')
self.eq(node.get('founded'), 1420070400000)
self.eq(node.get('dissolved'), 1546300800000)
self.nn(node.get('logo'))
self.len(1, await core.nodes('ou:org -> ou:orgtype'))
nodes = await snap.nodes('ou:name')
self.sorteq([x.ndef[1] for x in nodes], (normname,) + altnames)
nodes = await snap.nodes('ou:org:names*[=otheraltarrow]')
self.len(1, nodes)
opts = {'vars': {'name': name}}
nodes = await snap.nodes('ou:org:names*[=$name]', opts=opts)
self.len(0, nodes) # primary ou:org:name is not in ou:org:names
person0 = s_common.guid()
mprops = {
'title': 'Dancing Clown',
'start': '2001',
'end': '2010',
}
node = await snap.addNode('ou:member', (guid0, person0), mprops)
self.eq(node.ndef[1], (guid0, person0))
self.eq(node.get('title'), 'dancing clown')
self.eq(node.get('start'), 978307200000)
self.eq(node.get('end'), 1262304000000)
# ou:suborg
guid1 = s_common.guid()
subprops = {
'perc': 50,
'current': True,
}
node = await snap.addNode('ou:suborg', (guid0, guid1), subprops)
self.eq(node.ndef[1], (guid0, guid1))
self.eq(node.get('perc'), 50)
self.eq(node.get('current'), 1)
await self.asyncraises(s_exc.BadTypeValu, node.set('perc', -1))
await self.asyncraises(s_exc.BadTypeValu, node.set('perc', 101))
# ou:user
node = await snap.addNode('ou:user', (guid0, 'arrowman'))
self.eq(node.ndef[1], (guid0, 'arrowman'))
self.eq(node.get('org'), guid0)
self.eq(node.get('user'), 'arrowman')
# ou:hasalias
node = await snap.addNode('ou:hasalias', (guid0, 'EVILCORP'))
self.eq(node.ndef[1], (guid0, 'evilcorp'))
self.eq(node.get('alias'), 'evilcorp')
self.eq(node.get('org'), guid0)
# ou:orgnet4
node = await snap.addNode('ou:orgnet4', (guid0, ('192.168.1.1', '192.168.1.127')))
self.eq(node.ndef[1], (guid0, (3232235777, 3232235903)))
self.eq(node.get('net'), (3232235777, 3232235903))
self.eq(node.get('org'), guid0)
# ou:orgnet6
node = await snap.addNode('ou:orgnet6', (guid0, ('fd00::1', 'fd00::127')))
self.eq(node.ndef[1], (guid0, ('fd00::1', 'fd00::127')))
self.eq(node.get('net'), ('fd00::1', 'fd00::127'))
self.eq(node.get('org'), guid0)
# ou:org:has
node = await snap.addNode('ou:org:has', (guid0, ('test:str', 'pretty floral bonnet')))
self.eq(node.ndef[1], (guid0, ('test:str', 'pretty floral bonnet')))
self.eq(node.get('org'), guid0)
self.eq(node.get('node'), ('test:str', 'pretty floral bonnet'))
self.eq(node.get('node:form'), 'test:str')
# ou:meet
place0 = s_common.guid()
m0 = s_common.guid()
mprops = {
'name': 'Working Lunch',
'start': '201604011200',
'end': '201604011300',
'place': place0,
}
node = await snap.addNode('ou:meet', m0, mprops)
self.eq(node.ndef[1], m0)
self.eq(node.get('name'), 'working lunch')
self.eq(node.get('start'), 1459512000000)
self.eq(node.get('end'), 1459515600000)
self.eq(node.get('place'), place0)
mprops = {
'arrived': '201604011201',
'departed': '201604011259',
}
node = await snap.addNode('ou:meet:attendee', (m0, person0), mprops)
self.eq(node.ndef[1], (m0, person0))
self.eq(node.get('arrived'), 1459512060000)
self.eq(node.get('departed'), 1459515540000)
# ou:conference
c0 = s_common.guid()
cprops = {
'org': guid0,
'name': 'arrowcon 2018',
'base': 'arrowcon',
'start': '20180301',
'end': '20180303',
'place': place0,
'url': 'http://arrowcon.org/2018',
}
node = await snap.addNode('ou:conference', c0, cprops)
self.eq(node.ndef[1], c0)
self.eq(node.get('name'), 'arrowcon 2018')
self.eq(node.get('base'), 'arrowcon')
self.eq(node.get('org'), guid0)
self.eq(node.get('start'), 1519862400000)
self.eq(node.get('end'), 1520035200000)
self.eq(node.get('place'), place0)
self.eq(node.get('url'), 'http://arrowcon.org/2018')
cprops = {
'arrived': '201803010800',
'departed': '201803021500',
'role:staff': False,
'role:speaker': True,
'roles': ['usher', 'coatcheck'],
}
node = await snap.addNode('ou:conference:attendee', (c0, person0), cprops)
self.eq(node.ndef[1], (c0, person0))
self.eq(node.get('arrived'), 1519891200000)
self.eq(node.get('departed'), 1520002800000)
self.eq(node.get('role:staff'), 0)
self.eq(node.get('role:speaker'), 1)
self.eq(node.get('roles'), ('coatcheck', 'usher'))
# ou:conference:event
confguid = c0
con0 = s_common.guid()
cprops = {
'org': guid0,
'name': 'Steve Rogers',
'title': 'The First Avenger',
'orgname': 'Avengers',
'user': '******',
'web:acct': ('twitter.com', 'captainamerica'),
'dob': '1918-07-04',
'url': 'https://captainamerica.com/',
'email': '*****@*****.**',
'email:work': '*****@*****.**',
'phone': '12345678910',
'phone:fax': '12345678910',
'phone:work': '12345678910',
'address': '222 Avenger Row, Washington, DCSan Francisco, CA, 22222, USA',
}
pscon = await snap.addNode('ps:contact', con0, cprops)
c0 = s_common.guid()
cprops = {
'conference': confguid,
'name': 'arrowcon 2018 dinner',
'desc': 'arrowcon dinner',
'start': '201803011900',
'end': '201803012200',
'contact': con0,
'place': place0,
'url': 'http://arrowcon.org/2018/dinner',
}
node = await snap.addNode('ou:conference:event', c0, cprops)
self.eq(node.ndef[1], c0)
self.eq(node.get('name'), 'arrowcon 2018 dinner')
self.eq(node.get('desc'), 'arrowcon dinner')
self.eq(node.get('conference'), confguid)
self.eq(node.get('start'), 1519930800000)
self.eq(node.get('end'), 1519941600000)
self.eq(node.get('contact'), con0)
self.eq(node.get('place'), place0)
self.eq(node.get('url'), 'http://arrowcon.org/2018/dinner')
cprops = {
'arrived': '201803011923',
'departed': '201803012300',
'roles': ['staff', 'speaker'],
}
node = await snap.addNode('ou:conference:event:attendee', (c0, person0), cprops)
self.eq(node.ndef[1], (c0, person0))
self.eq(node.get('arrived'), 1519932180000)
self.eq(node.get('departed'), 1519945200000)
self.eq(node.get('roles'), ('speaker', 'staff'))
nodes = await core.nodes('[ ou:id:type=* :org=* :name=foobar ]')
self.len(1, nodes)
self.nn(nodes[0].get('org'))
self.eq('foobar', nodes[0].get('name'))
iden = await core.callStorm('ou:id:type return($node.value())')
opts = {'vars': {'type': iden}}
nodes = await core.nodes('[ ou:id:number=($type, visi) :status=woot :issued=202002 :expires=2021 ]', opts=opts)
self.len(1, nodes)
self.eq(('ou:id:number', (iden, 'visi')), nodes[0].ndef)
self.eq(iden, nodes[0].get('type'))
self.eq('visi', nodes[0].get('value'))
self.eq('woot', nodes[0].get('status'))
self.eq(1580515200000, nodes[0].get('issued'))
self.eq(1609459200000, nodes[0].get('expires'))
opts = {'vars': {'type': iden}}
nodes = await core.nodes('[ ou:id:update=* :number=($type, visi) :status=revoked :time=202003]', opts=opts)
self.len(1, nodes)
self.eq((iden, 'visi'), nodes[0].get('number'))
self.eq('revoked', nodes[0].get('status'))
self.eq(1583020800000, nodes[0].get('time'))
nodes = await core.nodes('[ ou:org=* :desc=hehe :hq=* :locations=(*, *) :dns:mx=(hehe.com, haha.com)]')
self.len(1, nodes)
self.eq('hehe', nodes[0].get('desc'))
opts = {'vars': {'iden': nodes[0].ndef[1]}}
self.len(3, await core.nodes('ou:org=$iden -> ps:contact', opts=opts))
self.len(1, await core.nodes('ou:org=$iden :hq -> ps:contact', opts=opts))
self.len(2, await core.nodes('ou:org=$iden :locations -> ps:contact', opts=opts))
self.len(2, await core.nodes('ou:org=$iden :dns:mx -> inet:fqdn', opts=opts))
nodes = await core.nodes('''[
ou:attendee=*
:person=*
:arrived=201202
:departed=201203
:meet=*
:preso=*
:conference=*
:conference:event=*
:roles+=staff
:roles+=STAFF
]''')
self.len(1, nodes)
self.eq(('staff',), nodes[0].get('roles'))
self.eq(1328054400000, nodes[0].get('arrived'))
self.eq(1330560000000, nodes[0].get('departed'))
self.len(1, await core.nodes('ou:attendee -> ps:contact'))
self.len(1, await core.nodes('ou:attendee -> ou:meet'))
self.len(1, await core.nodes('ou:attendee -> ou:preso'))
self.len(1, await core.nodes('ou:attendee -> ou:conference'))
self.len(1, await core.nodes('ou:attendee -> ou:conference:event'))
pres = s_common.guid()
nodes = await core.nodes(f'''[
ou:preso={pres}
:title=syn101
:desc=squeee
:time=20200808
:duration=2:00:00
:place=*
:loc=us.nv.lasvegas
:conference=*
:organizer=*
:sponsors=(*,)
:presenters=(*,*)
:deck:file=*
:recording:file=*
:deck:url=http://vertex.link/syn101deck
:attendee:url=http://vertex.link/syn101live
:recording:url=http://vertex.link/syn101recording
]''')
self.len(1, nodes)
self.eq('syn101', nodes[0].get('title'))
self.eq('squeee', nodes[0].get('desc'))
self.eq(1596844800000, nodes[0].get('time'))
self.eq(7200000, nodes[0].get('duration'))
self.eq('http://vertex.link/syn101deck', nodes[0].get('deck:url'))
self.eq('http://vertex.link/syn101live', nodes[0].get('attendee:url'))
self.eq('http://vertex.link/syn101recording', nodes[0].get('recording:url'))
self.nn(nodes[0].get('deck:file'))
self.nn(nodes[0].get('recording:file'))
self.eq('us.nv.lasvegas', nodes[0].get('loc'))
self.len(1, await core.nodes(f'ou:preso={pres} -> ou:conference'))
self.len(1, await core.nodes(f'ou:preso={pres} :sponsors -> ps:contact'))
self.len(1, await core.nodes(f'ou:preso={pres} :organizer -> ps:contact'))
self.len(2, await core.nodes(f'ou:preso={pres} :presenters -> ps:contact'))
cont = s_common.guid()
nodes = await core.nodes(f'''[
ou:contest={cont}
:name="defcon ctf 2020"
:type="cyber ctf"
:family="defcon ctf"
:start=20200808
:end=20200811
:url=http://vertex.link/contest
:loc=us.nv.lasvegas
:place=*
:latlong=(20, 30)
:conference=*
:contests=(*,*)
:sponsors=(*,)
:organizers=(*,)
:participants=(*,)
]''')
self.len(1, nodes)
self.eq('defcon ctf 2020', nodes[0].get('name'))
self.eq('cyber ctf', nodes[0].get('type'))
self.eq('defcon ctf', nodes[0].get('family'))
self.eq(1596844800000, nodes[0].get('start'))
self.eq(1597104000000, nodes[0].get('end'))
self.eq('http://vertex.link/contest', nodes[0].get('url'))
self.eq((20, 30), nodes[0].get('latlong'))
self.eq('us.nv.lasvegas', nodes[0].get('loc'))
self.len(2, await core.nodes(f'ou:contest={cont} -> ou:contest'))
self.len(1, await core.nodes(f'ou:contest={cont} -> ou:conference'))
self.len(1, await core.nodes(f'ou:contest={cont} :sponsors -> ps:contact'))
self.len(1, await core.nodes(f'ou:contest={cont} :organizers -> ps:contact'))
self.len(1, await core.nodes(f'ou:contest={cont} :participants -> ps:contact'))
nodes = await core.nodes('''[
ou:contest:result=(*, *)
:rank=1
:score=20
:url=http://vertex.link/contest/result
]''')
self.len(1, nodes)
self.nn(nodes[0].get('contest'))
self.nn(nodes[0].get('participant'))
self.eq(1, nodes[0].get('rank'))
self.eq(20, nodes[0].get('score'))
self.eq('http://vertex.link/contest/result', nodes[0].get('url'))
self.len(1, await core.nodes('ou:contest:result -> ps:contact'))
self.len(1, await core.nodes('ou:contest:result -> ou:contest'))
opts = {'vars': {'ind': s_common.guid()}}
nodes = await core.nodes('[ ou:org=* :industries=($ind, $ind) ]', opts=opts)
self.len(1, nodes)
self.len(1, nodes[0].get('industries'))
async def test_layer_nodeedits(self):
async with self.getTestCoreAndProxy() as (core0, prox0):
nodelist0 = []
nodes = await core0.nodes('[ test:str=foo ]')
nodelist0.extend(nodes)
nodes = await core0.nodes(
'[ inet:ipv4=1.2.3.4 .seen=(2012,2014) +#foo.bar=(2012, 2014) ]'
)
nodelist0.extend(nodes)
nodelist0 = [node.pack() for node in nodelist0]
editlist = []
layr = core0.getLayer()
async for offs, nodeedits in prox0.syncLayerNodeEdits(0):
editlist.append(nodeedits)
if offs == layr.nodeeditlog.index() - 1:
break
async with self.getTestCore() as core1:
url = core1.getLocalUrl('*/layer')
async with await s_telepath.openurl(url) as layrprox:
for nodeedits in editlist:
self.nn(await layrprox.storNodeEdits(nodeedits))
nodelist1 = []
nodelist1.extend(await core1.nodes('test:str'))
nodelist1.extend(await core1.nodes('inet:ipv4'))
nodelist1 = [node.pack() for node in nodelist1]
self.eq(nodelist0, nodelist1)
layr = core1.view.layers[0]
# Empty the layer to try again
await layr.truncate()
async with await s_telepath.openurl(url) as layrprox:
for nodeedits in editlist:
self.none(await
layrprox.storNodeEditsNoLift(nodeedits))
nodelist1 = []
nodelist1.extend(await core1.nodes('test:str'))
nodelist1.extend(await core1.nodes('inet:ipv4'))
nodelist1 = [node.pack() for node in nodelist1]
self.eq(nodelist0, nodelist1)
meta = {
'user': s_common.guid(),
'time': 0,
}
await layr.truncate()
for nodeedits in editlist:
self.none(await
layrprox.storNodeEditsNoLift(nodeedits,
meta=meta))
lastoffs = layr.nodeeditlog.index()
for nodeedit in layr.nodeeditlog.sliceBack(lastoffs, 2):
self.eq(meta, nodeedit[1][1])
async def test_storm_svcs(self):
with self.getTestDir() as dirn:
async with self.getTestDmon() as dmon:
dmon.share('prim', NoService())
dmon.share('real', RealService())
dmon.share('boom', BoomService())
dmon.share('dead', DeadService())
dmon.share('lift', LifterService())
host, port = dmon.addr
lurl = f'tcp://127.0.0.1:{port}/real'
purl = f'tcp://127.0.0.1:{port}/prim'
burl = f'tcp://127.0.0.1:{port}/boom'
curl = f'tcp://127.0.0.1:{port}/lift'
durl = f'tcp://127.0.0.1:{port}/dead'
async with self.getTestCore(dirn=dirn) as core:
await core.nodes(f'service.add fake {lurl}')
iden = core.getStormSvcs()[0].iden
await core.nodes(f'service.add prim {purl}')
await core.nodes(f'service.add boom {burl}')
await core.nodes(f'service.add lift {curl}')
evts = {
'add': {
'storm': '$lib.queue.add(foo)',
},
'del': {
'storm': '$lib.queue.del(foo)',
},
}
with self.raises(s_exc.NoSuchStormSvc):
await core.setStormSvcEvents(s_common.guid(), evts)
with self.raises(s_exc.NoSuchStormSvc):
await core._runStormSvcAdd(s_common.guid())
# force a wait for command loads
await core.nodes('$lib.service.wait(fake)')
await core.nodes('$lib.service.wait(prim)')
await core.nodes('$lib.service.wait(boom)')
await core.nodes('$lib.service.wait(lift)')
# check that new commands are displayed properly in help
msgs = await core.stormlist('help')
self.stormIsInPrint('service: fake', msgs)
self.stormIsInPrint('package: foo', msgs)
self.stormIsInPrint('foobar', msgs)
# ensure that the initializer ran, but only the initializers for
# RealService and BoomService, since the others should have failed
queue = core.multiqueue.list()
self.len(1, queue)
self.eq('vertex', queue[0]['name'])
nodes = await core.nodes('inet:ipv4=8.8.8.8')
self.len(1, nodes)
self.eq(nodes[0].ndef[1], 134744072)
self.nn(core.getStormCmd('ohhai'))
self.none(core.getStormCmd('goboom'))
nodes = await core.nodes(
'[ ps:name=$lib.service.get(prim).lower() ]')
self.len(1, nodes)
self.eq(nodes[0].ndef[1], 'asdf')
nodes = await core.nodes('[ inet:ipv4=5.5.5.5 ] | ohhai')
self.len(2, nodes)
self.eq(nodes[0].get('asn'), 20)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x05050505))
self.eq(nodes[1].get('asn'), 20)
self.eq(nodes[1].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes(
'for $ipv4 in $lib.service.get(fake).ipv4s() { [inet:ipv4=$ipv4] }'
)
self.len(3, nodes)
nodes = await core.nodes(
'[ inet:ipv4=1.2.3.4 :asn=20 ] | foobar | +:asn=40')
self.len(1, nodes)
self.none(await core.getStormPkg('boom'))
self.none(core.getStormCmd('badcmd'))
# execute a pure storm service without inbound nodes
# even though it has invalid add/del, it should still work
nodes = await core.nodes('lifter')
self.len(1, nodes)
# modconf data is available to commands
msgs = await core.stormlist(
'$real_lib = $lib.import("foo.bar") $real_lib.printmodconf()'
)
self.stormIsInPrint(f'svciden={iden}', msgs)
self.stormIsInPrint('key=valu', msgs)
# Check some service related permissions
user = await core.auth.addUser('user')
# No permissions is a failure too!
msgs = await core.stormlist('$svc=$lib.service.get(fake)',
{'user': user.iden})
self.stormIsInErr(
f'must have permission service.get.{iden}', msgs)
# Old permissions still wrk for now but cause warnings
await user.addRule((True, ('service', 'get', 'fake')))
msgs = await core.stormlist('$svc=$lib.service.get(fake)',
{'user': user.iden})
self.stormIsInWarn(
'service.get.<servicename> permissions are deprecated.',
msgs)
await user.delRule((True, ('service', 'get', 'fake')))
# storm service permissions should use svcidens
await user.addRule((True, ('service', 'get', iden)))
msgs = await core.stormlist(
'$svc=$lib.service.get(fake) $lib.print($svc)',
{'user': user.iden})
self.stormIsInPrint('storm:proxy', msgs)
self.len(0, [m for m in msgs if m[0] == 'warn'])
msgs = await core.stormlist(
f'$svc=$lib.service.get({iden}) $lib.print($svc)',
{'user': user.iden})
self.stormIsInPrint('storm:proxy', msgs)
self.len(0, [m for m in msgs if m[0] == 'warn'])
msgs = await core.stormlist(
f'$svc=$lib.service.get(real) $lib.print($svc)',
{'user': user.iden})
self.stormIsInPrint('storm:proxy', msgs)
self.len(0, [m for m in msgs if m[0] == 'warn'])
q = '$hasfoo=$lib.service.has($svc) if $hasfoo {$lib.print(yes)} else {$lib.print(no)}'
msgs = await core.stormlist(q, {'vars': {'svc': 'foo'}})
self.stormIsInPrint('no', msgs)
msgs = await core.stormlist(q, {'vars': {'svc': 'real'}})
self.stormIsInPrint('yes', msgs)
msgs = await core.stormlist(q, {'vars': {'svc': 'fake'}})
self.stormIsInPrint('yes', msgs)
msgs = await core.stormlist(q, {'vars': {'svc': iden}})
self.stormIsInPrint('yes', msgs)
# Since there ws a chnage to how $lib.service.wait handles permissions, anyone that can
# get a service can also wait for it, so ensure that those permissions still work.
# lib.service.wait can still be called both ways (iden or name)
msgs = await core.stormlist(
'$svc=$lib.service.wait(fake) $lib.print(yup)',
{'user': user.iden})
self.len(0, [m for m in msgs if m[0] == 'err'])
self.stormIsInPrint('yup', msgs)
msgs = await core.stormlist(
f'$svc=$lib.service.wait({iden}) $lib.print(yup)',
{'user': user.iden})
self.len(0, [m for m in msgs if m[0] == 'err'])
self.stormIsInPrint('yup', msgs)
await user.delRule((True, ('service', 'get', iden)))
await user.addRule((True, ('service', 'get')))
msgs = await core.stormlist(
f'$svc=$lib.service.wait({iden}) $lib.print(yup)',
{'user': user.iden})
self.len(0, [m for m in msgs if m[0] == 'err'])
self.stormIsInPrint('yup', msgs)
async with self.getTestCore(dirn=dirn) as core:
nodes = await core.nodes('$lib.service.wait(fake)')
nodes = await core.nodes('[ inet:ipv4=6.6.6.6 ] | ohhai')
self.len(2, nodes)
self.eq(nodes[0].get('asn'), 20)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x06060606))
self.eq(nodes[1].get('asn'), 20)
self.eq(nodes[1].ndef, ('inet:ipv4', 0x01020304))
# reach in and close the proxies
for ssvc in core.getStormSvcs():
await ssvc.proxy._t_proxy.fini()
nodes = await core.nodes('[ inet:ipv4=6.6.6.6 ] | ohhai')
self.len(2, nodes)
# haven't deleted the service yet, so still should be there
queue = core.multiqueue.list()
self.len(1, queue)
self.eq('vertex', queue[0]['name'])
await core.delStormSvc(iden)
# make sure stormcmd got deleted
self.none(core.getStormCmd('ohhai'))
# ensure del event ran
q = 'for ($o, $m) in $lib.queue.get(vertex).gets(wait=10) {return (($o, $m))}'
retn = await core.callStorm(q)
self.eq(retn, (0, 'done'))
# specifically call teardown
for svc in core.getStormSvcs():
mesgs = await core.stormlist(f'service.del {svc.iden}')
mesgs = [
m[1].get('mesg') for m in mesgs if m[0] == 'print'
]
self.len(1, mesgs)
self.isin(f'removed {svc.iden} ({svc.name})', mesgs[0])
self.len(0, core.getStormSvcs())
# make sure all the dels ran, except for the BoomService (which should fail)
nodes = await core.nodes('inet:ipv4')
ans = {
'1.2.3.4', '5.5.5.5', '6.6.6.6', '8.8.8.8',
'123.123.123.123'
}
reprs = set(map(lambda k: k.repr(), nodes))
self.eq(ans, reprs)
badiden = []
async def badSetStormSvcEvents(iden, evts):
badiden.append(iden)
raise s_exc.SynErr('Kaboom')
sdef = {
'name': 'dead',
'iden': s_common.guid(),
'url': durl,
}
with patchcore(core, 'setStormSvcEvents',
badSetStormSvcEvents):
svci = await core.addStormSvc(sdef)
self.true(await core.waitStormSvc('dead', timeout=0.2))
await core.delStormSvc(svci.get('iden'))
self.len(1, badiden)
self.eq(svci.get('iden'), badiden.pop())
async def badRunStormSvcAdd(iden):
badiden.append(iden)
raise s_exc.SynErr('Kaboom')
with patchcore(core, '_runStormSvcAdd', badRunStormSvcAdd):
svci = await core.addStormSvc(sdef)
self.true(await core.waitStormSvc('dead', timeout=0.2))
await core.delStormSvc(svci.get('iden'))
self.len(1, badiden)
self.eq(svci.get('iden'), badiden[0])
async def test_it_forms_hostexec(self):
# forms related to the host execution model
async with self.getTestCore() as core:
async with await core.snap() as snap:
exe = 'sha256:' + 'a' * 64
port = 80
tick = s_common.now()
host = s_common.guid()
proc = s_common.guid()
mutex = 'giggleXX_X0'
pipe = 'pipe\\mynamedpipe'
user = '******'
pid = 20
key = 'HKEY_LOCAL_MACHINE\\Foo\\Bar'
ipv4 = 0x01020304
ipv6 = '::1'
addr4 = f'tcp://1.2.3.4:{port}'
addr6 = f'udp://[::1]:{port}'
url = 'http://www.google.com/sekrit.html'
raw_path = r'c:\Windows\System32\rar.exe'
norm_path = r'c:/windows/system32/rar.exe'
src_proc = s_common.guid()
src_path = r'c:/temp/ping.exe'
cmd0 = 'rar a -r yourfiles.rar *.txt'
fpath = 'c:/temp/yourfiles.rar'
fbyts = 'sha256:' + 'b' * 64
pprops = {
'exe': exe,
'pid': pid,
'cmd': cmd0,
'host': host,
'time': tick,
'user': user,
'path': raw_path,
'src:exe': src_path,
'src:proc': src_proc,
}
node = await snap.addNode('it:exec:proc', proc, pprops)
self.eq(node.ndef[1], proc)
self.eq(node.get('exe'), exe)
self.eq(node.get('pid'), pid)
self.eq(node.get('cmd'), cmd0)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('user'), user)
self.eq(node.get('path'), norm_path)
self.eq(node.get('src:exe'), src_path)
self.eq(node.get('src:proc'), src_proc)
m0 = s_common.guid()
mprops = {
'exe': exe,
'proc': proc,
'name': mutex,
'host': host,
'time': tick,
}
node = await snap.addNode('it:exec:mutex', m0, mprops)
self.eq(node.ndef[1], m0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('name'), mutex)
p0 = s_common.guid()
pipeprops = {
'exe': exe,
'proc': proc,
'name': pipe,
'host': host,
'time': tick,
}
node = await snap.addNode('it:exec:pipe', p0, pipeprops)
self.eq(node.ndef[1], p0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('name'), pipe)
u0 = s_common.guid()
uprops = {
'proc': proc,
'host': host,
'exe': exe,
'time': tick,
'url': url,
'client': addr4,
}
node = await snap.addNode('it:exec:url', u0, uprops)
self.eq(node.ndef[1], u0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('url'), url)
self.eq(node.get('client'), addr4)
self.eq(node.get('client:ipv4'), ipv4)
self.eq(node.get('client:port'), port)
u1 = s_common.guid()
uprops['client'] = addr6
node = await snap.addNode('it:exec:url', u1, uprops)
self.eq(node.ndef[1], u1)
self.eq(node.get('client'), addr6)
self.eq(node.get('client:ipv6'), ipv6)
self.eq(node.get('client:port'), port)
b0 = s_common.guid()
bprops = {
'proc': proc,
'host': host,
'exe': exe,
'time': tick,
'server': addr4
}
node = await snap.addNode('it:exec:bind', b0, bprops)
self.eq(node.ndef[1], b0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('server'), addr4)
self.eq(node.get('server:ipv4'), ipv4)
self.eq(node.get('server:port'), port)
b1 = s_common.guid()
bprops['server'] = addr6
node = await snap.addNode('it:exec:bind', b1, bprops)
self.eq(node.ndef[1], b1)
self.eq(node.get('server'), addr6)
self.eq(node.get('server:ipv6'), ipv6)
self.eq(node.get('server:port'), port)
faprops = {
'exe': exe,
'host': host,
'proc': proc,
'file': fbyts,
'time': tick,
'path': fpath,
}
fa0 = s_common.guid()
node = await snap.addNode('it:exec:file:add', fa0, faprops)
self.eq(node.ndef[1], fa0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fr0 = s_common.guid()
node = await snap.addNode('it:exec:file:read', fr0, faprops)
self.eq(node.ndef[1], fr0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fw0 = s_common.guid()
node = await snap.addNode('it:exec:file:write', fw0, faprops)
self.eq(node.ndef[1], fw0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fd0 = s_common.guid()
node = await snap.addNode('it:exec:file:del', fd0, faprops)
self.eq(node.ndef[1], fd0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
file0 = s_common.guid()
fsprops = {
'host': host,
'path': fpath,
'file': fbyts,
'ctime': tick,
'mtime': tick + 1,
'atime': tick + 2,
'user': user,
'group': 'domainadmin'
}
node = await snap.addNode('it:fs:file', file0, fsprops)
self.eq(node.ndef[1], file0)
self.eq(node.get('host'), host)
self.eq(node.get('user'), user)
self.eq(node.get('group'), 'domainadmin')
self.eq(node.get('file'), fbyts)
self.eq(node.get('ctime'), tick)
self.eq(node.get('mtime'), tick + 1)
self.eq(node.get('atime'), tick + 2)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
rprops = {
'host': host,
'proc': proc,
'exe': exe,
'time': tick,
'reg': '*',
}
forms = (
'it:exec:reg:get',
'it:exec:reg:set',
'it:exec:reg:del',
)
for form in forms:
rk0 = s_common.guid()
nprops = rprops.copy()
node = await snap.addNode(form, rk0, nprops)
self.eq(node.ndef[1], rk0)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('exe'), exe)
self.eq(node.get('time'), tick)
self.nn(node.get('reg'))
