def test_lib_crypto_tnfl_cryptseq(self):
txk = s_common.buid()
rxk = s_common.buid()
crypter1 = s_tinfoil.CryptSeq(rxk, txk)
crypter2 = s_tinfoil.CryptSeq(txk, rxk)
mesg = ('hehe', {'key': 'valu'})
self.eq(str(crypter1._tx_sn), 'count(0)')
self.eq(str(crypter2._rx_sn), 'count(0)')
ct = crypter1.encrypt(mesg)
self.isinstance(ct, bytes)
self.eq(str(crypter1._tx_sn), 'count(1)')
pt = crypter2.decrypt(ct)
self.eq(str(crypter2._rx_sn), 'count(1)')
self.eq(mesg, pt)
self.raises(s_exc.CryptoErr, crypter1.decrypt, ct)
self.eq(str(crypter1._rx_sn), 'count(0)')
self.raises(s_exc.CryptoErr, crypter2.decrypt, ct)
self.eq(
str(crypter2._rx_sn),
'count(2)') # even though we fail, we've incremented the seqn valu
async def test_axon_wput(self):
async with self.getTestCore() as core:
axon = core.axon
axon.addHttpApi('/api/v1/pushfile', HttpPushFile, {'cell': axon})
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
size, sha256 = await fd.save()
host, port = await axon.addHttpsPort(0, host='127.0.0.1')
async with axon.getLocalProxy() as proxy:
resp = await proxy.wput(sha256, f'https://127.0.0.1:{port}/api/v1/pushfile', method='PUT', ssl=False)
self.eq(True, resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(sha256)}}
q = f'return($lib.axon.wput($sha256, "https://127.0.0.1:{port}/api/v1/pushfile", ssl=(0)))'
resp = await core.callStorm(q, opts=opts)
self.eq(True, resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
resp = await core.callStorm(q, opts=opts)
self.eq(False, resp['ok'])
self.isin('Axon does not contain the requested file.', resp.get('mesg'))
q = f'''
$fields = $lib.list(
$lib.dict(name=file, sha256=$sha256, filename=file),
$lib.dict(name=zip_password, value=test)
)
$resp = $lib.inet.http.post("https://127.0.0.1:{port}/api/v1/pushfile",
fields=$fields, ssl_verify=(0))
return($resp)
'''
opts = {'vars': {'sha256': s_common.ehex(sha256)}}
resp = await core.callStorm(q, opts=opts)
self.true(resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
resp = await core.callStorm(q, opts=opts)
self.false(resp['ok'])
self.isin('Axon does not contain the requested file.', resp.get('err'))
async with axon.getLocalProxy() as proxy:
resp = await proxy.postfiles(fields, f'https://127.0.0.1:{port}/api/v1/pushfile', ssl=False)
self.true(resp['ok'])
self.eq(200, resp['code'])
conf = {'http:proxy': 'socks5://user:[email protected]:1'}
async with self.getTestAxon(conf=conf) as axon:
async with axon.getLocalProxy() as proxy:
resp = await proxy.postfiles(fields, f'https://127.0.0.1:{port}/api/v1/pushfile', ssl=False)
self.false(resp['ok'])
self.isin('Can not connect to proxy 127.0.0.1:1', resp.get('err', ''))
def test_common_buid(self):
byts = b'deadb33f00010203'
iden = s_common.buid()
self.isinstance(iden, bytes)
self.len(32, iden)
# Buids are random by default
iden2 = s_common.buid()
self.ne(iden, iden2)
# buids may be derived from any msgpackable valu which is stable
iden3 = s_common.buid(byts)
evalu = b'\xde\x8a\x8a\x88\xbc \xd4\xc1\x81J\xf5\xc7\xbf\xbc\xd2T6\xba\xd0\xf1\x10\xaa\x07<\xfa\xe5\xfc\x8c\x93\xeb\xb4 '
self.len(32, iden3)
self.eq(iden3, evalu)
def test_common_buid(self):
byts = b'deadb33f00010203'
iden = s_common.buid()
self.isinstance(iden, bytes)
self.len(32, iden)
# Buids are random by default
iden2 = s_common.buid()
self.ne(iden, iden2)
# buids may be derived from any msgpackable valu which is stable
iden3 = s_common.buid(byts)
evalu = b'\xde\x8a\x8a\x88\xbc \xd4\xc1\x81J\xf5\xc7\xbf\xbc\xd2T6\xba\xd0\xf1\x10\xaa\x07<\xfa\xe5\xfc\x8c\x93\xeb\xb4 '
self.len(32, iden3)
self.eq(iden3, evalu)
async def test_axon_limits(self):
async with self.getTestAxon(conf={'max:count': 10}) as axon:
for i in range(10):
await axon.put(s_common.buid())
with self.raises(s_exc.HitLimit):
await axon.put(s_common.buid())
async with self.getTestAxon(conf={'max:bytes': 320}) as axon:
for i in range(10):
await axon.put(s_common.buid())
with self.raises(s_exc.HitLimit):
await axon.put(s_common.buid())
def getLiftOpsV2(self, prop, valu, cmpr='='):
if valu is None:
iops = (('pref', b'\x00'),)
return (
('indx', ('byprop', prop.pref, iops)),
)
if cmpr == '=':
norm, info = self.norm(valu)
iops = (
('eq', b'\x00' + s_common.buid(norm)),
)
return (
('indx', ('byprop', prop.pref, iops)),
)
if cmpr == 'contains=':
norm, info = self.arraytype.norm(valu)
byts = self.arraytype.indx(norm)
iops = (
('eq', b'\x01' + self.arraytype.indx(norm)),
)
return (
('indx', ('byprop', prop.pref, iops)),
)
# TODO we *could* retrieve and munge the iops from arraytype...
mesg = f'Array type has no lift by: {cmpr}.'
raise s_exc.NoSuchCmpr(mesg=mesg)
async def _getAddNodeEdits(self, name, valu, props=None):
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
if self.buidprefetch:
norm, info = form.type.norm(valu)
buid = s_common.buid((form.name, norm))
node = await self.getNodeByBuid(buid)
if node is not None:
if props is not None:
return (node, await self.getNodeAdds(form,
valu,
props=props,
addnode=False))
else:
return (node, [(buid, form.name, [])])
return (None, await self.getNodeAdds(form, valu, props=props))
def getStorNode(self, form):
ndef = (form.name, form.type.norm(self.name)[0])
buid = s_common.buid(ndef)
ctor = '.'.join([self.__class__.__module__, self.__class__.__qualname__])
props = {
'doc': self.info.get('doc'),
'ctor': ctor,
}
opts = {k: v for k, v in self.opts.items()}
if opts:
props['opts'] = opts
if self.subof is not None:
props['subof'] = self.subof
pnorms = {}
for prop, valu in props.items():
formprop = form.props.get(prop)
if formprop is not None and valu is not None:
pnorms[prop] = formprop.type.norm(valu)[0]
return (buid, {
'ndef': ndef,
'props': pnorms,
})
async def test_storm_node_iden(self):
async with self.getTestCore() as core:
nodes = await core.nodes(
'[ test:int=10 test:str=$node.iden() ] +test:str')
iden = s_common.ehex(s_common.buid(('test:int', 10)))
self.eq(nodes[0].ndef, ('test:str', iden))
self.len(1, nodes)
async def test_cortex_remote_layer(self):
async with self.getRemoteCores() as (directcore, core):
# We write to directcore and make sure we can read from core
await s_common.aspin(directcore.eval('[ test:str=woot :tick=2015 ]'))
layr = core.view.layers[1]
self.true(isinstance(layr, s_remotelayer.RemoteLayer))
self.len(1, [x async for x in layr.iterFormRows('test:str')])
self.len(1, [x async for x in layr.iterPropRows('test:str', 'tick')])
iden = s_common.guid()
buid = s_common.buid(('test:str', 'woot'))
props = await layr.getBuidProps(buid)
self.eq('woot', props.get('*test:str'))
await layr.setOffset(iden, 200)
self.eq(200, await layr.getOffset(iden))
self.ne((), tuple([x async for x in layr.splices(0, 200)]))
self.eq(s_modelrev.maxvers, await layr.getModelVers())
await self.asyncraises(s_exc.SynErr, layr.setModelVers((9, 9, 9)))
async def test_it_reveng(self):
async with self.getTestCore() as core:
baseFile = s_common.ehex(s_common.buid())
fva = 0x404438
fopt = {'vars': {'file': baseFile,
'func': s_common.guid(),
'fva': fva}}
vstr = 'VertexBrandArtisanalBinaries'
sopt = {'vars': {'func': fopt['vars']['func'],
'string': vstr}}
fnode = await core.eval('[it:reveng:filefunc=($file, $func) :va=$fva]', opts=fopt).list()
snode = await core.eval('[it:reveng:funcstr=($func, $string)]', opts=sopt).list()
self.len(1, fnode)
self.eq(f'sha256:{baseFile}', fnode[0].get('file'))
self.eq(fva, fnode[0].get('va'))
self.len(1, snode)
self.eq(fnode[0].get('function'), snode[0].get('function'))
self.eq(vstr, snode[0].get('string'))
funcnode = await core.eval('it:reveng:function [ :name="FunkyFunction" :description="Test Function" ]').list()
self.len(1, funcnode)
self.eq("FunkyFunction", funcnode[0].get('name'))
self.eq("Test Function", funcnode[0].get('description'))
nodes = await core.eval(f'file:bytes={baseFile} -> it:reveng:filefunc :function -> it:reveng:funcstr:function').list()
self.len(1, nodes)
self.eq(vstr, nodes[0].get('string'))
def getStorNode(self, form):
ndef = (form.name, form.type.norm(self.full)[0])
buid = s_common.buid(ndef)
props = {
'doc': self.info.get('doc', ''),
'type': self.type.name,
'relname': self.name,
'univ': self.isuniv,
'base': self.name.split(':')[-1],
'ro': int(self.info.get('ro', False)),
'extmodel': self.isext,
}
if self.form is not None:
props['form'] = self.form.name
pnorms = {}
for prop, valu in props.items():
formprop = form.props.get(prop)
if formprop is not None and valu is not None:
pnorms[prop] = formprop.type.norm(valu)[0]
return (buid, {'props': pnorms, 'ndef': ndef})
async def test_cortex_remote_layer(self):
async with self.getRemoteCores() as (directcore, core):
# We write to directcore and make sure we can read from core
await s_common.aspin(
directcore.eval('[ test:str=woot :tick=2015 ]'))
layr = core.view.layers[1]
self.true(isinstance(layr, s_remotelayer.RemoteLayer))
self.len(1, [x async for x in layr.iterFormRows('test:str')])
self.len(1,
[x async for x in layr.iterPropRows('test:str', 'tick')])
self.len(2, [x async for x in layr.iterUnivRows('.created')])
iden = s_common.guid()
buid = s_common.buid(('test:str', 'woot'))
props = await layr.getBuidProps(buid)
self.eq('woot', props.get('*test:str'))
await layr.setOffset(iden, 200)
self.eq(200, await layr.getOffset(iden))
await layr.delOffset(iden)
self.eq(0, await layr.getOffset(iden))
self.ne((), tuple([x async for x in layr.splices(0, 200)]))
self.eq(s_modelrev.maxvers, await layr.getModelVers())
await self.asyncraises(s_exc.SynErr, layr.setModelVers((9, 9, 9)))
def getStorNode(self, form):
ndef = (form.name, form.type.norm(self.name)[0])
buid = s_common.buid(ndef)
props = {
'doc': self.info.get('doc', self.type.info.get('doc', '')),
'type': self.type.name,
}
if form.name == 'syn:form':
props['runt'] = self.isrunt
elif form.name == 'syn:prop':
props['univ'] = False
props['extmodel'] = False
props['form'] = self.name
pnorms = {}
for prop, valu in props.items():
formprop = form.props.get(prop)
if formprop is not None and valu is not None:
pnorms[prop] = formprop.type.norm(valu)[0]
return (buid, {
'ndef': ndef,
'props': pnorms,
})
def getRuntInfo(self):
buid = s_common.buid(('syn:cron', self.iden))
return buid, (
('*syn:cron', self.iden),
('doc', self.doc),
('name', self.name),
('storm', self.query),
)
async def test_stable_uids(self):
with self.withStableUids():
guid = s_common.guid()
self.eq('000000', guid[:6])
guid2 = s_common.guid()
self.ne(guid, guid2)
guid = s_common.guid(42)
self.ne('000000', guid[:6])
buid = s_common.buid()
self.eq(b'\00\00\00\00\00\00', buid[:6])
buid2 = s_common.buid()
self.ne(buid, buid2)
buid = s_common.buid(42)
self.ne(b'\00\00\00\00\00\00', buid[:6])
def indx(self, norm):
# return a tuple of indx bytes and the layer will know what to do
vals = []
# prop=[foo,bar]
vals.append(b'\x00' + s_common.buid(norm))
# prop*contains=foo
[vals.append(b'\x01' + self.arraytype.indx(v)) for v in norm]
return vals
async def addNode(self, name, valu, props=None):
'''
Add a node by form name and value with optional props.
Args:
name (str): The form of node to add.
valu (obj): The value for the node.
props (dict): Optional secondary properties for the node.
Notes:
If a props dictionary is provided, it may be mutated during node construction.
Returns:
s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
try:
if self.buidprefetch:
norm, info = form.type.norm(valu)
node = await self.getNodeByBuid(
s_common.buid((form.name, norm)))
if node is not None:
# TODO implement node.setNodeProps()
if props is not None:
for p, v in props.items():
await node.set(p, v)
return node
adds = await self.getNodeAdds(form, valu, props=props)
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception as e:
if not self.strict:
await self.warn(f'addNode: {e}')
return None
raise
nodes = await self.applyNodeEdits(adds)
assert len(nodes) >= 1
# Adds is top-down, so the first node is what we want
return nodes[0]
async def getNodeByNdef(self, ndef):
'''
Return a single Node by (form,valu) tuple.
Args:
ndef ((str,obj)): A (form,valu) ndef tuple.
Returns:
(synapse.lib.node.Node): The Node or None.
'''
buid = s_common.buid(ndef)
return await self.getNodeByBuid(buid)
async def getNodeByNdef(self, ndef):
'''
Return a single Node by (form,valu) tuple.
Args:
ndef ((str,obj)): A (form,valu) ndef tuple. valu must be
normalized.
Returns:
(synapse.lib.node.Node): The Node or None.
'''
buid = s_common.buid(ndef)
return await self.getNodeByBuid(buid)
async def _initTestRunts(self):
modl = self.core.model
fnme = 'test:runt'
form = modl.form(fnme)
now = s_common.now()
data = [
(' BEEP ', {
'tick': modl.type('time').norm('2001')[0],
'lulz': 'beep.sys',
'.created': now
}),
('boop', {
'tick': modl.type('time').norm('2010')[0],
'.created': now
}),
('blah', {
'tick': modl.type('time').norm('2010')[0],
'lulz': 'blah.sys'
}),
('woah', {}),
]
for pprop, propd in data:
props = {}
pnorm, _ = form.type.norm(pprop)
for k, v in propd.items():
prop = form.props.get(k)
if prop:
norm, _ = prop.type.norm(v)
props[k] = norm
props.setdefault('.created', s_common.now())
rows = [('*' + fnme, pnorm)]
for k, v in props.items():
rows.append((k, v))
buid = s_common.buid((fnme, pnorm))
self._runtsByBuid[buid] = rows
# Allow for indirect lookup to a set of buids
self._runtsByPropValu[fnme].append(buid)
self._runtsByPropValu[(fnme, pnorm)].append(buid)
for k, propvalu in props.items():
prop = fnme + ':' + k
if k.startswith('.'):
prop = fnme + k
self._runtsByPropValu[prop].append(buid)
if modl.prop(prop).type.indx(propvalu):
# Can the secondary property be indexed for lift?
self._runtsByPropValu[(prop, propvalu)].append(buid)
def test_lib_crypto_tnfl_cryptseq(self):
txk = s_common.buid()
rxk = s_common.buid()
crypter1 = s_tinfoil.CryptSeq(rxk, txk)
crypter2 = s_tinfoil.CryptSeq(txk, rxk)
mesg = ('hehe', {'key': 'valu'})
self.eq(str(crypter1._tx_sn), 'count(0)')
self.eq(str(crypter2._rx_sn), 'count(0)')
ct = crypter1.encrypt(mesg)
self.isinstance(ct, bytes)
self.eq(str(crypter1._tx_sn), 'count(1)')
pt = crypter2.decrypt(ct)
self.eq(str(crypter2._rx_sn), 'count(1)')
self.eq(mesg, pt)
self.raises(s_exc.CryptoErr, crypter1.decrypt, ct)
self.eq(str(crypter1._rx_sn), 'count(0)')
self.raises(s_exc.CryptoErr, crypter2.decrypt, ct)
self.eq(str(crypter2._rx_sn), 'count(2)') # even though we fail, we've incremented the seqn valu
def _getNodeFnib(self, name, valu):
'''
return a form, norm, info, buid tuple
'''
form = self.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
try:
norm, info = form.type.norm(valu)
except Exception as e:
raise s_exc.BadPropValu(prop=form.name, valu=valu, mesg=str(e))
buid = s_common.buid((form.name, norm))
return form, norm, info, buid
def _getNodeFnib(self, name, valu):
'''
return a form, norm, info, buid tuple
'''
form = self.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
try:
norm, info = form.type.norm(valu)
except Exception as e:
raise s_exc.BadPropValu(prop=form.name, valu=valu, mesg=str(e))
buid = s_common.buid((form.name, norm))
return form, norm, info, buid
async def getNodeByNdef(self, ndef):
'''
Return a single Node() instance by (form,valu) tuple.
'''
name, valu = ndef
form = self.model.forms.get(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
norm, info = form.type.norm(valu)
buid = s_common.buid((form.name, norm))
async with await self.snap() as snap:
return await snap.getNodeByBuid(buid)
def _filtSaveByts(self, files):
todo = []
with self.lenv.begin() as xact:
curs = xact.cursor(db=self.blobhas)
for byts in files:
sha256 = hashlib.sha256(byts).digest()
if curs.get(sha256):
continue
todo.append((s_common.buid(), sha256, byts))
return todo
def getStorNode(self, form):
ndef = (form.name, form.type.norm(self.name)[0])
buid = s_common.buid(ndef)
props = {
'doc': self.info.get('doc', ''),
'type': self.type.name,
}
pnorms = {}
for prop, valu in props.items():
formprop = form.props.get(prop)
if formprop is not None and valu is not None:
pnorms[prop] = formprop.type.norm(valu)[0]
return (buid, {'ndef': ndef, 'props': pnorms})
async def revModel20210528(self, layers):
cmdtype = self.core.model.type('it:cmd')
cmdprop = self.core.model.prop('it:exec:proc:cmd')
for layr in layers:
done = set()
nodeedits = []
meta = {
'time': s_common.now(),
'user': self.core.auth.rootuser.iden
}
async def save():
await layr.storNodeEdits(nodeedits, meta)
done.clear()
nodeedits.clear()
async for buid, propvalu in layr.iterPropRows(
'it:exec:proc', 'cmd'):
cmdnorm = cmdtype.norm(propvalu)[0]
if cmdnorm != propvalu:
nodeedits.append(
(buid, 'it:exec:proc',
((s_layer.EDIT_PROP_SET,
('cmd', cmdnorm, propvalu, s_layer.STOR_TYPE_UTF8),
()), )), )
if cmdnorm not in done:
cmdbuid = s_common.buid(('it:cmd', cmdnorm))
nodeedits.append(
(cmdbuid, 'it:cmd',
((s_layer.EDIT_NODE_ADD,
(cmdnorm, s_layer.STOR_TYPE_UTF8), ()), )), )
done.add(cmdnorm)
if len(nodeedits) >= 1000:
await save()
if nodeedits:
await save()
async def editNodeNdef(self, oldv, newv):
oldb = s_common.buid(oldv)
for layr in self.layers:
indx = await layr.getFormIndx(oldb)
# save off any old indx valu so we can scan for them
if indx is not None:
self.slab.put(oldb, indx, overwrite=False, db=self.oldb2indx)
await layr.editNodeNdef(oldv, newv)
if self.ndefdelay is not None:
self.ndefdelay.append((oldv, newv))
return
await self.editNdefProps(oldv, newv)
async def walk(n, p):
valu = n.props.get(p)
if valu is None:
return None
prop = n.form.prop(p)
if prop is None:
return None
if prop.modl.form(prop.type.name) is None:
return None
buid = s_common.buid((prop.type.name, valu))
step = cache.get(buid, s_common.novalu)
if step is s_common.novalu:
step = cache[buid] = await node.snap.getNodeByBuid(buid)
return step
async def setNodeForm(self, layr, buid, name, oldv, newv):
'''
Reset the primary property for the given buid.
'''
form = self.core.model.form(name)
norm, info = form.type.norm(newv)
subtodo = []
subs = info.get('subs')
if subs is not None:
subtodo = list(subs.items())
newb = s_common.buid((name, norm))
ops = []
ops.extend(form.getDelOps(buid))
ops.extend(form.getSetOps(newb, norm))
# scoop up any set operations for form subs
for subn, subv in subtodo:
subp = self.core.model.prop(name + ':' + subn)
if subp is None:
continue
ops.extend(subp.getDelOps(buid))
ops.extend(subp.getSetOps(newb, subv))
# rewrite the primary property and subs
await layr.stor(ops)
# update props in all layers
await self.setNodeBuid(name, buid, newb)
await self.setPropsByType(name, oldv, norm, info)
oldndef = (name, oldv)
newndef = (name, norm)
await self.editNdefProps(oldndef, newndef)
async def _initTestRunts(self):
modl = self.core.model
fnme = 'test:runt'
form = modl.form(fnme)
now = s_common.now()
data = [(' BEEP ', {'tick': modl.type('time').norm('2001')[0], 'lulz': 'beep.sys', '.created': now}),
('boop', {'tick': modl.type('time').norm('2010')[0], '.created': now}),
('blah', {'tick': modl.type('time').norm('2010')[0], 'lulz': 'blah.sys'}),
('woah', {}),
]
for pprop, propd in data:
props = {}
pnorm, _ = form.type.norm(pprop)
for k, v in propd.items():
prop = form.props.get(k)
if prop:
norm, _ = prop.type.norm(v)
props[k] = norm
props.setdefault('.created', s_common.now())
rows = [('*' + fnme, pnorm)]
for k, v in props.items():
rows.append((k, v))
buid = s_common.buid((fnme, pnorm))
self._runtsByBuid[buid] = rows
# Allow for indirect lookup to a set of buids
self._runtsByPropValu[fnme].append(buid)
self._runtsByPropValu[(fnme, pnorm)].append(buid)
for k, propvalu in props.items():
prop = fnme + ':' + k
if k.startswith('.'):
prop = fnme + k
self._runtsByPropValu[prop].append(buid)
if modl.prop(prop).type.indx(propvalu):
# Can the secondary property be indexed for lift?
self._runtsByPropValu[(prop, propvalu)].append(buid)
def _addRuntRows(self, form, valu, props, buidcache, propcache):
buid = s_common.buid((form, valu))
if buid in buidcache:
return
rows = [('*' + form, valu)]
props.setdefault('.created', s_common.now())
for k, v in props.items():
rows.append((k, v))
buidcache[buid] = rows
propcache[form].append(buid)
propcache[(form, valu)].append(buid)
for k, propvalu in props.items():
prop = form + ':' + k
if k.startswith('.'):
prop = form + k
propcache[prop].append(buid)
# Can the secondary property be indexed for lift?
if self.model.prop(prop).type.indx(propvalu):
propcache[(prop, propvalu)].append(buid)
def _addModelRuntRows(self, form, valu, props):
buid = s_common.buid((form, valu))
if buid in self._modelRuntsByBuid:
return
rows = [('*' + form, valu)]
props.setdefault('.created', s_common.now())
for k, v in props.items():
rows.append((k, v))
self._modelRuntsByBuid[buid] = rows
self._modelRuntsByPropValu[form].append(buid)
self._modelRuntsByPropValu[(form, valu)].append(buid)
for k, propvalu in props.items():
prop = form + ':' + k
if k.startswith('.'):
prop = form + k
self._modelRuntsByPropValu[prop].append(buid)
# Can the secondary property be indexed for lift?
if self.model.prop(prop).type.indx(propvalu):
self._modelRuntsByPropValu[(prop, propvalu)].append(buid)
def getStorNode(self, form):
ndef = (form.name, form.type.norm(self.iden)[0])
buid = s_common.buid(ndef)
props = {
'doc': self.tdef.get('doc', ''),
'name': self.tdef.get('name', ''),
'vers': self.tdef.get('ver', 1),
'cond': self.tdef.get('cond'),
'storm': self.tdef.get('storm'),
'enabled': self.tdef.get('enabled'),
'user': self.tdef.get('user'),
}
tag = self.tdef.get('tag')
if tag is not None:
props['tag'] = tag
formprop = self.tdef.get('form')
if formprop is not None:
props['form'] = formprop
prop = self.tdef.get('prop')
if prop is not None:
props['prop'] = prop
pnorms = {}
for prop, valu in props.items():
formprop = form.props.get(prop)
if formprop is not None and valu is not None:
pnorms[prop] = formprop.type.norm(valu)[0]
return (buid, {
'ndef': ndef,
'props': pnorms,
})
async def editNdefProps(self, oldndef, newndef):
'''
Change all props as a result of an ndef change.
'''
oldbuid = s_common.buid(oldndef)
oldname, oldvalu = oldndef
newname, newvalu = newndef
rename = newname != oldname
# we only need to update secondary props if they have diff vals
# ( vs for example a pure rename )
if oldvalu != newvalu:
# get the indx bytes for the *value* of the ndef
indx = self.slab.get(oldbuid, db=self.oldb2indx)
if indx is not None:
# the only way for indx to be None is if we dont have the node...
for prop in self.core.model.getPropsByType(newname):
coff = prop.getCompOffs()
for layr in self.layers:
async for buid, valu in layr.iterPropIndx(prop.form.name, prop.name, indx):
await layr.storPropSet(buid, prop, newvalu)
# for now, assume any comp sub is on the same layer as it's form prop
if coff is not None:
ndef = await layr.getNodeNdef(buid)
edit = list(ndef[1])
edit[coff] = newvalu
await self.editNodeNdef(ndef, (ndef[0], edit))
for prop in self.core.model.getPropsByType('ndef'):
formsub = self.core.model.prop(prop.full + ':' + 'form')
coff = prop.getCompOffs()
for layr in self.layers:
async for buid, valu in layr.iterPropIndx(prop.form.name, prop.name, oldbuid):
await layr.storPropSet(buid, prop, newndef)
if rename and formsub is not None:
await layr.storPropSet(buid, formsub, newname)
if coff is not None:
# for now, assume form and prop on the same layer...
ndef = await layr.getNodeNdef(buid)
edit = list(ndef[1])
edit[coff] = newndef
await self.editNodeNdef(ndef, (ndef[0], edit))
async def _setops(self, name, valu, editatom, init=False):
'''
Generate operations to set a property on a node.
'''
prop = self.form.prop(name)
if prop is None:
if self.snap.strict:
raise s_exc.NoSuchProp(name=name)
await self.snap.warn(f'NoSuchProp: name={name}')
return False
if self.isrunt:
if prop.info.get('ro'):
raise s_exc.IsRuntForm(mesg='Cannot set read-only props on runt nodes',
form=self.form.full, prop=name, valu=valu)
return await self.snap.core.runRuntPropSet(self, prop, valu)
curv = self.props.get(name)
# normalize the property value...
try:
norm, info = prop.type.norm(valu)
except Exception as e:
mesg = f'Bad property value: {prop.full}={valu!r}'
return await self.snap._raiseOnStrict(s_exc.BadPropValu, mesg, name=prop.name, valu=valu, emesg=str(e))
# do we already have the value?
if curv == norm:
return False
if curv is not None and not init:
if prop.info.get('ro'):
if self.snap.strict:
raise s_exc.ReadOnlyProp(name=prop.full)
# not setting a set-once prop unless we are init...
await self.snap.warn(f'ReadOnlyProp: name={prop.full}')
return False
# check for type specific merging...
norm = prop.type.merge(curv, norm)
if curv == norm:
return False
sops = prop.getSetOps(self.buid, norm)
editatom.sops.extend(sops)
# self.props[prop.name] = norm
editatom.npvs.append((self, prop, curv, norm))
# do we have any auto nodes to add?
auto = self.snap.model.form(prop.type.name)
if auto is not None:
buid = s_common.buid((auto.name, norm))
await self.snap._addNodeFnibOps((auto, norm, info, buid), editatom)
# does the type think we have special auto nodes to add?
# ( used only for adds which do not meet the above block )
for autoname, autovalu in info.get('adds', ()):
auto = self.snap.model.form(autoname)
autonorm, autoinfo = auto.type.norm(autovalu)
buid = s_common.buid((auto.name, autonorm))
await self.snap._addNodeFnibOps((auto, autovalu, autoinfo, buid), editatom)
# do we need to set any sub props?
subs = info.get('subs')
if subs is not None:
for subname, subvalu in subs.items():
full = prop.name + ':' + subname
subprop = self.form.prop(full)
if subprop is None:
continue
await self._setops(full, subvalu, editatom, init=init)
return True
async def editNodeNdef(self, oldv, newv):
'''
Migration-only method
Notes:
Precondition: buid cache must be disabled
'''
assert self.buidcache.disabled
oldb = s_common.buid(oldv)
newb = s_common.buid(newv)
pvoldval = s_msgpack.en((oldb,))
pvnewval = s_msgpack.en((newb,))
oldfenc = oldv[0].encode() + b'\x00'
newfenc = newv[0].encode() + b'\x00'
newprel = b'*' + newv[0].encode()
newnindx = self.core.model.prop(newv[0]).type.indx(newv[1])
# avoid any potential iter/edit issues...
todo = list(self.layrslab.scanByPref(oldb, db=self.bybuid))
for lkey, lval in todo:
proputf8 = lkey[32:]
valu, indx = s_msgpack.un(lval)
# for the *<form> prop, the byprop index has <form><00><00><indx>
if proputf8[0] == 42:
newpropkey = newfenc + b'\x00' + newnindx
if indx is not None:
oldpropkey = oldfenc + b'\x00' + indx
if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover
logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}')
self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop)
byts = s_msgpack.en((newv[1], newnindx))
self.layrslab.put(newb + newprel, byts, db=self.bybuid)
else:
# <prop><00><indx>
propindx = proputf8 + b'\x00' + indx
if proputf8[0] in (46, 35): # ".univ" or "#tag"
self.layrslab.put(propindx, pvnewval, dupdata=True, db=self.byuniv)
self.layrslab.delete(propindx, pvoldval, db=self.byuniv)
oldpropkey = oldfenc + propindx
newpropkey = newfenc + propindx
if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover
logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}')
self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop)
self.layrslab.put(newb + proputf8, lval, db=self.bybuid)
self.layrslab.delete(lkey, db=self.bybuid)
def indx(self, norm):
return s_common.buid(norm)
def getWaitFor(self, valu):
norm, info = self.type.norm(valu)
buid = s_common.buid((self.name, norm))
evnt = asyncio.Event()
self.waits[buid].append(evnt)
return evnt
