async def getNodeByBuid(self, buid):
'''
Retrieve a node tuple by binary id.
Args:
buid (bytes): The binary ID for the node.
Returns:
Optional[s_node.Node]: The node object or None.
'''
node = self.livenodes.get(buid)
if node is not None:
return node
props = {}
proplayr = {}
for layr in self.layers:
layerprops = await layr.getBuidProps(buid)
props.update(layerprops)
proplayr.update({k: layr for k in layerprops})
node = s_node.Node(self, buid, props.items(), proplayr=proplayr)
# Give other tasks a chance to run
await asyncio.sleep(0)
if node.ndef is None:
return None
# Add node to my buidcache
self.buidcache.append(node)
self.livenodes[buid] = node
return node
async def getRowNodes(self, rows, rawprop, cmpr=None):
'''
Join a row generator into (row, Node()) tuples.
A row generator yields tuple rows where the first
valu is the buid of a node.
Args:
rows: A generator of (layer_idx, (buid, ...)) tuples.
rawprop(str): "raw" propname i.e. if a tag, starts with "#". Used for filtering so that we skip the props
for a buid if we're asking from a higher layer than the row was from (and hence, we'll presumable
get/have gotten the row when that layer is lifted.
cmpr (func): A secondary comparison function used to filter nodes.
Yields:
(tuple): (row, node)
'''
count = 0
async for origlayer, row in rows:
count += 1
if not count % 5:
await asyncio.sleep(0) # give other tasks some time
props = {}
buid = row[0]
node = self.buidcache.cache.get(buid)
# Evaluate layers top-down to more quickly abort if we've found a higher layer with the property set
for layeridx in range(len(self.layers) - 1, -1, -1):
layr = self.layers[layeridx]
layerprops = await layr.getBuidProps(buid)
# We mark this node to drop iff we see the prop set in this layer *and* we're looking at the props
# from a higher (i.e. closer to write, higher idx) layer.
if layeridx > origlayer and rawprop in layerprops:
props = None
break
if node is None:
for k, v in layerprops.items():
if k not in props:
props[k] = v
if props is None:
continue
if node is None:
node = s_node.Node(self, buid, props.items())
if node and node.ndef is not None:
self.buidcache.put(buid, node)
if node.ndef is not None:
if cmpr:
if rawprop == node.form.name:
valu = node.ndef[1]
else:
valu = node.get(rawprop)
if valu is None:
# cmpr required to evaluate something; cannot know if this
# node is valid or not without the prop being present.
continue
if not cmpr(valu):
continue
yield row, node
async def getRuntNodes(self, full, valu=None, cmpr=None):
todo = s_common.todo('runRuntLift', full, valu, cmpr)
async for sode in self.core.dyniter('cortex', todo):
node = s_node.Node(self, sode)
node.isrunt = True
yield node
async def _getNodeByBuid(self, buid):
props = {}
for layr in self.layers:
layerprops = await layr.getBuidProps(buid)
props.update(layerprops)
node = s_node.Node(self, buid, props.items())
# Give other tasks a chance to run
await asyncio.sleep(0)
return None if node.ndef is None else node
async def _joinSodes(self, buid, sodes):
node = self.livenodes.get(buid)
if node is not None:
await asyncio.sleep(0)
return node
ndef = None
tags = {}
props = {}
nodedata = {}
tagprops = {}
bylayer = {
'ndef': None,
'tags': {},
'props': {},
'tagprops': {},
}
for (layr, sode) in sodes:
form = sode.get('form')
valt = sode.get('valu')
if valt is not None:
ndef = (form, valt[0])
bylayer['ndef'] = layr
storprops = sode.get('props')
if storprops is not None:
for prop, (valu, stype) in storprops.items():
props[prop] = valu
bylayer['props'][prop] = layr
stortags = sode.get('tags')
if stortags is not None:
tags.update(stortags)
bylayer['tags'].update({p: layr for p in stortags.keys()})
stortagprops = sode.get('tagprops')
if stortagprops is not None:
for tagprop, (valu, stype) in stortagprops.items():
tagprops[tagprop] = valu
bylayer['tagprops'][tagprop] = layr
stordata = sode.get('nodedata')
if stordata is not None:
nodedata.update(stordata)
if ndef is None:
await asyncio.sleep(0)
return None
pode = (buid, {
'ndef': ndef,
'tags': tags,
'props': props,
'nodedata': nodedata,
'tagprops': tagprops,
})
node = s_node.Node(self, pode, bylayer=bylayer)
self.livenodes[buid] = node
self.buidcache.append(node)
await asyncio.sleep(0)
return node
async def getRowNodes(self, rows, rawprop, cmpf=None):
'''
Join a row generator into (row, Node()) tuples.
A row generator yields tuples of node buid, rawprop dict
Args:
rows: A generator of (layer_idx, (buid, ...)) tuples.
rawprop(str): "raw" propname e.g. if a tag, starts with "#". Used
for filtering so that we skip the props for a buid if we're
asking from a higher layer than the row was from (and hence,
we'll presumable get/have gotten the row when that layer is
lifted.
cmpf (func): A comparison function used to filter nodes.
Yields:
(tuple): (row, node)
'''
count = 0
async for origlayer, row in rows:
count += 1
if not count % 5:
await asyncio.sleep(0) # give other tasks some time
buid, rawprops = row
node = self.livenodes.get(buid)
if node is None:
props = {} # rawprop: valu
proplayr = {} # rawprop: layr
for layeridx, layr in enumerate(self.layers):
if layeridx == origlayer:
layerprops = rawprops
else:
layerprops = await layr.getBuidProps(buid)
props.update(layerprops)
proplayr.update({k: layr for k in layerprops})
node = s_node.Node(self,
buid,
props.items(),
proplayr=proplayr)
if node.ndef is None:
continue
# Add node to my buidcache
self.buidcache.append(node)
self.livenodes[buid] = node
# If the node's prop I'm filtering on came from a different layer, skip it
rawrawprop = ('*' if rawprop == node.form.name else '') + rawprop
if node.proplayr[rawrawprop] != self.layers[origlayer]:
continue
if cmpf:
if rawprop == node.form.name:
valu = node.ndef[1]
else:
valu = node.get(rawprop)
if valu is None:
# cmpr required to evaluate something; cannot know if this
# node is valid or not without the prop being present.
continue
if not cmpf(valu):
continue
yield row, node
async def getRuntNodes(self, full, valu=None, cmpr='='):
async for buid, rows in self.core.runRuntLift(full, valu, cmpr):
node = s_node.Node(self, buid, rows)
if node.ndef is not None:
yield node
async def _addNodeFnibOps(self, fnib, editatom, props=None):
'''
Add a node via (form, norm, info, buid) and add ops to editatom
'''
form, norm, info, buid = fnib
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=norm)
if props is None:
props = {}
# Check if this buid is already under construction
node = editatom.getNodeBeingMade(buid)
if node is not None:
return node
# Check if this buid is already fully made
node = await self.getNodeByBuid(buid)
if node is not None:
return node
# Another editatom might have created in another task during the above call, so check again
node = editatom.getNodeBeingMade(buid)
if node is not None:
return node
if props is None:
props = {}
# lets build a node...
node = s_node.Node(self, None)
node.buid = buid
node.form = form
node.ndef = (form.name, norm)
sops = form.getSetOps(buid, norm)
editatom.sops.extend(sops)
editatom.addNode(node)
# update props with any subs from form value
subs = info.get('subs')
if subs is not None:
for name, valu in subs.items():
if form.prop(name) is not None:
props[name] = valu
# update props with any defvals we are missing
for name, valu in form.defvals.items():
props.setdefault(name, valu)
# set all the properties with init=True
for name, valu in props.items():
await node._setops(name, valu, editatom, init=True)
# set our global properties
tick = s_common.now()
await node._setops('.created', tick, editatom, init=True)
return None
async def _joinStorNode(self, buid, cache):
node = self.livenodes.get(buid)
if node is not None:
await asyncio.sleep(0)
return node
ndef = None
tags = {}
props = {}
nodedata = {}
tagprops = {}
bylayer = {
'ndef': None,
'tags': {},
'props': {},
'tagprops': {},
}
for layr in self.layers:
sode = cache.get(layr.iden)
if sode is None:
sode = await layr.getStorNode(buid)
info = sode[1]
storndef = info.get('ndef')
if storndef is not None:
ndef = storndef
bylayer['ndef'] = layr
storprops = info.get('props')
if storprops is not None:
props.update(storprops)
bylayer['props'].update({p: layr for p in storprops.keys()})
stortags = info.get('tags')
if stortags is not None:
tags.update(stortags)
bylayer['tags'].update({p: layr for p in stortags.keys()})
stortagprops = info.get('tagprops')
if stortagprops is not None:
tagprops.update(stortagprops)
bylayer['tagprops'].update({p: layr for p in stortagprops.keys()})
stordata = info.get('nodedata')
if stordata is not None:
nodedata.update(stordata)
if ndef is None:
return None
fullnode = (buid, {
'ndef': ndef,
'tags': tags,
'props': props,
'nodedata': nodedata,
'tagprops': tagprops,
})
node = s_node.Node(self, fullnode, bylayer=bylayer)
self.livenodes[buid] = node
self.buidcache.append(node)
# moved here from getNodeByBuid() to cover more
await asyncio.sleep(0)
return node
