async def getNodeByBuid(self, buid): ''' Retrieve a node tuple by binary id. Args: buid (bytes): The binary ID for the node. Returns: Optional[s_node.Node]: The node object or None. ''' node = self.livenodes.get(buid) if node is not None: return node props = {} proplayr = {} for layr in self.layers: layerprops = await layr.getBuidProps(buid) props.update(layerprops) proplayr.update({k: layr for k in layerprops}) node = s_node.Node(self, buid, props.items(), proplayr=proplayr) # Give other tasks a chance to run await asyncio.sleep(0) if node.ndef is None: return None # Add node to my buidcache self.buidcache.append(node) self.livenodes[buid] = node return node
async def getRowNodes(self, rows, rawprop, cmpr=None): ''' Join a row generator into (row, Node()) tuples. A row generator yields tuple rows where the first valu is the buid of a node. Args: rows: A generator of (layer_idx, (buid, ...)) tuples. rawprop(str): "raw" propname i.e. if a tag, starts with "#". Used for filtering so that we skip the props for a buid if we're asking from a higher layer than the row was from (and hence, we'll presumable get/have gotten the row when that layer is lifted. cmpr (func): A secondary comparison function used to filter nodes. Yields: (tuple): (row, node) ''' count = 0 async for origlayer, row in rows: count += 1 if not count % 5: await asyncio.sleep(0) # give other tasks some time props = {} buid = row[0] node = self.buidcache.cache.get(buid) # Evaluate layers top-down to more quickly abort if we've found a higher layer with the property set for layeridx in range(len(self.layers) - 1, -1, -1): layr = self.layers[layeridx] layerprops = await layr.getBuidProps(buid) # We mark this node to drop iff we see the prop set in this layer *and* we're looking at the props # from a higher (i.e. closer to write, higher idx) layer. if layeridx > origlayer and rawprop in layerprops: props = None break if node is None: for k, v in layerprops.items(): if k not in props: props[k] = v if props is None: continue if node is None: node = s_node.Node(self, buid, props.items()) if node and node.ndef is not None: self.buidcache.put(buid, node) if node.ndef is not None: if cmpr: if rawprop == node.form.name: valu = node.ndef[1] else: valu = node.get(rawprop) if valu is None: # cmpr required to evaluate something; cannot know if this # node is valid or not without the prop being present. continue if not cmpr(valu): continue yield row, node
async def getRuntNodes(self, full, valu=None, cmpr=None): todo = s_common.todo('runRuntLift', full, valu, cmpr) async for sode in self.core.dyniter('cortex', todo): node = s_node.Node(self, sode) node.isrunt = True yield node
async def _getNodeByBuid(self, buid): props = {} for layr in self.layers: layerprops = await layr.getBuidProps(buid) props.update(layerprops) node = s_node.Node(self, buid, props.items()) # Give other tasks a chance to run await asyncio.sleep(0) return None if node.ndef is None else node
async def _joinSodes(self, buid, sodes): node = self.livenodes.get(buid) if node is not None: await asyncio.sleep(0) return node ndef = None tags = {} props = {} nodedata = {} tagprops = {} bylayer = { 'ndef': None, 'tags': {}, 'props': {}, 'tagprops': {}, } for (layr, sode) in sodes: form = sode.get('form') valt = sode.get('valu') if valt is not None: ndef = (form, valt[0]) bylayer['ndef'] = layr storprops = sode.get('props') if storprops is not None: for prop, (valu, stype) in storprops.items(): props[prop] = valu bylayer['props'][prop] = layr stortags = sode.get('tags') if stortags is not None: tags.update(stortags) bylayer['tags'].update({p: layr for p in stortags.keys()}) stortagprops = sode.get('tagprops') if stortagprops is not None: for tagprop, (valu, stype) in stortagprops.items(): tagprops[tagprop] = valu bylayer['tagprops'][tagprop] = layr stordata = sode.get('nodedata') if stordata is not None: nodedata.update(stordata) if ndef is None: await asyncio.sleep(0) return None pode = (buid, { 'ndef': ndef, 'tags': tags, 'props': props, 'nodedata': nodedata, 'tagprops': tagprops, }) node = s_node.Node(self, pode, bylayer=bylayer) self.livenodes[buid] = node self.buidcache.append(node) await asyncio.sleep(0) return node
async def getRowNodes(self, rows, rawprop, cmpf=None): ''' Join a row generator into (row, Node()) tuples. A row generator yields tuples of node buid, rawprop dict Args: rows: A generator of (layer_idx, (buid, ...)) tuples. rawprop(str): "raw" propname e.g. if a tag, starts with "#". Used for filtering so that we skip the props for a buid if we're asking from a higher layer than the row was from (and hence, we'll presumable get/have gotten the row when that layer is lifted. cmpf (func): A comparison function used to filter nodes. Yields: (tuple): (row, node) ''' count = 0 async for origlayer, row in rows: count += 1 if not count % 5: await asyncio.sleep(0) # give other tasks some time buid, rawprops = row node = self.livenodes.get(buid) if node is None: props = {} # rawprop: valu proplayr = {} # rawprop: layr for layeridx, layr in enumerate(self.layers): if layeridx == origlayer: layerprops = rawprops else: layerprops = await layr.getBuidProps(buid) props.update(layerprops) proplayr.update({k: layr for k in layerprops}) node = s_node.Node(self, buid, props.items(), proplayr=proplayr) if node.ndef is None: continue # Add node to my buidcache self.buidcache.append(node) self.livenodes[buid] = node # If the node's prop I'm filtering on came from a different layer, skip it rawrawprop = ('*' if rawprop == node.form.name else '') + rawprop if node.proplayr[rawrawprop] != self.layers[origlayer]: continue if cmpf: if rawprop == node.form.name: valu = node.ndef[1] else: valu = node.get(rawprop) if valu is None: # cmpr required to evaluate something; cannot know if this # node is valid or not without the prop being present. continue if not cmpf(valu): continue yield row, node
async def getRuntNodes(self, full, valu=None, cmpr='='): async for buid, rows in self.core.runRuntLift(full, valu, cmpr): node = s_node.Node(self, buid, rows) if node.ndef is not None: yield node
async def _addNodeFnibOps(self, fnib, editatom, props=None): ''' Add a node via (form, norm, info, buid) and add ops to editatom ''' form, norm, info, buid = fnib if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=norm) if props is None: props = {} # Check if this buid is already under construction node = editatom.getNodeBeingMade(buid) if node is not None: return node # Check if this buid is already fully made node = await self.getNodeByBuid(buid) if node is not None: return node # Another editatom might have created in another task during the above call, so check again node = editatom.getNodeBeingMade(buid) if node is not None: return node if props is None: props = {} # lets build a node... node = s_node.Node(self, None) node.buid = buid node.form = form node.ndef = (form.name, norm) sops = form.getSetOps(buid, norm) editatom.sops.extend(sops) editatom.addNode(node) # update props with any subs from form value subs = info.get('subs') if subs is not None: for name, valu in subs.items(): if form.prop(name) is not None: props[name] = valu # update props with any defvals we are missing for name, valu in form.defvals.items(): props.setdefault(name, valu) # set all the properties with init=True for name, valu in props.items(): await node._setops(name, valu, editatom, init=True) # set our global properties tick = s_common.now() await node._setops('.created', tick, editatom, init=True) return None
async def _joinStorNode(self, buid, cache): node = self.livenodes.get(buid) if node is not None: await asyncio.sleep(0) return node ndef = None tags = {} props = {} nodedata = {} tagprops = {} bylayer = { 'ndef': None, 'tags': {}, 'props': {}, 'tagprops': {}, } for layr in self.layers: sode = cache.get(layr.iden) if sode is None: sode = await layr.getStorNode(buid) info = sode[1] storndef = info.get('ndef') if storndef is not None: ndef = storndef bylayer['ndef'] = layr storprops = info.get('props') if storprops is not None: props.update(storprops) bylayer['props'].update({p: layr for p in storprops.keys()}) stortags = info.get('tags') if stortags is not None: tags.update(stortags) bylayer['tags'].update({p: layr for p in stortags.keys()}) stortagprops = info.get('tagprops') if stortagprops is not None: tagprops.update(stortagprops) bylayer['tagprops'].update({p: layr for p in stortagprops.keys()}) stordata = info.get('nodedata') if stordata is not None: nodedata.update(stordata) if ndef is None: return None fullnode = (buid, { 'ndef': ndef, 'tags': tags, 'props': props, 'nodedata': nodedata, 'tagprops': tagprops, }) node = s_node.Node(self, fullnode, bylayer=bylayer) self.livenodes[buid] = node self.buidcache.append(node) # moved here from getNodeByBuid() to cover more await asyncio.sleep(0) return node