def putmulti(self, kvpairs, dupdata=False, append=False, db=None): ''' Returns: Tuple of number of items consumed, number of items added ''' if self.readonly: raise s_exc.IsReadOnly() # Log playback isn't compatible with generators if not isinstance(kvpairs, list): kvpairs = list(kvpairs) realdb, dupsort = self.dbnames[db] try: self.dirty = True if not self.recovering: self._logXactOper(self.putmulti, kvpairs, dupdata=dupdata, append=append, db=db) with self.xact.cursor(db=realdb) as curs: return curs.putmulti(kvpairs, dupdata=dupdata, append=append) except lmdb.MapFullError: return self._handle_mapfull()
async def addNode(self, name, valu, props=None): ''' Add a node by form name and value with optional props. Args: name (str): The form of node to add. valu (obj): The value for the node. props (dict): Optional secondary properties for the node. Notes: If a props dictionary is provided, it may be mutated during node construction. Returns: s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) form = self.core.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=valu) try: if self.buidprefetch: norm, info = form.type.norm(valu) node = await self.getNodeByBuid( s_common.buid((form.name, norm))) if node is not None: # TODO implement node.setNodeProps() if props is not None: for p, v in props.items(): await node.set(p, v) return node adds = await self.getNodeAdds(form, valu, props=props) except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except Exception as e: if not self.strict: await self.warn(f'addNode: {e}') return None raise nodes = await self.applyNodeEdits(adds) assert len(nodes) >= 1 # Adds is top-down, so the first node is what we want return nodes[0]
def _xact_action(self, calling_func, xact_func, lkey, *args, db=None, **kwargs): if self.readonly: raise s_exc.IsReadOnly() realdb, dupsort = self.dbnames[db] try: self.dirty = True if not self.recovering: self._logXactOper(calling_func, lkey, *args, db=db, **kwargs) return xact_func(self.xact, lkey, *args, db=realdb, **kwargs) except lmdb.MapFullError: return self._handle_mapfull()
def putmulti(self, kvpairs, dupdata=False, append=False, db=_DefaultDB): if self.readonly: raise s_exc.IsReadOnly() try: self.dirty = True if not self.recovering: self._logXactOper(self.putmulti, kvpairs, dupdata=dupdata, append=True, db=db) with self.xact.cursor(db=db.db) as curs: retn = curs.putmulti(kvpairs, dupdata=dupdata, append=append) return retn except lmdb.MapFullError: return self._handle_mapfull()
def _xact_action(self, calling_func, xact_func, lkey, *args, db=None, **kwargs): if self.readonly: raise s_exc.IsReadOnly() if db is None: db = _DefaultDB try: self.dirty = True if not self.recovering: self._logXactOper(calling_func, lkey, *args, db=db, **kwargs) return xact_func(self.xact, lkey, *args, db=db.db, **kwargs) except lmdb.MapFullError: return self._handle_mapfull()
def dropdb(self, name): ''' Deletes an **entire database** (i.e. a table), losing all data. ''' if self.readonly: raise s_exc.IsReadOnly() while True: try: if not self.dbexists(name): return db = self.initdb(name) self.dirty = True self.xact.drop(db.db, delete=True) self.forcecommit() return except lmdb.MapFullError: self._handle_mapfull()
async def _push(self, event: str, *args: List[Any], **kwargs: Dict[str, Any]) -> Any: ''' Execute the change handler for the mesg Note: This method is considered 'protected', in that it should not be called from something other than self. ''' nexsiden = self.nexsiden if self.nexsroot is not None: # Distribute through the change root if self.nexsroot.readonly: raise s_exc.IsReadOnly() offs, retn = await self.nexsroot.issue(nexsiden, event, args, kwargs) await self.nexsroot.waitForOffset(offs) return retn # There's no change distribution, so directly execute return await self._nexshands[event](self, *args, **kwargs)
async def addNodes(self, nodedefs): ''' Add/merge nodes in bulk. The addNodes API is designed for bulk adds which will also set properties, add tags, add edges, and set nodedata to existing nodes. Nodes are specified as a list of the following tuples: ( (form, valu), {'props':{}, 'tags':{}}) Args: nodedefs (list): A list of nodedef tuples. Returns: (list): A list of xact messages. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) oldstrict = self.strict self.strict = False try: for nodedefn in nodedefs: try: node = await self._addNodeDef(nodedefn) if node is not None: yield node await asyncio.sleep(0) except asyncio.CancelledError: raise except Exception as e: if oldstrict: raise await self.warn(f'addNodes failed on {nodedefn}: {e}') await asyncio.sleep(0) finally: self.strict = oldstrict
async def addNode(self, name, valu, props=None): ''' Add a node by form name and value with optional props. Args: name (str): The form of node to add. valu (obj): The value for the node. props (dict): Optional secondary properties for the node. Notes: If a props dictionary is provided, it may be mutated during node construction. Returns: s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) form = self.core.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=valu) try: adds = self.getNodeAdds(form, valu, props=props) except Exception as e: if not self.strict: await self.warn(f'addNode: {e}') return None raise # depth first, so the last one is our added node nodes = await self.applyNodeEdits(adds) return nodes[-1]
async def applyNodeEdits(self, edits): ''' Sends edits to the write layer and evaluates the consequences (triggers, node object updates) ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) meta = await self.getSnapMeta() todo = s_common.todo('storNodeEdits', edits, meta) results = await self.core.dyncall(self.wlyr.iden, todo) wlyr = self.wlyr nodes = [] callbacks = [] actualedits = [] # List[Tuple[buid, form, changes]] # make a pass through the returned edits, apply the changes to our Nodes() # and collect up all the callbacks to fire at once at the end. It is # critical to fire all callbacks after applying all Node() changes. for buid, sode, postedits in results: cache = {wlyr.iden: sode} node = await self._joinStorNode(buid, cache) if node is None: # We got part of a node but no ndef continue nodes.append(node) if postedits: actualedits.append((buid, node.form.name, postedits)) for edit in postedits: etyp, parms, _ = edit if etyp == s_layer.EDIT_NODE_ADD: node.bylayer['ndef'] = wlyr.iden callbacks.append((node.form.wasAdded, (node, ), {})) callbacks.append((self.view.runNodeAdd, (node, ), {})) continue if etyp == s_layer.EDIT_NODE_DEL: callbacks.append((node.form.wasDeleted, (node, ), {})) callbacks.append((self.view.runNodeDel, (node, ), {})) continue if etyp == s_layer.EDIT_PROP_SET: (name, valu, oldv, stype) = parms prop = node.form.props.get(name) if prop is None: # pragma: no cover logger.warning( f'applyNodeEdits got EDIT_PROP_SET for bad prop {name} on form {node.form}' ) continue node.props[name] = valu node.bylayer['props'][name] = wlyr.iden callbacks.append((prop.wasSet, (node, oldv), {})) callbacks.append( (self.view.runPropSet, (node, prop, oldv), {})) continue if etyp == s_layer.EDIT_PROP_DEL: (name, oldv, stype) = parms prop = node.form.props.get(name) if prop is None: # pragma: no cover logger.warning( f'applyNodeEdits got EDIT_PROP_DEL for bad prop {name} on form {node.form}' ) continue node.props.pop(name, None) node.bylayer['props'].pop(name, None) callbacks.append((prop.wasDel, (node, oldv), {})) callbacks.append( (self.view.runPropSet, (node, prop, oldv), {})) continue if etyp == s_layer.EDIT_TAG_SET: (tag, valu, oldv) = parms node.tags[tag] = valu node.bylayer['tags'][tag] = wlyr.iden callbacks.append( (self.view.runTagAdd, (node, tag, valu), {})) callbacks.append((self.wlyr.fire, ('tag:add', ), { 'tag': tag, 'node': node.iden() })) continue if etyp == s_layer.EDIT_TAG_DEL: (tag, oldv) = parms node.tags.pop(tag, None) node.bylayer['tags'].pop(tag, None) callbacks.append( (self.view.runTagDel, (node, tag, oldv), {})) callbacks.append((self.wlyr.fire, ('tag:del', ), { 'tag': tag, 'node': node.iden() })) continue if etyp == s_layer.EDIT_TAGPROP_SET: (tag, prop, valu, oldv, stype) = parms node.tagprops[(tag, prop)] = valu node.bylayer['tags'][(tag, prop)] = wlyr.iden continue if etyp == s_layer.EDIT_TAGPROP_DEL: (tag, prop, oldv, stype) = parms node.tagprops.pop((tag, prop), None) node.bylayer['tags'].pop((tag, prop), None) continue [await func(*args, **kwargs) for (func, args, kwargs) in callbacks] if actualedits: providen, provstack = self.core.provstor.stor() if providen is not None: await self.fire('prov:new', time=meta['time'], user=meta['user'], prov=providen, provstack=provstack) await self.fire('node:edits', edits=actualedits) return nodes
async def addNodes(self, nodedefs): ''' Add/merge nodes in bulk. The addNodes API is designed for bulk adds which will also set properties, add tags, add edges, and set nodedata to existing nodes. Nodes are specified as a list of the following tuples: ( (form, valu), {'props':{}, 'tags':{}}) Args: nodedefs (list): A list of nodedef tuples. Returns: (list): A list of xact messages. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) nodeedits = [] buids = set() n2buids = set() for (formname, formvalu), forminfo in nodedefs: try: props = forminfo.get('props') # remove any universal created props... if props is not None: props.pop('.created', None) oldstrict = self.strict self.strict = False try: (node, editset) = await self._getAddNodeEdits(formname, formvalu, props=props) if editset is not None: buid, form, edits = editset[0] tags = forminfo.get('tags') tagprops = forminfo.get('tagprops') if tagprops is not None: if tags is None: tags = {} for tag in tagprops.keys(): tags[tag] = (None, None) if tags is not None: tagedits = await self._getAddTagEdits(node, tags) edits.extend(tagedits) if tagprops is not None: tpedits = await self._getAddTagPropEdits( node, tagprops) edits.extend(tpedits) nodedata = forminfo.get('nodedata') if nodedata is not None: try: for name, data in nodedata.items(): # make sure we have valid nodedata if not (isinstance(name, str)): await self.warn( f'Nodedata key is not a string: {name}' ) continue s_common.reqjsonsafe(data) edits.append((s_layer.EDIT_NODEDATA_SET, (name, data, None), ())) except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except Exception as e: await self.warn( f'Failed adding node data on {formname}, {formvalu}, {forminfo}: {e}' ) n2edits = [] for verb, n2iden in forminfo.get('edges', ()): # check for embedded ndef rather than n2iden if isinstance(n2iden, (list, tuple)): n2formname, n2valu = n2iden n2form = self.core.model.form(n2formname) if n2form is None: await self.warn( f'Failed to make n2 edge node for {n2iden}: invalid form' ) continue if n2form.isrunt: await self.warn( f'Edges cannot be used with runt nodes: {n2formname}' ) continue try: n2norm, _ = n2form.type.norm(n2valu) n2buid = s_common.buid( (n2form.name, n2norm)) if not (n2buid in n2buids or n2buid in buids): _, n2editset = await self._getAddNodeEdits( n2formname, n2valu) n2edits.extend(n2editset) except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except: await self.warn( f'Failed to make n2 edge node for {n2iden}' ) continue n2iden = s_common.ehex(n2buid) n2buids.add(n2buid) # make sure a valid iden and verb were passed in elif not (isinstance(n2iden, str) and len(n2iden) == 64): await self.warn(f'Invalid n2 iden {n2iden}') continue if not (isinstance(verb, str)): await self.warn(f'Invalid edge verb {verb}') continue edits.append( (s_layer.EDIT_EDGE_ADD, (verb, n2iden), ())) nodeedits.append((buid, form, edits)) nodeedits.extend(n2edits) buids.add(buid) await asyncio.sleep(0) except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except Exception as e: if not oldstrict: await self.warn( f'addNodes failed on {formname}, {formvalu}, {forminfo}: {e}' ) await asyncio.sleep(0) continue raise finally: self.strict = oldstrict if len(buids) >= 1000: nodes = await self.applyNodeEdits(nodeedits) for node in nodes: if node.buid in buids: yield node await asyncio.sleep(0) nodedits = [] buids.clear() n2buids.clear() except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except Exception: logger.exception(f'Error making node: [{formname}={formvalu}]') nodes = await self.applyNodeEdits(nodeedits) for node in nodes: if node.buid in buids: yield node await asyncio.sleep(0)
async def set(self, name, valu, init=False): ''' Set a property on the node. Args: name (str): The name of the property. valu (obj): The value of the property. init (bool): Set to True to disable read-only enforcement Returns: (bool): True if the property was changed. ''' if self.snap.readonly: mesg = 'Cannot set property in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) prop = self.form.prop(name) if prop is None: if self.snap.strict: mesg = f'No property named {name}.' raise s_exc.NoSuchProp(mesg=mesg, name=name, form=self.form.name) await self.snap.warn(f'NoSuchProp: name={name}') return False if self.form.isrunt: if prop.info.get('ro'): mesg = 'Cannot set read-only props on runt nodes' raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full, prop=name, valu=valu) await self.snap.core.runRuntPropSet(self, prop, valu) return True curv = self.props.get(name) # normalize the property value... try: norm, info = prop.type.norm(valu) except Exception as e: mesg = f'Bad property value: {prop.full}={valu!r}' return await self.snap._raiseOnStrict(s_exc.BadTypeValu, mesg, name=prop.name, valu=valu, emesg=str(e)) # do we already have the value? if curv == norm: return False await self.snap.core._callPropSetHook(self, prop, norm) if curv is not None and not init: if prop.info.get('ro'): if self.snap.strict: raise s_exc.ReadOnlyProp(name=prop.full) # not setting a set-once prop unless we are init... return False # check for type specific merging... norm = prop.type.merge(curv, norm) if curv == norm: return False props = {prop.name: norm} nodeedits = await self.snap.getNodeAdds(self.form, self.ndef[1], props, addnode=False) await self.snap.applyNodeEdits(nodeedits) return True