def putmulti(self, kvpairs, dupdata=False, append=False, db=None):
'''
Returns:
Tuple of number of items consumed, number of items added
'''
if self.readonly:
raise s_exc.IsReadOnly()
# Log playback isn't compatible with generators
if not isinstance(kvpairs, list):
kvpairs = list(kvpairs)
realdb, dupsort = self.dbnames[db]
try:
self.dirty = True
if not self.recovering:
self._logXactOper(self.putmulti,
kvpairs,
dupdata=dupdata,
append=append,
db=db)
with self.xact.cursor(db=realdb) as curs:
return curs.putmulti(kvpairs, dupdata=dupdata, append=append)
except lmdb.MapFullError:
return self._handle_mapfull()
async def addNode(self, name, valu, props=None):
'''
Add a node by form name and value with optional props.
Args:
name (str): The form of node to add.
valu (obj): The value for the node.
props (dict): Optional secondary properties for the node.
Notes:
If a props dictionary is provided, it may be mutated during node construction.
Returns:
s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
try:
if self.buidprefetch:
norm, info = form.type.norm(valu)
node = await self.getNodeByBuid(
s_common.buid((form.name, norm)))
if node is not None:
# TODO implement node.setNodeProps()
if props is not None:
for p, v in props.items():
await node.set(p, v)
return node
adds = await self.getNodeAdds(form, valu, props=props)
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception as e:
if not self.strict:
await self.warn(f'addNode: {e}')
return None
raise
nodes = await self.applyNodeEdits(adds)
assert len(nodes) >= 1
# Adds is top-down, so the first node is what we want
return nodes[0]
def _xact_action(self, calling_func, xact_func, lkey, *args, db=None, **kwargs):
if self.readonly:
raise s_exc.IsReadOnly()
realdb, dupsort = self.dbnames[db]
try:
self.dirty = True
if not self.recovering:
self._logXactOper(calling_func, lkey, *args, db=db, **kwargs)
return xact_func(self.xact, lkey, *args, db=realdb, **kwargs)
except lmdb.MapFullError:
return self._handle_mapfull()
def putmulti(self, kvpairs, dupdata=False, append=False, db=_DefaultDB):
if self.readonly:
raise s_exc.IsReadOnly()
try:
self.dirty = True
if not self.recovering:
self._logXactOper(self.putmulti, kvpairs, dupdata=dupdata, append=True, db=db)
with self.xact.cursor(db=db.db) as curs:
retn = curs.putmulti(kvpairs, dupdata=dupdata, append=append)
return retn
except lmdb.MapFullError:
return self._handle_mapfull()
def _xact_action(self, calling_func, xact_func, lkey, *args, db=None, **kwargs):
if self.readonly:
raise s_exc.IsReadOnly()
if db is None:
db = _DefaultDB
try:
self.dirty = True
if not self.recovering:
self._logXactOper(calling_func, lkey, *args, db=db, **kwargs)
return xact_func(self.xact, lkey, *args, db=db.db, **kwargs)
except lmdb.MapFullError:
return self._handle_mapfull()
def dropdb(self, name):
'''
Deletes an **entire database** (i.e. a table), losing all data.
'''
if self.readonly:
raise s_exc.IsReadOnly()
while True:
try:
if not self.dbexists(name):
return
db = self.initdb(name)
self.dirty = True
self.xact.drop(db.db, delete=True)
self.forcecommit()
return
except lmdb.MapFullError:
self._handle_mapfull()
async def _push(self, event: str, *args: List[Any],
**kwargs: Dict[str, Any]) -> Any:
'''
Execute the change handler for the mesg
Note:
This method is considered 'protected', in that it should not be called from something other than self.
'''
nexsiden = self.nexsiden
if self.nexsroot is not None: # Distribute through the change root
if self.nexsroot.readonly:
raise s_exc.IsReadOnly()
offs, retn = await self.nexsroot.issue(nexsiden, event, args,
kwargs)
await self.nexsroot.waitForOffset(offs)
return retn
# There's no change distribution, so directly execute
return await self._nexshands[event](self, *args, **kwargs)
async def addNodes(self, nodedefs):
'''
Add/merge nodes in bulk.
The addNodes API is designed for bulk adds which will
also set properties, add tags, add edges, and set nodedata to existing nodes.
Nodes are specified as a list of the following tuples:
( (form, valu), {'props':{}, 'tags':{}})
Args:
nodedefs (list): A list of nodedef tuples.
Returns:
(list): A list of xact messages.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
oldstrict = self.strict
self.strict = False
try:
for nodedefn in nodedefs:
try:
node = await self._addNodeDef(nodedefn)
if node is not None:
yield node
await asyncio.sleep(0)
except asyncio.CancelledError:
raise
except Exception as e:
if oldstrict:
raise
await self.warn(f'addNodes failed on {nodedefn}: {e}')
await asyncio.sleep(0)
finally:
self.strict = oldstrict
async def addNode(self, name, valu, props=None):
'''
Add a node by form name and value with optional props.
Args:
name (str): The form of node to add.
valu (obj): The value for the node.
props (dict): Optional secondary properties for the node.
Notes:
If a props dictionary is provided, it may be mutated during node construction.
Returns:
s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
try:
adds = self.getNodeAdds(form, valu, props=props)
except Exception as e:
if not self.strict:
await self.warn(f'addNode: {e}')
return None
raise
# depth first, so the last one is our added node
nodes = await self.applyNodeEdits(adds)
return nodes[-1]
async def applyNodeEdits(self, edits):
'''
Sends edits to the write layer and evaluates the consequences (triggers, node object updates)
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
meta = await self.getSnapMeta()
todo = s_common.todo('storNodeEdits', edits, meta)
results = await self.core.dyncall(self.wlyr.iden, todo)
wlyr = self.wlyr
nodes = []
callbacks = []
actualedits = [] # List[Tuple[buid, form, changes]]
# make a pass through the returned edits, apply the changes to our Nodes()
# and collect up all the callbacks to fire at once at the end. It is
# critical to fire all callbacks after applying all Node() changes.
for buid, sode, postedits in results:
cache = {wlyr.iden: sode}
node = await self._joinStorNode(buid, cache)
if node is None:
# We got part of a node but no ndef
continue
nodes.append(node)
if postedits:
actualedits.append((buid, node.form.name, postedits))
for edit in postedits:
etyp, parms, _ = edit
if etyp == s_layer.EDIT_NODE_ADD:
node.bylayer['ndef'] = wlyr.iden
callbacks.append((node.form.wasAdded, (node, ), {}))
callbacks.append((self.view.runNodeAdd, (node, ), {}))
continue
if etyp == s_layer.EDIT_NODE_DEL:
callbacks.append((node.form.wasDeleted, (node, ), {}))
callbacks.append((self.view.runNodeDel, (node, ), {}))
continue
if etyp == s_layer.EDIT_PROP_SET:
(name, valu, oldv, stype) = parms
prop = node.form.props.get(name)
if prop is None: # pragma: no cover
logger.warning(
f'applyNodeEdits got EDIT_PROP_SET for bad prop {name} on form {node.form}'
)
continue
node.props[name] = valu
node.bylayer['props'][name] = wlyr.iden
callbacks.append((prop.wasSet, (node, oldv), {}))
callbacks.append(
(self.view.runPropSet, (node, prop, oldv), {}))
continue
if etyp == s_layer.EDIT_PROP_DEL:
(name, oldv, stype) = parms
prop = node.form.props.get(name)
if prop is None: # pragma: no cover
logger.warning(
f'applyNodeEdits got EDIT_PROP_DEL for bad prop {name} on form {node.form}'
)
continue
node.props.pop(name, None)
node.bylayer['props'].pop(name, None)
callbacks.append((prop.wasDel, (node, oldv), {}))
callbacks.append(
(self.view.runPropSet, (node, prop, oldv), {}))
continue
if etyp == s_layer.EDIT_TAG_SET:
(tag, valu, oldv) = parms
node.tags[tag] = valu
node.bylayer['tags'][tag] = wlyr.iden
callbacks.append(
(self.view.runTagAdd, (node, tag, valu), {}))
callbacks.append((self.wlyr.fire, ('tag:add', ), {
'tag': tag,
'node': node.iden()
}))
continue
if etyp == s_layer.EDIT_TAG_DEL:
(tag, oldv) = parms
node.tags.pop(tag, None)
node.bylayer['tags'].pop(tag, None)
callbacks.append(
(self.view.runTagDel, (node, tag, oldv), {}))
callbacks.append((self.wlyr.fire, ('tag:del', ), {
'tag': tag,
'node': node.iden()
}))
continue
if etyp == s_layer.EDIT_TAGPROP_SET:
(tag, prop, valu, oldv, stype) = parms
node.tagprops[(tag, prop)] = valu
node.bylayer['tags'][(tag, prop)] = wlyr.iden
continue
if etyp == s_layer.EDIT_TAGPROP_DEL:
(tag, prop, oldv, stype) = parms
node.tagprops.pop((tag, prop), None)
node.bylayer['tags'].pop((tag, prop), None)
continue
[await func(*args, **kwargs) for (func, args, kwargs) in callbacks]
if actualedits:
providen, provstack = self.core.provstor.stor()
if providen is not None:
await self.fire('prov:new',
time=meta['time'],
user=meta['user'],
prov=providen,
provstack=provstack)
await self.fire('node:edits', edits=actualedits)
return nodes
async def addNodes(self, nodedefs):
'''
Add/merge nodes in bulk.
The addNodes API is designed for bulk adds which will
also set properties, add tags, add edges, and set nodedata to existing nodes.
Nodes are specified as a list of the following tuples:
( (form, valu), {'props':{}, 'tags':{}})
Args:
nodedefs (list): A list of nodedef tuples.
Returns:
(list): A list of xact messages.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
nodeedits = []
buids = set()
n2buids = set()
for (formname, formvalu), forminfo in nodedefs:
try:
props = forminfo.get('props')
# remove any universal created props...
if props is not None:
props.pop('.created', None)
oldstrict = self.strict
self.strict = False
try:
(node, editset) = await self._getAddNodeEdits(formname,
formvalu,
props=props)
if editset is not None:
buid, form, edits = editset[0]
tags = forminfo.get('tags')
tagprops = forminfo.get('tagprops')
if tagprops is not None:
if tags is None:
tags = {}
for tag in tagprops.keys():
tags[tag] = (None, None)
if tags is not None:
tagedits = await self._getAddTagEdits(node, tags)
edits.extend(tagedits)
if tagprops is not None:
tpedits = await self._getAddTagPropEdits(
node, tagprops)
edits.extend(tpedits)
nodedata = forminfo.get('nodedata')
if nodedata is not None:
try:
for name, data in nodedata.items():
# make sure we have valid nodedata
if not (isinstance(name, str)):
await self.warn(
f'Nodedata key is not a string: {name}'
)
continue
s_common.reqjsonsafe(data)
edits.append((s_layer.EDIT_NODEDATA_SET,
(name, data, None), ()))
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception as e:
await self.warn(
f'Failed adding node data on {formname}, {formvalu}, {forminfo}: {e}'
)
n2edits = []
for verb, n2iden in forminfo.get('edges', ()):
# check for embedded ndef rather than n2iden
if isinstance(n2iden, (list, tuple)):
n2formname, n2valu = n2iden
n2form = self.core.model.form(n2formname)
if n2form is None:
await self.warn(
f'Failed to make n2 edge node for {n2iden}: invalid form'
)
continue
if n2form.isrunt:
await self.warn(
f'Edges cannot be used with runt nodes: {n2formname}'
)
continue
try:
n2norm, _ = n2form.type.norm(n2valu)
n2buid = s_common.buid(
(n2form.name, n2norm))
if not (n2buid in n2buids
or n2buid in buids):
_, n2editset = await self._getAddNodeEdits(
n2formname, n2valu)
n2edits.extend(n2editset)
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except:
await self.warn(
f'Failed to make n2 edge node for {n2iden}'
)
continue
n2iden = s_common.ehex(n2buid)
n2buids.add(n2buid)
# make sure a valid iden and verb were passed in
elif not (isinstance(n2iden, str)
and len(n2iden) == 64):
await self.warn(f'Invalid n2 iden {n2iden}')
continue
if not (isinstance(verb, str)):
await self.warn(f'Invalid edge verb {verb}')
continue
edits.append(
(s_layer.EDIT_EDGE_ADD, (verb, n2iden), ()))
nodeedits.append((buid, form, edits))
nodeedits.extend(n2edits)
buids.add(buid)
await asyncio.sleep(0)
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception as e:
if not oldstrict:
await self.warn(
f'addNodes failed on {formname}, {formvalu}, {forminfo}: {e}'
)
await asyncio.sleep(0)
continue
raise
finally:
self.strict = oldstrict
if len(buids) >= 1000:
nodes = await self.applyNodeEdits(nodeedits)
for node in nodes:
if node.buid in buids:
yield node
await asyncio.sleep(0)
nodedits = []
buids.clear()
n2buids.clear()
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception:
logger.exception(f'Error making node: [{formname}={formvalu}]')
nodes = await self.applyNodeEdits(nodeedits)
for node in nodes:
if node.buid in buids:
yield node
await asyncio.sleep(0)
async def set(self, name, valu, init=False):
'''
Set a property on the node.
Args:
name (str): The name of the property.
valu (obj): The value of the property.
init (bool): Set to True to disable read-only enforcement
Returns:
(bool): True if the property was changed.
'''
if self.snap.readonly:
mesg = 'Cannot set property in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
prop = self.form.prop(name)
if prop is None:
if self.snap.strict:
mesg = f'No property named {name}.'
raise s_exc.NoSuchProp(mesg=mesg,
name=name,
form=self.form.name)
await self.snap.warn(f'NoSuchProp: name={name}')
return False
if self.form.isrunt:
if prop.info.get('ro'):
mesg = 'Cannot set read-only props on runt nodes'
raise s_exc.IsRuntForm(mesg=mesg,
form=self.form.full,
prop=name,
valu=valu)
await self.snap.core.runRuntPropSet(self, prop, valu)
return True
curv = self.props.get(name)
# normalize the property value...
try:
norm, info = prop.type.norm(valu)
except Exception as e:
mesg = f'Bad property value: {prop.full}={valu!r}'
return await self.snap._raiseOnStrict(s_exc.BadTypeValu,
mesg,
name=prop.name,
valu=valu,
emesg=str(e))
# do we already have the value?
if curv == norm:
return False
await self.snap.core._callPropSetHook(self, prop, norm)
if curv is not None and not init:
if prop.info.get('ro'):
if self.snap.strict:
raise s_exc.ReadOnlyProp(name=prop.full)
# not setting a set-once prop unless we are init...
return False
# check for type specific merging...
norm = prop.type.merge(curv, norm)
if curv == norm:
return False
props = {prop.name: norm}
nodeedits = await self.snap.getNodeAdds(self.form,
self.ndef[1],
props,
addnode=False)
await self.snap.applyNodeEdits(nodeedits)
return True
