async def _runJob(self, user, appt):
'''
Actually run the storm query, updating the appropriate statistics and results
'''
count = 0
appt.isrunning = True
appt.laststarttime = time.time()
appt.startcount += 1
await self._storeAppt(appt)
with s_provenance.claim('cron', iden=appt.iden):
logger.info('Agenda executing for iden=%s, user=%s, query={%s}', appt.iden, user.name, appt.query)
starttime = time.time()
try:
async for _ in self.core.eval(appt.query, user=user): # NOQA
count += 1
except asyncio.CancelledError:
result = 'cancelled'
raise
except Exception as e:
result = f'raised exception {e}'
logger.exception('Agenda job %s raised exception', appt.iden)
else:
result = f'finished successfully with {count} nodes'
finally:
finishtime = time.time()
logger.info('Agenda completed query for iden=%s with result "%s" took %0.3fs',
appt.iden, result, finishtime - starttime)
appt.lastfinishtime = finishtime
appt.isrunning = False
appt.lastresult = result
if not self.isfini:
await self._storeAppt(appt)
async def _runJob(self, user, appt):
'''
Actually run the storm query, updating the appropriate statistics and results
'''
count = 0
appt.isrunning = True
appt.laststarttime = time.time()
appt.startcount += 1
await self._storeAppt(appt)
with s_provenance.claim('cron', iden=appt.iden):
logger.info('Agenda executing for iden=%s, user=%s, query={%s}', appt.iden, user.name, appt.query)
starttime = time.time()
try:
async for _ in self.core.eval(appt.query, user=user): # NOQA
count += 1
except asyncio.CancelledError:
result = 'cancelled'
raise
except Exception as e:
result = f'raised exception {e}'
logger.exception('Agenda job %s raised exception', appt.iden)
else:
result = f'finished successfully with {count} nodes'
finally:
finishtime = time.time()
logger.info('Agenda completed query for iden=%s with result "%s" took %0.3fs',
appt.iden, result, finishtime - starttime)
appt.lastfinishtime = finishtime
appt.isrunning = False
appt.lastresult = result
if not self.isfini:
await self._storeAppt(appt)
async def execute(self, node, vars=None):
'''
Actually execute the query
'''
opts = {}
if not self.enabled:
return
if vars is not None:
opts['vars'] = vars
user = node.snap.core.auth.user(self.useriden)
if user is None:
logger.warning('Unknown user %s in stored trigger', self.useriden)
return
with s_provenance.claim('trig',
cond=self.cond,
form=self.form,
tag=self.tag,
prop=self.prop):
try:
await s_common.aspin(
node.storm(self.storm, opts=opts, user=user))
except asyncio.CancelledError: # pragma: no cover
raise
except Exception:
logger.exception(
'Trigger encountered exception running storm query %s',
self.storm)
async def execute(self, node, vars=None):
'''
Actually execute the query
'''
opts = {}
if not self.tdef.get('enabled'):
return
if vars is not None:
opts['vars'] = vars
useriden = self.tdef.get('user')
user = node.snap.core.auth.user(useriden)
if user is None:
logger.warning('Unknown user %s in stored trigger', useriden)
return
tag = self.tdef.get('tag')
cond = self.tdef.get('cond')
form = self.tdef.get('form')
prop = self.tdef.get('prop')
storm = self.tdef.get('storm')
with s_provenance.claim('trig', cond=cond, form=form, tag=tag, prop=prop):
try:
await s_common.aspin(node.storm(storm, opts=opts, user=user))
except (asyncio.CancelledError, s_exc.RecursionLimitHit):
raise
except Exception:
logger.exception('Trigger encountered exception running storm query %s', storm)
async def execute(self, node, vars=None, view=None):
'''
Actually execute the query
'''
opts = {}
if not self.tdef.get('enabled'):
return
useriden = self.tdef.get('user')
user = self.view.core.auth.user(useriden)
locked = user.info.get('locked')
if locked:
if not self.lockwarned:
self.lockwarned = True
logger.warning(
'Skipping trigger execution because user {useriden} is locked'
)
return
tag = self.tdef.get('tag')
cond = self.tdef.get('cond')
form = self.tdef.get('form')
prop = self.tdef.get('prop')
storm = self.tdef.get('storm')
query = self.view.core.getStormQuery(storm)
if view is None:
view = self.view.iden
opts = {
'user': useriden,
'view': view,
}
if vars is not None:
opts['vars'] = vars
with s_provenance.claim('trig',
cond=cond,
form=form,
tag=tag,
prop=prop):
try:
async with self.view.core.getStormRuntime(query,
opts=opts) as runt:
runt.addInput(node)
await s_common.aspin(runt.execute())
except (asyncio.CancelledError, s_exc.RecursionLimitHit):
raise
except Exception:
logger.exception(
'Trigger encountered exception running storm query %s',
storm)
async def iterStormQuery(self, query, genr=None):
with s_provenance.claim('storm', q=query.text, user=self.user.iden):
self.loadRuntVars(query)
# init any options from the query
# (but dont override our own opts)
for name, valu in query.opts.items():
self.opts.setdefault(name, valu)
async for node, path in query.iterNodePaths(self, genr=genr):
self.tick()
yield node, path
async def iterStormQuery(self, query, genr=None):
with s_provenance.claim('storm', q=query.text, user=self.user.iden):
# do a quick pass to determine which vars are per-node.
for oper in query.kids:
for name in oper.getRuntVars(self):
self.runtvars.add(name)
# init any options from the query
# (but dont override our own opts)
for name, valu in query.opts.items():
self.opts.setdefault(name, valu)
async for node, path in query.iterNodePaths(self, genr=genr):
self.tick()
yield node, path
async def _runJob(self, user, appt):
'''
Actually run the storm query, updating the appropriate statistics and results
'''
count = 0
appt.isrunning = True
appt.laststarttime = self._getNowTick()
appt.startcount += 1
await self._storeAppt(appt, nexs=True)
with s_provenance.claim('cron', iden=appt.iden):
logger.info(f'Agenda executing for iden={appt.iden}, name={appt.name} user={user.name}, view={appt.view}, query={appt.query}',
extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden, 'text': appt.query,
'username': user.name, 'view': appt.view}})
starttime = self._getNowTick()
success = False
try:
opts = {'user': user.iden, 'view': appt.view}
async for node in self.core.eval(appt.query, opts=opts):
count += 1
except asyncio.CancelledError:
result = 'cancelled'
raise
except Exception as e:
result = f'raised exception {e}'
logger.exception(f'Agenda job {appt.iden} {appt.name} raised exception',
extra={'synapse': {'iden': appt.iden, 'name': appt.name}}
)
else:
success = True
result = f'finished successfully with {count} nodes'
finally:
finishtime = self._getNowTick()
if not success:
appt.errcount += 1
appt.lasterrs.append(result)
took = finishtime - starttime
mesg = f'Agenda completed query for iden={appt.iden} name={appt.name} with result "{result}" ' \
f'took {took:.3f}s'
logger.info(mesg, extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden,
'result': result, 'username': user.name, 'took': took}})
appt.lastfinishtime = finishtime
appt.isrunning = False
appt.lastresult = result
if not self.isfini:
await self._storeAppt(appt, nexs=True)
async def _execute(self, node, vars=None, view=None):
opts = {}
locked = self.user.info.get('locked')
if locked:
if not self.lockwarned:
self.lockwarned = True
logger.warning(f'Skipping trigger execution because user {self.user.iden} is locked')
return
tag = self.tdef.get('tag')
cond = self.tdef.get('cond')
form = self.tdef.get('form')
prop = self.tdef.get('prop')
storm = self.tdef.get('storm')
query = await self.view.core.getStormQuery(storm)
if view is None:
view = self.view.iden
opts = {
'view': view,
'user': self.user.iden,
}
if vars is not None:
opts['vars'] = vars
self.startcount += 1
with s_provenance.claim('trig', cond=cond, form=form, tag=tag, prop=prop):
try:
async with self.view.core.getStormRuntime(query, opts=opts) as runt:
runt.addInput(node)
await s_common.aspin(runt.execute())
except (asyncio.CancelledError, s_exc.RecursionLimitHit):
raise
except Exception as e:
self.errcount += 1
self.lasterrs.append(str(e))
logger.exception('Trigger encountered exception running storm query %s', storm)
async def run(self, runt, genr):
name = self.kids[0].value()
argv = self.kids[1].value()
ctor = runt.snap.core.getStormCmd(name)
if ctor is None:
mesg = 'Storm command not found.'
raise s_exc.NoSuchName(name=name, mesg=mesg)
scmd = ctor(argv)
if not await scmd.hasValidOpts(runt.snap):
return
with s_provenance.claim('stormcmd', name=name, argv=argv):
async for item in scmd.execStormCmd(runt, genr):
yield item
async def _libIngest(self, name, data, seqn=None):
'''
Add nodes to the graph with a given ingest type.
Args:
name (str): Name of the ingest function to send data too.
data: Data to feed to the ingest function.
seqn: A tuple of (guid, offset) values used for tracking ingest data.
Notes:
This is using the Runtimes's Snap to call addFeedData().
Returns:
None or the sequence offset value.
'''
self.runt.reqLayerAllowed(('feed:data', *name.split('.')))
with s_provenance.claim('feed:data', name=name):
return await self.runt.snap.addFeedData(name, data, seqn)
async def execute(self, node, vars=None):
'''
Actually execute the query
'''
opts = {}
if vars is not None:
opts['vars'] = vars
user = node.snap.core.auth.user(self.useriden)
if user is None:
logger.warning('Unknown user %s in stored trigger', self.useriden)
return
with s_provenance.claim('trig', cond=self.cond, form=self.form, tag=self.tag, prop=self.prop):
try:
await s_common.aspin(node.storm(self.storm, opts=opts, user=user))
except asyncio.CancelledError: # pragma: no cover
raise
except Exception:
logger.exception('Trigger encountered exception running storm query %s', self.storm)
async def _libGenr(self, name, data):
'''
Yield nodes being added to the graph by adding data with a given ingest type.
Args:
name (str): Name of the ingest function to send data too.
data: Data to feed to the ingest function.
Notes:
This is using the Runtimes's Snap to call addFeedNodes().
This only yields nodes if the feed function yields nodes.
If the generator is not entirely consumed there is no guarantee
that all of the nodes which should be made by the feed function
will be made.
Returns:
s_node.Node: An async generator that yields nodes.
'''
self.runt.reqLayerAllowed(('feed:data', *name.split('.')))
with s_provenance.claim('feed:data', name=name):
return self.runt.snap.addFeedNodes(name, data)