def reprTag(pode, tag):
'''
Get the human readable value for the tag timestamp from the pode.
Args:
pode (tuple): A packed node.
tag (str): The tag to get the value for.
Notes:
The human readable value is only available if the node came from a
storm query execution where the ``repr`` key was passed into the
``opts`` argument with a True value.
If the tag does not have a timestamp, this returns a empty string.
Returns:
str: The human readable value for the tag. If the tag is not present, returns None.
'''
tag = tag.lstrip('#')
valu = pode[1]['tags'].get(tag)
if valu is None:
return None
if valu == (None, None):
return ''
mint = s_time.repr(valu[0])
maxt = s_time.repr(valu[1])
valu = f'({mint}, {maxt})'
return valu
def reprTag(pode, tag):
'''
Get the human readable value for the tag timestamp from the pode.
Args:
pode (tuple): A packed node.
tag (str): The tag to get the value for.
Notes:
The human readable value is only available if the node came from a
storm query execution where the ``repr`` key was passed into the
``opts`` argument with a True value.
If the tag does not have a timestamp, this returns a empty string.
Returns:
str: The human readable value for the tag. If the tag is not present, returns None.
'''
tag = tag.lstrip('#')
valu = pode[1]['tags'].get(tag)
if valu is None:
return None
if valu == (None, None):
return ''
mint = s_time.repr(valu[0])
maxt = s_time.repr(valu[1])
valu = f'({mint}, {maxt})'
return valu
def openLogFd(self, opts):
opath = self.locs.get('log:fp')
if opath:
self.printf(
'Must call --off to disable current file before starting a new file.'
)
return
fmt = opts.format
path = opts.path
nodes_only = opts.nodes_only
edits_only = opts.edits_only
if not path:
ts = s_time.repr(s_common.now(), True)
fn = f'storm_{ts}.{fmt}'
path = s_common.getSynPath('stormlogs', fn)
self.printf(f'Starting logfile at [{path}]')
q = queue.Queue()
fd = s_common.genfile(path)
# Seek to the end of the file. Allows a user to append to a file.
fd.seek(0, 2)
self.locs['log:fp'] = path
self.locs['log:fd'] = fd
self.locs['log:fmt'] = fmt
self.locs['log:queue'] = q
self.locs['log:thr'] = self.queueLoop()
self.locs['log:nodesonly'] = nodes_only
self.locs['log:editsonly'] = edits_only
self._cmd_cli.on('storm:mesg', self.onStormMesg)
def openLogFd(self, opts):
opath = self.locs.get('log:fp')
if opath:
self.printf('Must call --off to disable current file before starting a new file.')
return
fmt = opts.format
path = opts.path
nodes_only = opts.nodes_only
splice_only = opts.splices_only
if not path:
ts = s_time.repr(s_common.now(), True)
fn = f'storm_{ts}.{fmt}'
path = s_common.getSynPath('stormlogs', fn)
self.printf(f'Starting logfile at [{path}]')
q = queue.Queue()
fd = s_common.genfile(path)
# Seek to the end of the file. Allows a user to append to a file.
fd.seek(0, 2)
self.locs['log:fp'] = path
self.locs['log:fd'] = fd
self.locs['log:fmt'] = fmt
self.locs['log:queue'] = q
self.locs['log:thr'] = self.queueLoop()
self.locs['log:nodesonly'] = nodes_only
self.locs['log:splicesonly'] = splice_only
self._cmd_cli.on('storm:mesg', self.onStormMesg)
async def runCmdOpts(self, opts):
core = self.getCmdItem()
tasks = await core.ps()
isverbose = opts.get('verbose') or opts.get('v')
MAXFIELDLEN = 120
def clamp(field):
if isinstance(field, dict):
for key, valu in field.items():
field[key] = clamp(valu)
elif isinstance(field, str) and len(field) > MAXFIELDLEN:
field = field[:MAXFIELDLEN] + '...'
return field
for task in tasks:
self.printf('task iden: %s' % (task.get('iden'), ))
self.printf(' name: %s' % (task.get('name'), ))
self.printf(' user: %r' % (task.get('user'), ))
self.printf(' status: %r' % (task.get('status'), ))
metadata = task.get('info')
if metadata is not None and not isverbose:
metadata = clamp(metadata)
self.printf(' metadata: %r' % metadata)
self.printf(' start time: %s' %
(s_time.repr(task.get('tick', 0)), ))
self.printf('%d tasks found.' % (len(tasks, )))
def _onNode(self, mesg):
node = mesg[1]
opts = node[1].pop('_opts', {})
formname = node[0][0]
formvalu = node[1].get('repr')
if formvalu is None:
formvalu = str(node[0][1])
if opts.get('raw'):
self.printf(repr(node))
return
self.printf(f'{formname}={formvalu}')
if not opts.get('hide-props'):
for name, valu in sorted(node[1]['props'].items()):
valu = node[1]['reprs'].get(name, valu)
if name[0] != '.':
name = ':' + name
self.printf(f' {name} = {valu}')
if not opts.get('hide-tags'):
for tag in sorted(s_node.tags(node, leaf=True)):
valu = node[1]['tags'].get(tag)
if valu == (None, None):
self.printf(f' #{tag}')
continue
mint = s_time.repr(valu[0])
maxt = s_time.repr(valu[1])
self.printf(f' #{tag} = ({mint}, {maxt})')
async def runCmdOpts(self, opts):
core = self.getCmdItem()
tasks = await core.ps()
for task in tasks:
self.printf('task iden: %s' % (task.get('iden'),))
self.printf(' name: %s' % (task.get('name'),))
self.printf(' user: %r' % (task.get('user'),))
self.printf(' status: %r' % (task.get('status'),))
self.printf(' metadata: %r' % (task.get('info'),))
self.printf(' start time: %s' % (s_time.repr(task.get('tick', 0)),))
self.printf('%d tasks found.' % (len(tasks,)))
async def runCmdOpts(self, opts):
core = self.getCmdItem()
tasks = await core.ps()
for task in tasks:
self.printf('task iden: %s' % (task.get('iden'),))
self.printf(' name: %s' % (task.get('name'),))
self.printf(' user: %r' % (task.get('user'),))
self.printf(' status: %r' % (task.get('status'),))
self.printf(' metadata: %r' % (task.get('info'),))
self.printf(' start time: %s' % (s_time.repr(task.get('tick', 0)),))
self.printf('%d tasks found.' % (len(tasks,)))
def _onInit(self, mesg, opts):
tick = mesg[1].get('tick')
if tick is not None:
tick = s_time.repr(tick)
self.printf(f'Executing query at {tick}')
def repr(self, valu):
if valu == self.futsize:
return '?'
return s_time.repr(valu)
async def status(self, pprint=False):
'''
Provide sync summary by layer
Args:
pprint (bool): Whether to include pretty-printed layer status string in result
Returns:
(dict): Summary info with layer idens as keys
'''
retn = {}
for lyriden, _ in self.layers.items():
pulloffs = self.pull_offs.get(lyriden)
pushoffs = self.push_offs.get(lyriden)
queue = self._queues.get(lyriden)
queuestat = {}
if queue is not None:
queuestat = {
'isfini': queue.isfini,
'len': len(queue.linklist)
}
srclaststart = self.pull_last_start.get(lyriden)
if srclaststart is not None:
srclaststart = s_time.repr(srclaststart)
destlaststart = self.push_last_start.get(lyriden)
if destlaststart is not None:
destlaststart = s_time.repr(destlaststart)
pullstatus = self._pull_status.get(lyriden)
srctasksum = await self._getTaskSummary(
self.pull_tasks.get(lyriden))
desttasksum = await self._getTaskSummary(
self.push_tasks.get(lyriden))
errcnt = self.errcnts.get(lyriden, defv=0)
retn[lyriden] = {
'src:pullstatus': pullstatus,
'src:nextoffs': pulloffs,
'dest:nextoffs': pushoffs,
'queue': queuestat,
'src:task': srctasksum,
'dest:task': desttasksum,
'src:laststart': srclaststart,
'dest:laststart': destlaststart,
'errcnt': errcnt,
'migrmode_override': self.migrmode.valu,
}
if pprint:
outp = [
f'Layer: {lyriden}',
(f'{"":^6}|{"last_start":^25}|{"task_status":^15}|{"offset":^15}|{"read_status":^22}|'
f'{"queue_live":^15}|{"queue_len":^15}|{"err_cnt":^15}'),
'-' * (128 + 7)
]
# src side
tasksum = desttasksum.get('status', '-')
outp.append((
f' {"src":<5}| {srclaststart or "-":<24}| {tasksum:<14}| {pulloffs:<14,}| {pullstatus or "-":<21}|'
f' {"n/a":<14}| {"n/a":<14}| {"n/a":<14}'))
# dest side
tasksum = desttasksum.get('status', '-')
outp.append((
f' {"dest":<5}| {destlaststart or "-":<24}| {tasksum:<14}| {pushoffs:<14,}| {"n/a":<21}|'
f' {not queuestat.get("isfini", True)!s:<14}| {queuestat.get("len", 0):<14,}| {errcnt:<14}'
))
retn[lyriden]['pprint'] = '\n'.join(outp)
return retn
def repr(self, valu):
return s_time.repr(valu)
