def setup_debugger():
'''
Rig up breakpoint() behavior based on the debugger option. Return True
if we set up a debugger, False otherwise.
'''
if options.debugger not in SUPPORTED_DEBUGGERS:
vd.status(f'Skipping setup for unknown debugger: {options.debugger}')
return False
debugger_env = {
'remote-pdb': {
'PYTHONBREAKPOINT': 'remote_pdb.set_trace',
'REMOTE_PDB_HOST': '127.0.0.1',
'REMOTE_PDB_PORT': '4444',
},
'pudb': {
'PYTHONBREAKPOINT': 'pudb.set_trace',
},
'web-pdb': {
'PYTHONBREAKPOINT': 'web_pdb.set_trace',
},
}
os.environ.update(debugger_env[options.debugger])
return True
def _checkSelectedIndex(self):
if self._selectedMask.index is not self.frame.index:
# selection is no longer valid
vd.status('frame.index updated, clearing {} selected rows'.format(
self._selectedMask.sum()))
self._selectedMask = Series.from_element(False,
index=self.frame.index)
def setup_debugger():
"""
Rig up breakpoint() behavior based on the debugger option. Return True
if we set up a debugger, False otherwise.
"""
if vd.options.debugger not in SUPPORTED_DEBUGGERS:
vd.status(
f"Skipping setup for unknown debugger: {vd.options.debugger}")
return False
debugger_env = {
"remote-pdb": {
"PYTHONBREAKPOINT": "remote_pdb.set_trace",
"REMOTE_PDB_HOST": "127.0.0.1",
"REMOTE_PDB_PORT": "4444",
},
"pudb": {
"PYTHONBREAKPOINT": "pudb.set_trace",
},
"web-pdb": {
"PYTHONBREAKPOINT": "web_pdb.set_trace",
},
}
os.environ.update(debugger_env[vd.options.debugger])
return True
def goParentRow(_, by):
"""
While focused in a child "detail" sheet, navigate through rows
in the parent sheet.
"""
parent = vd.sheets[1]
newIndex = parent.cursorRowIndex + by
if newIndex < 0:
vd.status("Already at the top!")
return
elif newIndex >= len(parent.rows):
vd.status("Already at the bottom!")
return
# The goal here is to navigate around a parent window in a consistent way,
# updating a child view in the process. Find out whether the current
# sheet represents a detail view of the cursor _row_ or _cell_ in the
# parent sheet. Use that to determine how to update the child view.
#
# Edge case:
#
# * When scrolling through parent cells that would yield no child content
# (and therefore not open a sheet), we need a dummy stand-in sheet to
# keep the child window open and avoid breaking things. This happens
# when, for example, the parent cell is an empty list.
for entityType in CHILD_ENTITY_TYPES:
vs = _replaceDetailSheet(newIndex, entityType)
if vs:
break
def unselect(self, rows, status=True, progress=True):
"Unselect given rows. Don't show progress if progress=False; don't show status if status=False."
self.addUndoSelection()
before = self.nSelected
for r in (Progress(rows, 'unselecting') if progress else rows):
self.unselectRow(r)
if status:
vd.status('unselected %s/%s %s' %
(before - self.nSelected, before, self.rowtype))
def toggle_versioning(self):
"""Enable or disable support for S3 versioning."""
self.version_aware = not self.version_aware
self.fs.version_aware = self.version_aware
vd.status(f's3 versioning {"enabled" if self.version_aware else "disabled"}')
if self.currentThreads:
vd.debug("cancelling threads before reloading")
vd.cancelThread(*self.currentThreads)
self.reload()
def cursesMain(_scr, sheetlist):
'Populate VisiData object with sheets from a given list.'
colors.setup()
for vs in sheetlist:
vd.push(vs)
vd.status('Ctrl+H opens help')
return vd.mainloop(_scr)
def deleteBy(self, by):
'''Delete rows for which func(row) is true. Returns number of deleted rows.'''
# oldidx = self.cursorRowIndex # NOTE: not used
nRows = self.nRows
vd.addUndo(setattr, self, 'frame', self.frame)
self.frame = self.frame[~by].reindex(IndexAutoFactory)
ndeleted = nRows - self.nRows
vd.status('deleted %s %s' % (ndeleted, self.rowtype))
return ndeleted
def select(self, rows, status=True, progress=True):
"Bulk select given rows. Don't show progress if progress=False; don't show status if status=False."
self.addUndoSelection()
before = self.nSelected
if options.bulk_select_clear:
self.clearSelected()
for r in (Progress(rows, 'selecting') if progress else rows):
self.selectRow(r)
if status:
if options.bulk_select_clear:
msg = 'selected %s %s%s' % (self.nSelected, self.rowtype,
' instead' if before > 0 else '')
else:
msg = 'selected %s%s %s' % (self.nSelected - before, ' more'
if before > 0 else '', self.rowtype)
vd.status(msg)
def undo(vd, sheet):
if not options.undo:
vd.fail("options.undo not enabled")
# don't allow undo of first command on a sheet, which is always the command that created the sheet.
for cmdlogrow in sheet.cmdlog_sheet.rows[:0:-1]:
if cmdlogrow.undofuncs:
for undofunc, args, kwargs, in cmdlogrow.undofuncs:
undofunc(*args, **kwargs)
sheet.undone.append(cmdlogrow)
sheet.cmdlog_sheet.rows.remove(cmdlogrow)
vd.moveToReplayContext(cmdlogrow)
vd.clearCaches()
vd.status("%s undone" % cmdlogrow.longname)
return
vd.fail("nothing to undo on current sheet")
def setValues(self, rows, *values):
'Set our column value for given list of rows to `value`.'
vd.addUndoSetValues([self], rows)
for r, v in zip(rows, itertools.cycle(values)):
self.setValueSafe(r, v)
self.recalc()
return vd.status('set %d cells to %d values' %
(len(rows), len(values)))
def getSheet(vd, sheetname):
matchingSheets = [x for x in vd.sheets if x.name == sheetname]
if matchingSheets:
if len(matchingSheets) > 1:
vd.status('more than one sheet named "%s"' % sheetname)
return matchingSheets[0]
try:
sheetidx = int(sheetname)
return vd.sheets[sheetidx]
except ValueError:
pass
if sheetname == 'options':
vs = self.optionsSheet
vs.reload()
vs.vd = vd
return vs
def break_once(obj, func):
"""
Wrap obj.func() to perform initial debugger setup and trigger a breakpoint.
After one invocation, restore the original function.
"""
f = getattr(obj, func)
@wraps(f)
def wrapper(*args, **kwargs):
setup_debugger() and breakpoint() or vd.status(
"Skipping debugger setup")
f(*args, **kwargs)
setattr(obj, func, f)
vd.status(f"{func} function restored to original")
setattr(obj, func, wrapper)
vd.status(f"{func} function wrapped to initialize debugging")
def zoomFreqtblRow(sheet, by):
'''
Navigate a frequency table sheet, "zooming in" on matching rows from the
source sheet. Open matching rows in a disposable sheet one level up
in the stack - while using a split view, this means the non-active
window is a dedicated zoom pane.
'''
if sheet.cursorRowIndex == len(sheet.rows) - 1 and by == 1:
vd.status('Already at the bottom!')
return
if sheet.cursorRowIndex == 0 and by == -1:
vd.status('Already at the top!')
return
sheet.cursorDown(by)
vs = sheet.openRow(sheet.cursorRow)
vs.precious = False
if sheet.source.source is vd.sheets[1].source and not vd.sheets[1].precious:
vd.remove(vd.sheets[1])
vd.sheets.insert(1, vs)
def openurl_s3(p, filetype):
"""Open a sheet for an S3 path.
S3 directories (prefixes) require special handling, but files (objects)
can use standard VisiData "open" functions.
"""
# Non-obvious behavior here: For the default case, we don't want to send
# a custom endpoint to s3fs. However, using None as a default trips up
# VisiData's type detection for the endpoint option. So we use an empty
# string as the default instead, and convert back to None here.
endpoint = vd.options.vds3_endpoint or None
p = S3Path(
str(p.given),
version_aware=getattr(p, "version_aware", vd.options.vds3_version_aware),
version_id=getattr(p, "version_id", None),
)
p.fs.version_aware = p.version_aware
if p.fs.client_kwargs.get("endpoint_url", "") != endpoint:
p.fs.client_kwargs = {"endpoint_url": endpoint}
p.fs.connect()
if not p.fs.isfile(str(p.given)):
return S3DirSheet(p.name, source=p, version_aware=p.version_aware)
if not filetype:
filetype = p.ext or "txt"
openfunc = getattr(vd, f"open_{filetype.lower()}")
if not openfunc:
vd.warning(f"no loader found for {filetype} files, falling back to txt")
filetype = "txt"
openfunc = vd.open_txt
assert callable(openfunc), f"no function/method available to open {p.given}"
vs = openfunc(p)
vd.status(
f'opening {p.given} as {filetype} (version id: {p.version_id or "latest"})'
)
return vs
def goParentRow(sheet, by):
'''
While focused in a child "detail" split view, navigate through rows
in the parent sheet.
'''
parent = vd.sheets[1]
newIndex = parent.cursorRowIndex + by
if newIndex < 0:
vd.status('Already at the top!')
return
elif newIndex >= len(parent.rows):
vd.status('Already at the bottom!')
return
if not NoContentPlaceholder.emptyRowSheet:
NoContentPlaceholder.emptyRowSheet = load_pyobj(
'placeholder', NoContentPlaceholder.emptyRowMessage)
if not NoContentPlaceholder.emptyCellSheet:
NoContentPlaceholder.emptyCellSheet = load_pyobj(
'placeholder', NoContentPlaceholder.emptyCellMessage)
# The goal here is to intelligently navigate around a parent window,
# updating a child view in the process. Find out whether the current
# sheet represents a detail view of the cursor _row_ or _cell_ in the
# parent sheet. Use that to determine how to update the child view.
#
# Edge case:
#
# * When scrolling through parent cells that would yield no child content,
# we need a dummy stand-in sheet to keep the child window open.
ChildUpdate = namedtuple('ChildUpdate',
'parentRowIdx openCommand placeholder')
childUpdates = [
ChildUpdate(newIndex, 'open-cell',
NoContentPlaceholder.emptyCellSheet),
ChildUpdate(newIndex, 'open-row', NoContentPlaceholder.emptyRowSheet),
]
for childUpdate in childUpdates:
vs = _replaceDetailSheet(*childUpdate)
if vs:
break
def setValuesTyped(self, rows, *values):
'Set values on this column for rows, coerced to the column type. will stop on first exception in type().'
vd.addUndoSetValues([self], rows)
for r, v in zip(rows,
itertools.cycle(self.type(val) for val in values)):
self.setValueSafe(r, v)
self.recalc()
return vd.status('set %d cells to %d values' %
(len(rows), len(values)))
def select_duplicate_rows(sheet, duplicates=True):
"""
Given a sheet, sets the selection status in VisiData to `selected` for each
row that is a duplicate of a prior row.
If `duplicates = False`, then the behavior is reversed; sets the selection
status to `selected` for each row that is *not* a duplicate.
"""
before = len(sheet.selectedRows)
gen = gen_identify_duplicates(sheet)
prog = Progress(gen, gerund="selecting", total=sheet.nRows)
for row, is_dupe in prog:
if is_dupe == duplicates:
sheet.selectRow(row)
sel_count = len(sheet.selectedRows) - before
more_str = " more" if before > 0 else ""
vd.status(f"selected {sel_count}{more_str} {sheet.rowtype}")
def createPillowImage(dwg):
im = Image.new("RGB", (640, 480), color=(0, 0, 0))
draw = ImageDraw.Draw(im)
font = ImageFont.truetype(dwg.options.darkdraw_font,
dwg.options.darkdraw_font_size)
vd.clearCaches()
dwg._scr = mock.MagicMock(__bool__=mock.Mock(return_value=True),
getmaxyx=mock.Mock(return_value=(9999, 9999)))
dwg.draw(dwg._scr)
displayed = set()
for y in range(dwg.minY, dwg.maxY + 1):
for x in range(dwg.minX, dwg.maxX + 1):
rows = dwg._displayedRows.get((x, y), None)
if not rows: continue
r = rows[-1]
k = str(r)
if k in displayed: continue
if not r.text: continue
if x - r.x >= len(r.text): continue
i = x - r.x
s = r.text[i:]
fg, bg, attrs = colors.split_colorstr(r.color)
c = termcolor_to_rgb(fg)
xy = ((r.x + i) * 8, r.y * 16)
if bg:
draw.rectangle((xy, (xy[0] + 16, xy[1] + 8)),
fill=termcolor_to_rgb(bg))
draw.text(xy, s, font=font, fill=c)
if 'underline' in attrs:
draw.line((xy, (xy[0] + 16, xy[1])), fill=c)
draw.line(((xy[0], xy[1] + 8), (xy[0] + 16, xy[1] + 8)),
fill=c)
displayed.add(k)
vd.status(' '.join(displayed))
return im
def select_duplicate_rows(sheet, duplicates = True):
"""
Given a sheet, sets the selection status in VisiData to `selected` for each
row that is a duplicate of a prior row.
If `duplicates = False`, then the behavior is reversed; sets the selection
status to `selected` for each row that is *not* a duplicate.
"""
before = len(sheet.selectedRows)
for row, is_dupe in gen_identify_duplicates(sheet):
if is_dupe == duplicates:
sheet.selectRow(row)
sel_count = len(sheet.selectedRows) - before
more_str = " more" if before > 0 else ""
vd.status("selected {}{} {}".format(
sel_count,
more_str,
sheet.rowtype
))
Try to guess an appropriate vfake faketype for a given column and row set.
If we find a match, run with it. NO REGERTS.
"""
isNull = _isNullFunc()
for col in cols:
faketype = None
with suppress(StopIteration):
sample_value = next(hint for r in rows
if not isNull(hint := col.getValue(r)))
faketype = next(v for k, v in faketype_mapping.items()
if k(str(sample_value), col.name))
if not faketype:
vd.warning(f"Could not detect a fake type for column {col.name}")
continue
vd.status(f"Detected fake type {faketype} for column {col.name}")
vd.addUndoSetValues([col], rows)
col.setValuesFromFaker(faketype, rows)
BaseSheet.bindkey("zf", "setcol-fake")
BaseSheet.addCommand(
"gzf",
"setcol-fake-all",
'cursorCol.setValuesFromFaker(vd.input("faketype: ", type="faketype"), rows)',
)
BaseSheet.addCommand("z^F", "setcol-autofake",
"sheet.autofake([cursorCol], rows)")
BaseSheet.addCommand("gz^F", "setcols-autofake",
"sheet.autofake(columns, rows)")
def warning(vd, *args):
vd.status(*args, priority=1)
def mainloop(self, scr):
'Manage execution of keystrokes and subsequent redrawing of screen.'
scr.timeout(curses_timeout)
with contextlib.suppress(curses.error):
curses.curs_set(0)
numTimeouts = 0
prefixWaiting = False
vd.scrFull = scr
self.keystrokes = ''
while True:
if not self.sheets:
# if no more sheets, exit
return
sheet = self.sheets[0]
threading.current_thread().sheet = sheet
vd.drawThread = threading.current_thread()
sheet.ensureLoaded()
vd.setWindows(scr)
self.draw(vd.win1, self.sheets[0])
if vd.win2 and len(self.sheets) > 1:
self.draw(vd.win2, self.sheets[1])
else:
vd.win2.erase()
vd.win2.refresh()
keystroke = self.getkeystroke(scr, sheet)
if not keystroke and prefixWaiting and ESC in self.keystrokes: # timeout ESC
self.keystrokes = ''
if keystroke: # wait until next keystroke to clear statuses and previous keystrokes
numTimeouts = 0
if not prefixWaiting:
self.keystrokes = ''
self.statuses.clear()
if keystroke == 'KEY_MOUSE':
self.keystrokes = ''
clicktype = ''
try:
devid, x, y, z, bstate = curses.getmouse()
sheet.mouseX, sheet.mouseY = x, y
if bstate & curses.BUTTON_CTRL:
clicktype += "CTRL-"
bstate &= ~curses.BUTTON_CTRL
if bstate & curses.BUTTON_ALT:
clicktype += "ALT-"
bstate &= ~curses.BUTTON_ALT
if bstate & curses.BUTTON_SHIFT:
clicktype += "SHIFT-"
bstate &= ~curses.BUTTON_SHIFT
keystroke = clicktype + curses.mouseEvents.get(
bstate, str(bstate))
f = self.getMouse(scr, x, y, keystroke)
if f:
if isinstance(f, str):
for cmd in f.split():
sheet.exec_keystrokes(cmd)
else:
f(y, x, keystroke)
self.keystrokes = keystroke
keystroke = ''
except curses.error:
pass
except Exception as e:
self.exceptionCaught(e)
if keystroke in self.keystrokes[:-1]:
vd.warning('duplicate prefix')
self.keystrokes = ''
else:
self.keystrokes += keystroke
self.drawRightStatus(sheet._scr,
sheet) # visible for commands that wait for input
if not keystroke: # timeout instead of keypress
pass
elif keystroke == '^Q':
return self.lastErrors and '\n'.join(self.lastErrors[-1])
elif vd.bindkeys._get(self.keystrokes):
sheet.exec_keystrokes(self.keystrokes)
prefixWaiting = False
elif keystroke in self.allPrefixes:
prefixWaiting = True
else:
vd.status('no command for "%s"' % (self.keystrokes))
prefixWaiting = False
self.checkForFinishedThreads()
sheet.checkCursorNoExceptions()
# no idle redraw unless background threads are running
time.sleep(0) # yield to other threads which may not have started yet
if vd.unfinishedThreads:
scr.timeout(curses_timeout)
else:
numTimeouts += 1
if numTimeouts > timeouts_before_idle:
scr.timeout(-1)
else:
scr.timeout(curses_timeout)
'''
Try to guess an appropriate vfake faketype for a given column and row set.
If we find a match, run with it. NO REGERTS.
'''
isNull = isNullFunc()
for col in cols:
faketype = None
with suppress(StopIteration):
next(r for r in rows if not isNull(hint := col.getValue(r)))
faketype = next(v for k, v in faketype_mapping.items()
if k(str(hint), col.name))
if not faketype:
vd.warning(f'Could not detect a fake type for column {col.name}')
continue
vd.status(f'Detected fake type {faketype} for column {col.name}')
vd.addUndoSetValues([col], rows)
col.setValuesFromFaker(faketype, rows)
BaseSheet.bindkey("zf", "setcol-fake")
BaseSheet.addCommand(
"gzf",
"setcol-fake-all",
'cursorCol.setValuesFromFaker(input("faketype: ", type="faketype"), rows)',
)
BaseSheet.addCommand('z^F', 'setcol-autofake',
f'{__name__}.autofake([cursorCol], rows)')
BaseSheet.addCommand('gz^F', 'setcols-autofake',
f'{__name__}.autofake(columns, rows)')
def redo(vd, sheet):
sheet.undone or vd.fail("nothing to redo")
cmdlogrow = sheet.undone.pop()
vd.replayOne(cmdlogrow)
vd.status("%s redone" % cmdlogrow.longname)
def error(vd, *args):
'Log an error and raise an exception.'
vd.status(*args, priority=3)
raise ExpectedException(args[0] if args else '')
def wrapper(*args, **kwargs):
setup_debugger() and breakpoint() or vd.status(
"Skipping debugger setup")
f(*args, **kwargs)
setattr(obj, func, f)
vd.status(f"{func} function restored to original")
def fail(vd, *args):
vd.status(*args, priority=2)
raise ExpectedException(args[0] if args else '')
def debug(vd, *args, **kwargs):
if options.debug:
return vd.status(*args, **kwargs)
