def deleteSelected(self):
'Delete all selected rows.'
ndeleted = self.deleteBy(self.isSelected)
nselected = self.nSelected
self.clearSelected()
if ndeleted != nselected:
vd.warning(f'deleted {ndeleted}, expected {nselected}')
def gen_identify_duplicates(sheet):
"""
Takes a sheet, and returns a generator yielding a tuple for each row
encountered. The tuple's structure is `(row_object, is_dupe)`, where
is_dupe is True/False.
See note in Usage section above regarding how duplicates are determined.
"""
keyCols = sheet.keyCols
cols_to_check = None
if len(keyCols) == 0:
vd.warning("No key cols specified. Using all columns.")
cols_to_check = sheet.visibleCols
else:
cols_to_check = sheet.keyCols
seen = set()
for r in sheet.rows:
vals = tuple(col.getValue(r) for col in cols_to_check)
is_dupe = vals in seen
if not is_dupe:
seen.add(vals)
yield (r, is_dupe)
def set(self, k, v, obj='override'):
opt = self._get(k)
if opt:
curval = opt.value
t = type(curval)
if v is None and curval is not None:
v = t() # empty value
elif isinstance(
v, str) and t is bool: # special case for bool options
v = v and (
v[0] not in "0fFnN"
) # ''/0/false/no are false, everything else is true
elif type(v) is t: # if right type, no conversion
pass
elif curval is None: # if None, do not apply type conversion
pass
else:
v = t(v)
if curval != v and self._get(k, 'global').replayable:
if vd.cmdlog: # options set on init aren't recorded
vd.set_option(vd.cmdlog, k, v, obj)
else:
curval = None
vd.warning('setting unknown option %s' % k)
return self._set(k, v, obj)
def setValue(self, col, row, val):
'''
Update a column's value in the underlying Frame, loosening the
column's type as needed.
'''
dtype_old = col.sheet.frame.dtypes[col.expr]
f = col.sheet.frame.assign.loc[row.name, col.expr](val)
dtype_new = f.dtypes[col.expr]
if dtype_old != dtype_new:
vd.warning(f'Type of {val} does not match column {col.expr}. Changing type.')
col.type = self.dtype_to_type(dtype_new)
# assign back to frame, convert to StaticFrameAdapter
self.rows = StaticFrameAdapter(f)
def openurl_s3(p, filetype):
"""Open a sheet for an S3 path.
S3 directories (prefixes) require special handling, but files (objects)
can use standard VisiData "open" functions.
"""
# Non-obvious behavior here: For the default case, we don't want to send
# a custom endpoint to s3fs. However, using None as a default trips up
# VisiData's type detection for the endpoint option. So we use an empty
# string as the default instead, and convert back to None here.
endpoint = vd.options.vds3_endpoint or None
p = S3Path(
str(p.given),
version_aware=getattr(p, "version_aware", vd.options.vds3_version_aware),
version_id=getattr(p, "version_id", None),
)
p.fs.version_aware = p.version_aware
if p.fs.client_kwargs.get("endpoint_url", "") != endpoint:
p.fs.client_kwargs = {"endpoint_url": endpoint}
p.fs.connect()
if not p.fs.isfile(str(p.given)):
return S3DirSheet(p.name, source=p, version_aware=p.version_aware)
if not filetype:
filetype = p.ext or "txt"
openfunc = getattr(vd, f"open_{filetype.lower()}")
if not openfunc:
vd.warning(f"no loader found for {filetype} files, falling back to txt")
filetype = "txt"
openfunc = vd.open_txt
assert callable(openfunc), f"no function/method available to open {p.given}"
vs = openfunc(p)
vd.status(
f'opening {p.given} as {filetype} (version id: {p.version_id or "latest"})'
)
return vs
def sort(self):
'Sort rows according to the current self._ordering.'
try:
with Progress(gerund='sorting', total=self.nRows) as prog:
def sortkey(r):
ret = []
for col, reverse in self._ordering:
if isinstance(col, str):
col = self.column(col)
val = col.getTypedValue(r)
ret.append(Reversor(val) if reverse else val)
prog.addProgress(1)
return ret
# must not reassign self.rows: use .sort() instead of sorted()
self.rows.sort(key=sortkey)
except TypeError as e:
vd.warning('sort incomplete due to TypeError; change column type')
vd.exceptionCaught(e, status=False)
def exec_command(self, cmd, args='', vdglobals=None, keystrokes=None):
"Execute `cmd` tuple with `vdglobals` as globals and this sheet's attributes as locals. Returns True if user cancelled."
if not cmd:
vd.debug('no command "%s"' % keystrokes)
return True
escaped = False
err = ''
if vdglobals is None:
vdglobals = getGlobals()
self.sheet = self
try:
if vd.cmdlog:
vd.cmdlog.beforeExecHook(self, cmd, '', keystrokes)
code = compile(cmd.execstr, cmd.longname, 'exec')
vd.debug(cmd.longname)
exec(code, vdglobals, LazyChainMap(vd, self))
except EscapeException as e: # user aborted
vd.warning(str(e))
escaped = True
except Exception as e:
vd.debug(cmd.execstr)
err = vd.exceptionCaught(e)
escaped = True
try:
if vd.cmdlog:
# sheet may have changed
vd.cmdlog.afterExecSheet(vd.sheets[0] if vd.sheets else None,
escaped, err)
except Exception as e:
vd.exceptionCaught(e)
self.checkCursorNoExceptions()
vd.clearCaches()
return escaped
def workspace_id(self):
return self.hexify(f"ws-{'^' * 9}")
def eni_id(self):
return self.hexify(f"eni-{'^' * 17}")
try:
import plugins.vfake # noqa F401
vd.options.vfake_extra_providers = [
AmazonWebServicesProvider, VdCustomProvider
]
except Exception as err:
vd.warning(f"Error importing vfake dependency for vfake_extensions: {err}")
# Helper condition checker functions for autofake
def match(pat):
r = re.compile(pat)
def wrapper(val, _):
return re.match(r, val)
return wrapper
def is_public_ip(addr, _):
try:
@asyncthread
def autofake(cols, rows):
'''
Try to guess an appropriate vfake faketype for a given column and row set.
If we find a match, run with it. NO REGERTS.
'''
isNull = isNullFunc()
for col in cols:
faketype = None
with suppress(StopIteration):
next(r for r in rows if not isNull(hint := col.getValue(r)))
faketype = next(v for k, v in faketype_mapping.items()
if k(str(hint), col.name))
if not faketype:
vd.warning(f'Could not detect a fake type for column {col.name}')
continue
vd.status(f'Detected fake type {faketype} for column {col.name}')
vd.addUndoSetValues([col], rows)
col.setValuesFromFaker(faketype, rows)
BaseSheet.bindkey("zf", "setcol-fake")
BaseSheet.addCommand(
"gzf",
"setcol-fake-all",
'cursorCol.setValuesFromFaker(input("faketype: ", type="faketype"), rows)',
)
BaseSheet.addCommand('z^F', 'setcol-autofake',
f'{__name__}.autofake([cursorCol], rows)')
BaseSheet.addCommand('gz^F', 'setcols-autofake',
def mainloop(self, scr):
'Manage execution of keystrokes and subsequent redrawing of screen.'
scr.timeout(curses_timeout)
with contextlib.suppress(curses.error):
curses.curs_set(0)
numTimeouts = 0
prefixWaiting = False
vd.scrFull = scr
self.keystrokes = ''
while True:
if not self.sheets:
# if no more sheets, exit
return
sheet = self.sheets[0]
threading.current_thread().sheet = sheet
vd.drawThread = threading.current_thread()
sheet.ensureLoaded()
vd.setWindows(scr)
self.draw(vd.win1, self.sheets[0])
if vd.win2 and len(self.sheets) > 1:
self.draw(vd.win2, self.sheets[1])
else:
vd.win2.erase()
vd.win2.refresh()
keystroke = self.getkeystroke(scr, sheet)
if not keystroke and prefixWaiting and ESC in self.keystrokes: # timeout ESC
self.keystrokes = ''
if keystroke: # wait until next keystroke to clear statuses and previous keystrokes
numTimeouts = 0
if not prefixWaiting:
self.keystrokes = ''
self.statuses.clear()
if keystroke == 'KEY_MOUSE':
self.keystrokes = ''
clicktype = ''
try:
devid, x, y, z, bstate = curses.getmouse()
sheet.mouseX, sheet.mouseY = x, y
if bstate & curses.BUTTON_CTRL:
clicktype += "CTRL-"
bstate &= ~curses.BUTTON_CTRL
if bstate & curses.BUTTON_ALT:
clicktype += "ALT-"
bstate &= ~curses.BUTTON_ALT
if bstate & curses.BUTTON_SHIFT:
clicktype += "SHIFT-"
bstate &= ~curses.BUTTON_SHIFT
keystroke = clicktype + curses.mouseEvents.get(
bstate, str(bstate))
f = self.getMouse(scr, x, y, keystroke)
if f:
if isinstance(f, str):
for cmd in f.split():
sheet.exec_keystrokes(cmd)
else:
f(y, x, keystroke)
self.keystrokes = keystroke
keystroke = ''
except curses.error:
pass
except Exception as e:
self.exceptionCaught(e)
if keystroke in self.keystrokes[:-1]:
vd.warning('duplicate prefix')
self.keystrokes = ''
else:
self.keystrokes += keystroke
self.drawRightStatus(sheet._scr,
sheet) # visible for commands that wait for input
if not keystroke: # timeout instead of keypress
pass
elif keystroke == '^Q':
return self.lastErrors and '\n'.join(self.lastErrors[-1])
elif vd.bindkeys._get(self.keystrokes):
sheet.exec_keystrokes(self.keystrokes)
prefixWaiting = False
elif keystroke in self.allPrefixes:
prefixWaiting = True
else:
vd.status('no command for "%s"' % (self.keystrokes))
prefixWaiting = False
self.checkForFinishedThreads()
sheet.checkCursorNoExceptions()
# no idle redraw unless background threads are running
time.sleep(0) # yield to other threads which may not have started yet
if vd.unfinishedThreads:
scr.timeout(curses_timeout)
else:
numTimeouts += 1
if numTimeouts > timeouts_before_idle:
scr.timeout(-1)
else:
scr.timeout(curses_timeout)
def main_vd():
'Open the given sources using the VisiData interface.'
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('inputs', nargs='*', help='initial sources')
parser.add_argument('-f', dest='filetype', default='', help='uses loader for filetype instead of file extension')
parser.add_argument('-y', dest='confirm_overwrite', default=None, action='store_false', help='overwrites existing files without confirmation')
parser.add_argument('-p', '--play', dest='play', default=None, help='replays a saved .vd file within the interface')
parser.add_argument('-P', dest='preplay', action='append', default=[], help='VisiData command to preplay before cmdlog replay')
parser.add_argument('-b', '--batch', dest='batch', action='store_true', default=False, help='replays in batch mode (with no interface and all status sent to stdout)')
parser.add_argument('-o', '--output', dest='output', default=None, help='saves the final visible sheet to output at the end of replay')
parser.add_argument('-w', dest='replay_wait', default=0, help='time to wait between replayed commands, in seconds')
parser.add_argument('-d', dest='delimiter', help='delimiter to use for tsv/usv filetype')
parser.add_argument('--diff', dest='diff', default=None, help='show diffs from all sheets against this source')
parser.add_argument('-v', '--version', action='version', version=__version_info__)
args = vd.parseArgs(parser)
# fetch motd and plugins *after* options parsing/setting
visidata.PluginsSheet().reload()
domotd()
locale.setlocale(locale.LC_ALL, '')
flPipedInput = not sys.stdin.isatty()
flPipedOutput = not sys.stdout.isatty()
vd._stdin, vd._stdout = duptty() # always dup stdin/stdout
stdinSource = Path('-', fp=vd._stdin)
# parse args, including +sheetname:subsheet:4:3 starting at row:col on sheetname:subsheet[:...]
start_positions = [] # (list_of_sheetstr, str, str) # empty sheetstr means all sheets
startsheets, startrow, startcol = [], None, None
fmtargs = []
fmtkwargs = {}
inputs = []
for arg in args.inputs:
if arg.startswith('+'): # position cursor at start
if ':' in arg:
pos = arg[1:].split(':')
if len(pos) == 1:
startsheet = [Path(inputs[-1]).name] if inputs else None
start_positions.append((startsheet, pos[0], None))
elif len(pos) == 2:
startsheet = [Path(inputs[-1]).name] if inputs else None
startrow, startcol = pos
start_positions.append((None, startrow, startcol))
elif len(pos) >= 3:
startsheets = pos[:-2]
startrow, startcol = pos[-2:]
start_positions.append((startsheets, startrow, startcol))
else:
start_positions.append((None, arg[1:], None))
elif args.play and '=' in arg:
# parse 'key=value' pairs for formatting cmdlog template in replay mode
k, v = arg.split('=')
fmtkwargs[k] = v
elif arg == '-':
inputs.append(stdinSource)
else:
inputs.append(arg)
fmtargs.append(arg)
if args.diff:
vs = openSource(args.diff)
vd.push(vs)
vs.reload()
setDiffSheet(vs)
if args.batch:
options.undo = False
vd.status = lambda *args, **kwargs: print(*args, file=sys.stderr) # ignore kwargs (like priority)
vd.editline = lambda *args, **kwargs: ''
vd.execAsync = lambda func, *args, **kwargs: func(*args, **kwargs) # disable async
for cmd in args.preplay:
Sheet('').exec_keystrokes(cmd)
if not args.play:
if flPipedInput and not inputs: # '|vd' without explicit '-'
inputs.append(stdinSource)
sources = []
for src in inputs:
vs = openSource(src)
vd.cmdlog.openHook(vs, src)
sources.append(vs)
vd.sheets.extend(sources) # purposefully do not load everything
if not vd.sheets and not args.play and not args.batch:
vd.push(vd.vdmenu)
if not args.play:
if args.batch:
vd.push(sources[0])
sources[0].reload()
for startsheets, startrow, startcol in start_positions:
sheets = [] # sheets to apply startrow:startcol to
if not startsheets:
sheets = sources # apply row/col to all sheets
else:
vs = vd.getSheet(startsheets[0]) or sources[-1]
vd.sync(vs.ensureLoaded())
vd.clearCaches()
for startsheet in startsheets[1:]:
rowidx = vs.getRowIndexFromStr(startsheet)
if rowidx is None:
vs = None
vd.warning(f'no sheet "{startsheet}"')
break
vs = vs.rows[rowidx]
vd.sync(vs.ensureLoaded())
vd.clearCaches()
if vs:
vd.push(vs)
sheets = [vs]
if startrow:
for vs in sheets:
if vs:
vs.moveToRow(startrow) or vd.warning(f'{vs} has no row "{startrow}"')
if startcol:
for vs in sheets:
if vs:
vs.moveToCol(startcol) or vd.warning(f'{vs} has no column "{startcol}"')
if not args.batch:
run(vd.sheets[0])
else:
if args.play == '-':
vdfile = stdinSource
vdfile.name = 'stdin.vd'
else:
vdfile = Path(args.play)
vs = eval_vd(vdfile, *fmtargs, **fmtkwargs)
vd.sync(vs.reload())
if args.batch:
if vd.replay_sync(vs): # error
return 1
else:
vd.replay(vs)
run()
if vd.sheets and (flPipedOutput or args.output):
outpath = Path(args.output or '-')
saveSheets(outpath, vd.sheets[0], confirm_overwrite=False)
vd.sync()
vd._stdout.flush()
return 0
def bindkey(cls, keystrokes, longname):
oldlongname = vd.bindkeys._get(keystrokes, cls)
if oldlongname:
vd.warning('%s was already bound to %s' % (keystrokes, oldlongname))
vd.bindkeys.set(keystrokes, longname, cls)