# Usage: $0 [<options>] [<input> ...]
# $0 [<options>] --play <cmdlog> [--batch] [-w <waitsecs>] [-o <output>] [field=value ...]
__version__ = '2.-3'
__version_info__ = 'saul.pw/VisiData v' + __version__
import os
import io
import sys
import locale
from visidata import vd, option, options, status, run, Sheet
from visidata import Path, openSource, saveSheets, setDiffSheet, domotd
import visidata
option('config', '~/.visidatarc', 'config file to exec in Python')
option('preplay', '', 'longnames to preplay before replay')
# for --play
def eval_vd(logpath, *args, **kwargs):
'Instantiate logpath with args/kwargs replaced and replay all commands.'
log = logpath.read_text()
if args or kwargs:
log = log.format(*args, **kwargs)
src = Path(logpath.given, fp=io.StringIO(log), filesize=len(log))
vs = openSource(src, filetype=src.ext)
vs.name += '_vd'
vd.push(vs)
vs.vd = vd
return vs
import os
import contextlib
import itertools
import collections
from visidata import asyncthread, options, Progress, status, ColumnItem, SequenceSheet, Sheet, FileExistsError, getType, option, VisiData
from visidata import namedlist, filesize
option('delimiter',
'\t',
'field delimiter to use for tsv/usv filetype',
replay=True)
option('row_delimiter',
'\n',
'row delimiter to use for tsv/usv filetype',
replay=True)
option('tsv_safe_newline',
'\u001e',
'replacement for newline character when saving to tsv',
replay=True)
option('tsv_safe_tab',
'\u001f',
'replacement for tab character when saving to tsv',
replay=True)
def splitter(fp, delim='\n'):
'Generates one line/row/record at a time from fp, separated by delim'
buf = ''
while True:
from visidata import option, options, anytype, stacktrace, vd
from visidata import getType, typemap, isNumeric, isNullFunc
from visidata import asyncthread, dispwidth
from visidata import wrapply, TypedWrapper, TypedExceptionWrapper
from visidata import Extensible, AttrDict, undoAttrFunc
class InProgress(Exception):
@property
def stacktrace(self):
return ['calculation in progress']
INPROGRESS = TypedExceptionWrapper(None, exception=InProgress()) # sentinel
option('col_cache_size', 0,
'max number of cache entries in each cached column')
option('force_valid_colnames',
False,
'clean column names to be valid Python identifiers',
replay=True)
__all__ = [
'clean_to_id',
'Column',
'setitem',
'getattrdeep',
'setattrdeep',
'getitemdef',
'ColumnAttr',
'ColumnItem',
'SubColumnFunc',
import os.path
import functools
import cProfile
import threading
import collections
from visidata import VisiData, vd, option, options, status, globalCommand, Sheet, EscapeException
from visidata import ColumnAttr, Column, ENTER
from visidata import *
__all__ = [
'Progress', 'asynccache', 'async_deepcopy', 'elapsed_s', 'cancelThread',
'ThreadsSheet', 'ProfileSheet', 'codestr', 'asyncsingle'
]
option('profile', '', 'filename to save binary profiling data')
option('min_memory_mb', 0,
'minimum memory to continue loading and async processing')
theme('color_working', 'green', 'color of system running smoothly')
BaseSheet.init('currentThreads', list)
def asynccache(key=lambda *args, **kwargs: str(args) + str(kwargs)):
def _decorator(func):
'Function decorator, so first call to `func()` spawns a separate thread. Calls return the Thread until the wrapped function returns; subsequent calls return the cached return value.'
d = {} # per decoration cache
def _func(k, *args, **kwargs):
d[k] = func(*args, **kwargs)
import os
import stat
import subprocess
import contextlib
try:
import pwd
import grp
except ImportError:
pass # pwd,grp modules not available on Windows
from visidata import Column, Sheet, LazyComputeRow, asynccache, options, option, globalCommand
from visidata import Path, ENTER, date, asyncthread, confirm, fail, FileExistsError, VisiData
from visidata import CellColorizer, RowColorizer, modtime, filesize
option('dir_recurse', False, 'walk source path recursively on DirSheet')
option('dir_hidden', False, 'load hidden files on DirSheet')
@VisiData.lazy_property
def currentDirSheet(p):
'Support opening the current DirSheet from the vdmenu'
return DirSheet('.', source=Path('.'))
@asyncthread
def exec_shell(*args):
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if err or out:
lines = err.decode('utf8').splitlines() + out.decode('utf8').splitlines()
vd.push(TextSheet(' '.join(args), source=lines))
'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)'
)
@Sheet.api
def setSubst(sheet, cols, rows):
if not rows:
warning('no %s selected' % sheet.rowtype)
return
modified = 'column' if len(cols) == 1 else 'columns'
rex = vd.input("transform %s by regex: " % modified, type="regex-subst")
setValuesFromRegex(cols, rows, rex)
option('regex_flags',
'I',
'flags to pass to re.compile() [AILMSUX]',
replay=True)
option('regex_maxsplit', 0, 'maxsplit to pass to regex.split', replay=True)
option('default_sample_size',
100,
'number of rows to sample for regex.split',
replay=True)
def makeRegexSplitter(regex, origcol):
return lambda row, regex=regex, origcol=origcol, maxsplit=options.regex_maxsplit: regex.split(
origcol.getDisplayValue(row), maxsplit=maxsplit)
def makeRegexMatcher(regex, origcol):
def _regexMatcher(row):
Sheet.addCommand('zd', 'delete-cell', 'vd.clipcells = [cursorDisplay]; cursorCol.setValues([cursorRow], None)')
Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(sheet.cursorCol.getDisplayValue(r) for r in selectedRows); cursorCol.setValues(selectedRows, None)')
Sheet.addCommand('gzy', 'copy-cells', 'vd.clipcells = [sheet.cursorCol.getDisplayValue(r) for r in selectedRows]; status("%d values to clipboard" % len(vd.clipcells))')
Sheet.addCommand('gzp', 'paste-cells', 'for r, v in zip(selectedRows or rows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)')
Sheet.addCommand('Y', 'syscopy-row', 'saveToClipboard(sheet, [cursorRow], input("copy current row to system clipboard as filetype: ", value=options.save_filetype))')
Sheet.addCommand('gY', 'syscopy-selected', 'saveToClipboard(sheet, selectedRows or rows, input("copy rows to system clipboard as filetype: ", value=options.save_filetype))')
Sheet.addCommand('zY', 'syscopy-cell', 'copyToClipboard(cursorDisplay)')
Sheet.addCommand('gzY', 'syscopy-cells', 'copyToClipboard("\\n".join(sheet.cursorCol.getDisplayValue(r) for r in selectedRows))')
Sheet.bindkey('KEY_DC', 'delete-cell'),
Sheet.bindkey('gKEY_DC', 'delete-cells'),
option('clipboard_copy_cmd', '', 'command to copy stdin to system clipboard')
__clipboard_commands = [
('win32', 'clip', ''), # Windows Vista+
('darwin', 'pbcopy', 'w'), # macOS
(None, 'xclip', '-selection clipboard -filter'), # Linux etc.
(None, 'xsel', '--clipboard --input'), # Linux etc.
]
def detect_command(cmdlist):
'''Detect available clipboard util and return cmdline to copy data to the system clipboard.
cmddict is list of (platform, progname, argstr).'''
for platform, command, args in cmdlist:
if platform is None or sys.platform == platform:
path = shutil.which(command)
import itertools
from copy import copy
from visidata import vd, options, VisiData, BaseSheet, option
BaseSheet.init('undone', list) # list of CommandLogRow for redo after undo
option('undo', True, 'enable undo/redo')
@VisiData.api
def addUndo(vd, undofunc, *args, **kwargs):
'On undo of latest command, call undofunc()'
if options.undo:
r = vd.activeCommand
if not r:
return
r.undofuncs.append((undofunc, args, kwargs))
@VisiData.api
def undo(vd, sheet):
if not options.undo:
vd.fail("options.undo not enabled")
# don't allow undo of first command on a sheet, which is always the command that created the sheet.
for cmdlogrow in sheet.cmdlog_sheet.rows[:0:-1]:
if cmdlogrow.undofuncs:
for undofunc, args, kwargs, in cmdlogrow.undofuncs:
undofunc(*args, **kwargs)
sheet.undone.append(cmdlogrow)
'gY', 'syscopy-selected', 'syscopyRows(selectedRows)',
'yank (copy) selected rows to system clipboard (using options.clipboard_copy_cmd)'
)
Sheet.addCommand(
'zY', 'syscopy-cell', 'syscopyCells(cursorCol, [cursorRow])',
'yank (copy) current cell to system clipboard (using options.clipboard_copy_cmd)'
)
Sheet.addCommand(
'gzY', 'syscopy-cells', 'syscopyCells(cursorCol, selectedRows)',
'yank (copy) contents of current column from selected rows to system clipboard (using options.clipboard_copy_cmd'
)
Sheet.bindkey('KEY_DC', 'delete-cell'),
Sheet.bindkey('gKEY_DC', 'delete-cells'),
option('clipboard_copy_cmd', '', 'command to copy stdin to system clipboard')
option('clipboard_paste_cmd', '',
'command to get contents of system clipboard')
# mapping of OS to list of possible (command name, command args) for copy and
# paste commands
__copy_commands = {
# TODO TEST WINDOWS AND MAC
'win32': [('clip', '')],
'darwin': [('pbcopy', 'w')],
# try these for all other platforms
None: [('xclip', '-selection clipboard -filter'),
('xsel', '--clipboard --input')]
}
__paste_commands = {
# TODO TEST WINDOWS AND MAC
import textwrap
from visidata import vd, option, options, Sheet, ColumnItem, asyncthread
from visidata import globalCommand, error, stacktrace, VisiData
__all__ = ['TextSheet', 'ErrorSheet']
option('wrap', False, 'wrap text to fit window width on TextSheet')
option('save_filetype', 'tsv', 'specify default file type to save as', replay=True)
## text viewer
# rowdef: (linenum, str)
class TextSheet(Sheet):
'Displays any iterable source, with linewrap if wrap set in init kwargs or options.'
rowtype = 'lines'
filetype = 'txt'
columns = [
ColumnItem('linenum', 0, type=int, width=0),
ColumnItem('text', 1),
]
def iterload(self):
winWidth = min(self.columns[1].width or 78, self.windowWidth-2)
wrap = options.wrap
for startingLine, text in enumerate(self.source):
if wrap and text:
for i, L in enumerate(textwrap.wrap(str(text), width=winWidth)):
yield [startingLine+i+1, L]
else:
import os.path
import functools
import cProfile
import threading
import collections
from visidata import vd, option, options, status, globalCommand, Sheet, EscapeException
from visidata import elapsed_s, ColumnAttr, Column, ThreadsSheet, ENTER
option('profile', '', 'filename to save binary profiling data')
globalCommand('^_', 'toggle-profile',
'toggleProfiling(threading.current_thread())')
ThreadsSheet.addCommand(
ENTER, 'profile-row',
'vd.push(ProfileSheet(cursorRow.name+"_profile", source=cursorRow.profile.getstats()))'
)
min_thread_time_s = 0.10 # only keep threads that take longer than this number of seconds
def open_pyprof(p):
return ProfileSheet(p.name, p.open_bytes())
def toggleProfiling(t):
if not t.profile:
t.profile = cProfile.Profile()
t.profile.enable()
if not options.profile:
## Options
- `tabulate_format` specifies the default text-table format
"""
__version__ = "0.0.0"
from tabulate import tabulate
from visidata import (Sheet, addGlobals, copyToClipboard, Path, asyncthread,
option, options, status, confirm, fail, Progress)
DEFAULT_FORMAT = "simple"
option("tabulate_format", DEFAULT_FORMAT,
"Default format for 'tabulate' commands")
SUPPORTED_FORMATS = [
"plain",
"simple",
"github",
"grid",
"fancy_grid",
"pipe",
"orgtbl",
"jira",
"presto",
"psql",
"rst",
"mediawiki",
"moinmoin",
import collections
import itertools
from copy import copy
import textwrap
from visidata import VisiData, Extensible, globalCommand, ColumnAttr, ColumnItem, vd, ENTER, EscapeException, drawcache, drawcache_property, LazyChainMap, asyncthread, ExpectedException
from visidata import (options, theme, isNullFunc, isNumeric, Column, option, namedlist,
TypedExceptionWrapper, getGlobals, BaseSheet, UNLOADED,
vd, getType, clipdraw, ColorAttr, update_attr, colors, undoAttrFunc)
__all__ = ['RowColorizer', 'CellColorizer', 'ColumnColorizer', 'Sheet', 'IndexSheet', 'SheetsSheet', 'LazyComputeRow', 'SequenceSheet']
option('default_width', 20, 'default column width', replay=True) # TODO: make not replay and remove from markdown saver
option('textwrap_cells', True, 'wordwrap text for multiline rows')
option('cmd_after_edit', 'go-down', 'command longname to execute after successful edit')
option('quitguard', False, 'confirm before quitting last sheet')
option('debug', False, 'exit on error and display stacktrace')
option('skip', 0, 'skip N rows before header', replay=True)
option('header', 1, 'parse first N rows as column names', replay=True)
theme('force_256_colors', False, 'use 256 colors even if curses reports fewer')
theme('use_default_colors', False, 'curses use default terminal colors')
theme('disp_note_none', '⌀', 'visible contents of a cell whose value is None')
theme('disp_truncator', '…', 'indicator that the contents are only partially visible')
theme('disp_oddspace', '\u00b7', 'displayable character for odd whitespace')
theme('disp_more_left', '<', 'header note indicating more columns to the left')
theme('disp_more_right', '>', 'header note indicating more columns to the right')
theme('disp_error_val', '', 'displayed contents for computation exception')
from visidata import option, exceptionCaught, TypedWrapper, asyncthread
from visidata import wrapply, clean_to_id, isNumeric
option('graphviz_edge_labels', True,
'whether to include edge labels on graphviz diagrams')
si_levels = ['', 'k', 'M', 'G', 'T', 'P', 'Q']
def SI(n):
orig_n = n
try:
level = 0
while n > 1000:
n /= 1000
level += 1
return '%0.1f%s' % (n, si_levels[level])
except Exception as e:
exceptionCaught(e)
return orig_n
def is_valid(v):
if v is None:
return False
if isinstance(v, TypedWrapper):
return False
return True
@asyncthread
from visidata import globalCommand, BaseSheet, Column, options, vd, anytype, ENTER, asyncthread, option, Sheet, IndexSheet
from visidata import CellColorizer, RowColorizer
from visidata import ColumnAttr, ColumnEnum, ColumnItem
from visidata import getGlobals, TsvSheet, Path, Option
from visidata import undoAttrFunc, VisiData, vlen
option('visibility', 0, 'visibility level (0=low, 1=high)')
vd_system_sep = '\t'
@BaseSheet.lazy_property
def optionsSheet(sheet):
return OptionsSheet(sheet.name + "_options", source=sheet)
@VisiData.lazy_property
def globalOptionsSheet(vd):
return OptionsSheet('global_options', source='override')
class ColumnsSheet(Sheet):
rowtype = 'columns'
_rowtype = Column
_coltype = ColumnAttr
precious = False
class ValueColumn(Column):
'passthrough to the value on the source cursorRow'
def calcValue(self, srcCol):
# VisiData uses Python native int, float, str, and adds simple date, currency, and anytype.
import collections
import functools
import datetime
import locale
from visidata import option, options, TypedWrapper
#__all__ = ['anytype', 'vdtype', 'typemap', 'getType', 'typemap']
option('disp_float_fmt',
'%.02f',
'default fmtstr to format for float values',
replay=True)
option('disp_date_fmt',
'%Y-%m-%d',
'default fmtstr to strftime for date values',
replay=True)
try:
import dateutil.parser
except ImportError:
pass
# VisiDataType .typetype are e.g. int, float, str, and used internally in these ways:
#
# o = typetype(val) # for interpreting raw value
# o = typetype(str) # for conversion from string (when setting)
# o = typetype() # for default value to be used when conversion fails
#
# The resulting object o must be orderable and convertible to a string for display and certain outputs (like csv).
import contextlib
import os
import curses
import threading
import time
from unittest import mock
from visidata import vd, VisiData, colors, ESC, options, option
curses_timeout = 100 # curses timeout in ms
timeouts_before_idle = 10
option('disp_splitwin_pct', 0, 'height of second sheet on screen')
@VisiData.api
def draw(self, scr, sheet):
'Redraw full screen.'
scr.erase() # clear screen before every re-draw
sheet._scr = scr
self.drawLeftStatus(scr, sheet)
self.drawRightStatus(scr, sheet) # visible during this getkeystroke
try:
sheet.draw(scr)
except Exception as e:
self.exceptionCaught(e)
__author__ = 'Geekscrapy'
__version__ = '1.0'
__description__ = '''
Allows the user to fix the position on the page where they would like the
cursor to "stick". Helps to provide context about surrounding rows when
near the top and bottom of the page.
Usage: import this .py into .visidatarc, open a sheet, scroll a few
lines down, press "w" and scroll again!
Idea birthed here: https://github.com/saulpw/visidata/issues/561
'''
from visidata import option, options, status, Sheet
option(name='scroll_fix_enabled', default=False, helpstr='toggle scroll fix')
@Sheet.api
def toggle_scroll_fix(sheet):
# Disable scroll fix
if options.scroll_fix_enabled:
options.scroll_fix_enabled = False
status('scroll fix disabled')
Sheet.addCommand(None, 'go-down', 'cursorDown(+1)', 'go down')
Sheet.addCommand(None, 'go-up', 'cursorDown(-1)', 'go up')
# Enable scrollfix
else:
options.scroll_fix_enabled = True
from visidata import vd, Sheet, Progress, option, asyncthread, options, rotateRange, Fanout, undoAttrCopyFunc, copy
option('bulk_select_clear',
False,
'clear selected rows before new bulk selections',
replay=True)
Sheet.init('_selectedRows', dict) # rowid(row) -> row
@Sheet.api
def isSelected(self, row):
'True if given row is selected. O(log n).'
return self.rowid(row) in self._selectedRows
@Sheet.api
@asyncthread
def toggle(self, rows):
'Toggle selection of given `rows`.'
self.addUndoSelection()
for r in Progress(rows, 'toggling', total=len(self.rows)):
if not self.unselectRow(r):
self.selectRow(r)
@Sheet.api
def selectRow(self, row):
'Select given row. O(log n)'
self._selectedRows[self.rowid(row)] = row
asyncthread,
date,
error,
getGlobals,
open_txt,
option,
options,
status,
warning,
)
__version__ = '0.4'
option(
'vds3_endpoint',
'',
'alternate S3 endpoint, used for local testing or alternative S3-compatible services',
replay=True,
)
option(
'vds3_version_aware',
False,
'use object versioning in S3 buckets?',
replay=True,
)
option('vds3_glob', True, 'enable glob-matching for S3 paths', replay=True)
class S3Path(Path):
'''
A Path-like object representing an S3 file (object) or directory (prefix).
'''
import json
from visidata import options, option, status, date, deduceType
from visidata import PythonSheet, ColumnItem, stacktrace, asyncthread, Progress
from visidata import wrapply, TypedExceptionWrapper, TypedWrapper
option('json_indent', None, 'indent to use when saving json')
def open_json(p):
return JSONSheet(p.name, source=p, jsonlines=False)
def open_jsonl(p):
return JSONSheet(p.name, source=p, jsonlines=True)
class JSONSheet(PythonSheet):
@asyncthread
def reload(self):
self.colnames = {} # [colname] -> Column
self.columns.clear()
if not self.jsonlines:
try:
self.reload_json()
except ValueError as e:
status('trying jsonl')
self.jsonlines = True
if self.jsonlines:
'''motd: display a low-priority random Message Of The Day on startup.
Call `domotd()` to spawn an asyncthread to read and/or fetch
a motd file from a url. The file may be text or unheaded TSV, with one message per row in the first column.
Any Exception ends the thread silently.
options.motd_url may be set to another URL, or empty to disable entirely.
'''
import random
from visidata import option, options, asyncsingle, urlcache, status
from visidata import __version__
option('motd_url', 'https://visidata.org/motd-' + __version__,
'source of randomized startup messages')
@asyncsingle
def domotd():
try:
if options.motd_url:
p = urlcache(options.motd_url, days=1)
line = random.choice(list(p))
status(line.split('\t')[0], priority=-1)
except Exception:
pass
