def load_vd_sheet(inpath):
"""Load a file and return the VisiData
sheet object.
"""
vd.loadConfigAndPlugins(AttrDict({}))
sheet = vd.openSource(inpath)
sheet.reload()
vd.sync()
return sheet
def reload(self):
'Load rows and/or columns. Override in subclass.'
self.rows = []
with vd.Progress(gerund='loading', total=0):
for r in self.iterload():
self.addRow(r)
# if an ordering has been specified, sort the sheet
if self._ordering:
vd.sync(self.sort())
def test_s3_download(tmp_path, s3_setup, path_info):
"""Make sure that we can download files and nothing gets
lost along the way.
"""
sheet = load_vd_sheet(f"s3://{path_info.s3_bucket}")
sheet.download(sheet.rows, Path(tmp_path))
vd.sync()
assert {path_info.base_filename, path_info.gzip_filename} <= set(
f.name for f in Path(tmp_path).glob("**/*")
)
def join_rows(self, rows):
"""Open new sheets for the target rows and concatenate their contents."""
sheets = list(self.open_rows(rows))
for sheet in vd.Progress(sheets):
sheet.reload()
# Wait for all sheets to fully load before joining them.
# 'append' is the only join type that makes sense here,
# since we're joining freshly opened sheets with no key
# columns.
vd.sync()
vd.push(vd.createJoinedSheet(sheets, jointype="append"))
def save(self, vs, filetype):
'Copy rows to the system clipboard.'
# use NTF to generate filename and delete file on context exit
with tempfile.NamedTemporaryFile(suffix='.' + filetype) as temp:
vd.sync(saveSheets(Path(temp.name), vs))
p = subprocess.Popen(self.get_command('copy'),
stdin=open(temp.name,
'r',
encoding=options.encoding),
stdout=subprocess.DEVNULL,
close_fds=True)
p.communicate()
def iterrows(self):
if self.rows is UNLOADED:
try:
self.rows = []
for row in self.iterload():
self.addRow(row)
yield row
return
except ExpectedException:
vd.sync(self.reload())
for row in vd.Progress(self.rows):
yield row
def reload(self):
'Skip first options.skip rows; set columns from next options.header rows.'
itsource = self.iterload()
# skip the first options.skip rows
list(self.optlines(itsource, 'skip'))
# use the next options.header rows as columns
self.setCols(list(self.optlines(itsource, 'header')))
self.rows = []
# add the rest of the rows
for r in vd.Progress(itsource, gerund='loading', total=0):
self.addRow(r)
# if an ordering has been specified, sort the sheet
if self._ordering:
vd.sync(self.sort())
def main_vd():
'Open the given sources using the VisiData interface.'
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('inputs', nargs='*', help='initial sources')
parser.add_argument('-f', dest='filetype', default='', help='uses loader for filetype instead of file extension')
parser.add_argument('-y', dest='confirm_overwrite', default=None, action='store_false', help='overwrites existing files without confirmation')
parser.add_argument('-p', '--play', dest='play', default=None, help='replays a saved .vd file within the interface')
parser.add_argument('-P', dest='preplay', action='append', default=[], help='VisiData command to preplay before cmdlog replay')
parser.add_argument('-b', '--batch', dest='batch', action='store_true', default=False, help='replays in batch mode (with no interface and all status sent to stdout)')
parser.add_argument('-o', '--output', dest='output', default=None, help='saves the final visible sheet to output at the end of replay')
parser.add_argument('-w', dest='replay_wait', default=0, help='time to wait between replayed commands, in seconds')
parser.add_argument('-d', dest='delimiter', help='delimiter to use for tsv/usv filetype')
parser.add_argument('--diff', dest='diff', default=None, help='show diffs from all sheets against this source')
parser.add_argument('-v', '--version', action='version', version=__version_info__)
args = vd.parseArgs(parser)
# fetch motd and plugins *after* options parsing/setting
visidata.PluginsSheet().reload()
domotd()
locale.setlocale(locale.LC_ALL, '')
flPipedInput = not sys.stdin.isatty()
flPipedOutput = not sys.stdout.isatty()
vd._stdin, vd._stdout = duptty() # always dup stdin/stdout
stdinSource = Path('-', fp=vd._stdin)
# parse args, including +sheetname:subsheet:4:3 starting at row:col on sheetname:subsheet[:...]
start_positions = [] # (list_of_sheetstr, str, str) # empty sheetstr means all sheets
startsheets, startrow, startcol = [], None, None
fmtargs = []
fmtkwargs = {}
inputs = []
for arg in args.inputs:
if arg.startswith('+'): # position cursor at start
if ':' in arg:
pos = arg[1:].split(':')
if len(pos) == 1:
startsheet = [Path(inputs[-1]).name] if inputs else None
start_positions.append((startsheet, pos[0], None))
elif len(pos) == 2:
startsheet = [Path(inputs[-1]).name] if inputs else None
startrow, startcol = pos
start_positions.append((None, startrow, startcol))
elif len(pos) >= 3:
startsheets = pos[:-2]
startrow, startcol = pos[-2:]
start_positions.append((startsheets, startrow, startcol))
else:
start_positions.append((None, arg[1:], None))
elif args.play and '=' in arg:
# parse 'key=value' pairs for formatting cmdlog template in replay mode
k, v = arg.split('=')
fmtkwargs[k] = v
elif arg == '-':
inputs.append(stdinSource)
else:
inputs.append(arg)
fmtargs.append(arg)
if args.diff:
vs = openSource(args.diff)
vd.push(vs)
vs.reload()
setDiffSheet(vs)
if args.batch:
options.undo = False
vd.status = lambda *args, **kwargs: print(*args, file=sys.stderr) # ignore kwargs (like priority)
vd.editline = lambda *args, **kwargs: ''
vd.execAsync = lambda func, *args, **kwargs: func(*args, **kwargs) # disable async
for cmd in args.preplay:
Sheet('').exec_keystrokes(cmd)
if not args.play:
if flPipedInput and not inputs: # '|vd' without explicit '-'
inputs.append(stdinSource)
sources = []
for src in inputs:
vs = openSource(src)
vd.cmdlog.openHook(vs, src)
sources.append(vs)
vd.sheets.extend(sources) # purposefully do not load everything
if not vd.sheets and not args.play and not args.batch:
vd.push(vd.vdmenu)
if not args.play:
if args.batch:
vd.push(sources[0])
sources[0].reload()
for startsheets, startrow, startcol in start_positions:
sheets = [] # sheets to apply startrow:startcol to
if not startsheets:
sheets = sources # apply row/col to all sheets
else:
vs = vd.getSheet(startsheets[0]) or sources[-1]
vd.sync(vs.ensureLoaded())
vd.clearCaches()
for startsheet in startsheets[1:]:
rowidx = vs.getRowIndexFromStr(startsheet)
if rowidx is None:
vs = None
vd.warning(f'no sheet "{startsheet}"')
break
vs = vs.rows[rowidx]
vd.sync(vs.ensureLoaded())
vd.clearCaches()
if vs:
vd.push(vs)
sheets = [vs]
if startrow:
for vs in sheets:
if vs:
vs.moveToRow(startrow) or vd.warning(f'{vs} has no row "{startrow}"')
if startcol:
for vs in sheets:
if vs:
vs.moveToCol(startcol) or vd.warning(f'{vs} has no column "{startcol}"')
if not args.batch:
run(vd.sheets[0])
else:
if args.play == '-':
vdfile = stdinSource
vdfile.name = 'stdin.vd'
else:
vdfile = Path(args.play)
vs = eval_vd(vdfile, *fmtargs, **fmtkwargs)
vd.sync(vs.reload())
if args.batch:
if vd.replay_sync(vs): # error
return 1
else:
vd.replay(vs)
run()
if vd.sheets and (flPipedOutput or args.output):
outpath = Path(args.output or '-')
saveSheets(outpath, vd.sheets[0], confirm_overwrite=False)
vd.sync()
vd._stdout.flush()
return 0