def setValuesFromExpr(self, rows, expr):
compiledExpr = compile(expr, '<expr>', 'eval')
vd.addUndoSetValues([self], rows)
for row in Progress(rows, 'setting'):
self.setValueSafe(row, self.sheet.evalexpr(compiledExpr, row))
self.recalc()
status('set %d values = %s' % (len(rows), expr))
def copyCells(sheet, col, rows):
vd.clipcells = [col.getDisplayValue(r) for r in rows]
if not rows:
warning('no %s selected; clipboard emptied' % sheet.rowtype)
return
status('copied %d %s.%s to clipboard' %
(len(rows), sheet.rowtype, col.name))
def fillNullValues(col, rows):
'Fill null cells in col with the previous non-null value'
lastval = None
oldvals = [] # for undo
nullfunc = isNullFunc()
n = 0
rowsToFill = list(rows)
for r in Progress(col.sheet.rows, 'filling'): # loop over all rows
try:
val = col.getValue(r)
except Exception as e:
val = e
if nullfunc(val) and r in rowsToFill:
if lastval:
oldvals.append((col, r, val))
col.setValue(r, lastval)
n += 1
else:
lastval = val
def _undo():
for c, r, v in oldvals:
c.setValue(r, v)
vd.addUndo(_undo)
col.recalc()
status("filled %d values" % n)
def save_tsv(p, vs):
'Write sheet to file `fn` as TSV.'
delim = options.delimiter
trdict = tsv_trdict()
save_tsv_header(p, vs)
with p.open_text(mode='a') as fp:
for r in Progress(vs.rows):
dispvals = []
for col in vs.visibleCols:
v = col.getDisplayValue(r)
if isinstance(v, TypedWrapper):
if not options.save_errors:
continue
v = str(v)
if trdict:
v = str(v).translate(trdict)
dispvals.append(v)
fp.write(delim.join(dispvals))
fp.write('\n')
status('%s save finished' % p)
def deleteBy(self, func):
'Delete rows for which func(row) is true. Returns number of deleted rows.'
oldrows = copy(self.rows)
oldidx = self.cursorRowIndex
ndeleted = 0
row = None # row to re-place cursor after
while oldidx < len(oldrows):
if not func(oldrows[oldidx]):
row = self.rows[oldidx]
break
oldidx += 1
self.rows.clear()
for r in Progress(oldrows, 'deleting'):
if not func(r):
self.rows.append(r)
if r is row:
self.cursorRowIndex = len(self.rows) - 1
else:
ndeleted += 1
vd.addUndo(setattr, self, 'rows', oldrows)
status('deleted %s %s' % (ndeleted, self.rowtype))
return ndeleted
def saveToClipboard(sheet, rows, filetype=None):
'copy rows from sheet to system clipboard'
filetype = filetype or options.save_filetype
vs = copy(sheet)
vs.rows = rows
status('copying rows to clipboard')
clipboard().save(vs, filetype)
def reload(self):
self.rows = []
for sheet in self.sources:
for r in sheet.rows:
self.addRow((sheet, r))
self.columns = []
self.addColumn(ColumnItem('origin_sheet', 0))
allColumns = {}
for srcsheet in self.sources:
for srccol in srcsheet.visibleCols:
colsBySheet = allColumns.get(srccol.name, None)
if colsBySheet is None:
colsBySheet = {} # dict of [Sheet] -> Column
allColumns[srccol.name] = colsBySheet
if isinstance(srccol, ColumnExpr):
combinedCol = copy(srccol)
else:
combinedCol = ColumnConcat(srccol.name,
colsBySheet,
type=srccol.type)
self.addColumn(combinedCol)
if srcsheet in colsBySheet:
status('%s has multiple columns named "%s"' %
(srcsheet.name, srccol.name))
colsBySheet[srcsheet] = srccol
self.recalc(
) # to set .sheet on columns, needed if this reload becomes async
def syscopyRows(sheet, rows):
if not rows:
fail('no %s selected' % sheet.rowtype)
filetype = vd.input("copy %d %s to system clipboard as filetype: " %
(len(rows), sheet.rowtype),
value=options.save_filetype)
saveToClipboard(sheet, rows, filetype)
status('copied %d %s to system clipboard' % (len(rows), sheet.rowtype))
def domotd():
try:
if options.motd_url:
p = urlcache(options.motd_url, days=1)
line = random.choice(list(p))
status(line.split('\t')[0], priority=-1)
except Exception:
pass
def _save_table(sheet, fmt, p):
status(f"saving to {p.fqpn} as '{fmt}' table format")
tbl = to_tabulate_table(sheet, fmt)
with p.open_text(mode="w") as fp:
fp.write(tbl)
status(f"{p} save finished")
def toggleProfiling(t):
if not t.profile:
t.profile = cProfile.Profile()
t.profile.enable()
if not options.profile:
options.set('profile', 'vdprofile')
else:
t.profile.disable()
t.profile = None
options.set('profile', '')
status('profiling ' + ('ON' if t.profile else 'OFF'))
def edit_field(self, user_id):
"""Edits the currently selected value, encrypts it, and updates the database."""
user = self.current_user
row_id = self.cursorRow[0]
new_val = self.editCell(self.cursorVisibleColIndex)
encrypted_new_val = encrypt(new_val, self.current_key)
field = self.cursorCol.internal_field
edit_field_attr = getattr(Passwords, field)
visidata.status(row_id)
session.query(Passwords).filter_by(id=row_id).update({edit_field_attr: encrypted_new_val})
session.commit()
self.reload()
def reload(self):
self.colnames = {} # [colname] -> Column
self.columns.clear()
if not self.jsonlines:
try:
self.reload_json()
except ValueError as e:
status('trying jsonl')
self.jsonlines = True
if self.jsonlines:
self.reload_jsonl()
def save_tsv(p, vs):
'Write sheet to file `fn` as TSV.'
delim = options.get('delimiter', vs)
trdict = tsv_trdict(vs)
save_tsv_header(p, vs)
with p.open_text(mode='a') as fp:
for dispvals in genAllValues(vs.rows, vs.visibleCols, trdict, format=True):
fp.write(delim.join(dispvals))
fp.write('\n')
status('%s save finished' % p)
def vd_cli():
status(__version_info__)
rc = -1
try:
rc = main_vd()
except visidata.ExpectedException as e:
print('fail: ' + str(e))
except FileNotFoundError as e:
print(e)
if options.debug:
raise
sys.stderr.flush()
sys.stdout.flush()
os._exit(rc) # cleanup can be expensive with large datasets
def editText(vd, y, x, w, record=True, display=True, **kwargs):
'Wrap editline; if record=True, get input from the cmdlog in batch mode, save input to the cmdlog if display=True.'
v = None
if record and vd.cmdlog:
v = vd.getLastArgs()
if v is None:
v = vd.editline(vd.sheets[0]._scr, y, x, w, display=display, **kwargs)
if display:
status('"%s"' % v)
if record and vd.cmdlog:
vd.setLastArgs(v)
return v
def searchRegex(vd, sheet, moveCursor=False, reverse=False, **kwargs):
'Set row index if moveCursor, otherwise return list of row indexes.'
def findMatchingColumn(sheet, row, columns, func):
'Find column for which func matches the displayed value in this row'
for c in columns:
if func(c.getDisplayValue(row)):
return c
vd.searchContext.update(kwargs)
regex = kwargs.get("regex")
if regex:
vd.searchContext["regex"] = re.compile(
regex, sheet.regex_flags()) or error('invalid regex: %s' % regex)
regex = vd.searchContext.get("regex") or fail("no regex")
columns = vd.searchContext.get("columns")
if columns == "cursorCol":
columns = [sheet.cursorCol]
elif columns == "visibleCols":
columns = tuple(sheet.visibleCols)
elif isinstance(columns, Column):
columns = [columns]
if not columns:
error('bad columns')
searchBackward = vd.searchContext.get("backward")
if reverse:
searchBackward = not searchBackward
matchingRowIndexes = 0
for rowidx in rotateRange(len(sheet.rows),
sheet.cursorRowIndex,
reverse=searchBackward):
c = findMatchingColumn(sheet, sheet.rows[rowidx], columns,
regex.search)
if c:
if moveCursor:
sheet.cursorRowIndex = rowidx
sheet.cursorVisibleColIndex = sheet.visibleCols.index(c)
return
else:
matchingRowIndexes += 1
yield rowidx
status('%s matches for /%s/' % (matchingRowIndexes, regex.pattern))
def rotateRange(n, idx, reverse=False):
'Wraps an iter starting from idx. Yields indices from idx to n and then 0 to idx.'
if reverse:
rng = range(idx - 1, -1, -1)
rng2 = range(n - 1, idx - 1, -1)
else:
rng = range(idx + 1, n)
rng2 = range(0, idx + 1)
wrapped = False
with Progress(total=n) as prog:
for r in itertools.chain(rng, rng2):
prog.addProgress(1)
if not wrapped and r in rng2:
status('search wrapped')
wrapped = True
yield r
def save_tsv(vd, p, vs):
'Write sheet to file `fn` as TSV.'
unitsep = vs.options.delimiter
rowsep = vs.options.row_delimiter
trdict = vs.safe_trdict()
with p.open_text(mode='w') as fp:
colhdr = unitsep.join(
col.name.translate(trdict)
for col in vs.visibleCols) + options.row_delimiter
fp.write(colhdr)
for dispvals in vs.iterdispvals(format=True):
fp.write(unitsep.join(dispvals.values()))
fp.write(rowsep)
status('%s save finished' % p)
def _toplevelTryFunc(func, *args, status=status, **kwargs):
with ThreadProfiler(threading.current_thread()) as prof:
t = threading.current_thread()
t.name = func.__name__
try:
t.status = func(*args, **kwargs)
except EscapeException as e: # user aborted
t.status = 'aborted by user'
if status:
status('%s aborted' % t.name, priority=2)
except Exception as e:
t.exception = e
t.status = 'exception'
vd.exceptionCaught(e)
if t.sheet:
t.sheet.currentThreads.remove(t)
def toggle_scroll_fix(sheet):
# Disable scroll fix
if options.scroll_fix_enabled:
options.scroll_fix_enabled = False
status('scroll fix disabled')
Sheet.addCommand(None, 'go-down', 'cursorDown(+1)', 'go down')
Sheet.addCommand(None, 'go-up', 'cursorDown(-1)', 'go up')
# Enable scrollfix
else:
options.scroll_fix_enabled = True
status('scroll fix enabled')
Sheet.addCommand(None, 'go-up',
'sheet.topRowIndex -= 1; sheet.cursorRowIndex -= 1',
'move cursor up with a fixed position')
Sheet.addCommand(
None, 'go-down',
'sheet.topRowIndex = sheet.nRows - sheet.nScreenRows if sheet.topRowIndex + sheet.nScreenRows >= sheet.nRows else sheet.topRowIndex + 1; sheet.cursorRowIndex += 1',
'move cursor down with a fixed position')
def openurl_s3(p, filetype):
'''
Open a sheet for an S3 path. S3 directories (prefixes) require special handling,
but files (objects) can use standard VisiData "open" functions.
'''
from s3fs import S3FileSystem
# Non-obvious behavior here: For the default case, we don't want to send
# a custom endpoint to s3fs. However, using None as a default trips up
# VisiData's type detection for the endpoint option. So we use an empty
# string as the default instead, and convert back to None here.
endpoint = options.vds3_endpoint or None
version_aware = options.vds3_version_aware
# We can reuse an existing S3FileSystem as long as no relevant options
# have changed since it was created.
if (not S3Path.fs or S3Path.fs.version_aware != version_aware
or S3Path.fs.client_kwargs.get('endpoint_url', '') != endpoint):
S3Path.fs = S3FileSystem(client_kwargs={'endpoint_url': endpoint},
version_aware=version_aware)
p = S3Path(p.given)
if not p.is_file():
return S3DirSheet(p.name, source=p)
if not filetype:
filetype = p.ext or 'txt'
openfunc = getGlobals().get('open_' + filetype.lower())
if not openfunc:
warning(f'no loader found for {filetype} files, falling back to txt')
filetype = 'txt'
openfunc = open_txt
vs = openfunc(p)
status(f'opening {p.given} as {filetype}')
return vs
def syscopyCells(sheet, col, rows):
if not rows:
fail('no %s selected' % sheet.rowtype)
clipboard().copy("\n".join(col.getDisplayValue(r) for r in rows))
status('copied %s from %d %s to system clipboard' %
(col.name, len(rows), sheet.rowtype))
def copyToClipboard(value):
'copy single value to system clipboard'
clipboard().copy(value)
status('copied value to clipboard')
def copyRows(sheet, rows):
vd.cliprows = list((sheet, i, r) for i, r in enumerate(rows))
if not rows:
warning('no %s selected; clipboard emptied' % sheet.rowtype)
else:
status('copied %d %s to clipboard' % (len(rows), sheet.rowtype))
