def reload(self): ''' Refresh the current S3 directory (prefix) listing. Force a refresh from the S3 filesystem to avoid using cached responses and missing recent changes. ''' import re self.columns = [] self.use_glob_matching = self.options.vds3_glob and re.search( r'[*?\[\]]', self.source.given) if not (self.use_glob_matching or self.source.fs.exists(self.source.given)): error(f'unable to open S3 path: {self.source.given}') for col in ( Column('name', getter=self.object_display_name), Column('type', getter=lambda _, row: row.get('type')), Column('size', type=int, getter=lambda _, row: row.get('Size')), Column('modtime', type=date, getter=lambda _, row: row.get('LastModified')), ): self.addColumn(col) super().reload()
def addShellColumns(cmd, sheet): shellcol = ColumnShell(cmd, source=sheet, width=0) for i, c in enumerate([ shellcol, Column(cmd+'_stdout', srccol=shellcol, getter=lambda col,row: col.srccol.getValue(row)[0]), Column(cmd+'_stderr', srccol=shellcol, getter=lambda col,row: col.srccol.getValue(row)[1]), ]): sheet.addColumn(c, index=sheet.cursorColIndex+i+1)
class DirSheet(Sheet): 'Sheet displaying directory, using ENTER to open a particular file. Edited fields are applied to the filesystem.' rowtype = 'files' # rowdef: Path columns = [ Column('directory', getter=lambda col,row: row.parent if str(row.parent) == '.' else str(row.parent) + '/'), Column('filename', getter=lambda col,row: row.name + row.suffix), Column('abspath', width=0, type=str), Column('ext', getter=lambda col,row: row.is_dir() and '/' or row.ext), Column('size', type=int, getter=lambda col,row: filesize(row)), Column('modtime', type=date, getter=lambda col,row: modtime(row)), Column('owner', width=0, getter=lambda col,row: pwd.getpwuid(row.stat().st_uid).pw_name), Column('group', width=0, getter=lambda col,row: grp.getgrgid(row.stat().st_gid).gr_name), Column('mode', width=0, getter=lambda col,row: '{:o}'.format(row.stat().st_mode)), Column('filetype', width=0, cache=True, getter=lambda col,row: subprocess.Popen(['file', '--brief', row], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()), ] nKeys = 2 _ordering = [('modtime', True)] # sort by reverse modtime initially def iterload(self): def _walkfiles(p): basepath = str(p) for folder, subdirs, files in os.walk(basepath): subfolder = folder[len(basepath)+1:] if subfolder in ['.', '..']: continue fpath = Path(folder) yield fpath for fn in files: yield fpath/fn def _listfiles(p): basepath = str(p) for fn in os.listdir(basepath): yield p/fn basepath = str(self.source) folders = set() f = _walkfiles if options.dir_recurse else _listfiles hidden_files = options.dir_hidden for p in f(self.source): if hidden_files and p.name.startswith('.'): continue yield p
class OptionsSheet(Sheet): _rowtype = Option # rowdef: Option rowtype = 'options' precious = False columns = ( ColumnAttr('option', 'name'), Column( 'value', getter=lambda col, row: col.sheet.diffOption(row.name), setter=lambda col, row, val: options.set(row.name, val, col.sheet. source), ), Column('default', getter=lambda col, row: options.get(row.name, 'global')), Column( 'description', width=40, getter=lambda col, row: options._get(row.name, 'global').helpstr), ColumnAttr('replayable'), ) colorizers = [ CellColorizer( 3, None, lambda s, c, r, v: v.value if r and c in s.columns[1:3] and r.name.startswith('color_') else None), ] nKeys = 1 def diffOption(self, optname): val = options.get(optname, self.source) default = options.get(optname, 'global') return val if val != default else '' def editOption(self, row): currentValue = options.get(row.name, self.source) vd.addUndo(options.set, row.name, currentValue, self.source) if isinstance(row.value, bool): options.set(row.name, not currentValue, self.source) else: options.set(row.name, self.editCell(1, value=currentValue), self.source) def reload(self): self.rows = [] for k in options.keys(): opt = options._get(k) self.addRow(opt) self.columns[ 1].name = 'global_value' if self.source == 'override' else 'sheet_value'
def addRegexColumns(regexMaker, vs, colIndex, origcol, regexstr): regexstr or vd.fail('regex required') regex = re.compile(regexstr, vs.regex_flags()) func = regexMaker(regex, origcol) n = options.default_sample_size if n and n < len(vs.rows): exampleRows = random.sample(vs.rows, max( 0, n - 1)) # -1 to account for included cursorRow else: exampleRows = vs.rows ncols = 0 # number of new columns added already for r in Progress(exampleRows + [vs.cursorRow]): try: m = func(r) if not m: continue except Exception as e: vd.exceptionCaught(e) for _ in range(len(m) - ncols): c = Column( origcol.name + '_re' + str(ncols), getter=lambda col, row, i=ncols, func=func: func(row)[i], origCol=origcol) vs.addColumn(c, index=colIndex + ncols + 1) ncols += 1
def reload(self): if isinstance(self.source, Frame): frame = self.source else: # vd.fail(f'no support for loading {self.source.__class__}') raise NotImplementedError( f'no support for loading a Frame from {self.source}') # If the index is not an IndexAutoFactory, try to move it onto the Frame. If this fails it might mean we are trying to unset an auto index post selection if frame.index.depth > 1 or frame.index._map: # if it is not an IndexAutoFactory frame = frame.unset_index() # VisiData assumes string column names if frame.columns.dtype != str: frame = frame.relabel(columns=frame.columns.astype(str)) dtypes = frame.dtypes self.columns = [] for col in (c for c in frame.columns if not c.startswith('__vd_')): self.addColumn( Column( col, type=self.dtype_to_type(dtypes[col]), getter=self.getValue, setter=self.setValue, expr=col, )) self.rows = StaticFrameAdapter(frame) self._selectedMask = Series.from_element(False, index=frame.index)
class ColorSheet(Sheet): rowtype = 'colors' # rowdef: color number as assigned in the colors object columns = [ Column('color', type=int), Column('R', getter=lambda col, row: curses.color_content( curses.pair_number(colors[row]) - 1)[0]), Column('G', getter=lambda col, row: curses.color_content( curses.pair_number(colors[row]) - 1)[1]), Column('B', getter=lambda col, row: curses.color_content( curses.pair_number(colors[row]) - 1)[2]), ] colorizers = [Colorizer('row', 7, lambda s, c, r, v: r)] def reload(self): self.rows = sorted(colors.keys(), key=lambda n: wrapply(int, n))
def draw(self, scr): 'Draw entire screen onto the `scr` curses object.' vd.clearCaches() if not self.columns: if options.debug: self.addColumn(Column()) else: return drawparams = { 'isNull': isNullFunc(), 'topsep': options.disp_rowtop_sep, 'midsep': options.disp_rowmid_sep, 'botsep': options.disp_rowbot_sep, 'endsep': options.disp_rowend_sep, 'keytopsep': options.disp_keytop_sep, 'keymidsep': options.disp_keymid_sep, 'keybotsep': options.disp_keybot_sep, 'endtopsep': options.disp_endtop_sep, 'endmidsep': options.disp_endmid_sep, 'endbotsep': options.disp_endbot_sep, 'colsep': options.disp_column_sep, 'keysep': options.disp_keycol_sep, 'selectednote': options.disp_selected_note, } self._rowLayout = {} # [rowidx] -> (y, height) self.calcColLayout() numHeaderRows = self.nHeaderRows vcolidx = 0 headerRow = 0 for vcolidx, colinfo in sorted(self._visibleColLayout.items()): self.drawColHeader(scr, headerRow, numHeaderRows, vcolidx) y = headerRow + numHeaderRows rows = self.rows[self.topRowIndex:min(self.topRowIndex+self.nScreenRows, self.nRows)] self.checkCursorNoExceptions() for rowidx, row in enumerate(rows): if y >= self.windowHeight-1: break rowcattr = self._colorize(None, row) y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y, **drawparams) if vcolidx+1 < self.nVisibleCols: scr.addstr(headerRow, self.windowWidth-2, options.disp_more_right, colors.color_column_sep) scr.refresh()
def reload(self): """Reload the current S3 directory (prefix) listing.""" self.columns = [] if not ( self.use_glob_matching or self.fs.exists(self.source.given) or self.fs.isdir(self.source.given) ): vd.fail(f"unable to open S3 path: {self.source.given}") for col in ( Column("name", getter=self.object_display_name), Column("type", getter=lambda _, row: row.get("type")), Column("size", type=int, getter=lambda _, row: row.get("size")), Column("modtime", type=date, getter=lambda _, row: row.get("LastModified")), ): self.addColumn(col) if self.version_aware: self.addColumn( Column("latest", type=bool, getter=lambda _, row: row.get("IsLatest")) ) self.addColumn( Column( "version_id", type=str, getter=lambda _, row: row.get("VersionId"), width=0, ) ) super().reload()
class ProfileSheet(Sheet): columns = [ Column('funcname', getter=lambda col, row: codestr(row.code)), Column('filename', getter=lambda col, row: os.path.split(row.code.co_filename)[-1] if not isinstance(row.code, str) else ''), Column('linenum', type=int, getter=lambda col, row: row.code.co_firstlineno if not isinstance(row.code, str) else None), Column('inlinetime_us', type=int, getter=lambda col, row: row.inlinetime * 1000000), Column('totaltime_us', type=int, getter=lambda col, row: row.totaltime * 1000000), ColumnAttr('callcount', type=int), ColumnAttr('reccallcount', type=int), ColumnAttr('calls'), Column('callers', getter=lambda col, row: col.sheet.callers[row.code]), ] nKeys = 3 def reload(self): self.rows = self.source self.orderBy(self.column('inlinetime_us'), reverse=True) self.callers = collections.defaultdict( list) # [row.code] -> list(code) for r in self.rows: calls = getattr(r, 'calls', None) if calls: for callee in calls: self.callers[callee.code].append(r)
class StaticFrameIndexSheet(IndexSheet): rowtype = 'sheets' columns = [ Column('sheet', getter=lambda col, row: row.source.name), ColumnAttr('name', width=0), ColumnAttr('nRows', type=int), ColumnAttr('nCols', type=int), ] def iterload(self): for sheetname in self.source.keys(): # this will combine self.name, sheetname into one name yield StaticFrameSheet(self.name, sheetname, source=self.source[sheetname])
class ColumnsSheet(Sheet): rowtype = 'columns' _rowtype = Column _coltype = ColumnAttr precious = False class ValueColumn(Column): 'passthrough to the value on the source cursorRow' def calcValue(self, srcCol): return srcCol.getDisplayValue(srcCol.sheet.cursorRow) def setValue(self, srcCol, val): srcCol.setValue(srcCol.sheet.cursorRow, val) columns = [ ColumnAttr('sheet', type=str), ColumnAttr('name', width=options.default_width), ColumnAttr('width', type=int), ColumnAttr('height', type=int, width=0), ColumnEnum('type', getGlobals(), default=anytype), ColumnAttr('fmtstr'), ValueColumn('value', width=options.default_width), Column('expr', getter=lambda col, row: getattr(row, 'expr', ''), setter=lambda col, row, val: setattr(row, 'expr', val)), ColumnAttr('ncalcs', type=int, width=0, cache=False), ColumnAttr('maxtime', type=float, width=0, cache=False), ColumnAttr('totaltime', type=float, width=0, cache=False), ] nKeys = 2 colorizers = [ RowColorizer(7, 'color_key_col', lambda s, c, r, v: r and r.keycol), RowColorizer(8, 'color_hidden_col', lambda s, c, r, v: r and r.hidden), ] def reload(self): if len(self.source) == 1: self.rows = self.source[0].columns self.cursorRowIndex = self.source[0].cursorColIndex self.columns[0].hide() # hide 'sheet' column if only one sheet else: self.rows = [ col for vs in self.source for col in vs.visibleCols if vs is not self ] def newRow(self): c = type(self.source[0])._coltype() c.recalc(self.source[0]) return c
class HelpSheet(Sheet): 'Show all commands available to the source sheet.' rowtype = 'commands' precious = False columns = [ ColumnAttr('sheet'), ColumnAttr('longname'), Column('keystrokes', getter=lambda col, row: col.sheet.revbinds.get(row.longname)), Column('description', getter=lambda col, row: col.sheet.cmddict[ (row.sheet, row.longname)].helpstr), ColumnAttr('execstr', width=0), ColumnAttr('logged', 'replayable', width=0), ] nKeys = 2 @asyncthread def reload(self): from pkg_resources import resource_filename cmdlist = TsvSheet('cmdlist', source=Path( resource_filename(__name__, 'commands.tsv'))) cmdlist.reload_sync() self.cmddict = {} for cmdrow in cmdlist.rows: self.cmddict[(cmdrow.sheet, cmdrow.longname)] = cmdrow self.revbinds = { longname: keystrokes for (keystrokes, _), longname in bindkeys.iter(self.source) if keystrokes not in self.revbinds } self.rows = [] for (k, o), v in commands.iter(self.source): self.addRow(v) v.sheet = o
class ThreadsSheet(Sheet): rowtype = 'threads' precious = False columns = [ ColumnAttr('name'), Column('process_time', type=float, getter=lambda col, row: elapsed_s(row)), ColumnAttr('profile'), ColumnAttr('status'), ColumnAttr('exception'), ] def reload(self): self.rows = vd.threads
class StatusSheet(Sheet): precious = False rowtype = 'statuses' # rowdef: (priority, args, nrepeats) columns = [ ColumnItem('priority', 0, type=int, width=0), ColumnItem('nrepeats', 2, type=int, width=0), ColumnItem('args', 1, width=0), Column('message', getter=lambda col, row: composeStatus(row[1], row[2])), ] colorizers = [ RowColorizer(1, 'color_error', lambda s, c, r, v: r and r[0] == 3), RowColorizer(1, 'color_warning', lambda s, c, r, v: r and r[0] in [1, 2]), ] def reload(self): self.rows = self.source
class ColumnsSheet(Sheet): rowtype = 'columns' precious = False class ValueColumn(Column): 'passthrough to the value on the source cursorRow' def calcValue(self, srcCol): return srcCol.getDisplayValue(srcCol.sheet.cursorRow) def setValue(self, srcCol, val): srcCol.setValue(self.sheet.source.cursorRow, val) columns = [ ColumnAttr('sheet', type=str), ColumnAttr('name', width=options.default_width), ColumnAttr('width', type=int), ColumnEnum('type', getGlobals(), default=anytype), ColumnAttr('fmtstr'), ValueColumn('value', width=options.default_width), Column('expr', getter=lambda col, row: getattr(row, 'expr', ''), setter=lambda col, row, val: setattr(row, 'expr', val)), ] nKeys = 2 colorizers = Sheet.colorizers + [ Colorizer( 'row', 7, lambda self, c, r, v: options.color_key_col if r.keycol else None), Colorizer( 'row', 8, lambda self, c, r, v: options.color_hidden_col if r.hidden else None), ] def reload(self): if len(self.source) == 1: self.rows = self.source[0].columns self.cursorRowIndex = self.source[0].cursorColIndex self.columns[0].hide() # hide 'sheet' column if only one sheet else: self.rows = [ col for vs in self.source for col in vs.visibleCols if vs is not self ]
class StatusSheet(Sheet): precious = False rowtype = 'statuses' # rowdef: (priority, args, nrepeats) columns = [ ColumnItem('priority', 0, type=int, width=0), ColumnItem('nrepeats', 2, type=int, width=0), ColumnItem('args', 1, width=0), Column('message', getter=lambda col, row: composeStatus(row[1], row[2])), ] colorizers = [ Colorizer( 'row', 1, lambda s, c, r, v: options.color_error if r[0] == 3 else None), Colorizer( 'row', 1, lambda s, c, r, v: options.color_warning if r[0] in [1, 2] else None), ] def reload(self): self.rows = vd.statusHistory[::-1]
def __init__(self, name='', **kwargs): super().__init__(name=name, **kwargs) self.rows = UNLOADED # list of opaque row objects (UNLOADED before first reload) self.cursorRowIndex = 0 # absolute index of cursor into self.rows self.cursorVisibleColIndex = 0 # index of cursor into self.visibleCols self._topRowIndex = 0 # cursorRowIndex of topmost row self.leftVisibleColIndex = 0 # cursorVisibleColIndex of leftmost column self.rightVisibleColIndex = 0 # as computed during draw() self._rowLayout = {} # [rowidx] -> (y, w) self._visibleColLayout = {} # [vcolidx] -> (x, w) # list of all columns in display order self.columns = kwargs.get('columns') or [copy(c) for c in self.columns] or [Column('')] self._colorizers = [] self.recalc() # set .sheet on columns and start caches self.setKeys(self.columns[:self.nKeys]) # initial list of key columns self.__dict__.update(kwargs) # also done earlier in BaseSheet.__init__
class DirSheet(Sheet): 'Sheet displaying directory, using ENTER to open a particular file. Edited fields are applied to the filesystem.' rowtype = 'files' # rowdef: Path columns = [ DeferredSetColumn( 'directory', getter=lambda col, row: row.parent.relpath(col.sheet.source. resolve()), setter=lambda col, row, val: col.sheet.moveFile(row, val)), DeferredSetColumn( 'filename', getter=lambda col, row: row.name + row.ext, setter=lambda col, row, val: col.sheet.renameFile(row, val)), DeferredSetColumn( 'pathname', width=0, getter=lambda col, row: row.resolve(), setter=lambda col, row, val: os.rename(row.resolve(), val)), Column('ext', getter=lambda col, row: row.is_dir() and '/' or row.suffix), DeferredSetColumn( 'size', type=int, getter=lambda col, row: row.stat().st_size, setter=lambda col, row, val: os.truncate(row.resolve(), int(val))), DeferredSetColumn( 'modtime', type=date, getter=lambda col, row: row.stat().st_mtime, setter=lambda col, row, val: os.utime( row.resolve(), times=((row.stat().st_atime, float(val))))), DeferredSetColumn( 'owner', width=0, getter=lambda col, row: pwd.getpwuid(row.stat().st_uid).pw_name, setter=lambda col, row, val: os.chown(row.resolve(), pwd.getpwnam(val).pw_uid, -1 )), DeferredSetColumn( 'group', width=0, getter=lambda col, row: grp.getgrgid(row.stat().st_gid).gr_name, setter=lambda col, row, val: os.chown(row.resolve(), -1, grp.getgrnam(val).pw_gid)), DeferredSetColumn( 'mode', width=0, getter=lambda col, row: '{:o}'.format(row.stat().st_mode), setter=lambda col, row, val: os.chmod(row.resolve(), int(val, 8))), Column('filetype', width=0, cache=True, getter=lambda col, row: subprocess.Popen( ['file', '--brief', row.resolve()], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()), ] colorizers = [ # CellColorizer(4, None, lambda s,c,r,v: s.colorOwner(s,c,r,v)), CellColorizer(8, 'color_change_pending', lambda s, c, r, v: s.changed(c, r)), RowColorizer(9, 'color_delete_pending', lambda s, c, r, v: r in s.toBeDeleted), ] nKeys = 2 @staticmethod def colorOwner(sheet, col, row, val): ret = '' if col.name == 'group': mode = row.stat().st_mode if mode & stat.S_IXGRP: ret = 'bold ' if mode & stat.S_IWGRP: return ret + 'green' if mode & stat.S_IRGRP: return ret + 'yellow' elif col.name == 'owner': mode = row.stat().st_mode if mode & stat.S_IXUSR: ret = 'bold ' if mode & stat.S_IWUSR: return ret + 'green' if mode & stat.S_IRUSR: return ret + 'yellow' def changed(self, col, row): try: return isinstance(col, DeferredSetColumn) and col.changed(row) except Exception: return False def deleteFiles(self, rows): for r in rows: if r not in self.toBeDeleted: self.toBeDeleted.append(r) def moveFile(self, row, val): fn = row.name + row.ext newpath = os.path.join(val, fn) if not newpath.startswith('/'): newpath = os.path.join(self.source.resolve(), newpath) parent = Path(newpath).parent if parent.exists(): if not parent.is_dir(): error('destination %s not a directory' % parent) else: with contextlib.suppress(FileExistsError): os.makedirs(parent.resolve()) os.rename(row.resolve(), newpath) row.fqpn = newpath self.restat(row) def renameFile(self, row, val): newpath = row.with_name(val) os.rename(row.resolve(), newpath.resolve()) row.fqpn = newpath self.restat(row) def removeFile(self, path): if path.is_dir(): os.rmdir(path.resolve()) else: os.remove(path.resolve()) def undoMod(self, row): for col in self.visibleCols: if getattr(col, '_modifiedValues', None) and id(row) in col._modifiedValues: del col._modifiedValues[id(row)] if row in self.toBeDeleted: self.toBeDeleted.remove(row) self.restat(row) def save(self, *rows): changes = [] deletes = {} for r in list( rows or self.rows): # copy list because elements may be removed if r in self.toBeDeleted: deletes[id(r)] = r else: for col in self.visibleCols: if self.changed(col, r): changes.append((col, r)) if not changes and not deletes: fail('nothing to save') cstr = '' if changes: cstr += 'change %d attributes' % len(changes) if deletes: if cstr: cstr += ' and ' cstr += 'delete %d files' % len(deletes) confirm('really %s? ' % cstr) self._commit(changes, deletes) @asyncthread def _commit(self, changes, deletes): oldrows = self.rows self.rows = [] for r in oldrows: try: if id(r) in deletes: self.removeFile(r) else: self.rows.append(r) except Exception as e: exceptionCaught(e) for col, row in changes: try: col.realsetter(col, row, col._modifiedValues[id(row)]) self.restat(r) except Exception as e: exceptionCaught(e) @asyncthread def reload(self): self.toBeDeleted = [] self.rows = [] basepath = self.source.resolve() for folder, subdirs, files in os.walk(basepath): subfolder = folder[len(basepath) + 1:] if subfolder.startswith('.'): continue for fn in files: if fn.startswith('.'): continue p = Path(os.path.join(folder, fn)) self.rows.append(p) # sort by modtime initially self.rows.sort(key=lambda row: row.stat().st_mtime, reverse=True) def restat(self, row): row.stat(force=True)
aggregator('list', list) aggregators['q3'] = quantiles(3) aggregators['q4'] = quantiles(4) aggregators['q5'] = quantiles(5) aggregators['q10'] = quantiles(10) # returns keys of the row with the max value aggregators['keymax'] = _defaggr( 'keymax', anytype, lambda col, rows: col.sheet.rowkey(max(col.getValueRows(rows))[1])) ColumnsSheet.columns += [ Column('aggregators', getter=lambda col, row: ' '.join( x.__name__ for x in getattr(row, 'aggregators', [])), setter=lambda col, row, val: setattr( row, 'aggregators', list(aggregators[k] for k in (val or '').split()))) ] def addAggregators(cols, aggrnames): 'add aggregator for each aggrname to each of cols' for aggrname in aggrnames: aggrs = aggregators.get(aggrname) aggrs = aggrs if isinstance(aggrs, list) else [aggrs] for aggr in aggrs: for c in cols: if not hasattr(c, 'aggregators'): c.aggregators = [] if aggr and aggr not in c.aggregators:
def combineColumns(cols): 'Return Column object formed by joining fields in given columns.' return Column("+".join(c.name for c in cols), getter=lambda col, row, cols=cols, ch=' ': ch.join( c.getDisplayValue(row) for c in cols))