def walk_dir_animated(path, maxdircnt=1000):
""" Walk a directory, printing status updates along the way. """
p = AnimatedProgress(
'Walking {}...'.format(path),
frames=Frames.dots_orbit.as_rainbow(),
show_time=True,
)
rootcnt = 0
print('\nStarting animated progress.')
with p:
for root, dirs, files in os.walk(path):
rootcnt += 1
if rootcnt % 50 == 0:
p.text = 'Walking {}...'.format(C(root, 'cyan'))
if rootcnt > maxdircnt:
# Stop is called because we are printing before the
# AnimatedProgress is finished running.
p.stop()
print('\nFinished walking {} directories.'.format(
C(maxdircnt, 'blue', style='bright')))
break
else:
# AnimatedProgress still running, `stop` it before printing.
p.stop()
print_err('\nNever made it to {} directories ({}).'.format(
C(maxdircnt, 'blue', style='bright'),
C(rootcnt, 'red', style='bright'),
))
print('\nFinished with animated progress.')
return 0
def write_sitemap(
rootdir, filepath='sitemap.xml',
domain=None, include='.html', exclude=None):
if not rootdir:
rootdir = os.getcwd()
mapstr = get_sitemap(
rootdir,
domain=domain,
include=include,
exclude=exclude,
)
if not mapstr:
print_err('\nNo site map to write!')
return 1
if not filepath:
print(mapstr)
return 0
try:
with open(filepath, 'w') as f:
f.write(mapstr)
except EnvironmentError as ex:
print_err(f'Unable to write sitemap: {ex.filename}\n{ex}')
return 1
print(C(': ').join('Wrote sitemap', C(filepath, 'blue')))
return 0
def format_file_name(s):
""" Format a file name for printing. """
return str(C('').join(
'\n',
C(s, 'blue', style='bright'),
':',
))
def print_err(*args, **kwargs):
""" Print a message to stderr by default. """
if kwargs.get('file', None) is None:
kwargs['file'] = sys.stderr
nothing = object()
value = nothing
with suppress(KeyError):
value = kwargs.pop('value')
error = nothing
with suppress(KeyError):
error = kwargs.pop('error')
msg = kwargs.get('sep', ' ').join(
str(a) if isinstance(a, C) else str(C(a, 'red'))
for a in args
)
if (value is nothing) and (error is nothing):
print(msg, **kwargs)
return None
# Label/value pair.
if value is not nothing:
msg = C(': ').join(msg, C(value, 'blue'))
if error is not nothing:
msg = C('\n ').join(msg, C(error, 'red', style='bright'))
print(msg, **kwargs)
return None
def partlist_compare_fmt(lista, listb):
""" Show two parts lists side by side, highlighting any differences.
Returns a formatted str if the lists differ, otherwise returns
an empty str ('').
"""
if lista == listb:
return ''
lena, lenb = len(lista), len(listb)
maxwidth = 80
halfwidth = maxwidth // 2
lines = []
for i in range(max(lena, lenb)):
try:
itema = lista[i]
except IndexError:
itema = NotSet
try:
itemb = listb[i]
except IndexError:
itemb = NotSet
symbol = '==' if itema == itemb else '!='
colora = 'red' if itema is NotSet else 'cyan'
colorb = 'red' if itemb is NotSet or itemb != itema else 'cyan'
lines.append(
str(
C('\n').join(
' {}'.format(C(itema, colora).ljust(halfwidth)),
'{} {}'.format(symbol, C(itemb, colorb)),
)))
return '\n'.join(lines)
def __colr__(self):
return C(': ').join(
C('Large file', 'red'),
C(' ').join(
C('{} bytes'.format(self.size), 'blue').join('(', ')'),
C(self.filepath, 'cyan'),
))
def __colr__(self):
""" Format this TigerPart as a Colr when passed directly to Colr().
"""
just_default = 3
just = {
'length': 7,
'part': 5,
'no': 10,
}
justtype_default = '>'
justtype = {
'part': '<',
'no': '<',
}
missing_default = C('None', 'dimgrey').join('<', '>', fore='dimgrey')
missing = {
'completed': '0',
}
pcs = []
for key in self.header:
keylow = key.lower()
val = getattr(self, keylow, None)
if not val:
val = missing.get(keylow, missing_default)
if keylow == 'length':
val = '{:0.2f}'.format(float(val))
pcs.append(
'{k}: {v:{justtype}{just}}'.format(
k=C(key, 'blue'),
v=C(val, 'cyan'),
justtype=justtype.get(keylow, justtype_default),
just=just.get(keylow, just_default),
)
)
return C(' ').join(pcs)
def list_tests(package='test', full=False, patterns=None):
""" List all discoverable tests. """
test_info = filter_test_info(
patterns,
load_test_info(package=package),
)
for modulename, cases in test_info.items():
modulefmt = C(modulename, 'blue', style='bright')
casenames = {type(c).__name__: c for c in cases}
if not full:
print(modulefmt(':'))
for casename in sorted(casenames):
methodnames = cases[casenames[casename]]
casefmt = C(casename, 'cyan')
if not full:
print(' {}'.format(casefmt))
for methodname in sorted(methodnames):
methodfmt = C(methodname, 'green')
if full:
print(C('.').join(modulefmt, casefmt, methodfmt))
else:
print(' {}'.format(methodfmt))
if full and (not methodnames):
# Methods were filtered out.
print(C('.').join(modulefmt, casefmt))
if full and (not casenames):
# Methods and cases were filtered out.
print(modulefmt)
return 0
def debug(*args, **kwargs):
""" Print a message only if DEBUG is truthy. """
if not (DEBUG and args):
return None
# Include parent class name when given.
parent = kwargs.get('parent', None)
with suppress(KeyError):
kwargs.pop('parent')
# Go back more than once when given.
backlevel = kwargs.get('back', 1)
with suppress(KeyError):
kwargs.pop('back')
frame = inspect.currentframe()
# Go back a number of frames (usually 1).
while backlevel > 0:
frame = frame.f_back
backlevel -= 1
fname = os.path.split(frame.f_code.co_filename)[-1]
lineno = frame.f_lineno
if parent:
func = '{}.{}'.format(parent.__class__.__name__, frame.f_code.co_name)
else:
func = frame.f_code.co_name
lineinfo = '{}:{} {}: '.format(
C(fname, 'yellow'), C(str(lineno).ljust(4), 'blue'),
C().join(C(func, 'magenta'), '()').ljust(20))
# Patch args to stay compatible with print().
pargs = list(C(a, 'green').str() for a in args)
pargs[0] = ''.join((lineinfo, pargs[0]))
print_err(*pargs, **kwargs)
def search_requirements(
pattern, filename=DEFAULT_FILE, ignorecase=True):
""" Search requirements lines for a text/regex pattern, and print
results as they are found.
Returns the number of results found.
"""
reqs = Requirementz.from_file(filename=filename)
try:
found = Requirementz(
requirements=reqs.search(pattern, ignorecase=ignorecase)
)
except re.error as ex:
print_err('\nInvalid regex pattern', value=pattern, error=ex)
return 1
total = len(found)
if not total:
print_err('\nNo entries found with', value=pattern)
return 1
print('\n'.join(found.iter_str(color=True, align=True)))
print(C(' ').join(
C('\nFound', 'cyan'),
colr_num(total, style='bright'),
C('{}.'.format('entry' if total == 1 else 'entries'), 'cyan'),
))
return 0
def list_duplicates(filename=DEFAULT_FILE):
""" Print any duplicate package names found in the file.
Returns the number of duplicates found.
"""
dupes = Requirementz.from_file(filename=filename).duplicates()
dupelen = len(dupes)
if not dupelen:
print(C('No duplicate requirements found.', 'cyan'))
return 0
print(
C(' ').join(
C('Found', 'cyan'),
colr_num(dupelen),
C(
'{} with duplicate entries:'.format(
'requirement' if dupelen == 1 else 'requirements',
),
'cyan',
)
)
)
for req, dupcount in dupes.items():
print('{name:>30} has {num} {plural}'.format(
name=colr_name(req.name),
num=colr_num(dupcount, style='bright'),
plural='duplicate' if dupcount == 1 else 'duplicates'
))
return sum(dupes.values())
def __str__(self):
# Amount to indent version info, plus 2 for ': '.
verindent = ' ' * (self.basename_rjust + 2)
# Width, not counting indent, allowed for relative paths.
filenamewidth = (TERMWIDTH + len(verindent)) - self.basename_rjust
verstrlines = []
for fi in self.versions:
pathfmt = C(fi.relpath, 'cyan')
minfile = self.get_min_file(fi.relpath)
if minfile:
pathfmt = C(' ').join(
pathfmt,
C('{} {}'.format(
C('and', 'cyan'),
C(minfile, 'green'),
)).join('(', ')', style='bright')
)
verstrlines.append(str(pathfmt))
verstr = FormatBlock(
'\n'.join(verstrlines)
).format(
prepend=verindent,
newlines=True,
strip_first=True,
width=filenamewidth,
)
return '{bname:>{bnamerjust}}: {verstr}'.format(
bname=self.basename,
bnamerjust=self.basename_rjust,
verstr=verstr
)
def rainbow_text(text,
spread=2,
freq=0.1,
offset=0,
bg=False,
fg=(255, 255, 255),
style=[],
rand=False):
res = ''
if rand:
offset = random.randint(0, 10) * 10
for (l, (r, g, b)) in _rainbow_rgb_chars(text,
spread=spread,
freq=freq,
offset=offset):
if bg:
char = C().b_rgb(r, g, b).rgb(fg[0], fg[1], fg[2], l)
for sty in style:
char = color(char, style=sty)
res += char
else:
char = C().rgb(r, g, b, l)
for sty in style:
char = color(char, style=sty)
res += char
return res
def rewrite_files(filepaths,
pat,
repl,
view_first=False,
silent_empty=False,
diff=False):
""" Rewrite multiple files, replacing `pat` matches with `repl` str.
"""
skipped = 0
totalchanges = 0
confirm_opts = {'silent': silent_empty, 'diff': diff}
for filepath in filepaths:
infile = parse_file_arg(filepath, mode='r', viewmode=view_first)
if infile is None:
return 1
rf = ReplaceFile(infile, pat, repl)
if view_first and (not rf.confirm_lines(**confirm_opts)):
skipped += 1
continue
totalchanges += rf.replace_cnt
rf.rewrite()
print(format_file_changes(rf))
statuspcs = [
C(': ').join('Files', CNum(len(filepaths))),
C(': ').join('Changes', CNum(totalchanges)),
]
if skipped:
statuspcs.append(C(': ').join('Skipped', CNum(skipped)))
print('\n{}'.format(C(', ').join(statuspcs)))
return 0 if totalchanges else 1
def format_accepted_values(headername):
""" Return a comma-separated list of acceptable values for a header,
or the 'any' string if it was used.
"""
if HEADERS[headername]['accepted'] is ANY:
return C(ANY, 'yellow')
return C(', ').join(
C(val, 'yellow') for val in HEADERS[headername]['accepted'])
def proces_image(self, image):
colors = self.get_colors(image)
seperator = '================================'
print(C().b_rgb(0, 0, 0).rgb(255, 255, 255, seperator))
print(C().b_rgb(0, 0, 0).rgb(255, 255, 255, image))
print(C().b_rgb(0, 0, 0).rgb(255, 255, 255, seperator))
for c in colors:
self.show_colors(c[1])
def format_lbl(lbl, val, rjust=0, indent=0):
""" Format a label/value pair string. """
return C('').join(
' ' * indent,
C(': ').join(
C(str(lbl).rjust(rjust), 'cyan'),
C(val, 'blue', style='bright'),
))
def __str__(self):
if self.msg:
try:
lbl, val = self.msg.split(':')
except ValueError:
return 'Invalid argument, {}'.format(self.msg)
msg = C('Invalid argument, {}'.format(lbl), 'red', style='bright')
return str(C(':').join(msg, C(val, 'blue', style='bright')))
return 'Invalid argument!'
def list_labelconfig():
""" Print label config to the console and return an exit status. """
labels = label_config_get(use_display_order=True)
for name, lblinfo in labels:
print('{}:'.format(C(name, 'blue', style='bright')))
for k, v in lblinfo.items():
print(' {:>8}: {}'.format(
C(k, 'blue'),
C(v, 'cyan'),
))
return 0
def print_test_names(names):
""" Print formatted test names. """
print(
C(':').join(
C('Parsed test names', 'cyan'),
C(len(names), 'blue', style='bright'),
))
for name in names:
print(C(name, 'blue'))
return 0 if names else 1
def status(label=None, msg=None):
""" Print a status message if running in the console, and log it also. """
if msg:
line = C(': ').join(
C(label, 'blue'),
C(msg, 'cyan'),
)
else:
line = C(label, 'cyan')
if sys.stdout.isatty():
print(line)
logger.info(fix_log_msg(line.stripped(), level=1))
def write_tiger_file(
mozfile, outdir, archive_dir=None, extra_data=False,
error_cb=None, success_cb=None):
""" Write a .tiger file from a MozaikFile.
Without callbacks given, it returns an exit status (0 or 1).
With callbacks it returns `error_cb(mozfile, msg)` or
`success_cb(mozfile, tigerpath)`
"""
tigerpath = os.path.join(outdir, mozfile.filepath)
use_err_cb = callable(error_cb)
use_success_cb = callable(success_cb)
if os.path.exists(tigerpath):
debug_err('Tiger file already exists: {}'.format(tigerpath))
tigerpath = increment_file_path(tigerpath)
debug_err('Made new tiger file path: {}'.format(tigerpath))
try:
with open(tigerpath, 'w') as f:
f.write(create_xml(mozfile, extra_data=extra_data))
except EnvironmentError as ex:
msg = 'Cannot write tiger file: {}\n{}'.format(
tigerpath,
ex,
)
print_err(msg)
return error_cb(mozfile, msg) if use_err_cb else 1
partlen = len(mozfile.parts)
plural = 'part' if partlen == 1 else 'parts'
msg = C(' ').join(
C('Created', 'blue'),
C(partlen, 'blue', style='bright'),
C(plural, 'blue'),
C('parts in', 'blue'),
)
status(msg, tigerpath)
if archive_dir in (None, '', '-'):
# No archiving was requested/set.
debug('No archiving {}.'.format(
'requested' if archive_dir == '-' else 'set up',
))
return success_cb(mozfile, tigerpath) if use_success_cb else 0
if outdir in (None, '-'):
debug('Archiving disabled due to output style.')
return success_cb(mozfile, tigerpath) if use_success_cb else 0
archived = archive_file(
mozfile.parent_file,
archive_dir,
created_files=[tigerpath]
)
exitstatus = 0 if archived else 1
return success_cb(mozfile, tigerpath) if use_success_cb else exitstatus
def preview_file(filepath):
""" Preview a Mozaik file as a Tiger file. """
try:
check_file(filepath)
except LargeFileError as ex:
msg = '\n'.join((
str(C(ex)),
'',
str(C('Continue anyway?', 'cyan')),
))
if not confirm(msg):
return 1
return TigerFiles.from_file(filepath, split_parts=True).print()
def list_packages(location=False):
""" List all installed packages. """
# Sort by name first.
pkgs = sorted(PKGS)
if location:
# Sort by location, but the name sort is kept.
pkgs = sorted(pkgs, key=lambda p: PKGS[p].location)
for pname in pkgs:
p = PKGS[pname]
print('{:<30} v. {:<12} {}'.format(
colr_name(p.project_name),
C(pkg_installed_version(pname), fore='cyan'),
C(p.location, fore='green'),
))
def highlighted(self,
line_nums=False,
indent=0,
matched=False,
diff=False):
""" If matched is True, the text that matched is highlighted,
otherwise the replacement is highlighted.
"""
if diff:
pcs = [
[C('-', 'red'),
self._highlighted_match.rstrip()],
[C('+', 'green'), self._highlighted.rstrip()],
]
if line_nums:
pcs = [[
xs[0],
C('').join(C(self.lineno, 'red' if i == 0 else 'green'),
':'), xs[1]
] for i, xs in enumerate(pcs)]
return '{}\n'.format(C('\n').join(C(' ').join(pc) for pc in pcs))
line = self._highlighted_match if matched else self._highlighted
if line_nums:
s = str(C(': ').join(C(self.lineno, 'blue'), line))
else:
s = line
return '{}\n'.format(''.join((' ' * indent, s.rstrip())))
def assertPartListEqual(self, a, b, msg=None, desc=None):
""" Like assertListEqual, but with better repr's for parts. """
if a == b:
return
lena, lenb = len(a), len(b)
diffindex = -1
for i in range(min(lena, lenb)):
itema = a[i]
itemb = b[i]
if itema != itemb:
diffindex = i
parta = itema
partb = itemb
diffmsg = 'Parts are not equal.'
break
else:
# One is a subset of the other.
diffindex = max(min(lena, lenb) - 1, 0)
try:
parta = a[diffindex]
except IndexError:
parta = None
try:
partb = b[diffindex]
except IndexError:
partb = None
if lena > lenb:
diffmsg = 'First list is larger.'
elif lenb > lena:
diffmsg = 'Second list is larger.'
else:
diffmsg = 'List lengths are the same.'
lines = [
str(C(msg or 'Lists are not equal.', 'red')),
]
if desc:
lines.append(' Description: {}'.format(C(desc, 'cyan')))
lines.append('\n'.join((
' Length A: {}',
' Length B: {}',
)).format(C(lena, 'blue'), C(lenb, 'blue')))
lines.append('\n{}'.format(partlist_compare_fmt(a, b)))
lines.append('{} First differing index: {}'.format(diffmsg, diffindex))
lines.append('\nFirst differing part:\n{}'.format(
part_diff(parta, partb)))
raise AssertionError('\n'.join(lines))
def polyfill_debug_err(*args, **kwargs):
kwargs['level'] = kwargs.get('level', 0) + 1
args = (
a if isinstance(a, C) else C(a, 'red')
for a in args
)
return debugprinter.debug(*args, **kwargs)
def draw_image(cols,rows,tileH,tileW,pix):
whiteToBlack = '@%#*+=-:. '
width,height = pix.size[0],pix.size[1]
for y in range(cols):
yStart = int(y*tileH)
yEnd = int((y+1)*tileH)
if y == cols-1:
yEnd = height
for x in range(rows):
xStart = int(x*tileW)
xEnd = int((x+1)*tileW)
if x == rows-1:
xEnd = width
tile = pix.crop((xStart, yStart, xEnd, yEnd))
lum = 255 - int(getAvgBrightness(tile))
clr = getAvgColor(tile)
char = whiteToBlack[int((lum*9)/255)]
print(C().color(char,fore=(clr[0],clr[1],clr[2])),end='')
print('')
def format_label(cls, item, default='None'):
""" Format a label for printing, if it hasn't been printed yet. """
if not cls.print_labels:
return ''
return '{}: '.format(
C(cls.get_label(item, default=default), 'dimgrey')
)
def _process_input(self, text: str):
document = Document(text)
sentences = document.split_to_sentences(sent_tokenize)
for sentence in sentences:
response = self.text_index.search(sentence, neighbours=2)
if response is None:
continue
sentence = sentence.get_tokens_by_indices(
sentence.get_alphabetic_tokens())
sys.stdout.write(str(sentence) + '\n')
sys.stdout.write(str(response) + '\n')
sys.stdout.write('====================\n')
for r, word in zip(response, sentence):
c = max(ri[0] for ri in r)
sys.stdout.write(
str(C().b_rgb(255 * min(1, 1 - (c - 0.5) / 0.5), 0,
0).rgb(255, 255, 255, word)))
sys.stdout.write(' ')
sys.stdout.write('\n')
for r, word in zip(response, sentence):
r.sort(key=lambda ri: -ri[0])
sys.stdout.write(word + ': ')
for ri in r:
sys.stdout.write(f'({ri[1]}: {ri[0]:0.3f}) ')
sys.stdout.write('\n')
sys.stdout.write('====================\n')
sys.stdout.flush()