def alphanum_join(element):
sep = alphanum(element)
new = []
for s in sep:
if not new:
new.append(s)
elif isstr(new[-1]) and isstr(s):
new[-1] = new[-1] + s
else:
new.append(s)
return new
def __init__(self,
abs,
remote=None,
mker=False,
w=None,
default=None,
quiet=None):
# print('making file')
# log('f1')
self.allow_autoload = False
self.IGNORE_DS_STORE = False
self.DELETE_DS_STORE = True
self._default = default
self.stop_np_iter = False
if isstr(abs) and not os.path.isabs(abs):
self.isSSH = 'test-3' in abs
# abs = os.path.join(mypwd(), abs)
abs = os.path.abspath(abs)
elif isstr(abs):
self.isSSH = 'test-3' in abs
elif isinstsafe(abs, File):
self.isSSH = abs.isSSH
abs = abs.abspath
elif abs.__class__.__name__ == 'WLFunction':
from wolframclient.language import wl # 1 sec import
from mlib.wolf.wolfpy import weval # 1 sec import
if str(weval(wl.Head(abs))) == 'CloudObject':
self.isSSH = False
url = abs[0]
if BASE_WOLFRAM_URL not in url:
err(f'{BASE_WOLFRAM_URL=} not in {url=}')
else:
abs = url.replace(BASE_WOLFRAM_URL, '')
else:
err(f'only use cloud objects, not {weval(wl.Head(abs))=}')
else:
err(f'abs is a {type(abs)}, but needs to be a string(path), File, or weird WLFunction thing'
)
self.abspath = abs
# this might change so they have to be functions
# self.isfile = os.path.isfile(self.abspath)
# self.isdir = os.path.isdir(self.abspath)
self.mker = mker
if mker:
self.mkdirs()
if w is not None:
self.write(w)
self.default_quiet = quiet
self._default_default_quiet = None # for muffleing
def numel(aaa):
if isstr(aaa) or istuple(aaa) or islist(aaa): return len(aaa)
elif isinstance(aaa, np.ndarray):
return aaa.size
# elif islist(aaa): return len(aaa)
# else: return aaa.size
else:
return None
def parse_inf(o):
if not isstr(o):
return o
else:
if o == 'inf':
return np.inf
elif o == '-inf':
return -np.inf
def contents(self, resource_root_path, resource_root_rel,
force_fix_to_abs):
ml = ''
# try:
for o in self.objs:
ml += o if isstr(o) else o.getCode(
resource_root_path, resource_root_rel, force_fix_to_abs)
# except:
# breakpoint()
return ml
def isreal(n):
if isstr(n):
if 'inf' in n:
return False
else:
err('strings cant be tested for real unless inf')
# try:
if n is None or abs(n) == np.inf:
return False
return np.isreal(n)
def command_list(*args):
if len(args) == 0:
return ['']
elif len(args) > 1:
return list(map(AbstractShell.com_arg, args))
elif is_non_str_itr(args[0]):
return list(map(AbstractShell.com_arg, args[0]))
elif isstr(args[0]):
return listmap(AbstractShell.com_arg, args[0].split(' '))
else:
return [AbstractShell.com_arg(args[0])]
def write(self, data: bytes):
if self.stopped: return
assert not isstr(data)
try:
data = data.decode('utf-8').replace('\r', '')
except UnicodeDecodeError as u:
data = "!!![UnicodeDecodeError]!!!"
for stop_on_s in stop_on:
if stop_on_s in data or data in stop_on_s:
last_lines = data.split('\n')
for last_line in last_lines:
if stop_on_s in last_line or last_line in stop_on_s:
break
if self.fun is not None:
self.fun(last_line, self.o)
print(last_line)
self.stopped = True
return
if self.fun is not None:
self.fun(data, self.o)
if not just_fun:
# self.output_buffer.extend(data.encode())
self.output_buffer.extend(data.encode())
def int_if_not_none_or_str(e):
if isnan(e) or isstr(e):
return e
else:
return int(e)
def getCode(self,
resource_root,
resource_root_rel,
force_fix_to_abs=False):
ml = '<!DOCTYPE html>'
# with Temp('temp.css') as f:
# f.write(self.style)
head_objs = [
'''<meta content="text/html;charset=utf-8" http-equiv="Content-Type">
<meta content="utf-8" http-equiv="encoding">''',
'<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">'
] + ([HTMLCSSLink(href='style.css')] if self.style_dot_css else []) + [
StyleTag(self.style)
# StyleTag(lesscpy.compile(f.abspath, minify=True))
]
if self.jQuery:
head_objs.extend([
ExternalScript(
src=
"https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"
),
ExternalScript(
src=
"https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"
),
HTMLCSSLink(
href=
"https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css"
)
])
for cdn in self.CDNs:
head_objs.append(ExternalScript(cdn))
# head_objs.append(ExternalScript(
# Blocked loading mixed active content
# src='http://cdn.jsdelivr.net/gh/bestiejs/platform.js/platform.js',
# ))
head_objs.extend(
listmap(lambda x: ExternalScript(src=x.replace('.coffee', '.js')),
self.javascript_files) +
([JScript(JS(self.js, onload=self.wrap_js))] if self.
js is not None else []))
columnChildren = []
for c in self.children:
columnChildren.append(c)
if isstr(c) or 'hidden' not in listkeys(c.attributes):
columnChildren.append(Br)
if columnChildren and columnChildren[-1] == Br:
columnChildren = columnChildren[:-1]
ml += HTMLRoot(
HTMLHead(*head_objs),
HTMLBody(*columnChildren,
style=self.bodyStyle,
**self.bodyAttributes,
identified=self.identified).getCode(
resource_root, resource_root_rel,
force_fix_to_abs)).getCode(resource_root,
resource_root_rel,
force_fix_to_abs)
return ml
def _k(cls, key):
assert isstr(key)
return key.lower()
def arr(v=(), dtype=None, ndims=None):
import numpy as np
# print('start arr')
# mlog.log('arr1')
if enable_debug:
breakpoint()
if ndims is None and (
isinstsafe(v, np.ndarray) or cn(v) == 'ImagingCore' or
(len(v) > 0 and isinstance(v[0], Iterable) and not isstr(v[0])
and not (hasattr(v[0], '__npitr__') and v[0].__npitr__ is False))):
return mparray(v, dtype=dtype)
# mlog.log('arr2')
if not isinstance(v, Iterable):
# mlog.log('arr3')
v = [v]
if len(v) > 0 and not isnumber(v[0]):
# mlog.log('arr4')
if ndims is None: ndims = 1
if dtype is None:
dtype = {str: object}.get(v[0].__class__, object)
shape = []
slices = []
stopper = None
# mlog.log('arr5')
for d in range(ndims):
if d > 0:
stopper = [stopper]
ref = v
slices += [None]
for dd in range(d):
ref = ref[0]
shape += [len(ref)]
# mlog.log('arr6')
v.append(stopper)
shape[0] = shape[0] + 1
ar = np.ndarray(shape=tuple(shape), dtype=dtype)
from mlib.file import File
# mlog.log('arr7')
try:
from mlib.boot.mlog import Muffle
# mlog.log('arr8')
with Muffle(*listfilt(will_break_numpy, v)):
# mlog.log('arr9')
sli = slice(*slices)
# mlog.log('arr10')
old__iter__ = []
for vv in v:
if hasattr(vv, '__npitr__') and (vv.__npitr__ == False):
vv.stop_np_iter = True
# mlog.log('arr10.1')
# if hasattr(type(vv), '__iter__') and hasattr(vv, '__npitr__') and vv.__npitr__ == False:
# mlog.log('arr10.2')
# breakpoint()
# old__iter__.append(type(vv).__dict__['__iter__'])
# mlog.log('arr10.3')
# delattr(type(vv), '__iter__')
# mlog.log('arr10.4')
# else:
# mlog.log('arr10.45')
# old__iter__.append(None)
# mlog.log('arr10.5')
if enable_debug:
breakpoint()
# mlog.log('arr10.6')
ar[sli] = v # THIS IS WHERE THE HORRIBLE UN-INTENDED UNNECCESARY ITERATING INTO OBJECTS IS HAPPENING ... SHOULD BE FIXED NOW!
# mlog.log('arr10.7')
# for i, iitteerr in enum(old__iter__):
# mlog.log('arr10.8')
# if iitteerr is not None:
# mlog.log('arr10.9')
# type(v[i]).__iter__ = iitteerr
for vv in v:
if hasattr(vv, '__npitr__') and (vv.__npitr__ == False):
vv.stop_np_iter = False
# mlog.log('arr11')
except File.NoLoadSupportException:
import traceback
traceback.print_exc()
# print('here1')
ar = mparray(ar, dtype=dtype)
# print('here2')
v.pop()
ar = ar[:-1]
assert len(v) == len(ar)
# print('here3')
return ar
else:
return mparray(v, dtype=dtype)
def table(cls, fd, force_wolf=False, debug=False):
from mlib.wolf.wolf_lang import wlexpr
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
wolf = 'Wolf' in cls.__name__ or force_wolf
if wolf:
from mlib.wolf.wolf_figs import addHeaderLabels, LinePlotGrid, OneWayOfShowingARaster
if cls == MPLFigsBackend:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
if BLACK_FIGS:
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.ax = cls.fig.add_subplot(111)
cls.ax.axis("off")
cls.tabl = None
data = fd.data
backgrounds = None
if fd.confuse:
low = fd.confuse_min
high = fd.confuse_max
# scaleBits = []
# for i in range(1, 21):
# scaleBits += [[[0, 0, i / 21]]]
scaleBits = JET().tolist()
show_nums = fd.show_nums
from copy import deepcopy
backgrounds = deepcopy(data)
for rrr in itr(data):
for c in itr(data[rrr]):
if BLACK_FIGS:
backgrounds[rrr][c] = cls.color(0, 0, 0)
else:
backgrounds[rrr][c] = cls.color(1, 1, 1) # 256?
if isnan(data[rrr][c]):
if wolf:
data[rrr][c] = ''
backgrounds[rrr][c] = cls.none()
else:
data[rrr][c] = [0, 0, 0]
# if BLACK_FIGS:
backgrounds[rrr][c] = cls.color(0, 0, 0)
# else:
# backgrounds[rrr][c] = cls.color(255, 255, 255) # 256?
elif not isstr(data[rrr][c]):
dat = data[rrr][c]
if isnan(dat): # wait! this dealt with above
b = None
elif high != 0:
if fd.y_log_scale:
b = ((dat**2) / (high**2))
else:
# try:
b = (dat / high)
if cls.debug: # and data == 0:
breakpoint()
# except:
else:
if fd.y_log_scale:
# b = np.log10(b)
b = dat**2
else:
b = dat
if show_nums:
if b is None:
data[rrr][c] = 'NaN'
else:
data[rrr][c] = sigfig(dat, 2)
else:
# data[rrr][c] = [0, 0, b]
# try:
if b is None: # NaN
breakpoint()
data[rrr][c]
else:
# data[rrr][c] = JET[round(b * 256) - 1].tolist() # causes zeros to show wrong!
data[rrr][c] = JET()[round(b * 255)].tolist()
# except:
if (fd.headers_included and rrr > 0
and c > 0) or not fd.headers_included:
backgrounds[rrr][c] = cls.color(0, 0, b)
block_len = fd.block_len
if block_len is not None and fd.headers_included is False:
divs = [[], []]
for i in range(len(data)):
if i == 1 or i % block_len == 0:
divs[0] += [True]
divs[1] += [True]
else:
divs[0] += [False]
divs[1] += [False]
if not fd.confuse or fd.headers_included:
if cls == MPLFigsBackend and not fd.headers_included:
# breakpoint()
tbl = cls.ax.table(
cellText=data,
# rowLabels=rows,
# rowColours=colors,
# colLabels=columns,
fontsize=fd.fontsize,
loc='center')
tbl.auto_set_font_size(False)
tbl.set_fontsize(fd.fontsize)
tbl.scale(1, (fd.fontsize + 10) / 10)
# tbl.auto_set_column_width([0])
tbl.auto_set_column_width(list(range(len(data[0]))))
else:
breakpoint()
for ri, row in enumerate(data):
for ci, el in enumerate(row):
if cls == MPLFigsBackend:
if cls.tabl is None:
cls.tabl = cls.ax.table([[1]], loc='center')
width = 1 / len(data[0])
height = 1 / len(data)
if ri == 0:
height = 0.05
if ci == 0:
width = 0.1
cell = cls.tabl.add_cell(
ri,
ci,
width,
height,
loc="center",
text=str(el) if el != 'Non' else "",
# text="TEST_TEXT",
# facecolor='green'
**({
'facecolor': backgrounds[ri][ci]
} if backgrounds is not None else {}))
cell.get_text().set_fontsize(20)
if len(data) > 5:
cell.get_text().set_fontsize(10)
if BLACK_FIGS:
cell.get_text().set_color('white')
else:
data[ri][ci] = cls.tableItem(
el, backgrounds[ri][ci])
if fd.top_header_label is not None or fd.side_header_label is not None:
if wolf:
data = addHeaderLabels(data, fd.top_header_label,
fd.side_header_label).tolist()
else:
cls.tabl.auto_set_font_size(False)
h = cls.tabl.get_celld()[(0, 0)].get_height()
w = cls.tabl.get_celld()[(0, 0)].get_width()
# Create an additional Header
# weird = "Header Header Header Header"
weird = fd.top_header_label * 4
weird = fd.top_header_label
header = [
cls.tabl.add_cell(
-1,
pos,
w,
h,
loc="center", # fontsize=40.0 #facecolor="red",
) for pos in range(1,
len(data[0]) + 1)
]
if len(header) > 2:
for idx, head in enum(header):
if idx == 0:
# head.visible_edges = "TBL"
head.visible_edges = ""
elif idx == len(header) - 1:
# head.visible_edges = "TBR"
head.visible_edges = ""
else:
# head.visible_edges = 'TB'
head.visible_edges = ""
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
elif len(header) == 2:
header[0].visible_edges = 'TBL'
header[1].visible_edges = 'TBR'
header[1].get_text().set_text(weird)
else:
header[0].visible_edges = 'TBLR'
header[0].get_text().set_text(weird)
# Create an additional Header
weird = fd.side_header_label * 4
weird = fd.side_header_label
header = [
cls.tabl.add_cell(pos,
-1,
w,
h,
loc="center",
facecolor="none")
for pos in range(0,
len(data) + 1)
]
if len(header) > 2:
for idx, head in enum(header):
if idx == 0:
head.visible_edges = "LTR"
head.visible_edges = ""
elif idx == len(header) - 1:
head.visible_edges = "LRB"
head.visible_edges = ""
else:
head.visible_edges = 'LR'
head.visible_edges = ""
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
# header[1].set_rotation(90.0)
elif len(header) == 2:
header[0].visible_edges = 'TLR'
header[1].visible_edges = 'BLR'
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
# header[1].set_rotation(90.0)
else:
header[0].visible_edges = 'TBLR'
header[0].get_text().set_text(weird)
header[0].set_fontsize(40.0)
# header[0].set_rotation(90.0)
if cls != MPLFigsBackend:
insets = [
Inset(
Rasterize(Grid(
data,
Dividers(False),
), RasterSize(), ImageSize(), Background()))
]
if fd.confuse and fd.block_len is not None and fd.block_len > 1:
if fd.confuse_is_identical:
for rrr in itr(data):
for c in itr(data[0]):
if c > rrr:
if BLACK_FIGS:
data[rrr][c] = [0, 0, 0]
else:
data[rrr][c] = [1, 1, 1] # 256?
if cls != MPLFigsBackend:
scale = Graphics(
[
Raster(scaleBits),
Inset(Text(round(low), fontSize=30), [0.5, -1]),
Inset(Text(round(high), fontSize=30), [0.5, 21]),
],
ImagePadding([[75, 75], [20, 20]]),
)
else:
# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
from mpl_toolkits.axes_grid1 import make_axes_locatable # 1 FULL SECOND IMPORT
divider = make_axes_locatable(cls.ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
# cmap = matplotlib.colors.Colormap('name', N=256)
# cmap = LinearSegmentedColormap.from_list(
# 'bluemap', li(scaleBits).reshape(20, 3), N=20)
from matplotlib.colors import LinearSegmentedColormap # 1 FULL SECOND IMPORT
cmap = LinearSegmentedColormap.from_list(
'jet',
li(scaleBits) # .reshape(20, 3)
,
N=len(scaleBits) # 20
)
import matplotlib
sm = matplotlib.cm.ScalarMappable(norm=None, cmap=cmap)
cbar = cls.fig.colorbar(
sm,
cax=cax,
orientation='vertical',
ticks=np.linspace(
# low,
0,
# high,
1,
num=4))
base_scale_ticks = np.linspace(low, high, num=4)
if fd.y_log_scale:
base_scale_ticks = [bst**2 for bst in base_scale_ticks]
cbar.ax.set_yticklabels(
[sigfig(n, 3) for n in base_scale_ticks])
gl = len(data)
nt = len(fd.row_headers)
line_values = np.linspace(fd.block_len, fd.block_len * nt, nt)
half = fd.block_len / 2
labellocs_labels = ziplist(line_values - half, fd.row_headers)
# ticks start from the bottom but I want these to start from the top
# labellocs_labels.reverse()
if wolf:
gridlines = LinePlotGrid(line_values,
triangle=fd.confuse_is_identical)
else:
# gl = line_values[-1]
listpoints = []
for i in line_values:
i = i - 0.5
if fd.confuse_is_identical:
listpoints += [[[i, gl - 0.5], [i, i]]]
listpoints += [[[i, i], [-0.5, i]]]
else:
listpoints += [[[i, -0.5], [i, gl - 0.5]]]
listpoints += [[[-0.5, i], [gl - 0.5, i]]]
listpoints = arr(listpoints)
for sub in listpoints:
# cls.ax.line(sub[:, 0], sub[:, 1], 'y--')
# cls.ax.plot(sub[:, 0], sub[:, 1], 'y--')
cls.ax.plot(sub[:, 0], sub[:, 1], 'k--')
# rasters start from the bottom but I want this to start from the top
if wolf:
rast = OneWayOfShowingARaster(Raster(reversed(data)), gl)
else:
cls.ax.imshow(list(data))
x_ticks = []
xt_mpl_t = []
xt_mpl_l = []
y_ticks = []
yt_mpl_t = []
yt_mpl_l = []
for t in labellocs_labels:
if wolf:
x_ticks += [
Text(
t[1],
coords=[t[0] / gl, -.02],
direction=[1, 0],
fontSize=DEFAULT_TICK_SIZE,
offset=[0, 0],
)
]
xt_mpl_t += [t[0] / gl]
xt_mpl_l += [t[1]]
for t in labellocs_labels:
if wolf:
y_ticks += [
Text(
t[1],
# y ticks need to be reversed
coords=[-.01, 1 - (t[0] / gl)],
direction=[1, 0],
fontSize=DEFAULT_TICK_SIZE,
offset=[1, 0],
)
]
# y ticks not reversed for mpl?
yt_mpl_t += [(t[0] / gl)]
yt_mpl_l += [t[1]]
if wolf:
insets = [
Inset(obj=Rasterize(scale),
pos=[1.2, 0],
opos=[Center, Bottom]),
Inset(obj=Rasterize(
rast,
ImageResolution(),
),
opos=[Left, Bottom]),
Inset(
obj=Rasterize(gridlines, ImageResolution(),
Background(wlexpr('None'))),
opos=[Left, Bottom],
background=Background(
# wl.Red
wlexpr('None')))
]
[insets.extend(ticks) for ticks in zip(x_ticks, y_ticks)]
insets += [
Inset(Rasterize(
Text(fd.title,
fontSize=(40 if fd.headers_included else 20)
if fd.title_size is None else fd.title_size),
Background(wlexpr('None'))),
scale=(1, 1),
pos=Scaled([0.5, 2]),
background=Background(wlexpr('None')))
]
rrr = Graphics(insets)
else:
title_obj = cls.ax.set_title(fd.title,
fontsize=fd.title_size / 3)
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
cls.ax.set_xticks(arr(xt_mpl_t) * gl)
cls.ax.set_xticklabels(xt_mpl_l, rotation=90)
# cls.ax.xticks(rotation=90)
cls.ax.set_yticks(arr(yt_mpl_t) * gl)
cls.ax.set_yticklabels(yt_mpl_l)
cax.axis(True)
cax.spines['left'].set_color(text_color)
cax.spines['bottom'].set_color(text_color)
cax.spines['top'].set_color(text_color)
cax.spines['right'].set_color(text_color)
cax.xaxis.label.set_color(text_color)
cax.yaxis.label.set_color(text_color)
cax.tick_params(axis='x', colors=text_color)
cax.tick_params(axis='y', colors=text_color)
# cax.set_xticks(li(xt_mpl_t) * gl)
# cax.set_xticklabels(xt_mpl_l, rotation=90)
# cls.ax.xticks(rotation=90)
# cax.set_yticks(li(yt_mpl_t) * gl)
# cax.set_yticklabels(yt_mpl_l)
if not wolf and (fd.confuse and fd.block_len is not None
and fd.block_len > 1) == False:
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size / 3)
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
plt.setp(title_obj, color=text_color)
if wolf:
return rrr
def _get_cfg(self):
assert len(self.registered_flags()) == len(set(self.registered_flags()))
freecfg = File('freecfg.json').load()
prof = 'default'
cfg = 'default'
changes = {}
flags = []
cell = False
for idx, a in enum(sys.argv):
if idx == 0: continue
elif a.startswith('--'):
k, v = tuple(a.replace('--', '').split('='))
if k == 'tic': continue
changes[k] = v
elif a.startswith('-'):
k, v = tuple(a.replace('-', '').split('='))
if k == 'prof':
prof = v
elif k == 'cfg':
cfg = v
else:
err('arguments with one dash (-) need to be prof= or cfg=')
elif cell or a in self.registered_flags():
if a == 'cell':
cell = True
flags += [a]
else:
err(f'invalid argument:{a} please see README')
prof = Project.CFG['profiles'][prof]
cfg = Project.CFG['configs'][cfg]
for k in listkeys(prof):
if k in listkeys(cfg):
prof_ntrain = prof[k]
for i, n in enum(cfg[k]):
if isstr(n) and n[0] == 'i':
cfg[k][i] = prof_ntrain[int(n[1])]
cfg = {**prof, **cfg, 'FLAGS': flags}
for k, v in listitems(changes):
if k not in listkeys(cfg):
err(f'invalid -- arguments: {k}, please see {Project.CFG.name} for configuration options')
if isinstance(cfg[k], bool):
v = bool(int(v))
cfg[k] = v
# hello from freecomp
for k, v in listitems(freecfg):
log(f'freecfg: {k}:{v}')
cfg[k] = v
# cfg['EPOCHS'] = freecfg['EPOCHS']
return obj(cfg)