def alphanum_join(element): sep = alphanum(element) new = [] for s in sep: if not new: new.append(s) elif isstr(new[-1]) and isstr(s): new[-1] = new[-1] + s else: new.append(s) return new
def __init__(self, abs, remote=None, mker=False, w=None, default=None, quiet=None): # print('making file') # log('f1') self.allow_autoload = False self.IGNORE_DS_STORE = False self.DELETE_DS_STORE = True self._default = default self.stop_np_iter = False if isstr(abs) and not os.path.isabs(abs): self.isSSH = 'test-3' in abs # abs = os.path.join(mypwd(), abs) abs = os.path.abspath(abs) elif isstr(abs): self.isSSH = 'test-3' in abs elif isinstsafe(abs, File): self.isSSH = abs.isSSH abs = abs.abspath elif abs.__class__.__name__ == 'WLFunction': from wolframclient.language import wl # 1 sec import from mlib.wolf.wolfpy import weval # 1 sec import if str(weval(wl.Head(abs))) == 'CloudObject': self.isSSH = False url = abs[0] if BASE_WOLFRAM_URL not in url: err(f'{BASE_WOLFRAM_URL=} not in {url=}') else: abs = url.replace(BASE_WOLFRAM_URL, '') else: err(f'only use cloud objects, not {weval(wl.Head(abs))=}') else: err(f'abs is a {type(abs)}, but needs to be a string(path), File, or weird WLFunction thing' ) self.abspath = abs # this might change so they have to be functions # self.isfile = os.path.isfile(self.abspath) # self.isdir = os.path.isdir(self.abspath) self.mker = mker if mker: self.mkdirs() if w is not None: self.write(w) self.default_quiet = quiet self._default_default_quiet = None # for muffleing
def numel(aaa): if isstr(aaa) or istuple(aaa) or islist(aaa): return len(aaa) elif isinstance(aaa, np.ndarray): return aaa.size # elif islist(aaa): return len(aaa) # else: return aaa.size else: return None
def parse_inf(o): if not isstr(o): return o else: if o == 'inf': return np.inf elif o == '-inf': return -np.inf
def contents(self, resource_root_path, resource_root_rel, force_fix_to_abs): ml = '' # try: for o in self.objs: ml += o if isstr(o) else o.getCode( resource_root_path, resource_root_rel, force_fix_to_abs) # except: # breakpoint() return ml
def isreal(n): if isstr(n): if 'inf' in n: return False else: err('strings cant be tested for real unless inf') # try: if n is None or abs(n) == np.inf: return False return np.isreal(n)
def command_list(*args): if len(args) == 0: return [''] elif len(args) > 1: return list(map(AbstractShell.com_arg, args)) elif is_non_str_itr(args[0]): return list(map(AbstractShell.com_arg, args[0])) elif isstr(args[0]): return listmap(AbstractShell.com_arg, args[0].split(' ')) else: return [AbstractShell.com_arg(args[0])]
def write(self, data: bytes): if self.stopped: return assert not isstr(data) try: data = data.decode('utf-8').replace('\r', '') except UnicodeDecodeError as u: data = "!!![UnicodeDecodeError]!!!" for stop_on_s in stop_on: if stop_on_s in data or data in stop_on_s: last_lines = data.split('\n') for last_line in last_lines: if stop_on_s in last_line or last_line in stop_on_s: break if self.fun is not None: self.fun(last_line, self.o) print(last_line) self.stopped = True return if self.fun is not None: self.fun(data, self.o) if not just_fun: # self.output_buffer.extend(data.encode()) self.output_buffer.extend(data.encode())
def int_if_not_none_or_str(e): if isnan(e) or isstr(e): return e else: return int(e)
def getCode(self, resource_root, resource_root_rel, force_fix_to_abs=False): ml = '<!DOCTYPE html>' # with Temp('temp.css') as f: # f.write(self.style) head_objs = [ '''<meta content="text/html;charset=utf-8" http-equiv="Content-Type"> <meta content="utf-8" http-equiv="encoding">''', '<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">' ] + ([HTMLCSSLink(href='style.css')] if self.style_dot_css else []) + [ StyleTag(self.style) # StyleTag(lesscpy.compile(f.abspath, minify=True)) ] if self.jQuery: head_objs.extend([ ExternalScript( src= "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js" ), ExternalScript( src= "https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js" ), HTMLCSSLink( href= "https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css" ) ]) for cdn in self.CDNs: head_objs.append(ExternalScript(cdn)) # head_objs.append(ExternalScript( # Blocked loading mixed active content # src='http://cdn.jsdelivr.net/gh/bestiejs/platform.js/platform.js', # )) head_objs.extend( listmap(lambda x: ExternalScript(src=x.replace('.coffee', '.js')), self.javascript_files) + ([JScript(JS(self.js, onload=self.wrap_js))] if self. js is not None else [])) columnChildren = [] for c in self.children: columnChildren.append(c) if isstr(c) or 'hidden' not in listkeys(c.attributes): columnChildren.append(Br) if columnChildren and columnChildren[-1] == Br: columnChildren = columnChildren[:-1] ml += HTMLRoot( HTMLHead(*head_objs), HTMLBody(*columnChildren, style=self.bodyStyle, **self.bodyAttributes, identified=self.identified).getCode( resource_root, resource_root_rel, force_fix_to_abs)).getCode(resource_root, resource_root_rel, force_fix_to_abs) return ml
def _k(cls, key): assert isstr(key) return key.lower()
def arr(v=(), dtype=None, ndims=None): import numpy as np # print('start arr') # mlog.log('arr1') if enable_debug: breakpoint() if ndims is None and ( isinstsafe(v, np.ndarray) or cn(v) == 'ImagingCore' or (len(v) > 0 and isinstance(v[0], Iterable) and not isstr(v[0]) and not (hasattr(v[0], '__npitr__') and v[0].__npitr__ is False))): return mparray(v, dtype=dtype) # mlog.log('arr2') if not isinstance(v, Iterable): # mlog.log('arr3') v = [v] if len(v) > 0 and not isnumber(v[0]): # mlog.log('arr4') if ndims is None: ndims = 1 if dtype is None: dtype = {str: object}.get(v[0].__class__, object) shape = [] slices = [] stopper = None # mlog.log('arr5') for d in range(ndims): if d > 0: stopper = [stopper] ref = v slices += [None] for dd in range(d): ref = ref[0] shape += [len(ref)] # mlog.log('arr6') v.append(stopper) shape[0] = shape[0] + 1 ar = np.ndarray(shape=tuple(shape), dtype=dtype) from mlib.file import File # mlog.log('arr7') try: from mlib.boot.mlog import Muffle # mlog.log('arr8') with Muffle(*listfilt(will_break_numpy, v)): # mlog.log('arr9') sli = slice(*slices) # mlog.log('arr10') old__iter__ = [] for vv in v: if hasattr(vv, '__npitr__') and (vv.__npitr__ == False): vv.stop_np_iter = True # mlog.log('arr10.1') # if hasattr(type(vv), '__iter__') and hasattr(vv, '__npitr__') and vv.__npitr__ == False: # mlog.log('arr10.2') # breakpoint() # old__iter__.append(type(vv).__dict__['__iter__']) # mlog.log('arr10.3') # delattr(type(vv), '__iter__') # mlog.log('arr10.4') # else: # mlog.log('arr10.45') # old__iter__.append(None) # mlog.log('arr10.5') if enable_debug: breakpoint() # mlog.log('arr10.6') ar[sli] = v # THIS IS WHERE THE HORRIBLE UN-INTENDED UNNECCESARY ITERATING INTO OBJECTS IS HAPPENING ... SHOULD BE FIXED NOW! # mlog.log('arr10.7') # for i, iitteerr in enum(old__iter__): # mlog.log('arr10.8') # if iitteerr is not None: # mlog.log('arr10.9') # type(v[i]).__iter__ = iitteerr for vv in v: if hasattr(vv, '__npitr__') and (vv.__npitr__ == False): vv.stop_np_iter = False # mlog.log('arr11') except File.NoLoadSupportException: import traceback traceback.print_exc() # print('here1') ar = mparray(ar, dtype=dtype) # print('here2') v.pop() ar = ar[:-1] assert len(v) == len(ar) # print('here3') return ar else: return mparray(v, dtype=dtype)
def table(cls, fd, force_wolf=False, debug=False): from mlib.wolf.wolf_lang import wlexpr from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT wolf = 'Wolf' in cls.__name__ or force_wolf if wolf: from mlib.wolf.wolf_figs import addHeaderLabels, LinePlotGrid, OneWayOfShowingARaster if cls == MPLFigsBackend: if BLACK_FIGS: cls.fig = plt.figure(figsize=(16, 12), facecolor='black') else: cls.fig = plt.figure(figsize=(16, 12)) if BLACK_FIGS: cls.ax = cls.fig.add_subplot(111, facecolor='black') else: cls.ax = cls.fig.add_subplot(111) cls.ax.axis("off") cls.tabl = None data = fd.data backgrounds = None if fd.confuse: low = fd.confuse_min high = fd.confuse_max # scaleBits = [] # for i in range(1, 21): # scaleBits += [[[0, 0, i / 21]]] scaleBits = JET().tolist() show_nums = fd.show_nums from copy import deepcopy backgrounds = deepcopy(data) for rrr in itr(data): for c in itr(data[rrr]): if BLACK_FIGS: backgrounds[rrr][c] = cls.color(0, 0, 0) else: backgrounds[rrr][c] = cls.color(1, 1, 1) # 256? if isnan(data[rrr][c]): if wolf: data[rrr][c] = '' backgrounds[rrr][c] = cls.none() else: data[rrr][c] = [0, 0, 0] # if BLACK_FIGS: backgrounds[rrr][c] = cls.color(0, 0, 0) # else: # backgrounds[rrr][c] = cls.color(255, 255, 255) # 256? elif not isstr(data[rrr][c]): dat = data[rrr][c] if isnan(dat): # wait! this dealt with above b = None elif high != 0: if fd.y_log_scale: b = ((dat**2) / (high**2)) else: # try: b = (dat / high) if cls.debug: # and data == 0: breakpoint() # except: else: if fd.y_log_scale: # b = np.log10(b) b = dat**2 else: b = dat if show_nums: if b is None: data[rrr][c] = 'NaN' else: data[rrr][c] = sigfig(dat, 2) else: # data[rrr][c] = [0, 0, b] # try: if b is None: # NaN breakpoint() data[rrr][c] else: # data[rrr][c] = JET[round(b * 256) - 1].tolist() # causes zeros to show wrong! data[rrr][c] = JET()[round(b * 255)].tolist() # except: if (fd.headers_included and rrr > 0 and c > 0) or not fd.headers_included: backgrounds[rrr][c] = cls.color(0, 0, b) block_len = fd.block_len if block_len is not None and fd.headers_included is False: divs = [[], []] for i in range(len(data)): if i == 1 or i % block_len == 0: divs[0] += [True] divs[1] += [True] else: divs[0] += [False] divs[1] += [False] if not fd.confuse or fd.headers_included: if cls == MPLFigsBackend and not fd.headers_included: # breakpoint() tbl = cls.ax.table( cellText=data, # rowLabels=rows, # rowColours=colors, # colLabels=columns, fontsize=fd.fontsize, loc='center') tbl.auto_set_font_size(False) tbl.set_fontsize(fd.fontsize) tbl.scale(1, (fd.fontsize + 10) / 10) # tbl.auto_set_column_width([0]) tbl.auto_set_column_width(list(range(len(data[0])))) else: breakpoint() for ri, row in enumerate(data): for ci, el in enumerate(row): if cls == MPLFigsBackend: if cls.tabl is None: cls.tabl = cls.ax.table([[1]], loc='center') width = 1 / len(data[0]) height = 1 / len(data) if ri == 0: height = 0.05 if ci == 0: width = 0.1 cell = cls.tabl.add_cell( ri, ci, width, height, loc="center", text=str(el) if el != 'Non' else "", # text="TEST_TEXT", # facecolor='green' **({ 'facecolor': backgrounds[ri][ci] } if backgrounds is not None else {})) cell.get_text().set_fontsize(20) if len(data) > 5: cell.get_text().set_fontsize(10) if BLACK_FIGS: cell.get_text().set_color('white') else: data[ri][ci] = cls.tableItem( el, backgrounds[ri][ci]) if fd.top_header_label is not None or fd.side_header_label is not None: if wolf: data = addHeaderLabels(data, fd.top_header_label, fd.side_header_label).tolist() else: cls.tabl.auto_set_font_size(False) h = cls.tabl.get_celld()[(0, 0)].get_height() w = cls.tabl.get_celld()[(0, 0)].get_width() # Create an additional Header # weird = "Header Header Header Header" weird = fd.top_header_label * 4 weird = fd.top_header_label header = [ cls.tabl.add_cell( -1, pos, w, h, loc="center", # fontsize=40.0 #facecolor="red", ) for pos in range(1, len(data[0]) + 1) ] if len(header) > 2: for idx, head in enum(header): if idx == 0: # head.visible_edges = "TBL" head.visible_edges = "" elif idx == len(header) - 1: # head.visible_edges = "TBR" head.visible_edges = "" else: # head.visible_edges = 'TB' head.visible_edges = "" header[1].get_text().set_text(weird) header[1].set_fontsize(40.0) elif len(header) == 2: header[0].visible_edges = 'TBL' header[1].visible_edges = 'TBR' header[1].get_text().set_text(weird) else: header[0].visible_edges = 'TBLR' header[0].get_text().set_text(weird) # Create an additional Header weird = fd.side_header_label * 4 weird = fd.side_header_label header = [ cls.tabl.add_cell(pos, -1, w, h, loc="center", facecolor="none") for pos in range(0, len(data) + 1) ] if len(header) > 2: for idx, head in enum(header): if idx == 0: head.visible_edges = "LTR" head.visible_edges = "" elif idx == len(header) - 1: head.visible_edges = "LRB" head.visible_edges = "" else: head.visible_edges = 'LR' head.visible_edges = "" header[1].get_text().set_text(weird) header[1].set_fontsize(40.0) # header[1].set_rotation(90.0) elif len(header) == 2: header[0].visible_edges = 'TLR' header[1].visible_edges = 'BLR' header[1].get_text().set_text(weird) header[1].set_fontsize(40.0) # header[1].set_rotation(90.0) else: header[0].visible_edges = 'TBLR' header[0].get_text().set_text(weird) header[0].set_fontsize(40.0) # header[0].set_rotation(90.0) if cls != MPLFigsBackend: insets = [ Inset( Rasterize(Grid( data, Dividers(False), ), RasterSize(), ImageSize(), Background())) ] if fd.confuse and fd.block_len is not None and fd.block_len > 1: if fd.confuse_is_identical: for rrr in itr(data): for c in itr(data[0]): if c > rrr: if BLACK_FIGS: data[rrr][c] = [0, 0, 0] else: data[rrr][c] = [1, 1, 1] # 256? if cls != MPLFigsBackend: scale = Graphics( [ Raster(scaleBits), Inset(Text(round(low), fontSize=30), [0.5, -1]), Inset(Text(round(high), fontSize=30), [0.5, 21]), ], ImagePadding([[75, 75], [20, 20]]), ) else: # create an axes on the right side of ax. The width of cax will be 5% # of ax and the padding between cax and ax will be fixed at 0.05 inch. from mpl_toolkits.axes_grid1 import make_axes_locatable # 1 FULL SECOND IMPORT divider = make_axes_locatable(cls.ax) cax = divider.append_axes("right", size="5%", pad=0.05) # cmap = matplotlib.colors.Colormap('name', N=256) # cmap = LinearSegmentedColormap.from_list( # 'bluemap', li(scaleBits).reshape(20, 3), N=20) from matplotlib.colors import LinearSegmentedColormap # 1 FULL SECOND IMPORT cmap = LinearSegmentedColormap.from_list( 'jet', li(scaleBits) # .reshape(20, 3) , N=len(scaleBits) # 20 ) import matplotlib sm = matplotlib.cm.ScalarMappable(norm=None, cmap=cmap) cbar = cls.fig.colorbar( sm, cax=cax, orientation='vertical', ticks=np.linspace( # low, 0, # high, 1, num=4)) base_scale_ticks = np.linspace(low, high, num=4) if fd.y_log_scale: base_scale_ticks = [bst**2 for bst in base_scale_ticks] cbar.ax.set_yticklabels( [sigfig(n, 3) for n in base_scale_ticks]) gl = len(data) nt = len(fd.row_headers) line_values = np.linspace(fd.block_len, fd.block_len * nt, nt) half = fd.block_len / 2 labellocs_labels = ziplist(line_values - half, fd.row_headers) # ticks start from the bottom but I want these to start from the top # labellocs_labels.reverse() if wolf: gridlines = LinePlotGrid(line_values, triangle=fd.confuse_is_identical) else: # gl = line_values[-1] listpoints = [] for i in line_values: i = i - 0.5 if fd.confuse_is_identical: listpoints += [[[i, gl - 0.5], [i, i]]] listpoints += [[[i, i], [-0.5, i]]] else: listpoints += [[[i, -0.5], [i, gl - 0.5]]] listpoints += [[[-0.5, i], [gl - 0.5, i]]] listpoints = arr(listpoints) for sub in listpoints: # cls.ax.line(sub[:, 0], sub[:, 1], 'y--') # cls.ax.plot(sub[:, 0], sub[:, 1], 'y--') cls.ax.plot(sub[:, 0], sub[:, 1], 'k--') # rasters start from the bottom but I want this to start from the top if wolf: rast = OneWayOfShowingARaster(Raster(reversed(data)), gl) else: cls.ax.imshow(list(data)) x_ticks = [] xt_mpl_t = [] xt_mpl_l = [] y_ticks = [] yt_mpl_t = [] yt_mpl_l = [] for t in labellocs_labels: if wolf: x_ticks += [ Text( t[1], coords=[t[0] / gl, -.02], direction=[1, 0], fontSize=DEFAULT_TICK_SIZE, offset=[0, 0], ) ] xt_mpl_t += [t[0] / gl] xt_mpl_l += [t[1]] for t in labellocs_labels: if wolf: y_ticks += [ Text( t[1], # y ticks need to be reversed coords=[-.01, 1 - (t[0] / gl)], direction=[1, 0], fontSize=DEFAULT_TICK_SIZE, offset=[1, 0], ) ] # y ticks not reversed for mpl? yt_mpl_t += [(t[0] / gl)] yt_mpl_l += [t[1]] if wolf: insets = [ Inset(obj=Rasterize(scale), pos=[1.2, 0], opos=[Center, Bottom]), Inset(obj=Rasterize( rast, ImageResolution(), ), opos=[Left, Bottom]), Inset( obj=Rasterize(gridlines, ImageResolution(), Background(wlexpr('None'))), opos=[Left, Bottom], background=Background( # wl.Red wlexpr('None'))) ] [insets.extend(ticks) for ticks in zip(x_ticks, y_ticks)] insets += [ Inset(Rasterize( Text(fd.title, fontSize=(40 if fd.headers_included else 20) if fd.title_size is None else fd.title_size), Background(wlexpr('None'))), scale=(1, 1), pos=Scaled([0.5, 2]), background=Background(wlexpr('None'))) ] rrr = Graphics(insets) else: title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size / 3) from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT plt.setp(title_obj, color=text_color) cls.ax.axis(True) cls.ax.spines['left'].set_color(text_color) cls.ax.spines['bottom'].set_color(text_color) cls.ax.xaxis.label.set_color(text_color) cls.ax.yaxis.label.set_color(text_color) cls.ax.tick_params(axis='x', colors=text_color) cls.ax.tick_params(axis='y', colors=text_color) cls.ax.set_xticks(arr(xt_mpl_t) * gl) cls.ax.set_xticklabels(xt_mpl_l, rotation=90) # cls.ax.xticks(rotation=90) cls.ax.set_yticks(arr(yt_mpl_t) * gl) cls.ax.set_yticklabels(yt_mpl_l) cax.axis(True) cax.spines['left'].set_color(text_color) cax.spines['bottom'].set_color(text_color) cax.spines['top'].set_color(text_color) cax.spines['right'].set_color(text_color) cax.xaxis.label.set_color(text_color) cax.yaxis.label.set_color(text_color) cax.tick_params(axis='x', colors=text_color) cax.tick_params(axis='y', colors=text_color) # cax.set_xticks(li(xt_mpl_t) * gl) # cax.set_xticklabels(xt_mpl_l, rotation=90) # cls.ax.xticks(rotation=90) # cax.set_yticks(li(yt_mpl_t) * gl) # cax.set_yticklabels(yt_mpl_l) if not wolf and (fd.confuse and fd.block_len is not None and fd.block_len > 1) == False: title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size / 3) from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT plt.setp(title_obj, color=text_color) if wolf: return rrr
def _get_cfg(self): assert len(self.registered_flags()) == len(set(self.registered_flags())) freecfg = File('freecfg.json').load() prof = 'default' cfg = 'default' changes = {} flags = [] cell = False for idx, a in enum(sys.argv): if idx == 0: continue elif a.startswith('--'): k, v = tuple(a.replace('--', '').split('=')) if k == 'tic': continue changes[k] = v elif a.startswith('-'): k, v = tuple(a.replace('-', '').split('=')) if k == 'prof': prof = v elif k == 'cfg': cfg = v else: err('arguments with one dash (-) need to be prof= or cfg=') elif cell or a in self.registered_flags(): if a == 'cell': cell = True flags += [a] else: err(f'invalid argument:{a} please see README') prof = Project.CFG['profiles'][prof] cfg = Project.CFG['configs'][cfg] for k in listkeys(prof): if k in listkeys(cfg): prof_ntrain = prof[k] for i, n in enum(cfg[k]): if isstr(n) and n[0] == 'i': cfg[k][i] = prof_ntrain[int(n[1])] cfg = {**prof, **cfg, 'FLAGS': flags} for k, v in listitems(changes): if k not in listkeys(cfg): err(f'invalid -- arguments: {k}, please see {Project.CFG.name} for configuration options') if isinstance(cfg[k], bool): v = bool(int(v)) cfg[k] = v # hello from freecomp for k, v in listitems(freecfg): log(f'freecfg: {k}:{v}') cfg[k] = v # cfg['EPOCHS'] = freecfg['EPOCHS'] return obj(cfg)