def toDict(self): import json d = self.__dict__ for k, value in d.items(): if isinstance(value, obj): d[k] = value.toDict() elif isinstsafe(value, JsonSerializable): d[k] = json.loads(value.to_json()) elif is_non_str_itr(value): # try: if not isinstance(value, dict) and len(arr(value).shape) == 0: d[k] = [] elif len(list(value)) > 0 and isinstance(list(value)[0], obj): d[k] = [v.toDict() for v in value] elif len(list(value)) > 0 and isinstsafe( list(value)[0], JsonSerializable): d[k] = [json.loads(v.to_json()) for v in value] # except: # breakpoint() return d
def __getitem__(self, item): if isinstsafe(item, WL_API): item = item.to_wl() if isinstsafe(self, APISymbol) or isinstsafe(self, APIPart): self: Union[APISymbol, APIPart] return APIPart(self.to_wl(), item, self.service) else: return APIPart(self.to_wl(), item)
def plot_example_rpeaks(self): time_mins = 'time (mins)' l = PlotData(item_type='line', y=self.ecg_flt, x=self.times() / 60.0, xlim=self.samplesToMins(self.RAND_SLICE), xlabel=time_mins, ylim='auto', hideYTicks=True) s = PlotData(item_type='scatter', y=self.ecg_flt[self.rPeaks], x=self.samplesToMins(self.rPeaks), item_color='b', xlim=self.samplesToMins(self.RAND_SLICE), title=self.alg.name() + ': example R peaks') plots = (l, s) if isinstsafe(self.alg, ECGLAB_QRS_Mod): l2 = PlotData(item_type='line', y=self.nopl, x=self.times() / 60.0, xlim=self.samplesToMins(self.RAND_SLICE), xlabel=time_mins, ylim='auto', item_color='g') plots = tuple(list(plots) + [l2]) t = MultiPlot(*plots) return t
def _attributes(self, resource_root_path, resource_root_rel, force_fix_to_abs): a = '' for k, v in self.attributes.items(): if k in [ 'onclick', 'onchange', 'onmouseover', 'onmouseout', 'onkeydown', 'onload' ]: # not sure why i cared so much to make this an err before warn('JS pipeline not prepared for html even handlers') if v is None: a += f' {k}' elif isinstsafe(v, CSS_Style_Attribute): a += f' {k}="{repr(v)}"' elif isinstance(v, MagicLink.TempAbsPath) or not isblankstr(v): if isinstance(v, MagicLink.TempAbsPath): if force_fix_to_abs: v = merge_overlapping(resource_root_path, v.abs) else: v = merge_overlapping(resource_root_path, v.abs) elif isinstance(v, MagicLink.TempRelToResPath): if force_fix_to_abs: v = merge_overlapping(resource_root_path, v.rel) else: v = os.path.join(resource_root_rel, v.rel) a += f' {k}="{v}"' return a
def __iter__(self): if self.stop_np_iter: return [] if isinstsafe(self, Folder): return iter(self.files) else: return iter(self.load())
def __len__(self) -> int: if isinstsafe(self, Folder): return len(self.files) else: # unfortunately numpy asks for len() of random things in its arrays all the time, and this causes files to load like crazy here and slow everything down. so this feature is not possible. Just manually use len(self.load()) if not self.allow_autoload: raise AttributeError else: return len(self.load())
def __init__(self, js, onload=True): if os.path.isfile(js): js = File(js) if isinstsafe(js, File): if js.ext == 'coffee': js = js.coffee2js() js = js.read() self._raw = js self._onload = onload
def __init__(self, abs, remote=None, mker=False, w=None, default=None, quiet=None): # print('making file') # log('f1') self.allow_autoload = False self.IGNORE_DS_STORE = False self.DELETE_DS_STORE = True self._default = default self.stop_np_iter = False if isstr(abs) and not os.path.isabs(abs): self.isSSH = 'test-3' in abs # abs = os.path.join(mypwd(), abs) abs = os.path.abspath(abs) elif isstr(abs): self.isSSH = 'test-3' in abs elif isinstsafe(abs, File): self.isSSH = abs.isSSH abs = abs.abspath elif abs.__class__.__name__ == 'WLFunction': from wolframclient.language import wl # 1 sec import from mlib.wolf.wolfpy import weval # 1 sec import if str(weval(wl.Head(abs))) == 'CloudObject': self.isSSH = False url = abs[0] if BASE_WOLFRAM_URL not in url: err(f'{BASE_WOLFRAM_URL=} not in {url=}') else: abs = url.replace(BASE_WOLFRAM_URL, '') else: err(f'only use cloud objects, not {weval(wl.Head(abs))=}') else: err(f'abs is a {type(abs)}, but needs to be a string(path), File, or weird WLFunction thing' ) self.abspath = abs # this might change so they have to be functions # self.isfile = os.path.isfile(self.abspath) # self.isdir = os.path.isdir(self.abspath) self.mker = mker if mker: self.mkdirs() if w is not None: self.write(w) self.default_quiet = quiet self._default_default_quiet = None # for muffleing
def file_cache_logic(f, namefile: File, myfolder: Optional[File], *args, log_level=LogLevel.PROGRESS, **kwargs): key = log_level.handle(f"no cached file for {f.__name__}, running ...", attempt_one_line=True) r = f(*args, **kwargs) assert isinstsafe( r, File ), f'filecache functions must return a single File, but this is a {type(r)}' r = r.copy_into(myfolder, next_new=True) namefile.save(r.name, silent=True) log_level.handle(f"completed and filecached", one_line_key=key) return r
def save(self, data, silent=None): import mlib.JsonSerializable as JsonSerializable for ext in save_extensions: new_data, was_converted = ext(data) if was_converted: data = new_data if isinstsafe(data, JsonSerializable.JsonSerializable): import json data = json.loads(data.to_json()) elif isinstance(data, JsonSerializable.obj): data = data.toDict() if not silent and not self.default_quiet or (silent is False): log('saving ' + self.abspath) if self.ext in ['yml', 'yaml']: import yaml self.mkparents() self.write(yaml.dump(data, sort_keys=False)) elif self.ext in JSON_EXTS: self.mkparents() import json self.write(json.dumps(data, indent=4)) elif self.ext == 'mat': self.mkparents() from scipy.io import loadmat, savemat savemat(self.abspath, data) elif self.ext in PICKLE_EXTS: self.mkparents() with open(self.abspath, 'wb') as f: import pickle pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) elif self.ext == 'png': self.mkparents() im_data = np.vectorize(np.uint8)(data) import imageio imageio.imwrite(self.abspath, im_data) else: err(f'saving does not yet support .{self.ext} files')
def openInSafari(url): if isinstsafe(url, File): url = url.url return kmscript( idd="FF3E0AC0-67D2-4378-B65A-1EF0FB60DCE7", param=url )
def __init__(self, js: JS, *args, **kwargs): assert isinstsafe(js, JS) super().__init__(f'\n{js.output()}\n', *args, **kwargs)
def __post_init__(self): if isinstsafe(self.abs, File): self.abs: File self.abs = self.abs.abspath
def jsonReprCP(o): import numpy as np # log('jsonReprCP of a ' + str(type(o))) if isinstsafe(o, np.ndarray): o = o.tolist() if not isitr(o) and isnan(o): return None if o == np.inf or o == -np.inf: return str(o) if isinstance(o, str) or isinstance(o, float) or isinstance( o, int) or o is None: return o if isinstance(o, np.int64): return int(o) if isinstance(o, np.float32): return float(o) if isitr(o) and not isinstance(o, str): cp = [] for vv in o: cp.append(jsonReprCP(vv)) return cp import copy cp = copy.deepcopy(o) # debug_on = False if hasattr(o, 'item_type') and o.item_type == 'violin': debug_on = True # import mdb; mdb.remote_breakpoint() for v in cp.__dict__: debug(f'V:{v}') # if debug_on: # breakpoint() val = getattr(cp, v) if isitr(val) and not isinstance(val, str): if isinstance(val, np.ndarray): setattr(cp, v, jsonReprCP(val.tolist())) else: newval = [] for vv in val: newval.append(jsonReprCP(vv)) setattr(cp, v, newval) if not isitr(val) and (val == np.inf or val == -np.inf): setattr(cp, v, str(val)) if isinstance(val, np.int64): setattr(cp, v, int(val)) import pandas if not isitr(val) and pandas.isnull(val): setattr(cp, v, None) if isinstance(val, JsonSerializable): setattr(cp, v, jsonReprCP(val).__dict__) if isitr(val) and not isempty(val) and isinstance( val[0], JsonSerializable): newval = [] for i in itr(val): newval.append(jsonReprCP(val[i]).__dict__) setattr(cp, v, newval) # if debug_on: # breakpoint() # if hasattr(o, 'item_type') and o.item_type == 'violin': # breakpoint() # import mdb; mdb.remote_breakpoint() return cp
def com_arg(a): if isinstsafe(a, os.PathLike): return a.__fspath__() else: return str(a)
def _pull(self): if isinstsafe(self.root_dict, ProxyDictRoot) or isinstsafe(self.root_dict, ProxyListRoot): self._sub_dict = self.root_dict.get_without_proxying(self.key) else: self._sub_dict = self.root_dict[self.key]
def arr(v=(), dtype=None, ndims=None): import numpy as np # print('start arr') # mlog.log('arr1') if enable_debug: breakpoint() if ndims is None and ( isinstsafe(v, np.ndarray) or cn(v) == 'ImagingCore' or (len(v) > 0 and isinstance(v[0], Iterable) and not isstr(v[0]) and not (hasattr(v[0], '__npitr__') and v[0].__npitr__ is False))): return mparray(v, dtype=dtype) # mlog.log('arr2') if not isinstance(v, Iterable): # mlog.log('arr3') v = [v] if len(v) > 0 and not isnumber(v[0]): # mlog.log('arr4') if ndims is None: ndims = 1 if dtype is None: dtype = {str: object}.get(v[0].__class__, object) shape = [] slices = [] stopper = None # mlog.log('arr5') for d in range(ndims): if d > 0: stopper = [stopper] ref = v slices += [None] for dd in range(d): ref = ref[0] shape += [len(ref)] # mlog.log('arr6') v.append(stopper) shape[0] = shape[0] + 1 ar = np.ndarray(shape=tuple(shape), dtype=dtype) from mlib.file import File # mlog.log('arr7') try: from mlib.boot.mlog import Muffle # mlog.log('arr8') with Muffle(*listfilt(will_break_numpy, v)): # mlog.log('arr9') sli = slice(*slices) # mlog.log('arr10') old__iter__ = [] for vv in v: if hasattr(vv, '__npitr__') and (vv.__npitr__ == False): vv.stop_np_iter = True # mlog.log('arr10.1') # if hasattr(type(vv), '__iter__') and hasattr(vv, '__npitr__') and vv.__npitr__ == False: # mlog.log('arr10.2') # breakpoint() # old__iter__.append(type(vv).__dict__['__iter__']) # mlog.log('arr10.3') # delattr(type(vv), '__iter__') # mlog.log('arr10.4') # else: # mlog.log('arr10.45') # old__iter__.append(None) # mlog.log('arr10.5') if enable_debug: breakpoint() # mlog.log('arr10.6') ar[sli] = v # THIS IS WHERE THE HORRIBLE UN-INTENDED UNNECCESARY ITERATING INTO OBJECTS IS HAPPENING ... SHOULD BE FIXED NOW! # mlog.log('arr10.7') # for i, iitteerr in enum(old__iter__): # mlog.log('arr10.8') # if iitteerr is not None: # mlog.log('arr10.9') # type(v[i]).__iter__ = iitteerr for vv in v: if hasattr(vv, '__npitr__') and (vv.__npitr__ == False): vv.stop_np_iter = False # mlog.log('arr11') except File.NoLoadSupportException: import traceback traceback.print_exc() # print('here1') ar = mparray(ar, dtype=dtype) # print('here2') v.pop() ar = ar[:-1] assert len(v) == len(ar) # print('here3') return ar else: return mparray(v, dtype=dtype)
# with open('_remote_script.py', 'r') as pyscript: # exec(pyscript.read().replace('return ', '_boot_remote_result=')) def exec_pycode_and_return(pycode): exec(teleported['script'].replace('return ', '_dummy_var')) # noinspection PyUnresolvedReferences return _dummy_var if teleported['folder_builder']: fb = FolderBuilder() exec(teleported['script']) _boot_remote_result = fb.finish() else: exec(teleported['script'].replace('return ', '_boot_remote_result=')) File('_boot_remote_result_folder').mkdirs() File('_boot_remote_result_folder').clear(silent=True) from mlib.boot.lang import isinstsafe if isinstsafe(_boot_remote_result, File): _boot_remote_result: Optional[File] _boot_remote_result.copy_into(File('_boot_remote_result_folder')) _boot_remote_result = None with open('_boot_remote_result.pkl', 'wb') as result: pickle.dump(_boot_remote_result, result, protocol=pickle.HIGHEST_PROTOCOL) boot.finish_dnn_remote() # test change for rsync
def __init__(self, css): if os.path.isfile(css): js = File(css) if isinstsafe(css, File): css = css.read() self._raw = css
def __enter__(self): for o in self.objs: if o is not None and isinstsafe(o, Muffleable): o.muffle()
def __exit__(self, exc_type, exc_val, exc_tb): for o in self.objs: if o is not None and isinstsafe(o, Muffleable): o.unmuffle()