def toDict(self):
import json
d = self.__dict__
for k, value in d.items():
if isinstance(value, obj):
d[k] = value.toDict()
elif isinstsafe(value, JsonSerializable):
d[k] = json.loads(value.to_json())
elif is_non_str_itr(value):
# try:
if not isinstance(value, dict) and len(arr(value).shape) == 0:
d[k] = []
elif len(list(value)) > 0 and isinstance(list(value)[0], obj):
d[k] = [v.toDict() for v in value]
elif len(list(value)) > 0 and isinstsafe(
list(value)[0], JsonSerializable):
d[k] = [json.loads(v.to_json()) for v in value]
# except:
# breakpoint()
return d
def __getitem__(self, item):
if isinstsafe(item, WL_API):
item = item.to_wl()
if isinstsafe(self, APISymbol) or isinstsafe(self, APIPart):
self: Union[APISymbol, APIPart]
return APIPart(self.to_wl(), item, self.service)
else:
return APIPart(self.to_wl(), item)
def plot_example_rpeaks(self):
time_mins = 'time (mins)'
l = PlotData(item_type='line',
y=self.ecg_flt,
x=self.times() / 60.0,
xlim=self.samplesToMins(self.RAND_SLICE),
xlabel=time_mins,
ylim='auto',
hideYTicks=True)
s = PlotData(item_type='scatter',
y=self.ecg_flt[self.rPeaks],
x=self.samplesToMins(self.rPeaks),
item_color='b',
xlim=self.samplesToMins(self.RAND_SLICE),
title=self.alg.name() + ': example R peaks')
plots = (l, s)
if isinstsafe(self.alg, ECGLAB_QRS_Mod):
l2 = PlotData(item_type='line',
y=self.nopl,
x=self.times() / 60.0,
xlim=self.samplesToMins(self.RAND_SLICE),
xlabel=time_mins,
ylim='auto',
item_color='g')
plots = tuple(list(plots) + [l2])
t = MultiPlot(*plots)
return t
def _attributes(self, resource_root_path, resource_root_rel,
force_fix_to_abs):
a = ''
for k, v in self.attributes.items():
if k in [
'onclick', 'onchange', 'onmouseover', 'onmouseout',
'onkeydown', 'onload'
]:
# not sure why i cared so much to make this an err before
warn('JS pipeline not prepared for html even handlers')
if v is None:
a += f' {k}'
elif isinstsafe(v, CSS_Style_Attribute):
a += f' {k}="{repr(v)}"'
elif isinstance(v, MagicLink.TempAbsPath) or not isblankstr(v):
if isinstance(v, MagicLink.TempAbsPath):
if force_fix_to_abs:
v = merge_overlapping(resource_root_path, v.abs)
else:
v = merge_overlapping(resource_root_path, v.abs)
elif isinstance(v, MagicLink.TempRelToResPath):
if force_fix_to_abs:
v = merge_overlapping(resource_root_path, v.rel)
else:
v = os.path.join(resource_root_rel, v.rel)
a += f' {k}="{v}"'
return a
def __iter__(self):
if self.stop_np_iter:
return []
if isinstsafe(self, Folder):
return iter(self.files)
else:
return iter(self.load())
def __len__(self) -> int:
if isinstsafe(self, Folder):
return len(self.files)
else:
# unfortunately numpy asks for len() of random things in its arrays all the time, and this causes files to load like crazy here and slow everything down. so this feature is not possible. Just manually use len(self.load())
if not self.allow_autoload:
raise AttributeError
else:
return len(self.load())
def __init__(self, js, onload=True):
if os.path.isfile(js):
js = File(js)
if isinstsafe(js, File):
if js.ext == 'coffee':
js = js.coffee2js()
js = js.read()
self._raw = js
self._onload = onload
def __init__(self,
abs,
remote=None,
mker=False,
w=None,
default=None,
quiet=None):
# print('making file')
# log('f1')
self.allow_autoload = False
self.IGNORE_DS_STORE = False
self.DELETE_DS_STORE = True
self._default = default
self.stop_np_iter = False
if isstr(abs) and not os.path.isabs(abs):
self.isSSH = 'test-3' in abs
# abs = os.path.join(mypwd(), abs)
abs = os.path.abspath(abs)
elif isstr(abs):
self.isSSH = 'test-3' in abs
elif isinstsafe(abs, File):
self.isSSH = abs.isSSH
abs = abs.abspath
elif abs.__class__.__name__ == 'WLFunction':
from wolframclient.language import wl # 1 sec import
from mlib.wolf.wolfpy import weval # 1 sec import
if str(weval(wl.Head(abs))) == 'CloudObject':
self.isSSH = False
url = abs[0]
if BASE_WOLFRAM_URL not in url:
err(f'{BASE_WOLFRAM_URL=} not in {url=}')
else:
abs = url.replace(BASE_WOLFRAM_URL, '')
else:
err(f'only use cloud objects, not {weval(wl.Head(abs))=}')
else:
err(f'abs is a {type(abs)}, but needs to be a string(path), File, or weird WLFunction thing'
)
self.abspath = abs
# this might change so they have to be functions
# self.isfile = os.path.isfile(self.abspath)
# self.isdir = os.path.isdir(self.abspath)
self.mker = mker
if mker:
self.mkdirs()
if w is not None:
self.write(w)
self.default_quiet = quiet
self._default_default_quiet = None # for muffleing
def file_cache_logic(f,
namefile: File,
myfolder: Optional[File],
*args,
log_level=LogLevel.PROGRESS,
**kwargs):
key = log_level.handle(f"no cached file for {f.__name__}, running ...",
attempt_one_line=True)
r = f(*args, **kwargs)
assert isinstsafe(
r, File
), f'filecache functions must return a single File, but this is a {type(r)}'
r = r.copy_into(myfolder, next_new=True)
namefile.save(r.name, silent=True)
log_level.handle(f"completed and filecached", one_line_key=key)
return r
def save(self, data, silent=None):
import mlib.JsonSerializable as JsonSerializable
for ext in save_extensions:
new_data, was_converted = ext(data)
if was_converted:
data = new_data
if isinstsafe(data, JsonSerializable.JsonSerializable):
import json
data = json.loads(data.to_json())
elif isinstance(data, JsonSerializable.obj):
data = data.toDict()
if not silent and not self.default_quiet or (silent is False):
log('saving ' + self.abspath)
if self.ext in ['yml', 'yaml']:
import yaml
self.mkparents()
self.write(yaml.dump(data, sort_keys=False))
elif self.ext in JSON_EXTS:
self.mkparents()
import json
self.write(json.dumps(data, indent=4))
elif self.ext == 'mat':
self.mkparents()
from scipy.io import loadmat, savemat
savemat(self.abspath, data)
elif self.ext in PICKLE_EXTS:
self.mkparents()
with open(self.abspath, 'wb') as f:
import pickle
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
elif self.ext == 'png':
self.mkparents()
im_data = np.vectorize(np.uint8)(data)
import imageio
imageio.imwrite(self.abspath, im_data)
else:
err(f'saving does not yet support .{self.ext} files')
def openInSafari(url):
if isinstsafe(url, File): url = url.url
return kmscript(
idd="FF3E0AC0-67D2-4378-B65A-1EF0FB60DCE7",
param=url
)
def __init__(self, js: JS, *args, **kwargs):
assert isinstsafe(js, JS)
super().__init__(f'\n{js.output()}\n', *args, **kwargs)
def __post_init__(self):
if isinstsafe(self.abs, File):
self.abs: File
self.abs = self.abs.abspath
def jsonReprCP(o):
import numpy as np
# log('jsonReprCP of a ' + str(type(o)))
if isinstsafe(o, np.ndarray):
o = o.tolist()
if not isitr(o) and isnan(o):
return None
if o == np.inf or o == -np.inf:
return str(o)
if isinstance(o, str) or isinstance(o, float) or isinstance(
o, int) or o is None:
return o
if isinstance(o, np.int64):
return int(o)
if isinstance(o, np.float32):
return float(o)
if isitr(o) and not isinstance(o, str):
cp = []
for vv in o:
cp.append(jsonReprCP(vv))
return cp
import copy
cp = copy.deepcopy(o)
# debug_on = False
if hasattr(o, 'item_type') and o.item_type == 'violin':
debug_on = True
# import mdb; mdb.remote_breakpoint()
for v in cp.__dict__:
debug(f'V:{v}')
# if debug_on:
# breakpoint()
val = getattr(cp, v)
if isitr(val) and not isinstance(val, str):
if isinstance(val, np.ndarray):
setattr(cp, v, jsonReprCP(val.tolist()))
else:
newval = []
for vv in val:
newval.append(jsonReprCP(vv))
setattr(cp, v, newval)
if not isitr(val) and (val == np.inf or val == -np.inf):
setattr(cp, v, str(val))
if isinstance(val, np.int64):
setattr(cp, v, int(val))
import pandas
if not isitr(val) and pandas.isnull(val):
setattr(cp, v, None)
if isinstance(val, JsonSerializable):
setattr(cp, v, jsonReprCP(val).__dict__)
if isitr(val) and not isempty(val) and isinstance(
val[0], JsonSerializable):
newval = []
for i in itr(val):
newval.append(jsonReprCP(val[i]).__dict__)
setattr(cp, v, newval)
# if debug_on:
# breakpoint()
# if hasattr(o, 'item_type') and o.item_type == 'violin':
# breakpoint()
# import mdb; mdb.remote_breakpoint()
return cp
def com_arg(a):
if isinstsafe(a, os.PathLike):
return a.__fspath__()
else:
return str(a)
def _pull(self):
if isinstsafe(self.root_dict, ProxyDictRoot) or isinstsafe(self.root_dict, ProxyListRoot):
self._sub_dict = self.root_dict.get_without_proxying(self.key)
else:
self._sub_dict = self.root_dict[self.key]
def arr(v=(), dtype=None, ndims=None):
import numpy as np
# print('start arr')
# mlog.log('arr1')
if enable_debug:
breakpoint()
if ndims is None and (
isinstsafe(v, np.ndarray) or cn(v) == 'ImagingCore' or
(len(v) > 0 and isinstance(v[0], Iterable) and not isstr(v[0])
and not (hasattr(v[0], '__npitr__') and v[0].__npitr__ is False))):
return mparray(v, dtype=dtype)
# mlog.log('arr2')
if not isinstance(v, Iterable):
# mlog.log('arr3')
v = [v]
if len(v) > 0 and not isnumber(v[0]):
# mlog.log('arr4')
if ndims is None: ndims = 1
if dtype is None:
dtype = {str: object}.get(v[0].__class__, object)
shape = []
slices = []
stopper = None
# mlog.log('arr5')
for d in range(ndims):
if d > 0:
stopper = [stopper]
ref = v
slices += [None]
for dd in range(d):
ref = ref[0]
shape += [len(ref)]
# mlog.log('arr6')
v.append(stopper)
shape[0] = shape[0] + 1
ar = np.ndarray(shape=tuple(shape), dtype=dtype)
from mlib.file import File
# mlog.log('arr7')
try:
from mlib.boot.mlog import Muffle
# mlog.log('arr8')
with Muffle(*listfilt(will_break_numpy, v)):
# mlog.log('arr9')
sli = slice(*slices)
# mlog.log('arr10')
old__iter__ = []
for vv in v:
if hasattr(vv, '__npitr__') and (vv.__npitr__ == False):
vv.stop_np_iter = True
# mlog.log('arr10.1')
# if hasattr(type(vv), '__iter__') and hasattr(vv, '__npitr__') and vv.__npitr__ == False:
# mlog.log('arr10.2')
# breakpoint()
# old__iter__.append(type(vv).__dict__['__iter__'])
# mlog.log('arr10.3')
# delattr(type(vv), '__iter__')
# mlog.log('arr10.4')
# else:
# mlog.log('arr10.45')
# old__iter__.append(None)
# mlog.log('arr10.5')
if enable_debug:
breakpoint()
# mlog.log('arr10.6')
ar[sli] = v # THIS IS WHERE THE HORRIBLE UN-INTENDED UNNECCESARY ITERATING INTO OBJECTS IS HAPPENING ... SHOULD BE FIXED NOW!
# mlog.log('arr10.7')
# for i, iitteerr in enum(old__iter__):
# mlog.log('arr10.8')
# if iitteerr is not None:
# mlog.log('arr10.9')
# type(v[i]).__iter__ = iitteerr
for vv in v:
if hasattr(vv, '__npitr__') and (vv.__npitr__ == False):
vv.stop_np_iter = False
# mlog.log('arr11')
except File.NoLoadSupportException:
import traceback
traceback.print_exc()
# print('here1')
ar = mparray(ar, dtype=dtype)
# print('here2')
v.pop()
ar = ar[:-1]
assert len(v) == len(ar)
# print('here3')
return ar
else:
return mparray(v, dtype=dtype)
# with open('_remote_script.py', 'r') as pyscript:
# exec(pyscript.read().replace('return ', '_boot_remote_result='))
def exec_pycode_and_return(pycode):
exec(teleported['script'].replace('return ', '_dummy_var'))
# noinspection PyUnresolvedReferences
return _dummy_var
if teleported['folder_builder']:
fb = FolderBuilder()
exec(teleported['script'])
_boot_remote_result = fb.finish()
else:
exec(teleported['script'].replace('return ', '_boot_remote_result='))
File('_boot_remote_result_folder').mkdirs()
File('_boot_remote_result_folder').clear(silent=True)
from mlib.boot.lang import isinstsafe
if isinstsafe(_boot_remote_result, File):
_boot_remote_result: Optional[File]
_boot_remote_result.copy_into(File('_boot_remote_result_folder'))
_boot_remote_result = None
with open('_boot_remote_result.pkl', 'wb') as result:
pickle.dump(_boot_remote_result,
result,
protocol=pickle.HIGHEST_PROTOCOL)
boot.finish_dnn_remote()
# test change for rsync
def __init__(self, css):
if os.path.isfile(css):
js = File(css)
if isinstsafe(css, File):
css = css.read()
self._raw = css
def __enter__(self):
for o in self.objs:
if o is not None and isinstsafe(o, Muffleable):
o.muffle()
def __exit__(self, exc_type, exc_val, exc_tb):
for o in self.objs:
if o is not None and isinstsafe(o, Muffleable):
o.unmuffle()
