def savepeaks(self):
if self.algmode == 'LOAD':
log('skipping savepeaks for ' + str(self) +
' because data was loaded from file')
#
return False
# obsolete stuff
# if 'heartbeatevents_py' in self.peakfile.load().keys():
# del self.peakfile['heartbeatevents_py']
# del self.peakfile['heartbeatevents_mat']
export = {
'latency': arr(self.rPeaks) + 1,
'type': ['ECG' for _ in itr(self.rPeaks)],
'urevent': [i + 1 for i in itr(self.rPeaks)]
}
self.peakfile[self.alg.name()] = {
'alg': {
'name': self.alg.__class__.__name__,
'version': self.alg.version,
'tag': self.alg.versions()[self.alg.version]
},
'py': arr(self.rPeaks),
'mat': arr(self.rPeaks) + 1,
'heartbeatevents': export
}
self.peakfile['heartbeatevents'] = export
return True
def nandiv(a, v):
r = arr()
for i in itr(a):
if isnan(a[i]):
r = append(r, None)
else:
r = append(r, a[i] / v)
return r
def will_overlap(wbx, wby):
for iii in itr(drawn_xs):
twbx = drawn_xs[iii]
twby = drawn_ys[iii]
if ((min(wbx) <= min(twbx)) and
(max(wbx) >= min(twbx))) or ((max(wbx) >= max(twbx)) and
(min(wbx) <= max(twbx))):
# x overlaps
if ((min(wby) <= min(twby)) and
(max(wby) >= min(twby))) or ((max(wby) >= max(twby))
and
(min(wby) <= max(twby))):
return True
# return True
return False
def build(self):
top = [''] + listkeys(self.net_coefs)
orig_top = deepcopy(top)
if self.EXCLUDE_DARIUS_SMALLER_TRAIN_SIZES:
top[1:] = [s.split('_')[0] for s in top[1:]]
full = [top]
first = self.net_coefs[orig_top[1]]
left = [pattern_strings[k] for k in first.keys()]
cols = [left]
for coefs in self.net_coefs.values():
col = [sigfig(v, 3) for v in coefs.values()]
cols.append(col)
for i in itr(left):
row = [col[i] for col in cols]
full.append(row)
TableData(
data=full,
title=f"Correlation Coefficients Between {method_strings[self.method_name]} of model activations and Perfect Classifier Patterns",
title_size=70
).draw(builder=self, tags=self.tags + ['table', 'CorrCoefTable'])
def _log_plot(log_data, savetofile, checkpoint_lines: List[str], pipeline_sections):
import matplotlib.pyplot as plt # 1 SECOND IMPORT
important_text = []
important_time = []
for lin, file_line, t in log_data:
for cl in checkpoint_lines:
if cl in file_line:
important_text.append(cl)
important_time.append(t)
@dataclass
class LoggedPipelineSection:
start: float
end: float
thread: str
process: str
pid: str
label: str
# subsections: Optional[List] = None # does do anything yet
index: int = -1 # MUST SET LATER
source: Optional[int] = None # might set later
sourceSec: Optional = None
x: int = 0
# set later
time_amount: Optional[float] = None
time_rel: Optional[float] = None
time_amount_rel: Optional[float] = None
y_center: Optional[float] = None
color: str = 'orange'
loggedSections = []
for sec, v in listitems(pipeline_sections):
if v['start'] and v['end']:
loggedSections.append(LoggedPipelineSection(
start=v['start'],
end=v['end'],
label=sec,
thread=v['thread'],
pid=v['pid'],
process=v['process']
))
total = log_data[-1][2]
important_text = [shorten_str(s, 20) for s in important_text]
fig, axs = plt.subplots(nrows=1)
table_ax = axs
table_ax.set_axis_off()
important_time = [round(t, 2) for t in important_time]
if important_time:
table = table_ax.table(
cellText=[[str(t)] for t in important_time],
rowLabels=important_text,
colLabels=['time'],
rowColours=["palegreen"] * (len(important_text) + 1),
colColours=["palegreen"] * 2,
colWidths=[0.5, 0.5],
cellLoc='center',
loc='center'
)
table_ax.set_title('Important Logs', fontweight="bold")
time_amounts = []
time_rels = []
time_amount_rels = []
y_centers = []
last = 0
for t in important_time:
time_amounts.append(t - last)
time_rels.append(t / total)
time_amount_rels.append(time_amounts[-1] / total)
y_centers.append(time_rels[-1] - (time_amount_rels[-1] / 2))
last = t
sizes = important_time
loggedSectionsTotal = loggedSections[0].end - loggedSections[0].start
for i, sec in enum(loggedSections):
sec.time_amount = sec.end - sec.start
# no need for time_rel?
sec.time_amount_rel = sec.time_amount / loggedSectionsTotal
sec.y_center = (((sec.end - (sec.time_amount / 2)) - loggedSections[0].start) / loggedSectionsTotal)
sec.index = i
loggedSections[0].y_center = 0.5
for sec in loggedSections:
candidates = []
for secsec in loggedSections:
if sec.start > secsec.start:
candidates.append(secsec)
candidates2 = []
for cand in candidates:
if sec.end < cand.end:
candidates2.append(cand)
elif sec.start > secsec.end:
pass
# OVERLAP!
# assert sec.start > secsec.end # throws error if there is overlap but not nesting
if candidates2:
secsec = max(candidates2, key=lambda x: x.start)
sec.source = secsec.index
sec.sourceSec = secsec
def count_recurse(sec):
if sec.sourceSec:
return 1 + count_recurse(sec.sourceSec)
else:
return 0
for sec in loggedSections:
sec.x = count_recurse(sec)
colors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']
while len(colors) < len(sizes):
colors = colors + colors
colors = colors[:len(sizes)]
if important_text:
plt.savefig(savetofile.abspath)
plt.clf()
maxX = max([sec.x for sec in loggedSections])
xstep = normX = 1 / maxX
for sec in loggedSections:
sec.x = sec.x / maxX
labels = [sec.label for sec in loggedSections]
values = [sec.time_amount for sec in loggedSections if sec.source is not None]
if True:
for i in itr(labels):
if i > 0:
labels[i] = labels[i] + f' ({format_sec_dur(values[i - 1])})'
labels[0] = labels[0] + f' ({format_sec_dur(loggedSections[0].time_amount)})'
jitter_step = xstep / 10
keepJittering = True
while keepJittering:
for sec, secsec in unique_pairs(loggedSections):
if sec.x == secsec.x:
if sec.thread != secsec.thread or sec.process != secsec.process or sec.pid != secsec.pid:
secsec.color = 'blue'
secsec.x += jitter_step
break
keepJittering = False
import plotly.graph_objects as go
fig = go.Figure(data=[go.Sankey(
# arrangement="fixed", # no cutoff, but overlap
arrangement="snap", # no overlap, but cutoff
# arrangement = "perpendicular", # overlap and cutoff (less of both)
# arrangement="freeform",# both overlap and cutoff
node=dict(
pad=15,
thickness=20,
line=dict(color="black", width=0.5),
label=labels,
y=[sec.y_center for sec in loggedSections],
x=(arr([sec.x for sec in loggedSections]) * 1.0).tolist(),
color=[sec.color for sec in loggedSections]
),
link=dict(
source=[sec.source for sec in loggedSections if sec.source is not None],
target=list(range(1, (len(loggedSections)))),
value=values
))])
fig.update_layout(
font_size=20,
)
html = _get_fig(fig, full_html=True, include_plotlyjs=True)
File(savetofile).res_pre_ext("_sankey").resrepext('html').write(html)
def test_record(self, ei):
nnstate.CURRENT_PRED_MAP = self.train_data.class_label_map
nnstate.CURRENT_TRUE_MAP = self.test_data.class_label_map
ds = self.test_data.dataset(self.HEIGHT_WIDTH)
steps = self.test_data.num_steps
log('Recording(1)... (ims=$,steps=$)', len(self.test_data), steps)
net_mets.cmat = zeros(len(listkeys(nnstate.CURRENT_PRED_MAP)),
len(listkeys(nnstate.CURRENT_TRUE_MAP)))
inter_lay_name = self.net.layers[self.INTER_LAY].name
inter_output_model = self.tf.python.keras.models.Model(
self.net.input,
self.net.get_layer(index=self.INTER_LAY).output)
y_pred = arr(
self.net.predict(
ds,
steps=steps,
verbose=Verbose.PROGRESS_BAR,
use_multiprocessing=True,
workers=16,
))
log('done recording(1)')
if len(
y_pred.shape
) == 3: # GNET has 3 outputs, all identical I guess but not sure
y_pred = y_pred[2]
log('Recording(2)... (ims=$,steps=$)', len(self.test_data), steps)
inter_activations = arr(
inter_output_model.predict(ds,
steps=steps,
verbose=Verbose.PROGRESS_BAR,
use_multiprocessing=True,
workers=16))
log('done recording(2)')
x, _ = self.test_data.x(self)
y = self.test_data.y(self)
y_true = arr(y).flatten()
raw_images = x
raw_images2 = []
if len(x.shape) == 5:
for batch in raw_images:
for im in batch:
raw_images2.append(im)
else:
raw_images2 = raw_images
raw_images = arr(raw_images2)
raw_images2 = []
for i in itr(raw_images):
raw_images2.append(raw_images[i].flatten())
raw_images = arr(raw_images2)
inter_shape = inter_activations.shape
inter_activations = np.reshape(inter_activations, (inter_shape[0], -1))
BLOCK_LEN = 10 # I'm writing this bc I think it was always 10 back when I ran this code
TEST_CLASS_MAP = nnstate.CURRENT_TRUE_MAP
clas_set = ClassSet(
[Class(name=k, index=v) for k, v in TEST_CLASS_MAP.items()])
def run_and_save_rsa(nam, mat1, layer_name=None, layer_i=None):
index_to_cn = {v: k for k, v in TEST_CLASS_MAP.items()}
feature_matrix = FeatureMatrix(
mat1, clas_set,
[Class(index_to_cn[iii], iii) for iii, yt in enum(y_true)])
feature_matrix.sort_by_class_name()
fd = feature_matrix.compare(rsa_norm).image_plot()
tit = f'L2-{nam}'
fd.title = f'{tit} ({nnstate.FLAGS.arch}{nnstate.FLAGS.ntrain}E{ei + 1})'
if nam == 'Inter':
fd.title = f'{fd.title}(Layer{layer_i}:{layer_name})'
save_dnn_data(fd, tit, f'CM{ei + 1}', 'mfig')
run_and_save_rsa('Output', y_pred, layer_name='Output', layer_i='-1')
run_and_save_rsa('Inter',
inter_activations,
layer_name=inter_lay_name,
layer_i=self.INTER_LAY)
run_and_save_rsa('Raw', raw_images)
for met in net_mets.METS_TO_USE():
met(y_true, y_pred)
log('done recording.')
def jsonReprCP(o):
import numpy as np
# log('jsonReprCP of a ' + str(type(o)))
if isinstsafe(o, np.ndarray):
o = o.tolist()
if not isitr(o) and isnan(o):
return None
if o == np.inf or o == -np.inf:
return str(o)
if isinstance(o, str) or isinstance(o, float) or isinstance(
o, int) or o is None:
return o
if isinstance(o, np.int64):
return int(o)
if isinstance(o, np.float32):
return float(o)
if isitr(o) and not isinstance(o, str):
cp = []
for vv in o:
cp.append(jsonReprCP(vv))
return cp
import copy
cp = copy.deepcopy(o)
# debug_on = False
if hasattr(o, 'item_type') and o.item_type == 'violin':
debug_on = True
# import mdb; mdb.remote_breakpoint()
for v in cp.__dict__:
debug(f'V:{v}')
# if debug_on:
# breakpoint()
val = getattr(cp, v)
if isitr(val) and not isinstance(val, str):
if isinstance(val, np.ndarray):
setattr(cp, v, jsonReprCP(val.tolist()))
else:
newval = []
for vv in val:
newval.append(jsonReprCP(vv))
setattr(cp, v, newval)
if not isitr(val) and (val == np.inf or val == -np.inf):
setattr(cp, v, str(val))
if isinstance(val, np.int64):
setattr(cp, v, int(val))
import pandas
if not isitr(val) and pandas.isnull(val):
setattr(cp, v, None)
if isinstance(val, JsonSerializable):
setattr(cp, v, jsonReprCP(val).__dict__)
if isitr(val) and not isempty(val) and isinstance(
val[0], JsonSerializable):
newval = []
for i in itr(val):
newval.append(jsonReprCP(val[i]).__dict__)
setattr(cp, v, newval)
# if debug_on:
# breakpoint()
# if hasattr(o, 'item_type') and o.item_type == 'violin':
# breakpoint()
# import mdb; mdb.remote_breakpoint()
return cp
def find_local_maxima(r_indices,
ecg_flt,
FIX_WIDTH=100,
CROP_FIRST_LAST=False,
AUTO=True,
ABS=True):
# (searchback)
if isint(FIX_WIDTH):
FIX_WIDTH = (FIX_WIDTH, FIX_WIDTH)
if ABS:
myabs = abs
else:
myabs = lambda x: x
r_indices = flat(r_indices)
y = []
for i in itr(r_indices):
y.append(ecg_flt[r_indices[i]])
newlats = r_indices
marks = []
if AUTO:
log('automatically fixing all heartbeats')
with Progress(len(r_indices), 'searching back on', 'marks') as prog:
for i in itr(r_indices):
if CROP_FIRST_LAST and (i == 0 or i == len(r_indices) - 1):
continue
fix_width_back = min(FIX_WIDTH[0], r_indices[0])
fix_width_forward = FIX_WIDTH[1]
the_lat = assert_int(r_indices[i])
mn = the_lat - fix_width_back
mx = the_lat + 1 + fix_width_forward
if len(ecg_flt) >= mx:
snippet = ecg_flt[mn:mx]
else:
snippet = ecg_flt[mn:len(ecg_flt)]
[M, I] = mymax(myabs(snippet))
fixed_lat = the_lat + I - fix_width_back
if M != ecg_flt[the_lat]:
if not AUTO:
log('i=' + num2str(i) + '/' + num2str(len(r_indices)))
plt.plot(ecg_flt)
plt.scatter([r_indices[i], fixed_lat], [y[i], M], [],
[[1, 0, 0], [0, 0, 1]])
plt.xlim([mn, mx])
plt.show()
s = input('fix?(y/n)')
else:
s = 'y'
if strcmp(s, 'y'):
if not AUTO:
disp('fixing')
newlats[i] = fixed_lat
marks = [marks, the_lat]
else:
disp('skipping')
prog.tick()
return newlats
def analyze_exp_group(eg: DNN_ExperimentGroup, cfg):
eg.compile_folder.deleteIfExists()
eg.metadata.copy_into(eg.compile_folder)
ARCH_LABELS = listmap(__['label'], eg.metadata['archs'])
NTRAINS = eg.metadata['ntrainims']
NEPOCHS = eg.metadata['nepochs']
[
a.during_compile(eg) for a in ANALYSES(mode=AnalysisMode.PIPELINE)
if a.should_run(cfg)
]
experiments = experiments_from_folder(eg.folder)
random_exp = experiments[0]
TrainTable = FinalResult(2,
'test/Matthews_Correlation_Coefficient.mfig',
data_exists=random_exp.folder[f'test'].exists,
is_table=True,
rows=ARCH_LABELS,
cols=NTRAINS)
random_exp.folder['log.pkl'].copy_into(eg.compile_folder)
def maybe_avg_result(pre, nepochs, is_table=False, dims=2, suf=None):
return AverageResult(
dims,
f'{pre}/CM{nepochs}.mfig' if suf is None else f'{pre}/{suf}',
data_exists=random_exp.folder[pre].exists,
is_table=is_table)
results_to_compile = []
for ni, ntrain in enum(NTRAINS):
MCC = maybe_avg_result('test',
None,
dims=1,
suf='Matthews_Correlation_Coefficient.mfig')
for ai, arch in enum(ARCH_LABELS):
if cfg.salience:
results_to_compile = [TrainTable]
else:
results_to_compile = [TrainTable, MCC]
if random_exp.folder['L2-Output'].exists:
maybe_avg_result(f'L2-Output', NEPOCHS)
maybe_avg_result(f'L2-Inter', NEPOCHS)
maybe_avg_result(f'L2-Raw', NEPOCHS)
if not cfg.salience:
results_to_compile.append(
maybe_avg_result(f'val', NEPOCHS, is_table=True))
results_to_compile = [
r for r in results_to_compile if r is not None
]
for exp in experiments.filtered(
lambda e: e.arch == arch and e.ntrain == ntrain, ):
for res in results_to_compile:
try:
res.append(res.exp_data(exp), (ai, ni, 0),
is_GNET=arch == 'GNET')
except:
breakpoint()
for res in results_to_compile:
if not res.data_exists: continue
if res.j is None:
log('about to breakpoint')
breakpoint()
else:
log('res.j is not none, so no breakpoint')
for vis in res.j.viss:
vis.make = True
if res.dims == 1:
LINE_INDEX = -1
avg = np.mean(res.data, axis=0).tolist()
res.data = arr2d()
res.j.viss.append(copy.deepcopy(res.template))
res.j.viss[LINE_INDEX].make = True
res.j.viss[LINE_INDEX].y = avg
res.j.viss[LINE_INDEX].item_colors = CONTRAST_COLORS[ai]
for v in itr(res.j.viss):
res.j.viss[v].title = f'MCC(nTrainIms={ntrain})'
res.j.viss[LINE_INDEX].y_label = arch
elif res.dims == 2:
if isinstance(res, AverageResult):
avg = np.mean(res.data, axis=2)
else:
avg = res.data[:, :, 0]
res.j.viss[0].confuse_max = flatmax(avg)
# res.j.viss[0].title_size = 30
if res.is_table:
avg = np.concatenate((res.row_headers[1:], avg),
axis=1)
avg = np.concatenate((res.col_headers, avg), axis=0)
if isinstance(res, AverageResult):
res.j.viss[0].data = avg.tolist()
res.j.viss[0].title = res.j.viss[0].title.replace(
" ", f'{ntrain} ', 1)
eg.compile_exp_res_folder[
f'{arch}_{ntrain}__{res.suffix.replace("/", "_")}'].save(
res.j)
elif ai == len(ARCH_LABELS) - 1 and ni == len(NTRAINS) - 1:
res.j.viss = [
ConfusionMatrix(data=avg.tolist(),
title="Final Training MCCs",
confuse_min=0,
confuse_max=1,
headers_included=True,
make=True,
side_header_label='Architecture',
top_header_label='#Train Images')
]
eg.compile_exp_res_folder[
f'Final_Train_MCC.mfig'] = res.j
for res in [
mcc for mcc in results_to_compile
if mcc.dims == 1 and mcc.data_exists
]:
eg.compile_exp_res_folder[StringExtension(res.suffix[1:]).r({
"test/":
"",
".":
f"_{ntrain}."
})].save(res.j)
def _fun(i): # cannot be lambda?
return [(i, j, fun.fun(data[i, :], data[j, :])) for j in itr(data)]
def compare(self, fun: Type[Correlation], GPU=False):
special_confuse_mat = zeros(len(self.data), len(self.data))
if (fun == PearsonCorrelation) and any([min(x) == max(x) for x in self.data]):
raise MathFail
# # Pearson's Correlation Coefficient fails if
# # two arrays are commpared that have a zero standard deviation product (divide by zero)
# # Using an if statement above, I should prevent this
data = self.data # pleasework
def _fun(i): # cannot be lambda?
return [(i, j, fun.fun(data[i, :], data[j, :])) for j in itr(data)]
def _fun_tf(data): # cannot be lambda?
return fun.fun_tf(data)
MULTIPROCESS = False
from pathos.multiprocessing import ProcessPool
if islinux() and MULTIPROCESS:
# slower than GPU
# BUGGY
# not optimized
with ProcessPool() as p:
# if islinux():
# mapid = randrange(0,10000)
# print(f'starting map {mapid}')
r = p.map(_fun, itr(self.data))
for results in r:
for rr in results:
special_confuse_mat[rr[0], rr[1]] = rr[2]
elif islinux() and GPU:
import tensorflow as tf
special_confuse_mat = tf.zeros((len(self.data), len(self.data)))
with tf.device('/GPU:0'):
special_confuse_mat = _fun_tf(self.data).numpy()
# results[net] = rsa.numpy()
# tfdata = tf.convert_to_tensor(self.data).cuda()
else:
r = listmap(_fun, itr(self.data))
for results in r:
for rr in results:
special_confuse_mat[rr[0], rr[1]] = rr[2]
return ComparisonMatrix(
data=nan_above_eye(naneye(special_confuse_mat)),
method_used=fun.__name__,
ground_truth=self.ground_truth,
class_set=self.class_set
)
def FormatWLInput(ss):
f_s = ''
OPEN_Ps = []
OPEN_Cs = []
OPEN_Bs = []
OPENS = ['(', '{', '[']
CLOSES = [')', '}', ']']
in_str = False
fragments = []
for ii, cc in enum(ss):
if in_str:
in_str = cc != '"'
else:
in_str = cc == '"'
if cc == '(':
OPEN_Ps.append(ii)
elif cc == '{':
OPEN_Cs.append(ii)
elif cc == '[':
OPEN_Bs.append(ii)
elif cc == ',':
pass
elif cc == ')':
fragments.append((OPEN_Ps.pop(), ii))
elif cc == '}':
fragments.append((OPEN_Cs.pop(), ii))
elif cc == ']':
fragments.append((OPEN_Bs.pop(), ii))
elif cc == ';':
pass
for ii, f in enum(fragments):
fragments[ii] = list(f) + [f[1] - f[0]]
frag_for_i = []
for ii in itr(ss):
added = False
for f in fragments:
if f[0] == ii or f[1] == ii:
frag_for_i.append(f)
added = True
break
if not added:
frag_for_i.append(None)
in_str = False
opening = False
closed = False
THRESHOLD = 10
for ii, cc in enum(ss):
newline_after = False
newline_before = False
frag = None
if in_str:
in_str = cc != '"'
else:
in_str = cc == '"'
if cc in OPENS:
opening = True
if cc in [',', ';']:
newline_after = True
frag = (0, 0, 100)
if opening and cc not in OPENS:
newline_before = True
opening = False
frag = frag_for_i[ii - 1]
if closed and cc not in CLOSES:
closed = False
if not closed and cc in CLOSES:
newline_before = True
frag = frag_for_i[ii]
closed = True
if newline_before and frag[2] >= THRESHOLD:
f_s += '\n'
f_s += cc
if newline_after and frag[2] >= THRESHOLD:
f_s += '\n'
return f_s
def table(cls, fd, force_wolf=False, debug=False):
from mlib.wolf.wolf_lang import wlexpr
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
wolf = 'Wolf' in cls.__name__ or force_wolf
if wolf:
from mlib.wolf.wolf_figs import addHeaderLabels, LinePlotGrid, OneWayOfShowingARaster
if cls == MPLFigsBackend:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
if BLACK_FIGS:
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.ax = cls.fig.add_subplot(111)
cls.ax.axis("off")
cls.tabl = None
data = fd.data
backgrounds = None
if fd.confuse:
low = fd.confuse_min
high = fd.confuse_max
# scaleBits = []
# for i in range(1, 21):
# scaleBits += [[[0, 0, i / 21]]]
scaleBits = JET().tolist()
show_nums = fd.show_nums
from copy import deepcopy
backgrounds = deepcopy(data)
for rrr in itr(data):
for c in itr(data[rrr]):
if BLACK_FIGS:
backgrounds[rrr][c] = cls.color(0, 0, 0)
else:
backgrounds[rrr][c] = cls.color(1, 1, 1) # 256?
if isnan(data[rrr][c]):
if wolf:
data[rrr][c] = ''
backgrounds[rrr][c] = cls.none()
else:
data[rrr][c] = [0, 0, 0]
# if BLACK_FIGS:
backgrounds[rrr][c] = cls.color(0, 0, 0)
# else:
# backgrounds[rrr][c] = cls.color(255, 255, 255) # 256?
elif not isstr(data[rrr][c]):
dat = data[rrr][c]
if isnan(dat): # wait! this dealt with above
b = None
elif high != 0:
if fd.y_log_scale:
b = ((dat**2) / (high**2))
else:
# try:
b = (dat / high)
if cls.debug: # and data == 0:
breakpoint()
# except:
else:
if fd.y_log_scale:
# b = np.log10(b)
b = dat**2
else:
b = dat
if show_nums:
if b is None:
data[rrr][c] = 'NaN'
else:
data[rrr][c] = sigfig(dat, 2)
else:
# data[rrr][c] = [0, 0, b]
# try:
if b is None: # NaN
breakpoint()
data[rrr][c]
else:
# data[rrr][c] = JET[round(b * 256) - 1].tolist() # causes zeros to show wrong!
data[rrr][c] = JET()[round(b * 255)].tolist()
# except:
if (fd.headers_included and rrr > 0
and c > 0) or not fd.headers_included:
backgrounds[rrr][c] = cls.color(0, 0, b)
block_len = fd.block_len
if block_len is not None and fd.headers_included is False:
divs = [[], []]
for i in range(len(data)):
if i == 1 or i % block_len == 0:
divs[0] += [True]
divs[1] += [True]
else:
divs[0] += [False]
divs[1] += [False]
if not fd.confuse or fd.headers_included:
if cls == MPLFigsBackend and not fd.headers_included:
# breakpoint()
tbl = cls.ax.table(
cellText=data,
# rowLabels=rows,
# rowColours=colors,
# colLabels=columns,
fontsize=fd.fontsize,
loc='center')
tbl.auto_set_font_size(False)
tbl.set_fontsize(fd.fontsize)
tbl.scale(1, (fd.fontsize + 10) / 10)
# tbl.auto_set_column_width([0])
tbl.auto_set_column_width(list(range(len(data[0]))))
else:
breakpoint()
for ri, row in enumerate(data):
for ci, el in enumerate(row):
if cls == MPLFigsBackend:
if cls.tabl is None:
cls.tabl = cls.ax.table([[1]], loc='center')
width = 1 / len(data[0])
height = 1 / len(data)
if ri == 0:
height = 0.05
if ci == 0:
width = 0.1
cell = cls.tabl.add_cell(
ri,
ci,
width,
height,
loc="center",
text=str(el) if el != 'Non' else "",
# text="TEST_TEXT",
# facecolor='green'
**({
'facecolor': backgrounds[ri][ci]
} if backgrounds is not None else {}))
cell.get_text().set_fontsize(20)
if len(data) > 5:
cell.get_text().set_fontsize(10)
if BLACK_FIGS:
cell.get_text().set_color('white')
else:
data[ri][ci] = cls.tableItem(
el, backgrounds[ri][ci])
if fd.top_header_label is not None or fd.side_header_label is not None:
if wolf:
data = addHeaderLabels(data, fd.top_header_label,
fd.side_header_label).tolist()
else:
cls.tabl.auto_set_font_size(False)
h = cls.tabl.get_celld()[(0, 0)].get_height()
w = cls.tabl.get_celld()[(0, 0)].get_width()
# Create an additional Header
# weird = "Header Header Header Header"
weird = fd.top_header_label * 4
weird = fd.top_header_label
header = [
cls.tabl.add_cell(
-1,
pos,
w,
h,
loc="center", # fontsize=40.0 #facecolor="red",
) for pos in range(1,
len(data[0]) + 1)
]
if len(header) > 2:
for idx, head in enum(header):
if idx == 0:
# head.visible_edges = "TBL"
head.visible_edges = ""
elif idx == len(header) - 1:
# head.visible_edges = "TBR"
head.visible_edges = ""
else:
# head.visible_edges = 'TB'
head.visible_edges = ""
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
elif len(header) == 2:
header[0].visible_edges = 'TBL'
header[1].visible_edges = 'TBR'
header[1].get_text().set_text(weird)
else:
header[0].visible_edges = 'TBLR'
header[0].get_text().set_text(weird)
# Create an additional Header
weird = fd.side_header_label * 4
weird = fd.side_header_label
header = [
cls.tabl.add_cell(pos,
-1,
w,
h,
loc="center",
facecolor="none")
for pos in range(0,
len(data) + 1)
]
if len(header) > 2:
for idx, head in enum(header):
if idx == 0:
head.visible_edges = "LTR"
head.visible_edges = ""
elif idx == len(header) - 1:
head.visible_edges = "LRB"
head.visible_edges = ""
else:
head.visible_edges = 'LR'
head.visible_edges = ""
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
# header[1].set_rotation(90.0)
elif len(header) == 2:
header[0].visible_edges = 'TLR'
header[1].visible_edges = 'BLR'
header[1].get_text().set_text(weird)
header[1].set_fontsize(40.0)
# header[1].set_rotation(90.0)
else:
header[0].visible_edges = 'TBLR'
header[0].get_text().set_text(weird)
header[0].set_fontsize(40.0)
# header[0].set_rotation(90.0)
if cls != MPLFigsBackend:
insets = [
Inset(
Rasterize(Grid(
data,
Dividers(False),
), RasterSize(), ImageSize(), Background()))
]
if fd.confuse and fd.block_len is not None and fd.block_len > 1:
if fd.confuse_is_identical:
for rrr in itr(data):
for c in itr(data[0]):
if c > rrr:
if BLACK_FIGS:
data[rrr][c] = [0, 0, 0]
else:
data[rrr][c] = [1, 1, 1] # 256?
if cls != MPLFigsBackend:
scale = Graphics(
[
Raster(scaleBits),
Inset(Text(round(low), fontSize=30), [0.5, -1]),
Inset(Text(round(high), fontSize=30), [0.5, 21]),
],
ImagePadding([[75, 75], [20, 20]]),
)
else:
# create an axes on the right side of ax. The width of cax will be 5%
# of ax and the padding between cax and ax will be fixed at 0.05 inch.
from mpl_toolkits.axes_grid1 import make_axes_locatable # 1 FULL SECOND IMPORT
divider = make_axes_locatable(cls.ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
# cmap = matplotlib.colors.Colormap('name', N=256)
# cmap = LinearSegmentedColormap.from_list(
# 'bluemap', li(scaleBits).reshape(20, 3), N=20)
from matplotlib.colors import LinearSegmentedColormap # 1 FULL SECOND IMPORT
cmap = LinearSegmentedColormap.from_list(
'jet',
li(scaleBits) # .reshape(20, 3)
,
N=len(scaleBits) # 20
)
import matplotlib
sm = matplotlib.cm.ScalarMappable(norm=None, cmap=cmap)
cbar = cls.fig.colorbar(
sm,
cax=cax,
orientation='vertical',
ticks=np.linspace(
# low,
0,
# high,
1,
num=4))
base_scale_ticks = np.linspace(low, high, num=4)
if fd.y_log_scale:
base_scale_ticks = [bst**2 for bst in base_scale_ticks]
cbar.ax.set_yticklabels(
[sigfig(n, 3) for n in base_scale_ticks])
gl = len(data)
nt = len(fd.row_headers)
line_values = np.linspace(fd.block_len, fd.block_len * nt, nt)
half = fd.block_len / 2
labellocs_labels = ziplist(line_values - half, fd.row_headers)
# ticks start from the bottom but I want these to start from the top
# labellocs_labels.reverse()
if wolf:
gridlines = LinePlotGrid(line_values,
triangle=fd.confuse_is_identical)
else:
# gl = line_values[-1]
listpoints = []
for i in line_values:
i = i - 0.5
if fd.confuse_is_identical:
listpoints += [[[i, gl - 0.5], [i, i]]]
listpoints += [[[i, i], [-0.5, i]]]
else:
listpoints += [[[i, -0.5], [i, gl - 0.5]]]
listpoints += [[[-0.5, i], [gl - 0.5, i]]]
listpoints = arr(listpoints)
for sub in listpoints:
# cls.ax.line(sub[:, 0], sub[:, 1], 'y--')
# cls.ax.plot(sub[:, 0], sub[:, 1], 'y--')
cls.ax.plot(sub[:, 0], sub[:, 1], 'k--')
# rasters start from the bottom but I want this to start from the top
if wolf:
rast = OneWayOfShowingARaster(Raster(reversed(data)), gl)
else:
cls.ax.imshow(list(data))
x_ticks = []
xt_mpl_t = []
xt_mpl_l = []
y_ticks = []
yt_mpl_t = []
yt_mpl_l = []
for t in labellocs_labels:
if wolf:
x_ticks += [
Text(
t[1],
coords=[t[0] / gl, -.02],
direction=[1, 0],
fontSize=DEFAULT_TICK_SIZE,
offset=[0, 0],
)
]
xt_mpl_t += [t[0] / gl]
xt_mpl_l += [t[1]]
for t in labellocs_labels:
if wolf:
y_ticks += [
Text(
t[1],
# y ticks need to be reversed
coords=[-.01, 1 - (t[0] / gl)],
direction=[1, 0],
fontSize=DEFAULT_TICK_SIZE,
offset=[1, 0],
)
]
# y ticks not reversed for mpl?
yt_mpl_t += [(t[0] / gl)]
yt_mpl_l += [t[1]]
if wolf:
insets = [
Inset(obj=Rasterize(scale),
pos=[1.2, 0],
opos=[Center, Bottom]),
Inset(obj=Rasterize(
rast,
ImageResolution(),
),
opos=[Left, Bottom]),
Inset(
obj=Rasterize(gridlines, ImageResolution(),
Background(wlexpr('None'))),
opos=[Left, Bottom],
background=Background(
# wl.Red
wlexpr('None')))
]
[insets.extend(ticks) for ticks in zip(x_ticks, y_ticks)]
insets += [
Inset(Rasterize(
Text(fd.title,
fontSize=(40 if fd.headers_included else 20)
if fd.title_size is None else fd.title_size),
Background(wlexpr('None'))),
scale=(1, 1),
pos=Scaled([0.5, 2]),
background=Background(wlexpr('None')))
]
rrr = Graphics(insets)
else:
title_obj = cls.ax.set_title(fd.title,
fontsize=fd.title_size / 3)
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
cls.ax.set_xticks(arr(xt_mpl_t) * gl)
cls.ax.set_xticklabels(xt_mpl_l, rotation=90)
# cls.ax.xticks(rotation=90)
cls.ax.set_yticks(arr(yt_mpl_t) * gl)
cls.ax.set_yticklabels(yt_mpl_l)
cax.axis(True)
cax.spines['left'].set_color(text_color)
cax.spines['bottom'].set_color(text_color)
cax.spines['top'].set_color(text_color)
cax.spines['right'].set_color(text_color)
cax.xaxis.label.set_color(text_color)
cax.yaxis.label.set_color(text_color)
cax.tick_params(axis='x', colors=text_color)
cax.tick_params(axis='y', colors=text_color)
# cax.set_xticks(li(xt_mpl_t) * gl)
# cax.set_xticklabels(xt_mpl_l, rotation=90)
# cls.ax.xticks(rotation=90)
# cax.set_yticks(li(yt_mpl_t) * gl)
# cax.set_yticklabels(yt_mpl_l)
if not wolf and (fd.confuse and fd.block_len is not None
and fd.block_len > 1) == False:
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size / 3)
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
plt.setp(title_obj, color=text_color)
if wolf:
return rrr
def violin(cls, fd):
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
maxY = None if fd.maxY is None or fd.maxY == 'inf' or not isreal(
fd.maxY) else float(fd.maxY)
minY = None if fd.minY is None or fd.minY == 'inf' or not isreal(
fd.minY) else float(fd.minY)
if maxY != None and minY != None:
diff = maxY - minY
pad = diff * (fd.y_pad_percent / 100)
maxY = maxY + pad
minY = minY - pad
if cls.fig is None:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
cls.ax = cls.fig.add_subplot(111)
try:
cls.ax.violinplot(fd.y, showmeans=True)
except FloatingPointError as e: # weird underflow error
if str(e) == 'underflow encountered in exp':
progress(
'funny violinplot error occurred again. rebuild locally')
cls.ax.plot([1, 2, 3])
return
else:
import traceback
traceback.print_exc()
breakpoint()
x = range(1, len(fd.x) + 1)
cls.ax.set_xticks(x)
cls.ax.set_xticklabels(fd.x)
# fd.x,
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size)
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
cls.ax.set_ylabel(fd.y_label)
cls.ax.set_xlabel(fd.x_label)
drawn_xs = []
drawn_ys = []
def will_overlap(wbx, wby):
for iii in itr(drawn_xs):
twbx = drawn_xs[iii]
twby = drawn_ys[iii]
if ((min(wbx) <= min(twbx)) and
(max(wbx) >= min(twbx))) or ((max(wbx) >= max(twbx)) and
(min(wbx) <= max(twbx))):
# x overlaps
if ((min(wby) <= min(twby)) and
(max(wby) >= min(twby))) or ((max(wby) >= max(twby))
and
(min(wby) <= max(twby))):
return True
# return True
return False
from scipy import stats
the_y = [np.mean(it) for it in fd.y]
for i in itr(fd.y):
for ii in itr(fd.y):
if i <= ii: continue
# breakpoint()
dh = .05 # default,
barh = .05 # default
will_be_y1 = max(the_y[i], the_y[ii]) + dh
will_be_y2 = will_be_y1 + barh
will_be_x = [i, ii]
will_be_y = [will_be_y1, will_be_y2]
while will_overlap(will_be_x, will_be_y):
dh += .05
will_be_y1 = max(the_y[i], the_y[ii]) + dh
will_be_y2 = will_be_y1 + barh
will_be_x = [i, ii]
will_be_y = [will_be_y1, will_be_y2]
drawn_xs.append(will_be_x)
drawn_ys.append(will_be_y)
cls.barplot_annotate_brackets(
i,
ii,
stats.ttest_ind(fd.y[i], fd.y[ii],
alternative='two-sided')[1],
x, # list(itr(fd.y)), #fd.x
the_y,
dh=dh,
barh=barh)
@classmethod
def bar(cls, fd):
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
maxY = None if fd.maxY is None or fd.maxY == 'inf' or not isreal(
fd.maxY) else float(fd.maxY)
minY = None if fd.minY is None or fd.minY == 'inf' or not isreal(
fd.minY) else float(fd.minY)
# maxX = None if fd.maxX is None or fd.maxX == '-inf' or not isreal(fd.maxX) else float(fd.maxX)
# minX = None if fd.minX is None or fd.minX == 'inf' or not isreal(fd.minX) else float(fd.minX)
if maxY != None and minY != None:
diff = maxY - minY
pad = diff * (fd.y_pad_percent / 100)
maxY = maxY + pad
minY = minY - pad
if cls.fig is None:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
cls.ax = cls.fig.add_subplot(111)
cls.ax.bar(fd.x,
fd.y,
color=listmap(cls.color, fd.item_colors),
yerr=fd.err)
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size)
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
@classmethod
def none(cls):
pass
@classmethod
def color(cls, *rgb):
rgb = arr(rgb) * 128
# if len(rgb.shape) == 3:
# if rgb.shape[0] == 1:
# rgb = rgb[0]
return '#%02x%02x%02x' % tuple(ints(rgb).tolist())
@classmethod
@abstractmethod
def tableItem(cls, o, background):
err('unused')