def end(self):
assert current_thread() == self.thread
assert current_process() == self.process
t = mstoc()
try:
assert get_manager().PIPELINE_SECTIONS[self.value]["start"] is not None, f'never started {self.value}'
except:
breakpoint()
assert get_manager().PIPELINE_SECTIONS[self.value]["end"] is None
if self.sankey:
get_manager().PIPELINE_SECTIONS[self.value]["end"] = t
s = ' done'
if self.stopwatch:
dur = get_manager().PIPELINE_SECTIONS[self.value]["end"] - \
get_manager().PIPELINE_SECTIONS[self.value][
"start"]
s += f' in {sec_or_ms_str(dur)}'
get_manager().PROCESS_STATUSES[self.process.name] = s
get_manager().THREAD_STATUSES[self.thread.name] = s
if self.log:
from mlib.parallel import isMainThreadAndProcess, print_parallel_status
if isMainThreadAndProcess():
progress(s, one_line_key=self.key)
else:
print_parallel_status(real_progress=True)
def finish_process(self, p, SW):
# if SW is None or SW.interact:
import mlib.err
p.expect(
'LETS_TRY_TO_HAVE_A_CLEAN_CONSOLE'
) # hopefully flush out my python command so I dont see it in my console!
progress('python script must have started')
def kill_on_finish_str(s, p):
log(s, silent=True) # LOGPIE
if ContainerBashScript.FINISH_STR in s or mlib.err.ERROR_EXIT_STR in s:
p.close()
stopper = p.log_to_stdout(
# fun=kill_on_finish_str,
# just_fun=True,
o=p,
print_not_log=True,
stop_on=[ContainerBashScript.FINISH_STR, mlib.err.ERROR_EXIT_STR])
# p.log_to_stdout(o=p, print_not_log=True)
import mdb
# noinspection PyUnresolvedReferences
i = p.expect([
ContainerBashScript.FINISH_STR, p.PROMPT, pexpect.EOF, '(pdb)',
mdb.INTERACT_STR, mlib.err.ERROR_EXIT_STR
])
if i == 0:
p.close()
elif i in [1, 3, 4]:
stopper.stop()
p.interact()
err("PDB")
elif i == 2:
p.close()
err('EOF')
elif i == 5:
raise RelayException('got error in OM')
# while p.alive():
# inp = input_while(lambda: p.alive())
# p.sendline(inp)
# p.interact()
# else:
# old
# p.pipe_and_close_on(ContainerBashScript.FINISH_STR, close_fun=lambda p: p.sendline('exit'))
if self.host.n is not None: self.host.tick_job_finish()
def start(self):
assert current_thread() == self.thread
assert current_process() == self.process
t = mstoc()
if 'L2' in self.value:
debug(f"IT IS A1 {type(get_manager().PIPELINE_SECTIONS)}")
debug(f"IT IS A2 {type(get_manager().PIPELINE_SECTIONS['Startup'])}")
debug(f"IT IS A3 {type(get_manager().PIPELINE_SECTIONS['Startup']['start'])}")
debug(f"IT IS B1 {type(get_manager().PIPELINE_SECTIONS)}")
debug(f"IT IS B2 {type(get_manager().PIPELINE_SECTIONS[self.value])}")
debug(f"IT IS AB3 {type(get_manager().PIPELINE_SECTIONS[self.value]['start'])}")
assert get_manager().PIPELINE_SECTIONS[self.value]["start"] is None
get_manager().PIPELINE_SECTIONS[self.value]["start"] = t
# print(f'on starting {self.value}, PIPELINE_SECTIONS.keys() is {[k for k in get_manager().PIPELINE_SECTIONS.keys()]}, and _MULTIPROCESS_MODE is {_MULTIPROCESS_MODE}')
# print(f'also, here is my dict: {dict(get_manager().PIPELINE_SECTIONS[self.value])}')
# PIPELINE_SECTION_FILE.save(PIPELINE_SECTIONS)
s = f'{self.value}... '
get_manager().PROCESS_STATUSES[self.process.name] = s
get_manager().THREAD_STATUSES[self.thread.name] = s
if self.log:
from mlib.parallel import isMainThreadAndProcess, print_parallel_status
if isMainThreadAndProcess():
self.key = progress(s, attempt_one_line=True)
else:
print_parallel_status(real_progress=True)
def makeAllPlots(cls,
figDats,
overwrite,
force=False,
debug=False,
silent=True):
will_maybe_do_more = []
for fd in figDats:
viss = fd.dataFile.loado(silent=True).fixInfs().viss
makes = [v for v in viss if force or v.make]
if makes:
info('showing...')
if not silent:
progress(f'exporting {fd.dataFile} ...')
will_maybe_do_more += [
cls.export_fd(makes, fd, overwrite, debug=debug)
]
return will_maybe_do_more
def sigfig(number, ndigits):
testnumber = abs(number)
if testnumber == 0:
return 0
if testnumber < 1e-305:
progress(
f'sigfig got an astronomically low number ({testnumber}) which will cause a divide by zero error. Returning zero.'
)
return 0
ref = 1
roundn = ndigits
while testnumber >= ref:
roundn -= 1
ref = ref * 10
ref = 0.1
while testnumber <= ref:
roundn += 1
ref = ref / 10
return round(number, roundn)
def get(self,
*files,
destination=None,
merge=None,
overwrite=False,
verbose=False):
if len(files) > 0:
self.host.get(*files, project_name=self.name, verbose=verbose)
# breakpoint()
# files = (File(File(f).name) for f in files) # gen expression! YOU CAN ONLY ITERATE IT ONCE!
files = [File(File(f).name) for f in files]
all_mb_files = []
for f in files:
mb_files = [(fi, si) for fi, si in f.files_recursive().map(
lambda fi: (fi, fi.size())) if si.mb() >= 1]
all_mb_files.extend(mb_files)
size = f.size(recursive=True)
progress(f'Downloaded file {f.name} is {size}')
the_len = len(all_mb_files)
progress(f'{the_len} files were 1 MB or larger' +
(':' if the_len else ''))
for mb_file, size in sorted(all_mb_files,
key=lambda t: t[1].num_bytes):
progress(f'\t{size}\t{mb_file.rel_to(pwd())}')
if destination is not None and Folder(
destination).abspath != Folder(pwd()).abspath:
Folder(destination).mkdirs()
for f in files:
f.moveinto(destination, overwrite=overwrite)
if merge is not None:
for f in files:
f.mergeinto(Folder(merge), overwrite=overwrite)
f.deleteIfExists()
def _darius_and_shobhita_acts(
N_PER_CLASS,
ACT_SIZE,
SHOBHITA=True,
EXCLUDE_DARIUS_SMALLER_TRAIN_SIZES=True
):
T_SIZES = [
25,
50,
100,
150,
200
] # 6 epochs for all, and only 70% for training
if EXCLUDE_DARIUS_SMALLER_TRAIN_SIZES:
T_SIZES = [T_SIZES[-1]]
NETS = listkeys(RSA_LAYERS)
if SHOBHITA:
# noinspection PyRedeclaration
T_SIZES = [100000] # num epochs?
NETS = ["LSTM"]
else:
NETS.remove("LSTM")
ACTIVATIONS_FOLDER = OM_DATA_FOLDER['activations']
if not SHOBHITA:
imgActivations = ACTIVATIONS_FOLDER.resolve('imgActivationsForRSA')
activations = {}
for net_folder in imgActivations.folders:
modelname = net_folder.name
if modelname not in activations:
activations[modelname] = {}
for activations_mat in net_folder.files.filtered(
lambda x: x.ext == 'mat'
):
# breakpoint()
activations[modelname][activations_mat.name_pre_ext] = activations_mat
else:
folder = ACTIVATIONS_FOLDER['rsa_activations_shobhita2']
files = {f.name.split('Cat')[1].split('_')[0]: f for f in folder.files}
activations = {
'LSTM': {c.name: folder[files[c.name].name] for c in RSA_CLASSES}
}
for arch in NETS:
arch_rand_perm = None
for size in T_SIZES:
net = arch
with section(f'preparing {net} activations'):
if not SHOBHITA: net = f'{net}_{size}'
acts_for_rsa = None
for c in [cls.name for cls in RSA_CLASSES]:
acts = activations[net][c].load(silent=True)
if not SHOBHITA:
acts = acts['imageActivations']
if arch_rand_perm is None:
progress(f"activation size of {net}: {len(acts[0])}")
arch_rand_perm = randperm(range(len(acts[0])))
acts = [a[arch_rand_perm][:ACT_SIZE] for a in acts[0:N_PER_CLASS]]
if acts_for_rsa is None:
acts_for_rsa = acts
else:
acts_for_rsa = concat(
acts_for_rsa,
acts,
axis=0
)
yield arch, net, FeatureMatrix(
data=acts_for_rsa,
ground_truth=np.repeat(RSA_CLASSES, int(len(acts_for_rsa) / len(RSA_CLASSES))).tolist(),
class_set=RSA_CLASSES
)
def violin(cls, fd):
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
maxY = None if fd.maxY is None or fd.maxY == 'inf' or not isreal(
fd.maxY) else float(fd.maxY)
minY = None if fd.minY is None or fd.minY == 'inf' or not isreal(
fd.minY) else float(fd.minY)
if maxY != None and minY != None:
diff = maxY - minY
pad = diff * (fd.y_pad_percent / 100)
maxY = maxY + pad
minY = minY - pad
if cls.fig is None:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
cls.ax = cls.fig.add_subplot(111)
try:
cls.ax.violinplot(fd.y, showmeans=True)
except FloatingPointError as e: # weird underflow error
if str(e) == 'underflow encountered in exp':
progress(
'funny violinplot error occurred again. rebuild locally')
cls.ax.plot([1, 2, 3])
return
else:
import traceback
traceback.print_exc()
breakpoint()
x = range(1, len(fd.x) + 1)
cls.ax.set_xticks(x)
cls.ax.set_xticklabels(fd.x)
# fd.x,
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size)
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
cls.ax.set_ylabel(fd.y_label)
cls.ax.set_xlabel(fd.x_label)
drawn_xs = []
drawn_ys = []
def will_overlap(wbx, wby):
for iii in itr(drawn_xs):
twbx = drawn_xs[iii]
twby = drawn_ys[iii]
if ((min(wbx) <= min(twbx)) and
(max(wbx) >= min(twbx))) or ((max(wbx) >= max(twbx)) and
(min(wbx) <= max(twbx))):
# x overlaps
if ((min(wby) <= min(twby)) and
(max(wby) >= min(twby))) or ((max(wby) >= max(twby))
and
(min(wby) <= max(twby))):
return True
# return True
return False
from scipy import stats
the_y = [np.mean(it) for it in fd.y]
for i in itr(fd.y):
for ii in itr(fd.y):
if i <= ii: continue
# breakpoint()
dh = .05 # default,
barh = .05 # default
will_be_y1 = max(the_y[i], the_y[ii]) + dh
will_be_y2 = will_be_y1 + barh
will_be_x = [i, ii]
will_be_y = [will_be_y1, will_be_y2]
while will_overlap(will_be_x, will_be_y):
dh += .05
will_be_y1 = max(the_y[i], the_y[ii]) + dh
will_be_y2 = will_be_y1 + barh
will_be_x = [i, ii]
will_be_y = [will_be_y1, will_be_y2]
drawn_xs.append(will_be_x)
drawn_ys.append(will_be_y)
cls.barplot_annotate_brackets(
i,
ii,
stats.ttest_ind(fd.y[i], fd.y[ii],
alternative='two-sided')[1],
x, # list(itr(fd.y)), #fd.x
the_y,
dh=dh,
barh=barh)
@classmethod
def bar(cls, fd):
from matplotlib import pyplot as plt # 1 FULL SECOND IMPORT
maxY = None if fd.maxY is None or fd.maxY == 'inf' or not isreal(
fd.maxY) else float(fd.maxY)
minY = None if fd.minY is None or fd.minY == 'inf' or not isreal(
fd.minY) else float(fd.minY)
# maxX = None if fd.maxX is None or fd.maxX == '-inf' or not isreal(fd.maxX) else float(fd.maxX)
# minX = None if fd.minX is None or fd.minX == 'inf' or not isreal(fd.minX) else float(fd.minX)
if maxY != None and minY != None:
diff = maxY - minY
pad = diff * (fd.y_pad_percent / 100)
maxY = maxY + pad
minY = minY - pad
if cls.fig is None:
if BLACK_FIGS:
cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
cls.ax = cls.fig.add_subplot(111, facecolor='black')
else:
cls.fig = plt.figure(figsize=(16, 12))
cls.ax = cls.fig.add_subplot(111)
cls.ax.bar(fd.x,
fd.y,
color=listmap(cls.color, fd.item_colors),
yerr=fd.err)
title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size)
plt.setp(title_obj, color=text_color)
cls.ax.axis(True)
cls.ax.spines['left'].set_color(text_color)
cls.ax.spines['bottom'].set_color(text_color)
cls.ax.xaxis.label.set_color(text_color)
cls.ax.yaxis.label.set_color(text_color)
cls.ax.tick_params(axis='x', colors=text_color)
cls.ax.tick_params(axis='y', colors=text_color)
@classmethod
def none(cls):
pass
@classmethod
def color(cls, *rgb):
rgb = arr(rgb) * 128
# if len(rgb.shape) == 3:
# if rgb.shape[0] == 1:
# rgb = rgb[0]
return '#%02x%02x%02x' % tuple(ints(rgb).tolist())
@classmethod
@abstractmethod
def tableItem(cls, o, background):
err('unused')
def run(self,
python_file_name,
SW: Optional,
srun: Optional[SRun],
EXTRA_LOGGING=False,
use_singularity=False,
non_singularity_run_args=(),
rsync_also=()):
p: SSHExpectProcess = None
def login():
nonlocal p
with PipelineSection("logging in to open mind", log=True):
p = self.ssh()
login_thread = Thread(target=login)
login_thread.start()
@run_in_thread
def status_sync():
login_thread.join()
status_p = self.ssh()
status_p.sendatprompt('mkdir status')
status_p.sendatprompt('cd status')
LOCAL_STATUS_FOLDER = File(
"/Users/matt/Desktop/registered/todo/flow/status")
LOCAL_FREECOMP_STATUS_FOLDER = LOCAL_STATUS_FOLDER['freecomp']
while p.alive():
sleep(1)
the_ls = status_p.return_from_line("ls")
now = str(time())
if False: # DEBUG/DEV
formatls = now + '\n' + the_ls
LOCAL_FREECOMP_STATUS_FOLDER['ls.status'].write(formatls)
LOCAL_FREECOMP_STATUS_FOLDER['ls.status'].deleteIfExists(
silent=True)
for f in (ff.strip() for ff in the_ls.split('\n')):
thecat = status_p.return_from_line(f"cat {f}")
LOCAL_FREECOMP_STATUS_FOLDER[f].write(now + thecat)
USE_RSYNC = True
if USE_RSYNC: # NOT OPTIONAL NOW BC PORTAL FILE
def rsync_fun():
with PipelineSection("running rsync", log=True):
from mlib.open_mind import OM_MOUNT
# '-progress' # causes error?
rsync1 = [
'rsync',
'-vru',
# '-progress',
# https://galaxysd.github.io/20160302/Fastest-Way-Rsync
# supposedly speeds things up? ... Wow yes, from 10 to 8 seconds! Might be a coincidence but I'm happy!
'-e',
'ssh -T -o Compression=no -x',
# sshd_config requires special setup
# this is neccesary, or else python files I delete locally wont be deleted on OM
'--delete',
# of course this will mean I will need a seperate cache directory or some other exlcusion option. That's ok.
'''--filter=:- .gitignore''',
'--exclude=.git/',
'--exclude=*.pyc',
'.',
'../mlib/mlib',
'../mlib/mdb.py'
] + list(rsync_also) + [
f'[email protected]:{OM_MOUNT}/user/mjgroth/dnn'
]
# rsync2 = ['rsync', '-vru', '''--filter=:- .gitignore''', '--include=mlib/', '../mlib',
# '[email protected]:/om5/user/mjgroth/mlib']
rsync_p = SSHExpectProcess(rsync1)
rsync_p.login(longpass=True)
# rsync_p.interact()
rsync_p.all_output()
# rsync_p = SSHExpectProcess(rsync2)
# rsync_p.login(longpass=True)
# rsync_p.interact()
rsync_thread = Thread(target=rsync_fun)
rsync_thread.start()
else:
err('nope. not ready for this with new parallel system')
from _dnn_lib import git_push_both
with PipelineSection("git push"):
git_push_both()
from mlib.open_mind import OpenMindProject
self: OpenMindProject
self.git_pull_dnn_and_mlib(p)
from mlib.open_mind import OpenMindBashScript # should really make a super class
from mlib.open_mind import OpenMindProject
SW: Optional[OpenMindBashScript]
# p = self.ssh()
login_thread.join()
if EXTRA_LOGGING:
p.log_to_stdout()
if SW is not None:
[p.sendatprompt(f'module load {m}') for m in SW.modules]
if self.host.must_use_slurm():
assert srun is not None
if srun is not None:
with section(f"requesting slurm session: {srun}"):
import mlib.err
stopper = p.log_to_stdout(
# fun=kill_on_finish_str,
# just_fun=True,
o=p,
print_not_log=True,
stop_on=[
ContainerBashScript.FINISH_STR, mlib.err.ERROR_EXIT_STR
])
# log(f"requesting slurm session: {srun}")
p.sendatprompt(srun.command())
index = p.expect([
'srun: error',
# 'has been allocated' #doesnt always happen?
"tasks started", # need -v flag
"has been allocated resources"
])
stopper.stop()
if index == 0:
err('SRUN ERROR')
progress('slurm allowed job to start') # logpie
p.setprompt()
rsync_thread.join()
if isinstance(self, OpenMindProject):
if OpenMindProject.RUN_IN_VAGRANT:
self._take_into_vagrant(p)
info('bashing')
# p.sendatprompt('cd ')
with section("OM: running python script"):
if use_singularity:
p.sendatprompt(
f'bash {SW.name}'
# f'./dnn ' + SW.
) # why was I using sudo??? ohhhh I might have been using sudo in order to have write access to files? yes!! I was suing sudo because that is the only way files are writable
else:
# PYTHONPATH=../mlib
# p.p.logfile_send = None
progress('waiting for prompt to send py command')
p.prompt()
progress('got prompt, sending py command')
p.sendline(
f'if MPLBACKEND=Agg /om2/user/mjgroth/miniconda3/envs/dnn39/bin/python ./{python_file_name} '
+ ' '.join(non_singularity_run_args) +
'; then echo \"OK\"; else exit; fi')
# p.interact()
progress('python command sent')
self.finish_process(p, SW)
import mlib.boot.crunch
mlib.boot.crunch.take_om_logs(self)