def ANALYSES(mode: AnalysisMode):
[__import__(f.pymodname) for f in pwdf()['analyses'] if f.name != '__pycache__']
ans = listmap(lambda a: a(mode), _ANALYSES)
for a in ans:
a.check_shadow()
return ans
def EXP_FOLDER():
from lib.nn import nnstate
return pwdf()[root][nnstate.FLAGS.expid]
def get_report(md,
resources_root,
exp_name,
index_url,
database: Database = None,
api: SimpleAdminAPI = None,
editable=False,
web_resources_root=None,
show=False):
if exp_name not in listkeys(database):
database[exp_name] = {}
analysis_figdata = flat1([
a.get_report_figdata(exp_name, resources_root, database)
for a in ANALYSES(mode=AnalysisMode.PIPELINE)
])
all_arch_figs = []
ntrain_figs = md.ntrainims
nepochs = md.nepochs
for n in ntrain_figs:
for suffix in [
f'__val_CM{nepochs}', f'__L2-Output_CM{nepochs}',
f'__L2-Inter_CM{nepochs}', f'__L2-Raw_CM{nepochs}'
]:
arch_figs = listmap(__.label, md.archs)
arch_figs = listmap(
lambda a: (resources_root[f'{a}_{n}{suffix}.png'],
database.get_or_set_default(
'', exp_name, f'{a}_{n}{suffix}')), arch_figs)
all_arch_figs.extend(arch_figs)
mcc_name = f'Matthews_Correlation_Coefficient'
ntrain_figs = listmap(
lambda n:
(resources_root[f'{mcc_name}_{n}.png'],
database.get_or_set_default('', exp_name, f'{mcc_name}_{n}')),
ntrain_figs)
doc = HTMLPage(
f'index',
f'Symmetry Detection Report by Matt Groth',
f'',
f'{len(md.archs)} Architectures: ',
*listmap(__.label, md.archs),
f'Experiments: {md.nrepeats} per architecture',
f'Epochs: {md.nepochs} per experiment',
f'Batch Size: {md.batchsize}',
f'Training Images: {md.ntrainims}',
f'Normalized Individual Images: {md.normalized}',
FigureTable(
*analysis_figdata,
(resources_root['Final_Train_MCC.png'],
database.get_or_set_default('', exp_name, 'Final_Train_MCC')),
*ntrain_figs,
(pwdf()["/_figs/figs_misc/RSA_patterns/RSA_patterns.001.jpeg"],
database.get_or_set_default('', exp_name, 'RSA_patterns'))
if INCLUDE_RSA_SLIDE else None,
*all_arch_figs,
resources_root=web_resources_root,
exp_id=exp_name,
editable=editable),
Hyperlink('back to index', index_url),
api.apiElements(),
js=api.cs(),
style=DNN_REPORT_CSS,
show=show)
return doc
def write_sub_webpage(
htmlDoc: HTMLPage,
index_root,
rel_resource_root,
rel_root,
upload_resources=True,
WOLFRAM=False,
DEV=False
):
resource_root_rel = index_root[rel_resource_root].rel_to(index_root[rel_root])
resource_root_file = index_root[rel_resource_root]
r = f'\n~~Created Webpage: {htmlDoc.name}~~\n'
title = r
if WOLFRAM:
if upload_resources:
resource_root_file.wc.push_public()
if DEV:
index_root.edition_wolf_dev[rel_root].make_webpage(htmlDoc, resource_root_file.wcurl, resource_root_rel)
index_root.edition_wolf_dev[rel_root].wc.push_public()
r += f'\tPublic(DEV):{index_root.edition_wolf_dev[rel_root].wcurl}\n'
else:
index_root.edition_wolf_pub[rel_root].make_webpage(htmlDoc, resource_root_file.wcurl, resource_root_rel)
index_root.edition_wolf_pub[rel_root].wc.push_public()
r += f'\tPublic:{index_root.edition_wolf_pub[rel_root].wcurl}\n'
index_root.edition_git[rel_root].make_webpage(
htmlDoc,
resource_root=os.path.join(Project.GITHUB_LFS_IMAGE_ROOT, resource_root_file.rel_to(pwdf())),
resource_root_rel=os.path.join(Project.GITHUB_LFS_IMAGE_ROOT, resource_root_file.rel_to(pwdf())),
force_fix_to_abs=False
)
r += f'\tFor GitHub:{index_root.edition_git[rel_root].url}\n'
index_root.edition_local[rel_root].make_webpage(htmlDoc, url(resource_root_file), resource_root_rel)
local_page = index_root.edition_local[rel_root][htmlDoc.name + '.html']
r += f'\tLocal:{local_page.url}\n'
r += '~' * (len(title) - 2)
print(greens(r))
if htmlDoc.show:
atexit.register(
local_page.open
)
def send(self, *files, project_name):
for f in files:
assert f.parent.abspath == pwdf().abspath
class Project(SuperRunner, ABC):
INPUT_FILE = File('_input.txt')
REQS_FILE = File('reqs.json')
STATE = PermaDict('data/_metastate.json')
# noinspection PyMethodMayBeStatic,PyMethodParameters
def _default_config():
proto = {'placeholder1': None}
alll = {'placeholder2': None}
return {
'profiles': {
'proto' : proto,
'default': proto
},
'config' : {
'all' : alll,
'default': alll
}
}
CFG = File('cfg.yml', default=_default_config(), quiet=True)
DOCS_FOLDER = Folder('docs')
# LOCAL_DOCS_FOLDER = Folder('_docs')
RESOURCES_FOLDER = DOCS_FOLDER['resources']
SHADOW_RESOURCES = Folder('_Shadow_Resources')
FIGS_FOLDER = RESOURCES_FOLDER['figs']
DNN_FIGS_FOLDER = Folder('_figs')
DNN_WEB_FOLDER = Folder('_web')
DNN_FIGS_FIGS_FOLDER = DNN_FIGS_FOLDER['figs_dnn']
GITHUB_LFS_IMAGE_ROOT = os.path.join('https://media.githubusercontent.com/media/mgroth0/', pwdf().name, 'master')
PYCALL_FILE = RESOURCES_FOLDER['pycallgraph.png']
PYDEPS_OUTPUT = None
LOG_FILE = None
if main_mod_file() is not None:
PYDEPS_OUTPUT = RESOURCES_FOLDER[
f'{main_mod_file().name_pre_ext}.svg'
]
EXECUTABLE = main_mod_file().name_pre_ext
if File('git').exists:
GIT = Repo(pwd())
mbuild = False
extra_flags = []
clear_clear_cache_flags = [
'clear_cell_cache',
'cell_clear_cache',
'ccc'
]
def registered_flags(self): return [
'readme',
'build',
'cell',
] + self.extra_flags + self.clear_clear_cache_flags + listkeys(
self.fun_registry()
)
instructions = ''
configuration = ''
credits = ''
cfg = None
def _super_run(self, cfg_overrides=None):
if cfg_overrides is None:
cfg_overrides = {}
from mlib.web.html import HTMLObject
from mlib.web import shadow
from mlib.proj.stat import py_deps, class_model_report
self.prep_log_file(None)
cfg = self._get_cfg()
for k, v in listitems(cfg_overrides):
setattr(cfg, k, v)
self.cfg = cfg
if ismac():
# why was this so important again?
# self.daily(
# self.write_reqs
# )
# and this?
# self.daily(
# enable_py_call_graph,
# Project.PYCALL_FILE
# )
pass
with WOLFRAM:
if 'build' in cfg.FLAGS and self.mbuild:
assert len(cfg.FLAGS) == 1
err('anything that depends on mlib has to push that too')
build()
write_README(self)
self.push()
elif 'readme' in cfg.FLAGS:
assert len(cfg.FLAGS) == 1
write_README(self)
elif any(x in cfg.FLAGS for x in self.clear_clear_cache_flags):
assert len(cfg.FLAGS) == 1
clear_cell_cache()
elif 'cell' in cfg.FLAGS:
assert len(cfg.FLAGS) == 3
analysisFlag = cfg.FLAGS[1]
cellName = cfg.FLAGS[2]
analysisO = arr(ANALYSES(AnalysisMode.CELL)).first(
lambda o: cn(o) == analysisFlag or mn(o).split('.')[-1] == analysisFlag
)
cell = getattr(analysisO, cellName)
if cell.inputs[0] is not None:
inputs = cell.load_cached_input(analysisO)
cell(*inputs)
else:
cell()
else:
if ismac():
# need to have dailyOrFlag
# its asking me to delete nap online. not sure if I can do this so I'm commenting this out for now.
# self.daily(
# wolf_manager.manage
# )
run_in_daemon(
pingChecker) # this line wasnt nested in ismac b4 but got an error in openmind last time I ran
log('about to run with cfg')
self.run(cfg)
self.daily(
class_model_report, HTMLObject
)
if ismac():
self.daily(
# atexit.register,
py_deps,
main_mod_file(),
Project.PYDEPS_OUTPUT
)
# atexit.register(
if ismac() and shadow.enabled: # not doing this on openmind yet because it erases docs_local/results.html which I am using. need to fix this though
shadow.build_docs()
# )
if ismac():
reloadIdeaFilesFromDisk()
def push(self):
if self.GIT.is_dirty():
log(
f'A diff between the index and the commit’s tree your HEAD points to: {self.GIT.index.diff(self.GIT.head.commit)}')
log(
f'A diff between the index and the commit’s tree your HEAD points to: {self.GIT.index.diff(self.GIT.head.commit)}')
log(f'A list of untracked files: {self.GIT.untracked_files}')
inp = input('Ok to add, commit and push? [y/n] >')
inp = inp in ['y', 'Y']
if inp:
self.GIT.index.add('--all')
inp = "Commit Message: "
self.GIT.index.commit(inp.strip())
self.GIT.remotes[0].push()
else:
log('repo is not dirty')
@log_invokation
def _get_cfg(self):
assert len(self.registered_flags()) == len(set(self.registered_flags()))
freecfg = File('freecfg.json').load()
prof = 'default'
cfg = 'default'
changes = {}
flags = []
cell = False
for idx, a in enum(sys.argv):
if idx == 0: continue
elif a.startswith('--'):
k, v = tuple(a.replace('--', '').split('='))
if k == 'tic': continue
changes[k] = v
elif a.startswith('-'):
k, v = tuple(a.replace('-', '').split('='))
if k == 'prof':
prof = v
elif k == 'cfg':
cfg = v
else:
err('arguments with one dash (-) need to be prof= or cfg=')
elif cell or a in self.registered_flags():
if a == 'cell':
cell = True
flags += [a]
else:
err(f'invalid argument:{a} please see README')
prof = Project.CFG['profiles'][prof]
cfg = Project.CFG['configs'][cfg]
for k in listkeys(prof):
if k in listkeys(cfg):
prof_ntrain = prof[k]
for i, n in enum(cfg[k]):
if isstr(n) and n[0] == 'i':
cfg[k][i] = prof_ntrain[int(n[1])]
cfg = {**prof, **cfg, 'FLAGS': flags}
for k, v in listitems(changes):
if k not in listkeys(cfg):
err(f'invalid -- arguments: {k}, please see {Project.CFG.name} for configuration options')
if isinstance(cfg[k], bool):
v = bool(int(v))
cfg[k] = v
# hello from freecomp
for k, v in listitems(freecfg):
log(f'freecfg: {k}:{v}')
cfg[k] = v
# cfg['EPOCHS'] = freecfg['EPOCHS']
return obj(cfg)
def fun_registry(self):
if 'daily' not in listkeys(self.STATE):
self.STATE['daily'] = {}
return SubDictProxy(self.STATE, 'daily', makeObjs=False)
def daily(self, fun, *args):
self._daily(fun, fun.__name__, *args)
def daily_reminder(self, ss):
self._daily(lambda: input(ss), ss)
def _daily(self, fun, key, *args):
n = key
if n in self.cfg.FLAGS:
logy(f'running daily function FROM FLAG: {n}')
fun(*args)
elif n not in listkeys(self.fun_registry()):
logy(f'running daily function: {n}')
fun(*args)
self.fun_registry().update({n: time()})
elif self.fun_registry()[n] < time() - (3600 * 24):
logy(f'running daily function: {n}')
fun(*args)
self.fun_registry().update({n: time()})
else:
nex = self.fun_registry()[n] + (3600 * 24)
logc(f'{n} will run next in {nex - time()} seconds')
@staticmethod
def prep_log_file(filename, new=False):
if filename is None:
filename = os.path.basename(sys.argv[0]).replace('.py', '')
if ismac():
filename = f'_logs/local/{filename}.log'
else:
filename = f'_logs/remote/{filename}.log'
from mlib.file import Folder
filename = Folder(pwd())[filename]
if new:
filename = getNextIncrementalFile(filename)
if Project.LOG_FILE is None:
Project.LOG_FILE = File(filename)
Project.LOG_FILE.deleteIfExists()
Project.LOG_FILE.write('')
mlog.LOG_FILE = Project.LOG_FILE
if not mlog.QUIET: log(f'Initialized log file: {File(Project.LOG_FILE).relpath}')
@classmethod
def write_reqs(cls):
File('environment.yml').write(shell('conda env export').all_output())
reqs_conda = spshell(
f'{HOME}/miniconda3/bin/conda list -n {pwdf().name} -e'
).readlines_and_raise_if_err().filtered(
lambda l: 'pypi' not in l and (not l.strip().startswith("#"))
)
File('reqs_conda.txt').write('\n'.join(reqs_conda))
conda_prune(just_cache=True)
good2go = conda_prune()
return reqs_conda, good2go
def __init__(self, recipe_name, simg_name=None):
if simg_name is None:
simg_name = pwdf().name
self.recipe = SingularityRecipe(recipe_name, simg=simg_name) if recipe_name is not None else None
self.simg = self.recipe.simg if self.recipe is not None else File(simg_name)
self.img = self.recipe.img if self.recipe is not None else File(simg_name)
def __init__(self, host: Host, name: str = None):
self.host = host
if name is None:
name = pwdf().name
self.name = name