def mypwd(remote=None):
from mlib.boot.lang import pwd, ismac
if remote is None:
return pwd()
elif remote:
if not ismac():
return '/home/matt'
else:
return pwd()
else:
if ismac():
return pwd()
else:
return '/Users/matt'
def retrieve_api_info(self, name, hidden):
database_folder = {
False: self.EXP_FOLDER,
True: self.EXP_FOLDER.hidden_version(pwd())
}[hidden]
database_file = database_folder[f'{name}.json']
password_file = database_file.parent[
f'._{database_file.name_pre_ext}_password.txt']
return database_file, password_file
def checkWL():
STATUS_FILE = File(pwd() + '/WC/status')
SIGNAL_FILE = File(pwd() + '/WC/signal')
if not STATUS_FILE.exists() or STATUS_FILE.msecs() < time.time() - 2:
if SIGNAL_FILE.exists():
SIGNAL_FILE.delete()
pid = os.fork()
if pid == 0: # new process
# very interesting, and it worked, but overly complicated... why not just use an Wolfram APIFun as a more formal server?
# also, don't need this right now until I have a VM again.
# os.system(miniconda()+"/bin/python3 "+mitiliHome()+"/WC/wc.py"
# )
os._exit(0)
# print('in initFun for ' + str(os.getpid()) + "|" + sys.argv[0])
# eventually i think this is supposed to run on both systems so a flag is sent to the cloud for whichver most recently updated the data. but for now I don't wanna figure out which files should and which shouldn't be unisoned in the WC folder so lets deal with this later. besides, its mainly being used only for HEP right now.
# if ismac():
# very interesting, and it worked, but overly complicated... why not just use an Wolfram APIFun as a more formal server?
# also, don't need this right now until I have a VM again.
# checkWL()
def take_om_logs(OMP):
manager = get_manager()
from mlib.boot.lang import pwd
from mlib.boot.stream import listitems
from mlib.file import File, Folder
with PipelineSection('downloading pipeline section data', log=True):
OMP.get(
File(PIPELINE_SECTION_FILE).rel_to(pwd()),
destination=Folder('_data'),
overwrite=True
)
for seclabel, secdata in listitems(File(PIPELINE_SECTION_FILE).load(silent=True)):
while seclabel in manager.PIPELINE_SECTIONS:
seclabel = next_int_suffix(seclabel)
manager.PIPELINE_SECTIONS[seclabel] = secdata
def my_except_hook(exctype, value, tb):
if False:
exceptions.append((exctype, value, tb))
if exctype != Short_MException:
listing = traceback.format_exception(exctype, value, tb)
if False: # DEBUG
# if not ismac():
for i, line in enum(listing):
if line.startswith(' File "'):
linux_path = line.split('"')[1]
mac_path = pwd() + linux_path
listing[i] = listing[i].replace(linux_path, mac_path)
from mlib.proj.struct import REMOTE_CWD
listing[i] = listing[i].replace(REMOTE_CWD, '')
if exctype == MException:
del listing[-2]
listing[-2] = listing[-2].split('\n')[0] + '\n'
listing[-1] = listing[-1][0:-1]
print("".join(listing), file=sys.stderr)
print('ERROR ERROR ERROR')
# coming back here after a long time so... (these werent here before)
print(value)
for line in listing:
print(line)
# exit_for_real(1) #bad. with this, one exception thrown in pdb caused process to exit
sys.__excepthook__(exctype, value, tb)
else:
sys.__excepthook__(exctype, value, tb)
sys.exit(1)
if exctype == RelayException:
print(value)
else:
sys.__excepthook__(exctype, value, tb)
if auto_quit_on_exception:
print(ERROR_EXIT_STR)
# sys.exit(1)
# I think I've mismanaged all my threads...
if auto_quit_on_exception:
import os;
os._exit(1)
def prep_log_file(filename, new=False):
if filename is None:
filename = os.path.basename(sys.argv[0]).replace('.py', '')
if ismac():
filename = f'_logs/local/{filename}.log'
else:
filename = f'_logs/remote/{filename}.log'
from mlib.file import Folder
filename = Folder(pwd())[filename]
if new:
filename = getNextIncrementalFile(filename)
if Project.LOG_FILE is None:
Project.LOG_FILE = File(filename)
Project.LOG_FILE.deleteIfExists()
Project.LOG_FILE.write('')
mlog.LOG_FILE = Project.LOG_FILE
if not mlog.QUIET: log(f'Initialized log file: {File(Project.LOG_FILE).relpath}')
def get(self,
*files,
destination=None,
merge=None,
overwrite=False,
verbose=False):
if len(files) > 0:
self.host.get(*files, project_name=self.name, verbose=verbose)
# breakpoint()
# files = (File(File(f).name) for f in files) # gen expression! YOU CAN ONLY ITERATE IT ONCE!
files = [File(File(f).name) for f in files]
all_mb_files = []
for f in files:
mb_files = [(fi, si) for fi, si in f.files_recursive().map(
lambda fi: (fi, fi.size())) if si.mb() >= 1]
all_mb_files.extend(mb_files)
size = f.size(recursive=True)
progress(f'Downloaded file {f.name} is {size}')
the_len = len(all_mb_files)
progress(f'{the_len} files were 1 MB or larger' +
(':' if the_len else ''))
for mb_file, size in sorted(all_mb_files,
key=lambda t: t[1].num_bytes):
progress(f'\t{size}\t{mb_file.rel_to(pwd())}')
if destination is not None and Folder(
destination).abspath != Folder(pwd()).abspath:
Folder(destination).mkdirs()
for f in files:
f.moveinto(destination, overwrite=overwrite)
if merge is not None:
for f in files:
f.mergeinto(Folder(merge), overwrite=overwrite)
f.deleteIfExists()
def mitiliHome():
if ismac():
return pwd()
else:
return '/home/matt/mitili'
def dnn(cfg):
mode = cfg.MODE
log(f'MODE IS {mode}')
Project.DNN_FIGS_FIGS_FOLDER.mkdirs()
TEMP_FIGS_FOLDER = Folder(cfg.root)
last_eg = get_last_exp_group()
new_eg = None
new_fig_folder = None
muscle = Muscle(local=cfg.MUSCLE == 'local')
if cfg.CLEAR_EG_DATA and islinux():
Project.DNN_FIGS_FIGS_FOLDER.clear()
if 'JUSTRUN' in mode and cfg.SAVE_DATA:
TEMP_FIGS_FOLDER.mkdir().clear()
if 'next_exp_id' not in Project.STATE:
Project.STATE['next_exp_id'] = 1
if 'last_submitted_exp_group_name' not in Project.STATE:
Project.STATE['last_submitted_exp_group_name'] = ''
def check(a):
Project.STATE["last_submitted_exp_group_name"] = a
figs_folder, message = get_figs_folder(a)
return figs_folder is not None, figs_folder if figs_folder is not None else message
if cfg.EXPERIMENT_NAME is None:
new_fig_folder = answer_request.answer_request(
Project.STATE["last_submitted_exp_group_name"],
"Experiment Name:",
check,
gui=cfg.GUI)
else:
new_fig_folder = check(cfg.EXPERIMENT_NAME)[1]
new_fig_folder = File(new_fig_folder)
log(f'{new_fig_folder=}')
if 'JUSTRUN' in mode or 'PUSH' in mode:
if cfg.MUSCLE != 'local':
SyncedFolder(pwd()).sync(config='mitili', lpath='mitili')
if 'JUSTRUN' in mode:
cfg.tic = str(mlog.TIC)
experiments = experiments_from_cfg(cfg, advance_id=True)
jobs = make_jobs(cfg, muscle=muscle, experiments=experiments)
assert not cfg.GUI
muscle.run_all_jobs_main(jobs,
serial=cfg.RUN_EXPS_IN_SERIAL,
gui=cfg.GUI)
temp_eg = DNN_ExperimentGroup.temp(TEMP_FIGS_FOLDER)
temp_eg.save_md(cfg)
if cfg.SAVE_DATA:
new_eg = muscle.pull_data(TEMP_FIGS_FOLDER, cfg, new_fig_folder)
exp_group = new_eg or last_eg
log(f'MODE IS {mode}')
if 'COMPILE_TEST_ALL' in mode:
log('in CTA!')
analyze_exp_group(exp_group, cfg)
# the stuff below is only temporarily commented out
makefigs(exp_group.compile_folder, cfg.fig_backend, overwrite=True)
class Project(SuperRunner, ABC):
INPUT_FILE = File('_input.txt')
REQS_FILE = File('reqs.json')
STATE = PermaDict('data/_metastate.json')
# noinspection PyMethodMayBeStatic,PyMethodParameters
def _default_config():
proto = {'placeholder1': None}
alll = {'placeholder2': None}
return {
'profiles': {
'proto' : proto,
'default': proto
},
'config' : {
'all' : alll,
'default': alll
}
}
CFG = File('cfg.yml', default=_default_config(), quiet=True)
DOCS_FOLDER = Folder('docs')
# LOCAL_DOCS_FOLDER = Folder('_docs')
RESOURCES_FOLDER = DOCS_FOLDER['resources']
SHADOW_RESOURCES = Folder('_Shadow_Resources')
FIGS_FOLDER = RESOURCES_FOLDER['figs']
DNN_FIGS_FOLDER = Folder('_figs')
DNN_WEB_FOLDER = Folder('_web')
DNN_FIGS_FIGS_FOLDER = DNN_FIGS_FOLDER['figs_dnn']
GITHUB_LFS_IMAGE_ROOT = os.path.join('https://media.githubusercontent.com/media/mgroth0/', pwdf().name, 'master')
PYCALL_FILE = RESOURCES_FOLDER['pycallgraph.png']
PYDEPS_OUTPUT = None
LOG_FILE = None
if main_mod_file() is not None:
PYDEPS_OUTPUT = RESOURCES_FOLDER[
f'{main_mod_file().name_pre_ext}.svg'
]
EXECUTABLE = main_mod_file().name_pre_ext
if File('git').exists:
GIT = Repo(pwd())
mbuild = False
extra_flags = []
clear_clear_cache_flags = [
'clear_cell_cache',
'cell_clear_cache',
'ccc'
]
def registered_flags(self): return [
'readme',
'build',
'cell',
] + self.extra_flags + self.clear_clear_cache_flags + listkeys(
self.fun_registry()
)
instructions = ''
configuration = ''
credits = ''
cfg = None
def _super_run(self, cfg_overrides=None):
if cfg_overrides is None:
cfg_overrides = {}
from mlib.web.html import HTMLObject
from mlib.web import shadow
from mlib.proj.stat import py_deps, class_model_report
self.prep_log_file(None)
cfg = self._get_cfg()
for k, v in listitems(cfg_overrides):
setattr(cfg, k, v)
self.cfg = cfg
if ismac():
# why was this so important again?
# self.daily(
# self.write_reqs
# )
# and this?
# self.daily(
# enable_py_call_graph,
# Project.PYCALL_FILE
# )
pass
with WOLFRAM:
if 'build' in cfg.FLAGS and self.mbuild:
assert len(cfg.FLAGS) == 1
err('anything that depends on mlib has to push that too')
build()
write_README(self)
self.push()
elif 'readme' in cfg.FLAGS:
assert len(cfg.FLAGS) == 1
write_README(self)
elif any(x in cfg.FLAGS for x in self.clear_clear_cache_flags):
assert len(cfg.FLAGS) == 1
clear_cell_cache()
elif 'cell' in cfg.FLAGS:
assert len(cfg.FLAGS) == 3
analysisFlag = cfg.FLAGS[1]
cellName = cfg.FLAGS[2]
analysisO = arr(ANALYSES(AnalysisMode.CELL)).first(
lambda o: cn(o) == analysisFlag or mn(o).split('.')[-1] == analysisFlag
)
cell = getattr(analysisO, cellName)
if cell.inputs[0] is not None:
inputs = cell.load_cached_input(analysisO)
cell(*inputs)
else:
cell()
else:
if ismac():
# need to have dailyOrFlag
# its asking me to delete nap online. not sure if I can do this so I'm commenting this out for now.
# self.daily(
# wolf_manager.manage
# )
run_in_daemon(
pingChecker) # this line wasnt nested in ismac b4 but got an error in openmind last time I ran
log('about to run with cfg')
self.run(cfg)
self.daily(
class_model_report, HTMLObject
)
if ismac():
self.daily(
# atexit.register,
py_deps,
main_mod_file(),
Project.PYDEPS_OUTPUT
)
# atexit.register(
if ismac() and shadow.enabled: # not doing this on openmind yet because it erases docs_local/results.html which I am using. need to fix this though
shadow.build_docs()
# )
if ismac():
reloadIdeaFilesFromDisk()
def push(self):
if self.GIT.is_dirty():
log(
f'A diff between the index and the commit’s tree your HEAD points to: {self.GIT.index.diff(self.GIT.head.commit)}')
log(
f'A diff between the index and the commit’s tree your HEAD points to: {self.GIT.index.diff(self.GIT.head.commit)}')
log(f'A list of untracked files: {self.GIT.untracked_files}')
inp = input('Ok to add, commit and push? [y/n] >')
inp = inp in ['y', 'Y']
if inp:
self.GIT.index.add('--all')
inp = "Commit Message: "
self.GIT.index.commit(inp.strip())
self.GIT.remotes[0].push()
else:
log('repo is not dirty')
@log_invokation
def _get_cfg(self):
assert len(self.registered_flags()) == len(set(self.registered_flags()))
freecfg = File('freecfg.json').load()
prof = 'default'
cfg = 'default'
changes = {}
flags = []
cell = False
for idx, a in enum(sys.argv):
if idx == 0: continue
elif a.startswith('--'):
k, v = tuple(a.replace('--', '').split('='))
if k == 'tic': continue
changes[k] = v
elif a.startswith('-'):
k, v = tuple(a.replace('-', '').split('='))
if k == 'prof':
prof = v
elif k == 'cfg':
cfg = v
else:
err('arguments with one dash (-) need to be prof= or cfg=')
elif cell or a in self.registered_flags():
if a == 'cell':
cell = True
flags += [a]
else:
err(f'invalid argument:{a} please see README')
prof = Project.CFG['profiles'][prof]
cfg = Project.CFG['configs'][cfg]
for k in listkeys(prof):
if k in listkeys(cfg):
prof_ntrain = prof[k]
for i, n in enum(cfg[k]):
if isstr(n) and n[0] == 'i':
cfg[k][i] = prof_ntrain[int(n[1])]
cfg = {**prof, **cfg, 'FLAGS': flags}
for k, v in listitems(changes):
if k not in listkeys(cfg):
err(f'invalid -- arguments: {k}, please see {Project.CFG.name} for configuration options')
if isinstance(cfg[k], bool):
v = bool(int(v))
cfg[k] = v
# hello from freecomp
for k, v in listitems(freecfg):
log(f'freecfg: {k}:{v}')
cfg[k] = v
# cfg['EPOCHS'] = freecfg['EPOCHS']
return obj(cfg)
def fun_registry(self):
if 'daily' not in listkeys(self.STATE):
self.STATE['daily'] = {}
return SubDictProxy(self.STATE, 'daily', makeObjs=False)
def daily(self, fun, *args):
self._daily(fun, fun.__name__, *args)
def daily_reminder(self, ss):
self._daily(lambda: input(ss), ss)
def _daily(self, fun, key, *args):
n = key
if n in self.cfg.FLAGS:
logy(f'running daily function FROM FLAG: {n}')
fun(*args)
elif n not in listkeys(self.fun_registry()):
logy(f'running daily function: {n}')
fun(*args)
self.fun_registry().update({n: time()})
elif self.fun_registry()[n] < time() - (3600 * 24):
logy(f'running daily function: {n}')
fun(*args)
self.fun_registry().update({n: time()})
else:
nex = self.fun_registry()[n] + (3600 * 24)
logc(f'{n} will run next in {nex - time()} seconds')
@staticmethod
def prep_log_file(filename, new=False):
if filename is None:
filename = os.path.basename(sys.argv[0]).replace('.py', '')
if ismac():
filename = f'_logs/local/{filename}.log'
else:
filename = f'_logs/remote/{filename}.log'
from mlib.file import Folder
filename = Folder(pwd())[filename]
if new:
filename = getNextIncrementalFile(filename)
if Project.LOG_FILE is None:
Project.LOG_FILE = File(filename)
Project.LOG_FILE.deleteIfExists()
Project.LOG_FILE.write('')
mlog.LOG_FILE = Project.LOG_FILE
if not mlog.QUIET: log(f'Initialized log file: {File(Project.LOG_FILE).relpath}')
@classmethod
def write_reqs(cls):
File('environment.yml').write(shell('conda env export').all_output())
reqs_conda = spshell(
f'{HOME}/miniconda3/bin/conda list -n {pwdf().name} -e'
).readlines_and_raise_if_err().filtered(
lambda l: 'pypi' not in l and (not l.strip().startswith("#"))
)
File('reqs_conda.txt').write('\n'.join(reqs_conda))
conda_prune(just_cache=True)
good2go = conda_prune()
return reqs_conda, good2go
def MITILI_FOLDER():
if ismac():
return File(pwd())
else:
return File('/home/matt/mitili')
from packaging import version
def vers(s):
return version.parse(str(s))
def mexit(code, message):
log(message)
exit(code)
REMOTE_CWD = None
if islinux():
REMOTE_CWD = pwd()
log('Defining Project')
class Project(SuperRunner, ABC):
INPUT_FILE = File('_input.txt')
REQS_FILE = File('reqs.json')
STATE = PermaDict('data/_metastate.json')
# noinspection PyMethodMayBeStatic,PyMethodParameters
def _default_config():
proto = {'placeholder1': None}
alll = {'placeholder2': None}
return {
'profiles': {
'proto' : proto,
def rel(self):
from mlib.proj.struct import pwd
return os.path.relpath(self.abspath, pwd())
def ls(d=pwd()):
import os
dirlist = os.listdir(d)
return dirlist
def pwdf():
return Folder(pwd())