def mypwd(remote=None):
from mlib.boot.lang import pwd, ismac
if remote is None:
return pwd()
elif remote:
if not ismac():
return '/home/matt'
else:
return pwd()
else:
if ismac():
return pwd()
else:
return '/Users/matt'
def build_net(self, FLAGS):
import tensorflow as tf
LRN, PoolHelper = gnet_layer_classes()
if self.file.ext == '.h5':
self.net = tf.keras.models.load_model(self.file.abspath,
custom_objects={
'PoolHelper': PoolHelper,
'LRN': LRN
})
elif self.file.ext == 'onnx':
if ismac():
onnx_tf = f'{HOME}/miniconda3/envs/dnn/bin/onnx-tf'
else:
onnx_tf = f'matt/miniconda3/envs/dnn/bin/onnx-tf'
out = self.file.res_pre_ext("pb")
eshell(
f'{onnx_tf} convert -i {self.file.abspath} -o {out.abspath}')
# onnx-tf convert -i /path/to/input.onnx -o /path/to/output.pb
else:
err('')
if len(self.net.outputs) > 1:
found = False
for i, o in enum(self.net.outputs):
if 'prob' in o.name:
assert not found
self.OUTPUT_IDX = i
found = True
assert found
def __init__(self, name):
self.folder = OM_IM_DATA_ROOT[name]
self.metadata_file = self.folder['metadata.json']
self.trans = {_RAW: ImageDatasetTransformation(self, _RAW)}
self.splits = {
n: ImageDatasetSplit(self, n)
for n in [_TRAIN, _TEST, _EVAL]
}
if not ismac():
self.classes = ClassSet([
ImageDatasetClass(name=File(class_folder).name,
index=i,
dataset=self) for i, class_folder in esorted(
listvalues(self.splits)[0].folder(
listvalues(self.trans)[0]).paths)
])
if not ismac():
self.verify()
def __init__(self, mpath, lpath):
self.mpath = self.abspath
err('used to have line: self.lpath = mlib.file.abspath')
from mlib.boot.lang import ismac, islinux
if ismac():
thispath = mpath
else:
assert islinux()
thispath = lpath
super(SyncedDataFolder, self).__init__(thispath)
def eval(self, s):
if self.session is None:
kwargs = {} if ismac() else dict(
kernel='/home/matt/WOLFRAM/Executables/WolframKernel')
self._start_session(kwargs)
ev = self.session.evaluate_wrap_future(wl.UsingFrontEnd(s))
ev = ev.result()
if ev.messages is not None:
for m in ev.messages:
err(m)
return ev.result
class DNN_Job(Job):
SUCCESS_STR = "NRC IS FINISHED"
REMOTE_FOLDER = "~/mitili"
REMOTE_SCRIPT = "src/main/python/exec/work.py"
if ismac():
LOCAL_PY = f'{expanduser("~")}/miniconda3/envs/dnn/bin/python'
else:
os.environ["CONDA_HOME"] = '/om2/user/mjgroth/miniconda3'
LOCAL_PY = f'{os.environ["CONDA_HOME"]}/envs/dnn39/bin/python'
LOCAL_SCRIPT = f'work.py'
def __str__(self):
if self.job_args['expid'] == '0':
arch_str = ''
else:
arch_str = f'ARCH={self.job_args["arch"]}{self.job_args["ntrain"]}'
arch_str = lengthen_str(arch_str, 20)
return f'{super().__str__()}\tEXPID={self.job_args["expid"]}\t{arch_str}'
def run(self, cfg):
# keep modular
assert not (cfg.REGEN_DATA and cfg.OVERWRITE_NORMS
) # btw, both imply killing worker before exp
from lib.dnn_lib import dnn
from mlib.web.api import API
from mlib.web.database import Database
shadow.SHOW_INDEX = False
if len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'IMAGENET_COUNT':
import count_imagenet_data
count_imagenet_data.count()
return None
elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'RSA_MAIN':
# print('here1, doing Darius-RSA')
# import rsa_for_darius
import rsa_comp
rsa_comp.main()
finish_dnn_remote()
return None
elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] in self._human_exp_flags:
from lib.human_exp import human_exp
human_exp(cfg)
else:
flag_mode = ''.join(
arr(cfg.FLAGS).filtered(lambda s: s in self.MODES))
if not isblank(flag_mode): cfg.MODE = flag_mode
if isblank(cfg.MODE):
cfg.MODE = ''.join(self.MODES) # unnecessary?
if cfg.offline:
API.offline_mode = True
Database.offline_mode = True
from lib import makereport
makereport.MAKEREPORT_ONLINE = False
from mlib.km import kmscript # keep modular
if ismac():
kmscript('activate run tool window')
dnn(cfg)
finish_dnn_remote()
def prep_log_file(filename, new=False):
if filename is None:
filename = os.path.basename(sys.argv[0]).replace('.py', '')
if ismac():
filename = f'_logs/local/{filename}.log'
else:
filename = f'_logs/remote/{filename}.log'
from mlib.file import Folder
filename = Folder(pwd())[filename]
if new:
filename = getNextIncrementalFile(filename)
if Project.LOG_FILE is None:
Project.LOG_FILE = File(filename)
Project.LOG_FILE.deleteIfExists()
Project.LOG_FILE.write('')
mlog.LOG_FILE = Project.LOG_FILE
if not mlog.QUIET: log(f'Initialized log file: {File(Project.LOG_FILE).relpath}')
def mitiliHome():
if ismac():
return pwd()
else:
return '/home/matt/mitili'
def py():
if ismac():
return '/Users/matt/miniconda3/bin/python3'
else:
return '/home/matt/miniconda3/bin/python3'
def _super_run(self, cfg_overrides=None):
if cfg_overrides is None:
cfg_overrides = {}
from mlib.web.html import HTMLObject
from mlib.web import shadow
from mlib.proj.stat import py_deps, class_model_report
self.prep_log_file(None)
cfg = self._get_cfg()
for k, v in listitems(cfg_overrides):
setattr(cfg, k, v)
self.cfg = cfg
if ismac():
# why was this so important again?
# self.daily(
# self.write_reqs
# )
# and this?
# self.daily(
# enable_py_call_graph,
# Project.PYCALL_FILE
# )
pass
with WOLFRAM:
if 'build' in cfg.FLAGS and self.mbuild:
assert len(cfg.FLAGS) == 1
err('anything that depends on mlib has to push that too')
build()
write_README(self)
self.push()
elif 'readme' in cfg.FLAGS:
assert len(cfg.FLAGS) == 1
write_README(self)
elif any(x in cfg.FLAGS for x in self.clear_clear_cache_flags):
assert len(cfg.FLAGS) == 1
clear_cell_cache()
elif 'cell' in cfg.FLAGS:
assert len(cfg.FLAGS) == 3
analysisFlag = cfg.FLAGS[1]
cellName = cfg.FLAGS[2]
analysisO = arr(ANALYSES(AnalysisMode.CELL)).first(
lambda o: cn(o) == analysisFlag or mn(o).split('.')[-1] == analysisFlag
)
cell = getattr(analysisO, cellName)
if cell.inputs[0] is not None:
inputs = cell.load_cached_input(analysisO)
cell(*inputs)
else:
cell()
else:
if ismac():
# need to have dailyOrFlag
# its asking me to delete nap online. not sure if I can do this so I'm commenting this out for now.
# self.daily(
# wolf_manager.manage
# )
run_in_daemon(
pingChecker) # this line wasnt nested in ismac b4 but got an error in openmind last time I ran
log('about to run with cfg')
self.run(cfg)
self.daily(
class_model_report, HTMLObject
)
if ismac():
self.daily(
# atexit.register,
py_deps,
main_mod_file(),
Project.PYDEPS_OUTPUT
)
# atexit.register(
if ismac() and shadow.enabled: # not doing this on openmind yet because it erases docs_local/results.html which I am using. need to fix this though
shadow.build_docs()
# )
if ismac():
reloadIdeaFilesFromDisk()
from mlib.file import File, Folder, pwdf, getNextIncrementalFile
from mlib.inspect import mn
log('middle of some imports in struct.py')
from mlib.km import reloadIdeaFilesFromDisk
from mlib.obj import SuperRunner
from mlib.parallel import run_in_daemon
from mlib.proj.build.conda_prune import conda_prune
from mlib.proj.build.mbuild import build
from mlib.proj.build.readme import write_README
from mlib.shell import shell, spshell
from mlib.str import utf_decode
from mlib.term import log_invokation
from mlib.wolf.wolfpy import WOLFRAM
from mlib.analyses import ANALYSES, AnalysisMode, clear_cell_cache
log('top of struct.py import')
if ismac():
os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = '/usr/local/bin/git'
from git import Repo
def main_mod_file():
if hasattr(sys.modules['__main__'], '__file__'):
return File(os.path.abspath(sys.modules['__main__'].__file__))
def py():
if ismac():
return '/Users/matt/miniconda3/bin/python3'
else:
return '/home/matt/miniconda3/bin/python3'
def postsync(self):
if ismac():
self.sync()
WC['lastsave'] = 'mac'
else:
WC['lastsave'] = 'linux'
def presync(self):
lastsave = WC['lastsave']
if lastsave == 'linux' and ismac():
self.sync()
def miniconda():
if ismac():
return '/Users/matt/miniconda3'
else:
return '/home/matt/miniconda3'
def MITILI_FOLDER():
if ismac():
return File(pwd())
else:
return File('/home/matt/mitili')
def sync(self, config=None, batch=False, lpath=None):
gcloud_config()
from mlib.proj.struct import ismac
assert ismac()
import google_compute
assert google_compute.isrunning()
google_compute.gcloud_config()
com = ['/usr/local/bin/unison']
f = None
if config is not None:
assert config == 'mitili'
f = Temp('/Users/matt/.unison/mitili.pref')
com += [config]
com += '-root'
com += self.mpath
com += '-root'
if lpath is None:
lpath = self.lpath
if not lpath[0] == '/':
lpath.insert(0, '/')
com += 'ssh://test-3.us-central1-a.neat-beaker-261120' + self.lpath.replace(
'/home/matt', '')
if batch:
com += '-batch'
f.__enter__()
f.write('''
# Some regexps specifying names and paths to ignore
ignore = Name .DS_Store
ignore = Name .git
ignore = Name .idea
ignore = Name .gradle
ignore = Name gradle
ignore = Name .pass
ignore = Name __pycache__
ignore = Path WC/log.log
ignore = Name {.*,*,*/,.*/}.mat
ignore = Name {.*,*,*/,.*/}.png
ignore = Name {.*,*,*/,.*/}.pyc
ignore = Path {figures*}
ignore = Path {venv.old}
ignore = Path {cache}
ignore = Path {src/main/kotlin}
ignore = Path {build.gradle.kts}
ignore = Path {build}
ignore = Path {_figs}
ignore = Path {images}
ignore = Path {GoogLeNet}
ignore = Path {HEP/data}
ignore = Path {nap}
ignore = Path {plot.png}
''')
from mlib.shell import SSHProcess
p = SSHProcess(com)
def finishSync():
# # this has to be called or it will block
if p.alive():
p.readlines()
p.wait()
f.__exit__()
import atexit
atexit.register(finishSync)
p.login()
p.interact()
f.__exit__(None, None, None)