def mypwd(remote=None): from mlib.boot.lang import pwd, ismac if remote is None: return pwd() elif remote: if not ismac(): return '/home/matt' else: return pwd() else: if ismac(): return pwd() else: return '/Users/matt'
def build_net(self, FLAGS): import tensorflow as tf LRN, PoolHelper = gnet_layer_classes() if self.file.ext == '.h5': self.net = tf.keras.models.load_model(self.file.abspath, custom_objects={ 'PoolHelper': PoolHelper, 'LRN': LRN }) elif self.file.ext == 'onnx': if ismac(): onnx_tf = f'{HOME}/miniconda3/envs/dnn/bin/onnx-tf' else: onnx_tf = f'matt/miniconda3/envs/dnn/bin/onnx-tf' out = self.file.res_pre_ext("pb") eshell( f'{onnx_tf} convert -i {self.file.abspath} -o {out.abspath}') # onnx-tf convert -i /path/to/input.onnx -o /path/to/output.pb else: err('') if len(self.net.outputs) > 1: found = False for i, o in enum(self.net.outputs): if 'prob' in o.name: assert not found self.OUTPUT_IDX = i found = True assert found
def __init__(self, name): self.folder = OM_IM_DATA_ROOT[name] self.metadata_file = self.folder['metadata.json'] self.trans = {_RAW: ImageDatasetTransformation(self, _RAW)} self.splits = { n: ImageDatasetSplit(self, n) for n in [_TRAIN, _TEST, _EVAL] } if not ismac(): self.classes = ClassSet([ ImageDatasetClass(name=File(class_folder).name, index=i, dataset=self) for i, class_folder in esorted( listvalues(self.splits)[0].folder( listvalues(self.trans)[0]).paths) ]) if not ismac(): self.verify()
def __init__(self, mpath, lpath): self.mpath = self.abspath err('used to have line: self.lpath = mlib.file.abspath') from mlib.boot.lang import ismac, islinux if ismac(): thispath = mpath else: assert islinux() thispath = lpath super(SyncedDataFolder, self).__init__(thispath)
def eval(self, s): if self.session is None: kwargs = {} if ismac() else dict( kernel='/home/matt/WOLFRAM/Executables/WolframKernel') self._start_session(kwargs) ev = self.session.evaluate_wrap_future(wl.UsingFrontEnd(s)) ev = ev.result() if ev.messages is not None: for m in ev.messages: err(m) return ev.result
class DNN_Job(Job): SUCCESS_STR = "NRC IS FINISHED" REMOTE_FOLDER = "~/mitili" REMOTE_SCRIPT = "src/main/python/exec/work.py" if ismac(): LOCAL_PY = f'{expanduser("~")}/miniconda3/envs/dnn/bin/python' else: os.environ["CONDA_HOME"] = '/om2/user/mjgroth/miniconda3' LOCAL_PY = f'{os.environ["CONDA_HOME"]}/envs/dnn39/bin/python' LOCAL_SCRIPT = f'work.py' def __str__(self): if self.job_args['expid'] == '0': arch_str = '' else: arch_str = f'ARCH={self.job_args["arch"]}{self.job_args["ntrain"]}' arch_str = lengthen_str(arch_str, 20) return f'{super().__str__()}\tEXPID={self.job_args["expid"]}\t{arch_str}'
def run(self, cfg): # keep modular assert not (cfg.REGEN_DATA and cfg.OVERWRITE_NORMS ) # btw, both imply killing worker before exp from lib.dnn_lib import dnn from mlib.web.api import API from mlib.web.database import Database shadow.SHOW_INDEX = False if len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'IMAGENET_COUNT': import count_imagenet_data count_imagenet_data.count() return None elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'RSA_MAIN': # print('here1, doing Darius-RSA') # import rsa_for_darius import rsa_comp rsa_comp.main() finish_dnn_remote() return None elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] in self._human_exp_flags: from lib.human_exp import human_exp human_exp(cfg) else: flag_mode = ''.join( arr(cfg.FLAGS).filtered(lambda s: s in self.MODES)) if not isblank(flag_mode): cfg.MODE = flag_mode if isblank(cfg.MODE): cfg.MODE = ''.join(self.MODES) # unnecessary? if cfg.offline: API.offline_mode = True Database.offline_mode = True from lib import makereport makereport.MAKEREPORT_ONLINE = False from mlib.km import kmscript # keep modular if ismac(): kmscript('activate run tool window') dnn(cfg) finish_dnn_remote()
def prep_log_file(filename, new=False): if filename is None: filename = os.path.basename(sys.argv[0]).replace('.py', '') if ismac(): filename = f'_logs/local/{filename}.log' else: filename = f'_logs/remote/{filename}.log' from mlib.file import Folder filename = Folder(pwd())[filename] if new: filename = getNextIncrementalFile(filename) if Project.LOG_FILE is None: Project.LOG_FILE = File(filename) Project.LOG_FILE.deleteIfExists() Project.LOG_FILE.write('') mlog.LOG_FILE = Project.LOG_FILE if not mlog.QUIET: log(f'Initialized log file: {File(Project.LOG_FILE).relpath}')
def mitiliHome(): if ismac(): return pwd() else: return '/home/matt/mitili'
def py(): if ismac(): return '/Users/matt/miniconda3/bin/python3' else: return '/home/matt/miniconda3/bin/python3'
def _super_run(self, cfg_overrides=None): if cfg_overrides is None: cfg_overrides = {} from mlib.web.html import HTMLObject from mlib.web import shadow from mlib.proj.stat import py_deps, class_model_report self.prep_log_file(None) cfg = self._get_cfg() for k, v in listitems(cfg_overrides): setattr(cfg, k, v) self.cfg = cfg if ismac(): # why was this so important again? # self.daily( # self.write_reqs # ) # and this? # self.daily( # enable_py_call_graph, # Project.PYCALL_FILE # ) pass with WOLFRAM: if 'build' in cfg.FLAGS and self.mbuild: assert len(cfg.FLAGS) == 1 err('anything that depends on mlib has to push that too') build() write_README(self) self.push() elif 'readme' in cfg.FLAGS: assert len(cfg.FLAGS) == 1 write_README(self) elif any(x in cfg.FLAGS for x in self.clear_clear_cache_flags): assert len(cfg.FLAGS) == 1 clear_cell_cache() elif 'cell' in cfg.FLAGS: assert len(cfg.FLAGS) == 3 analysisFlag = cfg.FLAGS[1] cellName = cfg.FLAGS[2] analysisO = arr(ANALYSES(AnalysisMode.CELL)).first( lambda o: cn(o) == analysisFlag or mn(o).split('.')[-1] == analysisFlag ) cell = getattr(analysisO, cellName) if cell.inputs[0] is not None: inputs = cell.load_cached_input(analysisO) cell(*inputs) else: cell() else: if ismac(): # need to have dailyOrFlag # its asking me to delete nap online. not sure if I can do this so I'm commenting this out for now. # self.daily( # wolf_manager.manage # ) run_in_daemon( pingChecker) # this line wasnt nested in ismac b4 but got an error in openmind last time I ran log('about to run with cfg') self.run(cfg) self.daily( class_model_report, HTMLObject ) if ismac(): self.daily( # atexit.register, py_deps, main_mod_file(), Project.PYDEPS_OUTPUT ) # atexit.register( if ismac() and shadow.enabled: # not doing this on openmind yet because it erases docs_local/results.html which I am using. need to fix this though shadow.build_docs() # ) if ismac(): reloadIdeaFilesFromDisk()
from mlib.file import File, Folder, pwdf, getNextIncrementalFile from mlib.inspect import mn log('middle of some imports in struct.py') from mlib.km import reloadIdeaFilesFromDisk from mlib.obj import SuperRunner from mlib.parallel import run_in_daemon from mlib.proj.build.conda_prune import conda_prune from mlib.proj.build.mbuild import build from mlib.proj.build.readme import write_README from mlib.shell import shell, spshell from mlib.str import utf_decode from mlib.term import log_invokation from mlib.wolf.wolfpy import WOLFRAM from mlib.analyses import ANALYSES, AnalysisMode, clear_cell_cache log('top of struct.py import') if ismac(): os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = '/usr/local/bin/git' from git import Repo def main_mod_file(): if hasattr(sys.modules['__main__'], '__file__'): return File(os.path.abspath(sys.modules['__main__'].__file__)) def py(): if ismac(): return '/Users/matt/miniconda3/bin/python3' else: return '/home/matt/miniconda3/bin/python3'
def postsync(self): if ismac(): self.sync() WC['lastsave'] = 'mac' else: WC['lastsave'] = 'linux'
def presync(self): lastsave = WC['lastsave'] if lastsave == 'linux' and ismac(): self.sync()
def miniconda(): if ismac(): return '/Users/matt/miniconda3' else: return '/home/matt/miniconda3'
def MITILI_FOLDER(): if ismac(): return File(pwd()) else: return File('/home/matt/mitili')
def sync(self, config=None, batch=False, lpath=None): gcloud_config() from mlib.proj.struct import ismac assert ismac() import google_compute assert google_compute.isrunning() google_compute.gcloud_config() com = ['/usr/local/bin/unison'] f = None if config is not None: assert config == 'mitili' f = Temp('/Users/matt/.unison/mitili.pref') com += [config] com += '-root' com += self.mpath com += '-root' if lpath is None: lpath = self.lpath if not lpath[0] == '/': lpath.insert(0, '/') com += 'ssh://test-3.us-central1-a.neat-beaker-261120' + self.lpath.replace( '/home/matt', '') if batch: com += '-batch' f.__enter__() f.write(''' # Some regexps specifying names and paths to ignore ignore = Name .DS_Store ignore = Name .git ignore = Name .idea ignore = Name .gradle ignore = Name gradle ignore = Name .pass ignore = Name __pycache__ ignore = Path WC/log.log ignore = Name {.*,*,*/,.*/}.mat ignore = Name {.*,*,*/,.*/}.png ignore = Name {.*,*,*/,.*/}.pyc ignore = Path {figures*} ignore = Path {venv.old} ignore = Path {cache} ignore = Path {src/main/kotlin} ignore = Path {build.gradle.kts} ignore = Path {build} ignore = Path {_figs} ignore = Path {images} ignore = Path {GoogLeNet} ignore = Path {HEP/data} ignore = Path {nap} ignore = Path {plot.png} ''') from mlib.shell import SSHProcess p = SSHProcess(com) def finishSync(): # # this has to be called or it will block if p.alive(): p.readlines() p.wait() f.__exit__() import atexit atexit.register(finishSync) p.login() p.interact() f.__exit__(None, None, None)