def _apply_script_flags(self, opdef):
if opdef.flags_import in ([], False):
return
local_cache = {}
model_paths = op_util.opdef_model_paths(opdef)
flags_data = self._flags_data(opdef, model_paths, local_cache)
opdef.flags_dest = flags_data.pop("$dest", None)
import_data = {
name: flags_data[name]
for name in flags_data if self._is_import_flag(name, opdef)
}
opdef.merge_flags(ImportedFlagsOpProxy(import_data, opdef, self.log))
def python_script_opdef_loaded(self, opdef):
if opdef.output_scalars is not None:
return
assert opdef.main, opdef
main_mod = op_util.split_cmd(opdef.main)[0]
model_paths = op_util.opdef_model_paths(opdef)
try:
_path, mod_path = python_util.find_module(main_mod, model_paths)
except ImportError:
return
script = python_util.Script(mod_path)
if self.is_keras_script(script):
if opdef.output_scalars is None:
opdef.output_scalars = KERAS_OUTPUT_SCALARS
def _apply_script_flags(self, opdef):
if not opdef.flags_import and opdef.flags_dest:
# If we're not importing flags and we have an explicit
# flags dest, there's no need to look at the script.
return
local_cache = {}
model_paths = op_util.opdef_model_paths(opdef)
flags_data = self._flags_data(opdef, model_paths, local_cache)
self._apply_flags_dest(flags_data, opdef)
import_data = {
name: flags_data[name]
for name in flags_data if self._is_import_flag(name, opdef)
}
opdef.merge_flags(ImportedFlagsOpProxy(import_data, opdef, self.log))
def _flags_data(self, opdef, local_cache):
main_mod, base_args = _split_main_spec(opdef.main)
flags_dest = opdef.flags_dest
try:
flags_data = local_cache[(main_mod, flags_dest)]
except KeyError:
flags_import_util.log_flags_info("reading flags for main spec %r",
main_mod)
model_paths = op_util.opdef_model_paths(opdef)
flags_data = self._flags_data_(main_mod, base_args, model_paths,
opdef.flags_dest)
else:
flags_import_util.log_flags_info(
"using cached flags for main spec %r", main_mod)
return flags_data
def _model_paths(opdef):
"""Returns the model paths for opdef.
Guild is moving to an isolation scheme that requires these paths
NOT be included in a run system path. When they are, it's possible
that the run sourcecode dir does not contain all of the required
code and these model paths are relied on to silently provide
what's missing.
At the moment, cross package inheritance relies on these
paths. Until multiple packages can be merged into a single
sourcecode tree (tricky considering they use multiple roots that
share a common path to guild.yml) we rely on these paths to
support that functionality.
"""
return op_util.opdef_model_paths(opdef)
def _apply_script_flags(self, opdef):
if _flags_import_disabled(opdef):
return
import_all_marker = object()
to_import = _flags_to_import(opdef, import_all_marker)
to_skip = _flags_to_skip(opdef)
local_cache = {}
model_paths = op_util.opdef_model_paths(opdef)
flags_data = self._flags_data(opdef, model_paths, local_cache)
self._apply_flags_dest(flags_data, opdef)
import_data = {
name: flags_data[name]
for name in flags_data
if ((to_import is import_all_marker or name in to_import)
and not name in to_skip)
}
opdef.merge_flags(ImportedFlagsOpProxy(import_data, opdef, self.log))
def _apply_script_flags(self, opdef, local_cache):
_log_flags_info("### Script flags for %s", opdef.fullname)
if _flags_import_disabled(opdef):
_log_flags_info("flags import disabled - skipping")
return
import_all_marker = object()
to_import = _flags_to_import(opdef, import_all_marker)
to_skip = _flags_to_skip(opdef)
model_paths = op_util.opdef_model_paths(opdef)
flags_data = self._flags_data(opdef, model_paths, local_cache)
self._apply_flags_dest(flags_data, opdef)
import_data = {
name: flags_data[name]
for name in flags_data
if ((to_import is import_all_marker or name in to_import)
and not name in to_skip)
}
opdef.merge_flags(ImportedFlagsOpProxy(import_data, opdef, self.log))
def python_script_opdef(self, op):
if (op.compare is not None and op.output_scalars is not None):
return
assert op.main, op
plugin = pluginlib.for_name("python_script")
main_mod = op_util.split_main(op.main)[0]
model_paths = op_util.opdef_model_paths(op)
try:
_sys_path, mod_path = plugin.find_module(main_mod, model_paths)
except ImportError:
return
script = python_util.Script(mod_path)
if self.is_keras_script(script):
if op.compare is None:
flags = ["=%s" % f.name for f in op.flags]
op.compare = flags + KERAS_BASE_COMPARE
if op.output_scalars is None:
op.output_scalars = KERAS_OUTPUT_SCALARS
def _model_paths(opdef):
return op_util.opdef_model_paths(opdef)
