def run_chief_oracle():
"""Invoke chief oracle, and listen to the open port."""
logging.info('chief_oracle() starting...')
# Per specification, KerasTuner's behavior is controlled by env variables.
os.environ['KERASTUNER_ORACLE_IP'] = '0.0.0.0'
os.environ['KERASTUNER_ORACLE_PORT'] = self._master_port
os.environ['KERASTUNER_TUNER_ID'] = 'chief'
logging.info('Binding oracle chief server at: %s:%s',
os.environ['KERASTUNER_ORACLE_IP'],
os.environ['KERASTUNER_ORACLE_PORT'])
# By design of KerasTuner, chief oracle blocks forever.
tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
def _search(self, input_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, List[types.Artifact]]):
"""Conducts a single search loop, setting up chief oracle if necessary."""
# If not distributed, simply conduct search and return.
if self._tuner_id is None:
return tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
if _need_chief_oracle(exec_properties):
# If distributed search, and this node is chief, start a chief oracle
# process before conducting search by itself.
if self._is_chief:
# Tuner with chief oracle will block forever. As such, start it in
# a subprocess and manage its lifecycle by the main process.
# Note that the Tuner with chief oracle does not run search loop,
# hence does not run TensorFlow code in the subprocess.
self._chief_process = self._start_chief_oracle_in_subprocess(
input_dict, exec_properties)
# If distributed, both master and worker need to know where the oracle is.
# Per KerasTuner's interface, it is configured through env variables.
# This only affects the current main process, which is designed to be
# single-threaded. As such, mutation of this otherwise global state is
# safe.
os.environ['KERASTUNER_ORACLE_IP'] = self._master_addr
os.environ['KERASTUNER_ORACLE_PORT'] = self._master_port
logging.info('Oracle chief is known to be at: %s:%s',
os.environ['KERASTUNER_ORACLE_IP'],
os.environ['KERASTUNER_ORACLE_PORT'])
# Conduct tuner search loop, regardless of master or worker.
# There is only one Tuner instance in the current process, as such,
# controllling the id of the Tuner instance via environment variable
# is safe.
os.environ['KERASTUNER_TUNER_ID'] = self._tuner_id
logging.info('Setting KERASTUNER_TUNER_ID with %s',
os.environ['KERASTUNER_TUNER_ID'])
return tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
def _run_chief_oracle() -> None:
"""Invoke chief oracle, and listen to the open port."""
logging.info('chief_oracle() starting...')
# Per KerasTuner's specification, configuration of chief oracle is set
# by environment variables. This only affects the current sub-process
# which is single-threaded, but not the main process. As such, mutation
# of this otherwise global state is safe.
os.environ['KERASTUNER_ORACLE_IP'] = '0.0.0.0'
os.environ['KERASTUNER_ORACLE_PORT'] = self._master_port
os.environ['KERASTUNER_TUNER_ID'] = 'chief'
logging.info('Binding chief oracle server at: %s:%s',
os.environ['KERASTUNER_ORACLE_IP'],
os.environ['KERASTUNER_ORACLE_PORT'])
# By design of KerasTuner, chief oracle blocks forever. Ref.
# https://github.com/keras-team/keras-tuner/blob/e8b0ad3ecae471c73e17cb41f37e6f99202ac0dd/kerastuner/engine/base_tuner.py#L74-L76
tuner_executor.search(input_dict, exec_properties, _WORKING_DIRECTORY)
def run_chief_oracle():
"""Invoke chief oracle, and listen to the open port."""
logging.info('chief_oracle() starting...')
# Per KerasTuner's specification, configuration of chief oracle is set
# by environment variables. This only affects the current sub-process
# which is single-threaded, but not the main process. As such, mutation
# of this otherwise global state is safe.
os.environ['KERASTUNER_ORACLE_IP'] = '0.0.0.0'
os.environ['KERASTUNER_ORACLE_PORT'] = self._master_port
os.environ['KERASTUNER_TUNER_ID'] = 'chief'
logging.info('Binding chief oracle server at: %s:%s',
os.environ['KERASTUNER_ORACLE_IP'],
os.environ['KERASTUNER_ORACLE_PORT'])
# By design of KerasTuner, chief oracle blocks forever.
tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
def _search(self, input_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, List[types.Artifact]]):
"""Conducts a single search loop, setting up chief oracle if necessary."""
# If not distributed, simply conduct search and return.
if not self._is_distributed:
return tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
if _need_chief_oracle(exec_properties):
# If distributed search, and this node is chief, start a chief oracle
# process before conducting search by itself.
if self._is_chief:
# Tuner with chief oracle will block forever. As such, start it in
# a subprocess and manage its lifecycle by the main process.
# Note that the Tuner with chief oracle does not run search loop,
# hence does not run TensorFlow code in the subprocess.
self._chief_process = self._start_chief_oracle_in_subprocess(
input_dict, exec_properties)
# If distributed, both master and worker need to know where the oracle is.
os.environ['KERASTUNER_ORACLE_IP'] = self._master_addr
os.environ['KERASTUNER_ORACLE_PORT'] = self._master_port
logging.info('Oracle chief is known to be at: %s:%s',
os.environ['KERASTUNER_ORACLE_IP'],
os.environ['KERASTUNER_ORACLE_PORT'])
# Conduct tuner search loop, regardless of master or worker.
logging.info('Setting KERASTUNER_TUNER_ID with %s',
os.environ['KERASTUNER_TUNER_ID'])
os.environ['KERASTUNER_TUNER_ID'] = self._tuner_id
return tuner_executor.search(input_dict, exec_properties,
_WORKING_DIRECTORY)
