* ``acq-func`` : Acquisition function
* ``LCB`` :
* ``EI`` :
* ``PI`` :
* ``gp_hedge`` : (default)
"""
import signal
from deephyper.search.hps.optimizer import Optimizer
from deephyper.search import Search
from deephyper.search import util
logger = util.conf_logger('deephyper.search.hps.ambs')
SERVICE_PERIOD = 2 # Delay (seconds) between main loop iterations
CHECKPOINT_INTERVAL = 1 # How many jobs to complete between optimizer checkpoints
EXIT_FLAG = False
def on_exit(signum, stack):
global EXIT_FLAG
EXIT_FLAG = True
class AMBS(Search):
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem, run, evaluator, **kwargs)
logger.info("Initializing AMBS")
import os
import json
from random import random, seed
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.evaluator.evaluate import Encoder
dhlogger = util.conf_logger("deephyper.search.nas.random")
class Random(NeuralArchitectureSearch):
"""Search class to run a full random neural architecture search. The search is filling every available nodes as soon as they are detected. The master job is using only 1 MPI rank.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.nas.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
"""
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem=problem,
run=run,
evaluator=evaluator,
**kwargs)
seed(self.problem.seed)
self.free_workers = self.evaluator.num_workers
import os
import collections
import numpy as np
import json
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from deephyper.core.parser import add_arguments_from_signature
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.evaluator.evaluate import Encoder
dhlogger = util.conf_logger("deephyper.search.nas.regevo")
# def key(d):
# return json.dumps(dict(arch_seq=d['arch_seq']), cls=Encoder)
class RegularizedEvolution(NeuralArchitectureSearch):
"""Regularized evolution.
https://arxiv.org/abs/1802.01548
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.nas.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
population_size (int, optional): the number of individuals to keep in the population. Defaults to 100.
sample_size (int, optional): the number of individuals that should participate in each tournament. Defaults to 10.
"""
def __init__(self,
problem,
import os
import time
from inspect import signature
import numpy as np
import tensorflow as tf
import horovod.tensorflow.keras as hvd
from deephyper.core.exceptions import DeephyperRuntimeError
from deephyper.search import util
from deephyper.nas import arch as a
from deephyper.nas import train_utils as U
logger = util.conf_logger(__name__)
AUTOTUNE = tf.data.experimental.AUTOTUNE
class HorovodTrainerTrainValid:
def __init__(self, config, model):
self.cname = self.__class__.__name__
self.config = config
self.model = model
self.callbacks = []
self.data = self.config[a.data]
# hyperparameters
self.config_hp = self.config[a.hyperparameters]
import os
import json
from pprint import pprint, pformat
from mpi4py import MPI
import math
from deephyper.evaluator import Evaluator
from deephyper.search import util, Search
from deephyper.search.nas.agent import nas_ppo_sync_a3c
logger = util.conf_logger('deephyper.search.nas.ppo_a3c_sync')
def print_logs(runner):
logger.debug('num_episodes = {}'.format(runner.global_episode))
logger.debug(' workers = {}'.format(runner.workers))
def key(d):
return json.dumps(dict(arch_seq=d['arch_seq']))
LAUNCHER_NODES = int(os.environ.get('BALSAM_LAUNCHER_NODES', 1))
WORKERS_PER_NODE = int(os.environ.get('DEEPHYPER_WORKERS_PER_NODE', 1))
class NasPPOSyncA3C(Search):
"""Neural Architecture search using proximal policy gradient with synchronous optimization.
"""
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem, run, evaluator, **kwargs)
# set in super : self.problem
# set in super : self.run_func
import functools
import math
from sys import float_info
import numpy as np
from numpy import inf
from deephyper.search import util
from skopt import Optimizer as SkOptimizer
from skopt.learning import (
ExtraTreesRegressor,
GradientBoostingQuantileRegressor,
RandomForestRegressor,
)
logger = util.conf_logger("deephyper.search.hps.optimizer.optimizer")
def isnan(x):
if isinstance(x, float):
return math.isnan(x)
elif isinstance(x, np.float64):
return np.isnan(x)
else:
return False
def convert2np(x):
if x == "nan":
return np.nan
elif type(x) is float:
import os
import time
from inspect import signature
import numpy as np
import tensorflow as tf
from deephyper.core.exceptions import DeephyperRuntimeError
from deephyper.search import util
from deephyper.nas import arch as a
from deephyper.nas import train_utils as U
logger = util.conf_logger("deephyper.model.trainer")
import keras
class TrainerTrainValid:
def __init__(self, config, model):
self.cname = self.__class__.__name__
self.config = config
self.model = model
self.callbacks = []
self.data = self.config[a.data]
self.config_hp = self.config[a.hyperparameters]
self.optimizer_name = self.config_hp.get(a.optimizer, "adam")
self.optimizer_eps = self.config_hp.get("epsilon", None)
import tensorflow as tf
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from deephyper.evaluator.evaluate import Encoder
from deephyper.search.nas.baselines import logger
from deephyper.search.nas.env.neural_architecture_envs import \
NeuralArchitectureVecEnv
from deephyper.core.logs.logging import JsonMessage as jm
try:
from mpi4py import MPI
except ImportError:
MPI = None
dhlogger = util.conf_logger('deephyper.search.nas.rl')
class ReinforcementLearningSearch(NeuralArchitectureSearch):
"""Represents different kind of RL algorithms working with NAS.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.search.nas.model.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
alg (str): algorithm to use among ['ppo2',].
network (str/function): policy network.
num_envs (int): number of environments per agent to run in
parallel, it corresponds to the number of evaluation per
batch per agent.
cache_key (str): ...
import time
import gym
import numpy as np
from gym import spaces
from deephyper.search import util
from deephyper.search.nas.baselines.common.vec_env import VecEnv
from deephyper.core.logs.logging import JsonMessage as jm
try:
from mpi4py import MPI
except ImportError:
MPI = None
dhlogger = util.conf_logger("deephyper.search.nas.env.nas_env_1")
class NasEnv1(VecEnv):
"""Multiple environments neural architecture generation. One environment corresponds to one deep neural network architecture. The observation space corresponds to the action of previous steps.
Args:
num_envs (int): number of environments to run in parallel.
space (dict): neural architecture search space from the Problem.
evaluator (Evaluator): evaluator to use to evaluate deep neural networks generated.
structure (KerasStructure): structure to build deep neural networks.
"""
def __init__(self, num_envs, space, evaluator, structure):
assert num_envs >= 1
self.space = space
import numpy as np
import tensorflow as tf
from mpi4py import MPI
import deephyper.search.nas.utils.common.tf_util as U
from deephyper.search import util
from deephyper.search.nas.agent.utils import (reward_for_final_timestep,
traj_segment_generator)
from deephyper.search.nas.utils import logger
from deephyper.search.nas.utils._logging import JsonMessage as jm
from deephyper.search.nas.utils.common import (Dataset, explained_variance,
fmt_row, zipsame)
from deephyper.search.nas.utils.common.mpi_adam_async import MpiAdamAsync
from deephyper.search.nas.utils.common.mpi_moments import mpi_moments
dh_logger = util.conf_logger('deephyper.search.nas.agent.pposgd_async')
def add_vtarg_and_adv(seg, gamma, lam):
"""
Compute target value using TD(lambda) estimator, and advantage with GAE(lambda)
"""
new = np.append(
seg["new"], 0
) # last element is only used for last vtarg, but we already zeroed it if last new = 1
vpred = np.append(seg["vpred"], seg["nextvpred"])
T = len(seg["rew"])
seg["adv"] = gaelam = np.empty(T, 'float32')
rew = seg["rew"]
lastgaelam = 0
for t in reversed(range(T)):
import json
import os.path as osp
import numpy as np
import tensorflow as tf
from mpi4py import MPI
import deephyper.search.nas.utils.common.tf_util as U
from deephyper.evaluator import Evaluator
from deephyper.search.nas.env import NasEnv
from deephyper.search.nas.utils import bench, logger
from deephyper.search.nas.utils.common import set_global_seeds
from deephyper.search import util
from deephyper.search.nas.utils._logging import JsonMessage as jm
dh_logger = util.conf_logger('deephyper.search.nas.agent.nas_random')
def traj_segment_generator(env, horizon):
t = 0
ac = env.action_space.sample() # not used, just so we have the datatype
new = True # marks if we're on first timestep of an episode
ob = env.reset()
cur_ep_ret = 0 # return in current episode
cur_ep_len = 0 # len of current episode
ep_rets = [] # returns of completed episodes in this segment
ep_lens = [] # lengths of ...
ts_i2n_ep = {}
import signal
import random
from deephyper.search.hps.optimizer import GAOptimizer
from deephyper.search import Search
from deephyper.search import util
logger = util.conf_logger("deephyper.search.hps.ga")
SERVICE_PERIOD = 2 # Delay (seconds) between main loop iterations
CHECKPOINT_INTERVAL = 10 # How many jobs to complete between optimizer checkpoints
EXIT_FLAG = False
def on_exit(signum, stack):
global EXIT_FLAG
EXIT_FLAG = True
class GA(Search):
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem, run, evaluator, **kwargs)
logger.info("Initializing GA")
self.optimizer = GAOptimizer(self.problem, self.evaluator.num_workers,
self.args)
@staticmethod
def _extend_parser(parser):
parser.add_argument(
"--ga_num_gen",
default=100,
import collections
import copy
import ConfigSpace as CS
import numpy as np
from deephyper.problem import HpProblem
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.core.parser import add_arguments_from_signature
from deephyper.search import util
from deephyper.search.nas.regevo import RegularizedEvolution
dhlogger = util.conf_logger("deephyper.search.regevomixed")
class RegularizedEvolutionMixed(RegularizedEvolution):
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
import time
import gym
import numpy as np
from gym import spaces
from deephyper.search import util
from deephyper.search.nas.baselines.common.vec_env import VecEnv
from deephyper.core.logs.logging import JsonMessage as jm
try:
from mpi4py import MPI
except ImportError:
MPI = None
dhlogger = util.conf_logger(
'deephyper.search.nas.env.neural_architecture_envs')
class NeuralArchitectureVecEnv(VecEnv):
"""Multiple environments neural architecture generation. One environment corresponds to one deep neural network architecture.
Args:
num_envs (int): number of environments to run in parallel.
space (dict): neural architecture search space from the Problem.
evaluator (Evaluator): evaluator to use to evaluate deep neural networks generated.
structure (KerasStructure): structure to build deep neural networks.
"""
def __init__(self, num_envs, space, evaluator, structure):
assert num_envs >= 1
import tensorflow as tf
import numpy as np
import math
import traceback
from sklearn.metrics import mean_squared_error
import deephyper.search.nas.model.arch as a
import deephyper.search.nas.model.train_utils as U
from deephyper.search import util
from deephyper.search.nas.utils._logging import JsonMessage as jm
from deephyper.search.nas.model.trainer.train_valid import TrainerTrainValid
logger = util.conf_logger('deephyper.model.trainer')
class TrainerRegressorTrainValid(TrainerTrainValid):
def __init__(self, config, model):
super().__init__(config, model)
import collections
import json
import os
import copy
import numpy as np
from skopt import Optimizer as SkOptimizer
from skopt.learning import RandomForestRegressor
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.core.parser import add_arguments_from_signature
from deephyper.evaluator.evaluate import Encoder
from deephyper.search import util
from deephyper.search.nas.regevo import RegularizedEvolution
dhlogger = util.conf_logger("deephyper.search.nas.agebo")
# def key(d):
# return json.dumps(dict(arch_seq=d['arch_seq']), cls=Encoder)
class AgEBO(RegularizedEvolution):
"""Aging evolution with Bayesian Optimization.
This algorithm build on the 'Regularized Evolution' from https://arxiv.org/abs/1802.01548. It cumulates Hyperparameter optimization with bayesian optimisation and Neural architecture search with regularized evolution.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.nas.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
population_size (int, optional): the number of individuals to keep in the population. Defaults to 100.
from torch.optim import Adam
import numpy as np
import gym
from gym.spaces import Discrete, Box
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.evaluator.evaluate import Encoder
# from deephyper.search.nas.env.neural_architecture_envs import NeuralArchitectureVecEnv
from nascd.env.nasenv import NasEnv2
ENV = NasEnv2
dhlogger = util.conf_logger("nascd.search.rtg_pg")
class RtgPG(NeuralArchitectureSearch):
"""Search class to run a full random neural architecture search. The search is filling every available nodes as soon as they are detected. The master job is using only 1 MPI rank.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.search.nas.model.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
"""
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem=problem, run=run, evaluator=evaluator, **kwargs)
* ``LCB`` :
* ``EI`` :
* ``PI`` :
* ``gp_hedge`` : (default)
"""
import math
import signal
import numpy as np
import skopt
from deephyper.search import Search, util
from deephyper.core.logs.logging import JsonMessage as jm
dhlogger = util.conf_logger("deephyper.search.hps.ambs")
SERVICE_PERIOD = 2 # Delay (seconds) between main loop iterations
CHECKPOINT_INTERVAL = 1 # How many jobs to complete between optimizer checkpoints
EXIT_FLAG = False
def on_exit(signum, stack):
global EXIT_FLAG
EXIT_FLAG = True
class AMBS(Search):
def __init__(
self,
problem,
import math
import ConfigSpace as CS
import numpy as np
import skopt
from deephyper.problem import HpProblem
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.core.parser import add_arguments_from_signature
from deephyper.evaluator.evaluate import Encoder
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from skopt import Optimizer as SkOptimizer
dhlogger = util.conf_logger("deephyper.search.nas.ambsmixed")
class AMBSMixed(NeuralArchitectureSearch):
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
.. code-block:: python
Problem.hyperparameters(
...,
repeat=N
...
)
"""
import numpy as np
import tensorflow as tf
from deephyper.search import util
from deephyper.nas.run.alpha import run as run_alpha
logger = util.conf_logger("deephyper.search.nas.run")
def run(config: dict) -> float:
seed = config["seed"]
repeat = config["hyperparameters"].get("repeat", 1)
if seed is not None:
np.random.seed(seed)
seeds = np.random.randint(0, 2 ** 32 - 1, repeat)
res_list = []
for i in range(repeat):
tf.keras.backend.clear_session()
if seed is not None:
config["seed"] = seeds[i]
res = run_alpha(config)
import os
import json
from random import random, seed
from deephyper.search import util
from deephyper.search.nas import NeuralArchitectureSearch
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.evaluator.evaluate import Encoder
dhlogger = util.conf_logger('deephyper.search.nas.full_random')
class Random(NeuralArchitectureSearch):
"""Search class to run a full random neural architecture search. The search is filling every available nodes as soon as they are detected. The master job is using only 1 MPI rank.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.search.nas.model.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
"""
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem=problem,
run=run,
evaluator=evaluator,
**kwargs)
seed(self.problem.seed)
if evaluator == 'balsam':
balsam_launcher_nodes = int(
import json
import math
import os
from math import ceil, log
from pprint import pformat, pprint
import tensorflow as tf
from mpi4py import MPI
from deephyper.search import Search, util
from deephyper.search.nas.agent import nas_random
logger = util.conf_logger('deephyper.search.run_nas')
def print_logs(runner):
logger.debug('num_episodes = {}'.format(runner.global_episode))
logger.debug(' workers = {}'.format(runner.workers))
def key(d):
return json.dumps(dict(arch_seq=d['arch_seq']))
LAUNCHER_NODES = int(os.environ.get('BALSAM_LAUNCHER_NODES', 1))
WORKERS_PER_NODE = int(os.environ.get('DEEPHYPER_WORKERS_PER_NODE', 1))
class RandomAgents(Search):
"""Neural Architecture search using random search.
"""
import collections
import json
import os
import copy
import numpy as np
from skopt import Optimizer as SkOptimizer
from skopt.learning import RandomForestRegressor
from deephyper.core.logs.logging import JsonMessage as jm
from deephyper.core.parser import add_arguments_from_signature
from deephyper.evaluator.evaluate import Encoder
from deephyper.search import util
from deephyper.search.nas.regevo import RegularizedEvolution
dhlogger = util.conf_logger("deephyper.search.nas.ae_hpo_nas")
# def key(d):
# return json.dumps(dict(arch_seq=d['arch_seq']), cls=Encoder)
class AeHpoNas(RegularizedEvolution):
"""Aging evolution with Bayesian Optimization.
This algorithm build on the 'Regularized Evolution' from https://arxiv.org/abs/1802.01548. It cumulates Hyperparameter optimization with bayesian optimisation and Neural architecture search with regularized evolution.
Args:
problem (str): Module path to the Problem instance you want to use for the search (e.g. deephyper.benchmark.nas.linearReg.Problem).
run (str): Module path to the run function you want to use for the search (e.g. deephyper.nas.run.quick).
evaluator (str): value in ['balsam', 'subprocess', 'processPool', 'threadPool'].
population_size (int, optional): the number of individuals to keep in the population. Defaults to 100.
