def _iterate(self, s, budget=MAX_INT, skip_last=0):
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
# initial number of iterations per config
r = int(self.R * self.eta**(-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = sample_configurations(self.config_space, n)
time_elapsed = time.time() - start_time
self.logger.info(
"Choosing next batch of configurations took %.2f sec." %
time_elapsed)
with ParallelProcessEvaluator(self.eval_func,
n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() >= budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta**(-i)
n_resource = r * self.eta**i
self.logger.info(
"MFSE: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
val_losses = executor.parallel_execute(
T,
resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
def select_network_architectures(self,
algorithm_candidates,
dl_evaluator,
num_arch=1,
**kwargs):
if len(algorithm_candidates) == 1:
return algorithm_candidates
_archs = algorithm_candidates.copy()
if self.n_jobs > 1:
# self.executor = ParallelProcessEvaluator(dl_evaluator, n_worker=self.n_jobs)
with ParallelProcessEvaluator(dl_evaluator,
n_worker=self.n_jobs) as executor:
self.logger.info('Create parallel executor with n_jobs=%d' %
self.n_jobs)
while len(_archs) > num_arch:
_archs = self.exec_SEE(_archs, executor=executor)
else:
self.nas_evaluator = dl_evaluator
while len(_archs) > num_arch:
_archs = self.exec_SEE(_archs)
return _archs
def _iterate(self, s, budget=MAX_INT, skip_last=0):
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
# initial number of iterations per config
r = int(self.R * self.eta**(-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = self.get_candidate_configurations(n)
time_elapsed = time.time() - start_time
self.logger.info(
"Choosing next batch of configurations took %.2f sec." %
time_elapsed)
with ParallelProcessEvaluator(self.eval_func,
n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() >= budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta**(-i)
n_resource = r * self.eta**i
self.logger.info(
"BOHB: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
val_losses = executor.parallel_execute(
T,
resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
self.time_ticks.extend(
[time.time() - self.global_start_time] * len(T))
# Only update results using maximal resources
if self.config_generator != 'smac':
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.config_gen.new_result(T[_id], _val_loss)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
# Refit the surrogate model.
resource_val = self.iterate_r[-1]
if len(self.target_y[resource_val]) > 1:
if self.config_generator == 'smac':
normalized_y = std_normalization(self.target_y[resource_val])
self.surrogate.train(
convert_configurations_to_array(
self.target_x[resource_val]),
np.array(normalized_y, dtype=np.float64))
import time
sys.path.append(os.getcwd())
from solnml.utils.proc_thread.proc_func import kill_proc_tree
from solnml.components.computation.parallel_process import ParallelProcessEvaluator
def evaluate_func(a=None, data_node=None, name='fe', resource_ratio=0.1):
cnt = 0
for i in range(100000000):
cnt += 129
cnt %= 123200
for _ in range(100):
cnt += 3
return 2 * data_node
try:
executor = ParallelProcessEvaluator(evaluate_func, n_worker=3)
_configs = [1, 2, 3, 4, 5, 6]
res = executor.parallel_execute(_configs, resource_ratio=0.1)
print(res)
except Exception as e:
print(e)
finally:
pid = os.getpid()
kill_proc_tree(pid, including_parent=False)
print('Task finished.')
print('=' * 100)
def _iterate(self, s, budget=MAX_INT, skip_last=0):
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta ** s))
# initial number of iterations per config
r = int(self.R * self.eta ** (-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = self.mf_advisor.get_suggestions(n_suggestions=n)
time_elapsed = time.time() - start_time
self.logger.info("Choosing next batch of configurations took %.2f sec." % time_elapsed)
full_config_list = list()
full_perf_list = list()
with ParallelProcessEvaluator(self.eval_func, n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() > budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta ** (-i)
n_resource = r * self.eta ** i
self.logger.info("MFSE: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
if self.n_workers > 1:
# TODO: Time limit control
val_losses = executor.parallel_execute(T, resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(_val_loss)
else:
val_losses = list()
for config in T:
if time.time() - start_time > budget:
self.logger.warning('Time limit exceeded!')
break
try:
with time_limit(self.per_run_time_limit):
val_loss = self.eval_func(config, resource_ratio=float(n_resource / self.R),
eta=self.eta, first_iter=(i == 0))
except Exception as e:
# TODO: Distinguish error type
val_loss = np.inf
val_losses.append(val_loss)
if np.isfinite(val_loss):
self.target_x[int(n_resource)].append(config)
self.target_y[int(n_resource)].append(val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
full_config_list.extend(T)
full_perf_list.extend(val_losses)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
if len(self.target_y[self.iterate_r[-1]]) != 0:
observations = list()
for item in self.target_x:
config_dict = OrderedDict()
for i, config in enumerate(self.target_x[item]):
config_dict[config] = self.target_y[item][i]
observations.append(config_dict)
self.mf_advisor.update_mf_observations(observations)
return full_config_list, full_perf_list
def run(self, dl_evaluator):
start_time = time.time()
inc_config, inc_perf = None, np.inf
architecture_candidates = self.architectures.copy()
for _arch in architecture_candidates:
self.eval_hist_configs[_arch] = dict()
self.eval_hist_perfs[_arch] = dict()
self.evaluation_stats['timestamps'] = list()
self.evaluation_stats['val_scores'] = list()
with ParallelProcessEvaluator(dl_evaluator,
n_worker=self.n_jobs) as executor:
terminate_proc = False
while not terminate_proc:
r = 1
C = self.sample_configs_for_archs(
architecture_candidates,
self.N,
sampling_strategy=self.sampling_strategy)
while r < self.R or (r == self.R
and len(architecture_candidates) == 1):
for _arch in architecture_candidates:
if r not in self.eval_hist_configs[_arch]:
self.eval_hist_configs[_arch][r] = list()
self.eval_hist_perfs[_arch][r] = list()
self.logger.info('Evalutions [r=%d]' % r)
self.logger.info(
'Start to evaluate %d configurations with %d resource'
% (len(C), r))
self.logger.info('=' * 20)
_start_time = time.time()
if _start_time >= start_time + self.time_limit:
terminate_proc = True
break
if self.n_jobs > 1:
val_losses = executor.parallel_execute(
C,
resource_ratio=float(r / self.R),
eta=self.eta,
first_iter=(r == 1))
for _id, val_loss in enumerate(val_losses):
if np.isfinite(val_loss):
_arch = C[_id]['estimator']
self.eval_hist_configs[_arch][r].append(C[_id])
self.eval_hist_perfs[_arch][r].append(val_loss)
self.evaluation_stats['timestamps'].append(
time.time() - start_time)
self.evaluation_stats['val_scores'].append(
val_loss)
else:
val_losses = list()
for config in C:
val_loss = dl_evaluator(config,
resource_ratio=float(
r / self.R),
eta=self.eta,
first_iter=(r == 1))
val_losses.append(val_loss)
if np.isfinite(val_loss):
_arch = config['estimator']
self.eval_hist_configs[_arch][r].append(config)
self.eval_hist_perfs[_arch][r].append(val_loss)
self.evaluation_stats['timestamps'].append(
time.time() - start_time)
self.evaluation_stats['val_scores'].append(
val_loss)
self.logger.info('Evaluations [R=%d] took %.2f seconds' %
(r, time.time() - _start_time))
# Train surrogate
if self.sampling_strategy == 'bohb':
if r == self.R:
for i, _config in enumerate(C):
if np.isfinite(val_losses[i]):
_arch = _config['estimator']
self.tpe_config_gen[_arch].new_result(
_config, val_losses[i], r)
elif self.sampling_strategy == 'mfse':
for _arch in architecture_candidates: # Only update surrogate in candidates
normalized_y = std_normalization(
self.eval_hist_perfs[_arch][r])
if len(
self.eval_hist_configs[_arch]
[r]) == 0: # No configs for this architecture
continue
self.mfse_config_gen[_arch]['surrogate'].train(
convert_configurations_to_array(
self.eval_hist_configs[_arch][r]),
np.array(normalized_y, dtype=np.float64),
r=r)
if self.elimination_strategy == 'bandit':
indices = np.argsort(val_losses)
if len(C) >= self.eta:
C = [C[i] for i in indices]
reduced_num = int(len(C) / self.eta)
C = C[0:reduced_num]
else:
C = [C[indices[0]]]
else:
if r > 1:
val_losses_previous_iter = self.query_performance(
C, r // self.eta)
previous_inc_loss = np.min(
val_losses_previous_iter)
indices = np.argsort(val_losses)
C = [
C[idx] for idx in indices
if val_losses[idx] < previous_inc_loss
]
if inc_perf > val_losses[indices[0]]:
inc_perf = val_losses[indices[0]]
inc_config = C[0]
r *= self.eta
# Remove tmp model
if dl_evaluator.continue_training:
for filename in os.listdir(dl_evaluator.model_dir):
# Temporary model
if 'tmp_%s' % dl_evaluator.timestamp in filename:
try:
filepath = os.path.join(
dl_evaluator.model_dir, filename)
os.remove(filepath)
except Exception:
pass
archs, reduced_archs = [config['estimator']
for config in C], list()
# Preserve the partial-relationship order.
for _arch in archs:
if _arch not in reduced_archs:
reduced_archs.append(_arch)
architecture_candidates = reduced_archs
print('=' * 20)
print('Reduced architectures:', architecture_candidates)
print('=' * 20)
return inc_config, inc_perf
def _iterate(self, s, budget=MAX_INT, skip_last=0):
if self.weight_update_id > self.s_max:
self.update_weight()
self.weight_update_id += 1
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta ** s))
# initial number of iterations per config
r = int(self.R * self.eta ** (-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = self.fetch_candidate_configurations(n)
time_elapsed = time.time() - start_time
self.logger.info("Choosing next batch of configurations took %.2f sec." % time_elapsed)
with ParallelProcessEvaluator(self.eval_func, n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() >= budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta ** (-i)
n_resource = r * self.eta ** i
self.logger.info("MFSE: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
if self.n_workers > 1:
val_losses = executor.parallel_execute(T, resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(_val_loss)
else:
val_losses = list()
for config in T:
try:
val_loss = self.eval_func(config, resource_ratio=float(n_resource / self.R),
eta=self.eta, first_iter=(i == 0))
except Exception as e:
val_loss = np.inf
val_losses.append(val_loss)
if np.isfinite(val_loss):
self.target_x[int(n_resource)].append(config)
self.target_y[int(n_resource)].append(val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
for item in self.iterate_r[self.iterate_r.index(r):]:
if len(self.target_y[item]) == 0:
continue
normalized_y = std_normalization(self.target_y[item])
self.weighted_surrogate.train(convert_configurations_to_array(self.target_x[item]),
np.array(normalized_y, dtype=np.float64), r=item)