parser = argparse.ArgumentParser()
parser.add_argument("--dir", type=check_file_existence, required=True)
parser.add_argument("--param_names", type=check_param_names, required=True)
parser.add_argument("--algo_name", choices=SUPPORTED_ALGOS, required=True)
parser.add_argument("--env_name", choices=SUPPORTED_ENVS, required=True)
args = parser.parse_args()
logger = Log("analyze_number_of_executions_skipped")
logging.basicConfig(
filename=os.path.join(args.dir, "analyze_number_of_executions_skipped.txt"), filemode="w", level=logging.DEBUG
)
env_variables = instantiate_env_variables(
algo_name=args.algo_name,
discrete_action_space=False, # I do not care about this parameter in this case
env_name=args.env_name,
param_names=args.param_names,
)
iterations_dirs = glob.glob(os.path.join(args.dir, "n_iterations_*"))
iterations_dirs = filter_resampling_artifacts(files=iterations_dirs)
iterations_dirs_sorted = sorted(iterations_dirs, key=get_result_dir_iteration_number)
executions_skipped_dict = dict()
all_search_points = []
time_elapsed_per_run = []
time_taken_per_repetition = []
regression_time_per_repetition = []
all_frontier_points = []
executions_skipped = []
def get_binary_search_candidate(
t_env_variables: EnvVariables,
f_env_variables: EnvVariables,
algo_name: str,
env_name: str,
param_names,
discrete_action_space: bool,
buffer_env_predicate_pairs: BufferEnvPredicatePairs,
) -> EnvVariables:
original_max_iterations = 50
logger = Log("get_binary_search_candidate")
max_number_iterations = original_max_iterations
candidate_new_env_variables = copy.deepcopy(t_env_variables)
while True:
# compute all possible combinations of environments
candidates_dict = dict()
t_f_env_variables = random.choice([(t_env_variables, True),
(f_env_variables, False)])
for i in range(len(t_env_variables.get_params())):
new_value = (
t_env_variables.get_param(index=i).get_current_value() +
f_env_variables.get_param(index=i).get_current_value()) / 2
if i not in candidates_dict:
candidates_dict[i] = []
if (t_env_variables.get_param(index=i).get_current_value() !=
f_env_variables.get_param(index=i).get_current_value()):
candidates_dict[i].append(new_value)
for index in range(len(t_env_variables.get_params())):
if index not in candidates_dict:
candidates_dict[index] = []
if index != i:
candidates_dict[index].append(
t_f_env_variables[0].get_values()[index])
all_candidates = list(
itertools.product(*list(candidates_dict.values())))
logger.info("t_env: {}, f_env: {}".format(
t_env_variables.get_params_string(),
f_env_variables.get_params_string()))
logger.info("all candidates binary search: {}".format(all_candidates))
all_candidates_env_variables_filtered = []
all_candidates_env_variables = []
for candidate_values in all_candidates:
env_values = dict()
for i in range(len(t_f_env_variables[0].get_params())):
param_name = t_f_env_variables[0].get_param(index=i).get_name()
env_values[param_name] = candidate_values[i]
candidate_env_variables = instantiate_env_variables(
algo_name=algo_name,
discrete_action_space=discrete_action_space,
env_name=env_name,
param_names=param_names,
env_values=env_values,
)
# do not consider candidate = t_f_env_variables
if not candidate_env_variables.is_equal(
t_env_variables) and not candidate_env_variables.is_equal(
f_env_variables):
if not buffer_env_predicate_pairs.is_already_evaluated(
candidate_env_variables=candidate_env_variables):
all_candidates_env_variables_filtered.append(
candidate_env_variables)
all_candidates_env_variables.append(candidate_env_variables)
if len(all_candidates_env_variables_filtered) > 0:
candidate_new_env_variables = random.choice(
all_candidates_env_variables_filtered)
break
else:
assert len(
all_candidates
) > 0, "there must be at least one candidate env for binary search"
candidate_env_variables_already_evaluated = random.choice(
all_candidates_env_variables_filtered)
if t_f_env_variables[1]:
t_env_variables = copy.deepcopy(
candidate_env_variables_already_evaluated)
else:
f_env_variables = copy.deepcopy(
candidate_env_variables_already_evaluated)
max_number_iterations -= 1
if max_number_iterations == 0:
break
assert max_number_iterations > 0, "Could not binary mutate any param of envs {} and {} in {} steps".format(
t_env_variables.get_params_string(),
f_env_variables.get_params_string(), str(original_max_iterations))
assert not candidate_new_env_variables.is_equal(
t_env_variables
) and not candidate_new_env_variables.is_equal(
f_env_variables
), "candidate_env_variables {} must be different than t_env_variables {} and f_env_variables {}".format(
candidate_new_env_variables.get_params_string(),
t_env_variables.get_params_string(),
f_env_variables.get_params_string(),
)
return candidate_new_env_variables
parser.add_argument("--num_threads", type=int, default=0)
parser.add_argument("--sb_version", type=str, default="sb2")
parser.add_argument("--frontier_path", type=str, default=None)
parser.add_argument("--runs_for_probability_estimation", type=int, default=1)
args = parser.parse_args()
if args.random_seed:
args.seed = np.random.randint(2 ** 32 - 1, dtype="int64").item()
else:
args.seed = 0
env_kwargs = instantiate_env_variables(
algo_name=args.algo_name,
discrete_action_space=args.discrete_action_space,
env_name=args.env_name,
env_values=args.env_values,
param_names=args.param_names,
)
env_eval_callback = None
if args.instantiate_eval_callback:
env_eval_callback = instantiate_eval_callback(env_name=args.env_name)
if args.num_threads:
print(f"Setting torch.num_threads to {args.num_threads}")
th.set_num_threads(args.num_threads)
th.set_num_interop_threads(args.num_threads)
logger = Log("main")
agent = Agent(
param_names = None
if args.param_names:
try:
param_names = args.param_names.split(sep=",")
if len(param_names) != 2:
raise SyntaxError("2 param names must be specified: {}".format(
args.param_names))
except Exception:
raise SyntaxError("param names must be comma separated: {}".format(
args.param_names))
env_variables = instantiate_env_variables(
algo_name=args.algo_name,
discrete_action_space=args.discrete_action_space,
env_name=args.env_name,
param_names=param_names,
model_suffix=args.model_suffix,
)
env_eval_callback = instantiate_eval_callback(env_name=args.env_name)
if not args.stub_agent:
agent = Agent(
algo_name=args.algo_name,
env_name=args.env_name,
log_to_tensorboard=args.log_to_tensorboard,
tb_log_name=args.tb_log_name,
train_total_timesteps=args.train_total_timesteps,
n_eval_episodes=args.n_eval_episodes,
render=args.render,
num_envs=args.num_envs,
def __init__(
self,
agent: AbstractAgent,
num_iterations: int,
algo_name: str,
env_name: str,
tb_log_name: str,
continue_learning_suffix: str,
env_variables: EnvVariables,
param_names=None,
runs_for_probability_estimation: int = 1,
buffer_file: str = None,
archive_file: str = None,
executions_skipped_file: str = None,
parallelize_search: bool = False,
monitor_search_every: bool = False,
binary_search_epsilon: float = 0.05,
start_search_time: float = None,
starting_progress_report_number: int = 0,
stop_at_first_iteration: bool = False,
exp_suffix: str = None,
):
assert agent, "agent should have a value: {}".format(agent)
assert algo_name, "algo_name should have a value: {}".format(algo_name)
assert env_name, "env_name should have a value: {}".format(env_name)
self.agent = agent
self.num_iterations = num_iterations
self.init_env_variables = env_variables
self.previous_num_iterations = None
self.start_time = time.time()
self.logger = Log("Random")
self.param_names = param_names
self.all_params = env_variables.instantiate_env()
self.runs_for_probability_estimation = runs_for_probability_estimation
self.buffer_file = buffer_file
self.archive_file = archive_file
self.parallelize_search = parallelize_search
self.stop_at_first_iteration = stop_at_first_iteration
self.exp_suffix = exp_suffix
if param_names:
self.param_names_string = "_".join(param_names)
# TODO: refactor buffer restoring in abstract class extended by search algo
# (for now only random search and alphatest)
if buffer_file:
previously_saved_buffer = read_saved_buffer(
buffer_file=buffer_file)
index_last_slash = buffer_file.rindex("/")
self.algo_save_dir = buffer_file[:index_last_slash]
self.logger.debug(
"Algo save dir from restored execution: {}".format(
self.algo_save_dir))
self.buffer_env_predicate_pairs = BufferEnvPredicatePairs(
save_dir=self.algo_save_dir)
self.archive = Archive(save_dir=self.algo_save_dir,
epsilon=binary_search_epsilon)
# restore buffer
for buffer_item in previously_saved_buffer:
previous_env_variables = instantiate_env_variables(
algo_name=algo_name,
discrete_action_space=self.
all_params["discrete_action_space"],
env_name=env_name,
param_names=param_names,
env_values=buffer_item.get_env_values(),
)
self.buffer_env_predicate_pairs.append(
EnvPredicatePair(
env_variables=previous_env_variables,
pass_probability=buffer_item.get_pass_probability(),
predicate=buffer_item.is_predicate(),
regression_probability=buffer_item.
get_regression_probability(),
probability_estimation_runs=buffer_item.
get_probability_estimation_runs(),
regression_estimation_runs=buffer_item.
get_regression_estimation_runs(),
model_dirs=buffer_item.get_model_dirs(),
))
assert archive_file, (
"when buffer file is available so needs to be the archive file to "
"restore a previous execution")
try:
previous_num_iterations_buffer = get_result_file_iteration_number(
filename=buffer_file)
previous_num_iterations_archive = get_result_file_iteration_number(
filename=archive_file)
assert (previous_num_iterations_buffer ==
previous_num_iterations_archive
), "The two nums must coincide: {}, {}".format(
previous_num_iterations_buffer,
previous_num_iterations_archive)
previous_num_iterations = previous_num_iterations_buffer + 1
except ValueError as e:
raise ValueError(e)
self.previous_num_iterations = previous_num_iterations
self.logger.info(
"Restore previous execution of {} iterations.".format(
previous_num_iterations))
# restore archive
previously_saved_archive = read_saved_archive(
archive_file=archive_file)
t_env_variables = None
f_env_variables = None
for env_values, predicate in previously_saved_archive:
all_params = env_variables.instantiate_env()
previous_env_variables = instantiate_env_variables(
algo_name=algo_name,
discrete_action_space=all_params["discrete_action_space"],
env_name=env_name,
param_names=param_names,
env_values=env_values,
)
if predicate:
t_env_variables = previous_env_variables
else:
f_env_variables = previous_env_variables
if t_env_variables and f_env_variables:
self.archive.append(t_env_variables=t_env_variables,
f_env_variables=f_env_variables)
t_env_variables = None
f_env_variables = None
# restore executions skipped
previously_saved_executions_skipped = read_saved_buffer_executions_skipped(
buffer_executions_skipped_file=executions_skipped_file)
for buffer_executions_skipped_item in previously_saved_executions_skipped:
previous_env_variables_skipped = instantiate_env_variables(
algo_name=algo_name,
discrete_action_space=self.
all_params["discrete_action_space"],
env_name=env_name,
param_names=param_names,
env_values=buffer_executions_skipped_item.
env_values_skipped,
)
env_predicate_pair_skipped = EnvPredicatePair(
env_variables=previous_env_variables_skipped,
predicate=buffer_executions_skipped_item.predicate)
previous_env_variables_executed = instantiate_env_variables(
algo_name=algo_name,
discrete_action_space=self.
all_params["discrete_action_space"],
env_name=env_name,
param_names=param_names,
env_values=buffer_executions_skipped_item.
env_values_executed,
)
env_predicate_pair_executed = EnvPredicatePair(
env_variables=previous_env_variables_executed,
predicate=buffer_executions_skipped_item.predicate)
self.buffer_executions_skipped.append(
ExecutionSkipped(
env_predicate_pair_skipped=
env_predicate_pair_skipped,
env_predicate_pair_executed=
env_predicate_pair_executed,
search_component=buffer_executions_skipped_item.
search_component,
))
else:
attempt = 0
suffix = "n_iterations_"
if self.param_names:
suffix += self.param_names_string + "_"
if self.exp_suffix:
suffix += self.exp_suffix + "_"
suffix += str(num_iterations)
algo_save_dir = os.path.abspath(HOME + "/random/" + env_name +
"/" + algo_name + "/" + suffix +
"_" + str(attempt))
_algo_save_dir = algo_save_dir
while os.path.exists(_algo_save_dir):
attempt += 1
_algo_save_dir = algo_save_dir[:-1] + str(attempt)
self.algo_save_dir = _algo_save_dir
os.makedirs(self.algo_save_dir)
self.buffer_env_predicate_pairs = BufferEnvPredicatePairs(
save_dir=self.algo_save_dir)
# assuming initial env_variables satisfies the predicate of adequate performance
if self.runs_for_probability_estimation:
env_predicate_pair = EnvPredicatePair(
env_variables=self.init_env_variables,
predicate=True,
probability_estimation_runs=[True] *
self.runs_for_probability_estimation,
)
else:
env_predicate_pair = EnvPredicatePair(
env_variables=self.init_env_variables, predicate=True)
self.buffer_env_predicate_pairs.append(env_predicate_pair)
self.buffer_executions_skipped = BufferExecutionsSkipped(
save_dir=self.algo_save_dir)
self.archive = Archive(save_dir=self.algo_save_dir,
epsilon=binary_search_epsilon)
self.env_name = env_name
self.algo_name = algo_name
self.tb_log_name = tb_log_name
self.continue_learning_suffix = continue_learning_suffix
self.binary_search_epsilon = binary_search_epsilon
self.runner = Runner(
agent=self.agent,
runs_for_probability_estimation=self.
runs_for_probability_estimation,
)
self.monitor_search_every = monitor_search_every
self.monitor_progress = None
if self.monitor_search_every != -1 and self.monitor_search_every > 0:
self.monitor_progress = MonitorProgress(
algo_name=self.algo_name,
env_name=standardize_env_name(env_name=self.env_name),
results_dir=self.algo_save_dir,
param_names_string=self.param_names_string,
search_type="random",
start_search_time=start_search_time,
starting_progress_report_number=starting_progress_report_number,
)