def test_preset_n_and_ew(preset_args,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
"""
Test command arguments - check evaluation worker with number of workers
"""
ew_flag = ['-ew']
n_flag = ['-n', Def.Flags.enw]
p_valid_params = p_utils.validation_params(preset_args)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
]
# add flags to run command
test_ew_flag = a_utils.add_one_flag_value(flag=ew_flag)
test_n_flag = a_utils.add_one_flag_value(flag=n_flag)
run_cmd.extend(test_ew_flag)
run_cmd.extend(test_n_flag)
print(str(run_cmd))
try:
proc = subprocess.Popen(run_cmd,
stdout=clres.stdout,
stderr=clres.stdout)
try:
a_utils.validate_arg_result(flag=test_ew_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
a_utils.validate_arg_result(flag=test_n_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
except AssertionError:
# close process once get assert false
proc.kill()
# if test failed - print logs
screen.error(open(clres.stdout.name).read(), crash=False)
assert False
except OSError as e:
# if test launch failed due to OSError - skip test
pytest.skip(e)
proc.kill()
def test_preset_args(preset_args,
flag,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
""" Test command arguments - the test will check all flags one-by-one."""
p_valid_params = p_utils.validation_params(preset_args)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
]
if p_valid_params.reward_test_level:
lvl = ['-lvl', '{}'.format(p_valid_params.reward_test_level)]
run_cmd.extend(lvl)
# add flags to run command
test_flag = a_utils.add_one_flag_value(flag=flag)
run_cmd.extend(test_flag)
print(str(run_cmd))
# run command
p = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
# validate results
a_utils.validate_args_results(test_flag, clres, p, start_time, time_limit)
# Close process
p.kill()
def test_preset_args(preset_args,
flag,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
""" Test command arguments - the test will check all flags one-by-one."""
p_valid_params = p_utils.validation_params(preset_args)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
]
if p_valid_params.reward_test_level:
lvl = ['-lvl', '{}'.format(p_valid_params.reward_test_level)]
run_cmd.extend(lvl)
# add flags to run command
test_flag = a_utils.add_one_flag_value(flag=flag)
if flag[0] == "-cp":
seed = ['--seed', '42']
seed_flag = a_utils.add_one_flag_value(flag=seed)
run_cmd.extend(seed_flag)
run_cmd.extend(test_flag)
print(str(run_cmd))
try:
proc = subprocess.Popen(run_cmd,
stdout=clres.stdout,
stderr=clres.stdout)
try:
a_utils.validate_arg_result(flag=test_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
except AssertionError:
# close process once get assert false
proc.kill()
# if test failed - print logs
screen.error(open(clres.stdout.name).read(), crash=False)
assert False
except OSError as e:
# if test launch failed due to OSError - skip test
pytest.skip(e)
proc.kill()
def test_preset_seed(preset_args_for_seed,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
"""
Test command arguments - the test will check seed argument with all
presets
"""
def close_processes():
"""
close all processes that still active in the process list
"""
for i in range(seed_num):
proc[i].kill()
proc = []
seed_num = 2
flag = ["--seed", str(seed_num)]
p_valid_params = p_utils.validation_params(preset_args_for_seed)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args_for_seed),
'-e',
'{}'.format("ExpName_" + preset_args_for_seed),
]
if p_valid_params.trace_test_levels:
lvl = ['-lvl', '{}'.format(p_valid_params.trace_test_levels[0])]
run_cmd.extend(lvl)
# add flags to run command
test_flag = a_utils.add_one_flag_value(flag=flag)
run_cmd.extend(test_flag)
print(str(run_cmd))
for _ in range(seed_num):
proc.append(
subprocess.Popen(run_cmd, stdout=clres.stdout,
stderr=clres.stdout))
try:
a_utils.validate_arg_result(flag=test_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
except AssertionError:
close_processes()
# if test failed - print logs
screen.error(open(clres.stdout.name).read(), crash=False)
assert False
close_processes()
def clres(request):
"""
Create both file csv and log for testing
:yield: class of both files paths
"""
class CreateCsvLog:
"""
Create a test and log paths
"""
def __init__(self, csv, log, pattern):
self.exp_path = csv
self.stdout = open(log, 'w')
self.fn_pattern = pattern
@property
def experiment_path(self):
return self.exp_path
@property
def stdout_path(self):
return self.stdout
@experiment_path.setter
def experiment_path(self, val):
self.exp_path = val
@stdout_path.setter
def stdout_path(self, val):
self.stdout = open(val, 'w')
# get preset name from test request params
idx = 0 if 'preset' in list(request.node.funcargs.items())[0][0] else 1
p_name = list(request.node.funcargs.items())[idx][1]
p_valid_params = p_utils.validation_params(p_name)
sys.path.append('.')
test_name = 'ExpName_{}'.format(p_name)
test_path = os.path.join(Def.Path.experiments, test_name)
if path.exists(test_path):
shutil.rmtree(test_path)
# get the stdout for logs results
log_file_name = 'test_log_{}.txt'.format(p_name)
fn_pattern = '*.csv' if p_valid_params.num_workers > 1 else 'worker_0*.csv'
res = CreateCsvLog(test_path, log_file_name, fn_pattern)
yield res
# clean files
if path.exists(res.exp_path):
shutil.rmtree(res.exp_path)
if path.exists(res.stdout.name):
os.remove(res.stdout.name)
def test_preset_args(preset_args,
flag,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
""" Test command arguments - the test will check all flags one-by-one."""
p_valid_params = p_utils.validation_params(preset_args)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
]
if p_valid_params.reward_test_level:
lvl = ['-lvl', '{}'.format(p_valid_params.reward_test_level)]
run_cmd.extend(lvl)
# add flags to run command
test_flag = a_utils.add_one_flag_value(flag=flag)
run_cmd.extend(test_flag)
print(str(run_cmd))
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
try:
a_utils.validate_arg_result(flag=test_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
except AssertionError:
# close process once get assert false
proc.kill()
assert False
proc.kill()
def test_preset_n_and_ew_and_onnx(preset_args,
clres,
start_time=time.time(),
time_limit=Def.TimeOuts.test_time_limit):
"""
Test command arguments - check evaluation worker, number of workers and
onnx.
"""
ew_flag = ['-ew']
n_flag = ['-n', Def.Flags.enw]
onnx_flag = ['-onnx']
s_flag = ['-s', Def.Flags.css]
p_valid_params = p_utils.validation_params(preset_args)
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
]
# add flags to run command
test_ew_flag = a_utils.add_one_flag_value(flag=ew_flag)
test_n_flag = a_utils.add_one_flag_value(flag=n_flag)
test_onnx_flag = a_utils.add_one_flag_value(flag=onnx_flag)
test_s_flag = a_utils.add_one_flag_value(flag=s_flag)
run_cmd.extend(test_ew_flag)
run_cmd.extend(test_n_flag)
run_cmd.extend(test_onnx_flag)
run_cmd.extend(test_s_flag)
print(str(run_cmd))
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
try:
# Check csv files has been created
a_utils.validate_arg_result(flag=test_ew_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
# Check csv files created same as the number of the workers
a_utils.validate_arg_result(flag=test_n_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
# Check checkpoint files
a_utils.validate_arg_result(flag=test_s_flag,
p_valid_params=p_valid_params,
clres=clres,
process=proc,
start_time=start_time,
timeout=time_limit)
# TODO: add onnx check; issue found #257
except AssertionError:
# close process once get assert false
proc.kill()
assert False
proc.kill()
def test_restore_checkpoint(preset_args,
clres,
framework,
timeout=Def.TimeOuts.test_time_limit):
"""
Create checkpoints and restore them in second run.
:param preset_args: all preset that can be tested for argument tests
:param clres: logs and csv files
:param framework: name of the test framework
:param timeout: max time for test
"""
def _create_cmd_and_run(flag):
"""
Create default command with given flag and run it
:param flag: name of the tested flag, this flag will be extended to the
running command line
:return: active process
"""
run_cmd = [
'python3',
'rl_coach/coach.py',
'-p',
'{}'.format(preset_args),
'-e',
'{}'.format("ExpName_" + preset_args),
'--seed',
'{}'.format(4),
'-f',
'{}'.format(framework),
]
test_flag = a_utils.add_one_flag_value(flag=flag)
run_cmd.extend(test_flag)
print(str(run_cmd))
p = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
return p
start_time = time.time()
if framework == "mxnet":
# update preset name - for mxnet framework we are using *_DQN
preset_args = Def.Presets.mxnet_args_test[0]
# update logs paths
test_name = 'ExpName_{}'.format(preset_args)
test_path = os.path.join(Def.Path.experiments, test_name)
clres.experiment_path = test_path
clres.stdout_path = 'test_log_{}.txt'.format(preset_args)
p_valid_params = p_utils.validation_params(preset_args)
create_cp_proc = _create_cmd_and_run(flag=['--checkpoint_save_secs', '5'])
# wait for checkpoint files
csv_list = a_utils.get_csv_path(clres=clres)
assert len(csv_list) > 0
exp_dir = os.path.dirname(csv_list[0])
checkpoint_dir = os.path.join(exp_dir, Def.Path.checkpoint)
checkpoint_test_dir = os.path.join(Def.Path.experiments, Def.Path.test_dir)
if os.path.exists(checkpoint_test_dir):
shutil.rmtree(checkpoint_test_dir)
res = a_utils.is_reward_reached(csv_path=csv_list[0],
p_valid_params=p_valid_params,
start_time=start_time,
time_limit=timeout)
if not res:
screen.error(open(clres.stdout.name).read(), crash=False)
assert False
entities = a_utils.get_files_from_dir(checkpoint_dir)
assert len(entities) > 0
assert any(".ckpt." in file for file in entities)
# send CTRL+C to close experiment
create_cp_proc.send_signal(signal.SIGINT)
if os.path.isdir(checkpoint_dir):
shutil.copytree(exp_dir, checkpoint_test_dir)
shutil.rmtree(exp_dir)
create_cp_proc.kill()
checkpoint_test_dir = "{}/{}".format(checkpoint_test_dir,
Def.Path.checkpoint)
# run second time with checkpoint folder (restore)
restore_cp_proc = _create_cmd_and_run(
flag=['-crd', checkpoint_test_dir, '--evaluate'])
new_csv_list = test_utils.get_csv_path(clres=clres)
time.sleep(10)
csv = pd.read_csv(new_csv_list[0])
res = csv['Episode Length'].values[-1]
expected_reward = 100
assert res >= expected_reward, Def.Consts.ASSERT_MSG.format(
str(expected_reward), str(res))
restore_cp_proc.kill()
test_folder = os.path.join(Def.Path.experiments, Def.Path.test_dir)
if os.path.exists(test_folder):
shutil.rmtree(test_folder)
def test_restore_checkpoint(preset_args, clres, start_time=time.time(),
timeout=Def.TimeOuts.test_time_limit):
""" Create checkpoint and restore them in second run."""
def _create_cmd_and_run(flag):
run_cmd = [
'python3', 'rl_coach/coach.py',
'-p', '{}'.format(preset_args),
'-e', '{}'.format("ExpName_" + preset_args),
]
test_flag = a_utils.add_one_flag_value(flag=flag)
run_cmd.extend(test_flag)
p = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
return p
p_valid_params = p_utils.validation_params(preset_args)
create_cp_proc = _create_cmd_and_run(flag=['--checkpoint_save_secs', '5'])
# wait for checkpoint files
csv_list = a_utils.get_csv_path(clres=clres)
assert len(csv_list) > 0
exp_dir = os.path.dirname(csv_list[0])
checkpoint_dir = os.path.join(exp_dir, Def.Path.checkpoint)
checkpoint_test_dir = os.path.join(Def.Path.experiments, Def.Path.test_dir)
if os.path.exists(checkpoint_test_dir):
shutil.rmtree(checkpoint_test_dir)
assert a_utils.is_reward_reached(csv_path=csv_list[0],
p_valid_params=p_valid_params,
start_time=start_time, time_limit=timeout)
entities = a_utils.get_files_from_dir(checkpoint_dir)
assert len(entities) > 0
assert "checkpoint" in entities
assert any(".ckpt." in file for file in entities)
# send CTRL+C to close experiment
create_cp_proc.send_signal(signal.SIGINT)
csv = pd.read_csv(csv_list[0])
rewards = csv['Evaluation Reward'].values
rewards = rewards[~np.isnan(rewards)]
max_reward = np.amax(rewards)
if os.path.isdir(checkpoint_dir):
shutil.copytree(exp_dir, checkpoint_test_dir)
shutil.rmtree(exp_dir)
create_cp_proc.kill()
checkpoint_test_dir = "{}/{}".format(checkpoint_test_dir,
Def.Path.checkpoint)
# run second time with checkpoint folder (restore)
restore_cp_proc = _create_cmd_and_run(flag=['-crd', checkpoint_test_dir,
'--evaluate'])
new_csv_list = test_utils.get_csv_path(clres=clres)
time.sleep(10)
csv = pd.read_csv(new_csv_list[0])
res = csv['Episode Length'].values[-1]
assert res == max_reward, Def.Consts.ASSERT_MSG.format(str(max_reward),
str(res))
restore_cp_proc.kill()
test_folder = os.path.join(Def.Path.experiments, Def.Path.test_dir)
if os.path.exists(test_folder):
shutil.rmtree(test_folder)
