def naive_test():
'''run naive integration test'''
to_remove = [
'tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'
]
to_remove = list(map(lambda file: 'naive_test/' + file, to_remove))
remove_files(to_remove)
proc = subprocess.run(
['nnictl', 'create', '--config', 'naive_test/local.yml'])
assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
print('Spawning trials...')
nnimanager_log_path = get_nni_log_path(EXPERIMENT_URL)
current_trial = 0
for _ in range(120):
time.sleep(1)
tuner_status = read_last_line('naive_test/tuner_result.txt')
assessor_status = read_last_line('naive_test/assessor_result.txt')
experiment_status = is_experiment_done(nnimanager_log_path)
assert tuner_status != 'ERROR', 'Tuner exited with error'
assert assessor_status != 'ERROR', 'Assessor exited with error'
if experiment_status:
break
if tuner_status is not None:
for line in open('naive_test/tuner_result.txt'):
if line.strip() == 'ERROR':
break
trial = int(line.split(' ')[0])
if trial > current_trial:
current_trial = trial
print('Trial #%d done' % trial)
assert experiment_status, 'Failed to finish in 2 min'
ss1 = json.load(open('naive_test/search_space.json'))
ss2 = json.load(open('naive_test/tuner_search_space.json'))
assert ss1 == ss2, 'Tuner got wrong search space'
tuner_result = set(open('naive_test/tuner_result.txt'))
expected = set(open('naive_test/expected_tuner_result.txt'))
# Trials may complete before NNI gets assessor's result,
# so it is possible to have more final result than expected
assert tuner_result.issuperset(expected), 'Bad tuner result'
assessor_result = set(open('naive_test/assessor_result.txt'))
expected = set(open('naive_test/expected_assessor_result.txt'))
assert assessor_result == expected, 'Bad assessor result'
subprocess.run(['nnictl', 'stop'])
snooze()
def test_builtin_dispatcher(dispatch_type, dispatch_name):
'''test a dispatcher whose type is dispatch_type and name is dispatch_name'''
switch(dispatch_type, dispatch_name)
print('Testing %s...' % dispatch_name)
proc = subprocess.run(['nnictl', 'create', '--config', 'tuner_test/local.yml'])
assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
nnimanager_log_path = get_nni_log_path(EXPERIMENT_URL)
for _ in range(20):
time.sleep(3)
# check if experiment is done
experiment_status = is_experiment_done(nnimanager_log_path)
if experiment_status:
break
assert experiment_status, 'Failed to finish in 1 min'
