def launch_experiment(args, experiment_config, mode, experiment_id, config_version): '''follow steps to start rest server and start experiment''' # check packages for tuner package_name, module_name = None, None if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): package_name = experiment_config['tuner']['builtinTunerName'] module_name, _ = get_builtin_module_class_name('tuners', package_name) elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): package_name = experiment_config['advisor']['builtinAdvisorName'] module_name, _ = get_builtin_module_class_name('advisors', package_name) if package_name and module_name: try: stdout_full_path, stderr_full_path = get_log_path(experiment_id) with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file) except CalledProcessError: print_error('some errors happen when import package %s.' %(package_name)) print_log_content(experiment_id) if package_name in ['SMAC', 'BOHB', 'PPOTuner']: print_error(f'The dependencies for {package_name} can be installed through pip install nni[{package_name}]') raise if config_version == 1: log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else NNI_HOME_DIR else: log_dir = experiment_config['experimentWorkingDirectory'] if experiment_config.get('experimentWorkingDirectory') else NNI_HOME_DIR log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None #view experiment mode do not need debug function, when view an experiment, there will be no new logs created foreground = False if mode != 'view': foreground = args.foreground if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): log_level = 'debug' # start rest server if config_version == 1: platform = experiment_config['trainingServicePlatform'] elif isinstance(experiment_config['trainingService'], list): platform = 'hybrid' else: platform = experiment_config['trainingService']['platform'] rest_process, start_time = start_rest_server(args.port, platform, \ mode, experiment_id, foreground, log_dir, log_level) # save experiment information Experiments().add_experiment(experiment_id, args.port, start_time, platform, experiment_config.get('experimentName', 'N/A'), pid=rest_process.pid, logDir=log_dir) # Deal with annotation if experiment_config.get('useAnnotation'): path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') if not os.path.isdir(path): os.makedirs(path) path = tempfile.mkdtemp(dir=path) nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) experiment_config['trial']['codeDir'] = code_dir search_space = generate_search_space(code_dir) experiment_config['searchSpace'] = search_space assert search_space, ERROR_INFO % 'Generated search space is empty' elif config_version == 1: if experiment_config.get('searchSpacePath'): search_space = get_json_content(experiment_config.get('searchSpacePath')) experiment_config['searchSpace'] = search_space else: experiment_config['searchSpace'] = '' # check rest server running, _ = check_rest_server(args.port) if running: print_normal('Successfully started Restful server!') else: print_error('Restful server start failed!') print_log_content(experiment_id) try: kill_command(rest_process.pid) except Exception: raise Exception(ERROR_INFO % 'Rest server stopped!') exit(1) if config_version == 1 and mode != 'view': # set platform configuration set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ experiment_id, rest_process) # start a new experiment print_normal('Starting experiment...') # set debug configuration if mode != 'view' and experiment_config.get('debug') is None: experiment_config['debug'] = args.debug if config_version == 1: response = set_experiment_v1(experiment_config, mode, args.port, experiment_id) else: response = set_experiment_v2(experiment_config, mode, args.port, experiment_id) if response: if experiment_id is None: experiment_id = json.loads(response.text).get('experiment_id') else: print_error('Start experiment failed!') print_log_content(experiment_id) try: kill_command(rest_process.pid) except Exception: raise Exception(ERROR_INFO % 'Restful server stopped!') exit(1) if experiment_config.get('nniManagerIp'): web_ui_url_list = ['http://{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))] else: web_ui_url_list = get_local_urls(args.port) Experiments().update_experiment(experiment_id, 'webuiUrl', web_ui_url_list) print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) if mode != 'view' and args.foreground: try: while True: log_content = rest_process.stdout.readline().strip().decode('utf-8') print(log_content) except KeyboardInterrupt: kill_command(rest_process.pid) print_normal('Stopping experiment...')
def launch_experiment(args, experiment_config, mode, experiment_id): '''follow steps to start rest server and start experiment''' nni_config = Config(experiment_id) # check packages for tuner package_name, module_name = None, None if experiment_config.get('tuner') and experiment_config['tuner'].get( 'builtinTunerName'): package_name = experiment_config['tuner']['builtinTunerName'] module_name, _ = get_builtin_module_class_name('tuners', package_name) elif experiment_config.get('advisor') and experiment_config['advisor'].get( 'builtinAdvisorName'): package_name = experiment_config['advisor']['builtinAdvisorName'] module_name, _ = get_builtin_module_class_name('advisors', package_name) if package_name and module_name: try: stdout_full_path, stderr_full_path = get_log_path(experiment_id) with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: check_call([sys.executable, '-c', 'import %s' % (module_name)], stdout=stdout_file, stderr=stderr_file) except CalledProcessError: print_error('some errors happen when import package %s.' % (package_name)) print_log_content(experiment_id) if package_name in INSTALLABLE_PACKAGE_META: print_error('If %s is not installed, it should be installed through '\ '\'nnictl package install --name %s\'' % (package_name, package_name)) exit(1) log_dir = experiment_config['logDir'] if experiment_config.get( 'logDir') else None log_level = experiment_config['logLevel'] if experiment_config.get( 'logLevel') else None #view experiment mode do not need debug function, when view an experiment, there will be no new logs created foreground = False if mode != 'view': foreground = args.foreground if log_level not in [ 'trace', 'debug' ] and (args.debug or experiment_config.get('debug') is True): log_level = 'debug' # start rest server rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], \ mode, experiment_id, foreground, log_dir, log_level) nni_config.set_config('restServerPid', rest_process.pid) # Deal with annotation if experiment_config.get('useAnnotation'): path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') if not os.path.isdir(path): os.makedirs(path) path = tempfile.mkdtemp(dir=path) nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) experiment_config['trial']['codeDir'] = code_dir search_space = generate_search_space(code_dir) experiment_config['searchSpace'] = json.dumps(search_space) assert search_space, ERROR_INFO % 'Generated search space is empty' elif experiment_config.get('searchSpacePath'): search_space = get_json_content( experiment_config.get('searchSpacePath')) experiment_config['searchSpace'] = json.dumps(search_space) else: experiment_config['searchSpace'] = json.dumps('') # check rest server running, _ = check_rest_server(args.port) if running: print_normal('Successfully started Restful server!') else: print_error('Restful server start failed!') print_log_content(experiment_id) try: kill_command(rest_process.pid) except Exception: raise Exception(ERROR_INFO % 'Rest server stopped!') exit(1) if mode != 'view': # set platform configuration set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ experiment_id, rest_process) # start a new experiment print_normal('Starting experiment...') # save experiment information nnictl_experiment_config = Experiments() nnictl_experiment_config.add_experiment( experiment_id, args.port, start_time, experiment_config['trainingServicePlatform'], experiment_config['experimentName'], pid=rest_process.pid, logDir=log_dir) # set debug configuration if mode != 'view' and experiment_config.get('debug') is None: experiment_config['debug'] = args.debug response = set_experiment(experiment_config, mode, args.port, experiment_id) if response: if experiment_id is None: experiment_id = json.loads(response.text).get('experiment_id') else: print_error('Start experiment failed!') print_log_content(experiment_id) try: kill_command(rest_process.pid) except Exception: raise Exception(ERROR_INFO % 'Restful server stopped!') exit(1) if experiment_config.get('nniManagerIp'): web_ui_url_list = [ '{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port)) ] else: web_ui_url_list = get_local_urls(args.port) nni_config.set_config('webuiUrl', web_ui_url_list) print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) if mode != 'view' and args.foreground: try: while True: log_content = rest_process.stdout.readline().strip().decode( 'utf-8') print(log_content) except KeyboardInterrupt: kill_command(rest_process.pid) print_normal('Stopping experiment...')