def gen_new_config(config_file, training_service='local'): ''' Generates temporary config file for integration test, the file should be deleted after testing. ''' config = get_yml_content(config_file) new_config_file = config_file + '.tmp' it_config = get_yml_content('training_service.yml') # hack for kubeflow trial config if training_service == 'kubeflow': it_config[training_service]['trial']['worker']['command'] = config['trial']['command'] config['trial'].pop('command') if 'gpuNum' in config['trial']: config['trial'].pop('gpuNum') if training_service == 'frameworkcontroller': it_config[training_service]['trial']['taskRoles'][0]['command'] = config['trial']['command'] config['trial'].pop('command') if 'gpuNum' in config['trial']: config['trial'].pop('gpuNum') deep_update(config, it_config['all']) deep_update(config, it_config[training_service]) dump_yml_content(new_config_file, config) return new_config_file, config
def prepare_config_file(test_case_config, it_config, args): config_path = args.nni_source_dir + test_case_config['configFile'] test_yml_config = get_yml_content(config_path) # apply test case specific config if test_case_config.get('config') is not None: deep_update(test_yml_config, test_case_config['config']) # hack for windows if sys.platform == 'win32' and args.ts == 'local': test_yml_config['trial']['command'] = test_yml_config['trial'][ 'command'].replace('python3', 'python') # apply training service config # user's gpuNum, logCollection config is overwritten by the config in training_service.yml # the hack for kubeflow should be applied at last step update_training_service_config(test_yml_config, args.ts, test_case_config['configFile'], args.nni_source_dir, args.reuse_mode) # generate temporary config yml file to launch experiment new_config_file = config_path + '.tmp' dump_yml_content(new_config_file, test_yml_config) print(yaml.safe_dump(test_yml_config, default_flow_style=False), flush=True) return new_config_file
def update_training_service_config(args): config = get_yml_content(TRAINING_SERVICE_FILE) if args.nni_manager_ip is not None: config[args.ts]['nniManagerIp'] = args.nni_manager_ip if args.ts == 'pai': if args.pai_user is not None: config[args.ts]['paiConfig']['userName'] = args.pai_user if args.pai_pwd is not None: config[args.ts]['paiConfig']['passWord'] = args.pai_pwd if args.pai_host is not None: config[args.ts]['paiConfig']['host'] = args.pai_host if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.data_dir is not None: config[args.ts]['trial']['dataDir'] = args.data_dir if args.output_dir is not None: config[args.ts]['trial']['outputDir'] = args.output_dir elif args.ts == 'kubeflow': if args.nfs_server is not None: config[args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server if args.nfs_path is not None: config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path if args.keyvault_vaultname is not None: config[args.ts]['kubeflowConfig']['keyVault']['vaultName'] = args.keyvault_vaultname if args.keyvault_name is not None: config[args.ts]['kubeflowConfig']['keyVault']['name'] = args.keyvault_name if args.azs_account is not None: config[args.ts]['kubeflowConfig']['azureStorage']['accountName'] = args.azs_account if args.azs_share is not None: config[args.ts]['kubeflowConfig']['azureStorage']['azureShare'] = args.azs_share if args.nni_docker_image is not None: config[args.ts]['trial']['worker']['image'] = args.nni_docker_image dump_yml_content(TRAINING_SERVICE_FILE, config)
def convert_command(): '''convert command by platform''' if sys.platform != 'win32': return None config_files = glob.glob('./**/*.yml') + glob.glob('./**/**/*.yml') for config_file in config_files: print('processing {}'.format(config_file)) yml_content = get_yml_content(config_file) if yml_content.get('trial'): if yml_content['trial'].get('command'): yml_content['trial']['command'] = yml_content['trial']['command'].replace('python3', 'python') dump_yml_content(config_file, yml_content)
def switch(dispatch_type, dispatch_name): '''Change dispatch in config.yml''' config_path = 'tuner_test/local.yml' experiment_config = get_yml_content(config_path) if dispatch_name in ['GridSearch', 'BatchTuner']: experiment_config[dispatch_type.lower()] = { 'builtin' + dispatch_type + 'Name': dispatch_name } else: experiment_config[dispatch_type.lower()] = { 'builtin' + dispatch_type + 'Name': dispatch_name, 'classArgs': { 'optimize_mode': 'maximize' } } dump_yml_content(config_path, experiment_config)
def gen_new_config(config_file, training_service='local'): ''' Generates temporary config file for integration test, the file should be deleted after testing. ''' config = get_yml_content(config_file) new_config_file = config_file + '.tmp' ts = get_yml_content('training_service.yml')[training_service] print(config) print(ts) config.update(ts) print(config) dump_yml_content(new_config_file, config) return new_config_file, config
def switch(dispatch_type, dispatch_name): '''Change dispatch in config.yml''' config_path = get_config_file_path() experiment_config = get_yml_content(config_path) if dispatch_name in ['GridSearch', 'BatchTuner', 'Random']: experiment_config[dispatch_type.lower()] = { 'builtin' + dispatch_type + 'Name': dispatch_name } else: experiment_config[dispatch_type.lower()] = { 'builtin' + dispatch_type + 'Name': dispatch_name, 'classArgs': { 'optimize_mode': 'maximize' } } if dispatch_name == 'BatchTuner': experiment_config['searchSpacePath'] = 'batchtuner_search_space.json' else: experiment_config['searchSpacePath'] = 'search_space.json' dump_yml_content(config_path, experiment_config)
def gen_new_config(config_file, training_service='local'): ''' Generates temporary config file for integration test, the file should be deleted after testing. ''' config = get_yml_content(config_file) new_config_file = config_file + '.tmp' ts = get_yml_content('training_service.yml')[training_service] print(config) print(ts) # hack for kubeflow trial config if training_service == 'kubeflow': ts['trial']['worker']['command'] = config['trial']['command'] config['trial'].pop('command') if 'gpuNum' in config['trial']: config['trial'].pop('gpuNum') deep_update(config, ts) print(config) dump_yml_content(new_config_file, config) return new_config_file, config
def update_training_service_config(args): config = get_yml_content(TRAINING_SERVICE_FILE) if args.nni_manager_ip is not None: config[args.ts]['nniManagerIp'] = args.nni_manager_ip if args.ts == 'pai': if args.pai_user is not None: config[args.ts]['paiConfig']['userName'] = args.pai_user if args.pai_host is not None: config[args.ts]['paiConfig']['host'] = args.pai_host if args.pai_token is not None: config[args.ts]['paiConfig']['token'] = args.pai_token if args.pai_reuse is not None: config[args.ts]['paiConfig']['reuse'] = args.pai_reuse.lower( ) == 'true' if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.nni_manager_nfs_mount_path is not None: config[args.ts]['trial'][ 'nniManagerNFSMountPath'] = args.nni_manager_nfs_mount_path if args.container_nfs_mount_path is not None: config[args.ts]['trial'][ 'containerNFSMountPath'] = args.container_nfs_mount_path if args.pai_storage_config_name is not None: config[args.ts]['trial'][ 'paiStorageConfigName'] = args.pai_storage_config_name if args.vc is not None: config[args.ts]['trial']['virtualCluster'] = args.vc elif args.ts == 'kubeflow': if args.nfs_server is not None: config[ args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server if args.nfs_path is not None: config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path if args.keyvault_vaultname is not None: config[args.ts]['kubeflowConfig']['keyVault'][ 'vaultName'] = args.keyvault_vaultname if args.keyvault_name is not None: config[args.ts]['kubeflowConfig']['keyVault'][ 'name'] = args.keyvault_name if args.azs_account is not None: config[args.ts]['kubeflowConfig']['azureStorage'][ 'accountName'] = args.azs_account if args.azs_share is not None: config[args.ts]['kubeflowConfig']['azureStorage'][ 'azureShare'] = args.azs_share if args.nni_docker_image is not None: config[args.ts]['trial']['worker']['image'] = args.nni_docker_image elif args.ts == 'frameworkcontroller': if args.nfs_server is not None: config[args.ts]['frameworkcontrollerConfig']['nfs'][ 'server'] = args.nfs_server if args.nfs_path is not None: config[args.ts]['frameworkcontrollerConfig']['nfs'][ 'path'] = args.nfs_path if args.keyvault_vaultname is not None: config[args.ts]['frameworkcontrollerConfig']['keyVault'][ 'vaultName'] = args.keyvault_vaultname if args.keyvault_name is not None: config[args.ts]['frameworkcontrollerConfig']['keyVault'][ 'name'] = args.keyvault_name if args.azs_account is not None: config[args.ts]['frameworkcontrollerConfig']['azureStorage'][ 'accountName'] = args.azs_account if args.azs_share is not None: config[args.ts]['frameworkcontrollerConfig']['azureStorage'][ 'azureShare'] = args.azs_share if args.nni_docker_image is not None: config[args.ts]['trial']['taskRoles'][0][ 'image'] = args.nni_docker_image elif args.ts == 'remote': if args.remote_user is not None: config[args.ts]['machineList'][0]['username'] = args.remote_user if args.remote_host is not None: config[args.ts]['machineList'][0]['ip'] = args.remote_host if args.remote_port is not None: config[args.ts]['machineList'][0]['port'] = args.remote_port if args.remote_pwd is not None: config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd if args.remote_reuse is not None: config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower( ) == 'true' elif args.ts == 'adl': if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.checkpoint_storage_class is not None: config[args.ts]['trial']['checkpoint'][ 'storageClass'] = args.checkpoint_storage_class if args.checkpoint_storage_size is not None: config[args.ts]['trial']['checkpoint'][ 'storageSize'] = args.checkpoint_storage_size if args.adaptive is not None: config[args.ts]['trial']['adaptive'] = args.adaptive if args.adl_nfs_server is not None and args.adl_nfs_path is not None and args.adl_nfs_container_mount_path is not None: # default keys in nfs is empty, need to initialize config[args.ts]['trial']['nfs'] = {} config[args.ts]['trial']['nfs']['server'] = args.adl_nfs_server config[args.ts]['trial']['nfs']['path'] = args.adl_nfs_path config[args.ts]['trial']['nfs'][ 'container_mount_path'] = args.nadl_fs_container_mount_path elif args.ts == 'aml': if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.subscription_id is not None: config[ args.ts]['amlConfig']['subscriptionId'] = args.subscription_id if args.resource_group is not None: config[args.ts]['amlConfig']['resourceGroup'] = args.resource_group if args.workspace_name is not None: config[args.ts]['amlConfig']['workspaceName'] = args.workspace_name if args.compute_target is not None: config[args.ts]['amlConfig']['computeTarget'] = args.compute_target dump_yml_content(TRAINING_SERVICE_FILE, config)
def update_training_service_config(args): config = get_yml_content(TRAINING_SERVICE_FILE) if args.nni_manager_ip is not None: config[args.ts]['nniManagerIp'] = args.nni_manager_ip if args.ts == 'paiYarn': if args.pai_user is not None: config[args.ts]['paiYarnConfig']['userName'] = args.pai_user if args.pai_pwd is not None: config[args.ts]['paiYarnConfig']['passWord'] = args.pai_pwd if args.pai_host is not None: config[args.ts]['paiYarnConfig']['host'] = args.pai_host if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.data_dir is not None: config[args.ts]['trial']['dataDir'] = args.data_dir if args.output_dir is not None: config[args.ts]['trial']['outputDir'] = args.output_dir if args.vc is not None: config[args.ts]['trial']['virtualCluster'] = args.vc if args.ts == 'pai': if args.pai_user is not None: config[args.ts]['paiConfig']['userName'] = args.pai_user if args.pai_host is not None: config[args.ts]['paiConfig']['host'] = args.pai_host if args.pai_token is not None: config[args.ts]['paiConfig']['token'] = args.pai_token if args.pai_reuse is not None: config[args.ts]['paiConfig']['reuse'] = args.pai_reuse.lower( ) == 'true' if args.nni_docker_image is not None: config[args.ts]['trial']['image'] = args.nni_docker_image if args.nni_manager_nfs_mount_path is not None: config[args.ts]['trial'][ 'nniManagerNFSMountPath'] = args.nni_manager_nfs_mount_path if args.container_nfs_mount_path is not None: config[args.ts]['trial'][ 'containerNFSMountPath'] = args.container_nfs_mount_path if args.pai_storage_config_name is not None: config[args.ts]['trial'][ 'paiStorageConfigName'] = args.pai_storage_config_name if args.vc is not None: config[args.ts]['trial']['virtualCluster'] = args.vc elif args.ts == 'kubeflow': if args.nfs_server is not None: config[ args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server if args.nfs_path is not None: config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path if args.keyvault_vaultname is not None: config[args.ts]['kubeflowConfig']['keyVault'][ 'vaultName'] = args.keyvault_vaultname if args.keyvault_name is not None: config[args.ts]['kubeflowConfig']['keyVault'][ 'name'] = args.keyvault_name if args.azs_account is not None: config[args.ts]['kubeflowConfig']['azureStorage'][ 'accountName'] = args.azs_account if args.azs_share is not None: config[args.ts]['kubeflowConfig']['azureStorage'][ 'azureShare'] = args.azs_share if args.nni_docker_image is not None: config[args.ts]['trial']['worker']['image'] = args.nni_docker_image elif args.ts == 'frameworkcontroller': if args.nfs_server is not None: config[args.ts]['frameworkcontrollerConfig']['nfs'][ 'server'] = args.nfs_server if args.nfs_path is not None: config[args.ts]['frameworkcontrollerConfig']['nfs'][ 'path'] = args.nfs_path if args.keyvault_vaultname is not None: config[args.ts]['frameworkcontrollerConfig']['keyVault'][ 'vaultName'] = args.keyvault_vaultname if args.keyvault_name is not None: config[args.ts]['frameworkcontrollerConfig']['keyVault'][ 'name'] = args.keyvault_name if args.azs_account is not None: config[args.ts]['frameworkcontrollerConfig']['azureStorage'][ 'accountName'] = args.azs_account if args.azs_share is not None: config[args.ts]['frameworkcontrollerConfig']['azureStorage'][ 'azureShare'] = args.azs_share if args.nni_docker_image is not None: config[args.ts]['trial']['taskRoles'][0][ 'image'] = args.nni_docker_image elif args.ts == 'remote': if args.remote_user is not None: config[args.ts]['machineList'][0]['username'] = args.remote_user if args.remote_host is not None: config[args.ts]['machineList'][0]['ip'] = args.remote_host if args.remote_port is not None: config[args.ts]['machineList'][0]['port'] = args.remote_port if args.remote_pwd is not None: config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd if args.remote_reuse is not None: config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower( ) == 'true' dump_yml_content(TRAINING_SERVICE_FILE, config)