def gen_new_config(config_file, training_service='local'):
'''
Generates temporary config file for integration test, the file
should be deleted after testing.
'''
config = get_yml_content(config_file)
new_config_file = config_file + '.tmp'
it_config = get_yml_content('training_service.yml')
# hack for kubeflow trial config
if training_service == 'kubeflow':
it_config[training_service]['trial']['worker']['command'] = config['trial']['command']
config['trial'].pop('command')
if 'gpuNum' in config['trial']:
config['trial'].pop('gpuNum')
if training_service == 'frameworkcontroller':
it_config[training_service]['trial']['taskRoles'][0]['command'] = config['trial']['command']
config['trial'].pop('command')
if 'gpuNum' in config['trial']:
config['trial'].pop('gpuNum')
deep_update(config, it_config['all'])
deep_update(config, it_config[training_service])
dump_yml_content(new_config_file, config)
return new_config_file, config
def prepare_config_file(test_case_config, it_config, args):
config_path = args.nni_source_dir + test_case_config['configFile']
test_yml_config = get_yml_content(config_path)
# apply test case specific config
if test_case_config.get('config') is not None:
deep_update(test_yml_config, test_case_config['config'])
# hack for windows
if sys.platform == 'win32' and args.ts == 'local':
test_yml_config['trial']['command'] = test_yml_config['trial'][
'command'].replace('python3', 'python')
# apply training service config
# user's gpuNum, logCollection config is overwritten by the config in training_service.yml
# the hack for kubeflow should be applied at last step
update_training_service_config(test_yml_config, args.ts,
test_case_config['configFile'],
args.nni_source_dir, args.reuse_mode)
# generate temporary config yml file to launch experiment
new_config_file = config_path + '.tmp'
dump_yml_content(new_config_file, test_yml_config)
print(yaml.safe_dump(test_yml_config, default_flow_style=False),
flush=True)
return new_config_file
def update_training_service_config(args):
config = get_yml_content(TRAINING_SERVICE_FILE)
if args.nni_manager_ip is not None:
config[args.ts]['nniManagerIp'] = args.nni_manager_ip
if args.ts == 'pai':
if args.pai_user is not None:
config[args.ts]['paiConfig']['userName'] = args.pai_user
if args.pai_pwd is not None:
config[args.ts]['paiConfig']['passWord'] = args.pai_pwd
if args.pai_host is not None:
config[args.ts]['paiConfig']['host'] = args.pai_host
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.data_dir is not None:
config[args.ts]['trial']['dataDir'] = args.data_dir
if args.output_dir is not None:
config[args.ts]['trial']['outputDir'] = args.output_dir
elif args.ts == 'kubeflow':
if args.nfs_server is not None:
config[args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server
if args.nfs_path is not None:
config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path
if args.keyvault_vaultname is not None:
config[args.ts]['kubeflowConfig']['keyVault']['vaultName'] = args.keyvault_vaultname
if args.keyvault_name is not None:
config[args.ts]['kubeflowConfig']['keyVault']['name'] = args.keyvault_name
if args.azs_account is not None:
config[args.ts]['kubeflowConfig']['azureStorage']['accountName'] = args.azs_account
if args.azs_share is not None:
config[args.ts]['kubeflowConfig']['azureStorage']['azureShare'] = args.azs_share
if args.nni_docker_image is not None:
config[args.ts]['trial']['worker']['image'] = args.nni_docker_image
dump_yml_content(TRAINING_SERVICE_FILE, config)
def convert_command():
'''convert command by platform'''
if sys.platform != 'win32':
return None
config_files = glob.glob('./**/*.yml') + glob.glob('./**/**/*.yml')
for config_file in config_files:
print('processing {}'.format(config_file))
yml_content = get_yml_content(config_file)
if yml_content.get('trial'):
if yml_content['trial'].get('command'):
yml_content['trial']['command'] = yml_content['trial']['command'].replace('python3', 'python')
dump_yml_content(config_file, yml_content)
def switch(dispatch_type, dispatch_name):
'''Change dispatch in config.yml'''
config_path = 'tuner_test/local.yml'
experiment_config = get_yml_content(config_path)
if dispatch_name in ['GridSearch', 'BatchTuner']:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name
}
else:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name,
'classArgs': {
'optimize_mode': 'maximize'
}
}
dump_yml_content(config_path, experiment_config)
def gen_new_config(config_file, training_service='local'):
'''
Generates temporary config file for integration test, the file
should be deleted after testing.
'''
config = get_yml_content(config_file)
new_config_file = config_file + '.tmp'
ts = get_yml_content('training_service.yml')[training_service]
print(config)
print(ts)
config.update(ts)
print(config)
dump_yml_content(new_config_file, config)
return new_config_file, config
def switch(dispatch_type, dispatch_name):
'''Change dispatch in config.yml'''
config_path = get_config_file_path()
experiment_config = get_yml_content(config_path)
if dispatch_name in ['GridSearch', 'BatchTuner', 'Random']:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name
}
else:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name,
'classArgs': {
'optimize_mode': 'maximize'
}
}
if dispatch_name == 'BatchTuner':
experiment_config['searchSpacePath'] = 'batchtuner_search_space.json'
else:
experiment_config['searchSpacePath'] = 'search_space.json'
dump_yml_content(config_path, experiment_config)
def gen_new_config(config_file, training_service='local'):
'''
Generates temporary config file for integration test, the file
should be deleted after testing.
'''
config = get_yml_content(config_file)
new_config_file = config_file + '.tmp'
ts = get_yml_content('training_service.yml')[training_service]
print(config)
print(ts)
# hack for kubeflow trial config
if training_service == 'kubeflow':
ts['trial']['worker']['command'] = config['trial']['command']
config['trial'].pop('command')
if 'gpuNum' in config['trial']:
config['trial'].pop('gpuNum')
deep_update(config, ts)
print(config)
dump_yml_content(new_config_file, config)
return new_config_file, config
def update_training_service_config(args):
config = get_yml_content(TRAINING_SERVICE_FILE)
if args.nni_manager_ip is not None:
config[args.ts]['nniManagerIp'] = args.nni_manager_ip
if args.ts == 'pai':
if args.pai_user is not None:
config[args.ts]['paiConfig']['userName'] = args.pai_user
if args.pai_host is not None:
config[args.ts]['paiConfig']['host'] = args.pai_host
if args.pai_token is not None:
config[args.ts]['paiConfig']['token'] = args.pai_token
if args.pai_reuse is not None:
config[args.ts]['paiConfig']['reuse'] = args.pai_reuse.lower(
) == 'true'
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.nni_manager_nfs_mount_path is not None:
config[args.ts]['trial'][
'nniManagerNFSMountPath'] = args.nni_manager_nfs_mount_path
if args.container_nfs_mount_path is not None:
config[args.ts]['trial'][
'containerNFSMountPath'] = args.container_nfs_mount_path
if args.pai_storage_config_name is not None:
config[args.ts]['trial'][
'paiStorageConfigName'] = args.pai_storage_config_name
if args.vc is not None:
config[args.ts]['trial']['virtualCluster'] = args.vc
elif args.ts == 'kubeflow':
if args.nfs_server is not None:
config[
args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server
if args.nfs_path is not None:
config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path
if args.keyvault_vaultname is not None:
config[args.ts]['kubeflowConfig']['keyVault'][
'vaultName'] = args.keyvault_vaultname
if args.keyvault_name is not None:
config[args.ts]['kubeflowConfig']['keyVault'][
'name'] = args.keyvault_name
if args.azs_account is not None:
config[args.ts]['kubeflowConfig']['azureStorage'][
'accountName'] = args.azs_account
if args.azs_share is not None:
config[args.ts]['kubeflowConfig']['azureStorage'][
'azureShare'] = args.azs_share
if args.nni_docker_image is not None:
config[args.ts]['trial']['worker']['image'] = args.nni_docker_image
elif args.ts == 'frameworkcontroller':
if args.nfs_server is not None:
config[args.ts]['frameworkcontrollerConfig']['nfs'][
'server'] = args.nfs_server
if args.nfs_path is not None:
config[args.ts]['frameworkcontrollerConfig']['nfs'][
'path'] = args.nfs_path
if args.keyvault_vaultname is not None:
config[args.ts]['frameworkcontrollerConfig']['keyVault'][
'vaultName'] = args.keyvault_vaultname
if args.keyvault_name is not None:
config[args.ts]['frameworkcontrollerConfig']['keyVault'][
'name'] = args.keyvault_name
if args.azs_account is not None:
config[args.ts]['frameworkcontrollerConfig']['azureStorage'][
'accountName'] = args.azs_account
if args.azs_share is not None:
config[args.ts]['frameworkcontrollerConfig']['azureStorage'][
'azureShare'] = args.azs_share
if args.nni_docker_image is not None:
config[args.ts]['trial']['taskRoles'][0][
'image'] = args.nni_docker_image
elif args.ts == 'remote':
if args.remote_user is not None:
config[args.ts]['machineList'][0]['username'] = args.remote_user
if args.remote_host is not None:
config[args.ts]['machineList'][0]['ip'] = args.remote_host
if args.remote_port is not None:
config[args.ts]['machineList'][0]['port'] = args.remote_port
if args.remote_pwd is not None:
config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd
if args.remote_reuse is not None:
config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower(
) == 'true'
elif args.ts == 'adl':
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.checkpoint_storage_class is not None:
config[args.ts]['trial']['checkpoint'][
'storageClass'] = args.checkpoint_storage_class
if args.checkpoint_storage_size is not None:
config[args.ts]['trial']['checkpoint'][
'storageSize'] = args.checkpoint_storage_size
if args.adaptive is not None:
config[args.ts]['trial']['adaptive'] = args.adaptive
if args.adl_nfs_server is not None and args.adl_nfs_path is not None and args.adl_nfs_container_mount_path is not None:
# default keys in nfs is empty, need to initialize
config[args.ts]['trial']['nfs'] = {}
config[args.ts]['trial']['nfs']['server'] = args.adl_nfs_server
config[args.ts]['trial']['nfs']['path'] = args.adl_nfs_path
config[args.ts]['trial']['nfs'][
'container_mount_path'] = args.nadl_fs_container_mount_path
elif args.ts == 'aml':
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.subscription_id is not None:
config[
args.ts]['amlConfig']['subscriptionId'] = args.subscription_id
if args.resource_group is not None:
config[args.ts]['amlConfig']['resourceGroup'] = args.resource_group
if args.workspace_name is not None:
config[args.ts]['amlConfig']['workspaceName'] = args.workspace_name
if args.compute_target is not None:
config[args.ts]['amlConfig']['computeTarget'] = args.compute_target
dump_yml_content(TRAINING_SERVICE_FILE, config)
def update_training_service_config(args):
config = get_yml_content(TRAINING_SERVICE_FILE)
if args.nni_manager_ip is not None:
config[args.ts]['nniManagerIp'] = args.nni_manager_ip
if args.ts == 'paiYarn':
if args.pai_user is not None:
config[args.ts]['paiYarnConfig']['userName'] = args.pai_user
if args.pai_pwd is not None:
config[args.ts]['paiYarnConfig']['passWord'] = args.pai_pwd
if args.pai_host is not None:
config[args.ts]['paiYarnConfig']['host'] = args.pai_host
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.data_dir is not None:
config[args.ts]['trial']['dataDir'] = args.data_dir
if args.output_dir is not None:
config[args.ts]['trial']['outputDir'] = args.output_dir
if args.vc is not None:
config[args.ts]['trial']['virtualCluster'] = args.vc
if args.ts == 'pai':
if args.pai_user is not None:
config[args.ts]['paiConfig']['userName'] = args.pai_user
if args.pai_host is not None:
config[args.ts]['paiConfig']['host'] = args.pai_host
if args.pai_token is not None:
config[args.ts]['paiConfig']['token'] = args.pai_token
if args.pai_reuse is not None:
config[args.ts]['paiConfig']['reuse'] = args.pai_reuse.lower(
) == 'true'
if args.nni_docker_image is not None:
config[args.ts]['trial']['image'] = args.nni_docker_image
if args.nni_manager_nfs_mount_path is not None:
config[args.ts]['trial'][
'nniManagerNFSMountPath'] = args.nni_manager_nfs_mount_path
if args.container_nfs_mount_path is not None:
config[args.ts]['trial'][
'containerNFSMountPath'] = args.container_nfs_mount_path
if args.pai_storage_config_name is not None:
config[args.ts]['trial'][
'paiStorageConfigName'] = args.pai_storage_config_name
if args.vc is not None:
config[args.ts]['trial']['virtualCluster'] = args.vc
elif args.ts == 'kubeflow':
if args.nfs_server is not None:
config[
args.ts]['kubeflowConfig']['nfs']['server'] = args.nfs_server
if args.nfs_path is not None:
config[args.ts]['kubeflowConfig']['nfs']['path'] = args.nfs_path
if args.keyvault_vaultname is not None:
config[args.ts]['kubeflowConfig']['keyVault'][
'vaultName'] = args.keyvault_vaultname
if args.keyvault_name is not None:
config[args.ts]['kubeflowConfig']['keyVault'][
'name'] = args.keyvault_name
if args.azs_account is not None:
config[args.ts]['kubeflowConfig']['azureStorage'][
'accountName'] = args.azs_account
if args.azs_share is not None:
config[args.ts]['kubeflowConfig']['azureStorage'][
'azureShare'] = args.azs_share
if args.nni_docker_image is not None:
config[args.ts]['trial']['worker']['image'] = args.nni_docker_image
elif args.ts == 'frameworkcontroller':
if args.nfs_server is not None:
config[args.ts]['frameworkcontrollerConfig']['nfs'][
'server'] = args.nfs_server
if args.nfs_path is not None:
config[args.ts]['frameworkcontrollerConfig']['nfs'][
'path'] = args.nfs_path
if args.keyvault_vaultname is not None:
config[args.ts]['frameworkcontrollerConfig']['keyVault'][
'vaultName'] = args.keyvault_vaultname
if args.keyvault_name is not None:
config[args.ts]['frameworkcontrollerConfig']['keyVault'][
'name'] = args.keyvault_name
if args.azs_account is not None:
config[args.ts]['frameworkcontrollerConfig']['azureStorage'][
'accountName'] = args.azs_account
if args.azs_share is not None:
config[args.ts]['frameworkcontrollerConfig']['azureStorage'][
'azureShare'] = args.azs_share
if args.nni_docker_image is not None:
config[args.ts]['trial']['taskRoles'][0][
'image'] = args.nni_docker_image
elif args.ts == 'remote':
if args.remote_user is not None:
config[args.ts]['machineList'][0]['username'] = args.remote_user
if args.remote_host is not None:
config[args.ts]['machineList'][0]['ip'] = args.remote_host
if args.remote_port is not None:
config[args.ts]['machineList'][0]['port'] = args.remote_port
if args.remote_pwd is not None:
config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd
if args.remote_reuse is not None:
config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower(
) == 'true'
dump_yml_content(TRAINING_SERVICE_FILE, config)
