def test_param_override1():
conf = Config('confs/algos/darts.yaml;confs/datasets/cifar10.yaml')
assert not conf['nas']['eval']['trainer']['apex']['enabled']
assert not conf['nas']['eval']['loader']['apex']['enabled']
conf = Config('confs/algos/darts.yaml;confs/datasets/cifar10.yaml',
param_args=["--nas.eval.trainer.apex.enabled", "True"])
assert conf['nas']['eval']['trainer']['apex']['enabled']
assert conf['nas']['eval']['loader']['apex']['enabled']
def test_param_override2():
conf = Config('confs/algos/darts.yaml;confs/datasets/imagenet.yaml')
assert conf['nas']['eval']['trainer']['apex']['distributed_enabled']
assert conf['nas']['eval']['loader']['apex']['distributed_enabled']
conf = Config('confs/algos/darts.yaml;confs/datasets/imagenet.yaml',
param_args=[
"--nas.eval.trainer.apex.enabled", "True",
"--nas.eval.trainer.apex.distributed_enabled", "False"
])
assert not conf['nas']['eval']['trainer']['apex']['distributed_enabled']
assert not conf['nas']['eval']['loader']['apex']['distributed_enabled']
def main():
parser = argparse.ArgumentParser(description='Archai data install')
parser.add_argument('--dataroot',
type=str,
default='~/dataroot',
help='path to dataroot on local drive')
parser.add_argument(
'--dataset',
type=str,
default='cifar10',
help=
'Name of the dataset for which confs/dataset/name.yaml should exist and have name of folder or tar file it resides in'
)
args, extra_args = parser.parse_known_args()
pt_data_dir = os.environ.get('PT_DATA_DIR', '')
if not pt_data_dir:
raise RuntimeError(
'This script needs PT_DATA_DIR environment variable with path to dataroot on cloud drive'
)
pt_data_dir = utils.full_path(pt_data_dir)
print('pt_data_dir:', pt_data_dir)
conf_data_filepath = f'confs/datasets/{args.dataset}.yaml'
print('conf_data_filepath:', conf_data_filepath)
conf = Config(config_filepath=conf_data_filepath)
for dataset_key in ['dataset', 'dataset_search', 'dataset_eval']:
if dataset_key in conf:
conf_data = conf[dataset_key]
untar_dataset(dataset_key, pt_data_dir, conf_data, args.dataroot)
def imagenet_test():
conf = Config('confs/algos/darts.yaml;confs/datasets/imagenet.yaml', )
conf_loader = conf['nas']['eval']['loader']
dl_train, *_ = data.get_data(conf_loader)
def cancel_experiment(ws, experiment_name):
exp = get_experiment(ws, experiment_name)
print(
'Cancelling existing experiment with name: {}'.format(experiment_name))
for run in tqdm(list(exp.get_runs())):
run.cancel()
if __name__ == "__main__":
print("SDK Version:", azureml.core.VERSION)
set_diagnostics_collection(send_diagnostics=True)
# Read in config
conf = Config(config_filepath='~/aml_secrets/aml_secrets_rr2msrlabs.yaml')
# Config region
conf_aml = conf['aml_config']
conf_cluster = conf['cluster_config']
conf_docker = conf['azure_docker']
conf_experiment = conf['experiment']
# endregion
# Initialize workspace
# Make sure you have downloaded your workspace config
ws = Workspace.from_config(path=conf_aml['aml_config_file'])
print('Workspace name: ' + ws.name,
'Azure region: ' + ws.location,
'Subscription id: ' + ws.subscription_id,
'Resource group: ' + ws.resource_group,
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
from archai.common.config import Config
from archai.datasets import data
from torchvision import transforms
from archai.common.ml_utils import channel_norm
if __name__ == '__main__':
conf = Config(config_filepath='confs/datasets/flower102.yaml')
conf_dataset = conf['dataset']
ds_provider = data.create_dataset_provider(conf_dataset)
train_ds, _ = ds_provider.get_datasets(
True, False,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor()
]), transforms.Compose([]))
print(channel_norm(train_ds))
exit(0)
compute_target=compute_target,
entry_script='scripts/random/cifar_eval.py',
custom_docker_image=conf_docker['image_name'],
image_registry_details=image_registry_details,
user_managed=user_managed_dependencies,
source_directory_data_store=input_ds)
run = experiment.submit(est)
if __name__ == "__main__":
print("SDK Version:", azureml.core.VERSION)
set_diagnostics_collection(send_diagnostics=True)
# Read in config
conf = Config(config_filepath='~/aml_secrets/aml_config_dedey.yaml')
# Config region
conf_aml = conf['aml_config']
conf_cluster = conf['cluster_config']
conf_docker = conf['azure_docker']
conf_experiment = conf['experiment']
# endregion
# Initialize workspace
# Make sure you have downloaded your workspace config
ws = Workspace.from_config(path=conf_aml['aml_config_file'])
print('Workspace name: ' + ws.name,
'Azure region: ' + ws.location,
'Subscription id: ' + ws.subscription_id,
'Resource group: ' + ws.resource_group,
def get_dataroot() -> str:
conf = Config(config_filepath='confs/algos/manual.yaml')
return conf['dataset']['dataroot']
def test_serialize():
conf = Config()
conf['decay'] = 1
s = yaml.dump(conf)
conf2 = yaml.load(s, Loader=yaml.Loader)
assert len(conf2) == 1
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
from archai.common.config import Config
from archai.datasets import data
from torchvision import transforms
from archai.common.ml_utils import channel_norm
if __name__ == '__main__':
conf = Config(config_filepath='confs/datasets/cifar10.yaml')
conf_data = conf['dataset']
ds_provider = data.create_dataset_provider(conf_data)
train_ds, _ = ds_provider.get_datasets(
True, False,
transforms.Compose(
[transforms.Resize((32, 32)),
transforms.ToTensor()]), transforms.Compose([]))
print(channel_norm(train_ds))
exit(0)
def __init__(self, config_filepath: str) -> None:
# read in config
self.conf = Config(config_filepath)
# config region
self.conf_aml = self.conf['aml_config']
self.conf_storage = self.conf['storage']
self.conf_cluster = self.conf['cluster_config']
self.conf_docker = self.conf['azure_docker']
self.conf_experiment = self.conf['experiment']
# end region
# initialize workspace
self.ws = Workspace.from_config(path=self.conf_aml['aml_config_file'])
print('Workspace name: ' + self.ws.name,
'Azure region: ' + self.ws.location,
'Subscription id: ' + self.ws.subscription_id,
'Resource group: ' + self.ws.resource_group,
sep='\n')
# register blobs
# TODO: make blob registration more flexible
self.input_ds = Datastore.register_azure_blob_container(
workspace=self.ws,
datastore_name=self.conf_storage['input_datastore_name'],
container_name=self.conf_storage['input_container_name'],
account_name=self.conf_storage['input_azure_storage_account_name'],
account_key=self.conf_storage['input_azure_storage_account_key'],
create_if_not_exists=False)
self.output_ds = Datastore.register_azure_blob_container(
workspace=self.ws,
datastore_name=self.conf_storage['output_datastore_name'],
container_name=self.conf_storage['output_container_name'],
account_name=self.
conf_storage['output_azure_storage_account_name'],
account_key=self.conf_storage['output_azure_storage_account_key'],
create_if_not_exists=False)
# create compute cluster
try:
self.compute_target = ComputeTarget(
workspace=self.ws, name=self.conf_cluster['cluster_name'])
print(self.compute_target.get_status().serialize())
except Exception as e:
print('Encountered error trying to get the compute target')
print(f'Exception was {e}')
sys.exit(1)
self.project_folder = self.conf_experiment['project_folder']
# setup custom docker usage
self.image_registry_details = ContainerRegistry()
self.image_registry_details.address = self.conf_docker[
'image_registry_address']
self.image_registry_details.username = self.conf_docker[
'image_registry_username']
self.image_registry_details.password = self.conf_docker[
'image_registry_password']
self.user_managed_dependencies = True
