def test_init_no_optimizer(tmpdir):
config_dict = {"train_batch_size": 1, "fp16": {"enabled": True}}
config_path = create_config_from_dict(tmpdir, config_dict)
@distributed_test(world_size=1)
def _helper():
parser = argparse.ArgumentParser()
args = parser.parse_args(args='')
args.deepscale_config = config_path
args.local_rank = 0
hidden_dim = 10
model = SimpleModel(hidden_dim=hidden_dim)
model, _, _, _ = deepspeed.initialize(args=args, model=model)
data_loader = random_dataloader(model=model,
total_samples=5,
hidden_dim=hidden_dim,
device=model.device)
for n, batch in enumerate(data_loader):
loss = model(batch[0], batch[1])
with pytest.raises(AssertionError):
model.backward(loss)
with pytest.raises(AssertionError):
model.step()
_helper()
def test_temp_config_json(tmpdir):
config_dict = {
"train_batch_size": 1,
}
config_path = create_config_from_dict(tmpdir, config_dict)
config_json = json.load(open(config_path, 'r'))
assert 'train_batch_size' in config_json
def test_deprecated_deepscale_config(tmpdir):
config_dict = {
"train_batch_size": 1,
"optimizer": {
"type": "Adam",
"params": {
"lr": 0.00015
}
},
"fp16": {
"enabled": True
}
}
config_path = create_config_from_dict(tmpdir, config_dict)
parser = argparse.ArgumentParser()
args = parser.parse_args(args='')
args.deepscale_config = config_path
args.local_rank = 0
hidden_dim = 10
model = SimpleModel(hidden_dim)
@distributed_test(world_size=[1])
def _test_deprecated_deepscale_config(args, model, hidden_dim):
model, _, _, _ = deepspeed.initialize(
args=args,
model=model,
model_parameters=model.parameters(),
dist_init_required=False)
data_loader = random_dataloader(model=model,
total_samples=5,
hidden_dim=hidden_dim,
device=model.device)
for n, batch in enumerate(data_loader):
loss = model(batch[0], batch[1])
model.backward(loss)
model.step()
_test_deprecated_deepscale_config(args=args,
model=model,
hidden_dim=hidden_dim)
def test_autotuner_resources(tmpdir, active_resources):
config_dict = {
"autotuning": {
"enabled": True,
"exps_dir": os.path.join(tmpdir, 'exps_dir'),
"arg_mappings": {}
}
}
config_path = create_config_from_dict(tmpdir, config_dict)
args = dsrun.parse_args(
args=
f'--autotuning {TUNE_OPTION} foo.py --deepspeed_config {config_path}'.
split())
tuner = Autotuner(args=args, active_resources=active_resources)
expected_num_nodes = len(list(active_resources.keys()))
assert expected_num_nodes == tuner.exp_num_nodes
expected_num_gpus = min([len(v) for v in active_resources.values()])
assert expected_num_gpus == tuner.exp_num_gpus
