def __init__(self, client_id=0, model=None):
"""Initializing the trainer with the provided model.
Arguments:
model: The model to train. Must be a models.base_mindspore.Model subclass.
client_id: The ID of the client using this trainer (optional).
"""
super().__init__(client_id)
mindspore.context.set_context(mode=mindspore.context.PYNATIVE_MODE,
device_target='GPU')
if model is None:
self.model = models_registry.get()
# Initializing the loss criterion
loss_criterion = SoftmaxCrossEntropyWithLogits(sparse=True,
reduction='mean')
# Initializing the optimizer
optimizer = nn.Momentum(self.model.trainable_params(),
Config().trainer.learning_rate,
Config().trainer.momentum)
self.mindspore_model = mindspore.Model(
self.model,
loss_criterion,
optimizer,
metrics={"Accuracy": Accuracy()})
def test_freeze_none(self):
default_hparams = registry.get_default_hparams('cifar_resnet_20')
model = registry.get(default_hparams.model_hparams)
for k, v in model.named_parameters():
with self.subTest(tensor=k):
self.assertTrue(v.requires_grad)
def test_freeze_all(self):
default_hparams = registry.get_default_hparams('cifar_resnet_20')
default_hparams.model_hparams.others_frozen = True
default_hparams.model_hparams.output_frozen = True
default_hparams.model_hparams.batchnorm_frozen = True
model = registry.get(default_hparams.model_hparams)
for k, v in model.named_parameters():
with self.subTest(tensor=k):
self.assertFalse(v.requires_grad)
def test_freeze_output(self):
default_hparams = registry.get_default_hparams('cifar_resnet_20')
default_hparams.model_hparams.output_frozen = True
model = registry.get(default_hparams.model_hparams)
for k, v in model.named_parameters():
with self.subTest(tensor=k):
if k in model.output_layer_names:
self.assertFalse(v.requires_grad)
else:
self.assertTrue(v.requires_grad)
def setUp(self):
super().setUp()
__ = Config()
fields = [
'optimizer', 'lr_schedule', 'learning_rate', 'momentum',
'weight_decay', 'lr_gamma', 'lr_milestone_steps', 'lr_warmup_steps'
]
params = ['SGD', '', 0.1, 0.5, 0.0, 0.0, '', '']
Config().trainer = namedtuple('trainer', fields)(*params)
self.model = models_registry.get('resnet_18')
self.optimizer = optimizers.get_optimizer(self.model)
def test_save_load_exists(self):
hp = registry.get_default_hparams('cifar_resnet_20')
model = registry.get(hp.model_hparams)
step = Step.from_iteration(27, 17)
model_location = paths.model(self.root, step)
model_state = TestSaveLoadExists.get_state(model)
self.assertFalse(registry.exists(self.root, step))
self.assertFalse(os.path.exists(model_location))
# Test saving.
model.save(self.root, step)
self.assertTrue(registry.exists(self.root, step))
self.assertTrue(os.path.exists(model_location))
# Test loading.
model = registry.get(hp.model_hparams)
model.load_state_dict(torch.load(model_location))
self.assertStateEqual(model_state, TestSaveLoadExists.get_state(model))
model = registry.load(self.root, step, hp.model_hparams)
self.assertStateEqual(model_state, TestSaveLoadExists.get_state(model))
def test_freeze_batchnorm(self):
default_hparams = registry.get_default_hparams('cifar_resnet_20')
default_hparams.model_hparams.batchnorm_frozen = True
model = registry.get(default_hparams.model_hparams)
bn_names = []
for k, v in model.named_modules():
if isinstance(v, torch.nn.BatchNorm2d):
bn_names += [k + '.weight', k + '.bias']
for k, v in model.named_parameters():
with self.subTest(tensor=k):
if k in bn_names:
self.assertFalse(v.requires_grad)
else:
self.assertTrue(v.requires_grad)
def __init__(self, client_id=0, model=None):
"""Initializing the trainer with the provided model.
Arguments:
model: The model to train. Must be a models.base.Model subclass.
client_id: The ID of the client using this trainer (optional).
"""
super().__init__(client_id)
if model is None:
model = models_registry.get()
# Use data parallelism if multiple GPUs are available and the configuration specifies it
if Config().is_parallel():
logging.info("Using Data Parallelism.")
# DataParallel will divide and allocate batch_size to all available GPUs
self.model = nn.DataParallel(model)
else:
self.model = model
def import_data(self, database, file):
"""
Parses the given file object as a CSV file and adds the contents to the
given database using the first row as attribute names for each column.
"""
# Open the file in universal-newline mode to support CSV files created
# on different operating systems.
fh = open(file.temporary_file_path(), "rU")
reader = csv.reader(fh, delimiter=self.cleaned_data["delimiter"])
errors = []
docs = []
for row in reader:
if reader.line_num == 1:
# Get all non-empty column names using the first row of the
# data.
column_names = filter(lambda i: i, row)
continue
column_range = xrange(min(len(column_names), len(row)))
doc = {}
# TODO: Replace this for loop with a zip.
for i in column_range:
# Map row value to corresponding column name.
if len(row[i]) > 0:
doc[column_names[i]] = row[i]
# Skip empty documents.
if len(doc) == 0:
continue
if self.cleaned_data["model"]:
model = registered_models.get(self.cleaned_data["model"])
try:
doc = model.coerce(doc)
doc = model(**doc)
# doc["type"] = self.cleaned_data["model"].lower()
docs.append(doc)
except (KeyError, ValueError), e:
errors.append((doc, traceback.format_exc()))
else:
docs.append(doc)