def deserialize(self, model_dict, data=None, load_states=True):
"""
Loads per layer (weights, states) and other model parameters from the
dictionary passed.
Arguments:
model_dict (dict): dictionary describing the model including layers,
cost, optimizers, backend settings, etc.
generated by the serialize function
data (NervanaDataIterator): Data set (ignored, will be removed)
load_states (bool): if False, then only the weights will be loaded
into a model in which the layers have already been
created, otherwise will (re)create the layers from
the serialized parameters and set the learning
states as well
"""
if data is not None:
logger.warning('data is a deprecated argument and will be ignored')
if 'epoch_index' in model_dict:
self.epoch_index = model_dict['epoch_index']
if 'model' not in model_dict:
logger.error('Using old model serialization format. '
'Serialized the model into new format')
param_layers = [l for l in self.layers_to_optimize]
param_dict_list = model_dict['layer_params_states']
for l, ps in zip(param_layers, param_dict_list):
l.set_params(ps)
if 'states' in ps and load_states:
l.set_states(ps)
return
if 'backend' in model_dict:
if 'compat_mode' in model_dict['backend']:
self.be.compat_mode = model_dict['backend']['compat_mode']
else:
model_dict['backend'] = {}
typ = model_dict['model']['type']
main_container = load_class(typ)
if not hasattr(self, 'layers'):
self.layers = main_container.gen_class(
model_dict['model']['config'])
self.layers.load_weights(model_dict['model'], load_states)
if load_states and 'rng_state' in model_dict['backend']:
try:
self.be.rng_set_state(model_dict['backend']['rng_state'])
except ValueError as e:
# could come about when switching backend types (ex GPU to CPU)
logger.warning("Problems restoring existing RNG state: %s",
str(e))
# xxx - watkinspv, hack to load some data meta
if 'batch_meta' in model_dict:
self.batch_meta = model_dict['batch_meta']
def gen_class(cls, pdict):
if 'schedule' in pdict:
typ = pdict['schedule']['type']
scls = load_class(typ)
sched = scls.gen_class(pdict['schedule']['config'])
pdict['schedule'] = sched
return cls(**pdict)
def gen_class(cls, pdict):
if 'schedule' in pdict:
typ = pdict['schedule']['type']
scls = load_class(typ)
sched = scls.gen_class(pdict['schedule']['config'])
pdict['schedule'] = sched
return cls(**pdict)
def gen_class(cls, pdict):
costs = []
for cost in pdict['costs']:
typ = cost['type']
ccls = load_class(typ)
costs.append(ccls.gen_class(cost['config']))
pdict['costs'] = costs
return cls(**pdict)
def gen_class(cls, pdict):
costs = []
for cost in pdict["costs"]:
typ = cost["type"]
ccls = load_class(typ)
costs.append(ccls.gen_class(cost["config"]))
pdict["costs"] = costs
return cls(**pdict)
def gen_class(cls, pdict):
costs = []
for cost in pdict['costs']:
typ = cost['type']
ccls = load_class(typ)
costs.append(ccls.gen_class(cost['config']))
pdict['costs'] = costs
return cls(**pdict)
def deserialize(fn, datasets=None, inference=False):
"""
Helper function to load all objects from a serialized file,
this includes callbacks and datasets as well as the model, layers,
etc.
Arguments:
datasets (DataSet, optional): If the dataset is not serialized
in the file it can be passed in
as an argument. This will also
override any dataset in the serialized
file
inference (bool, optional): if true only the weights will be loaded, not
the states
Returns:
Model: the model object
Dataset: the data set object
Callback: the callbacks
"""
config_dict = load_obj(fn)
if datasets is not None:
logger.warn('Ignoring datasets serialized in archive file %s' % fn)
elif 'datasets' in config_dict:
ds_cls = load_class(config_dict['datasets']['type'])
dataset = ds_cls.gen_class(config_dict['datasets']['config'])
datasets = dataset.gen_iterators()
if 'train' in datasets:
data_iter = datasets['train']
else:
key = list(datasets.keys())[0]
data_iter = datasets[key]
logger.warn('Could not find training set iterator'
'using %s instead' % key)
model = Model(config_dict, data_iter)
callbacks = None
if 'callbacks' in config_dict:
# run through the callbacks looking for dataset objects
# replace them with the corresponding data set above
cbs = config_dict['callbacks']['callbacks']
for cb in cbs:
if 'config' not in cb:
cb['config'] = {}
for arg in cb['config']:
if type(cb['config']
[arg]) is dict and 'type' in cb['config'][arg]:
if cb['config'][arg]['type'] == 'Data':
key = cb['config'][arg]['name']
if key in datasets:
cb['config'][arg] = datasets[key]
else:
cb['config'][arg] = None
# now we can generate the callbacks
callbacks = Callbacks.load_callbacks(config_dict['callbacks'], model)
return (model, dataset, callbacks)
def gen_class(cls, pdict):
if 'schedule' in pdict:
typ = pdict['schedule']['type']
if typ.find('neon.') != 0:
typ = 'neon.optimizers.optimizer.' + typ
scls = load_class(typ)
sched = scls.gen_class(pdict['schedule']['config'])
pdict['schedule'] = sched
return cls(**pdict)
def deserialize(fn, datasets=None, inference=False):
"""
Helper function to load all objects from a serialized file,
this includes callbacks and datasets as well as the model, layers,
etc.
Arguments:
datasets (DataSet, optional): If the dataset is not serialized
in the file it can be passed in
as an argument. This will also
override any dataset in the serialized
file
inference (bool, optional): if true only the weights will be loaded, not
the states
Returns:
Model: the model object
Dataset: the data set object
Callback: the callbacks
"""
config_dict = load_obj(fn)
if datasets is not None:
logger.warn('Ignoring datasets serialized in archive file %s' % fn)
elif 'datasets' in config_dict:
ds_cls = load_class(config_dict['datasets']['type'])
dataset = ds_cls.gen_class(config_dict['datasets']['config'])
datasets = dataset.gen_iterators()
if 'train' in datasets:
data_iter = datasets['train']
else:
key = datasets.keys()[0]
data_iter = datasets[key]
logger.warn('Could not find training set iterator'
'using %s instead' % key)
model = Model(config_dict, data_iter)
callbacks = None
if 'callbacks' in config_dict:
# run through the callbacks looking for dataset objects
# replace them with the corresponding data set above
cbs = config_dict['callbacks']['callbacks']
for cb in cbs:
if 'config' not in cb:
cb['config'] = {}
for arg in cb['config']:
if type(cb['config'][arg]) is dict and 'type' in cb['config'][arg]:
if cb['config'][arg]['type'] == 'Data':
key = cb['config'][arg]['name']
if key in datasets:
cb['config'][arg] = datasets[key]
else:
cb['config'][arg] = None
# now we can generate the callbacks
callbacks = Callbacks.load_callbacks(config_dict['callbacks'], model)
return (model, dataset, callbacks)
def load_callbacks(cls, cdict, model, data=[]):
if type(cdict) is str:
cdict = load_obj(cdict)
callbacks = cls(model, output_file=cdict['output_file'])
callbacks.epoch_marker = cdict['epoch_marker']
callbacks.callbacks = []
for cb in cdict['callbacks']:
module = load_class(cb['type'])
callbacks.callbacks.append(module(**cb['config']))
return callbacks
def gen_class(cls, pdict):
costs = []
for cost in pdict["costs"]:
typ = cost["type"]
if typ.find("neon.") == -1:
typ = "neon.layers.layer." + typ
ccls = load_class(typ)
costs.append(ccls.gen_class(cost["config"]))
pdict["costs"] = costs
return cls(**pdict)
def load_callbacks(cls, cdict, model, data=[]):
if type(cdict) is str:
cdict = load_obj(cdict)
callbacks = cls(model, output_file=cdict["output_file"])
callbacks.epoch_marker = cdict["epoch_marker"]
callbacks.callbacks = []
for cb in cdict["callbacks"]:
module = load_class(cb["type"])
callbacks.callbacks.append(module(**cb["config"]))
return callbacks
def gen_class(cls, pdict):
costs = []
for cost in pdict['costs']:
typ = cost['type']
if typ.find('neon.') == -1:
typ = 'neon.layers.layer.' + typ
ccls = load_class(typ)
costs.append(ccls.gen_class(cost['config']))
pdict['costs'] = costs
return cls(**pdict)
def gen_class(cls, pdict):
costs = []
for cost in pdict['costs']:
typ = cost['type']
if typ.find('neon.') == -1:
typ = 'neon.layers.layer.' + typ
ccls = load_class(typ)
costs.append(ccls.gen_class(cost['config']))
pdict['costs'] = costs
return cls(**pdict)
def gen_class(cls, pdict):
for key in pdict['optimizer_mapping']:
# these should be optimizers
typ = pdict['optimizer_mapping'][key]['type']
ocls = load_class(typ)
if 'config' not in pdict['optimizer_mapping'][key]:
pdict['optimizer_mapping'][key]['config'] = {}
conf = pdict['optimizer_mapping'][key]['config']
pdict['optimizer_mapping'][key] = ocls.gen_class(conf)
return cls(**pdict)
def gen_class(cls, pdict):
for key in pdict['optimizer_mapping']:
# these should be optimizers
typ = pdict['optimizer_mapping'][key]['type']
ocls = load_class(typ)
if 'config' not in pdict['optimizer_mapping'][key]:
pdict['optimizer_mapping'][key]['config'] = {}
conf = pdict['optimizer_mapping'][key]['config']
pdict['optimizer_mapping'][key] = ocls.gen_class(conf)
return cls(**pdict)
def deserialize(self, model_dict, data=None, load_states=True):
"""
Loads per layer (weights, states) and other model parameters from the
dictionary passed.
Arguments:
model_dict (dict): dictionary describing the model including layers,
cost, optimizers, backend settings, etc.
generated by the serialize function
data (iterator): Data set (ignored, will be removed)
load_states (bool): if False, then only the weights will be loaded
into a model in which the layers have already been
created, otherwise will (re)create the layers from
the serialized parameters and set the learning
states as well
"""
if data is not None:
logger.warning('data is a deprecated argument and will be ignored')
if 'epoch_index' in model_dict:
self.epoch_index = model_dict['epoch_index']
if 'model' not in model_dict:
logger.error('Using old model serialization format. '
'Serialized the model into new format')
param_layers = [l for l in self.layers_to_optimize]
param_dict_list = model_dict['layer_params_states']
for l, ps in zip(param_layers, param_dict_list):
l.set_params(ps)
if 'states' in ps and load_states:
l.set_states(ps)
return
if 'backend' in model_dict:
if 'compat_mode' in model_dict['backend']:
self.be.compat_mode = model_dict['backend']['compat_mode']
else:
model_dict['backend'] = {}
typ = model_dict['model']['type']
main_container = load_class(typ)
if not hasattr(self, 'layers'):
self.layers = main_container.gen_class(model_dict['model']['config'])
self.layers.load_weights(model_dict['model'], load_states)
if load_states and 'rng_state' in model_dict['backend']:
try:
self.be.rng_set_state(model_dict['backend']['rng_state'])
except ValueError as e:
# could come about when switching backend types (ex GPU to CPU)
logger.warning("Problems restoring existing RNG state: %s", str(e))
def gen_class(cls, pdict):
for key in pdict['optimizer_mapping']:
# these should be optimizers
typ = pdict['optimizer_mapping'][key]['type']
if typ.find('neon.') != 0:
typ = 'neon.optimizers.optimizer.' + typ
ocls = load_class(typ)
if 'config' not in pdict['optimizer_mapping'][key]:
pdict['optimizer_mapping'][key]['config'] = {}
conf = pdict['optimizer_mapping'][key]['config']
pdict['optimizer_mapping'][key] = ocls.gen_class(conf)
return cls(**pdict)
def gen_class(cls, pdict):
layers = []
for layer in pdict['layers']:
typ = layer['type']
ccls = load_class(typ)
layers.append(ccls.gen_class(layer['config']))
# layers is special in that there may be parameters
# serialized which will be used elsewhere
lsave = pdict.pop('layers')
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def gen_class(cls, pdict):
layers = []
for layer in pdict['layers']:
typ = layer['type']
ccls = load_class(typ)
layers.append(ccls.gen_class(layer['config']))
# layers is special in that there may be parameters
# serialized which will be used elsewhere
lsave = pdict.pop('layers')
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def recursive_gen(pdict, key):
"""
helper method to check whether the definition
dictionary is defining a NervanaObject child,
if so it will instantiate that object and replace
the dictionary element with an instance of that object
"""
if type(pdict[key]) is dict and 'type' in pdict[key]:
if 'config' not in pdict[key]:
# assume empty params if 'config' key missing
# needed to keep yaml deserialization short
pdict[key]['config'] = {}
ccls = load_class(pdict[key]['type'])
pdict[key] = ccls.gen_class(pdict[key]['config'])
def gen_class(cls, pdict):
layers = []
for layer in pdict['layers']:
typ = layer['type']
if typ.find(__name__) != -1:
# this is a sequential layer
ccls = load_class(typ)
elif typ in globals():
# this may occur if full path not given
# be careful here because globals has stuff from outside this module
ccls = globals()[typ]
else:
# look in neon.layers.layer
if typ.find('neon.layers.layer') == -1:
typ = 'neon.layers.layer.' + typ
ccls = load_class(typ)
layers.append(ccls.gen_class(layer['config']))
# layers is special in that there may be parameters
# serialized which will be used elsewhere
lsave = pdict.pop('layers')
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def gen_class(cls, pdict):
layers = []
for layer in pdict['layers']:
typ = layer['type']
if typ.find(__name__) != -1:
# this is a sequential layer
ccls = load_class(typ)
elif typ in globals():
# this may occur if full path not given
# be careful here because globals has stuff from outside this module
ccls = globals()[typ]
else:
# look in neon.layers.layer
if typ.find('neon.layers.layer') == -1:
typ = 'neon.layers.layer.' + typ
ccls = load_class(typ)
layers.append(ccls.gen_class(layer['config']))
# layers is special in that there may be parameters
# serialized which will be used elsewhere
lsave = pdict.pop('layers')
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def gen_class(cls, pdict):
layers = []
for layer in pdict['layers']:
typ = layer['type']
ccls = load_class(typ)
layers.append(ccls.gen_class(layer['config']))
# the 'layers' key is special in that the layer
# parameters are in there and need to be saved the
# whole pdict['layers'] element can not be replaced
# with the just the layer objects like elsewhere
lsave = pdict.pop('layers')
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def gen_class(cls, pdict):
layers = [[], []]
for i, layer in enumerate(pdict['layers']):
typ = layer['type']
ccls = load_class(typ)
if i < pdict['num_encoder_layers']:
layers[0].append(ccls.gen_class(layer['config']))
else:
layers[1].append(ccls.gen_class(layer['config']))
# layers is special in that there may be parameters
# serialized which will be used elsewhere
lsave = pdict.pop('layers')
pdict.pop('num_encoder_layers', None)
new_cls = cls(layers=layers, **pdict)
pdict['layers'] = lsave
return new_cls
def deserialize(self, model_dict, data=None, load_states=True):
"""
Loads per layer (weights, states) and other model parameters from the
dictionary passed.
Arguments:
model_dict (dict): dictionary describing the model including layers,
cost, optimizers, backend settings, etc.
generated by the serialize function
data (NervanaDataIterator): data iterator used to initialize the
model
load_states (bool): if False, then only the weights will be loaded
into a model in which the layers have already been
created, otherwise will (re)create the layers from
the serialized parameters and set the learning
states as well
"""
if 'epoch_index' in model_dict:
self.epoch_index = model_dict['epoch_index']
if 'model' not in model_dict:
logger.error('Using old model serialization format. '
'Serialized the model into new format')
param_layers = [l for l in self.layers_to_optimize]
param_dict_list = model_dict['layer_params_states']
for l, ps in zip(param_layers, param_dict_list):
l.set_params(ps)
if 'states' in ps and load_states:
l.set_states(ps)
return
if 'backend' in model_dict:
if 'compat_mode' in model_dict['backend']:
self.be.compat_mode = model_dict['backend']['compat_mode']
else:
model_dict['backend'] = {}
typ = model_dict['model']['type']
main_container = load_class(typ)
self.layers = main_container.gen_class(model_dict['model']['config'])
# new serialization format
self.layers.load_weights(model_dict['model'], load_states)
cost = None
if 'cost' in model_dict:
cost_cls = load_class(model_dict['cost']['type'])
cost = cost_cls.gen_class(model_dict['cost']['config'])
opt = None
if 'optimizer' in model_dict:
opt_cls = load_class(model_dict['optimizer']['type'])
opt = opt_cls.gen_class(model_dict['optimizer']['config'])
if data is not None:
self.initialize(data, cost=cost)
if opt is not None:
self.optimizer = opt
else:
if self.be.__class__.__name__ == 'NervanaMGPU':
raise RuntimeError('Need dataset to initialize MGPU models')
if 'rng_state' in model_dict['backend']:
try:
self.be.rng_set_state(model_dict['backend']['rng_state'])
except ValueError as e:
# could come about when switching backend types (ex GPU to CPU)
logger.warning("Problems restoring existing RNG state: %s",
str(e))
def gen_class(cls, pdict):
key = "projection"
if pdict.get(key, None) is not None:
config = pdict[key].get("config", {})
pdict[key] = load_class(pdict[key]["type"]).gen_class(config)
return super(ResidualModule, cls).gen_class(pdict)
def deserialize(self, model_dict, data=None, weights_only=True):
"""
Loads per layer (weights, states) and other model parameters from the
dictionary passed.
Arguments:
model_dict (dict): dictionary describing the model including layers,
cost, optimizers, backend settings, etc.
generated by the serialize function
data (NervanaDataIterator): data iterator used to intialize the
model
weights_only (bool): if True, then only the weights will be loaded
into a model in which the layers have already been
created, otherwise will (re)create the layers from
the serialzed parameters and set the weights
"""
if 'epoch_index' in model_dict:
self.epoch_index = model_dict['epoch_index']
if 'model' not in model_dict:
logger.error('Using old model serialization format. '
'Serialized the model into new format')
param_layers = [l for l in self.layers_to_optimize]
param_dict_list = model_dict['layer_params_states']
for l, ps in zip(param_layers, param_dict_list):
l.set_params(ps)
if 'states' in ps:
l.set_states(ps)
return
if 'backend' in model_dict:
if 'compat_mode' in model_dict['backend']:
self.be.compat_mode = model_dict['backend']['compat_mode']
else:
model_dict['backend'] = {}
# new serialization format
if weights_only:
self.layers.load_weights(model_dict['model'])
return
typ = model_dict['model']['type']
main_container = load_class(typ)
self.layers = main_container.gen_class(model_dict['model']['config'])
cost = None
if 'cost' in model_dict:
cost_cls = load_class(model_dict['cost']['type'])
cost = cost_cls.gen_class(model_dict['cost']['config'])
opt = None
if 'optimizer' in model_dict:
opt_cls = load_class(model_dict['optimizer']['type'])
opt = opt_cls.gen_class(model_dict['optimizer']['config'])
if self.be.__class__.__name__ == 'NervanaMGPU':
assert data is not None, 'Need dataset to init MGPU models'
if data is not None:
self.initialize(data, cost=cost)
if opt is not None:
self.optimizer = opt
self.layers.load_weights(model_dict['model'])
if 'rng_state' in model_dict['backend']:
self.be.rng_set_state(model_dict['backend']['rng_state'])
